From 5981a908d4f0908d3bf542af6954ee2e7589e167 Mon Sep 17 00:00:00 2001 From: iamashwin99 <ashwin-kumar.karnad@mpsd.mpg.de> Date: Fri, 16 Jun 2023 13:54:27 +0200 Subject: [PATCH 01/20] reorganise files --- mpsd-software.py => mpsd-software-manager/mpsd-software.py | 0 test_mpsd-software.py => tests/test_mpsd-software.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename mpsd-software.py => mpsd-software-manager/mpsd-software.py (100%) rename test_mpsd-software.py => tests/test_mpsd-software.py (100%) diff --git a/mpsd-software.py b/mpsd-software-manager/mpsd-software.py similarity index 100% rename from mpsd-software.py rename to mpsd-software-manager/mpsd-software.py diff --git a/test_mpsd-software.py b/tests/test_mpsd-software.py similarity index 100% rename from test_mpsd-software.py rename to tests/test_mpsd-software.py -- GitLab From 01247c85b1cdb23800bc756756137d8c92a339d8 Mon Sep 17 00:00:00 2001 From: iamashwin99 <ashwin-kumar.karnad@mpsd.mpg.de> Date: Fri, 16 Jun 2023 14:31:50 +0200 Subject: [PATCH 02/20] add basic template --- pyproject.toml | 34 +++++++++++++++++++ src/mpsd_software_manager/__init__.py | 1 + .../mpsd_software_manager}/mpsd-software.py | 0 3 files changed, 35 insertions(+) create mode 100644 pyproject.toml create mode 100644 src/mpsd_software_manager/__init__.py rename {mpsd-software-manager => src/mpsd_software_manager}/mpsd-software.py (100%) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..919db5e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,34 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "mpsd_software_manager" +authors = [{name = "SSU-Computational Science (Fangohr et al)", email = "ssu-cs@mpsd.mpg.de"}] +license = {file = "LICENSE"} +classifiers = ["License :: OSI Approved :: MIT License"] +readme = "README.rst" + +dependencies = [ + "archspec", + "rich" +[project.scripts] +mpsd-software = "mpsd_software_manager.mpsd-software:main" + +[project.urls] +Home = "https://gitlab.gwdg.de/mpsd-cs/mpsd-software-manager/" + +[project.optional-dependencies] +dev = [ + "black", + "pytest", + "pytest-mock", + "pytest-cov", + "ruff", +] + +[tool.pytest.ini_options] +addopts = [ + "--import-mode=importlib", +] +pythonpath = "src" diff --git a/src/mpsd_software_manager/__init__.py b/src/mpsd_software_manager/__init__.py new file mode 100644 index 0000000..38d5d05 --- /dev/null +++ b/src/mpsd_software_manager/__init__.py @@ -0,0 +1 @@ +"""mpsd-software: tool for installation of software as on MPSD HPC.""" \ No newline at end of file diff --git a/mpsd-software-manager/mpsd-software.py b/src/mpsd_software_manager/mpsd-software.py similarity index 100% rename from mpsd-software-manager/mpsd-software.py rename to src/mpsd_software_manager/mpsd-software.py -- GitLab From f915f08ae119db2c529a2223f9ea63beca36713d Mon Sep 17 00:00:00 2001 From: iamashwin99 <ashwin-kumar.karnad@mpsd.mpg.de> Date: Fri, 16 Jun 2023 14:32:37 +0200 Subject: [PATCH 03/20] fix typo --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 919db5e..4cd5074 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,8 @@ readme = "README.rst" dependencies = [ "archspec", - "rich" + "rich", +] [project.scripts] mpsd-software = "mpsd_software_manager.mpsd-software:main" -- GitLab From f9d97d7fecefc1ae2127a444dccfeb6716ed2e6e Mon Sep 17 00:00:00 2001 From: iamashwin99 <ashwin-kumar.karnad@mpsd.mpg.de> Date: Fri, 16 Jun 2023 14:34:19 +0200 Subject: [PATCH 04/20] fix pytest import --- tests/test_mpsd-software.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_mpsd-software.py b/tests/test_mpsd-software.py index ce31308..c877ee9 100644 --- a/tests/test_mpsd-software.py +++ b/tests/test_mpsd-software.py @@ -11,7 +11,7 @@ import sys import pytest -mod = importlib.import_module("mpsd-software") +mod = importlib.import_module("mpsd_software_manager.mpsd-software") # set loglevel to debug - useful for understanding problems. # (if the tests pass, pytest doesn't show any output) -- GitLab From a7099294008ccf244c8b7b168c561e47186c08fb Mon Sep 17 00:00:00 2001 From: iamashwin99 <ashwin-kumar.karnad@mpsd.mpg.de> Date: Fri, 16 Jun 2023 14:38:32 +0200 Subject: [PATCH 05/20] fix ci --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 05f003e..c7b413b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -80,10 +80,10 @@ style: - pydocstyle --version - ruff . - black --check --diff . - - pydocstyle mpsd-software.py - - pydocstyle test_mpsd-software.py + - pydocstyle src/mpsd_software_manager/mpsd-software.py + - pydocstyle tests/test_mpsd-software.py # we could also use `ruff --select D` for pycodestyle. But the behaviour is not exactly the same. - - ./mpsd-software.py --version + - src/mpsd_software_manager/mpsd-software.py --version test-bullseye: -- GitLab From b456f25a16b54298b53894a6d00361d869cbbaee Mon Sep 17 00:00:00 2001 From: iamashwin99 <ashwin-kumar.karnad@mpsd.mpg.de> Date: Fri, 16 Jun 2023 14:39:43 +0200 Subject: [PATCH 06/20] black --- src/mpsd_software_manager/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mpsd_software_manager/__init__.py b/src/mpsd_software_manager/__init__.py index 38d5d05..58a3865 100644 --- a/src/mpsd_software_manager/__init__.py +++ b/src/mpsd_software_manager/__init__.py @@ -1 +1 @@ -"""mpsd-software: tool for installation of software as on MPSD HPC.""" \ No newline at end of file +"""mpsd-software: tool for installation of software as on MPSD HPC.""" -- GitLab From 41a6493eeb9e8e95631cd4d5a20448e089bba70b Mon Sep 17 00:00:00 2001 From: iamashwin99 <ashwin-kumar.karnad@mpsd.mpg.de> Date: Fri, 16 Jun 2023 14:42:12 +0200 Subject: [PATCH 07/20] fix ci --- .gitlab-ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c7b413b..504fd9c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -91,12 +91,11 @@ test-bullseye: image: debian:bullseye-slim script: - *prepare_debian - - pytest -v -l test_mpsd-software.py - + - pytest -v -l test-bookworm: stage: test image: debian:bookworm-slim script: - *prepare_debian - - pytest -v -l test_mpsd-software.py + - pytest -v -l -- GitLab From 94316262b928ea46e46c23128bbbece90fe9d7fb Mon Sep 17 00:00:00 2001 From: iamashwin99 <ashwin-kumar.karnad@mpsd.mpg.de> Date: Fri, 16 Jun 2023 14:49:32 +0200 Subject: [PATCH 08/20] update the default root_dir --- src/mpsd_software_manager/mpsd-software.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mpsd_software_manager/mpsd-software.py b/src/mpsd_software_manager/mpsd-software.py index 438dccc..bf408b1 100755 --- a/src/mpsd_software_manager/mpsd-software.py +++ b/src/mpsd_software_manager/mpsd-software.py @@ -1189,8 +1189,8 @@ def main(): # Carry out the action args = parser.parse_args() - # target dir is the place where this script exists. the - root_dir = Path(os.path.dirname(os.path.realpath(__file__))) + # root dir is the place where this script is called from + root_dir = Path(os.getcwd()) set_up_logging( args.loglevel, -- GitLab From 2d2d55624af800195401d284a9e6088abda7c997 Mon Sep 17 00:00:00 2001 From: iamashwin99 <ashwin-kumar.karnad@mpsd.mpg.de> Date: Fri, 16 Jun 2023 15:03:45 +0200 Subject: [PATCH 09/20] black --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4cd5074..8ad7286 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,6 @@ authors = [{name = "SSU-Computational Science (Fangohr et al)", email = "ssu-cs@ license = {file = "LICENSE"} classifiers = ["License :: OSI Approved :: MIT License"] readme = "README.rst" - dependencies = [ "archspec", "rich", -- GitLab From 39e7ec2205aabd29939bcbd15079e805fedbd90b Mon Sep 17 00:00:00 2001 From: iamashwin99 <ashwin-kumar.karnad@mpsd.mpg.de> Date: Fri, 16 Jun 2023 15:06:31 +0200 Subject: [PATCH 10/20] remove - in all names to avoid issue --- pyproject.toml | 3 ++- .../{mpsd-software.py => mpsd_software.py} | 0 tests/test_mpsd-software.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) rename src/mpsd_software_manager/{mpsd-software.py => mpsd_software.py} (100%) diff --git a/pyproject.toml b/pyproject.toml index 8ad7286..43fabe2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,13 +7,14 @@ name = "mpsd_software_manager" authors = [{name = "SSU-Computational Science (Fangohr et al)", email = "ssu-cs@mpsd.mpg.de"}] license = {file = "LICENSE"} classifiers = ["License :: OSI Approved :: MIT License"] +version = "2023.6.14" readme = "README.rst" dependencies = [ "archspec", "rich", ] [project.scripts] -mpsd-software = "mpsd_software_manager.mpsd-software:main" +mpsd-software = "mpsd_software_manager.mpsd_software:main" [project.urls] Home = "https://gitlab.gwdg.de/mpsd-cs/mpsd-software-manager/" diff --git a/src/mpsd_software_manager/mpsd-software.py b/src/mpsd_software_manager/mpsd_software.py similarity index 100% rename from src/mpsd_software_manager/mpsd-software.py rename to src/mpsd_software_manager/mpsd_software.py diff --git a/tests/test_mpsd-software.py b/tests/test_mpsd-software.py index c877ee9..6ef1d0d 100644 --- a/tests/test_mpsd-software.py +++ b/tests/test_mpsd-software.py @@ -11,7 +11,7 @@ import sys import pytest -mod = importlib.import_module("mpsd_software_manager.mpsd-software") +mod = importlib.import_module("mpsd_software_manager.mpsd_software") # set loglevel to debug - useful for understanding problems. # (if the tests pass, pytest doesn't show any output) -- GitLab From a65000ab9e36a0f65eee9f6f568ff2e4998ec419 Mon Sep 17 00:00:00 2001 From: iamashwin99 <ashwin-kumar.karnad@mpsd.mpg.de> Date: Fri, 16 Jun 2023 15:08:00 +0200 Subject: [PATCH 11/20] ignore python build artifacts --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 0b6d13d..3a1ec4c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,7 @@ **/*.pyc logs dev-23a/ -*.log \ No newline at end of file +*.log +dist/ +build/ +*.egg-info/ \ No newline at end of file -- GitLab From 94bb4ac83d33daad1f0c0716bd5f44bc15f33685 Mon Sep 17 00:00:00 2001 From: iamashwin99 <ashwin-kumar.karnad@mpsd.mpg.de> Date: Fri, 16 Jun 2023 15:15:18 +0200 Subject: [PATCH 12/20] update ci with name change --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 504fd9c..df9db09 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -80,10 +80,10 @@ style: - pydocstyle --version - ruff . - black --check --diff . - - pydocstyle src/mpsd_software_manager/mpsd-software.py + - pydocstyle src/mpsd_software_manager/mpsd_software.py - pydocstyle tests/test_mpsd-software.py # we could also use `ruff --select D` for pycodestyle. But the behaviour is not exactly the same. - - src/mpsd_software_manager/mpsd-software.py --version + - src/mpsd_software_manager/mpsd_software.py --version test-bullseye: -- GitLab From e06e9423abd63a18dd412d1f8672a9122906f267 Mon Sep 17 00:00:00 2001 From: iamashwin99 <ashwin-kumar.karnad@mpsd.mpg.de> Date: Fri, 16 Jun 2023 15:36:46 +0200 Subject: [PATCH 13/20] fix script hash not found when calling script as installed package --- src/mpsd_software_manager/mpsd_software.py | 36 ++++++++++++---------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/src/mpsd_software_manager/mpsd_software.py b/src/mpsd_software_manager/mpsd_software.py index bf408b1..1d9cb42 100755 --- a/src/mpsd_software_manager/mpsd_software.py +++ b/src/mpsd_software_manager/mpsd_software.py @@ -599,24 +599,28 @@ def record_script_execution_summary( cmd_line = " ".join(sys.argv) # script branch and commit hash with os_chdir(root_dir): - script_branch = ( - run( - ["git", "rev-parse", "--abbrev-ref", "HEAD"], - stdout=subprocess.PIPE, - check=True, + try: + script_branch = ( + run( + ["git", "rev-parse", "--abbrev-ref", "HEAD"], + stdout=subprocess.PIPE, + check=True, + ) + .stdout.decode() + .strip() ) - .stdout.decode() - .strip() - ) - script_commit_hash = ( - run( - ["git", "rev-parse", "--short", "HEAD"], - stdout=subprocess.PIPE, - check=True, + script_commit_hash = ( + run( + ["git", "rev-parse", "--short", "HEAD"], + stdout=subprocess.PIPE, + check=True, + ) + .stdout.decode() + .strip() ) - .stdout.decode() - .strip() - ) + except subprocess.CalledProcessError: + script_branch = "unknown" + script_commit_hash = "unknown" # spack-environments branch and commit hash from kwargs spe_branch = kwargs.get("spe_branch", None) spe_commit_hash = kwargs.get("spe_commit_hash", None) -- GitLab From e4d52a71627921801bc49aa3a1847302dc17f491 Mon Sep 17 00:00:00 2001 From: iamashwin99 <ashwin-kumar.karnad@mpsd.mpg.de> Date: Fri, 16 Jun 2023 15:44:29 +0200 Subject: [PATCH 14/20] rename tests for consistency --- tests/{test_mpsd-software.py => test_mpsd_software.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{test_mpsd-software.py => test_mpsd_software.py} (100%) diff --git a/tests/test_mpsd-software.py b/tests/test_mpsd_software.py similarity index 100% rename from tests/test_mpsd-software.py rename to tests/test_mpsd_software.py -- GitLab From 8cf6fd0f944d75b0ccb1465a172cf640666ebac5 Mon Sep 17 00:00:00 2001 From: iamashwin99 <ashwin-kumar.karnad@mpsd.mpg.de> Date: Fri, 16 Jun 2023 15:44:49 +0200 Subject: [PATCH 15/20] rename tests for consistency --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index df9db09..b70ecaf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -81,7 +81,7 @@ style: - ruff . - black --check --diff . - pydocstyle src/mpsd_software_manager/mpsd_software.py - - pydocstyle tests/test_mpsd-software.py + - pydocstyle tests/test_mpsd_software.py # we could also use `ruff --select D` for pycodestyle. But the behaviour is not exactly the same. - src/mpsd_software_manager/mpsd_software.py --version -- GitLab From aa4e80339d642eca2915a1e1043821b09943183e Mon Sep 17 00:00:00 2001 From: iamashwin99 <ashwin-kumar.karnad@mpsd.mpg.de> Date: Fri, 16 Jun 2023 15:49:30 +0200 Subject: [PATCH 16/20] update broken test ( not part of migration) --- tests/test_mpsd_software.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_mpsd_software.py b/tests/test_mpsd_software.py index 6ef1d0d..7b4fd77 100644 --- a/tests/test_mpsd_software.py +++ b/tests/test_mpsd_software.py @@ -190,13 +190,16 @@ def test_record_script_execution_summary(tmp_path): def test_install_environment_wrong_package_set(tmp_path): """Test exception is raised for non-existing package_set.""" - # Expect an Exception when wrong package_sets are provided - with pytest.raises(Exception): + # exits with exit code 1 when wrong package_sets are provided + with pytest.raises(SystemExit) as e: mod.install_environment( mpsd_release="dev-23a", package_sets=["wrong-package_set"], root_dir=(tmp_path), ) + assert e.type == SystemExit + assert e.value.code == 1 + def test_install_environment_wrong_mpsd_release(tmp_path): -- GitLab From 1f802441fcfd3b5407df5b2ee6eb8c8707910973 Mon Sep 17 00:00:00 2001 From: iamashwin99 <ashwin-kumar.karnad@mpsd.mpg.de> Date: Fri, 16 Jun 2023 15:59:10 +0200 Subject: [PATCH 17/20] black --- tests/test_mpsd_software.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_mpsd_software.py b/tests/test_mpsd_software.py index 7b4fd77..6c6c713 100644 --- a/tests/test_mpsd_software.py +++ b/tests/test_mpsd_software.py @@ -201,7 +201,6 @@ def test_install_environment_wrong_package_set(tmp_path): assert e.value.code == 1 - def test_install_environment_wrong_mpsd_release(tmp_path): """Test exception is raised for non-existing mpsd release.""" # Expect an Exception when wrong mpsd_release is provided (part of -- GitLab From 9b7bc44c34fb110a282e6580c93e57aa6a0ead76 Mon Sep 17 00:00:00 2001 From: Hans Fangohr <fangohr@users.noreply.github.com> Date: Fri, 16 Jun 2023 16:22:28 +0200 Subject: [PATCH 18/20] remove section on 'init' --- README.rst | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index c29a590..5adb22f 100644 --- a/README.rst +++ b/README.rst @@ -34,22 +34,23 @@ To install, for example, the ``foss2022a-serial`` toolchain: $ cd /home/user/mpsd-software -3. Initiate the installation at this location using:: - - $ mpsd-software init - - (This creates a hidden file ``.mpsd-software-root`` to tag the location for - as the root of the installation. All compiled files, logs etc are written in - or below this subdirectory.) - -4. From the same directory, run the command to install the ``foss2022a-serial`` +.. comment: + 3. Initiate the installation at this location using:: + + $ mpsd-software init + + (This creates a hidden file ``.mpsd-software-root`` to tag the location for + as the root of the installation. All compiled files, logs etc are written in + or below this subdirectory.) + +3. From the same directory, run the command to install the ``foss2022a-serial`` toolchain:: $ mpsd-software install dev-23a foss2022a-serial This will take some time (up to several hours depending on hardware). -5. To see the installation status, and the required ``module use`` command line +4. To see the installation status, and the required ``module use`` command line to activate the created modules, try the ``status`` command:: $ mpsd-software status dev-23a -- GitLab From b24cf9cf429fc745e50954f0f26a3615046e3052 Mon Sep 17 00:00:00 2001 From: Hans Fangohr <fangohr@users.noreply.github.com> Date: Fri, 16 Jun 2023 16:51:48 +0200 Subject: [PATCH 19/20] Add link to CI example. Can update later when this has moved to Gitlab. --- README.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 5adb22f..8008ca8 100644 --- a/README.rst +++ b/README.rst @@ -167,8 +167,10 @@ The compilation of the ``*-mpi`` toolchains needs linux headers installed. (TODO Working example ~~~~~~~~~~~~~~~ -We have a continuous integration example at XXX, that shows the complete -compilation cycle (including compilation of Octopus). (TODO) +There is an +[example](https://github.com/mpsd-computational-science/octopus-with-mpsd-software) +compilation that shows the complete compilation cycle (including compilation of +Octopus) using the ``foss2022a-serial`` toolchain. -- GitLab From cdfd2525e7cdd5214c11b7ca8970c87b0601e5c7 Mon Sep 17 00:00:00 2001 From: iamashwin99 <ashwin-kumar.karnad@mpsd.mpg.de> Date: Fri, 16 Jun 2023 17:01:13 +0200 Subject: [PATCH 20/20] add License --- LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..688828a --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2023 SSU-Computational Science Group (Fangohr et al) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. -- GitLab