diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index af91fa7d11c489ed3e42ba0f8c73fdd411e3f284..1ad39094f161660f238a7ecd312166adb88001b2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -77,6 +77,7 @@ style:
- *prepare_style
- black --version
- ruff --version
+ - pydocstyle --version
- ruff .
- black --check --diff .
- pydocstyle mpsd-software-environment.py
diff --git a/mpsd-software-environment.py b/mpsd-software-environment.py
index 02982e8c63392a351e0f6fef809bb7c3ba248a53..59a831b65c3f46b0eb1935bb1e690dc46da633a4 100755
--- a/mpsd-software-environment.py
+++ b/mpsd-software-environment.py
@@ -54,9 +54,9 @@ class os_chdir:
def run(*args, counter=[0], **kwargs):
"""
- Call subprocess.run(*args, **kwargs) and print logging data.
+ Run a subprocess and log the call.
- Conveniene function to call `subprocess.run` and provide some metadata
+ Convenience function to call `subprocess.run` and provide some metadata
about the call.
Parameters
@@ -193,6 +193,7 @@ def setup_log_cmd(
run(
["git", "rev-parse", "--abbrev-ref", "HEAD"],
stdout=subprocess.PIPE,
+ check=True,
)
.stdout.decode()
.strip()
@@ -201,6 +202,7 @@ def setup_log_cmd(
run(
["git", "rev-parse", "--short", "HEAD"],
stdout=subprocess.PIPE,
+ check=True,
)
.stdout.decode()
.strip()
@@ -250,19 +252,21 @@ def create_dir_structure(mpsd_release: str, script_dir: Path) -> None:
"git",
"clone",
config_vars["spack_environments_repo"],
- ]
+ ],
+ check=True,
)
with os_chdir("spack-environments"):
# Git fetch and checkout the release branch and git pull
# to be sure that the resulting repo is up to date
- run(["git", "fetch", "--all"])
- checkout_result = run(["git", "checkout", mpsd_release])
+ run(["git", "fetch", "--all"], check=True)
+ checkout_result = run(["git", "checkout", mpsd_release], check=True)
+
if checkout_result.returncode != 0:
raise Exception(
"Release branch does not exist in spack-environment repo \n."
"Check for typos."
)
- run(["git", "pull"])
+ run(["git", "pull"], check=True)
def get_release_info(mpsd_release: str, script_dir: Path) -> Tuple[str, str, List[str]]:
@@ -303,13 +307,15 @@ def get_release_info(mpsd_release: str, script_dir: Path) -> Tuple[str, str, Lis
with os_chdir("spack-environments"):
# Get the branch and commit hash of the spack-environments repo
spe_commit_hash = (
- run(["git", "rev-parse", "HEAD"], stdout=subprocess.PIPE)
+ run(["git", "rev-parse", "HEAD"], stdout=subprocess.PIPE, check=True)
.stdout.decode()
.strip()
)
spe_branch = (
run(
- ["git", "rev-parse", "--abbrev-ref", "HEAD"], stdout=subprocess.PIPE
+ ["git", "rev-parse", "--abbrev-ref", "HEAD"],
+ stdout=subprocess.PIPE,
+ check=True,
)
.stdout.decode()
.strip()
@@ -460,6 +466,7 @@ def install_environment(
f"bash {spack_setup_script} {' '.join(install_flags)} {toolchain} 2>&1 "
f"| tee -a {install_log_file} ",
shell=True,
+ check=True,
)
diff --git a/tests.py b/tests.py
index 04e1e4e099405555cb5837cf6ea7e89f816273f0..319e13cde23d794f5e422a60d1c1edbaca362960 100644
--- a/tests.py
+++ b/tests.py
@@ -11,6 +11,41 @@ import pytest
mod = importlib.import_module("mpsd-software-environment")
+def create_mock_git_repository(target_directory, create_directory=True):
+ """
+ Create a git repository in the directory `target_directory`.
+
+ Arguments
+ ---------
+ target_directory : pathlib.Path
+ - path at which the root of the repository should be located (i.e. `.git` folder)
+
+ create_directory : bool
+ - create `target_directory` and parent directories if True
+
+ """
+ # create directory first
+ if create_directory:
+ target_directory.mkdir(parents=True)
+
+ # then create git repository:
+ with mod.os_chdir(str(target_directory)):
+ subprocess.run("git init .", shell=True, check=True)
+ subprocess.run("echo 'fake content' > readme.txt", shell=True, check=True)
+ subprocess.run("git add readme.txt", shell=True, check=True)
+ subprocess.run("pwd", shell=True)
+
+ # if email and username are not available (such as on naked test container),
+ # git may complain. We set a temporary user for this one commit to work around
+ # that.
+ user_details = "-c user.name='Tes Ta' -c user.email='tester@some-ci.org'"
+ subprocess.run(
+ f'git {user_details} commit -m "first commit" readme.txt',
+ shell=True,
+ check=True,
+ )
+
+
def test_os_chdir(tmp_path):
"""Test the os_chdir context manager."""
# create a temporary directory for testing
@@ -30,39 +65,47 @@ def test_os_chdir(tmp_path):
def test_prepare_environment(tmp_path):
"""Simulate running preparation of environment.
-
+
Simulate running ./install-software-environment.py --release dev-23a \
--target-directory /tmp/test_prepare_env
prepare_env is run when cmd is not specified, we can test cmd='prepare'
and cmd=None to check both cases
"""
- script_dir = tmp_path / "test_prepare_env"
+ script_dir = tmp_path / "mpsd_opt" / "linux_debian_11"
spack_environments = "spack-environments"
mpsd_release_to_test = "dev-23a"
release_base_dir = script_dir / mpsd_release_to_test
# check that the test directory does not exist
assert not script_dir.exists()
+ # prepare_environment expects to be executed in git repository
+ # (mpsd-software-environments). It queries the commit on which we are to
+ # log that information. For this to work, we need to execute the command
+ # within a directory tree that has a git repository at the same or high
+ # level. Let's create one:
+ create_mock_git_repository(script_dir)
+
+ # now call the function we want to test
result = mod.prepare_environment(
- mpsd_release=mpsd_release_to_test, script_dir=(script_dir)
+ mpsd_release=mpsd_release_to_test, script_dir=script_dir
)
- # wait for 20 seconds for the git clone to finish
- # time.sleep(20)
+
# check if the directory now is created
assert release_base_dir.exists()
# check for spack-environments directory
assert spack_environments in os.listdir(release_base_dir)
- # check if the git branch is correctly checked out
- assert (
- subprocess.run(
- f"cd {str(release_base_dir/spack_environments)} && git branch",
- shell=True,
- capture_output=True,
- )
- .stdout.decode("utf-8")
- .split("\n")[0]
- == f"* {mpsd_release_to_test}"
+
+ # check if the git branch is correctly checked out. We expect output such as
+ # git_branch_stdout = '* dev-23a\n develop\n'
+ # The entry with the '* ' prefix is the active branch.
+ git_branch_output_raw = subprocess.run(
+ f"cd {str(release_base_dir/spack_environments)} && git branch",
+ shell=True,
+ capture_output=True,
)
+ git_branch_stdout = git_branch_output_raw.stdout.decode("utf-8")
+ assert f"* {mpsd_release_to_test}" in git_branch_stdout
+
# check that result is a list and contains atleast ['global','foss2021a-mpi']
assert isinstance(result, list)
assert "global" in result
@@ -91,6 +134,7 @@ def test_setup_log_cmd(tmp_path):
initial_bytes = 0
# run the prepare_env functionality
+ create_mock_git_repository(target_directory=script_dir, create_directory=True)
mod.prepare_environment(mpsd_release=mpsd_release_to_test, script_dir=(script_dir))
# check that logs/install-software-environment.log is updated
@@ -103,20 +147,19 @@ def test_setup_log_cmd(tmp_path):
assert "Spack environments branch: dev-23a " in last_line
-def test_install_environment(tmp_path):
- """Test the installation of a toolchain.
-
- This is a long test, its handy to test this with print statements printed to
- stdout, use:
- pytest -s
- Expect an Exception when wrong toolchains are provided
- """
+def test_install_environment_wrong_toolchain(tmp_path):
+ """Test exception is raised for non-existing toolchain."""
+ # Expect an Exception when wrong toolchains are provided
with pytest.raises(Exception):
mod.install_environment(
mpsd_release="dev-23a",
toolchains=["wrong-toolchain"],
script_dir=(tmp_path),
)
+
+
+def test_install_environment_wrong_mpsd_release(tmp_path):
+ """Test exception is raised for non-existing mpsd release."""
# Expect an Exception when wrong mpsd_release is provided (part of
# prepare_environment)
with pytest.raises(Exception):
@@ -125,11 +168,18 @@ def test_install_environment(tmp_path):
toolchains=["foss2021a-mpi"],
script_dir=(tmp_path),
)
- # prepare a test of global generic with only zlib to test the installation
- # prepare dev-23a release
- # script_dir = tmp_path / "test_global_generic"
- # for actaual installation avoid tmp_path as the lenght of the path is too long
- # and spack complains
+
+
+def test_install_environment_zlib():
+ """Test installation of toolchain."""
+ # Prepare a test installation of global generic
+ # with only zlib to test the installation
+ # This is a long test,
+ # its handy to test this with print statements printed to
+ # stdout, use:
+ # pytest -s
+ # for this installation avoid tmp_path as
+ # the length of the path becomes too long and spack complains
script_dir = Path("/tmp/test_global_generic")
if script_dir.exists():
shutil.rmtree(script_dir)
@@ -138,6 +188,7 @@ def test_install_environment(tmp_path):
toolchain_to_test = "global_generic"
mpsd_microarch = os.getenv("MPSD_MICROARCH", "UNKNOWN_MICROARCH")
release_base_dir = script_dir / mpsd_release_to_test
+ create_mock_git_repository(target_directory=script_dir, create_directory=False)
mod.prepare_environment(mpsd_release=mpsd_release_to_test, script_dir=(script_dir))
# Patch the spack environments to create a fake global_generic
# create a test toolchain