diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..bad44636b34555264f9823e84b29108d3b192b2d
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,4 @@
+.vscode
+**/*.pyc
+logs
+dev-23a/
\ No newline at end of file
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 0000000000000000000000000000000000000000..1ad39094f161660f238a7ecd312166adb88001b2
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,101 @@
+image: python:latest
+
+stages:
+ - test
+
+# prepare set up of latest python image to run style checks
+.prepare_style: &prepare_style
+ - cat /etc/issue
+ - python3 -V
+ - which python3
+ - python3 -m venv ../venv
+ - source ../venv/bin/activate
+ - which python3
+ - pwd
+ - ls -l
+ - pip install -U pip
+ - pip --version
+ - pip install pytest black ruff pydocstyle
+ - pytest --version
+ - cat /etc/issue
+
+
+# prepare set up of Debian system to run py.test
+.prepare_debian: &prepare_debian
+ - echo "Execute this command before any 'script:' commands."
+ - cat /etc/issue
+ - pwd
+ - ls -l
+
+ - echo "Install Python3"
+ - apt-get update
+ - apt-get install -y python3 python3-venv
+ - python3 -m venv --help
+ - python3 -m venv venv
+ - source venv/bin/activate
+ - which python3
+ - python3 --version
+
+ - echo "Install Python dependencies for running the tests"
+ - pip install -U pip
+ - pip --version
+ - pip install pytest black ruff archspec
+
+ - echo "Diagnostics - which versions are we using"
+ - python3 --version
+ - pytest --version
+ - black --version
+ - ruff --version
+
+ - echo "Install additional packages we need to run spack-setup.sh"
+ - apt-get install -y git rsync
+ - echo "Install additional packages we need to run spack"
+ # Taken from https://github.com/fangohr/oommf-in-spack/blob/main/Dockerfile
+ - apt-get install -y --no-install-recommends
+ autoconf
+ build-essential
+ ca-certificates
+ coreutils
+ curl
+ environment-modules
+ file
+ gfortran
+ git
+ openssh-server
+ unzip
+
+ - export MPSD_MICROARCH=$(archspec cpu)
+ - echo "Setting MPSD_MICROARCH variable to $MPSD_MICROARCH"
+ - echo "Which version of Debian are we running?"
+ - cat /etc/issue
+
+
+style:
+ stage: test
+ image: python:latest
+ script:
+ - *prepare_style
+ - black --version
+ - ruff --version
+ - pydocstyle --version
+ - ruff .
+ - black --check --diff .
+ - pydocstyle mpsd-software-environment.py
+ - pydocstyle tests.py
+ # we could also use `ruff --select D` for pycodestyle. But the behaviour is not exactly the same.
+
+
+test-bullseye:
+ stage: test
+ image: debian:bullseye-slim
+ script:
+ - *prepare_debian
+ - pytest -v -l tests.py
+
+test-bookworm:
+ stage: test
+ image: debian:bookworm-slim
+ script:
+ - *prepare_debian
+ - pytest -v -l tests.py
+
diff --git a/install-dev23a.sh b/install-dev23a.sh
index 368bbe81ce54d878f252aad8f3d5b4eff654dbae..c01e10b7078becf1735c680f3e5ac8b219910416 100755
--- a/install-dev23a.sh
+++ b/install-dev23a.sh
@@ -1,11 +1,17 @@
#!/bin/bash
+# Script to build all toolchains for this MPSD release ( 23a )
+# Run this script inside the cloned folder for eg:
+# mpsddeb@mpsd-hpc-ibm-022:/opt_mpsd/linux-debian11/mpsd-software-environments$ ./install-dev23a.sh
+
set -e
+cd ..
mkdir -p dev-23a
cd dev-23a
# clone repo if it doesn't exist yet
[ -d 'spack-environments' ] || git clone git@gitlab.gwdg.de:mpsd-cs/spack-environments.git
pushd spack-environments
git checkout dev-23a
+git pull
popd
mkdir -p sandybridge
cd sandybridge
@@ -13,3 +19,8 @@ cd sandybridge
../spack-environments/spack_setup.sh foss2021a-serial
../spack-environments/spack_setup.sh foss2021a-mpi
../spack-environments/spack_setup.sh foss2021a-cuda-mpi
+
+
+../spack-environments/spack_setup.sh foss2022a-serial
+../spack-environments/spack_setup.sh foss2022a-mpi
+../spack-environments/spack_setup.sh foss2022a-cuda-mpi
diff --git a/mpsd-software-environment.py b/mpsd-software-environment.py
new file mode 100755
index 0000000000000000000000000000000000000000..59a831b65c3f46b0eb1935bb1e690dc46da633a4
--- /dev/null
+++ b/mpsd-software-environment.py
@@ -0,0 +1,576 @@
+#!/usr/bin/env python3
+
+"""mpsd-software-environment: tool for installation of toolchains."""
+
+import argparse
+import datetime
+import os
+import subprocess
+import sys
+import time
+from pathlib import Path
+from typing import List, Tuple
+
+about_tool = """
+Build toolchains using Spack.\n
+
+This function builds toolchains for MPSD-HPC at the appropriate directory, \n
+for given system architecture and MPSD software stack version.\n
+The toolchains
+are built using the bash script spack_setup.sh, and the results are logged. """
+
+config_vars = {
+ "cmd_log_file": "install.log",
+ "build_log_file": (
+ "logs/mpsd_spack_ver_toolchains_"
+ f"{datetime.datetime.now().replace(microsecond=0).isoformat()}.log"
+ ),
+ # TODO: modify toolchains,mpsd_spack_ver when the variable is available
+ "spack_environments_repo": "https://gitlab.gwdg.de/mpsd-cs/spack-environments.git",
+}
+
+
+# Helper class to change directory via context manager
+class os_chdir:
+ """The os_chdir class is a context manager.
+
+ It changes the current directory to a specified directory
+ and returns to the original directory after execution.
+ """
+
+ def __init__(self, new_dir):
+ """Initialize, save original directory."""
+ self.new_dir = new_dir
+ self.saved_dir = os.getcwd()
+
+ def __enter__(self):
+ """Go to target directory (main action for context)."""
+ os.chdir(self.new_dir)
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ """On exist we return to original directory."""
+ os.chdir(self.saved_dir)
+
+
+def run(*args, counter=[0], **kwargs):
+ """
+ Run a subprocess and log the call.
+
+ Convenience function to call `subprocess.run` and provide some metadata
+ about the call.
+
+ Parameters
+ ----------
+ args : tuple
+ passed on to subprocess.run(*args). For example
+ ("ls -l") or (["ls", "-l"])
+ counter : TYPE, optional
+ list with one integer, starting from [0].
+ This is (a Python hack) to count the number of
+ calls of this function, so the different calls of subprocess.run
+ are easier to follow in the log files.
+ kwargs : dict
+ keyword-value arguments to be passed to subprocess.run. For example,
+ `shell=True`.
+
+ Returns
+ -------
+ process : subprocess.CompletedProcess
+ CompletedProcess object as returned by `subprocess.run` .
+
+ Examples
+ --------
+ >>> run(['date', '+%Y-%m-%d'])
+ ##-03 Starting subprocess.run(['date', '+%Y-%m-%d']) with options
+ ##-03 getcwd=/Users/fangohr/git/mpsd-software-environments
+ ##-03 COMMAND=date +%Y-%m-%d
+ 2023-05-30
+ ##-03 Completed in 0.0054s.
+ ##-03
+ CompletedProcess(args=['date', '+%Y-%m-%d'], returncode=0)
+
+ >>> run(['date +%Y-%m-%d'], shell=True)
+ ##-04 Starting subprocess.run(['date +%Y-%m-%d']) with options shell=True
+ ##-04 getcwd=/Users/fangohr/git/mpsd-software-environments
+ ##-04 COMMAND=date +%Y-%m-%d
+ 2023-05-30
+ ##-04 Completed in 0.0069s.
+ ##-04
+ CompletedProcess(args=['date +%Y-%m-%d'], returncode=0)
+ """
+ # token is printed in front of every meta-data line - useful for
+ # searching the logs. Starts with "##-00", then "##-01", ...
+ token = f"##-{counter[0]:02d}"
+
+ counter[0] += 1 # increase counter
+
+ # make command nicely readable: ["ls", "-l"] -> "ls -l"
+ assert isinstance(args, tuple)
+ assert len(args) == 1
+ arg = args[0]
+ # either args is a tuple containing a string | Example: ('ls -1',)
+ if isinstance(arg, str):
+ command = arg
+ # or we have a tuple containing a list of strings.
+ # Example: (['ls', '-1'],)
+ elif isinstance(arg, list):
+ command = " ".join(arg)
+ else:
+ # we do not expect this to happen
+ raise NotImplementedError(f"{arg=}, {args=}")
+
+ # make options (such as `shell=True`) nicely readable
+ options = ", ".join([f"{key}={value}" for key, value in kwargs.items()])
+
+ # provide information about upcoming subprocess.run call
+ print(f"{token} Starting subprocess.run({arg}) with options {options}")
+ print(f"{token} getcwd={os.getcwd()}")
+ print(f"{token} COMMAND={command}")
+
+ time_start = time.time()
+ process = subprocess.run(*args, **kwargs)
+ execution_time = time.time() - time_start
+
+ print(f"{token} Completed in {execution_time:.4f}s.")
+ print(f"{token}") # near-empty line to make reading logs easier
+ return process
+
+
+def setup_log_cmd(
+ mpsd_release: str, script_dir: str, msg: str = None, **kwargs
+) -> None:
+ """
+ Log the command used to build the toolchains.
+
+ It also logs information about the software environment installer branch,
+ the Spack environments branch, and the commit hashes of each.
+ It also logs steps taken
+ in the install process using the optional message argument.
+
+ Parameters
+ ----------
+ - mpsd_release : str
+ The name of the release to install toolchains for.
+ - script_dir : str
+ The path to the directory where the scripts are located.
+ - msg : str, optional
+ An optional message to log in the command log file.
+ - **kwargs : dict
+ A dictionary with values for
+ - spe_branch : str
+ The name of the Spack environments branch.
+ - spe_commit_hash : str
+ The commit hash of the Spack environments branch.
+
+ Returns
+ -------
+ - None
+ """
+ release_base_dir = script_dir / mpsd_release
+
+ # Write to the log file with the following format
+ # --------------------------------------------------
+ # 2023-02-29T23:32:01, install-software-environment.py --release 23b --install ALL
+ # Software environment installer branch: script_branch (commit hash: \
+ # script_commit_hash)
+ # Spack environments branch: dev-23a (commit hash: spe_commit_hash)
+ # MSGs
+ with os_chdir(release_base_dir):
+ with open(config_vars["cmd_log_file"], "a") as f:
+ if msg:
+ # Write the message to the log file
+ f.write(msg + "\n")
+ else:
+ # Write the header
+ f.write("-" * 50 + "\n")
+
+ # Gather data to log
+ # call statement:
+ cmd_line = " ".join(sys.argv)
+ # script branch and commit hash
+ with os_chdir(script_dir):
+ script_branch = (
+ run(
+ ["git", "rev-parse", "--abbrev-ref", "HEAD"],
+ stdout=subprocess.PIPE,
+ check=True,
+ )
+ .stdout.decode()
+ .strip()
+ )
+ script_commit_hash = (
+ run(
+ ["git", "rev-parse", "--short", "HEAD"],
+ stdout=subprocess.PIPE,
+ check=True,
+ )
+ .stdout.decode()
+ .strip()
+ )
+ # spack-environments branch and commit hash from kwargs
+ spe_branch = kwargs.get("spe_branch", None)
+ spe_commit_hash = kwargs.get("spe_commit_hash", None)
+
+ # Write to log file
+ f.write(f"{datetime.datetime.now().isoformat()}, {cmd_line}\n")
+ f.write(
+ f"Software environment installer branch: {script_branch} "
+ f"(commit hash: {script_commit_hash})\n"
+ )
+ f.write(
+ f"Spack environments branch: {spe_branch} "
+ f"(commit hash: {spe_commit_hash})\n"
+ )
+
+
+def create_dir_structure(mpsd_release: str, script_dir: Path) -> None:
+ """
+ Create the directory structure and clone spack environments repo.
+
+ The create_dir_structure function creates the directory structure for
+ the specified release and clones the Spack environments repository if it
+ doesn't exist.
+
+ Parameters
+ ----------
+ - mpsd_release: A string representing the MPSD release version.
+ - script_dir: A Path object representing the path to the scripts directory.
+
+ Returns
+ -------
+ - None
+ """
+ # Create the directory structure for the release
+ release_base_dir = script_dir / mpsd_release
+ release_base_dir.mkdir(parents=True, exist_ok=True)
+
+ with os_chdir(release_base_dir):
+ # Clone the spack-environments repo if it doesn't exist
+ if not os.path.exists("spack-environments"):
+ run(
+ [
+ "git",
+ "clone",
+ config_vars["spack_environments_repo"],
+ ],
+ check=True,
+ )
+ with os_chdir("spack-environments"):
+ # Git fetch and checkout the release branch and git pull
+ # to be sure that the resulting repo is up to date
+ run(["git", "fetch", "--all"], check=True)
+ checkout_result = run(["git", "checkout", mpsd_release], check=True)
+
+ if checkout_result.returncode != 0:
+ raise Exception(
+ "Release branch does not exist in spack-environment repo \n."
+ "Check for typos."
+ )
+ run(["git", "pull"], check=True)
+
+
+def get_release_info(mpsd_release: str, script_dir: Path) -> Tuple[str, str, List[str]]:
+ """
+ Get information about the specified release.
+
+ Get information about the specified release, such as the branch and commit hash
+ of the Spack environments repository and the available toolchains.
+
+ Parameters
+ ----------
+ mpsd_release : str
+ The name of the release to get information for.
+ script_dir : pathlib.Path
+ The base directory where releases are stored.
+
+ Returns
+ -------
+ spe_branch : str
+ The name of the branch for the Spack environments repository.
+ spe_commit_hash : str
+ The commit hash for the Spack environments repository.
+ available_toolchains : list
+ A list of strings representing the available toolchains for the release.
+
+ Raises
+ ------
+ FileNotFoundError
+ If the release directory does not exist. Run `create_dir_structure()` first.
+ """
+ # Get the info for release
+ release_base_dir = script_dir / mpsd_release
+ if not os.path.exists(release_base_dir):
+ raise FileNotFoundError(
+ "Release directory does not exist. Run create_dir_structure() first."
+ )
+ with os_chdir(release_base_dir):
+ with os_chdir("spack-environments"):
+ # Get the branch and commit hash of the spack-environments repo
+ spe_commit_hash = (
+ run(["git", "rev-parse", "HEAD"], stdout=subprocess.PIPE, check=True)
+ .stdout.decode()
+ .strip()
+ )
+ spe_branch = (
+ run(
+ ["git", "rev-parse", "--abbrev-ref", "HEAD"],
+ stdout=subprocess.PIPE,
+ check=True,
+ )
+ .stdout.decode()
+ .strip()
+ )
+ available_toolchains = os.listdir("toolchains")
+ return spe_branch, spe_commit_hash, available_toolchains
+
+
+def prepare_environment(mpsd_release: str, script_dir: Path) -> List[str]:
+ """
+ Create the directory structure for the given MPSD release.
+
+ It does the following steps:
+ Clones the spack-environments repository.
+ Determines the branch and commit hash of the spack-environments repository
+ and the available toolchains.
+ Logs the command usage.
+
+ Parameters
+ ----------
+ mpsd_release : str
+ The name of the MPSD release to prepare the environment for.
+ script_dir : pathlib.Path
+ The base directory to create the release folder and
+ clone the spack-environments repository into.
+
+ Returns
+ -------
+ available_toolchains : list
+ A list of available toolchains for the given MPSD release.
+ """
+ create_dir_structure(mpsd_release, script_dir)
+ spe_branch, spe_commit_hash, available_toolchains = get_release_info(
+ mpsd_release, script_dir
+ )
+ setup_log_cmd(
+ mpsd_release, script_dir, spe_branch=spe_branch, spe_commit_hash=spe_commit_hash
+ )
+ return available_toolchains
+
+
+def install_environment(
+ mpsd_release: str,
+ toolchains: List[str],
+ script_dir: Path,
+ force_reinstall: bool = False,
+ enable_build_cache: bool = False,
+) -> None:
+ """
+ Install the specified MPSD release and toolchains.
+
+ The function installs the toolchain to the specified directory, using Spack.
+
+ Parameters
+ ----------
+ mpsd_release : str
+ A string representing the MPSD release version.
+ toolchains : list of str
+ A list of strings representing the toolchains to install
+ (e.g., "foss2021a-mpi", "global_generic", "ALL").
+ script_dir : pathlib.Path
+ A Path object representing the path to the directory where
+ the release and toolchains will be installed.
+ force_reinstall : bool, optional
+ A boolean indicating whether to force a reinstallation
+ even if the release and toolchains already exist. Defaults to False.
+ enable_build_cache : bool, optional
+ A boolean indicating whether to build the build cache
+ when installing toolchains. Defaults to False.
+
+ Raises
+ ------
+ ValueError
+ If a requested toolchain is not available in the specified release.
+
+ Returns
+ -------
+ None
+ """
+ print(
+ f"Installing release {mpsd_release} with toolchains {toolchains} "
+ f"to {script_dir}"
+ )
+
+ # Set required variables
+ release_base_dir = script_dir / mpsd_release
+ os.environ.get("MPSD_OS", "UNKNOWN_OS")
+ mpsd_microarch = os.environ.get("MPSD_MICROARCH", "UNKNOWN_MICROARCH")
+ toolchain_dir = release_base_dir / mpsd_microarch
+ toolchain_dir.mkdir(parents=True, exist_ok=True)
+ spack_setup_script = release_base_dir / "spack-environments" / "spack_setup.sh"
+ install_flags = []
+ if not enable_build_cache:
+ install_flags.append("-b")
+
+ # run the prepare_environment function
+ available_toolchains = prepare_environment(mpsd_release, script_dir)
+ # Ensure that the requested toolchains are available in the release
+ if toolchains == "ALL":
+ toolchains = available_toolchains
+ elif toolchains == "NONE":
+ # No toolchains requested, so we only create the env and print the
+ # list of available toolchains
+ print(
+ "No toolchains requested. Available toolchains for release "
+ f"{mpsd_release} are: \n {available_toolchains}"
+ )
+ return
+
+ for toolchain in toolchains:
+ if toolchain not in available_toolchains:
+ raise ValueError(
+ f"Toolchain '{toolchain}' is not available in release {mpsd_release}."
+ )
+
+ # Install the toolchains
+ with os_chdir(toolchain_dir):
+ # run spack_setup_script with the toolchains as arguments
+ # if the log folder doesn't exist, create it
+ if not os.path.exists("logs"):
+ os.mkdir("logs")
+ for toolchain in toolchains:
+ # Set the install log file name to config_vars["install_log_file"]
+ # and replace _toolchains_ with the toolchain name and
+ # _mpsd_spack_ver_ with mpsd_release
+
+ print(f"Installing toolchain {toolchain} to {toolchain_dir}")
+ install_log_file = (
+ config_vars["build_log_file"]
+ .replace("mpsd_spack_ver_", f"{mpsd_release}_")
+ .replace("_toolchains_", f"_{toolchain}_")
+ )
+ # log the command
+ setup_log_cmd(
+ mpsd_release,
+ script_dir,
+ msg=f"installing {toolchain} and logging at {install_log_file}",
+ )
+ setup_log_cmd(
+ mpsd_release,
+ script_dir,
+ msg=(
+ f"CMD: bash {spack_setup_script} {' '.join(install_flags)}"
+ "{toolchain}"
+ ),
+ )
+ run(
+ f"bash {spack_setup_script} {' '.join(install_flags)} {toolchain} 2>&1 "
+ f"| tee -a {install_log_file} ",
+ shell=True,
+ check=True,
+ )
+
+
+def remove_environment(release, toolchains, target_dir):
+ """Remove release from installation."""
+ msg = f"Removing release {release} with toolchains {toolchains} from {target_dir}"
+ print(msg)
+ raise NotImplementedError(msg)
+
+
+def start_new_environment(release, from_release, target_dir):
+ """Start new MPSD software environment version."""
+ msg = f"Starting new release {release} from {from_release} to {target_dir}"
+ print(msg)
+ raise NotImplementedError(msg)
+
+
+def main():
+ """Execute main entry point."""
+ parser = argparse.ArgumentParser(description=about_tool)
+ subparsers = parser.add_subparsers(
+ dest="action", title="actions", description="valid actions", required=True
+ )
+ subparsers.required = True
+ list_of_cmds = [
+ ("prepare", "Prepare the environment for installation on the disk"),
+ ("install", "Install a software environment"),
+ ("reinstall", "Reinstall a software environment"),
+ ("remove", "Remove a software environment or toolchains from an environment"),
+ ("start-new", "Start a new software environment version"),
+ ]
+ for cmd, help_text in list_of_cmds:
+ subp = subparsers.add_parser(cmd, help=help_text)
+
+ if cmd == "start-new":
+ subp.add_argument(
+ "--from-release",
+ dest="from_release",
+ type=str,
+ required=True,
+ help="Release version to start from",
+ )
+ subp.add_argument(
+ "--to-release",
+ dest="to_release",
+ type=str,
+ required=True,
+ help="Release version to create",
+ )
+
+ else:
+ subp.add_argument(
+ "release",
+ type=str,
+ help="Release version to prepare, install, reinstall or remove",
+ )
+ if cmd in ["install", "reinstall", "remove"]:
+ # "install" command needs additional documentation
+ tool_chain_help = (
+ f"Pass a list of toolchains to command {cmd}. "
+ "Use '--toolchains ALL' to "
+ f"{cmd} all toolchains. If '--toolchain' is not "
+ "specified, list available toolchains for the release "
+ "(after environment has been prepared if not done yet)."
+ )
+
+ subp.add_argument(
+ "--toolchains", # first option defines attribute
+ # name `args.toolchains` in `args = parser_args()`
+ "--toolchain", # allow singular as alternative
+ # (-> creates attribute `args.toolchains` if used)
+ type=str,
+ dest="toolchains",
+ nargs="+",
+ default="NONE",
+ help=tool_chain_help,
+ )
+ subp.add_argument(
+ "--enable-build-cache",
+ action="store_true",
+ help=(
+ "Enable Spack build cache. Useful for reinstallation but "
+ "consumes time and disk space."
+ ),
+ )
+ # Carry out the action
+ args = parser.parse_args()
+
+ # target dir is the place where this script exists. the
+ # release `dev` in script_dir/dev-23a
+ script_dir = Path(os.path.dirname(os.path.realpath(__file__)))
+
+ # Check the command and run related function
+ if args.action == "remove":
+ remove_environment(args.release, args.toolchains, script_dir)
+ elif args.action == "start-new":
+ start_new_environment(args.from_release, args.to_release, script_dir)
+ elif args.action == "install":
+ install_environment(
+ args.release, args.toolchains, script_dir, False, args.enable_build_cache
+ )
+ elif args.action == "prepare":
+ prepare_environment(args.release, script_dir)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tests.py b/tests.py
new file mode 100644
index 0000000000000000000000000000000000000000..43d08a7be8cc80320ff7006f4718eda7eb4d8268
--- /dev/null
+++ b/tests.py
@@ -0,0 +1,319 @@
+"""Tests for mpsd-software-environment.py."""
+
+import importlib
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+
+mod = importlib.import_module("mpsd-software-environment")
+
+
+def create_mock_git_repository(target_directory, create_directory=True):
+ """
+ Create a git repository in the directory `target_directory`.
+
+ Arguments
+ ---------
+ target_directory : pathlib.Path
+ - path at which the root of the repository should be located (i.e. `.git` folder)
+
+ create_directory : bool
+ - create `target_directory` and parent directories if True
+
+ """
+ # create directory first
+ if create_directory:
+ target_directory.mkdir(parents=True)
+
+ # then create git repository:
+ with mod.os_chdir(str(target_directory)):
+ subprocess.run("git init .", shell=True, check=True)
+ subprocess.run("echo 'fake content' > readme.txt", shell=True, check=True)
+ subprocess.run("git add readme.txt", shell=True, check=True)
+ subprocess.run("pwd", shell=True)
+
+ # if email and username are not available (such as on naked test container),
+ # git may complain. We set a temporary user for this one commit to work around
+ # that.
+ user_details = "-c user.name='Tes Ta' -c user.email='tester@some-ci.org'"
+ subprocess.run(
+ f'git {user_details} commit -m "first commit" readme.txt',
+ shell=True,
+ check=True,
+ )
+
+
+def test_os_chdir(tmp_path):
+ """Test the os_chdir context manager."""
+ # create a temporary directory for testing
+ temp_dir = tmp_path / "test_os_chdir"
+ temp_dir.mkdir()
+
+ # initial current working directory
+ initial_cwd = os.getcwd()
+
+ # change to the temporary directory using os_chdir
+ with mod.os_chdir(str(temp_dir)):
+ assert os.getcwd() == str(temp_dir)
+
+ # current working directory should be back to initial directory
+ assert os.getcwd() == initial_cwd
+
+
+def test_run_method(tmp_path):
+ """Run tests for run method."""
+ run = mod.run
+
+ # test a command with options:
+ assert run(["date", "+%Y-%m-%d"]).returncode == 0
+ assert run("date +%Y-%m-%d", shell=True).returncode == 0
+
+ # tests interacting with the file system
+ with mod.os_chdir(str(tmp_path)):
+ # ensure single string command works
+ assert run(("ls -l"), shell=True).returncode == 0
+ # test spaces are handled correctly:
+ assert run(["touch", "file1", "file2"]).returncode == 0
+ assert os.path.exists("file1")
+ assert os.path.exists("file2")
+ # test output is captured:
+ assert (
+ b"Hello, world!\n"
+ in run(["echo", "Hello, world!"], capture_output=True).stdout
+ )
+
+ # check exceptions
+ with pytest.raises(FileNotFoundError):
+ run(["doesnotexistcommand"])
+
+ # check error code is checked
+ # 1. expect this to parse: return code is non-zero, but we don't check
+ run(["ls", "/doesnotexist"]),
+ # 2. expect this to fail:
+ with pytest.raises(subprocess.CalledProcessError):
+ run(["ls", "/doesnotexist"], check=True)
+
+
+def test_prepare_environment(tmp_path):
+ """Simulate running preparation of environment.
+
+ Simulate running ./install-software-environment.py --release dev-23a \
+ --target-directory /tmp/test_prepare_env
+ prepare_env is run when cmd is not specified, we can test cmd='prepare'
+ and cmd=None to check both cases
+ """
+ script_dir = tmp_path / "mpsd_opt" / "linux_debian_11"
+ spack_environments = "spack-environments"
+ mpsd_release_to_test = "dev-23a"
+ release_base_dir = script_dir / mpsd_release_to_test
+ # check that the test directory does not exist
+ assert not script_dir.exists()
+
+ # prepare_environment expects to be executed in git repository
+ # (mpsd-software-environments). It queries the commit on which we are to
+ # log that information. For this to work, we need to execute the command
+ # within a directory tree that has a git repository at the same or high
+ # level. Let's create one:
+ create_mock_git_repository(script_dir)
+
+ # now call the function we want to test
+ result = mod.prepare_environment(
+ mpsd_release=mpsd_release_to_test, script_dir=script_dir
+ )
+
+ # check if the directory now is created
+ assert release_base_dir.exists()
+ # check for spack-environments directory
+ assert spack_environments in os.listdir(release_base_dir)
+
+ # check if the git branch is correctly checked out. We expect output such as
+ # git_branch_stdout = '* dev-23a\n develop\n'
+ # The entry with the '* ' prefix is the active branch.
+ git_branch_output_raw = subprocess.run(
+ f"cd {str(release_base_dir/spack_environments)} && git branch",
+ shell=True,
+ capture_output=True,
+ )
+ git_branch_stdout = git_branch_output_raw.stdout.decode("utf-8")
+ assert f"* {mpsd_release_to_test}" in git_branch_stdout
+
+ # check that result is a list and contains atleast ['global','foss2021a-mpi']
+ assert isinstance(result, list)
+ assert "global" in result
+ assert "foss2021a-mpi" in result
+
+ # Expect an Exception when wrong mpsd_release is provided
+ with pytest.raises(Exception):
+ result = mod.prepare_environment(
+ mpsd_release="wrong-mpsd-release", script_dir=(script_dir)
+ )
+
+
+def test_setup_log_cmd(tmp_path):
+ """Check that log is updated.
+
+ Check that logs/install-software-environment.log is updated when the module is run
+ """
+ log_file = "install.log"
+
+ script_dir = tmp_path / "test_prepare_env"
+ mpsd_release_to_test = "dev-23a"
+ release_base_dir = script_dir / mpsd_release_to_test
+ if os.path.exists(release_base_dir / log_file):
+ initial_bytes = os.path.getsize(log_file)
+ else:
+ initial_bytes = 0
+
+ # run the prepare_env functionality
+ create_mock_git_repository(target_directory=script_dir, create_directory=True)
+ mod.prepare_environment(mpsd_release=mpsd_release_to_test, script_dir=(script_dir))
+
+ # check that logs/install-software-environment.log is updated
+ assert os.path.exists(release_base_dir / log_file)
+ assert os.path.getsize(release_base_dir / log_file) > initial_bytes
+
+ # Check that the log file has "Spack environments branch: dev-23a " in the last line
+ with open(release_base_dir / log_file, "r") as f:
+ last_line = f.readlines()[-1]
+ assert "Spack environments branch: dev-23a " in last_line
+
+
+def test_install_environment_wrong_toolchain(tmp_path):
+ """Test exception is raised for non-existing toolchain."""
+ # Expect an Exception when wrong toolchains are provided
+ with pytest.raises(Exception):
+ mod.install_environment(
+ mpsd_release="dev-23a",
+ toolchains=["wrong-toolchain"],
+ script_dir=(tmp_path),
+ )
+
+
+def test_install_environment_wrong_mpsd_release(tmp_path):
+ """Test exception is raised for non-existing mpsd release."""
+ # Expect an Exception when wrong mpsd_release is provided (part of
+ # prepare_environment)
+ with pytest.raises(Exception):
+ mod.install_environment(
+ mpsd_release="wrong-mpsd-release",
+ toolchains=["foss2021a-mpi"],
+ script_dir=(tmp_path),
+ )
+
+
+def test_install_environment_zlib():
+ """Test installation of toolchain."""
+ # Prepare a test installation of global generic
+ # with only zlib to test the installation
+ # This is a long test,
+ # its handy to test this with print statements printed to
+ # stdout, use:
+ # pytest -s
+ # for this installation avoid tmp_path as
+ # the length of the path becomes too long and spack complains
+ script_dir = Path("/tmp/test_global_generic")
+ if script_dir.exists():
+ shutil.rmtree(script_dir)
+ script_dir.mkdir(exist_ok=True, parents=True)
+ mpsd_release_to_test = "dev-23a"
+ toolchain_to_test = "global_generic"
+ mpsd_microarch = os.getenv("MPSD_MICROARCH", "UNKNOWN_MICROARCH")
+ release_base_dir = script_dir / mpsd_release_to_test
+ create_mock_git_repository(target_directory=script_dir, create_directory=False)
+ mod.prepare_environment(mpsd_release=mpsd_release_to_test, script_dir=(script_dir))
+ # Patch the spack environments to create a fake global_generic
+ # create a test toolchain
+ toolchain_src_dir = release_base_dir / "spack-environments" / "toolchains"
+ # with mod.os_chdir(toolchain_src_dir):
+ # subprocess.run(
+ # "cp -r foss2021a-mpi fuss1999a", shell=True, capture_output=True
+ # )
+ # add zlib as a spec to global_generic
+ with open(toolchain_src_dir / "global_generic" / "global_packages.list", "w") as f:
+ f.write("zlib@1.2.13 \n")
+
+ # add zlib to whitelist of module creation file by replacing anaconda3%gcc@10.2.1
+ # with zlib@1.2.13
+ # in release_base_dir / "spack-environments/spack_overlay/etc/spack/modules.yaml"
+ module_file = (
+ release_base_dir / "spack-environments/spack_overlay/etc/spack/modules.yaml"
+ )
+ with open(module_file, "r") as f:
+ lines = f.read().replace("anaconda3%gcc@10.2.1", "zlib@1.2.13")
+ with open(module_file, "w") as f:
+ f.write(lines)
+
+ # Replace gcc@10.2.1 with gcc#13.1.1 or available system gcc for testing on laptop
+ gcc_ver = (
+ subprocess.run(["gcc -dumpfullversion"], shell=True, capture_output=True)
+ .stdout.decode("utf-8")
+ .strip()
+ )
+ assert len(gcc_ver) > 3, f"Couldn't find gcc {gcc_ver=}"
+
+ setup_file = release_base_dir / "spack-environments/spack_setup.sh"
+ with open(setup_file, "r") as f:
+ lines = f.read().replace(
+ 'system_compiler="gcc@10.2.1"', f'system_compiler="gcc@{gcc_ver}"'
+ )
+ with open(setup_file, "w") as f:
+ f.write(lines)
+ # install global_generic toolchain
+ mod.install_environment(
+ mpsd_release=mpsd_release_to_test,
+ toolchains=[toolchain_to_test],
+ script_dir=script_dir,
+ enable_build_cache=False,
+ )
+ # test that the build log is created correctly
+ # check that a file with glob build_globale_generic_dev-23a*.log exists at
+ # release_base_dir/mpsd_microarch
+ # print("Debug here ")
+ # time.sleep(10)
+ build_log = list(
+ (release_base_dir / mpsd_microarch / "logs").glob(
+ f"{mpsd_release_to_test}_{toolchain_to_test}_*.log"
+ )
+ )
+ assert len(build_log) > 0
+ # take the most recent build log
+ build_log = sorted(build_log)[0]
+ # check that the build log contains statement ##### Installation finished
+ with open(build_log, "r") as f:
+ lines = f.read()
+ assert "##### Installation finished" in lines
+ build_log_file_name = os.path.basename(build_log)
+
+ # assert that install log files exists
+ assert os.path.exists(release_base_dir / "install.log")
+
+ # assert that the build log is written to the install log file
+ os.path.basename(build_log)
+ with open(release_base_dir / "install.log", "r") as f:
+ lines = f.read()
+ assert (
+ f"installing {toolchain_to_test} and logging at logs/{build_log_file_name}"
+ in lines
+ )
+ # assert that the module files are created correctly
+ assert os.path.exists(release_base_dir / mpsd_microarch)
+ assert os.path.exists(release_base_dir / mpsd_microarch / "lmod")
+ # assert that lmod/module-index.yaml contains zlib
+ with open(
+ release_base_dir / mpsd_microarch / "lmod" / "module-index.yaml", "r"
+ ) as f:
+ lines = f.read()
+ assert "zlib" in lines
+
+
+def test_interface(tmp_path):
+ """Test other things (not implemented yet)."""
+ pass
+ # ensure that installing without toolchains only passes the available toolchains
+ # check that the script branch and hash are correct when running the script
+ # check that the help message is printed when no arguments are provided
+ # check that the help message is printed when -h is provided
+ # check that the error messages are also logged to the log file