diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..bad44636b34555264f9823e84b29108d3b192b2d
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,4 @@
+.vscode
+**/*.pyc
+logs
+dev-23a/
\ No newline at end of file
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 0000000000000000000000000000000000000000..99c03e1bba7b4d16c4ba5579549413e668be1908
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,52 @@
+image: python:latest
+
+stages:
+ - test-style
+ - test
+
+before_script:
+ - cat /etc/issue
+ - python -V
+ - which python
+ - python -m venv ../venv
+ - source ../venv/bin/activate
+ - which python
+ - pwd
+ - pip install -U pip
+ - pip --version
+ - pip install pytest black ruff
+ - pytest --version
+
+
+style:
+ stage: test-style
+ image: python:latest
+ script:
+ - black --version
+ - ruff --version
+ - ruff . || true
+ - black --check --diff . || true
+
+
+test3-09:
+ stage: test
+ image: python:3.9
+ script:
+ - echo "run tests here later"
+ - python --version
+
+test3-10:
+ stage: test
+ image: python:3.10
+ script:
+ - echo "run tests here later"
+ - python --version
+
+
+test3-11:
+ stage: test
+ image: python:3.11
+ script:
+ - echo "run tests here later"
+ - python --version
+
diff --git a/mpsd-software-environment.py b/mpsd-software-environment.py
new file mode 100755
index 0000000000000000000000000000000000000000..efdd2c47fab1192fe5f0fd6810419c353e6e75da
--- /dev/null
+++ b/mpsd-software-environment.py
@@ -0,0 +1,384 @@
+#!/usr/bin/env python3
+import os
+import subprocess
+import datetime
+import argparse
+import sys
+from pathlib import Path
+from typing import List, Tuple
+import datetime
+
+about_tool = """
+Build toolchains using Spack.\n
+
+This function builds toolchains for MPSD-HPC at the appropriate directory, \n
+for given system architecture and MPSD software stack version.\n
+The toolchains are built using the bash script spack_setup.sh, and the results are logged.
+"""
+config_vars = {
+ "cmd_log_file": "install.log",
+ "build_log_file": f"logs/mpsd_spack_ver_toolchains_{datetime.datetime.now().replace(microsecond=0).isoformat()}.log", # TODO: modify toolchains,mpsd_spack_ver when the variable is available
+ "spack_environments_repo": "https://gitlab.gwdg.de/mpsd-cs/spack-environments.git",
+}
+
+
+# Helper class to change directory via context manager
+class os_chdir:
+ """
+ The os_chdir class is a context manager that
+ changes the current directory to a specified directory
+ and returns to the original directory after execution.
+ """
+
+ def __init__(self, new_dir):
+ self.new_dir = new_dir
+ self.saved_dir = os.getcwd()
+
+ def __enter__(self):
+ os.chdir(self.new_dir)
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ os.chdir(self.saved_dir)
+
+
+def setup_log_cmd(
+ mpsd_release: str, script_dir: str, msg: str = None, *args, **kwargs
+) -> None:
+ """
+ The setup_log_cmd function logs the command used to build the toolchains,
+ along with information about the software environment installer branch, the Spack environments branch,
+ and the commit hashes of each. It also logs steps taken in install process using the optional message argument.
+
+ Args:
+ mpsd_release (str): The name of the release to install toolchains for.
+ script_dir (str): The path to the directory where the scripts are located.
+ msg (str, optional): An optional message to log in the command log file.
+
+ Returns:
+ None
+ """
+ release_base_dir = script_dir / mpsd_release
+
+ with os_chdir(release_base_dir):
+ # Write to the log file with the following format
+ # --------------------------------------------------
+ # 2023-02-29T23:32:01, install-software-environment.py --release dev-23a --install ALL
+ # Software environment installer branch: script_branch (commit hash: script_commit_hash)
+ # Spack environments branch: dev-23a (commit hash: spe_commit_hash)
+ # MSGs
+ with open(config_vars["cmd_log_file"], "a") as f:
+ if msg:
+ # Write the message to the log file
+ f.write(msg + "\n")
+ else:
+ # Write the header
+ f.write("-" * 50 + "\n")
+
+ # Gather data to log
+ # call statement:
+ cmd_line = " ".join(sys.argv)
+ # script branch and commit hash
+ with os_chdir(script_dir):
+ script_branch = (
+ subprocess.run(
+ ["git", "rev-parse", "--abbrev-ref", "HEAD"],
+ stdout=subprocess.PIPE,
+ )
+ .stdout.decode()
+ .strip()
+ )
+ script_commit_hash = (
+ subprocess.run(
+ ["git", "rev-parse", "--short", "HEAD"],
+ stdout=subprocess.PIPE,
+ )
+ .stdout.decode()
+ .strip()
+ )
+ # spack-environments branch and commit hash from kwargs
+ spe_branch = kwargs.get("spe_branch", None)
+ spe_commit_hash = kwargs.get("spe_commit_hash", None)
+
+ # Write to log file
+ f.write(f"{datetime.datetime.now().isoformat()}, {cmd_line}\n")
+ f.write(
+ f"Software environment installer branch: {script_branch} (commit hash: {script_commit_hash})\n"
+ )
+ f.write(
+ f"Spack environments branch: {spe_branch} (commit hash: {spe_commit_hash})\n"
+ )
+
+
+def create_dir_structure(mpsd_release: str, script_dir: Path) -> None:
+ """
+ The create_dir_structure function creates the directory structure for the specified release
+ and clones the Spack environments repository if it doesn't exist.
+
+ Args:
+ - mpsd_release: A string representing the MPSD release version.
+ - script_dir: A Path object representing the path to the scripts directory.
+
+ Returns:
+ - None
+ """
+ # Create the directory structure for the release
+ release_base_dir = script_dir / mpsd_release
+ release_base_dir.mkdir(parents=True, exist_ok=True)
+
+ with os_chdir(release_base_dir):
+ # Clone the spack-environments repo if it doesn't exist
+ if not os.path.exists("spack-environments"):
+ subprocess.run(
+ [
+ "git",
+ "clone",
+ config_vars["spack_environments_repo"],
+ ]
+ )
+ with os_chdir("spack-environments"):
+ # Git fetch and checkout the release branch and git pull to be sure that the resulting repo is up to date
+ subprocess.run(["git", "fetch", "--all"])
+ checkout_result = subprocess.run(["git", "checkout", mpsd_release])
+ if checkout_result.returncode != 0:
+ raise Exception(
+ "Release branch does not exist in spack-environment repo \n. Check for typos."
+ )
+ subprocess.run(["git", "pull"])
+
+
+def get_release_info(mpsd_release: str, script_dir: Path) -> Tuple[str, str, List[str]]:
+ """
+ Get information about the specified release, such as the branch and commit hash
+ of the Spack environments repository and the available toolchains.
+
+ Args:
+ - mpsd_release (str) : the name of the release to get information for.
+ - script_dir (pathlib.Path): the base directory where releases are stored.
+
+ Returns:
+ - spe_branch: str, the name of the branch for the Spack environments repository.
+ - spe_commit_hash: str, the commit hash for the Spack environments repository.
+ - available_toolchains: list, a list of strings representing the available toolchains for the release.
+
+ Raises:
+ - Exception: If the release directory does not exist. Run `create_dir_structure()` first.
+ """
+ # Get the info for release
+ release_base_dir = script_dir / mpsd_release
+ if not os.path.exists(release_base_dir):
+ raise Exception(
+ "Release directory does not exist. Run create_dir_structure() first."
+ )
+ with os_chdir(release_base_dir):
+ with os_chdir("spack-environments"):
+ # Get the branch and commit hash of the spack-environments repo
+ spe_commit_hash = (
+ subprocess.run(["git", "rev-parse", "HEAD"], stdout=subprocess.PIPE)
+ .stdout.decode()
+ .strip()
+ )
+ spe_branch = (
+ subprocess.run(
+ ["git", "rev-parse", "--abbrev-ref", "HEAD"], stdout=subprocess.PIPE
+ )
+ .stdout.decode()
+ .strip()
+ )
+ available_toolchains = os.listdir("toolchains")
+ return spe_branch, spe_commit_hash, available_toolchains
+
+
+def prepare_environment(mpsd_release: str, script_dir: Path) -> List[str]:
+ """
+ - Creates the directory structure for the given MPSD release and clones the spack-environments repository.
+ - Determines the branch and commit hash of the spack-environments repository and the available toolchains.
+ - Logs the command usage.
+
+ Args:
+ - mpsd_release (str): The name of the MPSD release to prepare the environment for.
+ - script_dir (pathlib.Path): The base directory to create the release folder and clone the spack-environments repository into.
+
+ Returns:
+ - available_toolchains (list): A list of available toolchains for the given MPSD release.
+ """
+ create_dir_structure(mpsd_release, script_dir)
+ spe_branch, spe_commit_hash, available_toolchains = get_release_info(
+ mpsd_release, script_dir
+ )
+ setup_log_cmd(
+ mpsd_release, script_dir, spe_branch=spe_branch, spe_commit_hash=spe_commit_hash
+ )
+ return available_toolchains
+
+
+def install_environment(
+ mpsd_release: str,
+ toolchains: List[str],
+ script_dir: Path,
+ force_reinstall: bool = False,
+ enable_build_cache: bool = False,
+) -> None:
+ """
+ Installs the specified MPSD release and toolchains to the specified directory using Spack.
+
+ Args:
+ mpsd_release: A string representing the MPSD release version.
+ toolchains: A list of strings representing the toolchains to install (e.g., "foss2021a-mpi", "global_generic", "ALL").
+ script_dir: A Path object representing the path to the directory where the release and toolchains will be installed.
+ force_reinstall: A boolean indicating whether to force a reinstallation even if the release and toolchains already exist. Defaults to False.
+ enable_build_cache: A boolean indicating whether to build the build cache when installing toolchains. Defaults to False.
+
+ Raises:
+ Exception: If a requested toolchain is not available in the specified release.
+
+ Returns:
+ None
+ """
+ print(
+ f"Installing release {mpsd_release} with toolchains {toolchains} to {script_dir}"
+ )
+
+ # Set required variables
+ release_base_dir = script_dir / mpsd_release
+ os.environ.get("MPSD_OS", "UNKNOWN_OS")
+ mpsd_microarch = os.environ.get("MPSD_MICROARCH", "UNKNOWN_MICROARCH")
+ toolchain_dir = release_base_dir / mpsd_microarch
+ toolchain_dir.mkdir(parents=True, exist_ok=True)
+ spack_setup_script = release_base_dir / "spack-environments" / "spack_setup.sh"
+ install_flags = []
+ if not enable_build_cache:
+ install_flags.append("-b")
+
+ # run the prepare_environment function
+ available_toolchains = prepare_environment(mpsd_release, script_dir)
+ # Ensure that the requested toolchains are available in the release
+ if toolchains == "ALL":
+ toolchains = available_toolchains
+ elif toolchains == "NONE":
+ # No toolchains requested, so we only create the env and print the list of available toolchains
+ print(
+ f"No toolchains requested. Available toolchains for release {mpsd_release} are: \n {available_toolchains}"
+ )
+ return
+
+ for toolchain in toolchains:
+ if toolchain not in available_toolchains:
+ raise Exception(
+ f"Toolchain '{toolchain}' is not available in release {mpsd_release}."
+ )
+
+ # Install the toolchains
+ with os_chdir(toolchain_dir):
+ # run spack_setup_script with the toolchains as arguments
+ # if the log folder doesn't exist, create it
+ if not os.path.exists("logs"):
+ os.mkdir("logs")
+ for toolchain in toolchains:
+ # Set the install log file name to config_vars["install_log_file"]
+ # and replace _toolchains_ with the toolchain name and _mpsd_spack_ver_ with mpsd_release
+
+ print(f"Installing toolchain {toolchain} to {toolchain_dir}")
+ install_log_file = (
+ config_vars["build_log_file"]
+ .replace("mpsd_spack_ver_", f"{mpsd_release}_")
+ .replace("_toolchains_", f"_{toolchain}_")
+ )
+ # log the command
+ setup_log_cmd(
+ mpsd_release,
+ script_dir,
+ msg=f"installing {toolchain} and logging at {install_log_file}",
+ )
+ setup_log_cmd(
+ mpsd_release,
+ script_dir,
+ msg=f"CMD: bash {spack_setup_script} {' '.join(install_flags)} {toolchain}",
+ )
+ subprocess.run(
+ f"bash {spack_setup_script} {' '.join(install_flags)} {toolchain} 2>&1 | tee -a {install_log_file} ",
+ shell=True,
+ )
+
+
+def remove_environment(release, toolchains, target_dir):
+ print(f"Removing release {release} with toolchains {toolchains} from {target_dir}")
+
+
+def start_new_environment(release, from_release, target_dir):
+ print(f"Starting new release {release} from {from_release} to {target_dir}")
+
+
+def main():
+ parser = argparse.ArgumentParser(description=about_tool)
+ subparsers = parser.add_subparsers(
+ dest="action", title="actions", description="valid actions", required=True
+ )
+ subparsers.required = True
+ list_of_cmds = [
+ ("prepare", "Prepare the environment for installation on the disk"),
+ ("install", "Install a software environment"),
+ ("reinstall", "Reinstall a software environment"),
+ ("remove", "Remove a software environment or toolchains from an environment"),
+ ("start-new", "Start a new software environment version"),
+ ]
+ for cmd, help_text in list_of_cmds:
+ subp = subparsers.add_parser(cmd, help=help_text)
+
+ if cmd == "start-new":
+ subp.add_argument(
+ "--from-release",
+ dest="from_release",
+ type=str,
+ required=True,
+ help="Release version to start from",
+ )
+ subp.add_argument(
+ "--to-release",
+ dest="to_release",
+ type=str,
+ required=True,
+ help="Release version to create",
+ )
+
+ else:
+ subp.add_argument(
+ "release", type=str, help="Release version to install or remove"
+ )
+ if cmd in ["install", "reinstall"]:
+ subp.add_argument(
+ "--toolchains", # first option defines attribute
+ # name `args.toolchains` in `args = parser_args()`
+ "--toolchain", # allow singular as alternative
+ # (-> creates attribute `args.toolchains` if used)
+ type=str,
+ dest="toolchains",
+ nargs="+",
+ default="NONE",
+ help="List of toolchains to install (use '--toolchains ALL' to install all toolchains). If nothing is specified, list of available toolchains for the release would be shown after creating the environment.",
+ )
+ subp.add_argument(
+ "--enable-build-cache",
+ action="store_true",
+ help="Enable Spack build cache. Useful for reinstallation but consumes time and disk space",
+ )
+ # Carry out the action
+ args = parser.parse_args()
+
+ # target dir is the place where this script exists. the release `dev` in script_dir/dev-23a
+ script_dir = Path(os.path.dirname(os.path.realpath(__file__)))
+
+ # Check the command and run related function
+ if args.action == "remove":
+ remove_environment(args.release, args.toolchains, script_dir)
+ elif args.action == "start-new":
+ start_new_environment(args.from_release, args.to_release, script_dir)
+ elif args.action == "install":
+ install_environment(
+ args.release, args.toolchains, script_dir, False, args.enable_build_cache
+ )
+ elif args.action == "prepare":
+ prepare_environment(args.release, script_dir)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tests.py b/tests.py
new file mode 100644
index 0000000000000000000000000000000000000000..31d578fed2b66416f112a0a6fe0b59985e027b35
--- /dev/null
+++ b/tests.py
@@ -0,0 +1,213 @@
+import os
+import pytest
+import importlib
+import subprocess
+import shutil
+from pathlib import Path
+
+mod = importlib.import_module("mpsd-software-environment")
+
+
+def test_os_chdir(tmp_path):
+ # create a temporary directory for testing
+ temp_dir = tmp_path / "test_os_chdir"
+ temp_dir.mkdir()
+
+ # initial current working directory
+ initial_cwd = os.getcwd()
+
+ # change to the temporary directory using os_chdir
+ with mod.os_chdir(str(temp_dir)):
+ assert os.getcwd() == str(temp_dir)
+
+ # current working directory should be back to initial directory
+ assert os.getcwd() == initial_cwd
+
+
+def test_prepare_environment(tmp_path):
+ # simulate running ./install-software-environment.py --release dev-23a --target-directory /tmp/test_prepare_env
+ # prepare_env is run when cmd is not specified, we can test cmd='prepare' and cmd=None to check both cases
+
+ script_dir = tmp_path / "test_prepare_env"
+ spack_environments = "spack-environments"
+ mpsd_release_to_test = "dev-23a"
+ release_base_dir = script_dir / mpsd_release_to_test
+ # check that the test directory does not exist
+ assert not script_dir.exists()
+
+ result = mod.prepare_environment(
+ mpsd_release=mpsd_release_to_test, script_dir=(script_dir)
+ )
+ # wait for 20 seconds for the git clone to finish
+ # time.sleep(20)
+ # check if the directory now is created
+ assert release_base_dir.exists()
+ # check for spack-environments directory
+ assert spack_environments in os.listdir(release_base_dir)
+ # check if the git branch is correctly checked out
+ assert (
+ subprocess.run(
+ f"cd {str(release_base_dir/spack_environments)} && git branch",
+ shell=True,
+ capture_output=True,
+ )
+ .stdout.decode("utf-8")
+ .split("\n")[0]
+ == f"* {mpsd_release_to_test}"
+ )
+ # check that result is a list and contains atleast ['global','foss2021a-mpi']
+ assert isinstance(result, list)
+ assert "global" in result
+ assert "foss2021a-mpi" in result
+
+ # Expect an Exception when wrong mpsd_release is provided
+ with pytest.raises(Exception):
+ result = mod.prepare_environment(
+ mpsd_release="wrong-mpsd-release", script_dir=(script_dir)
+ )
+
+
+def test_setup_log_cmd(tmp_path):
+ # check that logs/install-software-environment.log is updated when the module is run
+ log_file = "install.log"
+
+ script_dir = tmp_path / "test_prepare_env"
+ mpsd_release_to_test = "dev-23a"
+ release_base_dir = script_dir / mpsd_release_to_test
+ if os.path.exists(release_base_dir / log_file):
+ initial_bytes = os.path.getsize(log_file)
+ else:
+ initial_bytes = 0
+
+ # run the prepare_env functionality
+ mod.prepare_environment(mpsd_release=mpsd_release_to_test, script_dir=(script_dir))
+
+ # check that logs/install-software-environment.log is updated
+ assert os.path.exists(release_base_dir / log_file)
+ assert os.path.getsize(release_base_dir / log_file) > initial_bytes
+
+ # Check that the log file has "Spack environments branch: dev-23a " in the last line
+ with open(release_base_dir / log_file, "r") as f:
+ last_line = f.readlines()[-1]
+ assert "Spack environments branch: dev-23a " in last_line
+
+
+def test_install_environment(tmp_path):
+ # Test the installation part
+ # This is a long test, its handy to test this with print statements printed to stdout, use:
+ # pytest -s
+ # Expect an Exception when wrong toolchains are provided
+ with pytest.raises(Exception):
+ mod.install_environment(
+ mpsd_release="dev-23a",
+ toolchains=["wrong-toolchain"],
+ script_dir=(tmp_path),
+ )
+ # Expect an Exception when wrong mpsd_release is provided ( part of prepare_environment)
+ with pytest.raises(Exception):
+ mod.install_environment(
+ mpsd_release="wrong-mpsd-release",
+ toolchains=["foss2021a-mpi"],
+ script_dir=(tmp_path),
+ )
+ # prepare a test of global generic with only zlib to test the installation
+ # prepare dev-23a release
+ # script_dir = tmp_path / "test_global_generic"
+ # for actaual installation avoid tmp_path as the lenght of the path is too long and spack complains
+ script_dir = Path("/tmp/test_global_generic")
+ if script_dir.exists():
+ shutil.rmtree(script_dir)
+ script_dir.mkdir(exist_ok=True, parents=True)
+ mpsd_release_to_test = "dev-23a"
+ toolchain_to_test = "global_generic"
+ mpsd_microarch = os.getenv("MPSD_MICROARCH", "UNKNOWN_MICROARCH")
+ release_base_dir = script_dir / mpsd_release_to_test
+ mod.prepare_environment(mpsd_release=mpsd_release_to_test, script_dir=(script_dir))
+ # Patch the spack environments to create a fake global_generic
+ # create a test toolchain
+ toolchain_src_dir = release_base_dir / "spack-environments" / "toolchains"
+ # with mod.os_chdir(toolchain_src_dir):
+ # subprocess.run(
+ # "cp -r foss2021a-mpi fuss1999a", shell=True, capture_output=True
+ # )
+ # add zlib as a spec to global_generic
+ with open(toolchain_src_dir / "global_generic" / "global_packages.list", "w") as f:
+ f.write("zlib@1.2.13 \n")
+
+ # add zlib to whitelist of module creation file by replacing anaconda3%gcc@10.2.1 with zlib@1.2.13
+ # in release_base_dir / "spack-environments/spack_overlay/etc/spack/modules.yaml"
+ module_file = (
+ release_base_dir / "spack-environments/spack_overlay/etc/spack/modules.yaml"
+ )
+ with open(module_file, "r") as f:
+ lines = f.read().replace("anaconda3%gcc@10.2.1", "zlib@1.2.13")
+ with open(module_file, "w") as f:
+ f.write(lines)
+
+ # Replace gcc@10.2.1 with gcc#13.1.1 or available system gcc for testing on laptop
+ gcc_ver = (
+ subprocess.run(["gcc -dumpfullversion"], shell=True, capture_output=True)
+ .stdout.decode("utf-8")
+ .strip()
+ )
+ setup_file = release_base_dir / "spack-environments/spack_setup.sh"
+ with open(setup_file, "r") as f:
+ lines = f.read().replace(
+ 'system_compiler="gcc@10.2.1"', f'system_compiler="gcc@{gcc_ver}"'
+ )
+ with open(setup_file, "w") as f:
+ f.write(lines)
+ # install global_generic toolchain
+ mod.install_environment(
+ mpsd_release=mpsd_release_to_test,
+ toolchains=[toolchain_to_test],
+ script_dir=script_dir,
+ enable_build_cache=False,
+ )
+ # test that the build log is created correctly
+ # check that a file with glob build_globale_generic_dev-23a*.log exists at release_base_dir/mpsd_microarch
+ # print("Debug here ")
+ # time.sleep(10)
+ build_log = list(
+ (release_base_dir / mpsd_microarch / "logs").glob(
+ f"{mpsd_release_to_test}_{toolchain_to_test}_*.log"
+ )
+ )
+ assert len(build_log) > 0
+ # take the most recent build log
+ build_log = sorted(build_log)[0]
+ # check that the build log contains statement ##### Installation finished
+ with open(build_log, "r") as f:
+ lines = f.read()
+ assert "##### Installation finished" in lines
+ build_log_file_name = os.path.basename(build_log)
+
+ # assert that install log files exists
+ assert os.path.exists(release_base_dir / "install.log")
+
+ # assert that the build log is written to the install log file
+ os.path.basename(build_log)
+ with open(release_base_dir / "install.log", "r") as f:
+ lines = f.read()
+ assert (
+ f"installing {toolchain_to_test} and logging at logs/{build_log_file_name}"
+ in lines
+ )
+ # assert that the module files are created correctly
+ assert os.path.exists(release_base_dir / mpsd_microarch)
+ assert os.path.exists(release_base_dir / mpsd_microarch / "lmod")
+ # assert that lmod/module-index.yaml contains zlib
+ with open(
+ release_base_dir / mpsd_microarch / "lmod" / "module-index.yaml", "r"
+ ) as f:
+ lines = f.read()
+ assert "zlib" in lines
+
+
+def test_interface(tmp_path):
+ pass
+ # ensure that installing without toolchains only passes the available toolchains
+ # check that the script branch and hash are correct when running the script
+ # check that the help message is printed when no arguments are provided
+ # check that the help message is printed when -h is provided
+ # check that the error messages are also logged to the log file