Merge branch 'linux-debian11' into 'main'

Move linux-debian11 into main

See merge request mpsd-cs/mpsd-software-environments!19

.gitignore

+.vscode
+**/*.pyc
+logs
+dev-23a/
\ No newline at end of file

.gitlab-ci.yml

+image: python:latest
+
+stages:
+  - test
+
+# prepare set up of latest python image to run style checks
+.prepare_style: &prepare_style
+  - cat /etc/issue
+  - python3 -V
+  - which python3
+  - python3 -m venv ../venv
+  - source ../venv/bin/activate
+  - which python3
+  - pwd
+  - ls -l
+  - pip install -U pip
+  - pip --version
+  - pip install pytest black ruff pydocstyle
+  - pytest --version
+  - cat /etc/issue
+
+
+# prepare set up of Debian system to run py.test
+.prepare_debian: &prepare_debian
+  - echo "Execute this command before any 'script:' commands."
+  - cat /etc/issue
+  - pwd
+  - ls -l
+
+  - echo "Install Python3"
+  - apt-get update
+  - apt-get install -y python3 python3-venv
+  - python3 -m venv --help
+  - python3 -m venv venv
+  - source venv/bin/activate
+  - which python3
+  - python3 --version
+
+  - echo "Install Python dependencies for running the tests"
+  - pip install -U pip
+  - pip --version
+  - pip install pytest black ruff archspec
+
+  - echo "Diagnostics - which versions are we using"
+  - python3 --version
+  - pytest --version
+  - black --version
+  - ruff --version
+
+  - echo "Install additional packages we need to run spack-setup.sh"
+  - apt-get install -y git rsync
+  - echo "Install additional packages we need to run spack"
+  # Taken from https://github.com/fangohr/oommf-in-spack/blob/main/Dockerfile
+  - apt-get install -y --no-install-recommends
+            autoconf
+            build-essential
+            ca-certificates
+            coreutils
+            curl
+            environment-modules
+            file
+            gfortran
+            git
+            openssh-server
+            unzip
+
+  - export MPSD_MICROARCH=$(archspec cpu)
+  - echo "Setting MPSD_MICROARCH variable to $MPSD_MICROARCH"
+  - echo "Which version of Debian are we running?"
+  - cat /etc/issue
+
+
+style:
+  stage: test
+  image: python:latest
+  script:
+    - *prepare_style
+    - black --version
+    - ruff --version
+    - pydocstyle --version
+    - ruff .
+    - black --check --diff .
+    - pydocstyle mpsd-software-environment.py
+    - pydocstyle tests.py
+    # we could also use `ruff --select D` for pycodestyle. But the behaviour is not exactly the same.
+
+
+test-bullseye:
+  stage: test
+  image: debian:bullseye-slim
+  script:
+    - *prepare_debian
+    - pytest -v -l tests.py
+
+test-bookworm:
+  stage: test
+  image: debian:bookworm-slim
+  script:
+    - *prepare_debian
+    - pytest -v -l tests.py
+

install-dev23a.sh

 #!/bin/bash
+# Script to build all toolchains for this MPSD release ( 23a )
+# Run this script inside the cloned folder for eg:
+# mpsddeb@mpsd-hpc-ibm-022:/opt_mpsd/linux-debian11/mpsd-software-environments$ ./install-dev23a.sh 
+
 set -e
+cd ..
 mkdir -p dev-23a
 cd dev-23a
 # clone repo if it doesn't exist yet
 [ -d 'spack-environments' ] || git clone git@gitlab.gwdg.de:mpsd-cs/spack-environments.git 
 pushd spack-environments
 git checkout dev-23a
+git pull
 popd
 mkdir -p sandybridge
 cd sandybridge
@@ -13,3 +19,8 @@ cd sandybridge
 ../spack-environments/spack_setup.sh foss2021a-serial
 ../spack-environments/spack_setup.sh foss2021a-mpi
 ../spack-environments/spack_setup.sh foss2021a-cuda-mpi
+
+
+../spack-environments/spack_setup.sh foss2022a-serial
+../spack-environments/spack_setup.sh foss2022a-mpi
+../spack-environments/spack_setup.sh foss2022a-cuda-mpi

tests.py

+"""Tests for mpsd-software-environment.py."""
+
+import importlib
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+
+mod = importlib.import_module("mpsd-software-environment")
+
+
+def create_mock_git_repository(target_directory, create_directory=True):
+    """
+    Create a git repository in the directory `target_directory`.
+
+    Arguments
+    ---------
+    target_directory : pathlib.Path
+      - path at which the root of the repository should be located (i.e. `.git` folder)
+
+    create_directory : bool
+      - create `target_directory` and parent directories if True
+
+    """
+    # create directory first
+    if create_directory:
+        target_directory.mkdir(parents=True)
+
+    # then create git repository:
+    with mod.os_chdir(str(target_directory)):
+        subprocess.run("git init .", shell=True, check=True)
+        subprocess.run("echo 'fake content' > readme.txt", shell=True, check=True)
+        subprocess.run("git add readme.txt", shell=True, check=True)
+        subprocess.run("pwd", shell=True)
+
+        # if email and username are not available (such as on naked test container),
+        # git may complain. We set a temporary user for this one commit to work around
+        # that.
+        user_details = "-c user.name='Tes Ta' -c user.email='tester@some-ci.org'"
+        subprocess.run(
+            f'git {user_details} commit -m "first commit" readme.txt',
+            shell=True,
+            check=True,
+        )
+
+
+def test_os_chdir(tmp_path):
+    """Test the os_chdir context manager."""
+    # create a temporary directory for testing
+    temp_dir = tmp_path / "test_os_chdir"
+    temp_dir.mkdir()
+
+    # initial current working directory
+    initial_cwd = os.getcwd()
+
+    # change to the temporary directory using os_chdir
+    with mod.os_chdir(str(temp_dir)):
+        assert os.getcwd() == str(temp_dir)
+
+    # current working directory should be back to initial directory
+    assert os.getcwd() == initial_cwd
+
+
+def test_run_method(tmp_path):
+    """Run tests for run method."""
+    run = mod.run
+
+    # test a command with options:
+    assert run(["date", "+%Y-%m-%d"]).returncode == 0
+    assert run("date +%Y-%m-%d", shell=True).returncode == 0
+
+    # tests interacting with the file system
+    with mod.os_chdir(str(tmp_path)):
+        # ensure single string command works
+        assert run(("ls -l"), shell=True).returncode == 0
+        # test spaces are handled correctly:
+        assert run(["touch", "file1", "file2"]).returncode == 0
+        assert os.path.exists("file1")
+        assert os.path.exists("file2")
+        # test output is captured:
+        assert (
+            b"Hello, world!\n"
+            in run(["echo", "Hello, world!"], capture_output=True).stdout
+        )
+
+    # check exceptions
+    with pytest.raises(FileNotFoundError):
+        run(["doesnotexistcommand"])
+
+    # check error code is checked
+    # 1. expect this to parse: return code is non-zero, but we don't check
+    run(["ls", "/doesnotexist"]),
+    # 2. expect this to fail:
+    with pytest.raises(subprocess.CalledProcessError):
+        run(["ls", "/doesnotexist"], check=True)
+
+
+def test_prepare_environment(tmp_path):
+    """Simulate running preparation of environment.
+    
+    Simulate running ./install-software-environment.py --release dev-23a \
+      --target-directory /tmp/test_prepare_env
+    prepare_env is run when cmd is not specified, we can test cmd='prepare'
+    and cmd=None to check both cases
+    """
+    script_dir = tmp_path / "mpsd_opt" / "linux_debian_11"
+    spack_environments = "spack-environments"
+    mpsd_release_to_test = "dev-23a"
+    release_base_dir = script_dir / mpsd_release_to_test
+    # check that the test directory does not exist
+    assert not script_dir.exists()
+
+    # prepare_environment expects to be executed in git repository
+    # (mpsd-software-environments). It queries the commit on which we are to
+    # log that information. For this to work, we need to execute the command
+    # within a directory tree that has a git repository at the same or high
+    # level. Let's create one:
+    create_mock_git_repository(script_dir)
+
+    # now call the function we want to test
+    result = mod.prepare_environment(
+        mpsd_release=mpsd_release_to_test, script_dir=script_dir
+    )
+
+    # check if the directory now is created
+    assert release_base_dir.exists()
+    # check for spack-environments directory
+    assert spack_environments in os.listdir(release_base_dir)
+
+    # check if the git branch is correctly checked out. We expect output such as
+    # git_branch_stdout = '* dev-23a\n  develop\n'
+    # The entry with the '* ' prefix is the active branch.
+    git_branch_output_raw = subprocess.run(
+        f"cd {str(release_base_dir/spack_environments)} && git branch",
+        shell=True,
+        capture_output=True,
+    )
+    git_branch_stdout = git_branch_output_raw.stdout.decode("utf-8")
+    assert f"* {mpsd_release_to_test}" in git_branch_stdout
+
+    # check that result is a list and contains atleast ['global','foss2021a-mpi']
+    assert isinstance(result, list)
+    assert "global" in result
+    assert "foss2021a-mpi" in result
+
+    # Expect an Exception when wrong mpsd_release is provided
+    with pytest.raises(Exception):
+        result = mod.prepare_environment(
+            mpsd_release="wrong-mpsd-release", script_dir=(script_dir)
+        )
+
+
+def test_setup_log_cmd(tmp_path):
+    """Check that log is updated.
+
+    Check that logs/install-software-environment.log is updated when the module is run
+    """
+    log_file = "install.log"
+
+    script_dir = tmp_path / "test_prepare_env"
+    mpsd_release_to_test = "dev-23a"
+    release_base_dir = script_dir / mpsd_release_to_test
+    if os.path.exists(release_base_dir / log_file):
+        initial_bytes = os.path.getsize(log_file)
+    else:
+        initial_bytes = 0
+
+    # run the prepare_env functionality
+    create_mock_git_repository(target_directory=script_dir, create_directory=True)
+    mod.prepare_environment(mpsd_release=mpsd_release_to_test, script_dir=(script_dir))
+
+    # check that logs/install-software-environment.log is updated
+    assert os.path.exists(release_base_dir / log_file)
+    assert os.path.getsize(release_base_dir / log_file) > initial_bytes
+
+    # Check that the log file has "Spack environments branch: dev-23a " in the last line
+    with open(release_base_dir / log_file, "r") as f:
+        last_line = f.readlines()[-1]
+        assert "Spack environments branch: dev-23a " in last_line
+
+
+def test_install_environment_wrong_toolchain(tmp_path):
+    """Test exception is raised for non-existing toolchain."""
+    # Expect an Exception when wrong toolchains are provided
+    with pytest.raises(Exception):
+        mod.install_environment(
+            mpsd_release="dev-23a",
+            toolchains=["wrong-toolchain"],
+            script_dir=(tmp_path),
+        )
+
+
+def test_install_environment_wrong_mpsd_release(tmp_path):
+    """Test exception is raised for non-existing mpsd release."""
+    # Expect an Exception when wrong mpsd_release is provided (part of
+    # prepare_environment)
+    with pytest.raises(Exception):
+        mod.install_environment(
+            mpsd_release="wrong-mpsd-release",
+            toolchains=["foss2021a-mpi"],
+            script_dir=(tmp_path),
+        )
+
+
+def test_install_environment_zlib():
+    """Test installation of toolchain."""
+    # Prepare a test installation of global generic
+    # with only zlib to test the installation
+    # This is a long test,
+    # its handy to test this with print statements printed to
+    # stdout, use:
+    #   pytest -s
+    # for this installation avoid tmp_path as
+    # the length of the path becomes too long and spack complains
+    script_dir = Path("/tmp/test_global_generic")
+    if script_dir.exists():
+        shutil.rmtree(script_dir)
+    script_dir.mkdir(exist_ok=True, parents=True)
+    mpsd_release_to_test = "dev-23a"
+    toolchain_to_test = "global_generic"
+    mpsd_microarch = os.getenv("MPSD_MICROARCH", "UNKNOWN_MICROARCH")
+    release_base_dir = script_dir / mpsd_release_to_test
+    create_mock_git_repository(target_directory=script_dir, create_directory=False)
+    mod.prepare_environment(mpsd_release=mpsd_release_to_test, script_dir=(script_dir))
+    # Patch the spack environments to create a fake global_generic
+    # create a test toolchain
+    toolchain_src_dir = release_base_dir / "spack-environments" / "toolchains"
+    # with mod.os_chdir(toolchain_src_dir):
+    #     subprocess.run(
+    #         "cp -r foss2021a-mpi fuss1999a", shell=True, capture_output=True
+    #     )
+    # add zlib as a spec to global_generic
+    with open(toolchain_src_dir / "global_generic" / "global_packages.list", "w") as f:
+        f.write("zlib@1.2.13 \n")
+
+    # add zlib to whitelist of module creation file by replacing anaconda3%gcc@10.2.1
+    # with zlib@1.2.13
+    # in release_base_dir / "spack-environments/spack_overlay/etc/spack/modules.yaml"
+    module_file = (
+        release_base_dir / "spack-environments/spack_overlay/etc/spack/modules.yaml"
+    )
+    with open(module_file, "r") as f:
+        lines = f.read().replace("anaconda3%gcc@10.2.1", "zlib@1.2.13")
+    with open(module_file, "w") as f:
+        f.write(lines)
+
+    # Replace gcc@10.2.1 with gcc#13.1.1 or available system gcc for testing on laptop
+    gcc_ver = (
+        subprocess.run(["gcc -dumpfullversion"], shell=True, capture_output=True)
+        .stdout.decode("utf-8")
+        .strip()
+    )
+    assert len(gcc_ver) > 3, f"Couldn't find gcc {gcc_ver=}"
+
+    setup_file = release_base_dir / "spack-environments/spack_setup.sh"
+    with open(setup_file, "r") as f:
+        lines = f.read().replace(
+            'system_compiler="gcc@10.2.1"', f'system_compiler="gcc@{gcc_ver}"'
+        )
+    with open(setup_file, "w") as f:
+        f.write(lines)
+    # install global_generic toolchain
+    mod.install_environment(
+        mpsd_release=mpsd_release_to_test,
+        toolchains=[toolchain_to_test],
+        script_dir=script_dir,
+        enable_build_cache=False,
+    )
+    # test that the build log is created correctly
+    # check that a file with glob build_globale_generic_dev-23a*.log exists at
+    # release_base_dir/mpsd_microarch
+    # print("Debug here ")
+    # time.sleep(10)
+    build_log = list(
+        (release_base_dir / mpsd_microarch / "logs").glob(
+            f"{mpsd_release_to_test}_{toolchain_to_test}_*.log"
+        )
+    )
+    assert len(build_log) > 0
+    # take the most recent build log
+    build_log = sorted(build_log)[0]
+    # check that the build log contains statement ##### Installation finished
+    with open(build_log, "r") as f:
+        lines = f.read()
+        assert "##### Installation finished" in lines
+    build_log_file_name = os.path.basename(build_log)
+
+    # assert that install log files exists
+    assert os.path.exists(release_base_dir / "install.log")
+
+    # assert that the build log is written to the install log file
+    os.path.basename(build_log)
+    with open(release_base_dir / "install.log", "r") as f:
+        lines = f.read()
+        assert (
+            f"installing {toolchain_to_test} and logging at logs/{build_log_file_name}"
+            in lines
+        )
+    # assert that the module files are created correctly
+    assert os.path.exists(release_base_dir / mpsd_microarch)
+    assert os.path.exists(release_base_dir / mpsd_microarch / "lmod")
+    # assert that lmod/module-index.yaml contains zlib
+    with open(
+        release_base_dir / mpsd_microarch / "lmod" / "module-index.yaml", "r"
+    ) as f:
+        lines = f.read()
+        assert "zlib" in lines
+
+
+def test_interface(tmp_path):
+    """Test other things (not implemented yet)."""
+    pass
+    # ensure that installing without toolchains only passes the available toolchains
+    # check that the script branch and hash are correct when running the script
+    # check that the help message is printed when no arguments are provided
+    # check that the help message is printed when -h is provided
+    # check that the error messages are also logged to the log file