import importlib
import os
import shutil
import subprocess
from pathlib import Path
import pytest
mod = importlib.import_module("mpsd-software-environment")
def test_os_chdir(tmp_path):
# create a temporary directory for testing
temp_dir = tmp_path / "test_os_chdir"
temp_dir.mkdir()
# initial current working directory
initial_cwd = os.getcwd()
# change to the temporary directory using os_chdir
with mod.os_chdir(str(temp_dir)):
assert os.getcwd() == str(temp_dir)
# current working directory should be back to initial directory
assert os.getcwd() == initial_cwd
def test_run_method(tmp_path):
run = mod.run
# test a command with options:
assert run(["date", "+%Y-%m-%d"]).returncode == 0
assert run("date +%Y-%m-%d", shell=True).returncode == 0
with mod.os_chdir(str(tmp_path)):
# ensure single string command works
assert run(("ls -l")).returncode == 0
# test spaces are handled correctly:
assert run(["touch", "file1 file2"]).returncode == 0
assert os.path.exists("file1")
assert os.path.exists("file2")
# test output is captured:
assert (
b"Hello, world!\n"
in run(["echo", "Hello, world!"], capture_output=True).stdout
)
# test counter works:
assert b"##-04\n" in run(["echo", "Hello, world!"], capture_output=True).stdout
def test_prepare_environment(tmp_path):
# simulate running ./install-software-environment.py --release dev-23a \
# --target-directory /tmp/test_prepare_env
# prepare_env is run when cmd is not specified, we can test cmd='prepare'
# and cmd=None to check both cases
script_dir = tmp_path / "test_prepare_env"
spack_environments = "spack-environments"
mpsd_release_to_test = "dev-23a"
release_base_dir = script_dir / mpsd_release_to_test
# check that the test directory does not exist
assert not script_dir.exists()
result = mod.prepare_environment(
mpsd_release=mpsd_release_to_test, script_dir=(script_dir)
)
# wait for 20 seconds for the git clone to finish
# time.sleep(20)
# check if the directory now is created
assert release_base_dir.exists()
# check for spack-environments directory
assert spack_environments in os.listdir(release_base_dir)
# check if the git branch is correctly checked out
assert (
subprocess.run(
f"cd {str(release_base_dir/spack_environments)} && git branch",
shell=True,
capture_output=True,
)
.stdout.decode("utf-8")
.split("\n")[0]
== f"* {mpsd_release_to_test}"
)
# check that result is a list and contains atleast ['global','foss2021a-mpi']
assert isinstance(result, list)
assert "global" in result
assert "foss2021a-mpi" in result
# Expect an Exception when wrong mpsd_release is provided
with pytest.raises(Exception):
result = mod.prepare_environment(
mpsd_release="wrong-mpsd-release", script_dir=(script_dir)
)
def test_setup_log_cmd(tmp_path):
# check that logs/install-software-environment.log is updated when the module is run
log_file = "install.log"
script_dir = tmp_path / "test_prepare_env"
mpsd_release_to_test = "dev-23a"
release_base_dir = script_dir / mpsd_release_to_test
if os.path.exists(release_base_dir / log_file):
initial_bytes = os.path.getsize(log_file)
else:
initial_bytes = 0
# run the prepare_env functionality
mod.prepare_environment(mpsd_release=mpsd_release_to_test, script_dir=(script_dir))
# check that logs/install-software-environment.log is updated
assert os.path.exists(release_base_dir / log_file)
assert os.path.getsize(release_base_dir / log_file) > initial_bytes
# Check that the log file has "Spack environments branch: dev-23a " in the last line
with open(release_base_dir / log_file, "r") as f:
last_line = f.readlines()[-1]
assert "Spack environments branch: dev-23a " in last_line
def test_install_environment(tmp_path):
# Test the installation part
# This is a long test, its handy to test this with print statements printed to
# stdout, use:
# pytest -s
# Expect an Exception when wrong toolchains are provided
with pytest.raises(Exception):
mod.install_environment(
mpsd_release="dev-23a",
toolchains=["wrong-toolchain"],
script_dir=(tmp_path),
)
# Expect an Exception when wrong mpsd_release is provided (part of
# prepare_environment)
with pytest.raises(Exception):
mod.install_environment(
mpsd_release="wrong-mpsd-release",
toolchains=["foss2021a-mpi"],
script_dir=(tmp_path),
)
# prepare a test of global generic with only zlib to test the installation
# prepare dev-23a release
# script_dir = tmp_path / "test_global_generic"
# for actaual installation avoid tmp_path as the lenght of the path is too long
# and spack complains
script_dir = Path("/tmp/test_global_generic")
if script_dir.exists():
shutil.rmtree(script_dir)
script_dir.mkdir(exist_ok=True, parents=True)
mpsd_release_to_test = "dev-23a"
toolchain_to_test = "global_generic"
mpsd_microarch = os.getenv("MPSD_MICROARCH", "UNKNOWN_MICROARCH")
release_base_dir = script_dir / mpsd_release_to_test
mod.prepare_environment(mpsd_release=mpsd_release_to_test, script_dir=(script_dir))
# Patch the spack environments to create a fake global_generic
# create a test toolchain
toolchain_src_dir = release_base_dir / "spack-environments" / "toolchains"
# with mod.os_chdir(toolchain_src_dir):
# subprocess.run(
# "cp -r foss2021a-mpi fuss1999a", shell=True, capture_output=True
# )
# add zlib as a spec to global_generic
with open(toolchain_src_dir / "global_generic" / "global_packages.list", "w") as f:
f.write("zlib@1.2.13 \n")
# add zlib to whitelist of module creation file by replacing anaconda3%gcc@10.2.1
# with zlib@1.2.13
# in release_base_dir / "spack-environments/spack_overlay/etc/spack/modules.yaml"
module_file = (
release_base_dir / "spack-environments/spack_overlay/etc/spack/modules.yaml"
)
with open(module_file, "r") as f:
lines = f.read().replace("anaconda3%gcc@10.2.1", "zlib@1.2.13")
with open(module_file, "w") as f:
f.write(lines)
# Replace gcc@10.2.1 with gcc#13.1.1 or available system gcc for testing on laptop
gcc_ver = (
subprocess.run(["gcc -dumpfullversion"], shell=True, capture_output=True)
.stdout.decode("utf-8")
.strip()
)
setup_file = release_base_dir / "spack-environments/spack_setup.sh"
with open(setup_file, "r") as f:
lines = f.read().replace(
'system_compiler="gcc@10.2.1"', f'system_compiler="gcc@{gcc_ver}"'
)
with open(setup_file, "w") as f:
f.write(lines)
# install global_generic toolchain
mod.install_environment(
mpsd_release=mpsd_release_to_test,
toolchains=[toolchain_to_test],
script_dir=script_dir,
enable_build_cache=False,
)
# test that the build log is created correctly
# check that a file with glob build_globale_generic_dev-23a*.log exists at
# release_base_dir/mpsd_microarch
# print("Debug here ")
# time.sleep(10)
build_log = list(
(release_base_dir / mpsd_microarch / "logs").glob(
f"{mpsd_release_to_test}_{toolchain_to_test}_*.log"
)
)
assert len(build_log) > 0
# take the most recent build log
build_log = sorted(build_log)[0]
# check that the build log contains statement ##### Installation finished
with open(build_log, "r") as f:
lines = f.read()
assert "##### Installation finished" in lines
build_log_file_name = os.path.basename(build_log)
# assert that install log files exists
assert os.path.exists(release_base_dir / "install.log")
# assert that the build log is written to the install log file
os.path.basename(build_log)
with open(release_base_dir / "install.log", "r") as f:
lines = f.read()
assert (
f"installing {toolchain_to_test} and logging at logs/{build_log_file_name}"
in lines
)
# assert that the module files are created correctly
assert os.path.exists(release_base_dir / mpsd_microarch)
assert os.path.exists(release_base_dir / mpsd_microarch / "lmod")
# assert that lmod/module-index.yaml contains zlib
with open(
release_base_dir / mpsd_microarch / "lmod" / "module-index.yaml", "r"
) as f:
lines = f.read()
assert "zlib" in lines
def test_interface(tmp_path):
pass
# ensure that installing without toolchains only passes the available toolchains
# check that the script branch and hash are correct when running the script
# check that the help message is printed when no arguments are provided
# check that the help message is printed when -h is provided
# check that the error messages are also logged to the log file