diff --git a/development.rst b/development.rst
index f034c980cf4e54eb7b229063b89c4b24950fae5b..0c733bf1134a6be6d20f022406f6a4fa9ccf54dc 100644
--- a/development.rst
+++ b/development.rst
@@ -33,4 +33,6 @@ Here is a list of exit codes and what they mean:
| 20 | Requested package set is not available | Use 'available' command to see list of available package_sets |
| 30 | Current directory is already initialised | Check if you are in the right directory |
| 40 | Current directory is not initialised | Check if you are in the right directory, if so use 'init' command to initialise |
+| 50 | No package set is selected | Please specify package_sets to remove, or 'ALL' to remove all package_sets |
+| 60 | User didnt confirm removing the release | Please input 'y' if you are sure about removing the entire release |
+-----------+------------------------------------------+----------------------------------------------------------------------------------+
diff --git a/src/mpsd_software_manager/mpsd_software.py b/src/mpsd_software_manager/mpsd_software.py
index 73b4f4d6d151eb531f866ffe424928a8aa0a53e4..b26344a4b570773dd2e997dba95dd06e70ca3847 100755
--- a/src/mpsd_software_manager/mpsd_software.py
+++ b/src/mpsd_software_manager/mpsd_software.py
@@ -5,18 +5,17 @@
import argparse
import datetime
+import importlib.metadata
import logging
import os
+import re
import subprocess
import sys
import tempfile
import time
+from functools import cache
from pathlib import Path
from typing import List, Tuple, Union
-import re
-import shutil
-from functools import cache
-import importlib.metadata
__version__ = importlib.metadata.version(__package__ or __name__)
@@ -1080,39 +1079,177 @@ def install_environment(
def remove_environment(mpsd_release, root_dir, package_sets="NONE", force_remove=False):
- """Remove release from installation."""
+ """Remove release from installation.
+
+ Handle 3 situations :
+ 1. remove does not specify what to remove
+ -> warn and exit
+ 2. remove all package_sets from release
+ -> remove release folder except logs
+ 3. remove specific package_sets from release
+ -> remove spack environments via spack commands
+
+ Parameters
+ ----------
+ mpsd_release : str
+ A string representing the MPSD release version.
+ root_dir : pathlib.Path
+ A Path object representing the path to the directory where
+ the release and package_sets will be installed.
+ package_sets : list of str
+ A list of strings representing the package_sets to remove
+ (e.g., "foss2021a-mpi", "global_generic", "ALL").
+ force_remove : bool, optional
+ A boolean indicating whether to force remove the release.
+ If False, the user will be prompted to confirm the removal.
+ Defaults to False.
+
+ Raises
+ ------
+ ValueError
+
+ """
msg = (
f"Removing release {mpsd_release}"
f" with package_sets {package_sets} from {root_dir}"
)
logging.warning(msg)
+
if package_sets == "NONE":
logging.warning(
- "Please specify package_sets to remove, or 'ALL' to remove all toolchains"
+ "Please specify package_sets to remove, or 'ALL' to remove all package_sets"
)
- sys.exit(1)
+ sys.exit(50)
+ # 2nd case: remove the entire release for microarchitecture
+ dir_to_remove = root_dir / mpsd_release / get_native_microarchitecture()
if "ALL" in package_sets:
# we need to remove the entire release folder
logging.warning(
- f"Removing release {mpsd_release} from {root_dir}"
- "do you want to continue? [y/n]"
+ f"Removing release {mpsd_release}"
+ f"from {root_dir} for {get_native_microarchitecture()}"
)
- if force_remove or input().lower() == "y":
- folders_to_remove = os.listdir(root_dir / mpsd_release)
- # skip logs folder
- if "logs" in folders_to_remove:
- folders_to_remove.remove("logs")
- for folder in folders_to_remove:
- shutil.rmtree(root_dir / mpsd_release / folder)
- sys.exit(0)
+ if not force_remove:
+ logging.warning("do you want to continue? [y/n]")
+ if input().lower() != "y":
+ sys.exit(60)
+
+ # Set the remove log file name from create_log_file_names
+ build_log_path = get_log_file_path(mpsd_release, "remove", root_dir, "ALL")
+
+ logging.info(f"> Logging removal of {mpsd_release} at {build_log_path}")
+ folders_to_remove = os.listdir(dir_to_remove)
+ for folder in folders_to_remove:
+ # shutil.rmtree(dir_to_remove / folder) #dosent delete file
+ run(
+ f"rm -rf {dir_to_remove / folder} 2>&1 | tee -a {build_log_path}",
+ shell=True,
+ check=True,
+ )
+ logging.warning(f"Removed release {mpsd_release} from {root_dir}")
+ return
+ # 3rd case: remove specific package_sets from release
for package_set in package_sets:
# we load the spack environment and remove the package_set
- spack_env = ""
- commands_to_execute = [
- f"source {spack_env}",
- f"spack env remove -y {package_set}",
- ]
- run(" && ".join(commands_to_execute), shell=True, check=True)
+ build_log_path = get_log_file_path(
+ mpsd_release, "remove", root_dir, package_set
+ )
+ logging.info(f"> Logging removal of {package_set} at {build_log_path}")
+ if package_set not in ["global_generic", "global"]:
+ remove_spack_environment(
+ dir_to_remove / "spack", package_set, build_log_path
+ )
+ else:
+ # list all specs from the global_packages.list
+ spe_folder = root_dir / mpsd_release / "spack-environments"
+ package_list_file = (
+ spe_folder / "toolchains" / package_set / "global_packages.list"
+ )
+ with open(package_list_file, "r") as f:
+ package_dump = f.read()
+
+ # remove all content from # to the end of the line
+ package_dump = re.sub(r"#.*\n", "\n", package_dump)
+ # replace \\n with "" to remove line breaks
+ package_list = package_dump.replace("\\\n", "").split("\n")
+ # remove all empty lines
+ package_list = [line for line in package_list if line != ""]
+
+ # remove all packages in package_list
+ for package in package_list:
+ logging.info(f"Removing package {package} from installation")
+ remove_spack_package(dir_to_remove / "spack", package, build_log_path)
+
+
+def remove_spack_environment(spack_dir, environment_name, build_log_path=None):
+ """Remove spack environment including packages exclusive to it.
+
+ First activate the environment,
+ then uninstall all packages exclusive to the environment,
+ then deactivate the environment,
+ remove the environment,
+ and finally remove the environment lua file.
+
+ Parameters
+ ----------
+ spack_dir : pathlib.Path
+ A Path object representing the path to the spack directory.
+ environment_name : str
+ A string representing the name of the spack environment to remove.
+ build_log_path : pathlib.Path, optional
+ A Path object representing the path to where the logs will be teed
+ """
+ logging.warning(f"Removing spack environment {environment_name}")
+ spack_env = spack_dir / "share" / "spack" / "setup-env.sh"
+ commands_to_execute = [
+ f"export SPACK_ROOT={spack_dir}", # need to set SPACK_ROOT in dash and sh
+ f". {spack_env}",
+ f"spack env activate {environment_name}",
+ f"for spec in $(spack -e {environment_name} find" # this line continues
+ r' --format "{name}@{version}%{compiler.name}@{compiler.version}");do'
+ " spack uninstall -y $spec; done",
+ "spack env deactivate",
+ f"spack env remove -y {environment_name}",
+ ]
+ build_log_path = build_log_path or "/dev/null"
+ run(
+ "(" + " && ".join(commands_to_execute) + f") 2>&1 |tee -a {build_log_path}",
+ shell=True,
+ check=True,
+ )
+ # remove the environment lua file
+ lua_file = (
+ spack_dir / ".." / "lmod" / "Core" / "toolchains" / f"{environment_name}.lua"
+ )
+ run(f"rm {lua_file}", shell=True, check=True)
+
+
+def remove_spack_package(spack_dir, package, build_log_path=None):
+ """Remove spack package.
+
+ Used to remove global packages.
+
+ Parameters
+ ----------
+ spack_dir : pathlib.Path
+ A Path object representing the path to the spack directory.
+ package : str
+ A string representing the name of the spack package to remove.
+ build_log_path : pathlib.Path, optional
+ A Path object representing the path to where the logs will be teed
+
+ """
+ logging.info(f"Removing spack package {package}")
+ spack_env = spack_dir / "share" / "spack" / "setup-env.sh"
+ commands_to_execute = [
+ f"export SPACK_ROOT={spack_dir}", # need to set SPACK_ROOT in dash and sh
+ f". {spack_env}",
+ f"spack uninstall -y {package}",
+ ]
+ run(
+ "(" + " && ".join(commands_to_execute) + f") 2>&1 |tee -a {build_log_path}",
+ shell=True,
+ check=True,
+ )
def start_new_environment(release, from_release, target_dir):
@@ -1315,7 +1452,7 @@ def main():
("available", "What is available for installation?"),
("install", "Install a software environment"),
# ("reinstall", "Reinstall a package_set"),
- # ("remove", "Remove a package set"),
+ ("remove", "Remove a package set"),
# ("start-new", "Start a new MPSD software release version"),
("status", "Show status: what is installed?"),
("prepare", "Prepare installation of MPSD-release (dev only)"),
diff --git a/tests/test_mpsd_software.py b/tests/test_mpsd_software.py
index a65d5d84a7b7c1de4b55509c3692d8f6406a90b4..0a503315a89740ec09b1d98dab7b272e5c25d27b 100644
--- a/tests/test_mpsd_software.py
+++ b/tests/test_mpsd_software.py
@@ -1,15 +1,17 @@
"""Tests for mpsd-software-environment.py."""
+import copy
+import datetime
import importlib
+import logging
import os
import shutil
import subprocess
-from pathlib import Path
-import logging
-import datetime
import sys
+from pathlib import Path
import pytest
+import yaml
mod = importlib.import_module("mpsd_software_manager.mpsd_software")
@@ -256,7 +258,7 @@ def test_install_environment_zlib():
with open(
package_set_src_dir / "global_generic" / "global_packages.list", "w"
) as f:
- f.write("zlib@1.2.13 \n")
+ f.write("zlib@1.2.13 \nzstd@1.5.2\n")
# add zlib to whitelist of module creation file by replacing anaconda3%gcc@10.2.1
# with zlib@1.2.13
@@ -487,8 +489,11 @@ def create_fake_environment(tmp_path, mpsd_release, expected_toolchain_map=None)
toolchain_lmod_folder.mkdir(parents=True, exist_ok=True)
spack_folder = tmp_path / mpsd_release / microarch / "spack"
spack_folder.mkdir(parents=True, exist_ok=True)
- logs_folder = tmp_path / mpsd_release / microarch / "logs"
+ logs_folder = tmp_path / mpsd_release / "logs"
logs_folder.mkdir(parents=True, exist_ok=True)
+ # Simulate the creation of APEX.log
+ # (which is created by the main function)
+ (logs_folder / "APEX.log").touch()
for toolchain in expected_toolchain_map[microarch]:
toolchain_lua_file = toolchain_lmod_folder / f"{toolchain}.lua"
toolchain_lua_file.touch()
@@ -509,33 +514,6 @@ def test_environment_status(tmp_path):
assert set(toolchain_map[microarch]) == set(expected_toolchain_map[microarch])
-@pytest.mark.skip(reason="not implemented yet")
-def test_remove_environment(tmp_path):
- """Test that the remove_environment works as expected."""
- mpsd_release = "dev-23a"
- # create a fake environment
- create_fake_environment(tmp_path, mpsd_release)
- # check that the environment status is correct
- toolchain_map = mod.environment_status(mpsd_release, tmp_path)
- assert toolchain_map is not None
-
- # test removal without arguments (should sys.exit(1))
- create_fake_environment(tmp_path, mpsd_release)
- with pytest.raises(SystemExit):
- mod.remove_environment(mpsd_release, tmp_path, force_remove=True)
-
- # test removal of the complete environment
- mod.remove_environment(mpsd_release, tmp_path, ["ALL"], force_remove=True)
- toolchain_map = mod.environment_status(mpsd_release, tmp_path)
- assert toolchain_map is None
- # ensure that logs folder remains
- logs_folder = tmp_path / mpsd_release / "logs"
- assert logs_folder.exists()
-
- # test removal of a single toolchain
- # done in test_install_environment_zlib
-
-
def test_initialise_environment(tmp_path):
"""Test that init_file is created as expected."""
# test that the init file is created as expected
@@ -675,6 +653,227 @@ def test_argument_parsing_logic(mocker):
### Copy from 'install' when the time has come.)
+def test_remove_environment(tmp_path, mocker):
+ """Test that the remove_environment function works as expected."""
+
+ release_to_test = "dev-23a"
+
+ # check exit 50 when no package_sets are provided
+ with pytest.raises(SystemExit) as pytest_wrapped_e:
+ mod.remove_environment("dev-23a", tmp_path)
+ assert pytest_wrapped_e.type == SystemExit
+ assert pytest_wrapped_e.value.code == 50
+
+ # Test case2 - remove entire release
+ ## exit with 60 when force is not set
+ ### patch the input function to return 'n'
+ mocker.patch("builtins.input", return_value="n")
+ with pytest.raises(SystemExit) as pytest_wrapped_e:
+ mod.remove_environment(release_to_test, tmp_path, "ALL")
+ assert pytest_wrapped_e.type == SystemExit
+ assert pytest_wrapped_e.value.code == 60
+ ### patch the input function to return 'y'
+ mocker.patch("builtins.input", return_value="y")
+ # check that the release directory is removed and logs are kept
+ # create a release directory
+ create_fake_environment(tmp_path, release_to_test)
+ release_dir = tmp_path / release_to_test / mod.get_native_microarchitecture()
+ logs_dir = tmp_path / release_to_test / "logs"
+
+ toolchain_map = mod.environment_status(release_to_test, tmp_path)
+ assert toolchain_map is not None
+ mod.remove_environment(release_to_test, tmp_path, "ALL")
+
+ toolchain_map = mod.environment_status(release_to_test, tmp_path)
+ # check that no toolchain remains
+ assert toolchain_map is None
+ # check that the release directory is empty
+ assert len(list(release_dir.iterdir())) == 0
+ # check that the logs directory is non-empty
+ list_of_logs = list(logs_dir.iterdir())
+ assert len(list_of_logs) == 2 # APEX + remove_build_log
+ # check that one of the log has 'remove.log' as part of the name
+ assert "BUILD_ALL_remove.log" in ",".join([str(x) for x in list_of_logs])
+
+ # Test case3 - remove specific package_sets
+ # defined in test_remove_package_sets
+
+
+@pytest.fixture
+def simple_toolchain():
+ """returns a dict for a simple toolchain"""
+ dict = {
+ "spack": {
+ "specs": ["zlib@1.2.13"],
+ "view": True,
+ "concretizer": {"reuse": False, "unify": True},
+ }
+ }
+ return dict
+
+
+def install_test_release(tmp_path, simple_toolchain):
+ """Install a test release
+
+ with toolchain1 and toolchain2 to use for testing.
+ """
+
+ release_to_test = "dev-23a"
+ # prepare a release directory
+ mod.prepare_environment(mpsd_release=release_to_test, root_dir=tmp_path)
+
+ tmp_path / release_to_test / mod.get_native_microarchitecture()
+ spe_dir = tmp_path / release_to_test / "spack-environments"
+
+ # create sample toolchains
+ simple_toolchain_string = yaml.dump(simple_toolchain)
+ simple_toolchain_2 = copy.deepcopy(simple_toolchain)
+ simple_toolchain_2["spack"]["specs"] = ["zlib@1.2.13", "zstd@1.5.2"]
+ simple_toolchain2_string = yaml.dump(simple_toolchain_2)
+ toolchain1_dir = spe_dir / "toolchains" / "toolchain1"
+ toolchain2_dir = spe_dir / "toolchains" / "toolchain2"
+ toolchain1_dir.mkdir(parents=True)
+ toolchain2_dir.mkdir(parents=True)
+ (toolchain1_dir / "spack.yaml").write_text(simple_toolchain_string)
+ (toolchain2_dir / "spack.yaml").write_text(simple_toolchain2_string)
+
+ # install the release
+ mod.install_environment(
+ mpsd_release=release_to_test,
+ package_sets=["toolchain1", "toolchain2"],
+ root_dir=tmp_path,
+ )
+
+
+def test_remove_package_sets(tmp_path, simple_toolchain):
+ """Test removal of package_sets via spack."""
+
+ release_to_test = "dev-23a"
+
+ # Case1 - remove global / global_generic package_sets
+
+ # Case2 - remove specific package_sets (toolchains)
+ # Create a test install
+ install_test_release(tmp_path, simple_toolchain)
+ # check that the installation went through
+ release_dir = tmp_path / release_to_test / mod.get_native_microarchitecture()
+ assert len(list(release_dir.iterdir())) == 2 # spack and lmod
+ # check that the two toolchains are installed
+ environments_dir = release_dir / "spack" / "var" / "spack" / "environments"
+ set([environment.name for environment in environments_dir.iterdir()]) == set(
+ ["toolchain1", "toolchain2"]
+ )
+ # check that the two toolchains have the "handmade" module files
+ toolchains_list = list(mod.environment_status(release_to_test, tmp_path).values())[
+ 0
+ ]
+ assert set(toolchains_list) == set(["toolchain1", "toolchain2"])
+
+ # remove toolchain2
+ # toolchain1 contains - zlib@1.2
+ # toolchain2 contains - zlib@1.2 and zstd@1.5
+ # we check that removing toolchain2 removes zstd@1.5 but NOT zlib@1.2
+ # and the environment toolchain2 is also removed
+
+ mod.remove_environment(
+ mpsd_release=release_to_test,
+ root_dir=tmp_path,
+ package_sets=["toolchain2"],
+ force_remove=True,
+ )
+ # now check that only "toolchain1" is installed in environments_dir
+ assert set([environment.name for environment in environments_dir.iterdir()]) == set(
+ ["toolchain1"]
+ )
+
+ # check that the only one toolchains has the "handmade" module files
+ toolchains_list = list(mod.environment_status(release_to_test, tmp_path).values())[
+ 0
+ ]
+ assert set(toolchains_list) == set(["toolchain1"])
+
+ # check that zlib@1.2 is still installed
+ # spack location -i <package> exit 0 if installed and 1 if not installed
+ source_spack = (
+ f"export SPACK_ROOT={release_dir} &&"
+ f'. {release_dir / "spack" / "share" / "spack" / "setup-env.sh"}'
+ )
+ mod.run(f"{source_spack} && spack location -i zlib", shell=True, check=True)
+ # check that zstd@1.5 is not installed
+ # we are here flipping the exit code to check that it is not installed
+ mod.run(
+ f"{source_spack} && (spack location -i zstd && exit 1 || exit 0 )",
+ shell=True,
+ check=True,
+ )
+ # check that the logs directory contains a build log for remove cmd
+ # dev-23a_zen3_2023-08-11T15-55-54_BUILD_toolchain2_remove.log
+ logs_dir = tmp_path / release_to_test / "logs"
+ # remove_build_log is the last log file in the list
+ remove_build_log = sorted(list(logs_dir.iterdir()))[-1]
+ assert "toolchain2_remove.log" in remove_build_log.name
+ with open(remove_build_log, "r") as f:
+ logs = f.read()
+ assert "==> Will not uninstall zlib@" in logs
+ assert "==> Successfully removed environment 'toolchain2'" in logs
+
+
+def test_remove_global_package_sets():
+ """Test removal of global package_sets via spack."""
+ root_dir = Path("/tmp/test_global_generic")
+ release_to_test = "dev-23a"
+ if not root_dir.exists():
+ # we need the sample spack instance with global_generic
+ # this is already done in test_install_environment_zlib
+ # so we just need to call it
+ test_install_environment_zlib()
+ # check that zlib and zstd are installed
+ spack_dir = (
+ root_dir / release_to_test / mod.get_native_microarchitecture() / "spack"
+ )
+ source_spack = (
+ f"export SPACK_ROOT={spack_dir} &&"
+ f'. {spack_dir / "share" / "spack" / "setup-env.sh"}'
+ )
+ # check that zlib is installed
+ # location commands exits with non zero if not installed thus
+ # breaking failing test
+ mod.run(f"{source_spack} && spack location -i zlib", shell=True, check=True)
+ # check that zstd is installed
+ mod.run(f"{source_spack} && spack location -i zstd", shell=True, check=True)
+
+ # remove global_generic
+ mod.remove_environment(
+ mpsd_release=release_to_test,
+ root_dir=root_dir,
+ package_sets=["global_generic"],
+ force_remove=True,
+ )
+ # check that zstd@1.5 is not installed
+ # we are here flipping the exit code to check that it is not installed
+ mod.run(
+ f"{source_spack} && (spack location -i zstd && exit 1 || exit 0 )",
+ shell=True,
+ check=True,
+ )
+ # check that zlib is not installed
+ mod.run(
+ f"{source_spack} && (spack location -i zlib && exit 1 || exit 0 )",
+ shell=True,
+ check=True,
+ )
+ # check that the logs directory contains a build log for remove cmd
+ # dev-23a_zen3_2023-08-11T15-55-54_BUILD_toolchain2_remove.log
+ logs_dir = root_dir / release_to_test / "logs"
+ # remove_build_log is the last log file in the list
+ remove_build_log = sorted(list(logs_dir.iterdir()))[-1]
+ assert "global_generic_remove.log" in remove_build_log.name
+ with open(remove_build_log, "r") as f:
+ logs = f.read()
+ assert "==> Successfully uninstalled zstd" in logs
+ assert "==> Successfully uninstalled zlib" in logs
+
+
def test_interface(tmp_path):
"""Test other things (not implemented yet)."""
pass