Remove cmd

src/mpsd_software_manager/mpsd_software.py

@@ -14,7 +14,6 @@ import time
 from pathlib import Path
 from typing import List, Tuple, Union
 import re
-import shutil
 from functools import cache
 import importlib.metadata
 
@@ -648,64 +647,78 @@ def run(*args, counter=[0], **kwargs):
     return process
 
 
+def write_to_file(file_path: Path, content: str) -> None:
+    """Write content to file.
+
+    Parameters
+    ----------
+    file_path : Path
+        The path to the file to write to.
+    content : str
+        The content to write to the file.
+
+    Returns
+    -------
+    None
+
+    """
+    with open(file_path, "a") as f:
+        f.write(content)
+
+
+def write_to_cmd_log(root_dir: Path, msg: str) -> None:
+    """Write message to command log.
+
+    Parameters
+    ----------
+    root_dir : Path
+        The path where the script is initialized.
+    msg : str
+        The message to write to the command log.
+    Returns
+    -------
+    - None
+    """
+    cmd_log_file_path = root_dir / config_vars["cmd_log_file"]
+    write_to_file(cmd_log_file_path, msg)
+
+
 def record_script_execution_summary(
-    root_dir: Path, msg: Union[str, None] = None, **kwargs
+    root_dir: Path, apex_log_file: Union[Path, None] = None
 ) -> None:
     """Log the command used to build the package_set.
 
-    It also logs information about the spack-environments branch and commit hash,
-    as well as the version of the mpsd-software-manager user. It also logs
-    steps taken in the install process using the optional message argument.
+    A date time header is added to the log file each time the script is called,
+    following which the commands executed by the user is logged.
+    The APEX log file is also logged if it is created.
+
+    for example:
+
+    ```
+
+    2023-06-28T11:18:10.718020
+    $ mpsd-software install dev-23a foss2021a-mpi
+    ```
 
     Parameters
     ----------
-    - root_dir : str
-        The path to the directory where the scripts are located.
-    - msg : str, optional
-        An optional message to log in the command log file.
-    - **kwargs : dict
-        A dictionary with values for
-        - spe_branch : str
-            The name of the Spack environments branch.
-        - spe_commit_hash : str
-            The commit hash of the Spack environments branch.
+    - root_dir : Path
+        The path where the script is initialized.
 
     Returns
     -------
     - None
 
     """
-    # Write to the log file with the following format
-    # --------------------------------------------------
-    # 2023-06-20T15:30:37.965370, mpsd_software prepare dev-23a
-    # MPSD Software manager version: 2023.6.16
-    # Spack environments branch: dev-23a (commit hash: abcdefg)
-    # MSGs
-    with os_chdir(root_dir):
-        with open(config_vars["cmd_log_file"], "a") as f:
-            if msg:
-                # Write the message to the log file
-                f.write(msg + "\n")
-            else:
-                # Write the header
-                f.write("-" * 50 + "\n")
-
-                # Gather data to log
-                # call statement:
-                cmd_line = " ".join(sys.argv)
-                # spack-environments branch and commit hash from kwargs
-                spe_branch = kwargs.get("spe_branch", None)
-                spe_commit_hash = kwargs.get("spe_commit_hash", None)
-
-                # Write to log file
-                f.write(f"{datetime.datetime.now().isoformat()}, {cmd_line}\n")
-                # logs script version
-                f.write(f"MPSD Software manager version: {__version__}\n")
-                if spe_branch and spe_commit_hash:
-                    f.write(
-                        f"Spack environments branch: {spe_branch} "
-                        f"(commit hash: {spe_commit_hash})\n"
-                    )
+    cmd_line = command_name + " " + " ".join(sys.argv[1:])
+    date_time = datetime.datetime.now().replace(microsecond=0).isoformat()
+
+    msg = f"\n {date_time}\n"
+    msg += f"$ {cmd_line}\n"
+    if apex_log_file:
+        # apex_log_file is not known when `init` is called
+        msg += f"> logging to {apex_log_file}\n"
+    write_to_cmd_log(root_dir, msg)
 
 
 def clone_repo(
@@ -904,8 +917,12 @@ def prepare_environment(mpsd_release: str, root_dir: Path) -> List[str]:
     spe_branch, spe_commit_hash, available_package_sets = get_release_info(
         mpsd_release, root_dir
     )
-    record_script_execution_summary(
-        root_dir, spe_branch=spe_branch, spe_commit_hash=spe_commit_hash
+    write_to_cmd_log(
+        root_dir=root_dir,
+        msg=(
+            f"Spack environments branch: {spe_branch} "
+            f"(commit hash: {spe_commit_hash})\n"
+        ),
     )
     return available_package_sets
 
@@ -1048,20 +1065,10 @@ def install_environment(
                 mpsd_release, "install", root_dir, package_set
             )
 
+            # Log the build_log_path and the package_set_dir
             logging.info(f"Installing package_set {package_set} to {package_set_dir}")
+            logging.info(f"> Logging installation of {package_set} at {build_log_path}")
 
-            # log the command
-            record_script_execution_summary(
-                root_dir,
-                msg=f"installing {package_set} and logging at {build_log_path}",
-            )
-            record_script_execution_summary(
-                root_dir,
-                msg=(
-                    f"CMD: bash {spack_setup_script} {' '.join(install_flags)} "
-                    f"{package_set}"
-                ),
-            )
             run(
                 f"bash {spack_setup_script} "
                 f"{' '.join(install_flags)} {package_set} 2>&1 "
@@ -1072,47 +1079,174 @@ def install_environment(
 
 
 def remove_environment(mpsd_release, root_dir, package_sets="NONE", force_remove=False):
-    """Remove release from installation."""
+    """Remove release from installation.
+
+    Handle 3 situations :
+    1. remove does not specify what to remove
+        -> warn and exit
+    2. remove all package_sets from release
+        -> remove release folder except logs
+    3. remove specific package_sets from release
+        -> remove spack environments via spack commands
+
+    Parameters
+    ----------
+    mpsd_release : str
+        A string representing the MPSD release version.
+    root_dir : pathlib.Path
+        A Path object representing the path to the directory where
+        the release and package_sets will be installed.
+    package_sets : list of str
+        A list of strings representing the package_sets to remove
+        (e.g., "foss2021a-mpi", "global_generic", "ALL").
+    force_remove : bool, optional
+        A boolean indicating whether to force remove the release.
+        If False, the user will be prompted to confirm the removal.
+        Defaults to False.
+
+    Raises
+    ------
+    ValueError
+
+    """
     msg = (
         f"Removing release {mpsd_release}"
         f" with package_sets {package_sets} from {root_dir}"
     )
     logging.warning(msg)
-    # Handle 3 situations :
-    # 1. remove dosent specify what to remove  -> warn and exit
-    # 1. remove all package_sets from release -> remove release folder except logs
-    # 2. remove specific package_sets from release
-    #       -> remove spack environments via spack commands
 
     if package_sets == "NONE":
         logging.warning(
-            "Please specify package_sets to remove, or 'ALL' to remove all toolchains"
+            "Please specify package_sets to remove, or 'ALL' to remove all package_sets"
         )
-        sys.exit(1)
+        sys.exit(50)
+    # 2nd case: remove the entire release for microarchitecture
+    dir_to_remove = root_dir / mpsd_release / get_native_microarchitecture()
     if "ALL" in package_sets:
         # we need to remove the entire release folder
-        logging.warning(f"Removing release {mpsd_release} from {root_dir}")
+        logging.warning(
+            f"Removing release {mpsd_release}"
+            f"from {root_dir} for {get_native_microarchitecture()}"
+        )
         if not force_remove:
             logging.warning("do you want to continue? [y/n]")
-            if input().lower() == "y":
-                force_remove = True
-        if force_remove:
-            dir_to_remove = root_dir / mpsd_release / get_native_microarchitecture()
-            folders_to_remove = os.listdir(dir_to_remove)
-            # skip logs folder
-            if "logs" in folders_to_remove:
-                folders_to_remove.remove("logs")
-            for folder in folders_to_remove:
-                shutil.rmtree(dir_to_remove / folder)
-            return
+            if input().lower() != "y":
+                sys.exit(60)
+
+            # Set the remove log file name from create_log_file_names
+        build_log_path = get_log_file_path(mpsd_release, "remove", root_dir, "ALL")
+
+        logging.info(f"> Logging removal of {mpsd_release} at {build_log_path}")
+        folders_to_remove = os.listdir(dir_to_remove)
+        # skip logs folder
+        # if "logs" in folders_to_remove:
+        #     folders_to_remove.remove("logs")
+        for folder in folders_to_remove:
+            # shutil.rmtree(dir_to_remove / folder) #dosent delete file
+            run(
+                f"rm -rf {dir_to_remove / folder} 2>&1 | tee -a {build_log_path}",
+                shell=True,
+                check=True,
+            )
+        logging.warning(f"Removed release {mpsd_release} from {root_dir}")
+        return
+    # 3rd case: remove specific package_sets from release
     for package_set in package_sets:
         # we load the spack environment and remove the package_set
-        spack_env = ""
-        commands_to_execute = [
-            f"source {spack_env}",
-            f"spack env remove -y {package_set}",
-        ]
-        run(" && ".join(commands_to_execute), shell=True, check=True)
+        build_log_path = get_log_file_path(
+            mpsd_release, "remove", root_dir, package_set
+        )
+        logging.info(f"> Logging removal of {package_set} at {build_log_path}")
+        if package_set not in ["global_generic", "global"]:
+            remove_spack_environment(
+                dir_to_remove / "spack", package_set, build_log_path
+            )
+        else:
+            # list all specs from the global_packages.list
+            spe_folder = root_dir / mpsd_release / "spack-environments"
+            package_list_file = (
+                spe_folder / "toolchains" / package_set / "global_packages.list"
+            )
+            with open(package_list_file, "r") as f:
+                package_dump = f.read()
+
+            # remove all content from # to the end of the line
+            package_dump = re.sub(r"#.*\n", "\n", package_dump)
+            # replace \\n with "" to remove line breaks
+            package_list = package_dump.replace("\\\n", "").split("\n")
+            # remove all empty lines
+            package_list = [line for line in package_list if line != ""]
+
+            # remove all packages in package_list
+            for package in package_list:
+                logging.info(f"Removing package {package} from installation")
+                remove_spack_package(dir_to_remove / "spack", package, build_log_path)
+
+
+def remove_spack_environment(spack_dir, environment_name, build_log_path=None):
+    """Remove spack environment including packages exclusive to it.
+
+    First activate the environment,
+    then uninstall all packages exclusive to the environment,
+    then deactivate the environment,
+    and finally remove the environment.
+
+    Parameters
+    ----------
+    spack_dir : pathlib.Path
+        A Path object representing the path to the spack directory.
+    environment_name : str
+        A string representing the name of the spack environment to remove.
+    build_log_path : pathlib.Path, optional
+        A Path object representing the path to where the logs will be teed
+    """
+    logging.warning(f"Removing spack environment {environment_name}")
+    spack_env = spack_dir / "share" / "spack" / "setup-env.sh"
+    commands_to_execute = [
+        f"export SPACK_ROOT={spack_dir}",  # need to set SPACK_ROOT in dash and sh
+        f". {spack_env}",
+        f"spack env activate {environment_name}",
+        f"for spec in $(spack -e {environment_name} find"  # this line continues
+        r' --format "{name}@{version}%{compiler.name}@{compiler.version}");do'
+        " spack uninstall -y $spec; done",
+        "spack env deactivate",
+        f"spack env remove -y {environment_name}",
+    ]
+    build_log_path = build_log_path or "/dev/null"
+    run(
+        "(" + " && ".join(commands_to_execute) + f") 2>&1 |tee -a {build_log_path}",
+        shell=True,
+        check=True,
+    )
+
+
+def remove_spack_package(spack_dir, package, build_log_path=None):
+    """Remove spack package.
+
+    Used to remove global packages.
+
+    Parameters
+    ----------
+    spack_dir : pathlib.Path
+        A Path object representing the path to the spack directory.
+    package : str
+        A string representing the name of the spack package to remove.
+    build_log_path : pathlib.Path, optional
+        A Path object representing the path to where the logs will be teed
+
+    """
+    logging.info(f"Removing spack package {package}")
+    spack_env = spack_dir / "share" / "spack" / "setup-env.sh"
+    commands_to_execute = [
+        f"export SPACK_ROOT={spack_dir}",  # need to set SPACK_ROOT in dash and sh
+        f". {spack_env}",
+        f"spack uninstall -y {package}",
+    ]
+    run(
+        "(" + " && ".join(commands_to_execute) + f") 2>&1 |tee -a {build_log_path}",
+        shell=True,
+        check=True,
+    )
 
 
 def start_new_environment(release, from_release, target_dir):
@@ -1232,9 +1366,11 @@ def initialise_environment(root_dir: Path) -> None:
         # create the log file and fill it with the headers
         record_script_execution_summary(root_dir=root_dir)
         # record the msg in the log file
-        record_script_execution_summary(
+        init_log_msg = f"Initialising MPSD software instance at {root_dir}.\n"
+        init_log_msg += f"MPSD Software manager version: {__version__}\n"
+        write_to_cmd_log(
             root_dir=root_dir,
-            msg=f"Initialising MPSD software instance at {root_dir}.",
+            msg=init_log_msg,
         )
 
 
@@ -1313,7 +1449,7 @@ def main():
         ("available", "What is available for installation?"),
         ("install", "Install a software environment"),
         # ("reinstall", "Reinstall a package_set"),
-        # ("remove", "Remove a package set"),
+        ("remove", "Remove a package set"),
         # ("start-new", "Start a new MPSD software release version"),
         ("status", "Show status: what is installed?"),
         ("prepare", "Prepare installation of MPSD-release (dev only)"),
@@ -1403,23 +1539,24 @@ def main():
 
     # set up logging filename: we record activities that change the installation
     if args.action in ["init", "install", "prepare", "reinstall", "remove"]:
-        log_file = get_log_file_path(
+        apex_log_file = get_log_file_path(
             args.release,
             args.action,
             root_dir,
         )
     # some commands do not write any log_files:
     elif args.action in ["available", "status"]:
-        log_file = None
+        apex_log_file = None
     else:
         # sanity check
         raise NotImplementedError(f"Should never happen: unknown {args.action=}")
 
     set_up_logging(
         args.loglevel,
-        log_file,
+        apex_log_file,
     )
 
+    record_script_execution_summary(root_dir, apex_log_file)
     # Check the command and run related function
     if args.action == "remove":
         remove_environment(args.release, root_dir, args.package_set)