Remove cmd

src/mpsd_software_manager/mpsd_software.py

@@ -5,17 +5,17 @@
 
 import argparse
 import datetime
+import importlib.metadata
 import logging
 import os
+import re
 import subprocess
 import sys
 import tempfile
 import time
+from functools import cache
 from pathlib import Path
 from typing import List, Tuple, Union
-import re
-from functools import cache
-import importlib.metadata
 
 __version__ = importlib.metadata.version(__package__ or __name__)
 
@@ -647,64 +647,78 @@ def run(*args, counter=[0], **kwargs):
     return process
 
 
+def write_to_file(file_path: Path, content: str) -> None:
+    """Write content to file.
+
+    Parameters
+    ----------
+    file_path : Path
+        The path to the file to write to.
+    content : str
+        The content to write to the file.
+
+    Returns
+    -------
+    None
+
+    """
+    with open(file_path, "a") as f:
+        f.write(content)
+
+
+def write_to_cmd_log(root_dir: Path, msg: str) -> None:
+    """Write message to command log.
+
+    Parameters
+    ----------
+    root_dir : Path
+        The path where the script is initialized.
+    msg : str
+        The message to write to the command log.
+    Returns
+    -------
+    - None
+    """
+    cmd_log_file_path = root_dir / config_vars["cmd_log_file"]
+    write_to_file(cmd_log_file_path, msg)
+
+
 def record_script_execution_summary(
-    root_dir: Path, msg: Union[str, None] = None, **kwargs
+    root_dir: Path, apex_log_file: Union[Path, None] = None
 ) -> None:
     """Log the command used to build the package_set.
 
-    It also logs information about the spack-environments branch and commit hash,
-    as well as the version of the mpsd-software-manager user. It also logs
-    steps taken in the install process using the optional message argument.
+    A date time header is added to the log file each time the script is called,
+    following which the commands executed by the user is logged.
+    The APEX log file is also logged if it is created.
+
+    for example:
+
+    ```
+
+    2023-06-28T11:18:10.718020
+    $ mpsd-software install dev-23a foss2021a-mpi
+    ```
 
     Parameters
     ----------
-    - root_dir : str
-        The path to the directory where the scripts are located.
-    - msg : str, optional
-        An optional message to log in the command log file.
-    - **kwargs : dict
-        A dictionary with values for
-        - spe_branch : str
-            The name of the Spack environments branch.
-        - spe_commit_hash : str
-            The commit hash of the Spack environments branch.
+    - root_dir : Path
+        The path where the script is initialized.
 
     Returns
     -------
     - None
 
     """
-    # Write to the log file with the following format
-    # --------------------------------------------------
-    # 2023-06-20T15:30:37.965370, mpsd_software prepare dev-23a
-    # MPSD Software manager version: 2023.6.16
-    # Spack environments branch: dev-23a (commit hash: abcdefg)
-    # MSGs
-    with os_chdir(root_dir):
-        with open(config_vars["cmd_log_file"], "a") as f:
-            if msg:
-                # Write the message to the log file
-                f.write(msg + "\n")
-            else:
-                # Write the header
-                f.write("-" * 50 + "\n")
-
-                # Gather data to log
-                # call statement:
-                cmd_line = " ".join(sys.argv)
-                # spack-environments branch and commit hash from kwargs
-                spe_branch = kwargs.get("spe_branch", None)
-                spe_commit_hash = kwargs.get("spe_commit_hash", None)
-
-                # Write to log file
-                f.write(f"{datetime.datetime.now().isoformat()}, {cmd_line}\n")
-                # logs script version
-                f.write(f"MPSD Software manager version: {__version__}\n")
-                if spe_branch and spe_commit_hash:
-                    f.write(
-                        f"Spack environments branch: {spe_branch} "
-                        f"(commit hash: {spe_commit_hash})\n"
-                    )
+    cmd_line = command_name + " " + " ".join(sys.argv[1:])
+    date_time = datetime.datetime.now().replace(microsecond=0).isoformat()
+
+    msg = f"\n {date_time}\n"
+    msg += f"$ {cmd_line}\n"
+    if apex_log_file:
+        # apex_log_file is not known when `init` is called
+        msg += f"> logging to {apex_log_file}\n"
+    write_to_cmd_log(root_dir, msg)
 
 
 def clone_repo(
@@ -903,8 +917,12 @@ def prepare_environment(mpsd_release: str, root_dir: Path) -> List[str]:
     spe_branch, spe_commit_hash, available_package_sets = get_release_info(
         mpsd_release, root_dir
     )
-    record_script_execution_summary(
-        root_dir, spe_branch=spe_branch, spe_commit_hash=spe_commit_hash
+    write_to_cmd_log(
+        root_dir=root_dir,
+        msg=(
+            f"Spack environments branch: {spe_branch} "
+            f"(commit hash: {spe_commit_hash})\n"
+        ),
     )
     return available_package_sets
 
@@ -1047,20 +1065,10 @@ def install_environment(
                 mpsd_release, "install", root_dir, package_set
             )
 
+            # Log the build_log_path and the package_set_dir
             logging.info(f"Installing package_set {package_set} to {package_set_dir}")
+            logging.info(f"> Logging installation of {package_set} at {build_log_path}")
 
-            # log the command
-            record_script_execution_summary(
-                root_dir,
-                msg=f"installing {package_set} and logging at {build_log_path}",
-            )
-            record_script_execution_summary(
-                root_dir,
-                msg=(
-                    f"CMD: bash {spack_setup_script} {' '.join(install_flags)} "
-                    f"{package_set}"
-                ),
-            )
             run(
                 f"bash {spack_setup_script} "
                 f"{' '.join(install_flags)} {package_set} 2>&1 "
@@ -1074,7 +1082,7 @@ def remove_environment(mpsd_release, root_dir, package_sets="NONE", force_remove
     """Remove release from installation.
 
     Handle 3 situations :
-    1. remove dosent specify what to remove
+    1. remove does not specify what to remove
         -> warn and exit
     2. remove all package_sets from release
         -> remove release folder except logs
@@ -1089,7 +1097,7 @@ def remove_environment(mpsd_release, root_dir, package_sets="NONE", force_remove
         A Path object representing the path to the directory where
         the release and package_sets will be installed.
     package_sets : list of str
-        A list of strings representing the package_sets to install
+        A list of strings representing the package_sets to remove
         (e.g., "foss2021a-mpi", "global_generic", "ALL").
     force_remove : bool, optional
         A boolean indicating whether to force remove the release.
@@ -1109,9 +1117,9 @@ def remove_environment(mpsd_release, root_dir, package_sets="NONE", force_remove
 
     if package_sets == "NONE":
         logging.warning(
-            "Please specify package_sets to remove, or 'ALL' to remove all toolchains"
+            "Please specify package_sets to remove, or 'ALL' to remove all package_sets"
         )
-        sys.exit(50)  # TODO document this code.
+        sys.exit(50)
     # 2nd case: remove the entire release for microarchitecture
     dir_to_remove = root_dir / mpsd_release / get_native_microarchitecture()
     if "ALL" in package_sets:
@@ -1123,33 +1131,76 @@ def remove_environment(mpsd_release, root_dir, package_sets="NONE", force_remove
         if not force_remove:
             logging.warning("do you want to continue? [y/n]")
             if input().lower() != "y":
-                sys.exit(60)  # TODO document this code.
+                sys.exit(60)
+
+            # Set the remove log file name from create_log_file_names
+        build_log_path = get_log_file_path(mpsd_release, "remove", root_dir, "ALL")
 
+        logging.info(f"> Logging removal of {mpsd_release} at {build_log_path}")
         folders_to_remove = os.listdir(dir_to_remove)
-        # skip logs folder
-        # if "logs" in folders_to_remove:
-        #     folders_to_remove.remove("logs")
         for folder in folders_to_remove:
             # shutil.rmtree(dir_to_remove / folder) #dosent delete file
-            run(f"rm -rf {dir_to_remove / folder}", shell=True, check=True)
+            run(
+                f"rm -rf {dir_to_remove / folder} 2>&1 | tee -a {build_log_path}",
+                shell=True,
+                check=True,
+            )
         logging.warning(f"Removed release {mpsd_release} from {root_dir}")
         return
     # 3rd case: remove specific package_sets from release
     for package_set in package_sets:
         # we load the spack environment and remove the package_set
-        if package_set not in ["global_packages", "global"]:
-            remove_spack_environment(dir_to_remove / "spack", package_set)
+        build_log_path = get_log_file_path(
+            mpsd_release, "remove", root_dir, package_set
+        )
+        logging.info(f"> Logging removal of {package_set} at {build_log_path}")
+        if package_set not in ["global_generic", "global"]:
+            remove_spack_environment(
+                dir_to_remove / "spack", package_set, build_log_path
+            )
         else:
-            # TODO remove global packages by calling remove_spack_package
-            pass
+            # list all specs from the global_packages.list
+            spe_folder = root_dir / mpsd_release / "spack-environments"
+            package_list_file = (
+                spe_folder / "toolchains" / package_set / "global_packages.list"
+            )
+            with open(package_list_file, "r") as f:
+                package_dump = f.read()
+
+            # remove all content from # to the end of the line
+            package_dump = re.sub(r"#.*\n", "\n", package_dump)
+            # replace \\n with "" to remove line breaks
+            package_list = package_dump.replace("\\\n", "").split("\n")
+            # remove all empty lines
+            package_list = [line for line in package_list if line != ""]
+
+            # remove all packages in package_list
+            for package in package_list:
+                logging.info(f"Removing package {package} from installation")
+                remove_spack_package(dir_to_remove / "spack", package, build_log_path)
 
 
-def remove_spack_environment(spack_dir, environment_name):
-    """Remove spack environment."""
+def remove_spack_environment(spack_dir, environment_name, build_log_path=None):
+    """Remove spack environment including packages exclusive to it.
+
+    First activate the environment,
+    then uninstall all packages exclusive to the environment,
+    then deactivate the environment,
+    and finally remove the environment.
+
+    Parameters
+    ----------
+    spack_dir : pathlib.Path
+        A Path object representing the path to the spack directory.
+    environment_name : str
+        A string representing the name of the spack environment to remove.
+    build_log_path : pathlib.Path, optional
+        A Path object representing the path to where the logs will be teed
+    """
     logging.warning(f"Removing spack environment {environment_name}")
     spack_env = spack_dir / "share" / "spack" / "setup-env.sh"
     commands_to_execute = [
-        f"export SPACK_DIR={spack_dir}",
+        f"export SPACK_ROOT={spack_dir}",  # need to set SPACK_ROOT in dash and sh
         f". {spack_env}",
         f"spack env activate {environment_name}",
         f"for spec in $(spack -e {environment_name} find"  # this line continues
@@ -1158,18 +1209,41 @@ def remove_spack_environment(spack_dir, environment_name):
         "spack env deactivate",
         f"spack env remove -y {environment_name}",
     ]
-    run(" && ".join(commands_to_execute), shell=True, check=True)
+    build_log_path = build_log_path or "/dev/null"
+    run(
+        "(" + " && ".join(commands_to_execute) + f") 2>&1 |tee -a {build_log_path}",
+        shell=True,
+        check=True,
+    )
+
 
+def remove_spack_package(spack_dir, package, build_log_path=None):
+    """Remove spack package.
 
-def remove_spack_package(spack_dir, package):
-    """Remove spack package."""
+    Used to remove global packages.
+
+    Parameters
+    ----------
+    spack_dir : pathlib.Path
+        A Path object representing the path to the spack directory.
+    package : str
+        A string representing the name of the spack package to remove.
+    build_log_path : pathlib.Path, optional
+        A Path object representing the path to where the logs will be teed
+
+    """
     logging.info(f"Removing spack package {package}")
     spack_env = spack_dir / "share" / "spack" / "setup-env.sh"
     commands_to_execute = [
-        f"source {spack_env}",
+        f"export SPACK_ROOT={spack_dir}",  # need to set SPACK_ROOT in dash and sh
+        f". {spack_env}",
         f"spack uninstall -y {package}",
     ]
-    run(" && ".join(commands_to_execute), shell=True, check=True)
+    run(
+        "(" + " && ".join(commands_to_execute) + f") 2>&1 |tee -a {build_log_path}",
+        shell=True,
+        check=True,
+    )
 
 
 def start_new_environment(release, from_release, target_dir):
@@ -1289,9 +1363,11 @@ def initialise_environment(root_dir: Path) -> None:
         # create the log file and fill it with the headers
         record_script_execution_summary(root_dir=root_dir)
         # record the msg in the log file
-        record_script_execution_summary(
+        init_log_msg = f"Initialising MPSD software instance at {root_dir}.\n"
+        init_log_msg += f"MPSD Software manager version: {__version__}\n"
+        write_to_cmd_log(
             root_dir=root_dir,
-            msg=f"Initialising MPSD software instance at {root_dir}.",
+            msg=init_log_msg,
         )
 
 
@@ -1460,23 +1536,24 @@ def main():
 
     # set up logging filename: we record activities that change the installation
     if args.action in ["init", "install", "prepare", "reinstall", "remove"]:
-        log_file = get_log_file_path(
+        apex_log_file = get_log_file_path(
             args.release,
             args.action,
             root_dir,
         )
     # some commands do not write any log_files:
     elif args.action in ["available", "status"]:
-        log_file = None
+        apex_log_file = None
     else:
         # sanity check
         raise NotImplementedError(f"Should never happen: unknown {args.action=}")
 
     set_up_logging(
         args.loglevel,
-        log_file,
+        apex_log_file,
     )
 
+    record_script_execution_summary(root_dir, apex_log_file)
     # Check the command and run related function
     if args.action == "remove":
         remove_environment(args.release, root_dir, args.package_set)