Toolchain becomes package set part2

This is mostly a search and replace exercise (and thus probably correct). (Individual commits deal with kind-of single functions, and their associated tests.)

More complicated changes I have tried to move into separate merge requests.

mpsd-software.py

@@ -27,11 +27,19 @@ else:
 
 
 about_intro = f"""
-Build toolchains using Spack.\n
+Build software as on MPSD HPC.
 
-This function builds toolchains for MPSD-HPC at the appropriate directory,
-for given system architecture and MPSD software stack version. The toolchains
-are built using the bash script spack_setup.sh, and the results are logged.
+
+    This tool builds software package sets (including toolchains for Octopus).
+    It follows recipes as used on the MPSD HPC system and the (spack-based)
+    Octopus buildbot. Compiled software is organised into MPSD software release
+    versions (such as `dev-23a`) and CPU microarchitecture (such as `sandybridge`).
+    
+    Compiled packages and toolchains can be activated and used via `module load` as
+    on the HPC system.
+    
+    Further documentation is available in the README.rst file, online at
+    https://gitlab.gwdg.de/mpsd-cs/mpsd-software/-/blob/main/README.rst
 
 Command line usage:
 
@@ -45,23 +53,24 @@ about_epilog = f"""
 
 Examples:
 
-1. Query what package sets and toolchains are available in relase dev-23a
-
-   $> {sys.argv[0]} available dev-23a
-
-2. Install foss2022a-serial toolchain
-
-   $> {sys.argv[0]} install dev-23a foss2022a-serial
-
-3. Check what package sets and toolchains are installed from release dev-23a
-
-   $> {sys.argv[0]} status dev-23a
-
-Documentation: https://gitlab.gwdg.de/mpsd-cs/mpsd-software/-/blob/main/README.rst
+    1. Query what package sets and toolchains are available for installation in
+       release dev-23a
+    
+       $> {sys.argv[0]} available dev-23a
+    
+    2. Install foss2022a-serial toolchain from the dev-23a release
+    
+       $> {sys.argv[0]} install dev-23a foss2022a-serial
+    
+    3. Check what package sets and toolchains are installed from release dev-23a
+    
+       $> {sys.argv[0]} status dev-23a
+    
+       The `status` command also displays the `module use` command needed to load
+       the created modules.
 
 """
 
-
 call_date_iso = (
     datetime.datetime.now().replace(microsecond=0).isoformat().replace(":", "-")
 )
@@ -80,15 +89,15 @@ def create_log_file_names(
     microarch: str,
     action: str,
     date: str = call_date_iso,
-    toolchain: str = None,
+    package_set: str = None,
 ) -> Union[str, None]:
     """Create log file names.
 
     This function creates the log file names for either the installer or
     the build log files.
 
-    If a toolchain is given, then the build log file name is created.
-    if no toolchain is given, then the installer log file name is created.
+    If a package_set is given, then the build log file name is created.
+    if no package_set is given, then the installer log file name is created.
     The installer log file hosts the logs of the installer script, while
     the build log file hosts the logs of the build process as generated by the
     spack_setup.sh script.
@@ -104,8 +113,8 @@ def create_log_file_names(
     action : str
         action performed (install,remove,reinstall,prepare,status)
         only install and remove are valid for build log file.
-    toolchain : str
-        toolchain name (only for build log file)
+    package_set : str
+        package_set name (only for build log file)
 
     Returns
     -------
@@ -116,16 +125,16 @@ def create_log_file_names(
         If the action is not one that changes the files on disk ( info only actions)
         then None is returned.
     """
-    if toolchain:
-        # if toolchain is given, then  we build the build_log_file_name
+    if package_set:
+        # if package_set is given, then  we build the build_log_file_name
         if action in ["install", "remove"]:
             log_file_name = (
-                f"{mpsd_release}_{microarch}_{date}_BUILD_{toolchain}_{action}.log"
+                f"{mpsd_release}_{microarch}_{date}_BUILD_{package_set}_{action}.log"
             )
         else:
             return None
     else:
-        # if toolchain is not given, then we build the installer_log_file_name
+        # if package_set is not given, then we build the installer_log_file_name
         log_file_name = f"{mpsd_release}_{microarch}_{date}_APEX_{action}.log"
 
     return log_file_name
@@ -257,7 +266,7 @@ def set_up_logging(loglevel="warning", file_path=None):
 
        Typical and intended use:
 
-       print_log.info("Available toolchains are ...")
+       print_log.info("Available package_sets are ...")
 
        The major difference from the normal print command is that the output
        will be send to the stdout (as for print) AND the file with name
@@ -345,8 +354,8 @@ def set_up_logging(loglevel="warning", file_path=None):
     )
 
 
-def get_available_toolchains(mpsd_release: str) -> List[str]:
-    """Given a release, return the available toolchains.
+def get_available_package_sets(mpsd_release: str) -> List[str]:
+    """Given a release, return the available package_sets.
 
     This is based on the spack-environment's repository [1]. For this function
     to succeed, we need to have Internet access etc.
@@ -358,11 +367,11 @@ def get_available_toolchains(mpsd_release: str) -> List[str]:
 
     Returns
     -------
-    toolchains : List[str]
+    package_sets : List[str]
 
     Example
     -------
-    >>> get_available_toolchains('dev-23a')
+    >>> get_available_package_sets('dev-23a')
     ['foss2021a-cuda-mpi',
      'foss2021a-mpi',
      'foss2021a-serial',
@@ -373,36 +382,36 @@ def get_available_toolchains(mpsd_release: str) -> List[str]:
      'global_generic']
 
     """
-    logging.debug(f"get_available_toolchains({mpsd_release=})")
-    logging.info(f"Retrieving available toolchains for release {mpsd_release}")
+    logging.debug(f"get_available_package_sets({mpsd_release=})")
+    logging.info(f"Retrieving available package_sets for release {mpsd_release}")
     print_log = logging.getLogger("print")
 
     # create temporary directory
     tmp_dir = tempfile.TemporaryDirectory(prefix="mpsd-software-available-")
     tmp_dir_path = Path(tmp_dir.name)
 
-    # find toolchains by cloning repository and checking out right branch
+    # find package_sets by cloning repository and checking out right branch
     clone_repo(
         tmp_dir_path, config_vars["spack_environments_repo"], branch=mpsd_release
     )
-    # look for directories defining the toolchains
+    # look for directories defining the package_sets
 
-    toolchains = os.listdir(tmp_dir_path / "toolchains")
-    msg = f"Found toolchains {sorted(toolchains)}"
+    package_sets = os.listdir(tmp_dir_path / "toolchains")
+    msg = f"Found package_sets {sorted(package_sets)}"
     logging.debug(msg)
 
-    # the 'toolchains' split into toolchains (such as foss2022a-mpi) and sets
+    # the 'package_sets' split into toolchains (such as foss2022a-mpi) and sets
     # of packages. Here we split them into the two categories for a more useful
     # output:
     toolchain_list = [
         x.parents[0].name
         for x in list((tmp_dir_path / "toolchains").glob("*/spack.yaml"))
     ]
-    package_sets = [
+    package_set_list = [
         x.parents[0].name for x in list((tmp_dir_path / "toolchains").glob("*/*.list"))
     ]
     logging.debug(f"{toolchain_list=}")
-    logging.debug(f"{package_sets=}")
+    logging.debug(f"{package_set_list=}")
 
     # summarise toolchains found for use, and show packages provided for each
     # package_set:
@@ -411,7 +420,7 @@ def get_available_toolchains(mpsd_release: str) -> List[str]:
     )
     print_log.info("Toolchains: \n    " + "\n    ".join(sorted(toolchain_list)))
     print_log.info("Package sets:")
-    for package_set in package_sets:
+    for package_set in package_set_list:
         # get a list of all packages which
         # starts from the first line of the file
         # that have the regex pattern \w+@\w+
@@ -427,7 +436,7 @@ def get_available_toolchains(mpsd_release: str) -> List[str]:
     # remove temporary directory
     tmp_dir.cleanup()
 
-    return toolchains
+    return package_sets
 
 
 # Helper class to change directory via context manager
@@ -856,83 +865,83 @@ def get_native_microarchitecture():
 
 def install_environment(
     mpsd_release: str,
-    toolchains: List[str],
+    package_sets: List[str],
     root_dir: Path,
     enable_build_cache: bool = False,
 ) -> None:
     """
-    Install the specified MPSD release and toolchains.
+    Install the specified MPSD release and package_sets.
 
-    The function installs the toolchain to the specified directory, using Spack.
+    The function installs the package_set to the specified directory, using Spack.
 
     Parameters
     ----------
     mpsd_release : str
         A string representing the MPSD release version.
-    toolchains : list of str
-        A list of strings representing the toolchains to install
+    package_sets : list of str
+        A list of strings representing the package_sets to install
         (e.g., "foss2021a-mpi", "global_generic", "ALL").
     root_dir : pathlib.Path
         A Path object representing the path to the directory where
-        the release and toolchains will be installed.
+        the release and package_sets will be installed.
     enable_build_cache : bool, optional
         A boolean indicating whether to build the build cache
-        when installing toolchains. Defaults to False.
+        when installing package_sets. Defaults to False.
 
     Raises
     ------
     ValueError
-        If a requested toolchain is not available in the specified release.
+        If a requested package_set is not available in the specified release.
 
     Returns
     -------
     None
     """
     logging.info(
-        f"Installing release {mpsd_release} with toolchains {toolchains} "
+        f"Installing release {mpsd_release} with package_sets {package_sets} "
         f"to {root_dir}"
     )
 
     # Set required variables
     release_base_dir = root_dir / mpsd_release
     microarch = get_native_microarchitecture()
-    toolchain_dir = release_base_dir / microarch
-    toolchain_dir.mkdir(parents=True, exist_ok=True)
+    package_set_dir = release_base_dir / microarch
+    package_set_dir.mkdir(parents=True, exist_ok=True)
     spack_setup_script = release_base_dir / "spack-environments" / "spack_setup.sh"
     install_flags = []
     if not enable_build_cache:
         install_flags.append("-b")
 
     # run the prepare_environment function
-    available_toolchains = prepare_environment(mpsd_release, root_dir)
-    # Ensure that the requested toolchains are available in the release
-    if toolchains == "ALL":
-        toolchains = available_toolchains
-    elif toolchains == "NONE":
-        # No toolchains requested, so we only create the env and print the
-        # list of available toolchains
+    available_package_sets = prepare_environment(mpsd_release, root_dir)
+    # Ensure that the requested package_sets are available in the release
+    if package_sets == "ALL":
+        package_sets = available_package_sets
+    elif package_sets == "NONE":
+        # No package_sets requested, so we only create the env and print the
+        # list of available package_sets
         logging.warning(
-            "No toolchains requested. Available toolchains for release "
-            f"{mpsd_release} are: \n {available_toolchains}"
+            "No package_sets requested. Available package_sets for release "
+            f"{mpsd_release} are: \n {available_package_sets}"
         )
         print_log = logging.getLogger("print")
-        print_log.info(f"{available_toolchains=}")
+        print_log.info(f"{available_package_sets=}")
         return
 
-    for toolchain in toolchains:
-        if toolchain not in available_toolchains:
-            msg = f"Toolchain '{toolchain}' is not available in release {mpsd_release}."
-            msg += "Use 'available' command to see list of available toolchains."
+    for package_set in package_sets:
+        if package_set not in available_package_sets:
+            msg = f"Package_Set '{package_set}' is not available in release {mpsd_release}."
+            msg += "Use 'available' command to see list of available package_sets."
             logging.error(msg)
             sys.exit(1)
 
-    # Install the toolchains
-    with os_chdir(toolchain_dir):
-        # run spack_setup_script with the toolchains as arguments
-        for toolchain in toolchains:
+    # Install the package_sets
+    with os_chdir(package_set_dir):
+        # run spack_setup_script with the package_sets as arguments
+        for package_set in package_sets:
             # Set the install log file name from create_log_file_names
             build_log_file_name = create_log_file_names(
-                mpsd_release, microarch, "install", toolchain=toolchain
+                mpsd_release, microarch, "install", package_set=package_set
             )
             build_log_folder = release_base_dir / "logs"
             build_log_path = build_log_folder / build_log_file_name
@@ -940,33 +949,33 @@ def install_environment(
             if not os.path.exists(build_log_folder):
                 os.makedirs(build_log_folder)
 
-            logging.info(f"Installing toolchain {toolchain} to {toolchain_dir}")
+            logging.info(f"Installing package_set {package_set} to {package_set_dir}")
 
             # log the command
             record_script_execution_summary(
                 mpsd_release,
                 root_dir,
-                msg=f"installing {toolchain} and logging at {build_log_path}",
+                msg=f"installing {package_set} and logging at {build_log_path}",
             )
             record_script_execution_summary(
                 mpsd_release,
                 root_dir,
                 msg=(
                     f"CMD: bash {spack_setup_script} {' '.join(install_flags)} "
-                    f"{toolchain}"
+                    f"{package_set}"
                 ),
             )
             run(
-                f"bash {spack_setup_script} {' '.join(install_flags)} {toolchain} 2>&1 "
+                f"bash {spack_setup_script} {' '.join(install_flags)} {package_set} 2>&1 "
                 f"| tee -a {build_log_path} ",
                 shell=True,
                 check=True,
             )
 
 
-def remove_environment(release, toolchains, target_dir):
+def remove_environment(release, package_sets, target_dir):
     """Remove release from installation."""
-    msg = f"Removing release {release} with toolchains {toolchains} from {target_dir}"
+    msg = f"Removing release {release} with package_sets {package_sets} from {target_dir}"
     logging.info(msg)
     raise NotImplementedError(msg)
 
@@ -994,7 +1003,11 @@ def environment_status(mpsd_release: str, root_dir: Union[str, Path]) -> dict:
     -------
     toolchain_map : dict
         A dictionary containing available microarchitectures as keys and
-        a list of available toolchains as values for each microarchitecture.
+        a list of available package_sets as values for each microarchitecture.
+
+        Note: only toolchains can be reported at the moment (i.e. package_sets
+        such as global and global_generic are missing, even if installed).
+
     """
     msg = f"Showing status of release {mpsd_release} in {root_dir}"
     logging.info(msg)
@@ -1066,7 +1079,6 @@ def main():
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     parser.add_argument(
-        "--log",
         "-l",
         dest="loglevel",
         choices=["warning", "info", "debug"],
@@ -1082,7 +1094,7 @@ def main():
     )
     subparsers.required = True
     list_of_cmds = [
-        ("available", "Show software available for installation"),
+        ("available", "What is available for installation?"),
         ("install", "Install a software environment"),
         # ("reinstall", "Reinstall a software environment"),
         # ("remove", "Remove a software environment or toolchains from an environment"),
@@ -1117,18 +1129,18 @@ def main():
             )
             if cmd in ["install", "reinstall", "remove"]:
                 # "install" command needs additional documentation
-                tool_chain_help = (
+                package_set_help = (
                     f"One or more toolchains to command {cmd}. "
-                    "Use 'ALL' to refer to all available toolchains."
+                    "Use 'ALL' to refer to all available package sets."
                 )
 
                 subp.add_argument(
-                    "toolchain",  # first option defines attribute
-                    # name `args.toolchain` in `args = parser_args()`
+                    "package_set",  # first option defines attribute
+                    # name `args.package_set` in `args = parser_args()`
                     type=str,
                     nargs="+",
                     default="NONE",
-                    help=tool_chain_help,
+                    help=package_set_help,
                 )
                 subp.add_argument(
                     "--enable-build-cache",
@@ -1159,19 +1171,19 @@ def main():
 
     # Check the command and run related function
     if args.action == "remove":
-        remove_environment(args.release, args.toolchain, root_dir)
+        remove_environment(args.release, args.package_set, root_dir)
     elif args.action == "start-new":
         start_new_environment(args.from_release, args.to_release, root_dir)
     elif args.action == "install":
         install_environment(
-            args.release, args.toolchain, root_dir, args.enable_build_cache
+            args.release, args.package_set, root_dir, args.enable_build_cache
         )
     elif args.action == "status":
         _ = environment_status(args.release, root_dir)
     elif args.action == "prepare":
         prepare_environment(args.release, root_dir)
     elif args.action == "available":
-        get_available_toolchains(args.release)
+        get_available_package_sets(args.release)
     else:
         message = (
             f"No known action found ({args.action=}). Should probably never happen."