Toolchain becomes package set part2

This is mostly a search and replace exercise (and thus probably correct). (Individual commits deal with kind-of single functions, and their associated tests.)

More complicated changes I have tried to move into separate merge requests.

mpsd-software.py

@@ -354,8 +354,8 @@ def set_up_logging(loglevel="warning", file_path=None):
     )
 
 
-def get_available_toolchains(mpsd_release: str) -> List[str]:
-    """Given a release, return the available toolchains.
+def get_available_package_sets(mpsd_release: str) -> List[str]:
+    """Given a release, return the available package_sets.
 
     This is based on the spack-environment's repository [1]. For this function
     to succeed, we need to have Internet access etc.
@@ -367,11 +367,11 @@ def get_available_toolchains(mpsd_release: str) -> List[str]:
 
     Returns
     -------
-    toolchains : List[str]
+    package_sets : List[str]
 
     Example
     -------
-    >>> get_available_toolchains('dev-23a')
+    >>> get_available_package_sets('dev-23a')
     ['foss2021a-cuda-mpi',
      'foss2021a-mpi',
      'foss2021a-serial',
@@ -382,36 +382,36 @@ def get_available_toolchains(mpsd_release: str) -> List[str]:
      'global_generic']
 
     """
-    logging.debug(f"get_available_toolchains({mpsd_release=})")
-    logging.info(f"Retrieving available toolchains for release {mpsd_release}")
+    logging.debug(f"get_available_package_sets({mpsd_release=})")
+    logging.info(f"Retrieving available package_sets for release {mpsd_release}")
     print_log = logging.getLogger("print")
 
     # create temporary directory
     tmp_dir = tempfile.TemporaryDirectory(prefix="mpsd-software-available-")
     tmp_dir_path = Path(tmp_dir.name)
 
-    # find toolchains by cloning repository and checking out right branch
+    # find package_sets by cloning repository and checking out right branch
     clone_repo(
         tmp_dir_path, config_vars["spack_environments_repo"], branch=mpsd_release
     )
-    # look for directories defining the toolchains
+    # look for directories defining the package_sets
 
-    toolchains = os.listdir(tmp_dir_path / "toolchains")
-    msg = f"Found toolchains {sorted(toolchains)}"
+    package_sets = os.listdir(tmp_dir_path / "toolchains")
+    msg = f"Found package_sets {sorted(package_sets)}"
     logging.debug(msg)
 
-    # the 'toolchains' split into toolchains (such as foss2022a-mpi) and sets
+    # the 'package_sets' split into toolchains (such as foss2022a-mpi) and sets
     # of packages. Here we split them into the two categories for a more useful
     # output:
     toolchain_list = [
         x.parents[0].name
         for x in list((tmp_dir_path / "toolchains").glob("*/spack.yaml"))
     ]
-    package_sets = [
+    package_set_list = [
         x.parents[0].name for x in list((tmp_dir_path / "toolchains").glob("*/*.list"))
     ]
     logging.debug(f"{toolchain_list=}")
-    logging.debug(f"{package_sets=}")
+    logging.debug(f"{package_set_list=}")
 
     # summarise toolchains found for use, and show packages provided for each
     # package_set:
@@ -420,7 +420,7 @@ def get_available_toolchains(mpsd_release: str) -> List[str]:
     )
     print_log.info("Toolchains: \n    " + "\n    ".join(sorted(toolchain_list)))
     print_log.info("Package sets:")
-    for package_set in package_sets:
+    for package_set in package_set_list:
         # get a list of all packages which
         # starts from the first line of the file
         # that have the regex pattern \w+@\w+
@@ -436,7 +436,7 @@ def get_available_toolchains(mpsd_release: str) -> List[str]:
     # remove temporary directory
     tmp_dir.cleanup()
 
-    return toolchains
+    return package_sets
 
 
 # Helper class to change directory via context manager
@@ -690,7 +690,7 @@ def get_release_info(mpsd_release: str, root_dir: Path) -> Tuple[str, str, List[
     Get information about the specified release.
 
     Get information about the specified release, such as the branch and commit hash
-    of the Spack environments repository and the available toolchains.
+    of the Spack environments repository and the available package_sets.
 
     Parameters
     ----------
@@ -705,8 +705,8 @@ def get_release_info(mpsd_release: str, root_dir: Path) -> Tuple[str, str, List[
         The name of the branch for the Spack environments repository.
     spe_commit_hash : str
         The commit hash for the Spack environments repository.
-    available_toolchains : list
-        A list of strings representing the available toolchains for the release.
+    available_package_sets : list
+        A list of strings representing the available package_sets for the release.
 
     Raises
     ------
@@ -738,8 +738,8 @@ def get_release_info(mpsd_release: str, root_dir: Path) -> Tuple[str, str, List[
                 .stdout.decode()
                 .strip()
             )
-            available_toolchains = os.listdir("toolchains")
-    return spe_branch, spe_commit_hash, available_toolchains
+            available_package_sets = os.listdir("toolchains")
+    return spe_branch, spe_commit_hash, available_package_sets
 
 
 def prepare_environment(mpsd_release: str, root_dir: Path) -> List[str]:
@@ -749,7 +749,7 @@ def prepare_environment(mpsd_release: str, root_dir: Path) -> List[str]:
     It does the following steps:
     Clones the spack-environments repository.
     Determines the branch and commit hash of the spack-environments repository
-    and the available toolchains.
+    and the available package_sets.
     Logs the command usage.
 
     Parameters
@@ -762,8 +762,8 @@ def prepare_environment(mpsd_release: str, root_dir: Path) -> List[str]:
 
     Returns
     -------
-    available_toolchains : list
-        A list of available toolchains for the given MPSD release.
+    available_package_sets : list
+        A list of available package_sets for the given MPSD release.
 
     Example
     -------
@@ -799,13 +799,13 @@ def prepare_environment(mpsd_release: str, root_dir: Path) -> List[str]:
         f"Release {mpsd_release} is prepared in {release_base_dir}"
     )
 
-    spe_branch, spe_commit_hash, available_toolchains = get_release_info(
+    spe_branch, spe_commit_hash, available_package_sets = get_release_info(
         mpsd_release, root_dir
     )
     record_script_execution_summary(
         mpsd_release, root_dir, spe_branch=spe_branch, spe_commit_hash=spe_commit_hash
     )
-    return available_toolchains
+    return available_package_sets
 
 
 def get_native_microarchitecture():
@@ -865,83 +865,84 @@ def get_native_microarchitecture():
 
 def install_environment(
     mpsd_release: str,
-    toolchains: List[str],
+    package_sets: List[str],
     root_dir: Path,
     enable_build_cache: bool = False,
 ) -> None:
     """
-    Install the specified MPSD release and toolchains.
+    Install the specified MPSD release and package_sets.
 
-    The function installs the toolchain to the specified directory, using Spack.
+    The function installs the package_set to the specified directory, using Spack.
 
     Parameters
     ----------
     mpsd_release : str
         A string representing the MPSD release version.
-    toolchains : list of str
-        A list of strings representing the toolchains to install
+    package_sets : list of str
+        A list of strings representing the package_sets to install
         (e.g., "foss2021a-mpi", "global_generic", "ALL").
     root_dir : pathlib.Path
         A Path object representing the path to the directory where
-        the release and toolchains will be installed.
+        the release and package_sets will be installed.
     enable_build_cache : bool, optional
         A boolean indicating whether to build the build cache
-        when installing toolchains. Defaults to False.
+        when installing package_sets. Defaults to False.
 
     Raises
     ------
     ValueError
-        If a requested toolchain is not available in the specified release.
+        If a requested package_set is not available in the specified release.
 
     Returns
     -------
     None
     """
     logging.info(
-        f"Installing release {mpsd_release} with toolchains {toolchains} "
+        f"Installing release {mpsd_release} with package_sets {package_sets} "
         f"to {root_dir}"
     )
 
     # Set required variables
     release_base_dir = root_dir / mpsd_release
     microarch = get_native_microarchitecture()
-    toolchain_dir = release_base_dir / microarch
-    toolchain_dir.mkdir(parents=True, exist_ok=True)
+    package_set_dir = release_base_dir / microarch
+    package_set_dir.mkdir(parents=True, exist_ok=True)
     spack_setup_script = release_base_dir / "spack-environments" / "spack_setup.sh"
     install_flags = []
     if not enable_build_cache:
         install_flags.append("-b")
 
     # run the prepare_environment function
-    available_toolchains = prepare_environment(mpsd_release, root_dir)
-    # Ensure that the requested toolchains are available in the release
-    if toolchains == "ALL":
-        toolchains = available_toolchains
-    elif toolchains == "NONE":
-        # No toolchains requested, so we only create the env and print the
-        # list of available toolchains
+    available_package_sets = prepare_environment(mpsd_release, root_dir)
+    # Ensure that the requested package_sets are available in the release
+    if package_sets == "ALL":
+        package_sets = available_package_sets
+    elif package_sets == "NONE":
+        # No package_sets requested, so we only create the env and print the
+        # list of available package_sets
         logging.warning(
-            "No toolchains requested. Available toolchains for release "
-            f"{mpsd_release} are: \n {available_toolchains}"
+            "No package_sets requested. Available package_sets for release "
+            f"{mpsd_release} are: \n {available_package_sets}"
         )
         print_log = logging.getLogger("print")
-        print_log.info(f"{available_toolchains=}")
+        print_log.info(f"{available_package_sets=}")
         return
 
-    for toolchain in toolchains:
-        if toolchain not in available_toolchains:
-            msg = f"Toolchain '{toolchain}' is not available in release {mpsd_release}."
-            msg += "Use 'available' command to see list of available toolchains."
+    for package_set in package_sets:
+        if package_set not in available_package_sets:
+            msg = f"Package_Set '{package_set}' is not available"
+            msg += f" in release {mpsd_release}. "
+            msg += "Use 'available' command to see list of available package_sets."
             logging.error(msg)
             sys.exit(1)
 
-    # Install the toolchains
-    with os_chdir(toolchain_dir):
-        # run spack_setup_script with the toolchains as arguments
-        for toolchain in toolchains:
+    # Install the package_sets
+    with os_chdir(package_set_dir):
+        # run spack_setup_script with the package_sets as arguments
+        for package_set in package_sets:
             # Set the install log file name from create_log_file_names
             build_log_file_name = create_log_file_names(
-                mpsd_release, microarch, "install", package_set=toolchain
+                mpsd_release, microarch, "install", package_set=package_set
             )
             build_log_folder = release_base_dir / "logs"
             build_log_path = build_log_folder / build_log_file_name
@@ -949,33 +950,36 @@ def install_environment(
             if not os.path.exists(build_log_folder):
                 os.makedirs(build_log_folder)
 
-            logging.info(f"Installing toolchain {toolchain} to {toolchain_dir}")
+            logging.info(f"Installing package_set {package_set} to {package_set_dir}")
 
             # log the command
             record_script_execution_summary(
                 mpsd_release,
                 root_dir,
-                msg=f"installing {toolchain} and logging at {build_log_path}",
+                msg=f"installing {package_set} and logging at {build_log_path}",
             )
             record_script_execution_summary(
                 mpsd_release,
                 root_dir,
                 msg=(
                     f"CMD: bash {spack_setup_script} {' '.join(install_flags)} "
-                    f"{toolchain}"
+                    f"{package_set}"
                 ),
             )
             run(
-                f"bash {spack_setup_script} {' '.join(install_flags)} {toolchain} 2>&1 "
+                f"bash {spack_setup_script} "
+                f"{' '.join(install_flags)} {package_set} 2>&1 "
                 f"| tee -a {build_log_path} ",
                 shell=True,
                 check=True,
             )
 
 
-def remove_environment(release, toolchains, target_dir):
+def remove_environment(release, package_sets, target_dir):
     """Remove release from installation."""
-    msg = f"Removing release {release} with toolchains {toolchains} from {target_dir}"
+    msg = (
+        f"Removing release {release} with package_sets {package_sets} from {target_dir}"
+    )
     logging.info(msg)
     raise NotImplementedError(msg)
 
@@ -1003,7 +1007,11 @@ def environment_status(mpsd_release: str, root_dir: Union[str, Path]) -> dict:
     -------
     toolchain_map : dict
         A dictionary containing available microarchitectures as keys and
-        a list of available toolchains as values for each microarchitecture.
+        a list of available package_sets as values for each microarchitecture.
+
+        Note: only toolchains can be reported at the moment (i.e. package_sets
+        such as global and global_generic are missing, even if installed).
+
     """
     msg = f"Showing status of release {mpsd_release} in {root_dir}"
     logging.info(msg)
@@ -1179,7 +1187,7 @@ def main():
     elif args.action == "prepare":
         prepare_environment(args.release, root_dir)
     elif args.action == "available":
-        get_available_toolchains(args.release)
+        get_available_package_sets(args.release)
     else:
         message = (
             f"No known action found ({args.action=}). Should probably never happen."