Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • hmenke/mpsd-software-manager
  • mpsd-cs/mpsd-software-manager
2 results
Show changes
Commits on Source (44)
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
View file @ e798908a
......@@ -81,7 +81,7 @@ style:
- ruff .
- black --check --diff .
- pydocstyle mpsd-software.py
- pydocstyle tests.py
- pydocstyle test_mpsd-software.py
# we could also use `ruff --select D` for pycodestyle. But the behaviour is not exactly the same.
- ./mpsd-software.py --version
......@@ -91,12 +91,12 @@ test-bullseye:
image: debian:bullseye-slim
script:
- *prepare_debian
- pytest -v -l tests.py
- pytest -v -l test_mpsd-software.py
test-bookworm:
stage: test
image: debian:bookworm-slim
script:
- *prepare_debian
- pytest -v -l tests.py
- pytest -v -l test_mpsd-software.py
mpsd-software.py
View file @ e798908a
......@@ -2,7 +2,7 @@
"""mpsd-software: tool for installation of software as on MPSD HPC."""
__version__ = "2023.6.9"
__version__ = "2023.6.14"
import argparse
import datetime
......@@ -26,13 +26,50 @@ else:
rich_available = True
about_tool = """
Build toolchains using Spack.\n
about_intro = f"""
Build software as on MPSD HPC.
This function builds toolchains for MPSD-HPC at the appropriate directory, \n
for given system architecture and MPSD software stack version.\n
The toolchains
are built using the bash script spack_setup.sh, and the results are logged. """
This tool builds software package sets (including toolchains for Octopus).
It follows recipes as used on the MPSD HPC system and the (spack-based)
Octopus buildbot. Compiled software is organised into MPSD software release
versions (such as `dev-23a`) and CPU microarchitecture (such as `sandybridge`).
Compiled packages and toolchains can be activated and used via `module load` as
on the HPC system.
Further documentation is available in the README.rst file, online at
https://gitlab.gwdg.de/mpsd-cs/mpsd-software/-/blob/main/README.rst
Command line usage:
$> {sys.argv[0]}
"""
about_epilog = f"""
Examples:
1. Query what package sets and toolchains are available for installation in
release dev-23a
$> {sys.argv[0]} available dev-23a
2. Install foss2022a-serial toolchain from the dev-23a release
$> {sys.argv[0]} install dev-23a foss2022a-serial
3. Check what package sets and toolchains are installed from release dev-23a
$> {sys.argv[0]} status dev-23a
The `status` command also displays the `module use` command needed to load
the created modules.
"""
call_date_iso = (
datetime.datetime.now().replace(microsecond=0).isoformat().replace(":", "-")
......@@ -52,15 +89,15 @@ def create_log_file_names(
microarch: str,
action: str,
date: str = call_date_iso,
toolchain: str = None,
package_set: str = None,
) -> Union[str, None]:
"""Create log file names.
This function creates the log file names for either the installer or
the build log files.
If a toolchain is given, then the build log file name is created.
if no toolchain is given, then the installer log file name is created.
If a package_set is given, then the build log file name is created.
if no package_set is given, then the installer log file name is created.
The installer log file hosts the logs of the installer script, while
the build log file hosts the logs of the build process as generated by the
spack_setup.sh script.
......@@ -76,8 +113,8 @@ def create_log_file_names(
action : str
action performed (install,remove,reinstall,prepare,status)
only install and remove are valid for build log file.
toolchain : str
toolchain name (only for build log file)
package_set : str
package_set name (only for build log file)
Returns
-------
......@@ -88,16 +125,16 @@ def create_log_file_names(
If the action is not one that changes the files on disk ( info only actions)
then None is returned.
"""
if toolchain:
# if toolchain is given, then we build the build_log_file_name
if package_set:
# if package_set is given, then we build the build_log_file_name
if action in ["install", "remove"]:
log_file_name = (
f"{mpsd_release}_{microarch}_{date}_BUILD_{toolchain}_{action}.log"
f"{mpsd_release}_{microarch}_{date}_BUILD_{package_set}_{action}.log"
)
else:
return None
else:
# if toolchain is not given, then we build the installer_log_file_name
# if package_set is not given, then we build the installer_log_file_name
log_file_name = f"{mpsd_release}_{microarch}_{date}_APEX_{action}.log"
return log_file_name
......@@ -229,7 +266,7 @@ def set_up_logging(loglevel="warning", file_path=None):
Typical and intended use:
print_log.info("Available toolchains are ...")
print_log.info("Available package_sets are ...")
The major difference from the normal print command is that the output
will be send to the stdout (as for print) AND the file with name
......@@ -317,8 +354,8 @@ def set_up_logging(loglevel="warning", file_path=None):
)
def get_available_toolchains(mpsd_release: str) -> List[str]:
"""Given a release, return the available toolchains.
def get_available_package_sets(mpsd_release: str) -> List[str]:
"""Given a release, return the available package_sets.
This is based on the spack-environment's repository [1]. For this function
to succeed, we need to have Internet access etc.
......@@ -330,11 +367,11 @@ def get_available_toolchains(mpsd_release: str) -> List[str]:
Returns
-------
toolchains : List[str]
package_sets : List[str]
Example
-------
>>> get_available_toolchains('dev-23a')
>>> get_available_package_sets('dev-23a')
['foss2021a-cuda-mpi',
'foss2021a-mpi',
'foss2021a-serial',
......@@ -345,36 +382,36 @@ def get_available_toolchains(mpsd_release: str) -> List[str]:
'global_generic']
"""
logging.debug(f"get_available_toolchains({mpsd_release=})")
logging.info(f"Retrieving available toolchains for release {mpsd_release}")
logging.debug(f"get_available_package_sets({mpsd_release=})")
logging.info(f"Retrieving available package_sets for release {mpsd_release}")
print_log = logging.getLogger("print")
# create temporary directory
tmp_dir = tempfile.TemporaryDirectory(prefix="mpsd-software-available-")
tmp_dir_path = Path(tmp_dir.name)
# find toolchains by cloning repository and checking out right branch
# find package_sets by cloning repository and checking out right branch
clone_repo(
tmp_dir_path, config_vars["spack_environments_repo"], branch=mpsd_release
)
# look for directories defining the toolchains
# look for directories defining the package_sets
toolchains = os.listdir(tmp_dir_path / "toolchains")
msg = f"Found toolchains {sorted(toolchains)}"
package_sets = os.listdir(tmp_dir_path / "toolchains")
msg = f"Found package_sets {sorted(package_sets)}"
logging.debug(msg)
# the 'toolchains' split into toolchains (such as foss2022a-mpi) and sets
# the 'package_sets' split into toolchains (such as foss2022a-mpi) and sets
# of packages. Here we split them into the two categories for a more useful
# output:
toolchain_list = [
x.parents[0].name
for x in list((tmp_dir_path / "toolchains").glob("*/spack.yaml"))
]
package_sets = [
package_set_list = [
x.parents[0].name for x in list((tmp_dir_path / "toolchains").glob("*/*.list"))
]
logging.debug(f"{toolchain_list=}")
logging.debug(f"{package_sets=}")
logging.debug(f"{package_set_list=}")
# summarise toolchains found for use, and show packages provided for each
# package_set:
......@@ -383,7 +420,7 @@ def get_available_toolchains(mpsd_release: str) -> List[str]:
)
print_log.info("Toolchains: \n " + "\n ".join(sorted(toolchain_list)))
print_log.info("Package sets:")
for package_set in package_sets:
for package_set in package_set_list:
# get a list of all packages which
# starts from the first line of the file
# that have the regex pattern \w+@\w+
......@@ -399,7 +436,7 @@ def get_available_toolchains(mpsd_release: str) -> List[str]:
# remove temporary directory
tmp_dir.cleanup()
return toolchains
return package_sets
# Helper class to change directory via context manager
......@@ -514,13 +551,11 @@ def run(*args, counter=[0], **kwargs):
def record_script_execution_summary(
mpsd_release: str, root_dir: str, msg: str = None, **kwargs
) -> None:
"""
Log the command used to build the toolchains.
"""Log the command used to build the package_set.
It also logs information about the software environment installer branch,
the Spack environments branch, and the commit hashes of each.
It also logs steps taken
in the install process using the optional message argument.
It also logs information about the mpsd-software repository branch, the
spack-environments branch, and the commit hashes of each. It also logs
steps taken in the install process using the optional message argument.
Parameters
----------
......@@ -540,9 +575,8 @@ def record_script_execution_summary(
Returns
-------
- None
"""
release_base_dir = root_dir / mpsd_release
"""
# Write to the log file with the following format
# --------------------------------------------------
# 2023-02-29T23:32:01, install-software-environment.py --release 23b --install ALL
......@@ -550,7 +584,7 @@ def record_script_execution_summary(
# script_commit_hash)
# Spack environments branch: dev-23a (commit hash: spe_commit_hash)
# MSGs
with os_chdir(release_base_dir):
with os_chdir(root_dir):
with open(config_vars["cmd_log_file"], "a") as f:
if msg:
# Write the message to the log file
......@@ -655,7 +689,7 @@ def get_release_info(mpsd_release: str, root_dir: Path) -> Tuple[str, str, List[
Get information about the specified release.
Get information about the specified release, such as the branch and commit hash
of the Spack environments repository and the available toolchains.
of the Spack environments repository and the available package_sets.
Parameters
----------
......@@ -670,14 +704,16 @@ def get_release_info(mpsd_release: str, root_dir: Path) -> Tuple[str, str, List[
The name of the branch for the Spack environments repository.
spe_commit_hash : str
The commit hash for the Spack environments repository.
available_toolchains : list
A list of strings representing the available toolchains for the release.
available_package_sets : list
A list of strings representing the available package_sets for the release.
Raises
------
FileNotFoundError
If the release directory does not exist.
"""
# TODO - review this function: can we re-use get_available_package_sets?
# Get the info for release
release_base_dir = root_dir / mpsd_release
if not os.path.exists(release_base_dir):
......@@ -703,8 +739,8 @@ def get_release_info(mpsd_release: str, root_dir: Path) -> Tuple[str, str, List[
.stdout.decode()
.strip()
)
available_toolchains = os.listdir("toolchains")
return spe_branch, spe_commit_hash, available_toolchains
available_package_sets = os.listdir("toolchains")
return spe_branch, spe_commit_hash, available_package_sets
def prepare_environment(mpsd_release: str, root_dir: Path) -> List[str]:
......@@ -714,7 +750,7 @@ def prepare_environment(mpsd_release: str, root_dir: Path) -> List[str]:
It does the following steps:
Clones the spack-environments repository.
Determines the branch and commit hash of the spack-environments repository
and the available toolchains.
and the available package_sets.
Logs the command usage.
Parameters
......@@ -727,9 +763,24 @@ def prepare_environment(mpsd_release: str, root_dir: Path) -> List[str]:
Returns
-------
available_toolchains : list
A list of available toolchains for the given MPSD release.
available_package_sets : list
A list of available package_sets for the given MPSD release.
Example
-------
>>> prepare_environment('dev-23a', Path('.'))
['foss2021a-cuda-mpi',
'foss2021a-mpi',
'foss2021a-serial',
'foss2022a-cuda-mpi',
'foss2022a-mpi',
'foss2022a-serial',
'global',
'global_generic']
"""
# TODO review: - does this function need to return anything? If yes:
# TODO review: - can we re-use get_available_package sets?
logging.info(f"Preparing {mpsd_release=}")
# Creates the directory structure for the specified release and clone the
......@@ -752,13 +803,13 @@ def prepare_environment(mpsd_release: str, root_dir: Path) -> List[str]:
f"Release {mpsd_release} is prepared in {release_base_dir}"
)
spe_branch, spe_commit_hash, available_toolchains = get_release_info(
spe_branch, spe_commit_hash, available_package_sets = get_release_info(
mpsd_release, root_dir
)
record_script_execution_summary(
mpsd_release, root_dir, spe_branch=spe_branch, spe_commit_hash=spe_commit_hash
)
return available_toolchains
return available_package_sets
def get_native_microarchitecture():
......@@ -818,83 +869,84 @@ def get_native_microarchitecture():
def install_environment(
mpsd_release: str,
toolchains: List[str],
package_sets: List[str],
root_dir: Path,
enable_build_cache: bool = False,
) -> None:
"""
Install the specified MPSD release and toolchains.
Install the specified MPSD release and package_sets.
The function installs the toolchain to the specified directory, using Spack.
The function installs the package_set to the specified directory, using Spack.
Parameters
----------
mpsd_release : str
A string representing the MPSD release version.
toolchains : list of str
A list of strings representing the toolchains to install
package_sets : list of str
A list of strings representing the package_sets to install
(e.g., "foss2021a-mpi", "global_generic", "ALL").
root_dir : pathlib.Path
A Path object representing the path to the directory where
the release and toolchains will be installed.
the release and package_sets will be installed.
enable_build_cache : bool, optional
A boolean indicating whether to build the build cache
when installing toolchains. Defaults to False.
when installing package_sets. Defaults to False.
Raises
------
ValueError
If a requested toolchain is not available in the specified release.
If a requested package_set is not available in the specified release.
Returns
-------
None
"""
logging.info(
f"Installing release {mpsd_release} with toolchains {toolchains} "
f"Installing release {mpsd_release} with package_sets {package_sets} "
f"to {root_dir}"
)
# Set required variables
release_base_dir = root_dir / mpsd_release
microarch = get_native_microarchitecture()
toolchain_dir = release_base_dir / microarch
toolchain_dir.mkdir(parents=True, exist_ok=True)
package_set_dir = release_base_dir / microarch
package_set_dir.mkdir(parents=True, exist_ok=True)
spack_setup_script = release_base_dir / "spack-environments" / "spack_setup.sh"
install_flags = []
if not enable_build_cache:
install_flags.append("-b")
# run the prepare_environment function
available_toolchains = prepare_environment(mpsd_release, root_dir)
# Ensure that the requested toolchains are available in the release
if toolchains == "ALL":
toolchains = available_toolchains
elif toolchains == "NONE":
# No toolchains requested, so we only create the env and print the
# list of available toolchains
available_package_sets = prepare_environment(mpsd_release, root_dir)
# Ensure that the requested package_sets are available in the release
if package_sets == "ALL":
package_sets = available_package_sets
elif package_sets == "NONE":
# No package_sets requested, so we only create the env and print the
# list of available package_sets
logging.warning(
"No toolchains requested. Available toolchains for release "
f"{mpsd_release} are: \n {available_toolchains}"
"No package_sets requested. Available package_sets for release "
f"{mpsd_release} are: \n {available_package_sets}"
)
print_log = logging.getLogger("print")
print_log.info(f"{available_toolchains=}")
print_log.info(f"{available_package_sets=}")
return
for toolchain in toolchains:
if toolchain not in available_toolchains:
msg = f"Toolchain '{toolchain}' is not available in release {mpsd_release}."
msg += "Use 'available' command to see list of available toolchains."
for package_set in package_sets:
if package_set not in available_package_sets:
msg = f"Package_Set '{package_set}' is not available"
msg += f" in release {mpsd_release}. "
msg += "Use 'available' command to see list of available package_sets."
logging.error(msg)
sys.exit(1)
# Install the toolchains
with os_chdir(toolchain_dir):
# run spack_setup_script with the toolchains as arguments
for toolchain in toolchains:
# Install the package_sets
with os_chdir(package_set_dir):
# run spack_setup_script with the package_sets as arguments
for package_set in package_sets:
# Set the install log file name from create_log_file_names
build_log_file_name = create_log_file_names(
mpsd_release, microarch, "install", toolchain=toolchain
mpsd_release, microarch, "install", package_set=package_set
)
build_log_folder = release_base_dir / "logs"
build_log_path = build_log_folder / build_log_file_name
......@@ -902,33 +954,36 @@ def install_environment(
if not os.path.exists(build_log_folder):
os.makedirs(build_log_folder)
logging.info(f"Installing toolchain {toolchain} to {toolchain_dir}")
logging.info(f"Installing package_set {package_set} to {package_set_dir}")
# log the command
record_script_execution_summary(
mpsd_release,
root_dir,
msg=f"installing {toolchain} and logging at {build_log_path}",
msg=f"installing {package_set} and logging at {build_log_path}",
)
record_script_execution_summary(
mpsd_release,
root_dir,
msg=(
f"CMD: bash {spack_setup_script} {' '.join(install_flags)} "
f"{toolchain}"
f"{package_set}"
),
)
run(
f"bash {spack_setup_script} {' '.join(install_flags)} {toolchain} 2>&1 "
f"bash {spack_setup_script} "
f"{' '.join(install_flags)} {package_set} 2>&1 "
f"| tee -a {build_log_path} ",
shell=True,
check=True,
)
def remove_environment(release, toolchains, target_dir):
def remove_environment(release, package_sets, target_dir):
"""Remove release from installation."""
msg = f"Removing release {release} with toolchains {toolchains} from {target_dir}"
msg = (
f"Removing release {release} with package_sets {package_sets} from {target_dir}"
)
logging.info(msg)
raise NotImplementedError(msg)
......@@ -956,7 +1011,11 @@ def environment_status(mpsd_release: str, root_dir: Union[str, Path]) -> dict:
-------
toolchain_map : dict
A dictionary containing available microarchitectures as keys and
a list of available toolchains as values for each microarchitecture.
a list of available package_sets as values for each microarchitecture.
Note: only toolchains can be reported at the moment (i.e. package_sets
such as global and global_generic are missing, even if installed).
"""
msg = f"Showing status of release {mpsd_release} in {root_dir}"
logging.info(msg)
......@@ -1022,9 +1081,12 @@ def environment_status(mpsd_release: str, root_dir: Union[str, Path]) -> dict:
def main():
"""Execute main entry point."""
parser = argparse.ArgumentParser(description=about_tool)
parser = argparse.ArgumentParser(
description=about_intro,
epilog=about_epilog,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument(
"--log",
"-l",
dest="loglevel",
choices=["warning", "info", "debug"],
......@@ -1040,11 +1102,11 @@ def main():
)
subparsers.required = True
list_of_cmds = [
("available", "Show software available for installation"),
("available", "What is available for installation?"),
("install", "Install a software environment"),
# ("reinstall", "Reinstall a software environment"),
# ("remove", "Remove a software environment or toolchains from an environment"),
# ("start-new", "Start a new software environment version"),
# ("reinstall", "Reinstall a package_set"),
# ("remove", "Remove a package set"),
# ("start-new", "Start a new MPSD software release version"),
("status", "Show status: what is installed?"),
("prepare", "Prepare installation of MPSD-release (dev only)"),
]
......@@ -1075,24 +1137,18 @@ def main():
)
if cmd in ["install", "reinstall", "remove"]:
# "install" command needs additional documentation
tool_chain_help = (
f"Pass a list of toolchains to command {cmd}. "
"Use '--toolchains ALL' to "
f"{cmd} all toolchains. If '--toolchain' is not "
"specified, list available toolchains for the release "
"(after environment has been prepared if not done yet)."
package_set_help = (
f"One or more package sets (like toolchains) to be {cmd}ed. "
"Use 'ALL' to refer to all available package sets."
)
subp.add_argument(
"--toolchains", # first option defines attribute
# name `args.toolchains` in `args = parser_args()`
"--toolchain", # allow singular as alternative
# (-> creates attribute `args.toolchains` if used)
"package_set", # first option defines attribute
# name `args.package_set` in `args = parser_args()`
type=str,
dest="toolchains",
nargs="+",
default="NONE",
help=tool_chain_help,
help=package_set_help,
)
subp.add_argument(
"--enable-build-cache",
......@@ -1113,21 +1169,29 @@ def main():
get_installer_log_file_path(args.release, args.action, root_dir),
)
# sanity check for common mistakes in command line arguments
if args.release.endswith("/"): # happens easily with autocompletion
logging.error(
f"You provided mpsd-release='{args.release}'. "
"Did you mean '{args.release.rstrip('/')}'?"
)
sys.exit(1)
# Check the command and run related function
if args.action == "remove":
remove_environment(args.release, args.toolchains, root_dir)
remove_environment(args.release, args.package_set, root_dir)
elif args.action == "start-new":
start_new_environment(args.from_release, args.to_release, root_dir)
elif args.action == "install":
install_environment(
args.release, args.toolchains, root_dir, args.enable_build_cache
args.release, args.package_set, root_dir, args.enable_build_cache
)
elif args.action == "status":
_ = environment_status(args.release, root_dir)
elif args.action == "prepare":
prepare_environment(args.release, root_dir)
elif args.action == "available":
get_available_toolchains(args.release)
get_available_package_sets(args.release)
else:
message = (
f"No known action found ({args.action=}). Should probably never happen."
......
tests.py → test_mpsd-software.py
View file @ e798908a
......@@ -7,6 +7,7 @@ import subprocess
from pathlib import Path
import logging
import datetime
import sys
import pytest
......@@ -168,8 +169,7 @@ def test_record_script_execution_summary(tmp_path):
root_dir = tmp_path / "test_prepare_env"
mpsd_release_to_test = "dev-23a"
release_base_dir = root_dir / mpsd_release_to_test
if os.path.exists(release_base_dir / cmd_log_file):
if os.path.exists(root_dir / cmd_log_file):
initial_bytes = os.path.getsize(cmd_log_file)
else:
initial_bytes = 0
......@@ -179,22 +179,22 @@ def test_record_script_execution_summary(tmp_path):
mod.prepare_environment(mpsd_release=mpsd_release_to_test, root_dir=(root_dir))
# check that logs/install-software-environment.log is updated
assert os.path.exists(release_base_dir / cmd_log_file)
assert os.path.getsize(release_base_dir / cmd_log_file) > initial_bytes
assert os.path.exists(root_dir / cmd_log_file)
assert os.path.getsize(root_dir / cmd_log_file) > initial_bytes
# Check that the log file has "Spack environments branch: dev-23a " in the last line
with open(release_base_dir / cmd_log_file, "r") as f:
with open(root_dir / cmd_log_file, "r") as f:
last_line = f.readlines()[-1]
assert "Spack environments branch: dev-23a " in last_line
def test_install_environment_wrong_toolchain(tmp_path):
"""Test exception is raised for non-existing toolchain."""
# Expect an Exception when wrong toolchains are provided
def test_install_environment_wrong_package_set(tmp_path):
"""Test exception is raised for non-existing package_set."""
# Expect an Exception when wrong package_sets are provided
with pytest.raises(Exception):
mod.install_environment(
mpsd_release="dev-23a",
toolchains=["wrong-toolchain"],
package_sets=["wrong-package_set"],
root_dir=(tmp_path),
)
......@@ -206,13 +206,14 @@ def test_install_environment_wrong_mpsd_release(tmp_path):
with pytest.raises(Exception):
mod.install_environment(
mpsd_release="wrong-mpsd-release",
toolchains=["foss2021a-mpi"],
package_sets=["foss2021a-mpi"],
root_dir=(tmp_path),
)
@pytest.mark.skipif(sys.platform == "darwin", reason="install not working on OSX")
def test_install_environment_zlib():
"""Test installation of toolchain."""
"""Test installation of package_set."""
# Prepare a test installation of global generic
# with only zlib to test the installation
# This is a long test,
......@@ -226,21 +227,23 @@ def test_install_environment_zlib():
shutil.rmtree(root_dir)
root_dir.mkdir(exist_ok=True, parents=True)
mpsd_release_to_test = "dev-23a"
toolchain_to_test = "global_generic"
package_set_to_test = "global_generic"
cmd_log_file = mod.config_vars["cmd_log_file"]
microarch = mod.get_native_microarchitecture()
release_base_dir = root_dir / mpsd_release_to_test
create_mock_git_repository(target_directory=root_dir, create_directory=False)
mod.prepare_environment(mpsd_release=mpsd_release_to_test, root_dir=(root_dir))
# Patch the spack environments to create a fake global_generic
# create a test toolchain
toolchain_src_dir = release_base_dir / "spack-environments" / "toolchains"
# with mod.os_chdir(toolchain_src_dir):
# create a test package_set
package_set_src_dir = release_base_dir / "spack-environments" / "toolchains"
# with mod.os_chdir(package_set_src_dir):
# subprocess.run(
# "cp -r foss2021a-mpi fuss1999a", shell=True, capture_output=True
# )
# add zlib as a spec to global_generic
with open(toolchain_src_dir / "global_generic" / "global_packages.list", "w") as f:
with open(
package_set_src_dir / "global_generic" / "global_packages.list", "w"
) as f:
f.write("zlib@1.2.13 \n")
# add zlib to whitelist of module creation file by replacing anaconda3%gcc@10.2.1
......@@ -270,14 +273,14 @@ def test_install_environment_zlib():
with open(setup_file, "w") as f:
f.write(lines)
# install global_generic toolchain
# install global_generic package_set
mod.set_up_logging(
"WARNING",
mod.get_installer_log_file_path(mpsd_release_to_test, "install", root_dir),
)
mod.install_environment(
mpsd_release=mpsd_release_to_test,
toolchains=[toolchain_to_test],
package_sets=[package_set_to_test],
root_dir=root_dir,
enable_build_cache=False,
)
......@@ -302,14 +305,14 @@ def test_install_environment_zlib():
os.path.basename(build_log)
# assert that install log files exists
assert os.path.exists(release_base_dir / cmd_log_file)
assert os.path.exists(root_dir / cmd_log_file)
# assert that the build log is written to the install log file
os.path.basename(build_log)
with open(release_base_dir / cmd_log_file, "r") as f:
with open(root_dir / cmd_log_file, "r") as f:
lines = f.read()
assert (
f"installing {toolchain_to_test} and logging at {str(build_log)}" in lines
f"installing {package_set_to_test} and logging at {str(build_log)}" in lines
)
# assert that the module files are created correctly
assert os.path.exists(release_base_dir / microarch)
......@@ -330,7 +333,7 @@ def test_install_environment_zlib():
)
mod.install_environment(
mpsd_release=mpsd_release_to_test,
toolchains=[toolchain_to_test],
package_sets=[package_set_to_test],
root_dir=root_dir,
enable_build_cache=False,
)
......@@ -381,14 +384,14 @@ def test_metadata_logging(tmp_path):
assert len(read_dict) == len(keys)
def test_get_available_toolchains():
def test_get_available_package_sets():
"""
Test that available toolchains are reported correctly.
Test that available package_sets are reported correctly.
Needs internet access to succeed.
"""
toolchains = mod.get_available_toolchains("dev-23a")
assert sorted(toolchains) == sorted(
package_sets = mod.get_available_package_sets("dev-23a")
assert sorted(package_sets) == sorted(
[
"foss2021a-cuda-mpi",
"foss2021a-mpi",
......@@ -409,18 +412,18 @@ def test_create_log_file_names():
microarch = "sandybridge"
date = datetime.datetime.now().replace(microsecond=0).isoformat()
action = "install"
toolchain = "foss2021a"
package_set = "foss2021a"
# test build_log_file_name generation
build_log_file_name = create_log_file_names(
microarch=microarch,
mpsd_release=mpsd_release,
date=date,
action=action,
toolchain=toolchain,
package_set=package_set,
)
assert (
build_log_file_name
== f"{mpsd_release}_{microarch}_{date}_BUILD_{toolchain}_{action}.log"
== f"{mpsd_release}_{microarch}_{date}_BUILD_{package_set}_{action}.log"
)
installer_log_file_name = create_log_file_names(
microarch=microarch,
......@@ -438,7 +441,7 @@ def test_create_log_file_names():
mpsd_release=mpsd_release,
date=date,
action="status",
toolchain=toolchain,
package_set=package_set,
)
assert build_log_file_name is None
......@@ -472,7 +475,7 @@ def test_environment_status(tmp_path):
def test_interface(tmp_path):
"""Test other things (not implemented yet)."""
pass
# ensure that installing without toolchains only passes the available toolchains
# ensure that installing without package_sets only passes the available package_sets
# check that the script branch and hash are correct when running the script
# check that the help message is printed when no arguments are provided
# check that the help message is printed when -h is provided
......