diff --git a/mpsd-software-environment.py b/mpsd-software-environment.py
index e61cec7283ec062af0ad7a0496f25ef2cc60ddf3..1fd2888bce6de4654358be7803f04603fa5cf87d 100755
--- a/mpsd-software-environment.py
+++ b/mpsd-software-environment.py
@@ -47,7 +47,7 @@ config_vars = {
def create_log_file_names(
mpsd_release: str,
- mpsd_microarch: str,
+ microarch: str,
action: str,
date: str = call_date_iso,
toolchain: str = None,
@@ -67,7 +67,7 @@ def create_log_file_names(
----------
mpsd_release : str
MPSD software stack version
- mpsd_microarch : str
+ microarch : str
system architecture
date : str
date of the call ins iso format
@@ -90,13 +90,13 @@ def create_log_file_names(
# if toolchain is given, then we build the build_log_file_name
if action in ["install", "remove"]:
log_file_name = (
- f"{mpsd_release}_{mpsd_microarch}_{date}_BUILD_{toolchain}_{action}.log"
+ f"{mpsd_release}_{microarch}_{date}_BUILD_{toolchain}_{action}.log"
)
else:
return None
else:
# if toolchain is not given, then we build the installer_log_file_name
- log_file_name = f"{mpsd_release}_{mpsd_microarch}_{date}_APEX_{action}.log"
+ log_file_name = f"{mpsd_release}_{microarch}_{date}_APEX_{action}.log"
return log_file_name
@@ -148,17 +148,17 @@ def read_metadata_from_logfile(logfile: Union[str, Path]) -> dict:
}
-def get_installer_log_file_path(mpsd_release: str, cmd: str, script_dir: str) -> str:
+def get_installer_log_file_path(mpsd_release: str, cmd: str, root_dir: str) -> str:
"""Get installer log file path."""
# Get machine configs
os.environ.get("MPSD_OS", "UNKNOWN_OS")
- mpsd_microarch = get_native_microarchitecture()
+ microarch = get_native_microarchitecture()
# parse logging first
# decide the log_file_name
installer_log_name = create_log_file_names(
- mpsd_release=mpsd_release, mpsd_microarch=mpsd_microarch, action=cmd
+ mpsd_release=mpsd_release, microarch=microarch, action=cmd
)
- log_folder = script_dir / mpsd_release / "logs"
+ log_folder = root_dir / mpsd_release / "logs"
# if the log_folder dosent exist, dont log this message if
# the command is a info-only command
if cmd not in ["status", "available"]:
@@ -449,20 +449,21 @@ def run(*args, counter=[0], **kwargs):
# provide information about upcoming subprocess.run call
logging.info(f"{token} Starting subprocess.run('{command}') with options {options}")
- logging.debug(f"{token} getcwd={os.getcwd()}")
- logging.debug(f"{token} exact call: subprocess.run({arg})")
+ logging.debug(f"""{token} getcwd={os.getcwd()}""")
+ logging.debug(f"""{token} subprocess.run("{arg}")""")
time_start = time.time()
process = subprocess.run(*args, **kwargs)
execution_time = time.time() - time_start
+ logging.debug(f"{token} {process=}")
logging.debug(f"{token} Completed in {execution_time:.4f}s.")
logging.debug(f"{token}") # near-empty line to make reading logs easier
return process
-def setup_log_cmd(
- mpsd_release: str, script_dir: str, msg: str = None, **kwargs
+def record_script_execution_summary(
+ mpsd_release: str, root_dir: str, msg: str = None, **kwargs
) -> None:
"""
Log the command used to build the toolchains.
@@ -476,7 +477,7 @@ def setup_log_cmd(
----------
- mpsd_release : str
The name of the release to install toolchains for.
- - script_dir : str
+ - root_dir : str
The path to the directory where the scripts are located.
- msg : str, optional
An optional message to log in the command log file.
@@ -491,7 +492,7 @@ def setup_log_cmd(
-------
- None
"""
- release_base_dir = script_dir / mpsd_release
+ release_base_dir = root_dir / mpsd_release
# Write to the log file with the following format
# --------------------------------------------------
@@ -513,7 +514,7 @@ def setup_log_cmd(
# call statement:
cmd_line = " ".join(sys.argv)
# script branch and commit hash
- with os_chdir(script_dir):
+ with os_chdir(root_dir):
script_branch = (
run(
["git", "rev-parse", "--abbrev-ref", "HEAD"],
@@ -548,9 +549,7 @@ def setup_log_cmd(
)
-def clone_repo(
- target_path: Path, repo_url: str, branch=None, skip_if_exists=True
-) -> None:
+def clone_repo(target_path: Path, repo_url: str, branch=None) -> None:
"""Clone repo locally. Optionally checkout a branch.
Parameters
@@ -595,7 +594,7 @@ def clone_repo(
run(["git", "pull"], check=True)
-def get_release_info(mpsd_release: str, script_dir: Path) -> Tuple[str, str, List[str]]:
+def get_release_info(mpsd_release: str, root_dir: Path) -> Tuple[str, str, List[str]]:
"""
Get information about the specified release.
@@ -606,7 +605,7 @@ def get_release_info(mpsd_release: str, script_dir: Path) -> Tuple[str, str, Lis
----------
mpsd_release : str
The name of the release to get information for.
- script_dir : pathlib.Path
+ root_dir : pathlib.Path
The base directory where releases are stored.
Returns
@@ -624,9 +623,9 @@ def get_release_info(mpsd_release: str, script_dir: Path) -> Tuple[str, str, Lis
If the release directory does not exist.
"""
# Get the info for release
- release_base_dir = script_dir / mpsd_release
+ release_base_dir = root_dir / mpsd_release
if not os.path.exists(release_base_dir):
- logging.debug(f"get_release_info({mpsd_release=}, {script_dir=})")
+ logging.debug(f"get_release_info({mpsd_release=}, {root_dir=})")
raise FileNotFoundError(
f"{release_base_dir} does not exist.\n"
f"Hint: `prepare {mpsd_release}` may fix this."
@@ -652,7 +651,7 @@ def get_release_info(mpsd_release: str, script_dir: Path) -> Tuple[str, str, Lis
return spe_branch, spe_commit_hash, available_toolchains
-def prepare_environment(mpsd_release: str, script_dir: Path) -> List[str]:
+def prepare_environment(mpsd_release: str, root_dir: Path) -> List[str]:
"""
Create the directory structure for the given MPSD release.
@@ -666,7 +665,7 @@ def prepare_environment(mpsd_release: str, script_dir: Path) -> List[str]:
----------
mpsd_release : str
The name of the MPSD release to prepare the environment for.
- script_dir : pathlib.Path
+ root_dir : pathlib.Path
The base directory to create the release folder and
clone the spack-environments repository into.
@@ -679,21 +678,28 @@ def prepare_environment(mpsd_release: str, script_dir: Path) -> List[str]:
# Spack environments repository if it doesn't exist:
# Create the directory structure for the release
- release_base_dir = script_dir / mpsd_release
+ release_base_dir = root_dir / mpsd_release
release_base_dir.mkdir(parents=True, exist_ok=True)
repo_path = release_base_dir / "spack-environments"
if repo_path.exists():
logging.debug(f"directory {repo_path} exists already, not touching")
+ logging.debug(
+ "XXX TODO: should we run a git pull here to get the latest version? XXX"
+ )
else:
repo_url = config_vars["spack_environments_repo"]
logging.info(f"cloning repository {repo_path} from {repo_url}")
clone_repo(repo_path, repo_url, branch=mpsd_release)
+ logging.getLogger("print").info(
+ f"Release {mpsd_release} is prepared in {release_base_dir}"
+ )
+
spe_branch, spe_commit_hash, available_toolchains = get_release_info(
- mpsd_release, script_dir
+ mpsd_release, root_dir
)
- setup_log_cmd(
- mpsd_release, script_dir, spe_branch=spe_branch, spe_commit_hash=spe_commit_hash
+ record_script_execution_summary(
+ mpsd_release, root_dir, spe_branch=spe_branch, spe_commit_hash=spe_commit_hash
)
return available_toolchains
@@ -701,13 +707,13 @@ def prepare_environment(mpsd_release: str, script_dir: Path) -> List[str]:
def get_native_microarchitecture():
"""Return native microarchitecture.
- On MPSD machines, there should be an environment variable "MPSD_MICROARCH".
+ On MPSD machines, there should be an environment variable "microarch".
We try to read that. If it fails, we use the 'archspec cpu' command.
If that fails, we ask the user to install it.
Returns
-------
- MPSD_MICROARCH : str
+ microarch : str
Example
-------
@@ -756,7 +762,7 @@ def get_native_microarchitecture():
def install_environment(
mpsd_release: str,
toolchains: List[str],
- script_dir: Path,
+ root_dir: Path,
force_reinstall: bool = False,
enable_build_cache: bool = False,
) -> None:
@@ -772,7 +778,7 @@ def install_environment(
toolchains : list of str
A list of strings representing the toolchains to install
(e.g., "foss2021a-mpi", "global_generic", "ALL").
- script_dir : pathlib.Path
+ root_dir : pathlib.Path
A Path object representing the path to the directory where
the release and toolchains will be installed.
force_reinstall : bool, optional
@@ -793,13 +799,13 @@ def install_environment(
"""
logging.info(
f"Installing release {mpsd_release} with toolchains {toolchains} "
- f"to {script_dir}"
+ f"to {root_dir}"
)
# Set required variables
- release_base_dir = script_dir / mpsd_release
- mpsd_microarch = get_native_microarchitecture()
- toolchain_dir = release_base_dir / mpsd_microarch
+ release_base_dir = root_dir / mpsd_release
+ microarch = get_native_microarchitecture()
+ toolchain_dir = release_base_dir / microarch
toolchain_dir.mkdir(parents=True, exist_ok=True)
spack_setup_script = release_base_dir / "spack-environments" / "spack_setup.sh"
install_flags = []
@@ -807,7 +813,7 @@ def install_environment(
install_flags.append("-b")
# run the prepare_environment function
- available_toolchains = prepare_environment(mpsd_release, script_dir)
+ available_toolchains = prepare_environment(mpsd_release, root_dir)
# Ensure that the requested toolchains are available in the release
if toolchains == "ALL":
toolchains = available_toolchains
@@ -838,7 +844,7 @@ def install_environment(
for toolchain in toolchains:
# Set the install log file name from create_log_file_names
build_log_file_name = create_log_file_names(
- mpsd_release, mpsd_microarch, "install", toolchain=toolchain
+ mpsd_release, microarch, "install", toolchain=toolchain
)
build_log_folder = release_base_dir / "logs"
build_log_path = build_log_folder / build_log_file_name
@@ -849,14 +855,14 @@ def install_environment(
logging.info(f"Installing toolchain {toolchain} to {toolchain_dir}")
# log the command
- setup_log_cmd(
+ record_script_execution_summary(
mpsd_release,
- script_dir,
+ root_dir,
msg=f"installing {toolchain} and logging at {build_log_path}",
)
- setup_log_cmd(
+ record_script_execution_summary(
mpsd_release,
- script_dir,
+ root_dir,
msg=(
f"CMD: bash {spack_setup_script} {' '.join(install_flags)} "
f"{toolchain}"
@@ -967,24 +973,24 @@ def main():
args = parser.parse_args()
# target dir is the place where this script exists. the
- script_dir = Path(os.path.dirname(os.path.realpath(__file__)))
+ root_dir = Path(os.path.dirname(os.path.realpath(__file__)))
set_up_logging(
args.loglevel,
- get_installer_log_file_path(args.release, args.action, script_dir),
+ get_installer_log_file_path(args.release, args.action, root_dir),
)
# Check the command and run related function
if args.action == "remove":
- remove_environment(args.release, args.toolchains, script_dir)
+ remove_environment(args.release, args.toolchains, root_dir)
elif args.action == "start-new":
- start_new_environment(args.from_release, args.to_release, script_dir)
+ start_new_environment(args.from_release, args.to_release, root_dir)
elif args.action == "install":
install_environment(
- args.release, args.toolchains, script_dir, False, args.enable_build_cache
+ args.release, args.toolchains, root_dir, False, args.enable_build_cache
)
elif args.action == "prepare":
- prepare_environment(args.release, script_dir)
+ prepare_environment(args.release, root_dir)
elif args.action == "available":
get_available_toolchains_upstream(args.release)
else:
diff --git a/tests.py b/tests.py
index c284591902e312c7b7d662a9da542894443e0681..685ce134653ab52da9b67c82d52e6d786862b8ef 100644
--- a/tests.py
+++ b/tests.py
@@ -112,23 +112,23 @@ def test_prepare_environment(tmp_path):
prepare_env is run when cmd is not specified, we can test cmd='prepare'
and cmd=None to check both cases
"""
- script_dir = tmp_path / "mpsd_opt" / "linux_debian_11"
+ root_dir = tmp_path / "mpsd_opt" / "linux_debian_11"
spack_environments = "spack-environments"
mpsd_release_to_test = "dev-23a"
- release_base_dir = script_dir / mpsd_release_to_test
+ release_base_dir = root_dir / mpsd_release_to_test
# check that the test directory does not exist
- assert not script_dir.exists()
+ assert not root_dir.exists()
# prepare_environment expects to be executed in git repository
# (mpsd-software-environments). It queries the commit on which we are to
# log that information. For this to work, we need to execute the command
# within a directory tree that has a git repository at the same or high
# level. Let's create one:
- create_mock_git_repository(script_dir)
+ create_mock_git_repository(root_dir)
# now call the function we want to test
result = mod.prepare_environment(
- mpsd_release=mpsd_release_to_test, script_dir=script_dir
+ mpsd_release=mpsd_release_to_test, root_dir=root_dir
)
# check if the directory now is created
@@ -155,28 +155,28 @@ def test_prepare_environment(tmp_path):
# Expect an Exception when wrong mpsd_release is provided
with pytest.raises(Exception):
result = mod.prepare_environment(
- mpsd_release="wrong-mpsd-release", script_dir=(script_dir)
+ mpsd_release="wrong-mpsd-release", root_dir=(root_dir)
)
-def test_setup_log_cmd(tmp_path):
+def test_record_script_execution_summary(tmp_path):
"""Check that log is updated.
Check that logs/install-software-environment.log is updated when the module is run
"""
cmd_log_file = mod.config_vars["cmd_log_file"]
- script_dir = tmp_path / "test_prepare_env"
+ root_dir = tmp_path / "test_prepare_env"
mpsd_release_to_test = "dev-23a"
- release_base_dir = script_dir / mpsd_release_to_test
+ release_base_dir = root_dir / mpsd_release_to_test
if os.path.exists(release_base_dir / cmd_log_file):
initial_bytes = os.path.getsize(cmd_log_file)
else:
initial_bytes = 0
# run the prepare_env functionality
- create_mock_git_repository(target_directory=script_dir, create_directory=True)
- mod.prepare_environment(mpsd_release=mpsd_release_to_test, script_dir=(script_dir))
+ create_mock_git_repository(target_directory=root_dir, create_directory=True)
+ mod.prepare_environment(mpsd_release=mpsd_release_to_test, root_dir=(root_dir))
# check that logs/install-software-environment.log is updated
assert os.path.exists(release_base_dir / cmd_log_file)
@@ -195,7 +195,7 @@ def test_install_environment_wrong_toolchain(tmp_path):
mod.install_environment(
mpsd_release="dev-23a",
toolchains=["wrong-toolchain"],
- script_dir=(tmp_path),
+ root_dir=(tmp_path),
)
@@ -207,7 +207,7 @@ def test_install_environment_wrong_mpsd_release(tmp_path):
mod.install_environment(
mpsd_release="wrong-mpsd-release",
toolchains=["foss2021a-mpi"],
- script_dir=(tmp_path),
+ root_dir=(tmp_path),
)
@@ -221,17 +221,17 @@ def test_install_environment_zlib():
# pytest -s
# for this installation avoid tmp_path as
# the length of the path becomes too long and spack complains
- script_dir = Path("/tmp/test_global_generic")
- if script_dir.exists():
- shutil.rmtree(script_dir)
- script_dir.mkdir(exist_ok=True, parents=True)
+ root_dir = Path("/tmp/test_global_generic")
+ if root_dir.exists():
+ shutil.rmtree(root_dir)
+ root_dir.mkdir(exist_ok=True, parents=True)
mpsd_release_to_test = "dev-23a"
toolchain_to_test = "global_generic"
cmd_log_file = mod.config_vars["cmd_log_file"]
- mpsd_microarch = mod.get_native_microarchitecture()
- release_base_dir = script_dir / mpsd_release_to_test
- create_mock_git_repository(target_directory=script_dir, create_directory=False)
- mod.prepare_environment(mpsd_release=mpsd_release_to_test, script_dir=(script_dir))
+ microarch = mod.get_native_microarchitecture()
+ release_base_dir = root_dir / mpsd_release_to_test
+ create_mock_git_repository(target_directory=root_dir, create_directory=False)
+ mod.prepare_environment(mpsd_release=mpsd_release_to_test, root_dir=(root_dir))
# Patch the spack environments to create a fake global_generic
# create a test toolchain
toolchain_src_dir = release_base_dir / "spack-environments" / "toolchains"
@@ -273,23 +273,23 @@ def test_install_environment_zlib():
# install global_generic toolchain
mod.set_up_logging(
"WARNING",
- mod.get_installer_log_file_path(mpsd_release_to_test, "install", script_dir),
+ mod.get_installer_log_file_path(mpsd_release_to_test, "install", root_dir),
)
mod.install_environment(
mpsd_release=mpsd_release_to_test,
toolchains=[toolchain_to_test],
- script_dir=script_dir,
+ root_dir=root_dir,
enable_build_cache=False,
)
# test that the build log is created correctly
# check that a file with glob build_globale_generic_dev-23a*.log exists at
- # release_base_dir/mpsd_microarch
+ # release_base_dir/microarch
# print("Debug here ")
# time.sleep(10)
build_log = list(
(release_base_dir / "logs").glob(
- f"{mpsd_release_to_test}_{mpsd_microarch}_*_install.log"
+ f"{mpsd_release_to_test}_{microarch}_*_install.log"
)
)
assert len(build_log) == 2
@@ -312,12 +312,10 @@ def test_install_environment_zlib():
f"installing {toolchain_to_test} and logging at {str(build_log)}" in lines
)
# assert that the module files are created correctly
- assert os.path.exists(release_base_dir / mpsd_microarch)
- assert os.path.exists(release_base_dir / mpsd_microarch / "lmod")
+ assert os.path.exists(release_base_dir / microarch)
+ assert os.path.exists(release_base_dir / microarch / "lmod")
# assert that lmod/module-index.yaml contains zlib
- with open(
- release_base_dir / mpsd_microarch / "lmod" / "module-index.yaml", "r"
- ) as f:
+ with open(release_base_dir / microarch / "lmod" / "module-index.yaml", "r") as f:
lines = f.read()
assert "zlib" in lines
@@ -328,17 +326,17 @@ def test_install_environment_zlib():
importlib.reload(mod)
mod.set_up_logging(
"WARNING",
- mod.get_installer_log_file_path(mpsd_release_to_test, "install", script_dir),
+ mod.get_installer_log_file_path(mpsd_release_to_test, "install", root_dir),
)
mod.install_environment(
mpsd_release=mpsd_release_to_test,
toolchains=[toolchain_to_test],
- script_dir=script_dir,
+ root_dir=root_dir,
enable_build_cache=False,
)
build_log = list(
(release_base_dir / "logs").glob(
- f"{mpsd_release_to_test}_{mpsd_microarch}_*_install.log"
+ f"{mpsd_release_to_test}_{microarch}_*_install.log"
)
)
assert len(build_log) == 4
@@ -387,13 +385,13 @@ def test_create_log_file_names():
"""Test that the log file names are created correctly."""
create_log_file_names = mod.create_log_file_names
mpsd_release = "dev-23a"
- mpsd_microarch = "sandybridge"
+ microarch = "sandybridge"
date = datetime.datetime.now().replace(microsecond=0).isoformat()
action = "install"
toolchain = "foss2021a"
# test build_log_file_name generation
build_log_file_name = create_log_file_names(
- mpsd_microarch=mpsd_microarch,
+ microarch=microarch,
mpsd_release=mpsd_release,
date=date,
action=action,
@@ -401,21 +399,21 @@ def test_create_log_file_names():
)
assert (
build_log_file_name
- == f"{mpsd_release}_{mpsd_microarch}_{date}_BUILD_{toolchain}_{action}.log"
+ == f"{mpsd_release}_{microarch}_{date}_BUILD_{toolchain}_{action}.log"
)
installer_log_file_name = create_log_file_names(
- mpsd_microarch=mpsd_microarch,
+ microarch=microarch,
mpsd_release=mpsd_release,
date=date,
action=action,
)
assert (
installer_log_file_name
- == f"{mpsd_release}_{mpsd_microarch}_{date}_APEX_{action}.log"
+ == f"{mpsd_release}_{microarch}_{date}_APEX_{action}.log"
)
# test no build log file for incorrect action
build_log_file_name = create_log_file_names(
- mpsd_microarch=mpsd_microarch,
+ microarch=microarch,
mpsd_release=mpsd_release,
date=date,
action="status",