diff --git a/mpsd-software-environment.py b/mpsd-software-environment.py
index 313682a2187320c635df88a14fd3540f622190fe..3f6b33feb0e17659c36fd8caa61e6885d37e918d 100755
--- a/mpsd-software-environment.py
+++ b/mpsd-software-environment.py
@@ -31,19 +31,75 @@ for given system architecture and MPSD software stack version.\n
The toolchains
are built using the bash script spack_setup.sh, and the results are logged. """
+call_date_iso = (
+ datetime.datetime.now().replace(microsecond=0).isoformat().replace(":", "-")
+)
config_vars = {
- "cmd_log_file": "install.log",
- "build_log_file": (
- "logs/mpsd_spack_ver_toolchains_"
- f"{datetime.datetime.now().replace(microsecond=0).isoformat()}.log"
- ),
+ # kept inside the mpsd_release folder
+ "cmd_log_file": "script_execution_summary.log",
+ # Metadata tags
"metadata_tag_open": "!<meta>",
"metadata_tag_close": "</meta>!",
- # TODO: modify toolchains,mpsd_spack_ver when the variable is available
"spack_environments_repo": "https://gitlab.gwdg.de/mpsd-cs/spack-environments.git",
}
+def create_log_file_names(
+ mpsd_release: str,
+ mpsd_microarch: str,
+ action: str,
+ date: str = call_date_iso,
+ toolchain: str = None,
+) -> Union[str, None]:
+ """Create log file names.
+
+ This function creates the log file names for either the installer or
+ the build log files.
+
+ If a toolchain is given, then the build log file name is created.
+ if no toolchain is given, then the installer log file name is created.
+ The installer log file hosts the logs of the installer script, while
+ the build log file hosts the logs of the build process as generated by the
+ spack_setup.sh script.
+
+ Parameters
+ ----------
+ mpsd_release : str
+ MPSD software stack version
+ mpsd_microarch : str
+ system architecture
+ date : str
+ date of the call ins iso format
+ action : str
+ action performed (install,remove,reinstall,prepare,status)
+ only install and remove are valid for build log file.
+ toolchain : str
+ toolchain name (only for build log file)
+
+ Returns
+ -------
+ str or None
+ log file name
+ installer_log_file_name or build_log_file_name depending on the
+ parameters given.
+ If the action is not one that changes the files on disk ( info only actions)
+ then None is returned.
+ """
+ if toolchain:
+ # if toolchain is given, then we build the build_log_file_name
+ if action in ["install", "remove"]:
+ log_file_name = (
+ f"{mpsd_release}_{mpsd_microarch}_{date}_BUILD_{toolchain}_{action}.log"
+ )
+ else:
+ return None
+ else:
+ # if toolchain is not given, then we build the installer_log_file_name
+ log_file_name = f"{mpsd_release}_{mpsd_microarch}_{date}_APEX_{action}.log"
+
+ return log_file_name
+
+
def log_metadata(key: str, value: str) -> None:
"""Log metadata to the log file.
@@ -91,7 +147,29 @@ def read_metadata_from_logfile(logfile: Union[str, Path]) -> dict:
}
-def set_up_logging(loglevel="warning", filename=None):
+def get_installer_log_file_path(mpsd_release: str, cmd: str, script_dir: str) -> str:
+ """Get installer log file path."""
+ # Get machine configs
+ os.environ.get("MPSD_OS", "UNKNOWN_OS")
+ mpsd_microarch = get_native_microarchitecture()
+ # parse logging first
+ # decide the log_file_name
+ installer_log_name = create_log_file_names(
+ mpsd_release=mpsd_release, mpsd_microarch=mpsd_microarch, action=cmd
+ )
+ log_folder = script_dir / mpsd_release / "logs"
+ # if the log_folder dosent exist, dont log this message if
+ # the command is a info-only command
+ if cmd not in ["status", "available"]:
+ if not os.path.exists(log_folder):
+ os.makedirs(log_folder)
+ installer_log_file = log_folder / installer_log_name
+ else:
+ installer_log_file = None
+ return installer_log_file
+
+
+def set_up_logging(loglevel="warning", file_path=None):
"""Set up logging.
This function sets up the logging configuration for the script.
@@ -106,7 +184,7 @@ def set_up_logging(loglevel="warning", filename=None):
- warning (default): only print statements if something is unexpected
- info (show more detailed progress)
- debug (show very detailed output)
- filename : str
+ file_path : str
- filename to save logging messages into
If loglevel is 'debug', save line numbers in log messages.
@@ -188,11 +266,16 @@ def set_up_logging(loglevel="warning", filename=None):
shell_handler.setFormatter(shell_formatter)
# use the log_level_numeric to decide how much logging is sent to shell
shell_handler.setLevel(log_level_numeric)
- logger.addHandler(shell_handler)
+
+ # Here we set the handlers of the RootLogger to be just the one we want.
+ # The reason is that the logging module will add a <StreamHandler <stderr>
+ # (NOTSET)> handler if logging.info/logging.debug/... is used before we
+ # come across this line. And we do not want that additional handler.
+ logger.handlers = [shell_handler]
# if filename provided, write log messages to that file, too.
- if filename:
- file_handler = logging.FileHandler(filename)
+ if file_path:
+ file_handler = logging.FileHandler(file_path)
# if we have a file, we write all information in there.
# We could change the level, for example restrict to only DEBUG and above with
# file_handler.setLevel(logging.DEBUG)
@@ -216,9 +299,9 @@ def set_up_logging(loglevel="warning", filename=None):
print_log.addHandler(ch)
# if filename provided, write output of print_log to that file, too
- if filename:
+ if file_path:
# create, format and add file handler
- fh = logging.FileHandler(filename)
+ fh = logging.FileHandler(file_path)
fh.setFormatter(formatter)
print_log.addHandler(fh)
@@ -226,8 +309,8 @@ def set_up_logging(loglevel="warning", filename=None):
# short message
#
logging.debug(
- f"Logging has been setup, loglevel={loglevel.upper()}"
- + f"{filename=} {rich_available=}"
+ f"Logging has been setup, loglevel={loglevel.upper()} "
+ + f"{file_path=} {rich_available=}"
)
@@ -658,8 +741,7 @@ def install_environment(
# Set required variables
release_base_dir = script_dir / mpsd_release
- os.environ.get("MPSD_OS", "UNKNOWN_OS")
- mpsd_microarch = os.environ.get("MPSD_MICROARCH", "UNKNOWN_MICROARCH")
+ mpsd_microarch = get_native_microarchitecture()
toolchain_dir = release_base_dir / mpsd_microarch
toolchain_dir.mkdir(parents=True, exist_ok=True)
spack_setup_script = release_base_dir / "spack-environments" / "spack_setup.sh"
@@ -697,33 +779,35 @@ def install_environment(
if not os.path.exists("logs"):
os.mkdir("logs")
for toolchain in toolchains:
- # Set the install log file name to config_vars["install_log_file"]
- # and replace _toolchains_ with the toolchain name and
- # _mpsd_spack_ver_ with mpsd_release
+ # Set the install log file name from create_log_file_names
+ build_log_file_name = create_log_file_names(
+ mpsd_release, mpsd_microarch, "install", toolchain=toolchain
+ )
+ build_log_folder = release_base_dir / "logs"
+ build_log_path = build_log_folder / build_log_file_name
+ # if logs folder dosent exist, create it
+ if not os.path.exists(build_log_folder):
+ os.makedirs(build_log_folder)
logging.info(f"Installing toolchain {toolchain} to {toolchain_dir}")
- install_log_file = (
- config_vars["build_log_file"]
- .replace("mpsd_spack_ver_", f"{mpsd_release}_")
- .replace("_toolchains_", f"_{toolchain}_")
- )
+
# log the command
setup_log_cmd(
mpsd_release,
script_dir,
- msg=f"installing {toolchain} and logging at {install_log_file}",
+ msg=f"installing {toolchain} and logging at {build_log_path}",
)
setup_log_cmd(
mpsd_release,
script_dir,
msg=(
- f"CMD: bash {spack_setup_script} {' '.join(install_flags)}"
- "{toolchain}"
+ f"CMD: bash {spack_setup_script} {' '.join(install_flags)} "
+ f"{toolchain}"
),
)
run(
f"bash {spack_setup_script} {' '.join(install_flags)} {toolchain} 2>&1 "
- f"| tee -a {install_log_file} ",
+ f"| tee -a {build_log_path} ",
shell=True,
check=True,
)
@@ -824,13 +908,14 @@ def main():
# Carry out the action
args = parser.parse_args()
- # parse logging first
- set_up_logging(args.loglevel)
-
# target dir is the place where this script exists. the
- # release `dev` in script_dir/dev-23a
script_dir = Path(os.path.dirname(os.path.realpath(__file__)))
+ set_up_logging(
+ args.loglevel,
+ get_installer_log_file_path(args.release, args.action, script_dir),
+ )
+
# Check the command and run related function
if args.action == "remove":
remove_environment(args.release, args.toolchains, script_dir)
diff --git a/tests.py b/tests.py
index c7103b4f04ca2ef93f716c2d7e95379da33118c2..c284591902e312c7b7d662a9da542894443e0681 100644
--- a/tests.py
+++ b/tests.py
@@ -6,13 +6,15 @@ import shutil
import subprocess
from pathlib import Path
import logging
+import datetime
+
import pytest
mod = importlib.import_module("mpsd-software-environment")
# set loglevel to debug - useful for understanding problems.
# (if the tests pass, pytest doesn't show any output)
-mod.set_up_logging(loglevel="debug", filename="tests.log")
+mod.set_up_logging(loglevel="debug", file_path="tests.log")
logging.debug(f"We have set up logging from {__file__}")
@@ -162,13 +164,13 @@ def test_setup_log_cmd(tmp_path):
Check that logs/install-software-environment.log is updated when the module is run
"""
- log_file = "install.log"
+ cmd_log_file = mod.config_vars["cmd_log_file"]
script_dir = tmp_path / "test_prepare_env"
mpsd_release_to_test = "dev-23a"
release_base_dir = script_dir / mpsd_release_to_test
- if os.path.exists(release_base_dir / log_file):
- initial_bytes = os.path.getsize(log_file)
+ if os.path.exists(release_base_dir / cmd_log_file):
+ initial_bytes = os.path.getsize(cmd_log_file)
else:
initial_bytes = 0
@@ -177,11 +179,11 @@ def test_setup_log_cmd(tmp_path):
mod.prepare_environment(mpsd_release=mpsd_release_to_test, script_dir=(script_dir))
# check that logs/install-software-environment.log is updated
- assert os.path.exists(release_base_dir / log_file)
- assert os.path.getsize(release_base_dir / log_file) > initial_bytes
+ assert os.path.exists(release_base_dir / cmd_log_file)
+ assert os.path.getsize(release_base_dir / cmd_log_file) > initial_bytes
# Check that the log file has "Spack environments branch: dev-23a " in the last line
- with open(release_base_dir / log_file, "r") as f:
+ with open(release_base_dir / cmd_log_file, "r") as f:
last_line = f.readlines()[-1]
assert "Spack environments branch: dev-23a " in last_line
@@ -225,7 +227,8 @@ def test_install_environment_zlib():
script_dir.mkdir(exist_ok=True, parents=True)
mpsd_release_to_test = "dev-23a"
toolchain_to_test = "global_generic"
- mpsd_microarch = os.getenv("MPSD_MICROARCH", "UNKNOWN_MICROARCH")
+ cmd_log_file = mod.config_vars["cmd_log_file"]
+ mpsd_microarch = mod.get_native_microarchitecture()
release_base_dir = script_dir / mpsd_release_to_test
create_mock_git_repository(target_directory=script_dir, create_directory=False)
mod.prepare_environment(mpsd_release=mpsd_release_to_test, script_dir=(script_dir))
@@ -266,7 +269,12 @@ def test_install_environment_zlib():
)
with open(setup_file, "w") as f:
f.write(lines)
+
# install global_generic toolchain
+ mod.set_up_logging(
+ "WARNING",
+ mod.get_installer_log_file_path(mpsd_release_to_test, "install", script_dir),
+ )
mod.install_environment(
mpsd_release=mpsd_release_to_test,
toolchains=[toolchain_to_test],
@@ -278,30 +286,30 @@ def test_install_environment_zlib():
# release_base_dir/mpsd_microarch
# print("Debug here ")
# time.sleep(10)
+
build_log = list(
- (release_base_dir / mpsd_microarch / "logs").glob(
- f"{mpsd_release_to_test}_{toolchain_to_test}_*.log"
+ (release_base_dir / "logs").glob(
+ f"{mpsd_release_to_test}_{mpsd_microarch}_*_install.log"
)
)
- assert len(build_log) > 0
+ assert len(build_log) == 2
# take the most recent build log
- build_log = sorted(build_log)[0]
+ build_log = sorted(build_log)[1]
# check that the build log contains statement ##### Installation finished
with open(build_log, "r") as f:
lines = f.read()
assert "##### Installation finished" in lines
- build_log_file_name = os.path.basename(build_log)
+ os.path.basename(build_log)
# assert that install log files exists
- assert os.path.exists(release_base_dir / "install.log")
+ assert os.path.exists(release_base_dir / cmd_log_file)
# assert that the build log is written to the install log file
os.path.basename(build_log)
- with open(release_base_dir / "install.log", "r") as f:
+ with open(release_base_dir / cmd_log_file, "r") as f:
lines = f.read()
assert (
- f"installing {toolchain_to_test} and logging at logs/{build_log_file_name}"
- in lines
+ f"installing {toolchain_to_test} and logging at {str(build_log)}" in lines
)
# assert that the module files are created correctly
assert os.path.exists(release_base_dir / mpsd_microarch)
@@ -313,13 +321,35 @@ def test_install_environment_zlib():
lines = f.read()
assert "zlib" in lines
+ # install again to ensure that
+ # commands that skip creation of folders when
+ # they are already present works as expected
+ # reload the module to ensure that date changes
+ importlib.reload(mod)
+ mod.set_up_logging(
+ "WARNING",
+ mod.get_installer_log_file_path(mpsd_release_to_test, "install", script_dir),
+ )
+ mod.install_environment(
+ mpsd_release=mpsd_release_to_test,
+ toolchains=[toolchain_to_test],
+ script_dir=script_dir,
+ enable_build_cache=False,
+ )
+ build_log = list(
+ (release_base_dir / "logs").glob(
+ f"{mpsd_release_to_test}_{mpsd_microarch}_*_install.log"
+ )
+ )
+ assert len(build_log) == 4
+
def test_metadata_logging(tmp_path):
"""Test that metadata is logged and read correctly."""
# Test that the metadata is logged correctly
filename = tmp_path / "test-metadata.log"
print(f"Writing to {filename}")
- mod.set_up_logging(loglevel="debug", filename=filename)
+ mod.set_up_logging(loglevel="debug", file_path=filename)
# our test data
keys = ["important_key", "important_key2"]
@@ -353,6 +383,47 @@ def test_metadata_logging(tmp_path):
assert len(read_dict) == len(keys)
+def test_create_log_file_names():
+ """Test that the log file names are created correctly."""
+ create_log_file_names = mod.create_log_file_names
+ mpsd_release = "dev-23a"
+ mpsd_microarch = "sandybridge"
+ date = datetime.datetime.now().replace(microsecond=0).isoformat()
+ action = "install"
+ toolchain = "foss2021a"
+ # test build_log_file_name generation
+ build_log_file_name = create_log_file_names(
+ mpsd_microarch=mpsd_microarch,
+ mpsd_release=mpsd_release,
+ date=date,
+ action=action,
+ toolchain=toolchain,
+ )
+ assert (
+ build_log_file_name
+ == f"{mpsd_release}_{mpsd_microarch}_{date}_BUILD_{toolchain}_{action}.log"
+ )
+ installer_log_file_name = create_log_file_names(
+ mpsd_microarch=mpsd_microarch,
+ mpsd_release=mpsd_release,
+ date=date,
+ action=action,
+ )
+ assert (
+ installer_log_file_name
+ == f"{mpsd_release}_{mpsd_microarch}_{date}_APEX_{action}.log"
+ )
+ # test no build log file for incorrect action
+ build_log_file_name = create_log_file_names(
+ mpsd_microarch=mpsd_microarch,
+ mpsd_release=mpsd_release,
+ date=date,
+ action="status",
+ toolchain=toolchain,
+ )
+ assert build_log_file_name is None
+
+
def test_interface(tmp_path):
"""Test other things (not implemented yet)."""
pass