diff --git a/src/mpsd_software_manager/mpsd_software.py b/src/mpsd_software_manager/mpsd_software.py
index c6b414d572cddf904b6134eba15924fb646d7a55..458c26f390ade9a6be3d9eb17b4c7efa985b2160 100755
--- a/src/mpsd_software_manager/mpsd_software.py
+++ b/src/mpsd_software_manager/mpsd_software.py
@@ -90,12 +90,59 @@ config_vars = {
}
-def create_log_file_names(
+def log_metadata(key: str, value: str) -> None:
+ """Log metadata to the log file.
+
+ This function logs metadata to the log file. The metadata is
+ enclosed in a tag, so that it can be easily found in the log file.
+ logging module is used to write the metadata to the log file.
+
+ Parameters
+ ----------
+ key : str
+ key of the metadata
+ value : str
+ value of the metadata
+ returns : None
+ """
+ logging.info(
+ f"{config_vars['metadata_tag_open']}{key}:{value}{config_vars['metadata_tag_close']}"
+ )
+
+
+def read_metadata_from_logfile(logfile: Union[str, Path]) -> dict:
+ """Read metadata from the log file.
+
+ This function reads metadata from the log file. The metadata is
+ enclosed in a tag, so that it can be easily found in the log file.
+
+ Parameters
+ ----------
+ logfile : str or Path
+ log file name
+ returns : dict
+ dictionary containing the metadata
+ """
+ with open(logfile, "r") as f:
+ log_text = f.read()
+ # check for all data that matches the regex
+ # metadata_tag_open {key}:{value} metadata_tag_close
+ # and return a dictionary with all the matches
+ return {
+ match.group(1): match.group(2)
+ for match in re.finditer(
+ rf"{config_vars['metadata_tag_open']}(\w+):(\w+){config_vars['metadata_tag_close']}",
+ log_text,
+ )
+ }
+
+
+def create_log_file_name(
mpsd_release: str,
microarch: str,
action: str,
date: str = call_date_iso,
- package_set: str = None,
+ package_set: Union[str, None] = None,
) -> Union[str, None]:
"""Create log file names.
@@ -119,7 +166,7 @@ def create_log_file_names(
action : str
action performed (install,remove,reinstall,prepare,status)
only install and remove are valid for build log file.
- package_set : str
+ package_set : str or None
package_set name (only for build log file)
Returns
@@ -128,8 +175,48 @@ def create_log_file_names(
log file name
installer_log_file_name or build_log_file_name depending on the
parameters given.
- If the action is not one that changes the files on disk ( info only actions)
+ If the action is not one that changes the files on disk (info only actions)
then None is returned.
+
+ Examples
+ --------
+ # installer log file name for `mpsd-software install dev-23a foss2021a-mpi`
+ >>> create_log_file_name(
+ ... "dev-23a",
+ ... "sandybridge",
+ ... "install",
+ ... "2023-07-03T12-27-52",
+ ... )
+ 'dev-23a_sandybridge_2023-07-03T12-27-52_APEX_install.log'
+
+ # build log file name for `mpsd-software install dev-23a foss2021a-mpi`
+ >>> create_log_file_name(
+ ... "dev-23a",
+ ... "sandybridge",
+ ... "install",
+ ... "2023-07-03T12-27-52",
+ ... "foss2021a-mpi",
+ ... )
+ 'dev-23a_sandybridge_2023-07-03T12-27-52_BUILD_foss2021a-mpi_install.log'
+
+ # installer log file name for `mpsd-software status dev-23a`
+ >>> create_log_file_name(
+ ... "dev-23a",
+ ... "sandybridge",
+ ... "status",
+ ... "2023-07-03T12-27-52",
+ ... )
+ 'dev-23a_sandybridge_2023-07-03T12-27-52_APEX_status.log'
+
+ # build log file name for `mpsd-software status dev-23a` (no log file is created)
+ >>> create_log_file_name(
+ ... "dev-23a",
+ ... "sandybridge",
+ ... "status",
+ ... "2023-07-03T12-27-52",
+ ... "foss2021a-mpi",
+ ... )
+ (None)
"""
if package_set:
# if package_set is given, then we build the build_log_file_name
@@ -146,73 +233,97 @@ def create_log_file_names(
return log_file_name
-def log_metadata(key: str, value: str) -> None:
- """Log metadata to the log file.
-
- This function logs metadata to the log file. The metadata is
- enclosed in a tag, so that it can be easily found in the log file.
- logging module is used to write the metadata to the log file.
-
- Parameters
- ----------
- key : str
- key of the metadata
- value : str
- value of the metadata
- returns : None
- """
- logging.info(
- f"{config_vars['metadata_tag_open']}{key}:{value}{config_vars['metadata_tag_close']}"
- )
+def get_log_file_path(
+ mpsd_release: str, cmd: str, root_dir: Path, package_set: Union[str, None] = None
+) -> Union[Path, None]:
+ """Get log file path.
+ This function creates the log file paths for either the installer or
+ the build log files.
-def read_metadata_from_logfile(logfile: Union[str, Path]) -> dict:
- """Read metadata from the log file.
+ If a package_set is given, then the build log file path is returned.
+ if no package_set is given, then the installer log file path is returned.
- This function reads metadata from the log file. The metadata is
- enclosed in a tag, so that it can be easily found in the log file.
+ If the logs folder does not exist, then it is created.
Parameters
----------
- logfile : str or Path
- log file name
- returns : dict
- dictionary containing the metadata
- """
- with open(logfile, "r") as f:
- log_text = f.read()
- # check for all data that matches the regex
- # metadata_tag_open {key}:{value} metadata_tag_close
- # and return a dictionary with all the matches
- return {
- match.group(1): match.group(2)
- for match in re.finditer(
- rf"{config_vars['metadata_tag_open']}(\w+):(\w+){config_vars['metadata_tag_close']}",
- log_text,
- )
- }
+ mpsd_release : str
+ MPSD software stack version
+ cmd : str
+ command to be executed
+ root_dir : str
+ root directory of the mpsd software stack
+ package_set : str
+ package_set name (only for build log file)
+ Returns
+ -------
+ Path or None
+ log file path
+ installer_log_file_path or build_log_file_path depending on the
+ parameters given.
-def get_installer_log_file_path(mpsd_release: str, cmd: str, root_dir: str) -> str:
- """Get installer log file path."""
+ Examples
+ --------
+ # installer log file path for `mpsd-software install dev-23a foss2021a-mpi`
+ >>> get_log_file_path(
+ ... "dev-23a",
+ ... "install",
+ ... Path(
+ ... "/tmp/root_dir"
+ ... ),
+ ... )
+ PosixPath('/tmp/root_dir/dev-23a/logs/dev-23a_zen3_2023-07-03T12-28-55_APEX_install.log')
+
+ # build log file path for `mpsd-software install dev-23a foss2021a-mpi`
+ >>> get_log_file_path(
+ ... "dev-23a",
+ ... "install",
+ ... Path(
+ ... "/tmp/root_dir"
+ ... ),
+ ... "foss2021a-mpi",
+ ... )
+ PosixPath('/tmp/root_dir/dev-23a/logs/dev-23a_zen3_2023-07-03T12-28-55_BUILD_foss2021a-mpi_install.log')
+
+ # installer log file path for `mpsd-software status dev-23a`
+ >>> get_log_file_path(
+ ... "dev-23a",
+ ... "status",
+ ... Path(
+ ... "/tmp/root_dir"
+ ... ),
+ ... )
+ PosixPath('/tmp/root_dir/dev-23a/logs/dev-23a_zen3_2023-07-03T12-28-55_APEX_status.log')
+
+ # build log file path for `mpsd-software status dev-23a` (no log file is created)
+ >>> get_log_file_path(
+ ... "dev-23a",
+ ... "status",
+ ... Path(
+ ... "/tmp/root_dir"
+ ... ),
+ ... "foss2021a-mpi",
+ ... )
+ (None)
+ """
# Get machine configs
- os.environ.get("MPSD_OS", "UNKNOWN_OS")
microarch = get_native_microarchitecture()
- # parse logging first
- # decide the log_file_name
- installer_log_name = create_log_file_names(
- mpsd_release=mpsd_release, microarch=microarch, action=cmd
+ log_file_name = create_log_file_name(
+ mpsd_release=mpsd_release,
+ microarch=microarch,
+ action=cmd,
+ package_set=package_set,
)
log_folder = root_dir / mpsd_release / "logs"
- # if the log_folder dosent exist, dont log this message if
- # the command is a info-only command
- if cmd not in ["status", "available"]:
- if not os.path.exists(log_folder):
- os.makedirs(log_folder)
- installer_log_file = log_folder / installer_log_name
+ if log_file_name:
+ # if the log_folder dosent exist, create it
+ if not log_folder.exists():
+ log_folder.mkdir(parents=True)
+ return log_folder / log_file_name
else:
- installer_log_file = None
- return installer_log_file
+ return None
def set_up_logging(loglevel="warning", file_path=None):
@@ -556,7 +667,9 @@ def run(*args, counter=[0], **kwargs):
return process
-def record_script_execution_summary(root_dir: str, msg: str = None, **kwargs) -> None:
+def record_script_execution_summary(
+ root_dir: Path, msg: Union[str, None] = None, **kwargs
+) -> None:
"""Log the command used to build the package_set.
It also logs information about the spack-environments branch and commit hash,
@@ -928,14 +1041,9 @@ def install_environment(
# run spack_setup_script with the package_sets as arguments
for package_set in package_sets:
# Set the install log file name from create_log_file_names
- build_log_file_name = create_log_file_names(
- mpsd_release, microarch, "install", package_set=package_set
+ build_log_path = get_log_file_path(
+ mpsd_release, "install", root_dir, package_set
)
- build_log_folder = release_base_dir / "logs"
- build_log_path = build_log_folder / build_log_file_name
- # if logs folder dosent exist, create it
- if not os.path.exists(build_log_folder):
- os.makedirs(build_log_folder)
logging.info(f"Installing package_set {package_set} to {package_set_dir}")
@@ -1003,7 +1111,7 @@ def start_new_environment(release, from_release, target_dir):
raise NotImplementedError(msg)
-def environment_status(mpsd_release: str, root_dir: Union[str, Path]) -> dict:
+def environment_status(mpsd_release: str, root_dir: Path) -> Union[dict, None]:
"""Show status of release in installation.
Parameters
@@ -1020,6 +1128,7 @@ def environment_status(mpsd_release: str, root_dir: Union[str, Path]) -> dict:
toolchain_map : dict
A dictionary containing available microarchitectures as keys and
a list of available package_sets as values for each microarchitecture.
+ If the release is not installed/found, None is returned.
Note: only toolchains can be reported at the moment (i.e. package_sets
such as global and global_generic are missing, even if installed).
@@ -1271,7 +1380,7 @@ def main():
root_dir = get_root_dir()
# set up logging ( with file handler) for all actions (except init)
- log_file = get_installer_log_file_path(
+ log_file = get_log_file_path(
args.release,
args.action,
root_dir,
diff --git a/tests/test_mpsd_software.py b/tests/test_mpsd_software.py
index 88dcb23fd20f687cac15bd3936c744343ed517fb..0b5489e74fdc97830b26565ffb1051ff7c73d1bd 100644
--- a/tests/test_mpsd_software.py
+++ b/tests/test_mpsd_software.py
@@ -280,7 +280,7 @@ def test_install_environment_zlib():
# install global_generic package_set
mod.set_up_logging(
"WARNING",
- mod.get_installer_log_file_path(mpsd_release_to_test, "install", root_dir),
+ mod.get_log_file_path(mpsd_release_to_test, "install", root_dir),
)
mod.install_environment(
mpsd_release=mpsd_release_to_test,
@@ -333,7 +333,7 @@ def test_install_environment_zlib():
importlib.reload(mod)
mod.set_up_logging(
"WARNING",
- mod.get_installer_log_file_path(mpsd_release_to_test, "install", root_dir),
+ mod.get_log_file_path(mpsd_release_to_test, "install", root_dir),
)
mod.install_environment(
mpsd_release=mpsd_release_to_test,
@@ -417,16 +417,16 @@ def test_get_available_package_sets():
)
-def test_create_log_file_names():
+def test_create_log_file_name():
"""Test that the log file names are created correctly."""
- create_log_file_names = mod.create_log_file_names
+ create_log_file_name = mod.create_log_file_name
mpsd_release = "dev-23a"
microarch = "sandybridge"
date = datetime.datetime.now().replace(microsecond=0).isoformat()
action = "install"
package_set = "foss2021a"
# test build_log_file_name generation
- build_log_file_name = create_log_file_names(
+ build_log_file_name = create_log_file_name(
microarch=microarch,
mpsd_release=mpsd_release,
date=date,
@@ -437,7 +437,7 @@ def test_create_log_file_names():
build_log_file_name
== f"{mpsd_release}_{microarch}_{date}_BUILD_{package_set}_{action}.log"
)
- installer_log_file_name = create_log_file_names(
+ installer_log_file_name = create_log_file_name(
microarch=microarch,
mpsd_release=mpsd_release,
date=date,
@@ -448,7 +448,7 @@ def test_create_log_file_names():
== f"{mpsd_release}_{microarch}_{date}_APEX_{action}.log"
)
# test no build log file for incorrect action
- build_log_file_name = create_log_file_names(
+ build_log_file_name = create_log_file_name(
microarch=microarch,
mpsd_release=mpsd_release,
date=date,