Skip to content
Snippets Groups Projects
Commit 5ef97087 authored by Ashwin Kumar Karnad's avatar Ashwin Kumar Karnad
Browse files

Merge branch 'simplify-log-file-path-generation' into 'main' 

Simplify log file path generation

Closes #24

See merge request !96
parents 1ee726e4 9e54342d
No related branches found
No related tags found
1 merge request!96Simplify log file path generation
Pipeline #375493 passed
Stage: test
Hide whitespace changes
Inline Side-by-side
src/mpsd_software_manager/mpsd_software.py
+ 177
− 68
View file @ 5ef97087
......@@ -90,12 +90,59 @@ config_vars = {
}
def create_log_file_names(
def log_metadata(key: str, value: str) -> None:
"""Log metadata to the log file.
This function logs metadata to the log file. The metadata is
enclosed in a tag, so that it can be easily found in the log file.
logging module is used to write the metadata to the log file.
Parameters
----------
key : str
key of the metadata
value : str
value of the metadata
returns : None
"""
logging.info(
f"{config_vars['metadata_tag_open']}{key}:{value}{config_vars['metadata_tag_close']}"
)
def read_metadata_from_logfile(logfile: Union[str, Path]) -> dict:
"""Read metadata from the log file.
This function reads metadata from the log file. The metadata is
enclosed in a tag, so that it can be easily found in the log file.
Parameters
----------
logfile : str or Path
log file name
returns : dict
dictionary containing the metadata
"""
with open(logfile, "r") as f:
log_text = f.read()
# check for all data that matches the regex
# metadata_tag_open {key}:{value} metadata_tag_close
# and return a dictionary with all the matches
return {
match.group(1): match.group(2)
for match in re.finditer(
rf"{config_vars['metadata_tag_open']}(\w+):(\w+){config_vars['metadata_tag_close']}",
log_text,
)
}
def create_log_file_name(
mpsd_release: str,
microarch: str,
action: str,
date: str = call_date_iso,
package_set: str = None,
package_set: Union[str, None] = None,
) -> Union[str, None]:
"""Create log file names.
......@@ -119,7 +166,7 @@ def create_log_file_names(
action : str
action performed (install,remove,reinstall,prepare,status)
only install and remove are valid for build log file.
package_set : str
package_set : str or None
package_set name (only for build log file)
Returns
......@@ -128,8 +175,48 @@ def create_log_file_names(
log file name
installer_log_file_name or build_log_file_name depending on the
parameters given.
If the action is not one that changes the files on disk ( info only actions)
If the action is not one that changes the files on disk (info only actions)
then None is returned.
Examples
--------
# installer log file name for `mpsd-software install dev-23a foss2021a-mpi`
>>> create_log_file_name(
... "dev-23a",
... "sandybridge",
... "install",
... "2023-07-03T12-27-52",
... )
'dev-23a_sandybridge_2023-07-03T12-27-52_APEX_install.log'
# build log file name for `mpsd-software install dev-23a foss2021a-mpi`
>>> create_log_file_name(
... "dev-23a",
... "sandybridge",
... "install",
... "2023-07-03T12-27-52",
... "foss2021a-mpi",
... )
'dev-23a_sandybridge_2023-07-03T12-27-52_BUILD_foss2021a-mpi_install.log'
# installer log file name for `mpsd-software status dev-23a`
>>> create_log_file_name(
... "dev-23a",
... "sandybridge",
... "status",
... "2023-07-03T12-27-52",
... )
'dev-23a_sandybridge_2023-07-03T12-27-52_APEX_status.log'
# build log file name for `mpsd-software status dev-23a` (no log file is created)
>>> create_log_file_name(
... "dev-23a",
... "sandybridge",
... "status",
... "2023-07-03T12-27-52",
... "foss2021a-mpi",
... )
(None)
"""
if package_set:
# if package_set is given, then we build the build_log_file_name
......@@ -146,73 +233,97 @@ def create_log_file_names(
return log_file_name
def log_metadata(key: str, value: str) -> None:
"""Log metadata to the log file.
This function logs metadata to the log file. The metadata is
enclosed in a tag, so that it can be easily found in the log file.
logging module is used to write the metadata to the log file.
Parameters
----------
key : str
key of the metadata
value : str
value of the metadata
returns : None
"""
logging.info(
f"{config_vars['metadata_tag_open']}{key}:{value}{config_vars['metadata_tag_close']}"
)
def get_log_file_path(
mpsd_release: str, cmd: str, root_dir: Path, package_set: Union[str, None] = None
) -> Union[Path, None]:
"""Get log file path.
This function creates the log file paths for either the installer or
the build log files.
def read_metadata_from_logfile(logfile: Union[str, Path]) -> dict:
"""Read metadata from the log file.
If a package_set is given, then the build log file path is returned.
if no package_set is given, then the installer log file path is returned.
This function reads metadata from the log file. The metadata is
enclosed in a tag, so that it can be easily found in the log file.
If the logs folder does not exist, then it is created.
Parameters
----------
logfile : str or Path
log file name
returns : dict
dictionary containing the metadata
"""
with open(logfile, "r") as f:
log_text = f.read()
# check for all data that matches the regex
# metadata_tag_open {key}:{value} metadata_tag_close
# and return a dictionary with all the matches
return {
match.group(1): match.group(2)
for match in re.finditer(
rf"{config_vars['metadata_tag_open']}(\w+):(\w+){config_vars['metadata_tag_close']}",
log_text,
)
}
mpsd_release : str
MPSD software stack version
cmd : str
command to be executed
root_dir : str
root directory of the mpsd software stack
package_set : str
package_set name (only for build log file)
Returns
-------
Path or None
log file path
installer_log_file_path or build_log_file_path depending on the
parameters given.
def get_installer_log_file_path(mpsd_release: str, cmd: str, root_dir: str) -> str:
"""Get installer log file path."""
Examples
--------
# installer log file path for `mpsd-software install dev-23a foss2021a-mpi`
>>> get_log_file_path(
... "dev-23a",
... "install",
... Path(
... "/tmp/root_dir"
... ),
... )
PosixPath('/tmp/root_dir/dev-23a/logs/dev-23a_zen3_2023-07-03T12-28-55_APEX_install.log')
# build log file path for `mpsd-software install dev-23a foss2021a-mpi`
>>> get_log_file_path(
... "dev-23a",
... "install",
... Path(
... "/tmp/root_dir"
... ),
... "foss2021a-mpi",
... )
PosixPath('/tmp/root_dir/dev-23a/logs/dev-23a_zen3_2023-07-03T12-28-55_BUILD_foss2021a-mpi_install.log')
# installer log file path for `mpsd-software status dev-23a`
>>> get_log_file_path(
... "dev-23a",
... "status",
... Path(
... "/tmp/root_dir"
... ),
... )
PosixPath('/tmp/root_dir/dev-23a/logs/dev-23a_zen3_2023-07-03T12-28-55_APEX_status.log')
# build log file path for `mpsd-software status dev-23a` (no log file is created)
>>> get_log_file_path(
... "dev-23a",
... "status",
... Path(
... "/tmp/root_dir"
... ),
... "foss2021a-mpi",
... )
(None)
"""
# Get machine configs
os.environ.get("MPSD_OS", "UNKNOWN_OS")
microarch = get_native_microarchitecture()
# parse logging first
# decide the log_file_name
installer_log_name = create_log_file_names(
mpsd_release=mpsd_release, microarch=microarch, action=cmd
log_file_name = create_log_file_name(
mpsd_release=mpsd_release,
microarch=microarch,
action=cmd,
package_set=package_set,
)
log_folder = root_dir / mpsd_release / "logs"
# if the log_folder dosent exist, dont log this message if
# the command is a info-only command
if cmd not in ["status", "available"]:
if not os.path.exists(log_folder):
os.makedirs(log_folder)
installer_log_file = log_folder / installer_log_name
if log_file_name:
# if the log_folder dosent exist, create it
if not log_folder.exists():
log_folder.mkdir(parents=True)
return log_folder / log_file_name
else:
installer_log_file = None
return installer_log_file
return None
def set_up_logging(loglevel="warning", file_path=None):
......@@ -556,7 +667,9 @@ def run(*args, counter=[0], **kwargs):
return process
def record_script_execution_summary(root_dir: str, msg: str = None, **kwargs) -> None:
def record_script_execution_summary(
root_dir: Path, msg: Union[str, None] = None, **kwargs
) -> None:
"""Log the command used to build the package_set.
It also logs information about the spack-environments branch and commit hash,
......@@ -928,14 +1041,9 @@ def install_environment(
# run spack_setup_script with the package_sets as arguments
for package_set in package_sets:
# Set the install log file name from create_log_file_names
build_log_file_name = create_log_file_names(
mpsd_release, microarch, "install", package_set=package_set
build_log_path = get_log_file_path(
mpsd_release, "install", root_dir, package_set
)
build_log_folder = release_base_dir / "logs"
build_log_path = build_log_folder / build_log_file_name
# if logs folder dosent exist, create it
if not os.path.exists(build_log_folder):
os.makedirs(build_log_folder)
logging.info(f"Installing package_set {package_set} to {package_set_dir}")
......@@ -1003,7 +1111,7 @@ def start_new_environment(release, from_release, target_dir):
raise NotImplementedError(msg)
def environment_status(mpsd_release: str, root_dir: Union[str, Path]) -> dict:
def environment_status(mpsd_release: str, root_dir: Path) -> Union[dict, None]:
"""Show status of release in installation.
Parameters
......@@ -1020,6 +1128,7 @@ def environment_status(mpsd_release: str, root_dir: Union[str, Path]) -> dict:
toolchain_map : dict
A dictionary containing available microarchitectures as keys and
a list of available package_sets as values for each microarchitecture.
If the release is not installed/found, None is returned.
Note: only toolchains can be reported at the moment (i.e. package_sets
such as global and global_generic are missing, even if installed).
......@@ -1271,7 +1380,7 @@ def main():
root_dir = get_root_dir()
# set up logging ( with file handler) for all actions (except init)
log_file = get_installer_log_file_path(
log_file = get_log_file_path(
args.release,
args.action,
root_dir,
......
tests/test_mpsd_software.py
+ 7
− 7
View file @ 5ef97087
......@@ -280,7 +280,7 @@ def test_install_environment_zlib():
# install global_generic package_set
mod.set_up_logging(
"WARNING",
mod.get_installer_log_file_path(mpsd_release_to_test, "install", root_dir),
mod.get_log_file_path(mpsd_release_to_test, "install", root_dir),
)
mod.install_environment(
mpsd_release=mpsd_release_to_test,
......@@ -333,7 +333,7 @@ def test_install_environment_zlib():
importlib.reload(mod)
mod.set_up_logging(
"WARNING",
mod.get_installer_log_file_path(mpsd_release_to_test, "install", root_dir),
mod.get_log_file_path(mpsd_release_to_test, "install", root_dir),
)
mod.install_environment(
mpsd_release=mpsd_release_to_test,
......@@ -417,16 +417,16 @@ def test_get_available_package_sets():
)
def test_create_log_file_names():
def test_create_log_file_name():
"""Test that the log file names are created correctly."""
create_log_file_names = mod.create_log_file_names
create_log_file_name = mod.create_log_file_name
mpsd_release = "dev-23a"
microarch = "sandybridge"
date = datetime.datetime.now().replace(microsecond=0).isoformat()
action = "install"
package_set = "foss2021a"
# test build_log_file_name generation
build_log_file_name = create_log_file_names(
build_log_file_name = create_log_file_name(
microarch=microarch,
mpsd_release=mpsd_release,
date=date,
......@@ -437,7 +437,7 @@ def test_create_log_file_names():
build_log_file_name
== f"{mpsd_release}_{microarch}_{date}_BUILD_{package_set}_{action}.log"
)
installer_log_file_name = create_log_file_names(
installer_log_file_name = create_log_file_name(
microarch=microarch,
mpsd_release=mpsd_release,
date=date,
......@@ -448,7 +448,7 @@ def test_create_log_file_names():
== f"{mpsd_release}_{microarch}_{date}_APEX_{action}.log"
)
# test no build log file for incorrect action
build_log_file_name = create_log_file_names(
build_log_file_name = create_log_file_name(
microarch=microarch,
mpsd_release=mpsd_release,
date=date,
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment