Skip to content
Snippets Groups Projects
Commit 5ef97087 authored by Ashwin Kumar Karnad's avatar Ashwin Kumar Karnad
Browse files

Merge branch 'simplify-log-file-path-generation' into 'main' 

Simplify log file path generation

Closes #24

See merge request !96
parents 1ee726e4 9e54342d
No related branches found
No related tags found
1 merge request!96Simplify log file path generation
Pipeline #375493 passed
Stage: test
Hide whitespace changes
Inline Side-by-side
src/mpsd_software_manager/mpsd_software.py
+ 177
− 68
View file @ 5ef97087
...@@ -90,12 +90,59 @@ config_vars = { ...@@ -90,12 +90,59 @@ config_vars = {
} }
def create_log_file_names( def log_metadata(key: str, value: str) -> None:
"""Log metadata to the log file.
This function logs metadata to the log file. The metadata is
enclosed in a tag, so that it can be easily found in the log file.
logging module is used to write the metadata to the log file.
Parameters
----------
key : str
key of the metadata
value : str
value of the metadata
returns : None
"""
logging.info(
f"{config_vars['metadata_tag_open']}{key}:{value}{config_vars['metadata_tag_close']}"
)
def read_metadata_from_logfile(logfile: Union[str, Path]) -> dict:
"""Read metadata from the log file.
This function reads metadata from the log file. The metadata is
enclosed in a tag, so that it can be easily found in the log file.
Parameters
----------
logfile : str or Path
log file name
returns : dict
dictionary containing the metadata
"""
with open(logfile, "r") as f:
log_text = f.read()
# check for all data that matches the regex
# metadata_tag_open {key}:{value} metadata_tag_close
# and return a dictionary with all the matches
return {
match.group(1): match.group(2)
for match in re.finditer(
rf"{config_vars['metadata_tag_open']}(\w+):(\w+){config_vars['metadata_tag_close']}",
log_text,
)
}
def create_log_file_name(
mpsd_release: str, mpsd_release: str,
microarch: str, microarch: str,
action: str, action: str,
date: str = call_date_iso, date: str = call_date_iso,
package_set: str = None, package_set: Union[str, None] = None,
) -> Union[str, None]: ) -> Union[str, None]:
"""Create log file names. """Create log file names.
...@@ -119,7 +166,7 @@ def create_log_file_names( ...@@ -119,7 +166,7 @@ def create_log_file_names(
action : str action : str
action performed (install,remove,reinstall,prepare,status) action performed (install,remove,reinstall,prepare,status)
only install and remove are valid for build log file. only install and remove are valid for build log file.
package_set : str package_set : str or None
package_set name (only for build log file) package_set name (only for build log file)
Returns Returns
...@@ -128,8 +175,48 @@ def create_log_file_names( ...@@ -128,8 +175,48 @@ def create_log_file_names(
log file name log file name
installer_log_file_name or build_log_file_name depending on the installer_log_file_name or build_log_file_name depending on the
parameters given. parameters given.
If the action is not one that changes the files on disk ( info only actions) If the action is not one that changes the files on disk (info only actions)
then None is returned. then None is returned.
Examples
--------
# installer log file name for `mpsd-software install dev-23a foss2021a-mpi`
>>> create_log_file_name(
... "dev-23a",
... "sandybridge",
... "install",
... "2023-07-03T12-27-52",
... )
'dev-23a_sandybridge_2023-07-03T12-27-52_APEX_install.log'
# build log file name for `mpsd-software install dev-23a foss2021a-mpi`
>>> create_log_file_name(
... "dev-23a",
... "sandybridge",
... "install",
... "2023-07-03T12-27-52",
... "foss2021a-mpi",
... )
'dev-23a_sandybridge_2023-07-03T12-27-52_BUILD_foss2021a-mpi_install.log'
# installer log file name for `mpsd-software status dev-23a`
>>> create_log_file_name(
... "dev-23a",
... "sandybridge",
... "status",
... "2023-07-03T12-27-52",
... )
'dev-23a_sandybridge_2023-07-03T12-27-52_APEX_status.log'
# build log file name for `mpsd-software status dev-23a` (no log file is created)
>>> create_log_file_name(
... "dev-23a",
... "sandybridge",
... "status",
... "2023-07-03T12-27-52",
... "foss2021a-mpi",
... )
(None)
""" """
if package_set: if package_set:
# if package_set is given, then we build the build_log_file_name # if package_set is given, then we build the build_log_file_name
...@@ -146,73 +233,97 @@ def create_log_file_names( ...@@ -146,73 +233,97 @@ def create_log_file_names(
return log_file_name return log_file_name
def log_metadata(key: str, value: str) -> None: def get_log_file_path(
"""Log metadata to the log file. mpsd_release: str, cmd: str, root_dir: Path, package_set: Union[str, None] = None
) -> Union[Path, None]:
This function logs metadata to the log file. The metadata is """Get log file path.
enclosed in a tag, so that it can be easily found in the log file.
logging module is used to write the metadata to the log file.
Parameters
----------
key : str
key of the metadata
value : str
value of the metadata
returns : None
"""
logging.info(
f"{config_vars['metadata_tag_open']}{key}:{value}{config_vars['metadata_tag_close']}"
)
This function creates the log file paths for either the installer or
the build log files.
def read_metadata_from_logfile(logfile: Union[str, Path]) -> dict: If a package_set is given, then the build log file path is returned.
"""Read metadata from the log file. if no package_set is given, then the installer log file path is returned.
This function reads metadata from the log file. The metadata is If the logs folder does not exist, then it is created.
enclosed in a tag, so that it can be easily found in the log file.
Parameters Parameters
---------- ----------
logfile : str or Path mpsd_release : str
log file name MPSD software stack version
returns : dict cmd : str
dictionary containing the metadata command to be executed
""" root_dir : str
with open(logfile, "r") as f: root directory of the mpsd software stack
log_text = f.read() package_set : str
# check for all data that matches the regex package_set name (only for build log file)
# metadata_tag_open {key}:{value} metadata_tag_close
# and return a dictionary with all the matches
return {
match.group(1): match.group(2)
for match in re.finditer(
rf"{config_vars['metadata_tag_open']}(\w+):(\w+){config_vars['metadata_tag_close']}",
log_text,
)
}
Returns
-------
Path or None
log file path
installer_log_file_path or build_log_file_path depending on the
parameters given.
def get_installer_log_file_path(mpsd_release: str, cmd: str, root_dir: str) -> str: Examples
"""Get installer log file path.""" --------
# installer log file path for `mpsd-software install dev-23a foss2021a-mpi`
>>> get_log_file_path(
... "dev-23a",
... "install",
... Path(
... "/tmp/root_dir"
... ),
... )
PosixPath('/tmp/root_dir/dev-23a/logs/dev-23a_zen3_2023-07-03T12-28-55_APEX_install.log')
# build log file path for `mpsd-software install dev-23a foss2021a-mpi`
>>> get_log_file_path(
... "dev-23a",
... "install",
... Path(
... "/tmp/root_dir"
... ),
... "foss2021a-mpi",
... )
PosixPath('/tmp/root_dir/dev-23a/logs/dev-23a_zen3_2023-07-03T12-28-55_BUILD_foss2021a-mpi_install.log')
# installer log file path for `mpsd-software status dev-23a`
>>> get_log_file_path(
... "dev-23a",
... "status",
... Path(
... "/tmp/root_dir"
... ),
... )
PosixPath('/tmp/root_dir/dev-23a/logs/dev-23a_zen3_2023-07-03T12-28-55_APEX_status.log')
# build log file path for `mpsd-software status dev-23a` (no log file is created)
>>> get_log_file_path(
... "dev-23a",
... "status",
... Path(
... "/tmp/root_dir"
... ),
... "foss2021a-mpi",
... )
(None)
"""
# Get machine configs # Get machine configs
os.environ.get("MPSD_OS", "UNKNOWN_OS")
microarch = get_native_microarchitecture() microarch = get_native_microarchitecture()
# parse logging first log_file_name = create_log_file_name(
# decide the log_file_name mpsd_release=mpsd_release,
installer_log_name = create_log_file_names( microarch=microarch,
mpsd_release=mpsd_release, microarch=microarch, action=cmd action=cmd,
package_set=package_set,
) )
log_folder = root_dir / mpsd_release / "logs" log_folder = root_dir / mpsd_release / "logs"
# if the log_folder dosent exist, dont log this message if if log_file_name:
# the command is a info-only command # if the log_folder dosent exist, create it
if cmd not in ["status", "available"]: if not log_folder.exists():
if not os.path.exists(log_folder): log_folder.mkdir(parents=True)
os.makedirs(log_folder) return log_folder / log_file_name
installer_log_file = log_folder / installer_log_name
else: else:
installer_log_file = None return None
return installer_log_file
def set_up_logging(loglevel="warning", file_path=None): def set_up_logging(loglevel="warning", file_path=None):
...@@ -556,7 +667,9 @@ def run(*args, counter=[0], **kwargs): ...@@ -556,7 +667,9 @@ def run(*args, counter=[0], **kwargs):
return process return process
def record_script_execution_summary(root_dir: str, msg: str = None, **kwargs) -> None: def record_script_execution_summary(
root_dir: Path, msg: Union[str, None] = None, **kwargs
) -> None:
"""Log the command used to build the package_set. """Log the command used to build the package_set.
It also logs information about the spack-environments branch and commit hash, It also logs information about the spack-environments branch and commit hash,
...@@ -928,14 +1041,9 @@ def install_environment( ...@@ -928,14 +1041,9 @@ def install_environment(
# run spack_setup_script with the package_sets as arguments # run spack_setup_script with the package_sets as arguments
for package_set in package_sets: for package_set in package_sets:
# Set the install log file name from create_log_file_names # Set the install log file name from create_log_file_names
build_log_file_name = create_log_file_names( build_log_path = get_log_file_path(
mpsd_release, microarch, "install", package_set=package_set mpsd_release, "install", root_dir, package_set
) )
build_log_folder = release_base_dir / "logs"
build_log_path = build_log_folder / build_log_file_name
# if logs folder dosent exist, create it
if not os.path.exists(build_log_folder):
os.makedirs(build_log_folder)
logging.info(f"Installing package_set {package_set} to {package_set_dir}") logging.info(f"Installing package_set {package_set} to {package_set_dir}")
...@@ -1003,7 +1111,7 @@ def start_new_environment(release, from_release, target_dir): ...@@ -1003,7 +1111,7 @@ def start_new_environment(release, from_release, target_dir):
raise NotImplementedError(msg) raise NotImplementedError(msg)
def environment_status(mpsd_release: str, root_dir: Union[str, Path]) -> dict: def environment_status(mpsd_release: str, root_dir: Path) -> Union[dict, None]:
"""Show status of release in installation. """Show status of release in installation.
Parameters Parameters
...@@ -1020,6 +1128,7 @@ def environment_status(mpsd_release: str, root_dir: Union[str, Path]) -> dict: ...@@ -1020,6 +1128,7 @@ def environment_status(mpsd_release: str, root_dir: Union[str, Path]) -> dict:
toolchain_map : dict toolchain_map : dict
A dictionary containing available microarchitectures as keys and A dictionary containing available microarchitectures as keys and
a list of available package_sets as values for each microarchitecture. a list of available package_sets as values for each microarchitecture.
If the release is not installed/found, None is returned.
Note: only toolchains can be reported at the moment (i.e. package_sets Note: only toolchains can be reported at the moment (i.e. package_sets
such as global and global_generic are missing, even if installed). such as global and global_generic are missing, even if installed).
...@@ -1271,7 +1380,7 @@ def main(): ...@@ -1271,7 +1380,7 @@ def main():
root_dir = get_root_dir() root_dir = get_root_dir()
# set up logging ( with file handler) for all actions (except init) # set up logging ( with file handler) for all actions (except init)
log_file = get_installer_log_file_path( log_file = get_log_file_path(
args.release, args.release,
args.action, args.action,
root_dir, root_dir,
......
tests/test_mpsd_software.py
+ 7
− 7
View file @ 5ef97087
...@@ -280,7 +280,7 @@ def test_install_environment_zlib(): ...@@ -280,7 +280,7 @@ def test_install_environment_zlib():
# install global_generic package_set # install global_generic package_set
mod.set_up_logging( mod.set_up_logging(
"WARNING", "WARNING",
mod.get_installer_log_file_path(mpsd_release_to_test, "install", root_dir), mod.get_log_file_path(mpsd_release_to_test, "install", root_dir),
) )
mod.install_environment( mod.install_environment(
mpsd_release=mpsd_release_to_test, mpsd_release=mpsd_release_to_test,
...@@ -333,7 +333,7 @@ def test_install_environment_zlib(): ...@@ -333,7 +333,7 @@ def test_install_environment_zlib():
importlib.reload(mod) importlib.reload(mod)
mod.set_up_logging( mod.set_up_logging(
"WARNING", "WARNING",
mod.get_installer_log_file_path(mpsd_release_to_test, "install", root_dir), mod.get_log_file_path(mpsd_release_to_test, "install", root_dir),
) )
mod.install_environment( mod.install_environment(
mpsd_release=mpsd_release_to_test, mpsd_release=mpsd_release_to_test,
...@@ -417,16 +417,16 @@ def test_get_available_package_sets(): ...@@ -417,16 +417,16 @@ def test_get_available_package_sets():
) )
def test_create_log_file_names(): def test_create_log_file_name():
"""Test that the log file names are created correctly.""" """Test that the log file names are created correctly."""
create_log_file_names = mod.create_log_file_names create_log_file_name = mod.create_log_file_name
mpsd_release = "dev-23a" mpsd_release = "dev-23a"
microarch = "sandybridge" microarch = "sandybridge"
date = datetime.datetime.now().replace(microsecond=0).isoformat() date = datetime.datetime.now().replace(microsecond=0).isoformat()
action = "install" action = "install"
package_set = "foss2021a" package_set = "foss2021a"
# test build_log_file_name generation # test build_log_file_name generation
build_log_file_name = create_log_file_names( build_log_file_name = create_log_file_name(
microarch=microarch, microarch=microarch,
mpsd_release=mpsd_release, mpsd_release=mpsd_release,
date=date, date=date,
...@@ -437,7 +437,7 @@ def test_create_log_file_names(): ...@@ -437,7 +437,7 @@ def test_create_log_file_names():
build_log_file_name build_log_file_name
== f"{mpsd_release}_{microarch}_{date}_BUILD_{package_set}_{action}.log" == f"{mpsd_release}_{microarch}_{date}_BUILD_{package_set}_{action}.log"
) )
installer_log_file_name = create_log_file_names( installer_log_file_name = create_log_file_name(
microarch=microarch, microarch=microarch,
mpsd_release=mpsd_release, mpsd_release=mpsd_release,
date=date, date=date,
...@@ -448,7 +448,7 @@ def test_create_log_file_names(): ...@@ -448,7 +448,7 @@ def test_create_log_file_names():
== f"{mpsd_release}_{microarch}_{date}_APEX_{action}.log" == f"{mpsd_release}_{microarch}_{date}_APEX_{action}.log"
) )
# test no build log file for incorrect action # test no build log file for incorrect action
build_log_file_name = create_log_file_names( build_log_file_name = create_log_file_name(
microarch=microarch, microarch=microarch,
mpsd_release=mpsd_release, mpsd_release=mpsd_release,
date=date, date=date,
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment