Skip to content
Snippets Groups Projects

Restructure log location

Merged Restructure log location
restructure-log-location into main
Merged Ashwin Kumar Karnad requested to merge restructure-log-location into main
Compare
and
Show latest version
2 files
+ 80
− 84
Compare changes
  • Side-by-side
  • Inline
Files
2
mpsd-software-environment.py
+ 51
− 54
@@ -12,7 +12,6 @@ import time
from pathlib import Path
from typing import List, Tuple, Union
import re
from string import Template
# If 'rich' is available ("pip install rich" or "apt-get install python3-rich"),
# then use coloured output, otherwise proceed as before
@@ -32,18 +31,13 @@ for given system architecture and MPSD software stack version.\n
The toolchains
are built using the bash script spack_setup.sh, and the results are logged. """
call_date_iso = datetime.datetime.now().replace(microsecond=0).isoformat()
call_date_iso = (
datetime.datetime.now().replace(microsecond=0).isoformat().replace(":", "-")
)
config_vars = {
# kept inside the mpsd_release folder
"cmd_log_file": "install.log",
# Placeholder installer log file name, placed at mpsd_microarch/logs
"installer_log_template": Template(
"${mpsd_release}_${mpsd_microarch}_${date}_${action}.log"
),
# Placeholder build log file name, placed at mpsd_microarch/logs
"build_log_template": Template(
"${mpsd_release}_${mpsd_microarch}_${date}_${toolchain}_${action}.log"
),
"cmd_log_file": "script_execution_summary.log",
# Metadata tags
"metadata_tag_open": "!<meta>",
"metadata_tag_close": "</meta>!",
"spack_environments_repo": "https://gitlab.gwdg.de/mpsd-cs/spack-environments.git",
@@ -56,17 +50,17 @@ def create_log_file_names(
action: str,
date: str = call_date_iso,
toolchain: str = None,
) -> Tuple[str, str]:
) -> Union[str, None]:
"""Create log file names.
This function creates the log file names for the installer and
This function creates the log file names for either the installer or
the build log files.
The installer log file is created
from the output of this script except
the toolchain build log that is handled by the spack_setup.sh script.
The build log `tee`'s the output of the spack_setup.sh script to the log file.
The log file names are created using the
template strings defined in config_vars.
If a toolchain is given, then the build log file name is created.
if no toolchain is given, then the installer log file name is created.
The installer log file hosts the logs of the installer script, while
the build log file hosts the logs of the build process as generated by the
spack_setup.sh script.
Parameters
----------
@@ -81,27 +75,29 @@ def create_log_file_names(
only install and remove are valid for build log file.
toolchain : str
toolchain name (only for build log file)
returns : tuple
tuple containing installer_log_file, build_log_file, file names
Returns
-------
str or None
log file name
installer_log_file_name or build_log_file_name depending on the
parameters given.
If the action is not one that changes the files on disk ( info only actions)
then None is returned.
"""
installer_log_file = config_vars["installer_log_template"].substitute(
mpsd_release=mpsd_release,
mpsd_microarch=mpsd_microarch,
date=date,
action=action,
)
if toolchain and action in ["install", "remove"]:
build_log_file = config_vars["build_log_template"].substitute(
mpsd_release=mpsd_release,
mpsd_microarch=mpsd_microarch,
date=date,
action=action,
toolchain=toolchain,
)
if toolchain:
# if toolchain is given, then we build the build_log_file_name
if action in ["install", "remove"]:
log_file_name = (
f"{mpsd_release}_{mpsd_microarch}_{date}_BUILD_{toolchain}_{action}.log"
)
else:
return None
else:
build_log_file = None
logging.warning("Incorrect action or toolchain name.")
return installer_log_file, build_log_file
# if toolchain is not given, then we build the installer_log_file_name
log_file_name = f"{mpsd_release}_{mpsd_microarch}_{date}_APEX_{action}.log"
return log_file_name
def log_metadata(key: str, value: str) -> None:
@@ -151,14 +147,14 @@ def read_metadata_from_logfile(logfile: Union[str, Path]) -> dict:
}
def get_installer_log_file(mpsd_release: str, cmd: str, script_dir: str) -> str:
"""Get installer log file name."""
def get_installer_log_file_path(mpsd_release: str, cmd: str, script_dir: str) -> str:
"""Get installer log file path."""
# Get machine configs
os.environ.get("MPSD_OS", "UNKNOWN_OS")
mpsd_microarch = get_native_microarchitecture()
# parse logging first
# decide the log_file_name
installer_log_name, build_log_name = create_log_file_names(
installer_log_name = create_log_file_names(
mpsd_release=mpsd_release, mpsd_microarch=mpsd_microarch, action=cmd
)
log_folder = script_dir / mpsd_release / "logs"
@@ -173,7 +169,7 @@ def get_installer_log_file(mpsd_release: str, cmd: str, script_dir: str) -> str:
return installer_log_file
def set_up_logging(loglevel="warning", filename=None):
def set_up_logging(loglevel="warning", file_path=None):
"""Set up logging.
This function sets up the logging configuration for the script.
@@ -188,7 +184,7 @@ def set_up_logging(loglevel="warning", filename=None):
- warning (default): only print statements if something is unexpected
- info (show more detailed progress)
- debug (show very detailed output)
filename : str
file_path : str
- filename to save logging messages into
If loglevel is 'debug', save line numbers in log messages.
@@ -270,7 +266,7 @@ def set_up_logging(loglevel="warning", filename=None):
shell_handler.setFormatter(shell_formatter)
# use the log_level_numeric to decide how much logging is sent to shell
shell_handler.setLevel(log_level_numeric)
# Here we set the handlers of the RootLogger to be just the one we want.
# The reason is that the logging module will add a <StreamHandler <stderr>
# (NOTSET)> handler if logging.info/logging.debug/... is used before we
@@ -278,8 +274,8 @@ def set_up_logging(loglevel="warning", filename=None):
logger.handlers = [shell_handler]
# if filename provided, write log messages to that file, too.
if filename:
file_handler = logging.FileHandler(filename)
if file_path:
file_handler = logging.FileHandler(file_path)
# if we have a file, we write all information in there.
# We could change the level, for example restrict to only DEBUG and above with
# file_handler.setLevel(logging.DEBUG)
@@ -303,9 +299,9 @@ def set_up_logging(loglevel="warning", filename=None):
print_log.addHandler(ch)
# if filename provided, write output of print_log to that file, too
if filename:
if file_path:
# create, format and add file handler
fh = logging.FileHandler(filename)
fh = logging.FileHandler(file_path)
fh.setFormatter(formatter)
print_log.addHandler(fh)
@@ -314,7 +310,7 @@ def set_up_logging(loglevel="warning", filename=None):
#
logging.debug(
f"Logging has been setup, loglevel={loglevel.upper()} "
+ f"{filename=} {rich_available=}"
+ f"{file_path=} {rich_available=}"
)
@@ -784,11 +780,11 @@ def install_environment(
os.mkdir("logs")
for toolchain in toolchains:
# Set the install log file name from create_log_file_names
_, build_log_file_name = create_log_file_names(
build_log_file_name = create_log_file_names(
mpsd_release, mpsd_microarch, "install", toolchain=toolchain
)
build_log_folder = release_base_dir / "logs"
build_log_file = build_log_folder / build_log_file_name
build_log_path = build_log_folder / build_log_file_name
# if logs folder dosent exist, create it
if not os.path.exists(build_log_folder):
os.makedirs(build_log_folder)
@@ -799,7 +795,7 @@ def install_environment(
setup_log_cmd(
mpsd_release,
script_dir,
msg=f"installing {toolchain} and logging at {build_log_file}",
msg=f"installing {toolchain} and logging at {build_log_path}",
)
setup_log_cmd(
mpsd_release,
@@ -811,7 +807,7 @@ def install_environment(
)
run(
f"bash {spack_setup_script} {' '.join(install_flags)} {toolchain} 2>&1 "
f"| tee -a {build_log_file} ",
f"| tee -a {build_log_path} ",
shell=True,
check=True,
)
@@ -916,7 +912,8 @@ def main():
script_dir = Path(os.path.dirname(os.path.realpath(__file__)))
set_up_logging(
args.loglevel, get_installer_log_file(args.release, args.action, script_dir)
args.loglevel,
get_installer_log_file_path(args.release, args.action, script_dir),
)
# Check the command and run related function