Restructure log location

Resolves #7 (closed)

mpsd-software-environment.py

@@ -12,7 +12,6 @@ import time
 from pathlib import Path
 from typing import List, Tuple, Union
 import re
-from string import Template
 
 # If 'rich' is available ("pip install rich" or "apt-get install python3-rich"),
 # then use coloured output, otherwise proceed as before
@@ -32,18 +31,13 @@ for given system architecture and MPSD software stack version.\n
 The toolchains
 are built using the bash script spack_setup.sh, and the results are logged. """
 
-call_date_iso = datetime.datetime.now().replace(microsecond=0).isoformat()
+call_date_iso = (
+    datetime.datetime.now().replace(microsecond=0).isoformat().replace(":", "-")
+)
 config_vars = {
     #  kept inside the mpsd_release folder
-    "cmd_log_file": "install.log",
-    #  Placeholder installer log file name, placed at mpsd_microarch/logs
-    "installer_log_template": Template(
-        "${mpsd_release}_${mpsd_microarch}_${date}_${action}.log"
-    ),
-    #  Placeholder build log file name, placed at mpsd_microarch/logs
-    "build_log_template": Template(
-        "${mpsd_release}_${mpsd_microarch}_${date}_${toolchain}_${action}.log"
-    ),
+    "cmd_log_file": "script_execution_summary.log",
+    #  Metadata tags
     "metadata_tag_open": "!<meta>",
     "metadata_tag_close": "</meta>!",
     "spack_environments_repo": "https://gitlab.gwdg.de/mpsd-cs/spack-environments.git",
@@ -56,17 +50,17 @@ def create_log_file_names(
     action: str,
     date: str = call_date_iso,
     toolchain: str = None,
-) -> Tuple[str, str]:
+) -> Union[str, None]:
     """Create log file names.
 
-    This function creates the log file names for the installer and
+    This function creates the log file names for either the installer or
     the build log files.
-    The installer log file is created
-    from the output of this script except
-    the toolchain build log that is handled by the spack_setup.sh script.
-    The build log `tee`'s the output of the spack_setup.sh script to the log file.
-    The log file names are created using the
-    template strings defined in config_vars.
+
+    If a toolchain is given, then the build log file name is created.
+    if no toolchain is given, then the installer log file name is created.
+    The installer log file hosts the logs of the installer script, while
+    the build log file hosts the logs of the build process as generated by the
+    spack_setup.sh script.
 
     Parameters
     ----------
@@ -81,27 +75,29 @@ def create_log_file_names(
         only install and remove are valid for build log file.
     toolchain : str
         toolchain name (only for build log file)
-    returns : tuple
-        tuple containing installer_log_file, build_log_file, file names
+
+    Returns
+    -------
+    str or None
+        log file name
+        installer_log_file_name or build_log_file_name depending on the
+        parameters given.
+        If the action is not one that changes the files on disk ( info only actions)
+        then None is returned.
     """
-    installer_log_file = config_vars["installer_log_template"].substitute(
-        mpsd_release=mpsd_release,
-        mpsd_microarch=mpsd_microarch,
-        date=date,
-        action=action,
-    )
-    if toolchain and action in ["install", "remove"]:
-        build_log_file = config_vars["build_log_template"].substitute(
-            mpsd_release=mpsd_release,
-            mpsd_microarch=mpsd_microarch,
-            date=date,
-            action=action,
-            toolchain=toolchain,
-        )
+    if toolchain:
+        # if toolchain is given, then  we build the build_log_file_name
+        if action in ["install", "remove"]:
+            log_file_name = (
+                f"{mpsd_release}_{mpsd_microarch}_{date}_BUILD_{toolchain}_{action}.log"
+            )
+        else:
+            return None
     else:
-        build_log_file = None
-        logging.warning("Incorrect action or toolchain name.")
-    return installer_log_file, build_log_file
+        # if toolchain is not given, then we build the installer_log_file_name
+        log_file_name = f"{mpsd_release}_{mpsd_microarch}_{date}_APEX_{action}.log"
+
+    return log_file_name
 
 
 def log_metadata(key: str, value: str) -> None:
@@ -151,7 +147,29 @@ def read_metadata_from_logfile(logfile: Union[str, Path]) -> dict:
     }
 
 
-def set_up_logging(loglevel="warning", filename=None):
+def get_installer_log_file_path(mpsd_release: str, cmd: str, script_dir: str) -> str:
+    """Get installer log file path."""
+    # Get machine configs
+    os.environ.get("MPSD_OS", "UNKNOWN_OS")
+    mpsd_microarch = get_native_microarchitecture()
+    # parse logging first
+    # decide the log_file_name
+    installer_log_name = create_log_file_names(
+        mpsd_release=mpsd_release, mpsd_microarch=mpsd_microarch, action=cmd
+    )
+    log_folder = script_dir / mpsd_release / "logs"
+    # if the log_folder dosent exist, dont log this message if
+    #  the command is a info-only command
+    if cmd not in ["status", "available"]:
+        if not os.path.exists(log_folder):
+            os.makedirs(log_folder)
+        installer_log_file = log_folder / installer_log_name
+    else:
+        installer_log_file = None
+    return installer_log_file
+
+
+def set_up_logging(loglevel="warning", file_path=None):
     """Set up logging.
 
     This function sets up the logging configuration for the script.
@@ -166,7 +184,7 @@ def set_up_logging(loglevel="warning", filename=None):
          - warning (default): only print statements if something is unexpected
          - info (show more detailed progress)
          - debug (show very detailed output)
-    filename : str
+    file_path : str
          - filename to save logging messages into
 
     If loglevel is 'debug', save line numbers in log messages.
@@ -248,11 +266,16 @@ def set_up_logging(loglevel="warning", filename=None):
     shell_handler.setFormatter(shell_formatter)
     # use the log_level_numeric to decide how much logging is sent to shell
     shell_handler.setLevel(log_level_numeric)
-    logger.addHandler(shell_handler)
+
+    # Here we set the handlers of the RootLogger to be just the one we want.
+    # The reason is that the logging module will add a <StreamHandler <stderr>
+    # (NOTSET)> handler if logging.info/logging.debug/... is used before we
+    # come across this line. And we do not want that additional handler.
+    logger.handlers = [shell_handler]
 
     # if filename provided, write log messages to that file, too.
-    if filename:
-        file_handler = logging.FileHandler(filename)
+    if file_path:
+        file_handler = logging.FileHandler(file_path)
         # if we have a file, we write all information in there.
         # We could change the level, for example restrict to only DEBUG and above with
         # file_handler.setLevel(logging.DEBUG)
@@ -276,9 +299,9 @@ def set_up_logging(loglevel="warning", filename=None):
     print_log.addHandler(ch)
 
     # if filename provided, write output of print_log to that file, too
-    if filename:
+    if file_path:
         # create, format and add file handler
-        fh = logging.FileHandler(filename)
+        fh = logging.FileHandler(file_path)
         fh.setFormatter(formatter)
         print_log.addHandler(fh)
 
@@ -286,8 +309,8 @@ def set_up_logging(loglevel="warning", filename=None):
     # short message
     #
     logging.debug(
-        f"Logging has been setup, loglevel={loglevel.upper()}"
-        + f"{filename=} {rich_available=}"
+        f"Logging has been setup, loglevel={loglevel.upper()} "
+        + f"{file_path=} {rich_available=}"
     )
 
 
@@ -757,11 +780,11 @@ def install_environment(
             os.mkdir("logs")
         for toolchain in toolchains:
             # Set the install log file name from create_log_file_names
-            _, build_log_file_name = create_log_file_names(
+            build_log_file_name = create_log_file_names(
                 mpsd_release, mpsd_microarch, "install", toolchain=toolchain
             )
             build_log_folder = release_base_dir / "logs"
-            build_log_file = build_log_folder / build_log_file_name
+            build_log_path = build_log_folder / build_log_file_name
             # if logs folder dosent exist, create it
             if not os.path.exists(build_log_folder):
                 os.makedirs(build_log_folder)
@@ -772,19 +795,19 @@ def install_environment(
             setup_log_cmd(
                 mpsd_release,
                 script_dir,
-                msg=f"installing {toolchain} and logging at {build_log_file}",
+                msg=f"installing {toolchain} and logging at {build_log_path}",
             )
             setup_log_cmd(
                 mpsd_release,
                 script_dir,
                 msg=(
-                    f"CMD: bash {spack_setup_script} {' '.join(install_flags)}"
-                    "{toolchain}"
+                    f"CMD: bash {spack_setup_script} {' '.join(install_flags)} "
+                    f"{toolchain}"
                 ),
             )
             run(
                 f"bash {spack_setup_script} {' '.join(install_flags)} {toolchain} 2>&1 "
-                f"| tee -a {build_log_file} ",
+                f"| tee -a {build_log_path} ",
                 shell=True,
                 check=True,
             )
@@ -885,28 +908,13 @@ def main():
     # Carry out the action
     args = parser.parse_args()
 
-    # Get machine configs
-    os.environ.get("MPSD_OS", "UNKNOWN_OS")
-    mpsd_microarch = get_native_microarchitecture()
     # target dir is the place where this script exists. the
     script_dir = Path(os.path.dirname(os.path.realpath(__file__)))
-    mpsd_release = args.release
-    # parse logging first
-    # decide the log_file_name
-    installer_log_name, build_log_name = create_log_file_names(
-        mpsd_release=mpsd_release, mpsd_microarch=mpsd_microarch, action=args.action
-    )
-    log_folder = script_dir / mpsd_release / mpsd_microarch / "logs"
-    # if the log_folder dosent exist, dont log this message if
-    #  the command is a info-only command
-    if args.action not in ["status", "available"]:
-        if not os.path.exists(log_folder):
-            os.makedirs(log_folder)
-        installer_log_file = log_folder / installer_log_name
-    else:
-        installer_log_file = None
 
-    set_up_logging(args.loglevel, installer_log_file)
+    set_up_logging(
+        args.loglevel,
+        get_installer_log_file_path(args.release, args.action, script_dir),
+    )
 
     # Check the command and run related function
     if args.action == "remove":