mpsd-software-environment.py

#!/usr/bin/env python3

"""mpsd-software-environment: tool for installation of toolchains."""

import argparse
import datetime
import logging
import os
import subprocess
import sys
import tempfile
import time
from pathlib import Path
from typing import List, Tuple, Union
import re

# If 'rich' is available ("pip install rich" or "apt-get install python3-rich"),
# then use coloured output, otherwise proceed as before
try:
    import rich.logging
except ModuleNotFoundError:
    rich_available = False
else:
    rich_available = True


about_tool = """
Build toolchains using Spack.\n

This function builds toolchains for MPSD-HPC at the appropriate directory, \n
for given system architecture and MPSD software stack version.\n
The toolchains
are built using the bash script spack_setup.sh, and the results are logged. """

call_date_iso = (
    datetime.datetime.now().replace(microsecond=0).isoformat().replace(":", "-")
)
config_vars = {
    #  kept inside the mpsd_release folder
    "cmd_log_file": "script_execution_summary.log",
    #  Metadata tags
    "metadata_tag_open": "!<meta>",
    "metadata_tag_close": "</meta>!",
    "spack_environments_repo": "https://gitlab.gwdg.de/mpsd-cs/spack-environments.git",
}


def create_log_file_names(
    mpsd_release: str,
    microarch: str,
    action: str,
    date: str = call_date_iso,
    toolchain: str = None,
) -> Union[str, None]:
    """Create log file names.

    This function creates the log file names for either the installer or
    the build log files.

    If a toolchain is given, then the build log file name is created.
    if no toolchain is given, then the installer log file name is created.
    The installer log file hosts the logs of the installer script, while
    the build log file hosts the logs of the build process as generated by the
    spack_setup.sh script.

    Parameters
    ----------
    mpsd_release : str
        MPSD software stack version
    microarch : str
        system architecture
    date : str
        date of the call ins iso format
    action : str
        action performed (install,remove,reinstall,prepare,status)
        only install and remove are valid for build log file.
    toolchain : str
        toolchain name (only for build log file)

    Returns
    -------
    str or None
        log file name
        installer_log_file_name or build_log_file_name depending on the
        parameters given.
        If the action is not one that changes the files on disk ( info only actions)
        then None is returned.
    """
    if toolchain:
        # if toolchain is given, then  we build the build_log_file_name
        if action in ["install", "remove"]:
            log_file_name = (
                f"{mpsd_release}_{microarch}_{date}_BUILD_{toolchain}_{action}.log"
            )
        else:
            return None
    else:
        # if toolchain is not given, then we build the installer_log_file_name
        log_file_name = f"{mpsd_release}_{microarch}_{date}_APEX_{action}.log"

    return log_file_name


def log_metadata(key: str, value: str) -> None:
    """Log metadata to the log file.

    This function logs metadata to the log file. The metadata is
    enclosed in a tag, so that it can be easily found in the log file.
    logging module is used to write the metadata to the log file.

    Parameters
    ----------
    key : str
        key of the metadata
    value : str
        value of the metadata
    returns : None
    """
    logging.info(
        f"{config_vars['metadata_tag_open']}{key}:{value}{config_vars['metadata_tag_close']}"
    )


def read_metadata_from_logfile(logfile: Union[str, Path]) -> dict:
    """Read metadata from the log file.

    This function reads metadata from the log file. The metadata is
    enclosed in a tag, so that it can be easily found in the log file.

    Parameters
    ----------
    logfile : str or Path
        log file name
    returns : dict
        dictionary containing the metadata
    """
    with open(logfile, "r") as f:
        log_text = f.read()
    # check for all data that matches the regex
    # metadata_tag_open {key}:{value} metadata_tag_close
    # and return a dictionary with all the matches
    return {
        match.group(1): match.group(2)
        for match in re.finditer(
            f"{config_vars['metadata_tag_open']}(\w+):(\w+){config_vars['metadata_tag_close']}",
            log_text,
        )
    }


def get_installer_log_file_path(mpsd_release: str, cmd: str, root_dir: str) -> str:
    """Get installer log file path."""
    # Get machine configs
    os.environ.get("MPSD_OS", "UNKNOWN_OS")
    microarch = get_native_microarchitecture()
    # parse logging first
    # decide the log_file_name
    installer_log_name = create_log_file_names(
        mpsd_release=mpsd_release, microarch=microarch, action=cmd
    )
    log_folder = root_dir / mpsd_release / "logs"
    # if the log_folder dosent exist, dont log this message if
    #  the command is a info-only command
    if cmd not in ["status", "available"]:
        if not os.path.exists(log_folder):
            os.makedirs(log_folder)
        installer_log_file = log_folder / installer_log_name
    else:
        installer_log_file = None
    return installer_log_file


def set_up_logging(loglevel="warning", file_path=None):
    """Set up logging.

    This function sets up the logging configuration for the script.
    It configures the log level, log format, and log handlers
    for both file and console(=shell) output.


    Parameters
    ----------
    loglevel : str or int
       Loglevels are:
         - warning (default): only print statements if something is unexpected
         - info (show more detailed progress)
         - debug (show very detailed output)
    file_path : str
         - filename to save logging messages into

    If loglevel is 'debug', save line numbers in log messages.

    Returns
    -------
    None.

    Logger instances are generally not passed around, but retrieved from the
    logging module as shown below (they are singletons).

    We provide two loggers: