mpsd_software.py

#!/usr/bin/env python3

"""mpsd-software: tool for installation of software as on MPSD HPC."""

__version__ = "2023.6.27"

import argparse
import datetime
import logging
import os
import subprocess
import sys
import tempfile
import time
from pathlib import Path
from typing import List, Tuple, Union
import re
import shutil

# If 'rich' is available ("pip install rich" or "apt-get install python3-rich"),
# then use coloured output, otherwise proceed as before
try:
    import rich.logging
except ModuleNotFoundError:
    rich_available = False
else:
    rich_available = True


about_intro = f"""
Build software as on MPSD HPC.


    This tool builds software package sets (including toolchains for Octopus).
    It follows recipes as used on the MPSD HPC system and the (spack-based)
    Octopus buildbot. Compiled software is organised into MPSD software release
    versions (such as `dev-23a`) and CPU microarchitecture (such as `sandybridge`).

    Compiled packages and toolchains can be activated and used via `module load` as
    on the HPC system.

    Further documentation is available in the README.rst file, online at
    https://gitlab.gwdg.de/mpsd-cs/mpsd-software-manager/-/blob/main/README.rst

Command line usage:

   $> {sys.argv[0]}

"""


about_epilog = f"""


Examples:

    1. Query what package sets and toolchains are available for installation in
       release dev-23a

       $> {sys.argv[0]} available dev-23a

    2. Install foss2022a-serial toolchain from the dev-23a release

       $> {sys.argv[0]} install dev-23a foss2022a-serial

    3. Check what package sets and toolchains are installed from release dev-23a

       $> {sys.argv[0]} status dev-23a

       The `status` command also displays the `module use` command needed to load
       the created modules.

"""

call_date_iso = (
    datetime.datetime.now().replace(microsecond=0).isoformat().replace(":", "-")
)
config_vars = {
    #  kept inside the mpsd_release folder
    "cmd_log_file": "mpsd-software.log",
    #  Metadata tags
    "metadata_tag_open": "!<meta>",
    "metadata_tag_close": "</meta>!",
    "spack_environments_repo": "https://gitlab.gwdg.de/mpsd-cs/spack-environments.git",
    "init_file": ".mpsd-software-root",
}


def create_log_file_names(
    mpsd_release: str,
    microarch: str,
    action: str,
    date: str = call_date_iso,
    package_set: str = None,
) -> Union[str, None]:
    """Create log file names.

    This function creates the log file names for either the installer or
    the build log files.

    If a package_set is given, then the build log file name is created.
    if no package_set is given, then the installer log file name is created.
    The installer log file hosts the logs of the installer script, while
    the build log file hosts the logs of the build process as generated by the
    spack_setup.sh script.

    Parameters
    ----------
    mpsd_release : str
        MPSD software stack version
    microarch : str
        system architecture
    date : str
        date of the call ins iso format
    action : str
        action performed (install,remove,reinstall,prepare,status)
        only install and remove are valid for build log file.
    package_set : str
        package_set name (only for build log file)

    Returns
    -------
    str or None
        log file name
        installer_log_file_name or build_log_file_name depending on the
        parameters given.
        If the action is not one that changes the files on disk ( info only actions)
        then None is returned.
    """
    if package_set:
        # if package_set is given, then  we build the build_log_file_name
        if action in ["install", "remove"]:
            log_file_name = (
                f"{mpsd_release}_{microarch}_{date}_BUILD_{package_set}_{action}.log"
            )
        else:
            return None
    else:
        # if package_set is not given, then we build the installer_log_file_name
        log_file_name = f"{mpsd_release}_{microarch}_{date}_APEX_{action}.log"

    return log_file_name


def log_metadata(key: str, value: str) -> None:
    """Log metadata to the log file.

    This function logs metadata to the log file. The metadata is
    enclosed in a tag, so that it can be easily found in the log file.
    logging module is used to write the metadata to the log file.

    Parameters
    ----------
    key : str
        key of the metadata
    value : str
        value of the metadata
    returns : None
    """
    logging.info(
        f"{config_vars['metadata_tag_open']}{key}:{value}{config_vars['metadata_tag_close']}"
    )


def read_metadata_from_logfile(logfile: Union[str, Path]) -> dict:
    """Read metadata from the log file.

    This function reads metadata from the log file. The metadata is
    enclosed in a tag, so that it can be easily found in the log file.

    Parameters
    ----------
    logfile : str or Path
        log file name
    returns : dict
        dictionary containing the metadata
    """
    with open(logfile, "r") as f:
        log_text = f.read()
    # check for all data that matches the regex
    # metadata_tag_open {key}:{value} metadata_tag_close
    # and return a dictionary with all the matches
    return {
        match.group(1): match.group(2)
        for match in re.finditer(
            f"{config_vars['metadata_tag_open']}(\w+):(\w+){config_vars['metadata_tag_close']}",
            log_text,
        )
    }


def get_installer_log_file_path(mpsd_release: str, cmd: str, root_dir: str) -> str:
    """Get installer log file path."""
    # Get machine configs
    os.environ.get("MPSD_OS", "UNKNOWN_OS")
    microarch = get_native_microarchitecture()
    # parse logging first
    # decide the log_file_name
    installer_log_name = create_log_file_names(
        mpsd_release=mpsd_release, microarch=microarch, action=cmd