mpsd_software.py

#!/usr/bin/env python3

"""mpsd-software: tool for installation of software as on MPSD HPC."""


import argparse
import datetime
import logging
import os
import subprocess
import sys
import tempfile
import time
from pathlib import Path
from typing import List, Tuple, Union
import re
import shutil
from functools import cache
import importlib.metadata

__version__ = importlib.metadata.version(__package__ or __name__)

import rich.logging

command_name = Path(sys.argv[0]).name

about_intro = f"""
Build software as on MPSD HPC.


    This tool builds software package sets (including toolchains for Octopus).
    It follows recipes as used on the MPSD HPC system and the (spack-based)
    Octopus buildbot. Compiled software is organised into MPSD software release
    versions (such as `dev-23a`) and CPU microarchitecture (such as `sandybridge`).

    Compiled packages and toolchains can be activated and used via `module load` as
    on the HPC system.

    Further documentation is available in the README.rst file, online at
    https://gitlab.gwdg.de/mpsd-cs/mpsd-software-manager/-/blob/main/README.rst

Command line usage:

   $> {command_name}

"""


about_epilog = f"""


Examples:

    1. Query what releases are available for installation

       $> {command_name} available

    2. Query what package sets and toolchains are available for installation in
       release dev-23a

       $> {command_name} available dev-23a

    3. Install foss2022a-serial toolchain from the dev-23a release

       $> {command_name} install dev-23a foss2022a-serial

    4. Check what package sets and toolchains are installed from release dev-23a

       $> {command_name} status dev-23a

       The `status` command also displays the `module use` command needed to load
       the created modules.

"""

call_date_iso = (
    datetime.datetime.now().replace(microsecond=0).isoformat().replace(":", "-")
)
config_vars = {
    #  kept inside the mpsd_release folder
    "cmd_log_file": "mpsd-software.log",
    #  Metadata tags
    "metadata_tag_open": "!<meta>",
    "metadata_tag_close": "</meta>!",
    "spack_environments_repo": "https://gitlab.gwdg.de/mpsd-cs/spack-environments.git",
    "init_file": ".mpsd-software-root",
}


def log_metadata(key: str, value: str) -> None:
    """Log metadata to the log file.

    This function logs metadata to the log file. The metadata is
    enclosed in a tag, so that it can be easily found in the log file.
    logging module is used to write the metadata to the log file.

    Parameters
    ----------
    key : str
        key of the metadata
    value : str
        value of the metadata
    returns : None
    """
    logging.info(
        f"{config_vars['metadata_tag_open']}{key}:{value}{config_vars['metadata_tag_close']}"
    )


def read_metadata_from_logfile(logfile: Union[str, Path]) -> dict:
    """Read metadata from the log file.

    This function reads metadata from the log file. The metadata is
    enclosed in a tag, so that it can be easily found in the log file.

    Parameters
    ----------
    logfile : str or Path
        log file name
    returns : dict
        dictionary containing the metadata
    """
    with open(logfile, "r") as f:
        log_text = f.read()
    # check for all data that matches the regex
    # metadata_tag_open {key}:{value} metadata_tag_close
    # and return a dictionary with all the matches
    return {
        match.group(1): match.group(2)
        for match in re.finditer(
            rf"{config_vars['metadata_tag_open']}(\w+):(\w+){config_vars['metadata_tag_close']}",
            log_text,
        )
    }


def create_log_file_name(
    mpsd_release: str,
    action: str,
    date: str = call_date_iso,
    package_set: Union[str, None] = None,
) -> Union[str, None]:
    """Create log file names.

    This function creates the log file names for either the installer or
    the build log files.

    If a package_set is given, then the build log file name is created.
    if no package_set is given, then the installer log file name is created.
    The installer log file hosts the logs of the installer script, while
    the build log file hosts the logs of the build process as generated by the
    spack_setup.sh script.

    Parameters
    ----------
    mpsd_release : str
        MPSD software stack version
    date : str
        date of the call ins iso format
    action : str
        action performed (install,remove,reinstall,prepare,status)
        only install and remove are valid for build log file.
    package_set : str or None
        package_set name (only for build log file)

    Returns
    -------
    str or None
        log file name
        installer_log_file_name or build_log_file_name depending on the
        parameters given.
        If the action is not one that changes the files on disk (info only actions)
        then None is returned.

    Examples
    --------
    # installer log file name for `mpsd-software install dev-23a foss2021a-mpi`
    >>> create_log_file_name(
    ...         "dev-23a",
    ...         "install",
    ...         "2023-07-03T12-27-52",
    ...     )
    'dev-23a_sandybridge_2023-07-03T12-27-52_APEX_install.log'

    # build log file name for `mpsd-software install dev-23a foss2021a-mpi`
    >>> create_log_file_name(
    ...     "dev-23a",
    ...     "install",
    ...     "2023-07-03T12-27-52",
    ...     "foss2021a-mpi",
    ... )
    'dev-23a_sandybridge_2023-07-03T12-27-52_BUILD_foss2021a-mpi_install.log'

    # installer log file name for `mpsd-software status dev-23a`
    >>> create_log_file_name(
    ...     "dev-23a",
    ...     "status",
    ...     "2023-07-03T12-27-52",
    ... )
    'dev-23a_sandybridge_2023-07-03T12-27-52_APEX_status.log'