Victor László Horváth · Martin Lang
--- a/src/mpsd_software_manager/__init__.py 0 → 100644
+++ b/src/mpsd_software_manager/__init__.py 0 → 100644
+"""Command-line tool to install software using Spack."""
+
+import sys
+from logging import getLogger
+from typing import Any
+
+from ._version import __version__ as __version__
--- a/src/mpsd_software_manager/__init__.py 0 → 100644
+++ b/src/mpsd_software_manager/__init__.py 0 → 100644
+"""Command-line tool to install software using Spack."""
+
+import sys
+from logging import getLogger
+from typing import Any
+
+from ._version import __version__ as __version__
+
+logger = getLogger(__name__)
+
+
+def silent_keyboard_interrupt(
-def silent_keyboard_interrupt(
+def _silent_keyboard_interrupt(
--- a/src/mpsd_software_manager/config.py 0 → 100644
+++ b/src/mpsd_software_manager/config.py 0 → 100644
+   ...
+   Config().<property>
+
+The config object should be initialised in the command line module.
+"""
+
+from __future__ import annotations
+
+from dataclasses import InitVar, dataclass, field
+from logging import getLogger
+from pathlib import Path
+from typing import Any
+
+from .util import abort, detect_cpu_microarch
+
+MPSD_SOFTWARE_ROOT_MARKER: str = ".mpsd-software-root"
-MPSD_SOFTWARE_ROOT_MARKER: str = ".mpsd-software-root"
+MPSD_SOFTWARE_ROOT_MARKER: Final[str] = ".mpsd-software-root"
--- a/src/mpsd_software_manager/config.py 0 → 100644
+++ b/src/mpsd_software_manager/config.py 0 → 100644
+
+from __future__ import annotations
+
+from dataclasses import InitVar, dataclass, field
+from logging import getLogger
+from pathlib import Path
+from typing import Any
+
+from .util import abort, detect_cpu_microarch
+
+MPSD_SOFTWARE_ROOT_MARKER: str = ".mpsd-software-root"
+
+logger = getLogger(__name__)
+
+
+class Singleton(type):
-class Singleton(type):
+class SingletonMeta(type):
--- a/src/mpsd_software_manager/init.py 0 → 100644
+++ b/src/mpsd_software_manager/init.py 0 → 100644
+import datetime
+from logging import getLogger
+from pathlib import Path
+
+import mpsd_software_manager as smgr
+
+from .config import MPSD_SOFTWARE_ROOT_MARKER
+
+logger = getLogger(__name__)
+
+
+def init(_args: argparse.Namespace) -> None:
+    """Initialise directory as mpsd software root.
+
+    - create .mpsd-software-root file as a marker
+    - initialise logging
--- a/src/mpsd_software_manager/log_handler.py 0 → 100644
+++ b/src/mpsd_software_manager/log_handler.py 0 → 100644
+
+from rich.logging import RichHandler
+
+from .config import Config
+
+logger = logging.getLogger(__name__)
+
+
+def init_stdout_logger(verbose: bool = False) -> None:
+    """Show log messages on stdout."""
+    level = logging.DEBUG if verbose else logging.INFO
+    logging.basicConfig(
+        format="%(message)s",
+        handlers=[RichHandler(level=level, show_time=False, show_path=False)],
+    )
+    # set only interal logging to a low level, ignore third party
-    # set only interal logging to a low level, ignore third party
+    # set only internal logging to a low level, ignore third party
--- a/src/mpsd_software_manager/log_handler.py 0 → 100644
+++ b/src/mpsd_software_manager/log_handler.py 0 → 100644
+    logging.getLogger("mpsd_software_manager").setLevel(level)
+
+
+def init_file_logger() -> None:
+    """Write log messages to a file."""
+    config = Config()
+    file_name = (
+        f"{config.microarch}-{datetime.datetime.now().isoformat(timespec='seconds')}"
+    )
+    config.log_root.mkdir(parents=True, exist_ok=True)
+    log_file = config.log_root / file_name
+    if log_file.exists():
+        logger.error("Log file '%s' exists already; skipping file logging", log_file)
+        return
+    else:
+        logger.info("Detailed log written to '%s'", log_file)
--- a/src/mpsd_software_manager/config.py 0 → 100644
+++ b/src/mpsd_software_manager/config.py 0 → 100644
+
+    release_root: Path = field(init=False)
+    microarch_root: Path = field(init=False)
+    spack_root: Path = field(init=False)
+    lmod_root: Path = field(init=False)
+    spack_environments_root: Path = field(init=False)
+    log_root: Path = field(init=False)
+
+    source_cache_root: Path = Path("/opt_mpsd/mpsd_spack_sources")
+    binary_cache_root: Path = field(init=False)
+    spack_user_cache_root: Path = field(init=False)
+
+    system_compiler: str = field(init=False)
+
+    spack_environments_repository = ("gitlab.gwdg.de", "mpsd-cs/spack-environments")
+    spack_repository = ("github.com", "mpsd-computational-science/spack")
-    spack_environments_repository = ("gitlab.gwdg.de", "mpsd-cs/spack-environments")
-    spack_repository = ("github.com", "mpsd-computational-science/spack")
+    spack_environments_repository = Repo(host="gitlab.gwdg.de", repo_name="mpsd-cs/spack-environments")
+    spack_repository = Repo(host="github.com", repo_name="mpsd-computational-science/spack")
--- a/src/mpsd_software_manager/status.py 0 → 100644
+++ b/src/mpsd_software_manager/status.py 0 → 100644
+
+logger = getLogger(__name__)
+
+
+def status(args: argparse.Namespace) -> None:
+    """Status of installed releases and package sets."""
+    require_valid_software_root(command="status")
+
+    if args.software_release is None:
+        mpsd_software_status()
+    elif args.package_set is None:
+        release_status(args.software_release)
+    else:
+        package_set_status(
+            args.software_release, args.package_set, args.package_details
+        )
--- a/src/mpsd_software_manager/install.py 0 → 100644
+++ b/src/mpsd_software_manager/install.py 0 → 100644
+
+    require_valid_software_root(command="install")
+    spack_environments_branch = validate_software_release_and_package_sets(
+        software_release, package_sets
+    )
+
+    config = Config()
+    config.resolve_paths(software_release)
+    config.source_cache_root = args.source_cache_path
+
+    init_file_logger()
+
+    spack.ensure_no_active_spack()
+
+    # logger.critical("git clones disabled")
+    initialise_dir(config.release_root, "release")
-    spack.ensure_no_active_spack()
-
-    # logger.critical("git clones disabled")
-    initialise_dir(config.release_root, "release")
+    spack.ensure_no_active_spack()
+
+    initialise_dir(config.release_root, "release")
--- a/src/mpsd_software_manager/install.py 0 → 100644
+++ b/src/mpsd_software_manager/install.py 0 → 100644
+from .config import Config, require_valid_software_root
+from .git_helper import initialise_or_update_repository
+from .log_handler import init_file_logger
+from .util import abort, initialise_dir
+
+logger = getLogger(__name__)
+
+
+def install(args: argparse.Namespace) -> None:
+    """Install package set(s)."""
+    software_release: str = args.software_release
+    package_sets: list[str] = args.package_set
+
+    # print(args)
+
+    require_valid_software_root(command="install")
-    package_sets: list[str] = args.package_set
-
-    # print(args)
-
-    require_valid_software_root(command="install")
+    package_sets: list[str] = args.package_set
+
+    require_valid_software_root(command="install")
--- a/src/mpsd_software_manager/spack.py 0 → 100644
+++ b/src/mpsd_software_manager/spack.py 0 → 100644
+    if replacements:
+        for placeholder, value in replacements:
+            content = content.replace(placeholder, value)
+
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    with dest.open("w") as f:
+        f.write(content)
+
+
+def add_mirror(type_: str, name: str, path: Path) -> None:
+    """Add mirror with given name if it is not yet configured and the path exists."""
+    if not path.is_dir():
+        logger.warning("Ignoring mirror '%s', '%s' is not a directory", name, path)
+        return
+    existing_mirrors = spack("mirror list").stdout
+    if name in existing_mirrors:
--- a/src/mpsd_software_manager/spack.py 0 → 100644
+++ b/src/mpsd_software_manager/spack.py 0 → 100644
+            is not None
+        ):
+            # compiler path: cc = /path/to/spack/root -> no system compiler
+            continue
+        system_compilers.append(compiler)
+
+    if len(system_compilers) == 0:
+        abort("Could not find a system compiler.")
+    elif (
+        requested_system_compiler is not None
+        and requested_system_compiler not in system_compilers
+    ):
+        abort(
+            f"Requested system compiler '{requested_system_compiler}' is not known"
+            " to Spack"
+        )
--- a/src/mpsd_software_manager/spack.py 0 → 100644
+++ b/src/mpsd_software_manager/spack.py 0 → 100644
+        )
+    elif requested_system_compiler is None and len(system_compilers) > 1:
+        abort(
+            f"Found multiple system compilers: '{system_compilers}'; specify preferred"
+            " system compiler on command line"
+        )
+
+    if requested_system_compiler is not None:
+        logger.debug(
+            "Using '%s' as system compiler per user request", requested_system_compiler
+        )
+        config.system_compiler = requested_system_compiler
+    else:
+        system_compiler = system_compilers[0]
+        logger.debug("Found system compiler '%s'", system_compiler)
+        config.system_compiler = system_compiler
-def find_system_compiler(requested_system_compiler: str | None) -> None:
-    """Auto-detect system compiler or use user-specified compiler."""
-    logger.info("spack: detecting system compiler")
-    spack("compiler find --scope site", log_callback=logger.debug)
-    spack_compilers = spack("compiler list", logger.debug).stdout
-    config = Config()
-    system_compilers = []
-    for compiler in re.findall(r"\n(\w+@[0-9.]+)", spack_compilers):
-        compiler_info = spack(f"compiler info {compiler}").stdout
-        if (
-            re.search(rf"\n\s*cc\s*=\s*{config.spack_root!s}", compiler_info)
-            is not None
-        ):
-            # compiler path: cc = /path/to/spack/root -> no system compiler
-            continue
-        system_compilers.append(compiler)
-
-    if len(system_compilers) == 0:
-        abort("Could not find a system compiler.")
-    elif (
-        requested_system_compiler is not None
-        and requested_system_compiler not in system_compilers
-    ):
-        abort(
-            f"Requested system compiler '{requested_system_compiler}' is not known"
-            " to Spack"
-        )
-    elif requested_system_compiler is None and len(system_compilers) > 1:
-        abort(
-            f"Found multiple system compilers: '{system_compilers}'; specify preferred"
-            " system compiler on command line"
-        )
-
-    if requested_system_compiler is not None:
-        logger.debug(
-            "Using '%s' as system compiler per user request", requested_system_compiler
-        )
-        config.system_compiler = requested_system_compiler
-    else:
-        system_compiler = system_compilers[0]
-        logger.debug("Found system compiler '%s'", system_compiler)
-        config.system_compiler = system_compiler
+def find_system_compilers() -> list[str]:
+    spack("compiler find --scope site", log_callback=logger.debug)
+    spack_compilers = spack("compiler list", logger.debug).stdout
+    config = Config()
+    system_compilers = []
+    for compiler in re.findall(r"\n(\w+@[0-9.]+)", spack_compilers):
+        compiler_info = spack(f"compiler info {compiler}").stdout
+        if (
+            re.search(rf"\n\s*cc\s*=\s*{config.spack_root!s}", compiler_info)
+            is not None
+        ):
+            # compiler path: cc = /path/to/spack/root -> no system compiler
+            continue
+        system_compilers.append(compiler)
+    return system_compilers
+
+def set_system_compiler(requested_system_compiler: str | None) -> None:
+    """Auto-detect system compiler or use user-specified compiler."""
+    logger.info("spack: detecting system compiler")
+    system_compilers = find_system_compilers()
+
+    if len(system_compilers) == 0:
+        abort("Could not find a system compiler.")
+    elif (
+        requested_system_compiler is not None
+        and requested_system_compiler not in system_compilers
+    ):
+        abort(
+            f"Requested system compiler '{requested_system_compiler}' is not known"
+            " to Spack"
+        )
+    elif requested_system_compiler is None and len(system_compilers) > 1:
+        abort(
+            f"Found multiple system compilers: '{system_compilers}'; specify preferred"
+            " system compiler on command line"
+        )
+
+    if requested_system_compiler is not None:
+        logger.debug(
+            "Using '%s' as system compiler per user request", requested_system_compiler
+        )
+        config.system_compiler = requested_system_compiler
+    else:
+        system_compiler = system_compilers[0]
+        logger.debug("Found system compiler '%s'", system_compiler)
+        config.system_compiler = system_compiler
--- a/src/mpsd_software_manager/spack.py 0 → 100644
+++ b/src/mpsd_software_manager/spack.py 0 → 100644
+        )
+
+    if requested_system_compiler is not None:
+        logger.debug(
+            "Using '%s' as system compiler per user request", requested_system_compiler
+        )
+        config.system_compiler = requested_system_compiler
+    else:
+        system_compiler = system_compilers[0]
+        logger.debug("Found system compiler '%s'", system_compiler)
+        config.system_compiler = system_compiler
+
+
+def update_custom_spack_config() -> None:
+    """Apply custom spack configuration from spack-environments repository."""
+    # copy
--- a/src/mpsd_software_manager/spack.py 0 → 100644
+++ b/src/mpsd_software_manager/spack.py 0 → 100644
+
+
+def install_package_set_from_file(package_set: str) -> None:
+    """Install list of packages globally."""
+    package_list = (
+        Config().spack_environments_root
+        / "toolchains"
+        / package_set
+        / "global_packages.list"
+    )
+    logger.info("Installing packages from %s", package_list)
+    with package_list.open() as f:
+        package_str = f.read()
+
+    packages = []
+    for package in package_str.replace("\\\n", "").split("\n"):
--- a/src/mpsd_software_manager/spack.py 0 → 100644
+++ b/src/mpsd_software_manager/spack.py 0 → 100644
+    command: str, log_callbacks: dict[str, Callable[[str], None]], **kwargs: Any
+) -> None:
+    """Run spack command and write stdout to log callback asynchronuously.
+
+    Implementation for asyncio code based on:
+    https://kevinmccarthy.org/2016/07/25/streaming-subprocess-stdin-and-stdout-with-asyncio-in-python/
+    """
+    loop = asyncio.get_event_loop()
+    returncode = loop.run_until_complete(
+        run_async_spack_shell_command(
+            f"spack {command}",
+            log_callbacks=log_callbacks,
+            env=spack_shell_env(),
+            **kwargs,
+        )
+    )
--- a/src/mpsd_software_manager/spack.py 0 → 100644
+++ b/src/mpsd_software_manager/spack.py 0 → 100644
+            Config().lmod_root / mpi_module / compiler_module / "octopus-dependencies"
+        )
+
+    write_lua_module(lmod_octopus_dependencies_root / "full.lua", octopus_dependencies)
+
+
+def write_lua_module(file_path: Path, modules: list[str]) -> None:
+    """Write lua metamodule with a list of dependencies; create directory if needed."""
+    logger.debug("writing lmod module file '%s'", file_path)
+    file_path.parent.mkdir(exist_ok=True)
+    with file_path.open("w") as f:
+        f.write("-- -*- lua -*-\n")
+        f.write("\n".join(f'depends_on("{module}")' for module in modules))
+
+
+def install_toolchain_compiler(spack_environment: str) -> dict[str, dict[str, str]]:
--- a/src/mpsd_software_manager/spack.py 0 → 100644
+++ b/src/mpsd_software_manager/spack.py 0 → 100644
+        }
+
+    logger.debug("Required toolchain compiler: %s", compilers)
+
+    for compiler_definition in compilers.values():
+        spack_install_package(compiler_definition["package"])
+        compiler_path = spack(
+            f"location -i {compiler_definition['package']}"
+        ).stdout.strip()
+        logger.debug(
+            spack(f"compiler find --scope site {compiler_path}").stdout.strip()
+        )
+
+    if len(compilers) == 2:
+        gcc = compilers["fallback"]
+        intel = compilers["default"]
--- a/src/mpsd_software_manager/spack.py 0 → 100644
+++ b/src/mpsd_software_manager/spack.py 0 → 100644
+    with (spack_environment_path(spack_environment) / "loads").open() as f:
+        loads_file = f.read()
+
+    MPI_MODULES = ["openmpi", "intel-oneapi-mpi"]
+    GENERIC_TOOLCHAIN_MODULES = [
+        "binutils",
+        "fftw",
+        "openblas",
+        "autoconf",
+        "libtool",
+        "automake",
+        "netlib-scalapack",
+        "cmake",
+        "ninja",
+        "intel-oneapi-mkl",
+    ]