From 71b272c99eacfc74c096fd0e9d9c26a2c42c257a Mon Sep 17 00:00:00 2001
From: Martin Lang <martin.lang@mpsd.mpg.de>
Date: Fri, 22 Nov 2024 14:07:51 +0100
Subject: [PATCH 1/3] Switch to jinja templating

---
 pyproject.toml                     |  3 +-
 src/mpsd_software_manager/spack.py | 98 +++++++++++++++++-------------
 2 files changed, 57 insertions(+), 44 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index bd9fbf2..d5c1303 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -7,11 +7,12 @@ name = "mpsd_software_manager"
 authors = [{name = "MPSD, SSU-Computational Science"}]
 license = {file = "LICENSE"}
 classifiers = ["License :: OSI Approved :: MIT License"]
-version = "1.0.dev1"
+version = "1.0.dev2"
 readme = "README.rst"
 requires-python = ">=3.9"
 dependencies = [
     "archspec",
+    "Jinja2",
     "rich",
     "python-gitlab",
     "PyYAML",
diff --git a/src/mpsd_software_manager/spack.py b/src/mpsd_software_manager/spack.py
index d5c69c2..0f4b5cb 100644
--- a/src/mpsd_software_manager/spack.py
+++ b/src/mpsd_software_manager/spack.py
@@ -15,6 +15,7 @@ from logging import getLogger
 from pathlib import Path
 from typing import Any, Callable
 
+import jinja2
 import yaml
 
 from .config import Config
@@ -89,21 +90,36 @@ def find_system_compiler(requested_system_compiler: str | None) -> None:
 
 def update_custom_spack_config() -> None:
     """Apply custom spack configuration from spack-environments repository."""
-    # copy
-    custom_config = Config().spack_environments_root / "spack_overlay"
-    assert custom_config.is_absolute()
+    spack_overlay_root = Config().spack_environments_root / "spack_overlay"
+    assert spack_overlay_root.is_absolute()
     files = [
-        elem.relative_to(custom_config)
-        for elem in custom_config.rglob("*")
+        elem.relative_to(spack_overlay_root)
+        for elem in spack_overlay_root.rglob("*")
         if elem.is_file()
     ]
-    for f in files:
-        copy_file(source=custom_config / f, dest=Config().spack_root / f)
+    # source_cache_root may not exist; as fallbacks we try to use the binary cache
+    # or a suitable location inside spack
+    if Config().source_cache_root.is_dir():
+        source_cache = Config().source_cache_root
+    elif Config().binary_cache_root.is_dir():
+        source_cache = Config().binary_cache_root
+    else:
+        source_cache = Config().spack_root / "var" / "spack" / "cache"
+
+    for template_file in files:
+        render_template(
+            template_dir=spack_overlay_root / template_file.parent,
+            template_name=template_file.name,
+            dest_dir=Config().spack_root / template_file.parent,
+            source_cache=source_cache,
+            spack_root=Config().spack_root,
+            lmod_root=Config().lmod_root,
+            system_compiler=Config().system_compiler,
+        )
 
 
 def configure_spack_mirrors(disable_binary_cache: bool) -> None:
     """Configure source and binary cache."""
-    # TODO check if source mirror is writable
     add_mirror("source", "mpsd_spack_sources", Config().source_cache_root)
     if not disable_binary_cache:
         Config().binary_cache_root.mkdir(exist_ok=True)
@@ -145,37 +161,35 @@ def update_caches(disable_binary_cache: bool) -> None:
         )
 
 
-def copy_file(
-    source: Path, dest: Path, replacements: list[tuple[str, str]] | None = None
+def render_template(
+    template_dir: Path, template_name: str, dest_dir: Path, **context: Any
 ) -> None:
-    """Copy file 'source' to 'dest' and replace template variables.
+    """Render given template and write output to dest_dir.
 
-    Existing files are overwritten without notice.
-    """
-    logger.debug("Copying '%s' to '%s' with template replacement", source, dest)
-    with source.open() as f:
-        content = f.read()
+    The template must have name "name.ext.jinja"; it is written to "dest_dir/name.ext".
 
-    # TODO replace with better templating
-    content = content.replace("##SYSTEM_COMPILER##", Config().system_compiler)
-    content = content.replace("$spack/../lmod", str(Config().lmod_root))
-    if Config().source_cache_root.is_dir():
-        source_cache = Config().source_cache_root
-    elif Config().binary_cache_root.is_dir():
-        source_cache = Config().binary_cache_root
-    else:
-        source_cache = Config().spack_root / "var" / "spack" / "cache"
-    content = content.replace("$spack/../mpsd-spack-cache", str(source_cache))
-    # replace all other occurrences of $spack variable
-    content = content.replace("$spack", str(Config().spack_root))
+    All keyword arguments are passed to the template engine as 'context'. Jinja ignores
+    unused elements in the context so passing arguments that are not used in a template
+    is fine. Likewise, missing variables do not result in an error but are just empty
+    strings in the output.
 
-    if replacements:
-        for placeholder, value in replacements:
-            content = content.replace(placeholder, value)
+    The guaranteed context is documented in:
+    https://gitlab.gwdg.de/mpsd-cs/spack-environments/-/tree/develop/docs/templating.org
 
-    dest.parent.mkdir(parents=True, exist_ok=True)
-    with dest.open("w") as f:
-        f.write(content)
+    Existing files are overwritten without notice.
+    """
+    dest_file = template_name.removesuffix(".jinja")
+    logger.debug(
+        "Rendering template '%s' to '%s'\nContext: %s",
+        template_dir / template_name,
+        dest_dir / dest_file,
+        context,
+    )
+    env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir))
+    template = env.get_template(template_name)
+    dest_dir.mkdir(parents=True, exist_ok=True)
+    with open(dest_dir / dest_file, "w") as f:
+        f.write(template.render(context))
 
 
 def add_mirror(type_: str, name: str, path: Path) -> None:
@@ -322,16 +336,14 @@ def install_package_set_from_environment(spack_environment: str) -> None:
         logger.debug(spack(f"env create {spack_environment}"))
 
     config = Config()
-    replacements = [("##TOOLCHAIN_COMPILER##", compilers["default"]["name"])]
+    context = {"toolchain_compiler": compilers["default"]["name"]}
     if "fallback" in compilers:
-        replacements.append(("##TOOLCHAIN_GCC##", compilers["fallback"]["name"]))
-    copy_file(
-        config.spack_environments_root
-        / "toolchains"
-        / spack_environment
-        / "spack.yaml",
-        spack_environment_path(spack_environment) / "spack.yaml",
-        replacements,
+        context["fallback_compiler"] = compilers["fallback"]["name"]
+    render_template(
+        config.spack_environments_root / "toolchains" / spack_environment,
+        "spack.yaml.jinja",
+        spack_environment_path(spack_environment),
+        **context,
     )
 
     logger.info("concretizing environment '%s'", spack_environment)
-- 
GitLab


From 74611eddfb4640b5c7cf435a1d28c3a3d02f17ad Mon Sep 17 00:00:00 2001
From: Martin Lang <martin.lang@mpsd.mpg.de>
Date: Fri, 22 Nov 2024 14:31:37 +0100
Subject: [PATCH 2/3] Add CHANGELOG

---
 CHANGELOG.rst | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 CHANGELOG.rst

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
new file mode 100644
index 0000000..85d8719
--- /dev/null
+++ b/CHANGELOG.rst
@@ -0,0 +1,17 @@
+1.0.dev2
+========
+
+- Switch to Jinja templating
+
+1.0.dev1
+========
+
+Re-write of the metamodule generation:
+- Toolchain metamodules are deprecated and will only be generated for the existing toolchains
+- The octopus-dependencies module does now come in two variants min and full
+
+
+1.0.dev0
+========
+
+- Re-write of the old ``spack_setup.sh`` and ``software-manager``.
-- 
GitLab


From 4973fee985c0921507cf36f9da1c688dafbd63c4 Mon Sep 17 00:00:00 2001
From: Martin Lang <martin.lang@mpsd.mpg.de>
Date: Fri, 22 Nov 2024 14:48:26 +0100
Subject: [PATCH 3/3] Stay on dev1

---
 CHANGELOG.rst  | 12 ++++--------
 pyproject.toml |  2 +-
 2 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 85d8719..7c63797 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -1,14 +1,10 @@
-1.0.dev2
-========
-
-- Switch to Jinja templating
-
 1.0.dev1
 ========
 
-Re-write of the metamodule generation:
-- Toolchain metamodules are deprecated and will only be generated for the existing toolchains
-- The octopus-dependencies module does now come in two variants min and full
+- Switch to Jinja templating
+- Re-write of the metamodule generation:
+  - Toolchain metamodules are deprecated and will only be generated for the existing toolchains
+  - The octopus-dependencies module does now come in two variants min and full
 
 
 1.0.dev0
diff --git a/pyproject.toml b/pyproject.toml
index d5c1303..97e312d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -7,7 +7,7 @@ name = "mpsd_software_manager"
 authors = [{name = "MPSD, SSU-Computational Science"}]
 license = {file = "LICENSE"}
 classifiers = ["License :: OSI Approved :: MIT License"]
-version = "1.0.dev2"
+version = "1.0.dev1"
 readme = "README.rst"
 requires-python = ">=3.9"
 dependencies = [
-- 
GitLab