From 71b272c99eacfc74c096fd0e9d9c26a2c42c257a Mon Sep 17 00:00:00 2001 From: Martin Lang <martin.lang@mpsd.mpg.de> Date: Fri, 22 Nov 2024 14:07:51 +0100 Subject: [PATCH 1/3] Switch to jinja templating --- pyproject.toml | 3 +- src/mpsd_software_manager/spack.py | 98 +++++++++++++++++------------- 2 files changed, 57 insertions(+), 44 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bd9fbf2..d5c1303 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,11 +7,12 @@ name = "mpsd_software_manager" authors = [{name = "MPSD, SSU-Computational Science"}] license = {file = "LICENSE"} classifiers = ["License :: OSI Approved :: MIT License"] -version = "1.0.dev1" +version = "1.0.dev2" readme = "README.rst" requires-python = ">=3.9" dependencies = [ "archspec", + "Jinja2", "rich", "python-gitlab", "PyYAML", diff --git a/src/mpsd_software_manager/spack.py b/src/mpsd_software_manager/spack.py index d5c69c2..0f4b5cb 100644 --- a/src/mpsd_software_manager/spack.py +++ b/src/mpsd_software_manager/spack.py @@ -15,6 +15,7 @@ from logging import getLogger from pathlib import Path from typing import Any, Callable +import jinja2 import yaml from .config import Config @@ -89,21 +90,36 @@ def find_system_compiler(requested_system_compiler: str | None) -> None: def update_custom_spack_config() -> None: """Apply custom spack configuration from spack-environments repository.""" - # copy - custom_config = Config().spack_environments_root / "spack_overlay" - assert custom_config.is_absolute() + spack_overlay_root = Config().spack_environments_root / "spack_overlay" + assert spack_overlay_root.is_absolute() files = [ - elem.relative_to(custom_config) - for elem in custom_config.rglob("*") + elem.relative_to(spack_overlay_root) + for elem in spack_overlay_root.rglob("*") if elem.is_file() ] - for f in files: - copy_file(source=custom_config / f, dest=Config().spack_root / f) + # source_cache_root may not exist; as fallbacks we try to use the binary cache + # or a suitable location inside spack + if Config().source_cache_root.is_dir(): + source_cache = Config().source_cache_root + elif Config().binary_cache_root.is_dir(): + source_cache = Config().binary_cache_root + else: + source_cache = Config().spack_root / "var" / "spack" / "cache" + + for template_file in files: + render_template( + template_dir=spack_overlay_root / template_file.parent, + template_name=template_file.name, + dest_dir=Config().spack_root / template_file.parent, + source_cache=source_cache, + spack_root=Config().spack_root, + lmod_root=Config().lmod_root, + system_compiler=Config().system_compiler, + ) def configure_spack_mirrors(disable_binary_cache: bool) -> None: """Configure source and binary cache.""" - # TODO check if source mirror is writable add_mirror("source", "mpsd_spack_sources", Config().source_cache_root) if not disable_binary_cache: Config().binary_cache_root.mkdir(exist_ok=True) @@ -145,37 +161,35 @@ def update_caches(disable_binary_cache: bool) -> None: ) -def copy_file( - source: Path, dest: Path, replacements: list[tuple[str, str]] | None = None +def render_template( + template_dir: Path, template_name: str, dest_dir: Path, **context: Any ) -> None: - """Copy file 'source' to 'dest' and replace template variables. + """Render given template and write output to dest_dir. - Existing files are overwritten without notice. - """ - logger.debug("Copying '%s' to '%s' with template replacement", source, dest) - with source.open() as f: - content = f.read() + The template must have name "name.ext.jinja"; it is written to "dest_dir/name.ext". - # TODO replace with better templating - content = content.replace("##SYSTEM_COMPILER##", Config().system_compiler) - content = content.replace("$spack/../lmod", str(Config().lmod_root)) - if Config().source_cache_root.is_dir(): - source_cache = Config().source_cache_root - elif Config().binary_cache_root.is_dir(): - source_cache = Config().binary_cache_root - else: - source_cache = Config().spack_root / "var" / "spack" / "cache" - content = content.replace("$spack/../mpsd-spack-cache", str(source_cache)) - # replace all other occurrences of $spack variable - content = content.replace("$spack", str(Config().spack_root)) + All keyword arguments are passed to the template engine as 'context'. Jinja ignores + unused elements in the context so passing arguments that are not used in a template + is fine. Likewise, missing variables do not result in an error but are just empty + strings in the output. - if replacements: - for placeholder, value in replacements: - content = content.replace(placeholder, value) + The guaranteed context is documented in: + https://gitlab.gwdg.de/mpsd-cs/spack-environments/-/tree/develop/docs/templating.org - dest.parent.mkdir(parents=True, exist_ok=True) - with dest.open("w") as f: - f.write(content) + Existing files are overwritten without notice. + """ + dest_file = template_name.removesuffix(".jinja") + logger.debug( + "Rendering template '%s' to '%s'\nContext: %s", + template_dir / template_name, + dest_dir / dest_file, + context, + ) + env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir)) + template = env.get_template(template_name) + dest_dir.mkdir(parents=True, exist_ok=True) + with open(dest_dir / dest_file, "w") as f: + f.write(template.render(context)) def add_mirror(type_: str, name: str, path: Path) -> None: @@ -322,16 +336,14 @@ def install_package_set_from_environment(spack_environment: str) -> None: logger.debug(spack(f"env create {spack_environment}")) config = Config() - replacements = [("##TOOLCHAIN_COMPILER##", compilers["default"]["name"])] + context = {"toolchain_compiler": compilers["default"]["name"]} if "fallback" in compilers: - replacements.append(("##TOOLCHAIN_GCC##", compilers["fallback"]["name"])) - copy_file( - config.spack_environments_root - / "toolchains" - / spack_environment - / "spack.yaml", - spack_environment_path(spack_environment) / "spack.yaml", - replacements, + context["fallback_compiler"] = compilers["fallback"]["name"] + render_template( + config.spack_environments_root / "toolchains" / spack_environment, + "spack.yaml.jinja", + spack_environment_path(spack_environment), + **context, ) logger.info("concretizing environment '%s'", spack_environment) -- GitLab From 74611eddfb4640b5c7cf435a1d28c3a3d02f17ad Mon Sep 17 00:00:00 2001 From: Martin Lang <martin.lang@mpsd.mpg.de> Date: Fri, 22 Nov 2024 14:31:37 +0100 Subject: [PATCH 2/3] Add CHANGELOG --- CHANGELOG.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 CHANGELOG.rst diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 0000000..85d8719 --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,17 @@ +1.0.dev2 +======== + +- Switch to Jinja templating + +1.0.dev1 +======== + +Re-write of the metamodule generation: +- Toolchain metamodules are deprecated and will only be generated for the existing toolchains +- The octopus-dependencies module does now come in two variants min and full + + +1.0.dev0 +======== + +- Re-write of the old ``spack_setup.sh`` and ``software-manager``. -- GitLab From 4973fee985c0921507cf36f9da1c688dafbd63c4 Mon Sep 17 00:00:00 2001 From: Martin Lang <martin.lang@mpsd.mpg.de> Date: Fri, 22 Nov 2024 14:48:26 +0100 Subject: [PATCH 3/3] Stay on dev1 --- CHANGELOG.rst | 12 ++++-------- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 85d8719..7c63797 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,14 +1,10 @@ -1.0.dev2 -======== - -- Switch to Jinja templating - 1.0.dev1 ======== -Re-write of the metamodule generation: -- Toolchain metamodules are deprecated and will only be generated for the existing toolchains -- The octopus-dependencies module does now come in two variants min and full +- Switch to Jinja templating +- Re-write of the metamodule generation: + - Toolchain metamodules are deprecated and will only be generated for the existing toolchains + - The octopus-dependencies module does now come in two variants min and full 1.0.dev0 diff --git a/pyproject.toml b/pyproject.toml index d5c1303..97e312d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "mpsd_software_manager" authors = [{name = "MPSD, SSU-Computational Science"}] license = {file = "LICENSE"} classifiers = ["License :: OSI Approved :: MIT License"] -version = "1.0.dev2" +version = "1.0.dev1" readme = "README.rst" requires-python = ">=3.9" dependencies = [ -- GitLab