diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 0000000000000000000000000000000000000000..7c6379752c173fbdcb7e0f76d46cca30e9027312 --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,13 @@ +1.0.dev1 +======== + +- Switch to Jinja templating +- Re-write of the metamodule generation: + - Toolchain metamodules are deprecated and will only be generated for the existing toolchains + - The octopus-dependencies module does now come in two variants min and full + + +1.0.dev0 +======== + +- Re-write of the old ``spack_setup.sh`` and ``software-manager``. diff --git a/pyproject.toml b/pyproject.toml index bd9fbf24ecd94089b7f89259b95faddd2b6f538c..97e312dc496b3092a28aa0cfde80d5d423991ffd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ readme = "README.rst" requires-python = ">=3.9" dependencies = [ "archspec", + "Jinja2", "rich", "python-gitlab", "PyYAML", diff --git a/src/mpsd_software_manager/spack.py b/src/mpsd_software_manager/spack.py index d5c69c2f66233671bc917caf78f20b8a8edfdf29..0f4b5cb38e56e68615f1d32d3ac95cddab31ecb6 100644 --- a/src/mpsd_software_manager/spack.py +++ b/src/mpsd_software_manager/spack.py @@ -15,6 +15,7 @@ from logging import getLogger from pathlib import Path from typing import Any, Callable +import jinja2 import yaml from .config import Config @@ -89,21 +90,36 @@ def find_system_compiler(requested_system_compiler: str | None) -> None: def update_custom_spack_config() -> None: """Apply custom spack configuration from spack-environments repository.""" - # copy - custom_config = Config().spack_environments_root / "spack_overlay" - assert custom_config.is_absolute() + spack_overlay_root = Config().spack_environments_root / "spack_overlay" + assert spack_overlay_root.is_absolute() files = [ - elem.relative_to(custom_config) - for elem in custom_config.rglob("*") + elem.relative_to(spack_overlay_root) + for elem in spack_overlay_root.rglob("*") if elem.is_file() ] - for f in files: - copy_file(source=custom_config / f, dest=Config().spack_root / f) + # source_cache_root may not exist; as fallbacks we try to use the binary cache + # or a suitable location inside spack + if Config().source_cache_root.is_dir(): + source_cache = Config().source_cache_root + elif Config().binary_cache_root.is_dir(): + source_cache = Config().binary_cache_root + else: + source_cache = Config().spack_root / "var" / "spack" / "cache" + + for template_file in files: + render_template( + template_dir=spack_overlay_root / template_file.parent, + template_name=template_file.name, + dest_dir=Config().spack_root / template_file.parent, + source_cache=source_cache, + spack_root=Config().spack_root, + lmod_root=Config().lmod_root, + system_compiler=Config().system_compiler, + ) def configure_spack_mirrors(disable_binary_cache: bool) -> None: """Configure source and binary cache.""" - # TODO check if source mirror is writable add_mirror("source", "mpsd_spack_sources", Config().source_cache_root) if not disable_binary_cache: Config().binary_cache_root.mkdir(exist_ok=True) @@ -145,37 +161,35 @@ def update_caches(disable_binary_cache: bool) -> None: ) -def copy_file( - source: Path, dest: Path, replacements: list[tuple[str, str]] | None = None +def render_template( + template_dir: Path, template_name: str, dest_dir: Path, **context: Any ) -> None: - """Copy file 'source' to 'dest' and replace template variables. + """Render given template and write output to dest_dir. - Existing files are overwritten without notice. - """ - logger.debug("Copying '%s' to '%s' with template replacement", source, dest) - with source.open() as f: - content = f.read() + The template must have name "name.ext.jinja"; it is written to "dest_dir/name.ext". - # TODO replace with better templating - content = content.replace("##SYSTEM_COMPILER##", Config().system_compiler) - content = content.replace("$spack/../lmod", str(Config().lmod_root)) - if Config().source_cache_root.is_dir(): - source_cache = Config().source_cache_root - elif Config().binary_cache_root.is_dir(): - source_cache = Config().binary_cache_root - else: - source_cache = Config().spack_root / "var" / "spack" / "cache" - content = content.replace("$spack/../mpsd-spack-cache", str(source_cache)) - # replace all other occurrences of $spack variable - content = content.replace("$spack", str(Config().spack_root)) + All keyword arguments are passed to the template engine as 'context'. Jinja ignores + unused elements in the context so passing arguments that are not used in a template + is fine. Likewise, missing variables do not result in an error but are just empty + strings in the output. - if replacements: - for placeholder, value in replacements: - content = content.replace(placeholder, value) + The guaranteed context is documented in: + https://gitlab.gwdg.de/mpsd-cs/spack-environments/-/tree/develop/docs/templating.org - dest.parent.mkdir(parents=True, exist_ok=True) - with dest.open("w") as f: - f.write(content) + Existing files are overwritten without notice. + """ + dest_file = template_name.removesuffix(".jinja") + logger.debug( + "Rendering template '%s' to '%s'\nContext: %s", + template_dir / template_name, + dest_dir / dest_file, + context, + ) + env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir)) + template = env.get_template(template_name) + dest_dir.mkdir(parents=True, exist_ok=True) + with open(dest_dir / dest_file, "w") as f: + f.write(template.render(context)) def add_mirror(type_: str, name: str, path: Path) -> None: @@ -322,16 +336,14 @@ def install_package_set_from_environment(spack_environment: str) -> None: logger.debug(spack(f"env create {spack_environment}")) config = Config() - replacements = [("##TOOLCHAIN_COMPILER##", compilers["default"]["name"])] + context = {"toolchain_compiler": compilers["default"]["name"]} if "fallback" in compilers: - replacements.append(("##TOOLCHAIN_GCC##", compilers["fallback"]["name"])) - copy_file( - config.spack_environments_root - / "toolchains" - / spack_environment - / "spack.yaml", - spack_environment_path(spack_environment) / "spack.yaml", - replacements, + context["fallback_compiler"] = compilers["fallback"]["name"] + render_template( + config.spack_environments_root / "toolchains" / spack_environment, + "spack.yaml.jinja", + spack_environment_path(spack_environment), + **context, ) logger.info("concretizing environment '%s'", spack_environment)