Skip to content
Snippets Groups Projects

Switch to jinja templating

Merged Martin Lang requested to merge jinja-templates into main
Files
3
@@ -15,6 +15,7 @@ from logging import getLogger
from pathlib import Path
from typing import Any, Callable
import jinja2
import yaml
from .config import Config
@@ -89,21 +90,36 @@ def find_system_compiler(requested_system_compiler: str | None) -> None:
def update_custom_spack_config() -> None:
"""Apply custom spack configuration from spack-environments repository."""
# copy
custom_config = Config().spack_environments_root / "spack_overlay"
assert custom_config.is_absolute()
spack_overlay_root = Config().spack_environments_root / "spack_overlay"
assert spack_overlay_root.is_absolute()
files = [
elem.relative_to(custom_config)
for elem in custom_config.rglob("*")
elem.relative_to(spack_overlay_root)
for elem in spack_overlay_root.rglob("*")
if elem.is_file()
]
for f in files:
copy_file(source=custom_config / f, dest=Config().spack_root / f)
# source_cache_root may not exist; as fallbacks we try to use the binary cache
# or a suitable location inside spack
if Config().source_cache_root.is_dir():
source_cache = Config().source_cache_root
elif Config().binary_cache_root.is_dir():
source_cache = Config().binary_cache_root
else:
source_cache = Config().spack_root / "var" / "spack" / "cache"
for template_file in files:
render_template(
template_dir=spack_overlay_root / template_file.parent,
template_name=template_file.name,
dest_dir=Config().spack_root / template_file.parent,
source_cache=source_cache,
spack_root=Config().spack_root,
lmod_root=Config().lmod_root,
system_compiler=Config().system_compiler,
)
def configure_spack_mirrors(disable_binary_cache: bool) -> None:
"""Configure source and binary cache."""
# TODO check if source mirror is writable
add_mirror("source", "mpsd_spack_sources", Config().source_cache_root)
if not disable_binary_cache:
Config().binary_cache_root.mkdir(exist_ok=True)
@@ -145,37 +161,35 @@ def update_caches(disable_binary_cache: bool) -> None:
)
def copy_file(
source: Path, dest: Path, replacements: list[tuple[str, str]] | None = None
def render_template(
template_dir: Path, template_name: str, dest_dir: Path, **context: Any
) -> None:
"""Copy file 'source' to 'dest' and replace template variables.
"""Render given template and write output to dest_dir.
Existing files are overwritten without notice.
"""
logger.debug("Copying '%s' to '%s' with template replacement", source, dest)
with source.open() as f:
content = f.read()
The template must have name "name.ext.jinja"; it is written to "dest_dir/name.ext".
# TODO replace with better templating
content = content.replace("##SYSTEM_COMPILER##", Config().system_compiler)
content = content.replace("$spack/../lmod", str(Config().lmod_root))
if Config().source_cache_root.is_dir():
source_cache = Config().source_cache_root
elif Config().binary_cache_root.is_dir():
source_cache = Config().binary_cache_root
else:
source_cache = Config().spack_root / "var" / "spack" / "cache"
content = content.replace("$spack/../mpsd-spack-cache", str(source_cache))
# replace all other occurrences of $spack variable
content = content.replace("$spack", str(Config().spack_root))
All keyword arguments are passed to the template engine as 'context'. Jinja ignores
unused elements in the context so passing arguments that are not used in a template
is fine. Likewise, missing variables do not result in an error but are just empty
strings in the output.
if replacements:
for placeholder, value in replacements:
content = content.replace(placeholder, value)
The guaranteed context is documented in:
https://gitlab.gwdg.de/mpsd-cs/spack-environments/-/tree/develop/docs/templating.org
dest.parent.mkdir(parents=True, exist_ok=True)
with dest.open("w") as f:
f.write(content)
Existing files are overwritten without notice.
"""
dest_file = template_name.removesuffix(".jinja")
logger.debug(
"Rendering template '%s' to '%s'\nContext: %s",
template_dir / template_name,
dest_dir / dest_file,
context,
)
env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir))
template = env.get_template(template_name)
dest_dir.mkdir(parents=True, exist_ok=True)
with open(dest_dir / dest_file, "w") as f:
f.write(template.render(context))
def add_mirror(type_: str, name: str, path: Path) -> None:
@@ -322,16 +336,14 @@ def install_package_set_from_environment(spack_environment: str) -> None:
logger.debug(spack(f"env create {spack_environment}"))
config = Config()
replacements = [("##TOOLCHAIN_COMPILER##", compilers["default"]["name"])]
context = {"toolchain_compiler": compilers["default"]["name"]}
if "fallback" in compilers:
replacements.append(("##TOOLCHAIN_GCC##", compilers["fallback"]["name"]))
copy_file(
config.spack_environments_root
/ "toolchains"
/ spack_environment
/ "spack.yaml",
spack_environment_path(spack_environment) / "spack.yaml",
replacements,
context["fallback_compiler"] = compilers["fallback"]["name"]
render_template(
config.spack_environments_root / "toolchains" / spack_environment,
"spack.yaml.jinja",
spack_environment_path(spack_environment),
**context,
)
logger.info("concretizing environment '%s'", spack_environment)
Loading