From 7a02c796a18156e33fe0e3902a352338842ffe79 Mon Sep 17 00:00:00 2001
From: Hans Fangohr <hans.fangohr@mpsd.mpg.de>
Date: Wed, 24 May 2023 07:14:57 +0200
Subject: [PATCH 1/7] allow CI to fail if ruff or black is unhappy
---
.gitlab-ci.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 99c03e1..88a56bd 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -24,8 +24,8 @@ style:
script:
- black --version
- ruff --version
- - ruff . || true
- - black --check --diff . || true
+ - ruff .
+ - black --check --diff .
test3-09:
--
GitLab
From 6478fd0920fcf5022be5c5688766ede03d046943 Mon Sep 17 00:00:00 2001
From: Hans Fangohr <hans.fangohr@mpsd.mpg.de>
Date: Wed, 24 May 2023 07:16:34 +0200
Subject: [PATCH 2/7] remove repeated import
---
mpsd-software-environment.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/mpsd-software-environment.py b/mpsd-software-environment.py
index efdd2c4..10434cb 100755
--- a/mpsd-software-environment.py
+++ b/mpsd-software-environment.py
@@ -6,7 +6,6 @@ import argparse
import sys
from pathlib import Path
from typing import List, Tuple
-import datetime
about_tool = """
Build toolchains using Spack.\n
--
GitLab
From 949e8fe8e6319ce1bf941f740e7a9d017b069b3c Mon Sep 17 00:00:00 2001
From: Hans Fangohr <hans.fangohr@mpsd.mpg.de>
Date: Wed, 24 May 2023 10:03:50 +0200
Subject: [PATCH 3/7] line-length editing to make ruff and black happy
- didn't pay attention to content
- didn't intend to change content or behaviour
- didn't execute tests
---
mpsd-software-environment.py | 126 +++++++++++++++++++++++------------
1 file changed, 85 insertions(+), 41 deletions(-)
diff --git a/mpsd-software-environment.py b/mpsd-software-environment.py
index 10434cb..3a5c81a 100755
--- a/mpsd-software-environment.py
+++ b/mpsd-software-environment.py
@@ -12,11 +12,15 @@ Build toolchains using Spack.\n
This function builds toolchains for MPSD-HPC at the appropriate directory, \n
for given system architecture and MPSD software stack version.\n
-The toolchains are built using the bash script spack_setup.sh, and the results are logged.
-"""
+The toolchains
+are built using the bash script spack_setup.sh, and the results are logged. """
config_vars = {
"cmd_log_file": "install.log",
- "build_log_file": f"logs/mpsd_spack_ver_toolchains_{datetime.datetime.now().replace(microsecond=0).isoformat()}.log", # TODO: modify toolchains,mpsd_spack_ver when the variable is available
+ "build_log_file": (
+ "logs/mpsd_spack_ver_toolchains_"
+ f"{datetime.datetime.now().replace(microsecond=0).isoformat()}.log"
+ ),
+ # TODO: modify toolchains,mpsd_spack_ver when the variable is available
"spack_environments_repo": "https://gitlab.gwdg.de/mpsd-cs/spack-environments.git",
}
@@ -43,10 +47,11 @@ class os_chdir:
def setup_log_cmd(
mpsd_release: str, script_dir: str, msg: str = None, *args, **kwargs
) -> None:
- """
- The setup_log_cmd function logs the command used to build the toolchains,
- along with information about the software environment installer branch, the Spack environments branch,
- and the commit hashes of each. It also logs steps taken in install process using the optional message argument.
+ """The setup_log_cmd function logs the command used to build the
+ toolchains, along with information about the software environment installer
+ branch, the Spack environments branch, and the commit hashes of each. It
+ also logs steps taken in install process using the optional message
+ argument.
Args:
mpsd_release (str): The name of the release to install toolchains for.
@@ -55,16 +60,18 @@ def setup_log_cmd(
Returns:
None
+
"""
release_base_dir = script_dir / mpsd_release
+ # Write to the log file with the following format
+ # --------------------------------------------------
+ # 2023-02-29T23:32:01, install-software-environment.py --release 23b --install ALL
+ # Software environment installer branch: script_branch (commit hash: \
+ # script_commit_hash)
+ # Spack environments branch: dev-23a (commit hash: spe_commit_hash)
+ # MSGs
with os_chdir(release_base_dir):
- # Write to the log file with the following format
- # --------------------------------------------------
- # 2023-02-29T23:32:01, install-software-environment.py --release dev-23a --install ALL
- # Software environment installer branch: script_branch (commit hash: script_commit_hash)
- # Spack environments branch: dev-23a (commit hash: spe_commit_hash)
- # MSGs
with open(config_vars["cmd_log_file"], "a") as f:
if msg:
# Write the message to the log file
@@ -101,17 +108,19 @@ def setup_log_cmd(
# Write to log file
f.write(f"{datetime.datetime.now().isoformat()}, {cmd_line}\n")
f.write(
- f"Software environment installer branch: {script_branch} (commit hash: {script_commit_hash})\n"
+ f"Software environment installer branch: {script_branch} "
+ f"(commit hash: {script_commit_hash})\n"
)
f.write(
- f"Spack environments branch: {spe_branch} (commit hash: {spe_commit_hash})\n"
+ f"Spack environments branch: {spe_branch} "
+ f"(commit hash: {spe_commit_hash})\n"
)
def create_dir_structure(mpsd_release: str, script_dir: Path) -> None:
- """
- The create_dir_structure function creates the directory structure for the specified release
- and clones the Spack environments repository if it doesn't exist.
+ """The create_dir_structure function creates the directory structure for
+ the specified release and clones the Spack environments repository if it
+ doesn't exist.
Args:
- mpsd_release: A string representing the MPSD release version.
@@ -119,6 +128,7 @@ def create_dir_structure(mpsd_release: str, script_dir: Path) -> None:
Returns:
- None
+
"""
# Create the directory structure for the release
release_base_dir = script_dir / mpsd_release
@@ -135,12 +145,14 @@ def create_dir_structure(mpsd_release: str, script_dir: Path) -> None:
]
)
with os_chdir("spack-environments"):
- # Git fetch and checkout the release branch and git pull to be sure that the resulting repo is up to date
+ # Git fetch and checkout the release branch and git pull
+ # to be sure that the resulting repo is up to date
subprocess.run(["git", "fetch", "--all"])
checkout_result = subprocess.run(["git", "checkout", mpsd_release])
if checkout_result.returncode != 0:
raise Exception(
- "Release branch does not exist in spack-environment repo \n. Check for typos."
+ "Release branch does not exist in spack-environment repo \n."
+ "Check for typos."
)
subprocess.run(["git", "pull"])
@@ -157,10 +169,12 @@ def get_release_info(mpsd_release: str, script_dir: Path) -> Tuple[str, str, Lis
Returns:
- spe_branch: str, the name of the branch for the Spack environments repository.
- spe_commit_hash: str, the commit hash for the Spack environments repository.
- - available_toolchains: list, a list of strings representing the available toolchains for the release.
+ - available_toolchains: list, a list of strings representing the available
+ toolchains for the release.
Raises:
- - Exception: If the release directory does not exist. Run `create_dir_structure()` first.
+ - Exception: If the release directory does not exist. Run `create_dir_structure()`
+ first.
"""
# Get the info for release
release_base_dir = script_dir / mpsd_release
@@ -189,16 +203,20 @@ def get_release_info(mpsd_release: str, script_dir: Path) -> Tuple[str, str, Lis
def prepare_environment(mpsd_release: str, script_dir: Path) -> List[str]:
"""
- - Creates the directory structure for the given MPSD release and clones the spack-environments repository.
- - Determines the branch and commit hash of the spack-environments repository and the available toolchains.
+ - Creates the directory structure for the given MPSD release and clones the
+ spack-environments repository.
+ - Determines the branch and commit hash of the spack-environments repository and
+ the available toolchains.
- Logs the command usage.
Args:
- mpsd_release (str): The name of the MPSD release to prepare the environment for.
- - script_dir (pathlib.Path): The base directory to create the release folder and clone the spack-environments repository into.
+ - script_dir (pathlib.Path): The base directory to create the release folder and
+ clone the spack-environments repository into.
Returns:
- - available_toolchains (list): A list of available toolchains for the given MPSD release.
+ - available_toolchains (list): A list of available toolchains for the given MPSD
+ release.
"""
create_dir_structure(mpsd_release, script_dir)
spe_branch, spe_commit_hash, available_toolchains = get_release_info(
@@ -217,24 +235,34 @@ def install_environment(
force_reinstall: bool = False,
enable_build_cache: bool = False,
) -> None:
- """
- Installs the specified MPSD release and toolchains to the specified directory using Spack.
+ """Installs the specified MPSD release and toolchains to the specified
+ directory using Spack.
Args:
mpsd_release: A string representing the MPSD release version.
- toolchains: A list of strings representing the toolchains to install (e.g., "foss2021a-mpi", "global_generic", "ALL").
- script_dir: A Path object representing the path to the directory where the release and toolchains will be installed.
- force_reinstall: A boolean indicating whether to force a reinstallation even if the release and toolchains already exist. Defaults to False.
- enable_build_cache: A boolean indicating whether to build the build cache when installing toolchains. Defaults to False.
+
+ toolchains: A list of strings representing the toolchains to install
+ (e.g., "foss2021a-mpi", "global_generic", "ALL").
+
+ script_dir: A Path object representing the path to the directory where
+ the release and toolchains will be installed.
+
+ force_reinstall: A boolean indicating whether to force a reinstallation
+ even if the release and toolchains already exist. Defaults to False.
+
+ enable_build_cache: A boolean indicating whether to build the build
+ cache when installing toolchains. Defaults to False.
Raises:
Exception: If a requested toolchain is not available in the specified release.
Returns:
None
+
"""
print(
- f"Installing release {mpsd_release} with toolchains {toolchains} to {script_dir}"
+ f"Installing release {mpsd_release} with toolchains {toolchains} "
+ f"to {script_dir}"
)
# Set required variables
@@ -254,9 +282,11 @@ def install_environment(
if toolchains == "ALL":
toolchains = available_toolchains
elif toolchains == "NONE":
- # No toolchains requested, so we only create the env and print the list of available toolchains
+ # No toolchains requested, so we only create the env and print the
+ # list of available toolchains
print(
- f"No toolchains requested. Available toolchains for release {mpsd_release} are: \n {available_toolchains}"
+ "No toolchains requested. Available toolchains for release "
+ f"{mpsd_release} are: \n {available_toolchains}"
)
return
@@ -274,7 +304,8 @@ def install_environment(
os.mkdir("logs")
for toolchain in toolchains:
# Set the install log file name to config_vars["install_log_file"]
- # and replace _toolchains_ with the toolchain name and _mpsd_spack_ver_ with mpsd_release
+ # and replace _toolchains_ with the toolchain name and
+ # _mpsd_spack_ver_ with mpsd_release
print(f"Installing toolchain {toolchain} to {toolchain_dir}")
install_log_file = (
@@ -291,10 +322,14 @@ def install_environment(
setup_log_cmd(
mpsd_release,
script_dir,
- msg=f"CMD: bash {spack_setup_script} {' '.join(install_flags)} {toolchain}",
+ msg=(
+ f"CMD: bash {spack_setup_script} {' '.join(install_flags)}"
+ "{toolchain}"
+ ),
)
subprocess.run(
- f"bash {spack_setup_script} {' '.join(install_flags)} {toolchain} 2>&1 | tee -a {install_log_file} ",
+ f"bash {spack_setup_script} {' '.join(install_flags)} {toolchain} 2>&1 "
+ f"| tee -a {install_log_file} ",
shell=True,
)
@@ -353,17 +388,26 @@ def main():
dest="toolchains",
nargs="+",
default="NONE",
- help="List of toolchains to install (use '--toolchains ALL' to install all toolchains). If nothing is specified, list of available toolchains for the release would be shown after creating the environment.",
+ help=(
+ "List of toolchains to install (use '--toolchains ALL' to "
+ "install all toolchains). If nothing is specified, list of"
+ "available toolchains for the release would be shown after"
+ "creating the environment."
+ ),
)
subp.add_argument(
"--enable-build-cache",
action="store_true",
- help="Enable Spack build cache. Useful for reinstallation but consumes time and disk space",
+ help=(
+ "Enable Spack build cache. Useful for reinstallation but"
+ "consumes time and disk space"
+ ),
)
# Carry out the action
args = parser.parse_args()
- # target dir is the place where this script exists. the release `dev` in script_dir/dev-23a
+ # target dir is the place where this script exists. the
+ # release `dev` in script_dir/dev-23a
script_dir = Path(os.path.dirname(os.path.realpath(__file__)))
# Check the command and run related function
--
GitLab
From ae162c17fb11636393d1741b551a19ada13e69ba Mon Sep 17 00:00:00 2001
From: Hans Fangohr <hans.fangohr@mpsd.mpg.de>
Date: Wed, 24 May 2023 10:06:10 +0200
Subject: [PATCH 4/7] execute all tests (functionality and style) at the same
time
triggered by discussion https://gitlab.gwdg.de/mpsd-cs/mpsd-software-environments/-/merge_requests/3#note_803951
---
.gitlab-ci.yml | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 88a56bd..478acf5 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,7 +1,6 @@
image: python:latest
stages:
- - test-style
- test
before_script:
@@ -19,7 +18,7 @@ before_script:
style:
- stage: test-style
+ stage: test
image: python:latest
script:
- black --version
--
GitLab
From c8078b350839e6b919bf7dff1bfeb61cf546eee8 Mon Sep 17 00:00:00 2001
From: Hans Fangohr <hans.fangohr@mpsd.mpg.de>
Date: Wed, 24 May 2023 10:07:19 +0200
Subject: [PATCH 5/7] white space tidy up
---
.gitlab-ci.yml | 25 ++++++++++++-------------
1 file changed, 12 insertions(+), 13 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 478acf5..e8098aa 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -4,17 +4,17 @@ stages:
- test
before_script:
- - cat /etc/issue
- - python -V
- - which python
- - python -m venv ../venv
- - source ../venv/bin/activate
- - which python
- - pwd
- - pip install -U pip
- - pip --version
- - pip install pytest black ruff
- - pytest --version
+ - cat /etc/issue
+ - python -V
+ - which python
+ - python -m venv ../venv
+ - source ../venv/bin/activate
+ - which python
+ - pwd
+ - pip install -U pip
+ - pip --version
+ - pip install pytest black ruff
+ - pytest --version
style:
@@ -24,7 +24,7 @@ style:
- black --version
- ruff --version
- ruff .
- - black --check --diff .
+ - black --check --diff .
test3-09:
@@ -48,4 +48,3 @@ test3-11:
script:
- echo "run tests here later"
- python --version
-
--
GitLab
From a277de9a223762de4b5a95b14084bc69ecb27373 Mon Sep 17 00:00:00 2001
From: Hans Fangohr <hans.fangohr@mpsd.mpg.de>
Date: Wed, 24 May 2023 10:07:53 +0200
Subject: [PATCH 6/7] remove empty line
---
.gitlab-ci.yml | 1 -
1 file changed, 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e8098aa..a83f93a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -16,7 +16,6 @@ before_script:
- pip install pytest black ruff
- pytest --version
-
style:
stage: test
image: python:latest
--
GitLab
From a0aefbfe1496ea1639b9f4788652dfacef8d77df Mon Sep 17 00:00:00 2001
From: Hans Fangohr <hans.fangohr@mpsd.mpg.de>
Date: Wed, 24 May 2023 10:11:01 +0200
Subject: [PATCH 7/7] line-wrapping to satisfy ruff
- no attention or change to contents
- tests not executed
---
tests.py | 21 ++++++++++++++-------
1 file changed, 14 insertions(+), 7 deletions(-)
diff --git a/tests.py b/tests.py
index 31d578f..d266751 100644
--- a/tests.py
+++ b/tests.py
@@ -25,8 +25,10 @@ def test_os_chdir(tmp_path):
def test_prepare_environment(tmp_path):
- # simulate running ./install-software-environment.py --release dev-23a --target-directory /tmp/test_prepare_env
- # prepare_env is run when cmd is not specified, we can test cmd='prepare' and cmd=None to check both cases
+ # simulate running ./install-software-environment.py --release dev-23a \
+ # --target-directory /tmp/test_prepare_env
+ # prepare_env is run when cmd is not specified, we can test cmd='prepare'
+ # and cmd=None to check both cases
script_dir = tmp_path / "test_prepare_env"
spack_environments = "spack-environments"
@@ -94,7 +96,8 @@ def test_setup_log_cmd(tmp_path):
def test_install_environment(tmp_path):
# Test the installation part
- # This is a long test, its handy to test this with print statements printed to stdout, use:
+ # This is a long test, its handy to test this with print statements printed to
+ # stdout, use:
# pytest -s
# Expect an Exception when wrong toolchains are provided
with pytest.raises(Exception):
@@ -103,7 +106,8 @@ def test_install_environment(tmp_path):
toolchains=["wrong-toolchain"],
script_dir=(tmp_path),
)
- # Expect an Exception when wrong mpsd_release is provided ( part of prepare_environment)
+ # Expect an Exception when wrong mpsd_release is provided (part of
+ # prepare_environment)
with pytest.raises(Exception):
mod.install_environment(
mpsd_release="wrong-mpsd-release",
@@ -113,7 +117,8 @@ def test_install_environment(tmp_path):
# prepare a test of global generic with only zlib to test the installation
# prepare dev-23a release
# script_dir = tmp_path / "test_global_generic"
- # for actaual installation avoid tmp_path as the lenght of the path is too long and spack complains
+ # for actaual installation avoid tmp_path as the lenght of the path is too long
+ # and spack complains
script_dir = Path("/tmp/test_global_generic")
if script_dir.exists():
shutil.rmtree(script_dir)
@@ -134,7 +139,8 @@ def test_install_environment(tmp_path):
with open(toolchain_src_dir / "global_generic" / "global_packages.list", "w") as f:
f.write("zlib@1.2.13 \n")
- # add zlib to whitelist of module creation file by replacing anaconda3%gcc@10.2.1 with zlib@1.2.13
+ # add zlib to whitelist of module creation file by replacing anaconda3%gcc@10.2.1
+ # with zlib@1.2.13
# in release_base_dir / "spack-environments/spack_overlay/etc/spack/modules.yaml"
module_file = (
release_base_dir / "spack-environments/spack_overlay/etc/spack/modules.yaml"
@@ -165,7 +171,8 @@ def test_install_environment(tmp_path):
enable_build_cache=False,
)
# test that the build log is created correctly
- # check that a file with glob build_globale_generic_dev-23a*.log exists at release_base_dir/mpsd_microarch
+ # check that a file with glob build_globale_generic_dev-23a*.log exists at
+ # release_base_dir/mpsd_microarch
# print("Debug here ")
# time.sleep(10)
build_log = list(
--
GitLab