Skip to content
Snippets Groups Projects
Commit d5465c9d authored by Ashwin Kumar Karnad's avatar Ashwin Kumar Karnad
Browse files

Merge branch 'activate-style-checks-in-ci' into 'linux-debian11' 

allow CI to fail if ruff or black is unhappy, and format source to make it pass

See merge request mpsd-cs/mpsd-software-environments!3
parents 3554a57b a0aefbfe
No related branches found
No related tags found
2 merge requests!19Move linux-debian11 into main,!3allow CI to fail if ruff or black is unhappy, and format source to make it pass
Pipeline #367677 passed
Stage: test
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
+ 15
− 18
View file @ d5465c9d
image: python:latest
stages:
- test-style
- test
before_script:
- cat /etc/issue
- python -V
- which python
- python -m venv ../venv
- source ../venv/bin/activate
- which python
- pwd
- pip install -U pip
- pip --version
- pip install pytest black ruff
- pytest --version
- cat /etc/issue
- python -V
- which python
- python -m venv ../venv
- source ../venv/bin/activate
- which python
- pwd
- pip install -U pip
- pip --version
- pip install pytest black ruff
- pytest --version
style:
stage: test-style
stage: test
image: python:latest
script:
- black --version
- ruff --version
- ruff . || true
- black --check --diff . || true
- ruff .
- black --check --diff .
test3-09:
......@@ -49,4 +47,3 @@ test3-11:
script:
- echo "run tests here later"
- python --version
mpsd-software-environment.py
+ 85
− 42
View file @ d5465c9d
......@@ -6,18 +6,21 @@ import argparse
import sys
from pathlib import Path
from typing import List, Tuple
import datetime
about_tool = """
Build toolchains using Spack.\n
This function builds toolchains for MPSD-HPC at the appropriate directory, \n
for given system architecture and MPSD software stack version.\n
The toolchains are built using the bash script spack_setup.sh, and the results are logged.
"""
The toolchains
are built using the bash script spack_setup.sh, and the results are logged. """
config_vars = {
"cmd_log_file": "install.log",
"build_log_file": f"logs/mpsd_spack_ver_toolchains_{datetime.datetime.now().replace(microsecond=0).isoformat()}.log", # TODO: modify toolchains,mpsd_spack_ver when the variable is available
"build_log_file": (
"logs/mpsd_spack_ver_toolchains_"
f"{datetime.datetime.now().replace(microsecond=0).isoformat()}.log"
),
# TODO: modify toolchains,mpsd_spack_ver when the variable is available
"spack_environments_repo": "https://gitlab.gwdg.de/mpsd-cs/spack-environments.git",
}
......@@ -44,10 +47,11 @@ class os_chdir:
def setup_log_cmd(
mpsd_release: str, script_dir: str, msg: str = None, *args, **kwargs
) -> None:
"""
The setup_log_cmd function logs the command used to build the toolchains,
along with information about the software environment installer branch, the Spack environments branch,
and the commit hashes of each. It also logs steps taken in install process using the optional message argument.
"""The setup_log_cmd function logs the command used to build the
toolchains, along with information about the software environment installer
branch, the Spack environments branch, and the commit hashes of each. It
also logs steps taken in install process using the optional message
argument.
Args:
mpsd_release (str): The name of the release to install toolchains for.
......@@ -56,16 +60,18 @@ def setup_log_cmd(
Returns:
None
"""
release_base_dir = script_dir / mpsd_release
# Write to the log file with the following format
# --------------------------------------------------
# 2023-02-29T23:32:01, install-software-environment.py --release 23b --install ALL
# Software environment installer branch: script_branch (commit hash: \
# script_commit_hash)
# Spack environments branch: dev-23a (commit hash: spe_commit_hash)
# MSGs
with os_chdir(release_base_dir):
# Write to the log file with the following format
# --------------------------------------------------
# 2023-02-29T23:32:01, install-software-environment.py --release dev-23a --install ALL
# Software environment installer branch: script_branch (commit hash: script_commit_hash)
# Spack environments branch: dev-23a (commit hash: spe_commit_hash)
# MSGs
with open(config_vars["cmd_log_file"], "a") as f:
if msg:
# Write the message to the log file
......@@ -102,17 +108,19 @@ def setup_log_cmd(
# Write to log file
f.write(f"{datetime.datetime.now().isoformat()}, {cmd_line}\n")
f.write(
f"Software environment installer branch: {script_branch} (commit hash: {script_commit_hash})\n"
f"Software environment installer branch: {script_branch} "
f"(commit hash: {script_commit_hash})\n"
)
f.write(
f"Spack environments branch: {spe_branch} (commit hash: {spe_commit_hash})\n"
f"Spack environments branch: {spe_branch} "
f"(commit hash: {spe_commit_hash})\n"
)
def create_dir_structure(mpsd_release: str, script_dir: Path) -> None:
"""
The create_dir_structure function creates the directory structure for the specified release
and clones the Spack environments repository if it doesn't exist.
"""The create_dir_structure function creates the directory structure for
the specified release and clones the Spack environments repository if it
doesn't exist.
Args:
- mpsd_release: A string representing the MPSD release version.
......@@ -120,6 +128,7 @@ def create_dir_structure(mpsd_release: str, script_dir: Path) -> None:
Returns:
- None
"""
# Create the directory structure for the release
release_base_dir = script_dir / mpsd_release
......@@ -136,12 +145,14 @@ def create_dir_structure(mpsd_release: str, script_dir: Path) -> None:
]
)
with os_chdir("spack-environments"):
# Git fetch and checkout the release branch and git pull to be sure that the resulting repo is up to date
# Git fetch and checkout the release branch and git pull
# to be sure that the resulting repo is up to date
subprocess.run(["git", "fetch", "--all"])
checkout_result = subprocess.run(["git", "checkout", mpsd_release])
if checkout_result.returncode != 0:
raise Exception(
"Release branch does not exist in spack-environment repo \n. Check for typos."
"Release branch does not exist in spack-environment repo \n."
"Check for typos."
)
subprocess.run(["git", "pull"])
......@@ -158,10 +169,12 @@ def get_release_info(mpsd_release: str, script_dir: Path) -> Tuple[str, str, Lis
Returns:
- spe_branch: str, the name of the branch for the Spack environments repository.
- spe_commit_hash: str, the commit hash for the Spack environments repository.
- available_toolchains: list, a list of strings representing the available toolchains for the release.
- available_toolchains: list, a list of strings representing the available
toolchains for the release.
Raises:
- Exception: If the release directory does not exist. Run `create_dir_structure()` first.
- Exception: If the release directory does not exist. Run `create_dir_structure()`
first.
"""
# Get the info for release
release_base_dir = script_dir / mpsd_release
......@@ -190,16 +203,20 @@ def get_release_info(mpsd_release: str, script_dir: Path) -> Tuple[str, str, Lis
def prepare_environment(mpsd_release: str, script_dir: Path) -> List[str]:
"""
- Creates the directory structure for the given MPSD release and clones the spack-environments repository.
- Determines the branch and commit hash of the spack-environments repository and the available toolchains.
- Creates the directory structure for the given MPSD release and clones the
spack-environments repository.
- Determines the branch and commit hash of the spack-environments repository and
the available toolchains.
- Logs the command usage.
Args:
- mpsd_release (str): The name of the MPSD release to prepare the environment for.
- script_dir (pathlib.Path): The base directory to create the release folder and clone the spack-environments repository into.
- script_dir (pathlib.Path): The base directory to create the release folder and
clone the spack-environments repository into.
Returns:
- available_toolchains (list): A list of available toolchains for the given MPSD release.
- available_toolchains (list): A list of available toolchains for the given MPSD
release.
"""
create_dir_structure(mpsd_release, script_dir)
spe_branch, spe_commit_hash, available_toolchains = get_release_info(
......@@ -218,24 +235,34 @@ def install_environment(
force_reinstall: bool = False,
enable_build_cache: bool = False,
) -> None:
"""
Installs the specified MPSD release and toolchains to the specified directory using Spack.
"""Installs the specified MPSD release and toolchains to the specified
directory using Spack.
Args:
mpsd_release: A string representing the MPSD release version.
toolchains: A list of strings representing the toolchains to install (e.g., "foss2021a-mpi", "global_generic", "ALL").
script_dir: A Path object representing the path to the directory where the release and toolchains will be installed.
force_reinstall: A boolean indicating whether to force a reinstallation even if the release and toolchains already exist. Defaults to False.
enable_build_cache: A boolean indicating whether to build the build cache when installing toolchains. Defaults to False.
toolchains: A list of strings representing the toolchains to install
(e.g., "foss2021a-mpi", "global_generic", "ALL").
script_dir: A Path object representing the path to the directory where
the release and toolchains will be installed.
force_reinstall: A boolean indicating whether to force a reinstallation
even if the release and toolchains already exist. Defaults to False.
enable_build_cache: A boolean indicating whether to build the build
cache when installing toolchains. Defaults to False.
Raises:
Exception: If a requested toolchain is not available in the specified release.
Returns:
None
"""
print(
f"Installing release {mpsd_release} with toolchains {toolchains} to {script_dir}"
f"Installing release {mpsd_release} with toolchains {toolchains} "
f"to {script_dir}"
)
# Set required variables
......@@ -255,9 +282,11 @@ def install_environment(
if toolchains == "ALL":
toolchains = available_toolchains
elif toolchains == "NONE":
# No toolchains requested, so we only create the env and print the list of available toolchains
# No toolchains requested, so we only create the env and print the
# list of available toolchains
print(
f"No toolchains requested. Available toolchains for release {mpsd_release} are: \n {available_toolchains}"
"No toolchains requested. Available toolchains for release "
f"{mpsd_release} are: \n {available_toolchains}"
)
return
......@@ -275,7 +304,8 @@ def install_environment(
os.mkdir("logs")
for toolchain in toolchains:
# Set the install log file name to config_vars["install_log_file"]
# and replace _toolchains_ with the toolchain name and _mpsd_spack_ver_ with mpsd_release
# and replace _toolchains_ with the toolchain name and
# _mpsd_spack_ver_ with mpsd_release
print(f"Installing toolchain {toolchain} to {toolchain_dir}")
install_log_file = (
......@@ -292,10 +322,14 @@ def install_environment(
setup_log_cmd(
mpsd_release,
script_dir,
msg=f"CMD: bash {spack_setup_script} {' '.join(install_flags)} {toolchain}",
msg=(
f"CMD: bash {spack_setup_script} {' '.join(install_flags)}"
"{toolchain}"
),
)
subprocess.run(
f"bash {spack_setup_script} {' '.join(install_flags)} {toolchain} 2>&1 | tee -a {install_log_file} ",
f"bash {spack_setup_script} {' '.join(install_flags)} {toolchain} 2>&1 "
f"| tee -a {install_log_file} ",
shell=True,
)
......@@ -354,17 +388,26 @@ def main():
dest="toolchains",
nargs="+",
default="NONE",
help="List of toolchains to install (use '--toolchains ALL' to install all toolchains). If nothing is specified, list of available toolchains for the release would be shown after creating the environment.",
help=(
"List of toolchains to install (use '--toolchains ALL' to "
"install all toolchains). If nothing is specified, list of"
"available toolchains for the release would be shown after"
"creating the environment."
),
)
subp.add_argument(
"--enable-build-cache",
action="store_true",
help="Enable Spack build cache. Useful for reinstallation but consumes time and disk space",
help=(
"Enable Spack build cache. Useful for reinstallation but"
"consumes time and disk space"
),
)
# Carry out the action
args = parser.parse_args()
# target dir is the place where this script exists. the release `dev` in script_dir/dev-23a
# target dir is the place where this script exists. the
# release `dev` in script_dir/dev-23a
script_dir = Path(os.path.dirname(os.path.realpath(__file__)))
# Check the command and run related function
......
tests.py
+ 14
− 7
View file @ d5465c9d
......@@ -25,8 +25,10 @@ def test_os_chdir(tmp_path):
def test_prepare_environment(tmp_path):
# simulate running ./install-software-environment.py --release dev-23a --target-directory /tmp/test_prepare_env
# prepare_env is run when cmd is not specified, we can test cmd='prepare' and cmd=None to check both cases
# simulate running ./install-software-environment.py --release dev-23a \
# --target-directory /tmp/test_prepare_env
# prepare_env is run when cmd is not specified, we can test cmd='prepare'
# and cmd=None to check both cases
script_dir = tmp_path / "test_prepare_env"
spack_environments = "spack-environments"
......@@ -94,7 +96,8 @@ def test_setup_log_cmd(tmp_path):
def test_install_environment(tmp_path):
# Test the installation part
# This is a long test, its handy to test this with print statements printed to stdout, use:
# This is a long test, its handy to test this with print statements printed to
# stdout, use:
# pytest -s
# Expect an Exception when wrong toolchains are provided
with pytest.raises(Exception):
......@@ -103,7 +106,8 @@ def test_install_environment(tmp_path):
toolchains=["wrong-toolchain"],
script_dir=(tmp_path),
)
# Expect an Exception when wrong mpsd_release is provided ( part of prepare_environment)
# Expect an Exception when wrong mpsd_release is provided (part of
# prepare_environment)
with pytest.raises(Exception):
mod.install_environment(
mpsd_release="wrong-mpsd-release",
......@@ -113,7 +117,8 @@ def test_install_environment(tmp_path):
# prepare a test of global generic with only zlib to test the installation
# prepare dev-23a release
# script_dir = tmp_path / "test_global_generic"
# for actaual installation avoid tmp_path as the lenght of the path is too long and spack complains
# for actaual installation avoid tmp_path as the lenght of the path is too long
# and spack complains
script_dir = Path("/tmp/test_global_generic")
if script_dir.exists():
shutil.rmtree(script_dir)
......@@ -134,7 +139,8 @@ def test_install_environment(tmp_path):
with open(toolchain_src_dir / "global_generic" / "global_packages.list", "w") as f:
f.write("zlib@1.2.13 \n")
# add zlib to whitelist of module creation file by replacing anaconda3%gcc@10.2.1 with zlib@1.2.13
# add zlib to whitelist of module creation file by replacing anaconda3%gcc@10.2.1
# with zlib@1.2.13
# in release_base_dir / "spack-environments/spack_overlay/etc/spack/modules.yaml"
module_file = (
release_base_dir / "spack-environments/spack_overlay/etc/spack/modules.yaml"
......@@ -165,7 +171,8 @@ def test_install_environment(tmp_path):
enable_build_cache=False,
)
# test that the build log is created correctly
# check that a file with glob build_globale_generic_dev-23a*.log exists at release_base_dir/mpsd_microarch
# check that a file with glob build_globale_generic_dev-23a*.log exists at
# release_base_dir/mpsd_microarch
# print("Debug here ")
# time.sleep(10)
build_log = list(
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment