Skip to content
Snippets Groups Projects
Commit 21865059 authored by Ashwin Kumar Karnad's avatar Ashwin Kumar Karnad
Browse files

run black

parent a76a28c2
No related branches found
No related tags found
2 merge requests!19Move linux-debian11 into main,!1Resolve "First draft for user interface for top level install command"
Hide whitespace changes
Inline Side-by-side
mpsd-software-environment.py
+ 66
− 22
View file @ 21865059
......@@ -42,7 +42,9 @@ class os_chdir:
os.chdir(self.saved_dir)
def setup_log_cmd(mpsd_release: str, script_dir: str, msg: str = None, *args, **kwargs) -> None:
def setup_log_cmd(
mpsd_release: str, script_dir: str, msg: str = None, *args, **kwargs
) -> None:
"""
The setup_log_cmd function logs the command used to build the toolchains,
along with information about the software environment installer branch, the Spack environments branch,
......@@ -79,12 +81,18 @@ def setup_log_cmd(mpsd_release: str, script_dir: str, msg: str = None, *args, **
# script branch and commit hash
with os_chdir(script_dir):
script_branch = (
subprocess.run(["git", "rev-parse", "--abbrev-ref", "HEAD"], stdout=subprocess.PIPE)
subprocess.run(
["git", "rev-parse", "--abbrev-ref", "HEAD"],
stdout=subprocess.PIPE,
)
.stdout.decode()
.strip()
)
script_commit_hash = (
subprocess.run(["git", "rev-parse", "--short", "HEAD"], stdout=subprocess.PIPE)
subprocess.run(
["git", "rev-parse", "--short", "HEAD"],
stdout=subprocess.PIPE,
)
.stdout.decode()
.strip()
)
......@@ -94,8 +102,12 @@ def setup_log_cmd(mpsd_release: str, script_dir: str, msg: str = None, *args, **
# Write to log file
f.write(f"{datetime.datetime.now().isoformat()}, {cmd_line}\n")
f.write(f"Software environment installer branch: {script_branch} (commit hash: {script_commit_hash})\n")
f.write(f"Spack environments branch: {spe_branch} (commit hash: {spe_commit_hash})\n")
f.write(
f"Software environment installer branch: {script_branch} (commit hash: {script_commit_hash})\n"
)
f.write(
f"Spack environments branch: {spe_branch} (commit hash: {spe_commit_hash})\n"
)
def create_dir_structure(mpsd_release: str, script_dir: Path) -> None:
......@@ -129,7 +141,9 @@ def create_dir_structure(mpsd_release: str, script_dir: Path) -> None:
subprocess.run(["git", "fetch", "--all"])
checkout_result = subprocess.run(["git", "checkout", mpsd_release])
if checkout_result.returncode != 0:
raise Exception("Release branch does not exist in spack-environment repo \n. Check for typos.")
raise Exception(
"Release branch does not exist in spack-environment repo \n. Check for typos."
)
subprocess.run(["git", "pull"])
......@@ -153,15 +167,21 @@ def get_release_info(mpsd_release: str, script_dir: Path) -> Tuple[str, str, Lis
# Get the info for release
release_base_dir = script_dir / mpsd_release
if not os.path.exists(release_base_dir):
raise Exception("Release directory does not exist. Run create_dir_structure() first.")
raise Exception(
"Release directory does not exist. Run create_dir_structure() first."
)
with os_chdir(release_base_dir):
with os_chdir("spack-environments"):
# Get the branch and commit hash of the spack-environments repo
spe_commit_hash = (
subprocess.run(["git", "rev-parse", "HEAD"], stdout=subprocess.PIPE).stdout.decode().strip()
subprocess.run(["git", "rev-parse", "HEAD"], stdout=subprocess.PIPE)
.stdout.decode()
.strip()
)
spe_branch = (
subprocess.run(["git", "rev-parse", "--abbrev-ref", "HEAD"], stdout=subprocess.PIPE)
subprocess.run(
["git", "rev-parse", "--abbrev-ref", "HEAD"], stdout=subprocess.PIPE
)
.stdout.decode()
.strip()
)
......@@ -183,8 +203,12 @@ def prepare_environment(mpsd_release: str, script_dir: Path) -> List[str]:
- available_toolchains (list): A list of available toolchains for the given MPSD release.
"""
create_dir_structure(mpsd_release, script_dir)
spe_branch, spe_commit_hash, available_toolchains = get_release_info(mpsd_release, script_dir)
setup_log_cmd(mpsd_release, script_dir, spe_branch=spe_branch, spe_commit_hash=spe_commit_hash)
spe_branch, spe_commit_hash, available_toolchains = get_release_info(
mpsd_release, script_dir
)
setup_log_cmd(
mpsd_release, script_dir, spe_branch=spe_branch, spe_commit_hash=spe_commit_hash
)
return available_toolchains
......@@ -211,7 +235,9 @@ def install_environment(
Returns:
None
"""
print(f"Installing release {mpsd_release} with toolchains {toolchains} to {script_dir}")
print(
f"Installing release {mpsd_release} with toolchains {toolchains} to {script_dir}"
)
# Set required variables
release_base_dir = script_dir / mpsd_release
......@@ -238,7 +264,9 @@ def install_environment(
for toolchain in toolchains:
if toolchain not in available_toolchains:
raise Exception(f"Toolchain '{toolchain}' is not available in release {mpsd_release}.")
raise Exception(
f"Toolchain '{toolchain}' is not available in release {mpsd_release}."
)
# Install the toolchains
with os_chdir(toolchain_dir):
......@@ -249,13 +277,24 @@ def install_environment(
for toolchain in toolchains:
# Set the install log file name to config_vars["install_log_file"]
# and replace _toolchains_ with the toolchain name and _mpsd_spack_ver_ with mpsd_release
print(f"Installing toolchain {toolchain} to {toolchain_dir}")
install_log_file = (
config_vars["build_log_file"]
.replace("mpsd_spack_ver_", f"{mpsd_release}_")
.replace("_toolchains_", f"_{toolchain}_")
)
# log the command
setup_log_cmd(
mpsd_release,
script_dir,
msg=f"installing {toolchain} and logging at {install_log_file}",
)
setup_log_cmd(
mpsd_release,
script_dir,
msg=f"CMD: bash {spack_setup_script} {' '.join(install_flags)} {toolchain}",
)
subprocess.run(
f"bash {spack_setup_script} {' '.join(install_flags)} {toolchain} 2>&1 | tee -a {install_log_file} ",
shell=True,
......@@ -272,7 +311,9 @@ def start_new_environment(release, from_release, target_dir):
def main():
parser = argparse.ArgumentParser(description=about_tool)
subparsers = parser.add_subparsers(dest="action", title="actions", description="valid actions", required=True)
subparsers = parser.add_subparsers(
dest="action", title="actions", description="valid actions", required=True
)
subparsers.required = True
list_of_cmds = [
("prepare", "Prepare the environment for installation on the disk"),
......@@ -301,13 +342,15 @@ def main():
)
else:
subp.add_argument("release", type=str, help="Release version to install or remove")
subp.add_argument(
"release", type=str, help="Release version to install or remove"
)
if cmd in ["install", "reinstall"]:
subp.add_argument(
"--toolchains", # first option defines attribute
# name `args.toolchains` in `args = parser_args()`
"--toolchain", # allow singular as alternative
# (-> creates attribute `args.toolchains` if used)
"--toolchains", # first option defines attribute
# name `args.toolchains` in `args = parser_args()`
"--toolchain", # allow singular as alternative
# (-> creates attribute `args.toolchains` if used)
type=str,
dest="toolchains",
nargs="+",
......@@ -331,11 +374,12 @@ def main():
elif args.action == "start-new":
start_new_environment(args.from_release, args.to_release, script_dir)
elif args.action == "install":
install_environment(args.release, args.toolchains, script_dir, False, args.enable_build_cache)
install_environment(
args.release, args.toolchains, script_dir, False, args.enable_build_cache
)
elif args.action == "prepare":
prepare_environment(args.release, script_dir)
if __name__ == "__main__":
main()
tests.py
+ 59
− 40
View file @ 21865059
......@@ -56,13 +56,13 @@ def test_prepare_environment(tmp_path):
.split("\n")[0]
== f"* {mpsd_release_to_test}"
)
# check that result is a list and contains atleast ['global','foss2021a-mpi']
# check that result is a list and contains atleast ['global','foss2021a-mpi']
assert isinstance(result, list)
assert "global" in result
assert "foss2021a-mpi" in result
# Expect an Exception when wrong mpsd_release is provided
with pytest.raises(Exception):
with pytest.raises(Exception):
result = mod.prepare_environment(
mpsd_release="wrong-mpsd-release", script_dir=(script_dir)
)
......@@ -71,12 +71,12 @@ def test_prepare_environment(tmp_path):
def test_setup_log_cmd(tmp_path):
# check that logs/install-software-environment.log is updated when the module is run
log_file = "install.log"
script_dir = tmp_path / "test_prepare_env"
spack_environments = "spack-environments"
mpsd_release_to_test = "dev-23a"
release_base_dir = script_dir / mpsd_release_to_test
if os.path.exists(release_base_dir/log_file):
if os.path.exists(release_base_dir / log_file):
initial_bytes = os.path.getsize(log_file)
else:
initial_bytes = 0
......@@ -87,11 +87,11 @@ def test_setup_log_cmd(tmp_path):
)
# check that logs/install-software-environment.log is updated
assert os.path.exists(release_base_dir/log_file)
assert os.path.getsize(release_base_dir/log_file) > initial_bytes
assert os.path.exists(release_base_dir / log_file)
assert os.path.getsize(release_base_dir / log_file) > initial_bytes
# Check that the log file has "Spack environments branch: dev-23a " in the last line
with open(release_base_dir/log_file, "r") as f:
with open(release_base_dir / log_file, "r") as f:
last_line = f.readlines()[-1]
assert "Spack environments branch: dev-23a " in last_line
......@@ -118,18 +118,17 @@ def test_install_environment(tmp_path):
# prepare dev-23a release
# script_dir = tmp_path / "test_global_generic"
# for actaual installation avoid tmp_path as the lenght of the path is too long and spack complains
script_dir = Path('/tmp/test_global_generic')
script_dir = Path("/tmp/test_global_generic")
if script_dir.exists():
shutil.rmtree(script_dir)
script_dir.mkdir(exist_ok=True, parents=True)
spack_environments = "spack-environments"
mpsd_release_to_test = "dev-23a"
toolchain_to_test = "global_generic"
mpsd_microarch=os.getenv("MPSD_MICROARCH",'UNKNOWN_MICROARCH')
mpsd_microarch = os.getenv("MPSD_MICROARCH", "UNKNOWN_MICROARCH")
release_base_dir = script_dir / mpsd_release_to_test
prepare_result = mod.prepare_environment(
mpsd_release=mpsd_release_to_test,
script_dir=(script_dir)
mpsd_release=mpsd_release_to_test, script_dir=(script_dir)
)
# Patch the spack environments to create a fake global_generic
# create a test toolchain
......@@ -144,51 +143,71 @@ def test_install_environment(tmp_path):
# add zlib to whitelist of module creation file by replacing anaconda3%gcc@10.2.1 with zlib@1.2.13
# in release_base_dir / "spack-environments/spack_overlay/etc/spack/modules.yaml"
module_file = release_base_dir / "spack-environments/spack_overlay/etc/spack/modules.yaml"
with open(module_file,'r') as f:
lines = f.read().replace('anaconda3%gcc@10.2.1','zlib@1.2.13')
with open(module_file,'w') as f:
module_file = (
release_base_dir / "spack-environments/spack_overlay/etc/spack/modules.yaml"
)
with open(module_file, "r") as f:
lines = f.read().replace("anaconda3%gcc@10.2.1", "zlib@1.2.13")
with open(module_file, "w") as f:
f.write(lines)
# Replace gcc@10.2.1 with gcc#13.1.1 or available system gcc for testing on laptop
gcc_ver = subprocess.run(['gcc -dumpfullversion'],shell=True,capture_output=True).stdout.decode('utf-8').strip()
gcc_ver = (
subprocess.run(["gcc -dumpfullversion"], shell=True, capture_output=True)
.stdout.decode("utf-8")
.strip()
)
setup_file = release_base_dir / "spack-environments/spack_setup.sh"
with open(setup_file,'r') as f:
lines = f.read().replace('system_compiler="gcc@10.2.1"',f'system_compiler="gcc@{gcc_ver}"')
with open(setup_file,'w') as f:
with open(setup_file, "r") as f:
lines = f.read().replace(
'system_compiler="gcc@10.2.1"', f'system_compiler="gcc@{gcc_ver}"'
)
with open(setup_file, "w") as f:
f.write(lines)
# install global_generic toolchain
install_result = mod.install_environment(
mpsd_release=mpsd_release_to_test,
toolchains=[toolchain_to_test],
script_dir=script_dir,
enable_build_cache=False
)
# assert that install log files exists
assert os.path.exists(release_base_dir/'install.log')
# assert that the module files are created correctly
assert os.path.exists(release_base_dir/mpsd_microarch)
assert os.path.exists(release_base_dir/mpsd_microarch/'lmod')
# assert that lmod/module-index.yaml contains zlib
with open(release_base_dir/mpsd_microarch/'lmod'/'module-index.yaml','r') as f:
lines = f.read()
assert 'zlib' in lines
mpsd_release=mpsd_release_to_test,
toolchains=[toolchain_to_test],
script_dir=script_dir,
enable_build_cache=False,
)
# test that the build log is created correctly
# check that a file with glob build_globale_generic_dev-23a*.log exists at release_base_dir/mpsd_microarch
# print("Debug here ")
# time.sleep(10)
build_log = list((release_base_dir/mpsd_microarch/"logs").glob(f'{mpsd_release_to_test}_{toolchain_to_test}_*.log'))
build_log = list(
(release_base_dir / mpsd_microarch / "logs").glob(
f"{mpsd_release_to_test}_{toolchain_to_test}_*.log"
)
)
assert len(build_log) > 0
# take the most recent build log
build_log = sorted(build_log)[0]
# check that the build log contains statement ##### Installation finished
with open(build_log,'r') as f:
# check that the build log contains statement ##### Installation finished
with open(build_log, "r") as f:
lines = f.read()
assert '##### Installation finished' in lines
assert "##### Installation finished" in lines
# assert that install log files exists
assert os.path.exists(release_base_dir / "install.log")
# assert that the build log is written to the install log file
build_log_file_name = os.path.basename(build_log)
with open(release_base_dir / "install.log", "r") as f:
lines = f.read()
assert (
"installing {toolchain_to_test} and logging at logs/{build_log_file_name}"
in lines
)
# assert that the module files are created correctly
assert os.path.exists(release_base_dir / mpsd_microarch)
assert os.path.exists(release_base_dir / mpsd_microarch / "lmod")
# assert that lmod/module-index.yaml contains zlib
with open(
release_base_dir / mpsd_microarch / "lmod" / "module-index.yaml", "r"
) as f:
lines = f.read()
assert "zlib" in lines
def test_interface(tmp_path):
......@@ -197,4 +216,4 @@ def test_interface(tmp_path):
# check that the script branch and hash are correct when running the script
# check that the help message is printed when no arguments are provided
# check that the help message is printed when -h is provided
# check that the error messages are also logged to the log file
\ No newline at end of file
# check that the error messages are also logged to the log file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment