From 66ab060fc2efea3a883324ed4a83f5c814523b47 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Fri, 3 Oct 2025 19:09:36 +0000 Subject: [PATCH 01/16] add acorn deploy scripting --- util/acorn/deploy/deploy.py | 103 +++++++++++++++++++++++++++++ util/acorn/deploy/deployments.yaml | 13 ++++ 2 files changed, 116 insertions(+) create mode 100755 util/acorn/deploy/deploy.py create mode 100644 util/acorn/deploy/deployments.yaml diff --git a/util/acorn/deploy/deploy.py b/util/acorn/deploy/deploy.py new file mode 100755 index 000000000..c07231c48 --- /dev/null +++ b/util/acorn/deploy/deploy.py @@ -0,0 +1,103 @@ +#!/usr/bin/env spack-python + +import argparse + +parser = argparse.ArgumentParser(description="Your script description") + +parser.add_argument('-s', '--scheduler', action='store_true', help="Enable scheduler") +parser.add_argument('-c', '--concretize-args', type=str, help="Concretize arguments (provide a single string)") +parser.add_argument('-i', '--install-args', type=str, help="Install arguments (provide a single string)") +parser.add_argument('-r', '--redeploy-existing', action='store_true', help="Redeploy existing deployments") + +parser.add_argument('deployments', nargs='*', help="List of deployments to apply (default is all)") + +args = parser.parse_args() + +import collections +import os +import yaml + +import spack.extensions +spack.extensions.load_extension("stack") +from spack.extensions.stack.cmd.stack_cmds.create import * +import spack.environment + +spack_stack_dir = os.getenv("SPACK_STACK_DIR") + +def get_env_dir_basename(deployment): + base = deployment["template"] + base = base.replace("unified-dev", "ue") + base = base.replace("-dev", "") + return "-".join([base, deployment["compiler"]]) + +def deployment_already_exists(env_dir_basename, args): + path_to_check = os.path.join(spack_stack_dir, "envs", env_dir_basename) + return os.path.isdir(path_to_check) + +def is_deployment_requested(env_dir_basename, deployment, args): + if deployment["template"] in args.deployments: + return True + if deployment["template"] + "/" + deployment["compiler"] in args.deployments: + return True + if args.redeploy_existing: + return True + return not deployment_already_exists(env_dir_basename, args) + +def get_create_env_settings(env_dir_basename, deployment, deployments): + dict = {} + dict["site"] = "acorn" + dict["template"] = deployment["template"] + dict["dir"] = os.path.join(spack_stack_dir, "envs") + dict["name"] = env_dir_basename + if "upstreams" in deployment: + upstream_full_paths = [] + for upstream_template in deployment["upstreams"]: + for _candidate_upstream in deployments.values(): + if (_candidate_upstream["template"] == upstream_template) and (_candidate_upstream["compiler"] == deployment["compiler"]): + upstream_basename = get_env_dir_basename(_candidate_upstream) + upstream_full_path = os.path.join(spack_stack_dir, "envs", upstream_basename, "install") + upstream_full_paths.append(upstream_full_path) + break + dict["upstreams"] = upstream_full_paths + dict["compiler"] = deployment["compiler"] + + return dict + +# Load deployments.yaml configuration +print("Loading deployments.yaml") +script_directory = os.path.dirname(os.path.abspath(__file__)) +deployments_yaml_path = os.path.join(script_directory, "deployments.yaml") +with open(deployments_yaml_path, "r") as f: + deployments_yaml = yaml.safe_load(f) + +# Generate deployments object, including iterating over compilers +deployments = collections.OrderedDict() + +for _deployment in deployments_yaml["core_deployments"]: # + deployments_yaml["addon_deployments"]: + for _compiler in _deployment["compilers"]: + deployment = _deployment.copy() + del(deployment["compilers"]) + deployment["compiler"] = _compiler + env_dir_basename = get_env_dir_basename(deployment) + deployments[env_dir_basename] = deployment + print("Registered deployment:", env_dir_basename) + +print("="*30) + +# Create and install each deployment +for env_dir_basename, deployment in deployments.items(): + if not is_deployment_requested(env_dir_basename, deployment, args): + continue + print("="*30) + # Create env based on config + stack_settings = get_create_env_settings(env_dir_basename, deployment, deployments) + env_dir_full_path = os.path.join(spack_stack_dir, "envs", env_dir_basename) + print(f"Creating environment for {deployment['template']}/{deployment['compiler']}") + print(f" at {env_dir_full_path}") + stack_env = StackEnv(**stack_settings) + stack_env.write() + stack_env.check_umask() + env = spack.environment.Environment(env_dir_full_path) + spack.environment.activate(env) + env.concretize() +## Install (PBS Pro if --scheduler/-s) diff --git a/util/acorn/deploy/deployments.yaml b/util/acorn/deploy/deployments.yaml new file mode 100644 index 000000000..a717de8e1 --- /dev/null +++ b/util/acorn/deploy/deployments.yaml @@ -0,0 +1,13 @@ +core_deployments: + - template: "unified-dev" + compilers: ["oneapi-2024.2.1"] + packages_to_install: + - ufs-weather-model-env + - global-workflow-env + - gsi-env + - madis + +#addon_deployments: +# - template: "test-addon" +# compilers: ["oneapi-2024.2.1"] +# upstream: ["unified-dev"] From 5737a3c90eb45c8c8e9ae7cde60356f4f616c3ee Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Fri, 3 Oct 2025 19:20:15 +0000 Subject: [PATCH 02/16] Acorn site config updates for spack v1 changes --- configs/sites/tier1/acorn/compilers.yaml | 133 ------------------ configs/sites/tier1/acorn/packages.yaml | 2 + ...ages_gcc.yaml => packages_gcc-11.2.0.yaml} | 0 ...el.yaml => packages_intel-19.1.3.304.yaml} | 0 .../tier1/acorn/packages_oneapi-2024.2.1.yaml | 78 ++++++++++ .../sites/tier1/acorn/packages_oneapi.yaml | 37 ----- 6 files changed, 80 insertions(+), 170 deletions(-) delete mode 100644 configs/sites/tier1/acorn/compilers.yaml rename configs/sites/tier1/acorn/{packages_gcc.yaml => packages_gcc-11.2.0.yaml} (100%) rename configs/sites/tier1/acorn/{packages_intel.yaml => packages_intel-19.1.3.304.yaml} (100%) create mode 100644 configs/sites/tier1/acorn/packages_oneapi-2024.2.1.yaml delete mode 100644 configs/sites/tier1/acorn/packages_oneapi.yaml diff --git a/configs/sites/tier1/acorn/compilers.yaml b/configs/sites/tier1/acorn/compilers.yaml deleted file mode 100644 index da0e33a1b..000000000 --- a/configs/sites/tier1/acorn/compilers.yaml +++ /dev/null @@ -1,133 +0,0 @@ -compilers: -- compiler: - spec: intel@19.1.3.304 - paths: - cc: cc - cxx: CC - f77: ftn - fc: ftn - flags: {} - operating_system: sles15 - modules: - - PrgEnv-intel/8.5.0 - - craype/2.7.17 - - intel/19.1.3.304 - - libfabric - environment: - set: - # OpenSUSE on WCOSS2 machines sets CONFIG_SITE so - # Automake-based builds are installed in lib64 - # which confuses some packages. - CONFIG_SITE: '' - extra_rpaths: [] -- compiler: - spec: intel@2022.2.0.262 - paths: - cc: cc - cxx: CC - f77: ftn - fc: ftn - flags: {} - operating_system: sles15 - modules: - - PrgEnv-intel/8.5.0 - - craype/2.7.17 - - intel-classic/2022.2.0.262 - - libfabric - environment: - prepend_path: - PATH: /opt/cray/pe/gcc/10.2.0/bin - LD_LIBRARY_PATH: /opt/cray/pe/gcc/10.2.0/snos/lib64 - CPATH: /opt/cray/pe/gcc/10.2.0/snos/include - set: - # OpenSUSE on WCOSS2 machines sets CONFIG_SITE so - # Automake-based builds are installed in lib64 - # which confuses some packages. - CONFIG_SITE: '' - extra_rpaths: [] -- compiler: - spec: gcc@11.2.0 - paths: - cc: cc - cxx: CC - f77: ftn - fc: ftn - flags: {} - operating_system: sles15 - modules: - - PrgEnv-gnu/8.3.3 - - craype/2.7.13 - - gcc/11.2.0 - - libfabric - environment: - set: - # OpenSUSE on WCOSS2 machines sets CONFIG_SITE so - # Automake-based builds are installed in lib64 - # which confuses some packages. - CONFIG_SITE: '' - extra_rpaths: [] -# Needed for Intel 19/2022 backend -- compiler: - spec: gcc@10.2.0 - paths: - cc: cc - cxx: CC - f77: ftn - fc: ftn - flags: {} - operating_system: sles15 - modules: - - PrgEnv-gnu/8.3.3 - - craype/2.7.13 - - gcc/10.2.0 - - libfabric - environment: - set: - # OpenSUSE on WCOSS2 machines sets CONFIG_SITE so - # Automake-based builds are installed in lib64 - # which confuses some packages. - CONFIG_SITE: '' - extra_rpaths: [] -- compiler: - spec: oneapi@2024.2.1 - paths: - cc: cc - cxx: CC - f77: ftn - fc: ftn - flags: {} - operating_system: sles15 - modules: - - PrgEnv-intel/8.5.0 - - craype/2.7.17 - - intel-oneapi/2024.2 - - libfabric - environment: - prepend_path: - PATH: /opt/cray/pe/gcc/12.1.0/bin - LD_LIBRARY_PATH: /opt/cray/pe/gcc/12.1.0/snos/lib64 - LIBRARY_PATH: /opt/cray/pe/gcc/12.1.0/snos/lib64 - CPATH: /opt/cray/pe/gcc/12.1.0/snos/include - set: - # OpenSUSE on WCOSS2 machines sets CONFIG_SITE so - # Automake-based builds are installed in lib64 - # which confuses some packages. - CONFIG_SITE: '' - extra_rpaths: [] -- compiler: - spec: gcc@12.1.0 - paths: - cc: /opt/cray/pe/gcc/12.1.0/bin/gcc - cxx: /opt/cray/pe/gcc/12.1.0/bin/g++ - f77: /opt/cray/pe/gcc/12.1.0/bin/gfortran - fc: /opt/cray/pe/gcc/12.1.0/bin/gfortran - flags: {} - operating_system: sles15 - modules: [] - environment: - set: - # OpenSUSE on WCOSS2 machines sets CONFIG_SITE so - # Automake-based builds are installed in lib64 - # which confuses some packages. - CONFIG_SITE: '' - extra_rpaths: [] diff --git a/configs/sites/tier1/acorn/packages.yaml b/configs/sites/tier1/acorn/packages.yaml index 42e0165dd..0161b2569 100644 --- a/configs/sites/tier1/acorn/packages.yaml +++ b/configs/sites/tier1/acorn/packages.yaml @@ -43,3 +43,5 @@ externals: - spec: grep@3.1 prefix: /usr + libxslt: + require: '~crypto' diff --git a/configs/sites/tier1/acorn/packages_gcc.yaml b/configs/sites/tier1/acorn/packages_gcc-11.2.0.yaml similarity index 100% rename from configs/sites/tier1/acorn/packages_gcc.yaml rename to configs/sites/tier1/acorn/packages_gcc-11.2.0.yaml diff --git a/configs/sites/tier1/acorn/packages_intel.yaml b/configs/sites/tier1/acorn/packages_intel-19.1.3.304.yaml similarity index 100% rename from configs/sites/tier1/acorn/packages_intel.yaml rename to configs/sites/tier1/acorn/packages_intel-19.1.3.304.yaml diff --git a/configs/sites/tier1/acorn/packages_oneapi-2024.2.1.yaml b/configs/sites/tier1/acorn/packages_oneapi-2024.2.1.yaml new file mode 100644 index 000000000..cf2b28909 --- /dev/null +++ b/configs/sites/tier1/acorn/packages_oneapi-2024.2.1.yaml @@ -0,0 +1,78 @@ +packages: + ### Compilers / compiler runtimes + intel-oneapi-compilers: + buildable: false + externals: + - spec: intel-oneapi-compilers@2024.2.1 ~envmods + modules: + - PrgEnv-intel/8.5.0 + - craype/2.7.17 + - intel-oneapi/2024.2 + - libfabric + extra_attributes: + compilers: + c: cc + cxx: CC + fortran: ftn + flags: + cflags: '--gcc-toolchain=/opt/cray/pe/gcc/12.1.0/snos' + cxxflags: '--gcc-toolchain=/opt/cray/pe/gcc/12.1.0/snos' + fflags: '-gcc-name=/opt/cray/pe/gcc/12.1.0/snos/bin/gcc -gxx-name=/opt/cray/pe/gcc/12.1.0/snos/bin/g++' + environment: + prepend_path: + PATH: /opt/cray/pe/gcc/12.1.0/snos/bin + LD_LIBRARY_PATH: /opt/cray/pe/gcc/12.1.0/snos/lib64 + LIBRARY_PATH: /opt/cray/pe/gcc/12.1.0/snos/lib64 + CPATH: /opt/cray/pe/gcc/12.1.0/snos/include + set: + CONFIG_SITE: '' + INTEL_COMPILER_TYPE: RECOMMENDED + gcc: + buildable: false + externals: + - spec: gcc@12.1.0 languages:='c,c++,fortran' + prefix: /opt/cray/pe/gcc/12.1.0 + extra_attributes: + compilers: + c: /opt/cray/pe/gcc/12.1.0/bin/gcc + cxx: /opt/cray/pe/gcc/12.1.0/bin/g++ + fortran: /opt/cray/pe/gcc/12.1.0/bin/gfortran + environment: + set: + # OpenSUSE on WCOSS2 machines sets CONFIG_SITE so + # Automake-based builds are installed in lib64 + # which confuses some packages. + CONFIG_SITE: '' + gcc-runtime: + require: '%gcc@12.1.0' + externals: + - spec: gcc-runtime@12.1.0 + prefix: /opt/cray/pe/gcc/12.1.0 + ### General package/provider settings & MPI external + mpi: + buildable: false + require: ['cray-mpich@8.1.29'] + cray-mpich: + externals: + - spec: cray-mpich@8.1.29~wrappers + prefix: /opt/cray/pe/mpich/8.1.29/ofi/intel/2022.1 + modules: + - libfabric + - craype-network-ofi + - cray-mpich/8.1.29 + - cray-pals + ### Disable MKL + intel-oneapi-mkl: + buildable: false + blas: + require: ['openblas'] + lapack: + require: ['openblas'] + fftw-api: + require: ['fftw'] + ectrans: + require:: ['~mkl', '+fftw'] + gsibec: + require: ['~mkl'] + py-numpy: + require:: ['^[virtuals=lapack,blas] openblas'] diff --git a/configs/sites/tier1/acorn/packages_oneapi.yaml b/configs/sites/tier1/acorn/packages_oneapi.yaml deleted file mode 100644 index 01e4538e1..000000000 --- a/configs/sites/tier1/acorn/packages_oneapi.yaml +++ /dev/null @@ -1,37 +0,0 @@ - packages: - all: - compiler:: [oneapi@2024.2] - providers: - mpi:: [cray-mpich@8.1.29] - # Remove the next three lines to switch to intel-oneapi-mkl - blas:: [openblas] - fftw-api:: [fftw] - lapack:: [openblas] - gcc-runtime: - require: '%gcc@12.1.0' - mpi: - buildable: False - cray-mpich: - externals: - - spec: cray-mpich@8.1.29~wrappers - modules: - - libfabric - - craype-network-ofi - - cray-mpich/8.1.29 - - cray-pals - intel-oneapi-mkl: - # Remove buildable: False and configure+uncomment externals section below to use intel-oneapi-mkl - buildable: False - #externals: - #- spec: intel-oneapi-mkl@2022.0.2%intel@2021.5.0 - # prefix: /apps/oneapi - # DH* Remove this section to switch to intel-oneapi-mkl - ectrans: - require:: - - '@1.2.0 ~mkl +fftw' - gsibec: - require:: - - '@1.2.1 ~mkl' - py-numpy: - require:: - - '^[virtuals=lapack,blas] openblas' From a4b7a95157c4853154769ceb3d5bd294a7cac200 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Fri, 3 Oct 2025 19:21:00 +0000 Subject: [PATCH 03/16] fetch_cargo_deps.py: fix for spack v1 update (spack->spack_repo) --- util/fetch_cargo_deps.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/util/fetch_cargo_deps.py b/util/fetch_cargo_deps.py index 250a94804..f2b8ae0bc 100755 --- a/util/fetch_cargo_deps.py +++ b/util/fetch_cargo_deps.py @@ -21,8 +21,8 @@ this_dir = os.path.dirname(os.path.abspath(sys.argv[0])) from spack.environment import active_environment -from spack.build_systems.cargo import CargoPackage -from spack.build_systems.python import PythonPackage +from spack_repo.builtin.build_systems.cargo import CargoPackage +from spack_repo.builtin.build_systems.python import PythonPackage from spack.package_base import PackageBase from spack.util.executable import Executable, which from llnl.util.filesystem import working_dir @@ -112,4 +112,4 @@ for root, dirs, files in os.walk(pkg.stage.source_path): if "Cargo.toml" in files: with working_dir(root): - cargo_exe("fetch") \ No newline at end of file + cargo_exe("fetch") From 927912edf2495f47dd7c1075cbd74b79398e6520 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Wed, 8 Oct 2025 18:09:25 +0000 Subject: [PATCH 04/16] eliminate 'llnl' deprecation warning --- .../lib/jcsda-emc/spack-stack/stack/cmd/stack_cmds/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/cmd/stack_cmds/create.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/cmd/stack_cmds/create.py index 31eae3615..e91ae2679 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/cmd/stack_cmds/create.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/cmd/stack_cmds/create.py @@ -3,7 +3,7 @@ import shutil from sys import platform -import llnl.util.tty as tty +import spack.llnl.util.tty as tty from spack.extensions.stack.container_env import StackContainer from spack.extensions.stack.stack_env import StackEnv, stack_path From a24d5f9be9c950a9bd3cc45f72adcd7229f2665e Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Wed, 8 Oct 2025 18:10:10 +0000 Subject: [PATCH 05/16] acorn: gcc/10.3 python doesn't have yaml somehow? --- configs/sites/tier1/acorn/setup.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/configs/sites/tier1/acorn/setup.sh b/configs/sites/tier1/acorn/setup.sh index 86ba92b44..966263370 100644 --- a/configs/sites/tier1/acorn/setup.sh +++ b/configs/sites/tier1/acorn/setup.sh @@ -1,5 +1,4 @@ -## Load modules to avoid various issues with old Python versions -module load gcc/10.3.0 python/3.11.7 +module load cray-python/3.11.7 ## Go, Rust repo setup for fetcher scripts export GOMODCACHE=${SPACK_STACK_DIR}/cache/go From f1e0c68d6412caa75e09e4b8cf5609c4a0318b31 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Wed, 8 Oct 2025 18:24:34 +0000 Subject: [PATCH 06/16] stack_env.py: copy envrepo from template dirs if present --- spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py index e3dc258d6..2dabde98f 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py @@ -253,6 +253,14 @@ def write(self): if self.site != "none": self._copy_site_includes() + # Copy any envrepo directory associated with the template (i.e., add-on env) + if self.template: + envrepo_path = os.path.join(self.template_path, "envrepo") + if os.path.isdir(envrepo_path): + logging.info(f"Copying envrepo/ from {self.template_path}") + target_envrepo_path = os.path.join(env_dir, "envrepo") + shutil.copytree(envrepo_path, target_envrepo_path, dirs_exist_ok=True) + # Copy common include files self._copy_common_includes() From 76d64a0cfe241551f3b11adf1c1ab943e0261e6f Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Thu, 9 Oct 2025 16:39:30 +0000 Subject: [PATCH 07/16] deploy.py: major updates; now works with addon envs --- util/acorn/deploy/deploy.py | 150 ++++++++++++++++++++++++++++++++---- 1 file changed, 134 insertions(+), 16 deletions(-) diff --git a/util/acorn/deploy/deploy.py b/util/acorn/deploy/deploy.py index c07231c48..09a3b6cd9 100755 --- a/util/acorn/deploy/deploy.py +++ b/util/acorn/deploy/deploy.py @@ -4,7 +4,8 @@ parser = argparse.ArgumentParser(description="Your script description") -parser.add_argument('-s', '--scheduler', action='store_true', help="Enable scheduler") +parser.add_argument('-n', '--no-scheduler', action='store_true', help="Run installation on local node (no job scheduler)") +parser.add_argument('-s', '--skip-go-rust-handling', action='store_true', help="Skip handling of Go/Rust dep fetching when using parallel job scheduler") parser.add_argument('-c', '--concretize-args', type=str, help="Concretize arguments (provide a single string)") parser.add_argument('-i', '--install-args', type=str, help="Install arguments (provide a single string)") parser.add_argument('-r', '--redeploy-existing', action='store_true', help="Redeploy existing deployments") @@ -14,16 +15,33 @@ args = parser.parse_args() import collections +from datetime import datetime +import logging import os +import subprocess import yaml +from contextlib import redirect_stdout, redirect_stderr +from types import SimpleNamespace + +logging.basicConfig(level=logging.CRITICAL) import spack.extensions spack.extensions.load_extension("stack") from spack.extensions.stack.cmd.stack_cmds.create import * +from spack.extensions.stack.meta_modules import setup_meta_modules import spack.environment +from spack.cmd.concretize import concretize +from spack.cmd.fetch import fetch +from spack.cmd.module import module +from spack import modules +from spack.util.executable import which spack_stack_dir = os.getenv("SPACK_STACK_DIR") +nowdate = datetime.now().strftime("%Y%m%d-%H%M") +logdir = os.path.join(spack_stack_dir, "deploy_logs") +os.makedirs(logdir, exist_ok=True) + def get_env_dir_basename(deployment): base = deployment["template"] base = base.replace("unified-dev", "ue") @@ -44,11 +62,11 @@ def is_deployment_requested(env_dir_basename, deployment, args): return not deployment_already_exists(env_dir_basename, args) def get_create_env_settings(env_dir_basename, deployment, deployments): - dict = {} - dict["site"] = "acorn" - dict["template"] = deployment["template"] - dict["dir"] = os.path.join(spack_stack_dir, "envs") - dict["name"] = env_dir_basename + config_dict = {} + config_dict["site"] = "acorn" + config_dict["template"] = deployment["template"] + config_dict["dir"] = os.path.join(spack_stack_dir, "envs") + config_dict["name"] = env_dir_basename if "upstreams" in deployment: upstream_full_paths = [] for upstream_template in deployment["upstreams"]: @@ -56,12 +74,33 @@ def get_create_env_settings(env_dir_basename, deployment, deployments): if (_candidate_upstream["template"] == upstream_template) and (_candidate_upstream["compiler"] == deployment["compiler"]): upstream_basename = get_env_dir_basename(_candidate_upstream) upstream_full_path = os.path.join(spack_stack_dir, "envs", upstream_basename, "install") - upstream_full_paths.append(upstream_full_path) + upstream_full_paths.append([upstream_full_path]) break - dict["upstreams"] = upstream_full_paths - dict["compiler"] = deployment["compiler"] - - return dict + config_dict["upstreams"] = upstream_full_paths + config_dict["compiler"] = deployment["compiler"] + + return config_dict + +def run_batch_install(batch_config, deployment, env_dir_full_path, specs_str, logfile, logfilepath): + if "walltime" in deployment: + walltime = deployment["walltime"] + elif "default_walltime" in batch_config: + walltime = batch_config["default_walltime"] + else: + assert False, "Set deployment-specific walltime or batch_config:default_walltime" + cmd = [ + "qsub", + "-o", logfilepath + ".batch_install", + "-e", logfilepath + ".batch_install", + "-A", batch_config["account"], + "-q", batch_config["queue"], + "-l", "walltime=" + walltime + ",select=1:ncpus=12", + "-V", "-Wblock=true", "--", + which("spack").path, "--env", env_dir_full_path, + "install", "--concurrent-packages", "3", "--jobs", "4", + ] + cmd.extend(specs_str) + subprocess.run(cmd, stdout=logfile, stderr=logfile, check=True) # Load deployments.yaml configuration print("Loading deployments.yaml") @@ -73,31 +112,110 @@ def get_create_env_settings(env_dir_basename, deployment, deployments): # Generate deployments object, including iterating over compilers deployments = collections.OrderedDict() -for _deployment in deployments_yaml["core_deployments"]: # + deployments_yaml["addon_deployments"]: +for _deployment in deployments_yaml["deployments"]: for _compiler in _deployment["compilers"]: deployment = _deployment.copy() del(deployment["compilers"]) deployment["compiler"] = _compiler env_dir_basename = get_env_dir_basename(deployment) deployments[env_dir_basename] = deployment - print("Registered deployment:", env_dir_basename) + print(f"Registered deployment: {deployment['template']}/{deployment['compiler']} ({env_dir_basename})") print("="*30) # Create and install each deployment for env_dir_basename, deployment in deployments.items(): if not is_deployment_requested(env_dir_basename, deployment, args): + print(f"Skipping existing deployment: {deployment['template']}/{deployment['compiler']} ({env_dir_basename})") continue print("="*30) # Create env based on config stack_settings = get_create_env_settings(env_dir_basename, deployment, deployments) env_dir_full_path = os.path.join(spack_stack_dir, "envs", env_dir_basename) + if os.path.isdir(env_dir_full_path) and args.redeploy_existing: + backup_dir_full_path = os.path.join(spack_stack_dir, "envs", env_dir_basename + "_bkp") + assert not os.path.isdir(backup_dir_full_path), f"Backup dir {backup_dir_full_path} already exists" + print(f"Moving {env_dir_full_path}\n to {backup_dir_full_path}") + os.rename(env_dir_full_path, backup_dir_full_path) + logfilepath = os.path.join(logdir, nowdate + f".{deployment['template']}.{deployment['compiler']}.log") + print(f"Log file: {logfilepath}") + logfile = open(logfilepath, "a") + logfile.write(str(deployment) + "\n") + logfile.write(str(stack_settings) + "\n") print(f"Creating environment for {deployment['template']}/{deployment['compiler']}") - print(f" at {env_dir_full_path}") + print(f" at {env_dir_full_path} ...") + # Use equivalent of 'spack stack create env' stack_env = StackEnv(**stack_settings) stack_env.write() stack_env.check_umask() env = spack.environment.Environment(env_dir_full_path) spack.environment.activate(env) - env.concretize() -## Install (PBS Pro if --scheduler/-s) + + # Concretize environment + concretize_args = SimpleNamespace( + test = False, ### + quiet = False, + fresh = True, ### + force = True, + ) + print(f"... concretizing ...") + with redirect_stdout(logfile), redirect_stderr(logfile): + concretize(None, concretize_args) + + # Fetch packages + if "packages_to_install" in deployment: + specs_str = deployment["packages_to_install"] + else: + specs_str = [] + fetch_args = SimpleNamespace( + missing = True, + no_checksum = False, + dependencies = bool(specs_str), + specs = specs_str, + ) + print(f"... fetching packages ...") + with redirect_stdout(logfile), redirect_stderr(logfile): + fetch(None, fetch_args) + + # Install packages + print("... installing" + (" specs: " +" ".join(specs_str) if specs_str else "") + " ...") + logfile.flush() + if args.no_scheduler: + specs = env.all_matching_specs(*specs_str) + env.install_specs(specs) + else: + if not args.skip_go_rust_handling: + run_batch_install(deployments_yaml["batch_config"], deployment, env_dir_full_path, ["rust", "go"], logfile, logfilepath) + subprocess.run( + os.path.join(spack_stack_dir, "util", "fetch_cargo_deps.py"), + stdout=logfile, + stderr=logfile, + check=True, + text=True, + ) + subprocess.run( + os.path.join(spack_stack_dir, "util", "fetch_go_deps.py"), + stdout=logfile, + stderr=logfile, + check=True, + text=True, + ) + run_batch_install(deployments_yaml["batch_config"], deployment, env_dir_full_path, specs_str, logfile, logfilepath) + + # Generate modules + print(f"... writing package modules ...") + subprocess.run( + ["spack", "-e", env_dir_full_path, "module", "lmod", "refresh", "--yes-to-all", "--upstream-modules"], + stdout=logfile, + stderr=logfile, + check=True, + text=True, + ) + + # Meta modules + print(f"... writing metamodules ...") + setup_meta_modules() + + logfile.close() + + print(f"... done.") From f4f219085cbeb99af5b503e1d83b15698e128da6 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Thu, 9 Oct 2025 16:40:11 +0000 Subject: [PATCH 08/16] deployments.yaml (acorn): consolidate core and addon sections --- util/acorn/deploy/deployments.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/util/acorn/deploy/deployments.yaml b/util/acorn/deploy/deployments.yaml index a717de8e1..e875d3ef1 100644 --- a/util/acorn/deploy/deployments.yaml +++ b/util/acorn/deploy/deployments.yaml @@ -1,4 +1,4 @@ -core_deployments: +deployments: - template: "unified-dev" compilers: ["oneapi-2024.2.1"] packages_to_install: @@ -6,8 +6,9 @@ core_deployments: - global-workflow-env - gsi-env - madis + walltime: 03:00:00 -#addon_deployments: -# - template: "test-addon" -# compilers: ["oneapi-2024.2.1"] -# upstream: ["unified-dev"] +batch_config: + account: NCEPLIBS-DEV + queue: dev + default_walltime: 02:00:00 From a8cf0113744678cfa861d7b0e8ed822a3ebac47f Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Thu, 9 Oct 2025 16:41:59 +0000 Subject: [PATCH 09/16] .gitignore: deploy_logs/ --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9c064ff8b..38038e6e3 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ __pycache__ cache .vscode doc/build +deploy_logs/ From b4094546b8956729d38bb3e901dbb3f3752f0809 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Thu, 9 Oct 2025 16:43:57 +0000 Subject: [PATCH 10/16] stack_env.py: re-order envrepo logic so template supersedes upstream --- .../jcsda-emc/spack-stack/stack/stack_env.py | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py index 2dabde98f..45917d361 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py @@ -253,14 +253,6 @@ def write(self): if self.site != "none": self._copy_site_includes() - # Copy any envrepo directory associated with the template (i.e., add-on env) - if self.template: - envrepo_path = os.path.join(self.template_path, "envrepo") - if os.path.isdir(envrepo_path): - logging.info(f"Copying envrepo/ from {self.template_path}") - target_envrepo_path = os.path.join(env_dir, "envrepo") - shutil.copytree(envrepo_path, target_envrepo_path, dirs_exist_ok=True) - # Copy common include files self._copy_common_includes() @@ -357,13 +349,14 @@ def write(self): "version of spack-stack, or really, really know what you are doing.") ) new_envrepo = os.path.join(self.env_dir(), "envrepo") + config_envrepo = False for upstream_path in all_upstreams[::-1]: envrepo_path = os.path.realpath(os.path.join(upstream_path, "../envrepo")) if os.path.isdir(envrepo_path): shutil.copytree(envrepo_path, new_envrepo, dirs_exist_ok=True) - if os.path.isdir(new_envrepo): - repo_cfg = "repos:[$env/envrepo]" - spack.config.add(repo_cfg, scope=env_scope) + config_envrepo = True + if config_envrepo: + spack.config.add("repos:[$env/envrepo]", scope=env_scope) if self.modifypkg: logging.info("Creating custom repo with packages %s" % ", ".join(self.modifypkg)) @@ -396,8 +389,16 @@ def write(self): if not pkg_found: logging.warning(f"WARNING: package '{pkg_name}' could not be found") logging.info("Adding custom repo 'envrepo' to env config") - repo_cfg = "repos:[$env/envrepo]" - spack.config.add(repo_cfg, scope=env_scope) + spack.config.add("repos:[$env/envrepo]", scope=env_scope) + + # Copy any envrepo directory associated with the template (i.e., add-on env) + if self.template: + template_envrepo_path = os.path.join(self.template_path, "envrepo") + if os.path.isdir(template_envrepo_path): + logging.info(f"Copying envrepo/ from {self.template_path}") + target_envrepo_path = os.path.join(env_dir, "envrepo") + shutil.copytree(template_envrepo_path, target_envrepo_path, dirs_exist_ok=True) + spack.config.add("repos:[$env/envrepo]", scope=env_scope) # Merge the original spack.yaml template back in # so it has the highest precedence From 81db12b62599b77406fa2adabf4b8a5814f02e20 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Thu, 9 Oct 2025 17:21:22 +0000 Subject: [PATCH 11/16] acorn: update gcc config in oneapi packages.yaml --- .../acorn/packages_intel-19.1.3.304.yaml | 60 ------------------- .../tier1/acorn/packages_oneapi-2024.2.1.yaml | 11 ++-- 2 files changed, 4 insertions(+), 67 deletions(-) delete mode 100644 configs/sites/tier1/acorn/packages_intel-19.1.3.304.yaml diff --git a/configs/sites/tier1/acorn/packages_intel-19.1.3.304.yaml b/configs/sites/tier1/acorn/packages_intel-19.1.3.304.yaml deleted file mode 100644 index 28b327d1a..000000000 --- a/configs/sites/tier1/acorn/packages_intel-19.1.3.304.yaml +++ /dev/null @@ -1,60 +0,0 @@ - packages: - all: - compiler:: [intel@19.1.3.304,intel@2022.0.2.262,gcc@10.2.0] - providers: - mpi:: [cray-mpich@8.1.9] - # Remove the next three lines to switch to intel-oneapi-mkl - blas:: [openblas] - fftw-api:: [fftw] - lapack:: [openblas] - mpi: - buildable: False - cray-mpich: - externals: - - spec: cray-mpich@8.1.9~wrappers - modules: - - libfabric - - craype-network-ofi - - cray-mpich/8.1.9 - - cray-pals - intel-oneapi-mkl: - # Remove buildable: False and configure+uncomment externals section below to use intel-oneapi-mkl - buildable: False - #externals: - #- spec: intel-oneapi-mkl@2022.0.2%intel@2021.5.0 - # prefix: /apps/oneapi - # DH* Remove this section to switch to intel-oneapi-mkl - ectrans: - require:: - - '@1.2.0 ~mkl +fftw' - gsibec: - require:: - - '@1.2.1 ~mkl' - py-numpy: - require:: - - '^[virtuals=lapack,blas] openblas' - - '@1.26 %gcc' - # *DH - eckit: - require: - - any_of: ["@1.23.1"] - when: "%intel@19.1.3.304" - eccodes: - require: - - any_of: ["@2.25.0"] - when: "%intel@19.1.3.304" - message: "2.25.0 is the last version to use C++11 (as opposed to C++17)" - gettext: - require: ['%gcc', '@0.21.1'] - py-scipy: - require: - - any_of: ["@1.10.1"] - when: "%intel@19.1.3.304" - cdo: - require: - - any_of: ["@2.3.0"] - when: "%intel@2022.0.2.262" - message: "2.3.0 is the last version to use C++17" - - any_of: ["@2.0.5"] - when: "%intel@19.1.3.304" - message: "2.0.5 is the last version to use C++14" diff --git a/configs/sites/tier1/acorn/packages_oneapi-2024.2.1.yaml b/configs/sites/tier1/acorn/packages_oneapi-2024.2.1.yaml index cf2b28909..2553756bc 100644 --- a/configs/sites/tier1/acorn/packages_oneapi-2024.2.1.yaml +++ b/configs/sites/tier1/acorn/packages_oneapi-2024.2.1.yaml @@ -31,12 +31,12 @@ packages: buildable: false externals: - spec: gcc@12.1.0 languages:='c,c++,fortran' - prefix: /opt/cray/pe/gcc/12.1.0 + prefix: /opt/cray/pe/gcc/12.1.0/snos/ extra_attributes: compilers: - c: /opt/cray/pe/gcc/12.1.0/bin/gcc - cxx: /opt/cray/pe/gcc/12.1.0/bin/g++ - fortran: /opt/cray/pe/gcc/12.1.0/bin/gfortran + c: /opt/cray/pe/gcc/12.1.0/snos/bin/gcc + cxx: /opt/cray/pe/gcc/12.1.0/snos/bin/g++ + fortran: /opt/cray/pe/gcc/12.1.0/snos/bin/gfortran environment: set: # OpenSUSE on WCOSS2 machines sets CONFIG_SITE so @@ -45,9 +45,6 @@ packages: CONFIG_SITE: '' gcc-runtime: require: '%gcc@12.1.0' - externals: - - spec: gcc-runtime@12.1.0 - prefix: /opt/cray/pe/gcc/12.1.0 ### General package/provider settings & MPI external mpi: buildable: false From c77fba273407f6a438d1a0a310040f1002cc5eed Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Thu, 9 Oct 2025 18:03:00 +0000 Subject: [PATCH 12/16] delete gcc config for acorn --- configs/sites/tier1/acorn/packages_gcc-11.2.0.yaml | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 configs/sites/tier1/acorn/packages_gcc-11.2.0.yaml diff --git a/configs/sites/tier1/acorn/packages_gcc-11.2.0.yaml b/configs/sites/tier1/acorn/packages_gcc-11.2.0.yaml deleted file mode 100644 index b801a9758..000000000 --- a/configs/sites/tier1/acorn/packages_gcc-11.2.0.yaml +++ /dev/null @@ -1,14 +0,0 @@ - packages: - all: - compiler:: [gcc@11.2.0] - providers: - mpi:: [cray-mpich@8.1.9] - mpi: - buildable: False - cray-mpich: - externals: - - spec: cray-mpich@8.1.9~wrappers - modules: - - libfabric - - craype-network-ofi - - cray-mpich/8.1.9 From 922d45b328e9236d11e2315c9bd52704d8a04d9a Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Fri, 10 Oct 2025 16:45:34 +0000 Subject: [PATCH 13/16] fetch_cargo_deps.py: first try to use each spec's rust/cargo; set PATH so it can find rustc --- util/fetch_cargo_deps.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/util/fetch_cargo_deps.py b/util/fetch_cargo_deps.py index f2b8ae0bc..9e1091a94 100755 --- a/util/fetch_cargo_deps.py +++ b/util/fetch_cargo_deps.py @@ -27,6 +27,7 @@ from spack.util.executable import Executable, which from llnl.util.filesystem import working_dir from spack.store import find +from spack.util.environment import EnvironmentModifications from spack.error import SpackError # Load the current environment @@ -75,11 +76,17 @@ pkg = spec.package pkg.do_stage() - if which("cargo"): + cargo_path = os.path.join(spec["rust"].prefix.bin, "cargo") + if os.path.exists(cargo_path): + cargo_bin = spec["rust"].prefix.bin + cargo_exe = Executable(cargo_path) + elif which("cargo"): + cargo_bin = "" cargo_exe = Executable("cargo") else: # cargo/rustup - cargo_path = os.path.join(cargo_home, "bin", "cargo") + cargo_bin = os.path.join(cargo_home, "bin") + cargo_path = os.path.join(cargo_bin, "cargo") print(f"Checking for {cargo_path} ...") if which(cargo_path): print(f" ... found {cargo_path}") @@ -109,7 +116,10 @@ os.chmod(cargo_wrapper, 0o744) cargo_exe = Executable(cargo_wrapper) + env = EnvironmentModifications() + if cargo_bin: + env.prepend_path("PATH", cargo_bin) for root, dirs, files in os.walk(pkg.stage.source_path): if "Cargo.toml" in files: with working_dir(root): - cargo_exe("fetch") + cargo_exe("fetch", env=env) From bad867649e6530b40169e5991058f5190eb8c4e2 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Fri, 10 Oct 2025 16:46:06 +0000 Subject: [PATCH 14/16] deploy.py: set up env for go+rust dep fetching --- util/acorn/deploy/deploy.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/util/acorn/deploy/deploy.py b/util/acorn/deploy/deploy.py index 09a3b6cd9..a76bf1059 100755 --- a/util/acorn/deploy/deploy.py +++ b/util/acorn/deploy/deploy.py @@ -186,8 +186,11 @@ def run_batch_install(batch_config, deployment, env_dir_full_path, specs_str, lo else: if not args.skip_go_rust_handling: run_batch_install(deployments_yaml["batch_config"], deployment, env_dir_full_path, ["rust", "go"], logfile, logfilepath) + shell_env = os.environ.copy() + shell_env["SPACK_ENV"] = env_dir_full_path subprocess.run( os.path.join(spack_stack_dir, "util", "fetch_cargo_deps.py"), + env=shell_env, stdout=logfile, stderr=logfile, check=True, @@ -195,6 +198,7 @@ def run_batch_install(batch_config, deployment, env_dir_full_path, specs_str, lo ) subprocess.run( os.path.join(spack_stack_dir, "util", "fetch_go_deps.py"), + env=shell_env, stdout=logfile, stderr=logfile, check=True, From a71f9dd324f188d7c9d6130e015a8cb2e50cc514 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Fri, 10 Oct 2025 18:26:46 +0000 Subject: [PATCH 15/16] deploy.py: flush output more --- util/acorn/deploy/deploy.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/util/acorn/deploy/deploy.py b/util/acorn/deploy/deploy.py index a76bf1059..6bb9c3951 100755 --- a/util/acorn/deploy/deploy.py +++ b/util/acorn/deploy/deploy.py @@ -186,6 +186,7 @@ def run_batch_install(batch_config, deployment, env_dir_full_path, specs_str, lo else: if not args.skip_go_rust_handling: run_batch_install(deployments_yaml["batch_config"], deployment, env_dir_full_path, ["rust", "go"], logfile, logfilepath) + logfile.flush() shell_env = os.environ.copy() shell_env["SPACK_ENV"] = env_dir_full_path subprocess.run( @@ -196,6 +197,7 @@ def run_batch_install(batch_config, deployment, env_dir_full_path, specs_str, lo check=True, text=True, ) + logfile.flush() subprocess.run( os.path.join(spack_stack_dir, "util", "fetch_go_deps.py"), env=shell_env, @@ -204,7 +206,9 @@ def run_batch_install(batch_config, deployment, env_dir_full_path, specs_str, lo check=True, text=True, ) + logfile.flush() run_batch_install(deployments_yaml["batch_config"], deployment, env_dir_full_path, specs_str, logfile, logfilepath) + logfile.flush() # Generate modules print(f"... writing package modules ...") @@ -215,6 +219,7 @@ def run_batch_install(batch_config, deployment, env_dir_full_path, specs_str, lo check=True, text=True, ) + logfile.flush() # Meta modules print(f"... writing metamodules ...") From 4f7cbc55160c8eefb5397349acd0fa1465fe95e0 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Fri, 10 Oct 2025 18:33:51 +0000 Subject: [PATCH 16/16] deploy.py: don't overwrite pbs pro outputs --- util/acorn/deploy/deploy.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/util/acorn/deploy/deploy.py b/util/acorn/deploy/deploy.py index 6bb9c3951..96c72d750 100755 --- a/util/acorn/deploy/deploy.py +++ b/util/acorn/deploy/deploy.py @@ -81,7 +81,7 @@ def get_create_env_settings(env_dir_basename, deployment, deployments): return config_dict -def run_batch_install(batch_config, deployment, env_dir_full_path, specs_str, logfile, logfilepath): +def run_batch_install(batch_config, deployment, env_dir_full_path, specs_str, logfile, logfilepath, suffix=".batch_install"): if "walltime" in deployment: walltime = deployment["walltime"] elif "default_walltime" in batch_config: @@ -90,8 +90,8 @@ def run_batch_install(batch_config, deployment, env_dir_full_path, specs_str, lo assert False, "Set deployment-specific walltime or batch_config:default_walltime" cmd = [ "qsub", - "-o", logfilepath + ".batch_install", - "-e", logfilepath + ".batch_install", + "-o", logfilepath + suffix, + "-e", logfilepath + suffix, "-A", batch_config["account"], "-q", batch_config["queue"], "-l", "walltime=" + walltime + ",select=1:ncpus=12", @@ -185,7 +185,7 @@ def run_batch_install(batch_config, deployment, env_dir_full_path, specs_str, lo env.install_specs(specs) else: if not args.skip_go_rust_handling: - run_batch_install(deployments_yaml["batch_config"], deployment, env_dir_full_path, ["rust", "go"], logfile, logfilepath) + run_batch_install(deployments_yaml["batch_config"], deployment, env_dir_full_path, ["rust", "go"], logfile, logfilepath, suffix=".rustgo") logfile.flush() shell_env = os.environ.copy() shell_env["SPACK_ENV"] = env_dir_full_path