diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 47a19d437..c4129358d 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -105,7 +105,8 @@ packages: - +tools ecflow: require: - - '@5.11.4' + # ecFlow 5.18.0 supports CMake 4; older releases require CMake 3. + - '@5.18.0' - +ui eckit: require: @@ -161,7 +162,10 @@ packages: - '@2.6.4' fms: require: - - precision=32,64 + - spec: precision=32,64 + when: '@:2025.03' + - spec: precision=mixed + when: '@2025.04:' - +quad_precision - +openmp - +pic @@ -170,6 +174,7 @@ packages: - any_of: - '@=2025.03' - '@=2024.03' + - '@=2026.01.01' - any_of: - +gfs_phys constants=GFS - ~gfs_phys constants=GEOS @@ -196,6 +201,14 @@ packages: gfsio: require: - '@1.4.2' + # py-greenlet@:3.0 requires python@:3.12; @3.1: has no upper cap + py-greenlet: + require: + - '@3.1:' + # 2025.7.0 supports Python 3.13 without Dask 2026's Setuptools-SCM 9 requirement. + py-dask: + require: + - '@2025.7.0' global-workflow-env: require: - '+uwtools' @@ -334,7 +347,6 @@ packages: - ~fortran openmpi: require: - - ~internal-hwloc - +two_level_namespace openssl: require: @@ -343,6 +355,10 @@ packages: p4est: require: - '@2.8.7' + # pmix+docs pulls in py-recommonmark -> py-sphinx@:6.1 -> python@:3.12 + pmix: + require: + - ~docs # Earlier versions of pango don't build on Blueback because of # a bug in finding the correct Python interpreter from spack. pango: @@ -352,10 +368,11 @@ packages: require: - '@2.6.2' - +pnetcdf - # DH* CAN WE UPDATE? NEED A NEW SHARED-INTEL PATCH? + # 1.14.1 fixes a GCC 15 build failure. The old oneAPI shared-intel + # linkage workaround applied only to 1.12.3 and is intentionally not used. parallel-netcdf: require: - - '@1.12.3' + - '@1.14.1' pflogger: require: - +mpi @@ -375,10 +392,10 @@ packages: # Turn off crypt variant for Python; this leads to build errors # with Intel in py-cryptography unless external curl and openssl # are removed, which itself is problematic. + # Note: crypt variant only exists for python@:3.12; python@3.13 removed it entirely. python: require: - - '@3.11' - - ~crypt + - '@3.13' # Not sure about the exact version requirements, but earlier # versions (definitely 4.5.3 and earlier) don't work with newer # py-setuptools (version 68 and later) because of a breaking @@ -386,8 +403,10 @@ packages: py-beautifulsoup4: require: - '@4.12.3:' + # py-cartopy@:0.22 requires py-numpy@:1; @0.24.0 caps python@:3.12; @0.24.1: allows python@3.10:3.14 py-cartopy: require: + - '@0.24.1:' - +plotting # See py-pycparser py-click: @@ -400,10 +419,18 @@ packages: # Pin to version 42 to avoid maturin dependency that breaks Intel oneAPI builds py-cryptography: require: - - '@42' + - '@46' + py-cython: + require: + - '@3.2.4' py-h5py: require: + - '@3.16' - ~mpi + # py-ipykernel@5.x requires python@3.6:3.9; @6.11: has no upper cap + py-ipykernel: + require: + - '@6.11:' # Needed to fix issues of older versions (e.g. 1.13.0) with newer Python and # newer python packages: "ValueError: Unknown classifier in field project.classifiers ..." py-hatchling: @@ -412,7 +439,7 @@ packages: # To avoid duplicate packages py-lxml: require: - - '@5.2.2' + - '@6.0.2' # https://github.com/JCSDA/spack-stack/issues/1276 py-matplotlib: require: @@ -420,26 +447,44 @@ packages: #- '@3.7.4' #- '@3.8.0' # https://github.com/JCSDA/spack-stack/issues/1971 + # py-netcdf4@:1.6 requires py-numpy@:1; @1.7: allows numpy@2 py-netcdf4: require: - ~mpi + - '@1.7:' + # py-pandas@:2.2.1 requires py-numpy@:1; @:2.2.2 requires python@:3.12 + # ~performance disables optional py-bottleneck and py-numexpr dependencies + # (py-numexpr fails to build with oneapi icpx; py-bottleneck pulls in py-maturin/rust) + # Sites with rust/icpx available can override with +performance in their site packages.yaml + py-pandas: + require: + - '@3:' + prefer: + - '~performance' + # 0.11.1 replaces the Versioneer-based build that breaks with Python 3.12+. + py-pandas-datareader: + require: + - '@0.11.1' # To avoid duplicate packages py-numpy: require: - - '@1' + - '@2.1:' # To avoid duplicate packages py-poetry-core: require: - - '@1.7' + - '@2.3.2' # See py-pycparser py-pooch: require: - '@1.7.0' + py-pytest: + require: + - '@9:' # Need older version of py-pycparser until we can update setuputils to >= 77 ? # https://github.com/pydata/xarray/issues/10588 py-pycparser: require: - - '@2.21' + - '@3.0' # Need at least py-pygithub@2.7 for self-hosted runner management py-pygithub: require: @@ -451,7 +496,7 @@ packages: # See py-pycparser py-pyparsing: require: - - '@3.1.2' + - '@3.3.2' # To avoid duplicate packages py-python-dateutil: require: @@ -463,29 +508,42 @@ packages: # To avoid duplicate packages py-ruamel-yaml: require: - - '@0.17.16' + - '@0.18:' + # py-ruamel-yaml-clib@:0.2.8 requires python@:3.12; @0.2.12: allows 3.13 + py-ruamel-yaml-clib: + require: + - '@0.2.12:' # https://github.com/JCSDA/spack-stack/issues/1942 # https://github.com/spack/spack-packages/issues/3695 py-setuptools: require: - - '@73.0.1' + - '@80:' + # py-sphinx@:6.1 requires python@:3.12; @6.2: allows 3.13 + py-sphinx: + require: + - '@6.2:' # See py-pycparser py-sphinxcontrib-bibtex: require: - '@2.5.0' py-torch: require: + - '@2.10:' - +custom-protobuf - ~mkldnn + # PyTorch 2.12 does not honor the recipe's USE_XPU=OFF setting when + # built through PEP 517. With oneAPI, Kineto then enables its XPU/PTI + # plugin, but the installed oneAPI toolkit does not provide PTI CMake + # configuration files. Re-enable once that forwarding issue is fixed. + - ~kineto # To avoid duplicate packages py-urllib3: require: - - '@1.26.20' - # OR - '@1.26.12' ? - # To avoid duplicate packages - py-versioneer: + - '@2.6.3' + # py-xarray@:2024.5 requires py-numpy@:1; @2024.7 caps py-pandas@2.1; @2025.7: allows numpy@2 + pandas@2.2: (pandas@3: ok) + py-xarray: require: - - '@=0.28' + - '@2025.7:' # On a per-site basis, either set qt to buildable: false # or add requirement to build with gcc for Intel oneAPI qt: @@ -528,7 +586,7 @@ packages: - '@10.0.10' uwtools: require: - - '@2.7.2' + - '@2.17.0' w3emc: require: - '@2.13.0' diff --git a/configs/common/packages_oneapi.yaml b/configs/common/packages_oneapi.yaml index ad4ea0e5d..a06621715 100644 --- a/configs/common/packages_oneapi.yaml +++ b/configs/common/packages_oneapi.yaml @@ -59,9 +59,6 @@ packages: oops: require: - +mkl - parallel-netcdf: - require: - - +shared-intel patchelf: require: - '%c=gcc' diff --git a/configs/sites/tier1/discover/README.md b/configs/sites/tier1/discover/README.md index f823e3279..463a1cfa6 100644 --- a/configs/sites/tier1/discover/README.md +++ b/configs/sites/tier1/discover/README.md @@ -1,4 +1,4 @@ -Discover (Tier 1 / JCSDA) +Discover (Tier 1 / GMAO SI Team) --- @@ -12,6 +12,7 @@ Discover (Tier 1 / JCSDA) - [Roles and modes](#roles-and-modes) - [The -H flag](#the--h-flag) - [The -e flag](#the--e-flag) + - [The -L / --disable-locks flag](#the--l---disable-locks-flag) - [The -s flag](#the--s-flag) - [The -a flag](#the--a-flag) - [The -p, -q, and --constraint flags](#the--p--q-and---constraint-flags) @@ -30,17 +31,16 @@ Discover (Tier 1 / JCSDA) ## Overview -This configuration provides the Spack Stack setup for the Tier 1 JCSDA installation on the NASA Discover supercomputer. It is maintained under the JCSDA `/discover/swdev/` space. +This configuration provides the Spack Stack setup for the Tier 1 GMAO SI Team installation on the NASA Discover supercomputer. It is maintained under the GMAO SI Team `/discover/nobackup/projects/gmao/SIteam/` space. The site builds the `unified-dev` environment template with the following compilers: - `gcc@=14.2.0` +- `gcc@=15.2.0` - `oneapi@=2024.2.0` - `oneapi@=2025.3.0` -Module files are managed with `lmod`. Shared caches are located under `/discover/swdev/jcsda/spack-stack/`. - -> **Note:** This site (`discover`) is distinct from the GMAO Tier 2 site (`discover-gmao`). The two sites use different cache locations, compiler sets, and environment templates. If you are doing GMAO-specific work, see `configs/sites/tier2/discover-gmao/README.md`. +Module files are managed with `lmod`. Shared caches are located under `/discover/nobackup/projects/gmao/SIteam/spack-stack/`. --- @@ -73,7 +73,7 @@ git clone --recurse-submodules https://github.com/GMAO-SI-Team/spack-stack.git - Activate the base spack-stack environment from the root of the repository: ```bash -cd spack-stack-dev +cd spack-stack-siteam . ./setup.sh ``` @@ -138,11 +138,28 @@ The `-e` flag tells the script to continue even if the environment directories a --- +### The -L / --disable-locks flag + +The `-L` / `--disable-locks` flag creates a job-local Spack user configuration containing `config:locks:false`, so Spack's package workers inherit the setting without changing the environment YAML. It also sets `--concurrent-packages=1`. Per-package build parallelism is unchanged. Use it only to work around a filesystem lock failure such as `OSError: [Errno 37] No locks available`, and only when this is the sole process modifying the environment's install tree. + +For example, an exclusive SLURM job retry for the oneAPI 2024.2.0 build can use: + +```bash +./util/gmao/batch_install.sh -r dev -m build -H discover -s \ + -C oneapi@=2024.2.0 -e -L +``` + +Do not use `-L` for normal shared-stack builds or when another Spack process might install, uninstall, or otherwise modify the same environment. Locks remain enabled by default. + +--- + ### The -s flag -The `-s` flag submits the `spack install` step to the SLURM scheduler via `salloc` rather than running it directly in your current shell. This is strongly recommended for full stack builds on Discover: +The `-s` flag submits the `spack install` step to the SLURM scheduler via `sbatch` rather than running it directly in your current shell. This is strongly recommended for full stack builds on Discover: - Allocates a single node with 120 tasks +- Limits each Spack package build to 24 jobs; the full-node allocation provides isolation rather than maximum build parallelism +- Does not use `$TSE_TMPDIR`: its inode quota is insufficient for large packages such as Rust; build stages remain in the site-configured project-space cache - Targets Milan nodes (`--constraint=mil`) by default - Walltime: 8 hours - Output is redirected to `spack...log` in the current directory @@ -165,7 +182,7 @@ When the account is `s1873`, the job defaults to `--partition=preops --qos=bench ### The -p, -q, and --constraint flags -Override the SLURM partition, QOS, and node constraint independently. These options only apply to NCCS hosts (`discover`, `discover-gmao`). +Override the SLURM partition, QOS, and node constraint independently: ```bash # Use a different partition @@ -183,6 +200,8 @@ Override the SLURM partition, QOS, and node constraint independently. These opti `-p` and `-q` also accept long-option forms (`--partition=`, `--qos=`). `--constraint` has no short form because `-c` and `-C` are already in use. +These flags override the site defaults regardless of the account. Useful when working under a non-`s1873` account that still has access to specific partitions, QOS levels, or node types. + --- ### Typical workflow @@ -197,7 +216,7 @@ On the very first run, use `-u` to populate the bootstrap, source, and cargo mir ./util/gmao/batch_install.sh -r dev -m build -H discover -s -u ``` -- `-u`: Builds and populates the bootstrap mirror, source cache, and cargo mirror under `/discover/swdev/jcsda/spack-stack/`. Only needed once (or when the caches need to be refreshed). +- `-u`: Builds and populates the bootstrap mirror, source cache, and cargo mirror under `/discover/nobackup/projects/gmao/SIteam/spack-stack/`. Only needed once (or when the caches need to be refreshed). - `-s`: Submits the install step to SLURM. - No `-e` needed since the environment directories do not exist yet. @@ -229,8 +248,8 @@ The script creates one environment per compiler under `envs/` (relative to the r | Mode | Environment names created | |---|---| -| `build` | `ue-gcc-14.2.0-build`, `ue-oneapi-2024.2.0-build`, `ue-oneapi-2025.3.0-build` | -| `install` | `ue-gcc-14.2.0`, `ue-oneapi-2024.2.0`, `ue-oneapi-2025.3.0` | +| `build` | `ue-gcc-14.2.0-build`, `ue-gcc-15.2.0-build`, `ue-oneapi-2024.2.0-build`, `ue-oneapi-2025.3.0-build` | +| `install` | `ue-gcc-14.2.0`, `ue-gcc-15.2.0`, `ue-oneapi-2024.2.0`, `ue-oneapi-2025.3.0` | The `ue-` prefix comes from the `unified-dev` template used by this site. @@ -241,11 +260,11 @@ The `ue-` prefix comes from the `unified-dev` template used by this site. After the `install` step completes, point your shell to the generated module files: ```bash -module use -a /path/to/spack-stack/envs/ue-gcc-14.2.0/install/modulefiles/Core +module use -a /path/to/spack-stack-siteam/envs/ue-gcc-15.2.0/install/modulefiles/Core module load stack-gcc stack-openmpi ``` -Replace `ue-gcc-14.2.0` with the appropriate environment name for your compiler of choice. +Replace `ue-gcc-15.2.0` with the appropriate environment name for your compiler of choice. --- @@ -257,7 +276,7 @@ First, activate spack-stack (required for interactive Spack commands) and activa ```bash . ./setup.sh -cd envs/ue-gcc-14.2.0 +cd envs/ue-gcc-15.2.0 spack env activate -p . ``` @@ -349,7 +368,7 @@ spack buildcache push -u local-binary spack buildcache update-index local-binary ``` -> **Warning:** This writes to the default shared build cache at `/discover/swdev/jcsda/spack-stack/build-cache`. Only do this if you have write access and intend to publish the change. +> **Warning:** This writes to the default shared build cache at `/discover/nobackup/projects/gmao/SIteam/spack-stack/build-cache`. Only do this if you have write access and intend to publish the change. --- @@ -368,7 +387,7 @@ If a package needs a new version or checksum update (e.g., a new upstream releas 3. Activate the relevant environment: ```bash - cd envs/ue-gcc-14.2.0 + cd envs/ue-gcc-15.2.0 spack env activate -p . ``` @@ -409,12 +428,13 @@ The following flags are less commonly used. The descriptions below reflect our b |---|---| | `-c BUILDCACHE_DIR` | Use a custom build cache directory. Required with `-r ops -m build` for operational partners who do not have write access to the shared mirrors. | | `-d ENV_DIRS` | Override the default environment directory (default: `envs/` in the repo root). | -| `-C COMPILERS` | Comma-separated list of compilers to build (e.g., `gcc@=14.2.0,oneapi@=2025.3.0`). Overrides the site defaults. | -| `-p` / `--partition` | Override the SLURM partition. Defaults to `preops` when account is `s1873`, otherwise unset. NCCS hosts only. | -| `-q` / `--qos` | Override the SLURM QOS. Defaults to `benchmark` when account is `s1873`, otherwise unset. NCCS hosts only. | -| `--constraint` | Override the SLURM node constraint. Defaults to `mil` (Milan nodes). No short form. NCCS hosts only. | +| `-C COMPILERS` | Comma-separated list of compilers to build (e.g., `gcc@=15.2.0,oneapi@=2025.3.0`). Overrides the site defaults. | +| `-p` / `--partition` | Override the SLURM partition. Defaults to `preops` when account is `s1873`, otherwise unset. | +| `-q` / `--qos` | Override the SLURM QOS. Defaults to `benchmark` when account is `s1873`, otherwise unset. | +| `--constraint` | Override the SLURM node constraint. Defaults to `mil` (Milan nodes). No short form (`-c`/`-C` are taken). | | `-t` | Run tests for specific third-party dependencies after installation. The list of packages to test is hardcoded in `batch_install.sh`. | | `-u` | Populate bootstrap, source, and cargo mirrors. Requires `-r dev -m build`. Only needed on first run or when refreshing caches. | +| `-L` / `--disable-locks` | Disable Spack install locks and serialize package installs for an isolated recovery run only. Per-package build parallelism is unchanged. Requires assurance that no other process is modifying the same install tree. | --- diff --git a/configs/sites/tier1/discover/config.yaml b/configs/sites/tier1/discover/config.yaml index 7e559f33f..b7f1f5624 100644 --- a/configs/sites/tier1/discover/config.yaml +++ b/configs/sites/tier1/discover/config.yaml @@ -1,2 +1,9 @@ config: build_jobs: 6 + + # Overrides for spack build and staging areas to speed up builds + # and avoid errors with Lustre file locking and xattr issues + build_stage: /discover/nobackup/projects/gmao/SIteam/spack-stack/cache/build_stage + test_stage: /discover/nobackup/projects/gmao/SIteam/spack-stack/cache/test_stage + source_cache: /discover/nobackup/projects/gmao/SIteam/spack-stack/cache/source_cache + misc_cache: /discover/nobackup/projects/gmao/SIteam/spack-stack/cache/misc_cache diff --git a/configs/sites/tier1/discover/mirrors.yaml b/configs/sites/tier1/discover/mirrors.yaml index 1cd8ba7db..9c06031b7 100644 --- a/configs/sites/tier1/discover/mirrors.yaml +++ b/configs/sites/tier1/discover/mirrors.yaml @@ -1,8 +1,8 @@ mirrors: local-source: - url: file:///discover/swdev/jcsda/spack-stack/source-cache + url: file:///discover/nobackup/projects/gmao/SIteam/spack-stack/source-cache binary: false local-binary: - url: file:///discover/swdev/jcsda/spack-stack/build-cache + url: file:///discover/nobackup/projects/gmao/SIteam/spack-stack/build-cache binary: true diff --git a/configs/sites/tier1/discover/modules.yaml b/configs/sites/tier1/discover/modules.yaml index 4e9771f38..348c1b215 100644 --- a/configs/sites/tier1/discover/modules.yaml +++ b/configs/sites/tier1/discover/modules.yaml @@ -2,3 +2,21 @@ modules: default: enable:: - lmod + # RPATH is sufficient for stack libraries. Avoid making every generated + # package module append lib/lib64 to LD_LIBRARY_PATH: a long path adds + # substantial process-launch overhead, especially for Intel MPI. + prefix_inspections:: + bin: + - PATH + man: + - MANPATH + share/man: + - MANPATH + share/aclocal: + - ACLOCAL_PATH + include: + - CPATH + lib/pkgconfig: + - PKG_CONFIG_PATH + lib64/pkgconfig: + - PKG_CONFIG_PATH diff --git a/configs/sites/tier1/discover/packages.yaml b/configs/sites/tier1/discover/packages.yaml index 0f2f94601..310f3d1c8 100644 --- a/configs/sites/tier1/discover/packages.yaml +++ b/configs/sites/tier1/discover/packages.yaml @@ -3,11 +3,22 @@ packages: ewok-env: require: - '+ecflow' + # Completion-index generation can hang during isolated offline builds. + awscli-v2: + require: + - '~autocomplete' # Problems building shared hdf-eos2 with Intel, not needed hdf-eos2: variants: ~shared met: variants: +python +grib2 +graphics +lidar2nc +modis + # mimalloc is vendored and downloaded via FetchContent; fails on air-gapped compute nodes + arrow: + require: + - '~mimalloc' + gdal: + require: + - '~arrow' ### All other external packages listed alphabetically autoconf: externals: @@ -79,6 +90,11 @@ packages: externals: - spec: git-lfs@3.4.0 prefix: /usr/local/other/git-lfs/3.4.0 + gh: + buildable: False + externals: + - spec: gh@2.96.0 + prefix: /discover/swdev/gmao_SIteam/gh gmake: externals: - spec: gmake@4.2.1 @@ -99,6 +115,10 @@ packages: externals: - spec: m4@1.4.18 prefix: /usr + ncurses: + externals: + - spec: ncurses@6.1 +termlib abi=6 + prefix: /usr openssh: externals: - spec: openssh@8.4p1 diff --git a/configs/sites/tier1/discover/packages_gcc-15.2.0.yaml b/configs/sites/tier1/discover/packages_gcc-15.2.0.yaml new file mode 100644 index 000000000..6bb650a8e --- /dev/null +++ b/configs/sites/tier1/discover/packages_gcc-15.2.0.yaml @@ -0,0 +1,60 @@ +packages: + all: + require: + - any_of: ['%gcc@=15.2.0'] + when: '%gcc' + mpi: + buildable: false + require: + - "openmpi@5.0.10" + gcc: + buildable: false + externals: + - spec: gcc@15.2.0 languages:='c,c++,fortran' + prefix: /usr/local/other/gcc/gcc-15.2.0 + modules: + - comp/gcc/15.2.0 + extra_attributes: + compilers: + c: /usr/local/other/gcc/gcc-15.2.0/bin/gcc + cxx: /usr/local/other/gcc/gcc-15.2.0/bin/g++ + fortran: /usr/local/other/gcc/gcc-15.2.0/bin/gfortran + openmpi: + buildable: false + externals: + - spec: openmpi@5.0.10 ~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath+lustre~internal-hwloc+two_level_namespace fabrics=ucx schedulers=slurm %gcc@15.2.0 + prefix: /discover/swdev/gmao_SIteam/MPI/openmpi/5.0.10/gcc-15.2.0 + modules: + - comp/gcc/15.2.0 + - mpi/openmpi/5.0.10/gcc-15.2.0 + extra_attributes: + environment: + set: + OMPI_MCA_orte_tmpdir_base: /tmp + OMP_STACKSIZE: 1G + OMPI_MCA_mca_base_component_show_load_errors: "0" + PMIX_MCA_mca_base_component_show_load_errors: "0" + OMPI_MCA_coll_hcoll_enable: "0" + OPENMPI: /discover/swdev/gmao_SIteam/MPI/openmpi/5.0.10/gcc-15.2.0 + prepend_path: + PATH: /discover/swdev/gmao_SIteam/MPI/openmpi/5.0.10/gcc-15.2.0/bin + LIBRARY_PATH: /discover/swdev/gmao_SIteam/MPI/openmpi/5.0.10/gcc-15.2.0/lib + LD_LIBRARY_PATH: /discover/swdev/gmao_SIteam/MPI/openmpi/5.0.10/gcc-15.2.0/lib + INCLUDE: /discover/swdev/gmao_SIteam/MPI/openmpi/5.0.10/gcc-15.2.0/include + MANPATH: /discover/swdev/gmao_SIteam/MPI/openmpi/5.0.10/gcc-15.2.0 + + # Testing shows we now need to pass in -lm for GCC + p4est: + require: + - 'ldflags="-lm"' + + # GCC 15 defaults to -std=c23 where 'false' is a keyword; parallel-netcdf 1.12.3 + # uses 'false' as an enum constant in ncmpidump.h, so force C11 mode. + parallel-netcdf: + require: + - 'cflags="-std=gnu11"' + + # ncurses 6.5 fails with GCC 15 due to NCURSES_BOOL conversion issues; 6.6 fixes this. + ncurses: + require: + - '@6.6' diff --git a/configs/sites/tier1/discover/packages_oneapi-2024.2.0.yaml b/configs/sites/tier1/discover/packages_oneapi-2024.2.0.yaml index 68ef40e65..3e2157f4a 100644 --- a/configs/sites/tier1/discover/packages_oneapi-2024.2.0.yaml +++ b/configs/sites/tier1/discover/packages_oneapi-2024.2.0.yaml @@ -8,11 +8,12 @@ packages: mpi: buildable: false require: - - "intel-oneapi-mpi@2021.13" + # Select mpiifort for the ifort compiler stack instead of the default mpiifx. + - "intel-oneapi-mpi@2021.13 +classic-names" intel-oneapi-compilers: buildable: false externals: - - spec: intel-oneapi-compilers@2024.2.0 + - spec: intel-oneapi-compilers@2024.2.0 +fix_rt_linkage prefix: /usr/local/intel/oneapi/2024 modules: - comp/intel/2024.2.0 @@ -44,7 +45,7 @@ packages: intel-oneapi-mpi: buildable: false externals: - - spec: intel-oneapi-mpi@2021.13 + - spec: intel-oneapi-mpi@2021.13 +classic-names prefix: /usr/local/intel/oneapi/2024 modules: - mpi/impi/2021.13 @@ -53,3 +54,11 @@ packages: externals: - spec: intel-oneapi-mkl@2024.2 prefix: /usr/local/intel/oneapi/2024 + + llvm: + buildable: false + externals: + - spec: llvm@22.1.7 + prefix: /discover/nobackup/projects/gmao/SIteam/comp/SLES-15/llvm-flang/22.1.7 + modules: + - comp/llvm-flang/22.1.7 diff --git a/configs/sites/tier1/discover/packages_oneapi-2025.3.0.yaml b/configs/sites/tier1/discover/packages_oneapi-2025.3.0.yaml index c0fdbedfe..3013a07b0 100644 --- a/configs/sites/tier1/discover/packages_oneapi-2025.3.0.yaml +++ b/configs/sites/tier1/discover/packages_oneapi-2025.3.0.yaml @@ -12,7 +12,7 @@ packages: intel-oneapi-compilers: buildable: false externals: - - spec: intel-oneapi-compilers@2025.3.0 + - spec: intel-oneapi-compilers@2025.3.0 +fix_rt_linkage prefix: /usr/local/intel/oneapi/2025 modules: - comp/intel/2025.3.0 @@ -47,6 +47,8 @@ packages: - spec: intel-oneapi-mpi@2021.17 prefix: /usr/local/intel/oneapi/2025 modules: + # mpi/impi/2021.17 requires this compiler module to be loaded first. + - comp/intel/2025.3.0 - mpi/impi/2021.17 intel-oneapi-mkl: buildable: false diff --git a/configs/sites/tier1/nas-toss5/README.md b/configs/sites/tier1/nas-toss5/README.md index bec7fd3dd..689100538 100644 --- a/configs/sites/tier1/nas-toss5/README.md +++ b/configs/sites/tier1/nas-toss5/README.md @@ -1,6 +1,6 @@ # How to Build **spack-stack** at NAS on TOSS5 -This guide documents how to build **spack-stack** on NASA NAS TOSS5 systems. The previous install process used three stages across compute and login nodes, but recent testing shows installation can now be done directly on a compute node. +This guide documents how to build **spack-stack** on NASA NAS TOSS5 systems. Use `util/gmao/batch_install.sh` for normal full-stack builds: it performs the internet-facing preparation on the login node and submits the offline install to PBS. --- @@ -11,6 +11,7 @@ This guide documents how to build **spack-stack** on NASA NAS TOSS5 systems. The - [Clone spack-stack](#clone-spack-stack) - [Obtain an Interactive Compute Node](#obtain-an-interactive-compute-node) - [Setup spack-stack](#setup-spack-stack) +- [Using batch_install.sh (recommended)](#using-batch_installsh-recommended) - [Create Environments](#create-environments) - [oneAPI - ifx Environment](#oneapi---ifx-environment) - [oneAPI - ifort Environment](#oneapi---ifort-environment) @@ -90,6 +91,48 @@ cd spack-stack-2.1.0 --- +## Using batch_install.sh (recommended) + +Run the script from a NAS TOSS5 login node. It creates and concretizes the environments, populates required mirrors on the login node, and—with `-s`—submits the install to a PBS compute node. The generated PBS job exports the shared Cargo mirror and sets `CARGO_NET_OFFLINE=true`, so Rust builds do not try to access the network. + +Start by checking the command that would run: + +```bash +./util/gmao/batch_install.sh -r dev -m build -H nas-toss5 -s -n +``` + +For a new environment, use `-u` once to populate the bootstrap, source, and Cargo mirrors before the offline build job starts: + +```bash +./util/gmao/batch_install.sh -r dev -m build -H nas-toss5 -s -u +``` + +For a failed build or a later retry, retain the environment with `-e` and normally omit `-u`: + +```bash +./util/gmao/batch_install.sh -r dev -m build -H nas-toss5 -s -e +``` + +After the build cache is populated, generate usable Tcl module files with the install mode: + +```bash +./util/gmao/batch_install.sh -r dev -m install -H nas-toss5 -s -e +``` + +Use `-C oneapi@=2024.2.0` or `-C oneapi@=2025.3.0` to work on only one compiler environment. `-o` (or `--concretize-only`) stops after concretization, and `--help` displays all accepted options. + +With `-s`, NAS TOSS5 requests one 240-task Turin PBS node for eight hours; the script caps each package build at 24 jobs and normally permits two independent package builds. If the filesystem reports `OSError: [Errno 37] No locks available`, an exclusive recovery job may use `-L` (or `--disable-locks`): it creates a job-local Spack configuration with locking disabled and serializes package builds. Do not use `-L` while any other process can modify the same install tree. + +The `-p`, `-q`, and `--constraint` options are Discover-specific and do not apply to NAS PBS jobs. + +--- + +## Manual workflow (fallback) + +The sections below describe the older manual workflow. Prefer `batch_install.sh` above for full-stack builds, especially when Rust packages are present. + +--- + ## Create Environments You only need to create each environment once. @@ -161,11 +204,11 @@ spack mirror create -a -d /swbuild/gmao_SIteam/spack-stack/source-cache Rust packages frequently require network access during build. Pre-fetch their dependencies: ```bash -export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror +export SPACK_CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror ../../util/fetch_cargo_deps.py ``` -> ⚠️ **Set `CARGO_HOME` on compute nodes** before running `spack install`. +> ⚠️ **Set `SPACK_CARGO_HOME` and `CARGO_NET_OFFLINE=true` on compute nodes** before running `spack install`. --- @@ -174,7 +217,8 @@ export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror Run installation on a **compute node**: ```bash -export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror +export SPACK_CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror +export CARGO_NET_OFFLINE=true spack install -j 16 --verbose --fail-fast --show-log-on-error --no-check-signature 2>&1 | tee log.install ; bell ``` @@ -229,17 +273,19 @@ Typical commands were: ```bash # Step 1 (compute node) -export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror +export SPACK_CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror +export CARGO_NET_OFFLINE=true spack install -j 16 --verbose --fail-fast --show-log-on-error --no-check-signature \ --only dependencies py-cryptography py-maturin py-rpds-py ecflow 2>&1 | tee log.install.deps-for-rust-and-ecflow ; bell # Step 2 (athfe login node) -export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror +export SPACK_CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror spack install -j 2 -p 1 --verbose --fail-fast --show-log-on-error --no-check-signature \ py-cryptography py-maturin py-rpds-py ecflow 2>&1 | tee log.install.rust-and-ecflow ; bell # Step 3 (compute node) -export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror +export SPACK_CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror +export CARGO_NET_OFFLINE=true spack install -j 16 --verbose --fail-fast --show-log-on-error --no-check-signature 2>&1 | tee log.install.after-cargo ; bell ``` diff --git a/configs/sites/tier1/nas-toss5/config.yaml b/configs/sites/tier1/nas-toss5/config.yaml index 714b3c10e..c948f3c36 100644 --- a/configs/sites/tier1/nas-toss5/config.yaml +++ b/configs/sites/tier1/nas-toss5/config.yaml @@ -3,7 +3,7 @@ config: # Overrides for spack build and staging areas to speed up builds # and avoid errors with Lustre file locking and xattr issues - build_stage: /swbuild/gmao_SIteam/spack-stack/cache/build_stage - test_stage: /swbuild/gmao_SIteam/spack-stack/cache/test_stage - source_cache: /swbuild/gmao_SIteam/spack-stack/cache/source_cache - misc_cache: /swbuild/gmao_SIteam/spack-stack/cache/misc_cache + build_stage: /swbuild/gmao_SIteam/spack-stack/cache-toss5/build_stage + test_stage: /swbuild/gmao_SIteam/spack-stack/cache-toss5/test_stage + source_cache: /swbuild/gmao_SIteam/spack-stack/cache-toss5/source_cache + misc_cache: /swbuild/gmao_SIteam/spack-stack/cache-toss5/misc_cache diff --git a/configs/sites/tier1/nas-toss5/modules.yaml b/configs/sites/tier1/nas-toss5/modules.yaml index aeb254c2e..69496b50a 100644 --- a/configs/sites/tier1/nas-toss5/modules.yaml +++ b/configs/sites/tier1/nas-toss5/modules.yaml @@ -2,6 +2,24 @@ modules: default: enable:: - tcl + # RPATH is sufficient for stack libraries. Avoid making every generated + # package module append lib/lib64 to LD_LIBRARY_PATH: a long path adds + # substantial process-launch overhead, especially for MPI launchers. + prefix_inspections:: + bin: + - PATH + man: + - MANPATH + share/man: + - MANPATH + share/aclocal: + - ACLOCAL_PATH + include: + - CPATH + lib/pkgconfig: + - PKG_CONFIG_PATH + lib64/pkgconfig: + - PKG_CONFIG_PATH tcl: include: # List of packages for which we need modules that are blacklisted by default diff --git a/configs/sites/tier1/nas-toss5/packages.yaml b/configs/sites/tier1/nas-toss5/packages.yaml index f0ad64588..9fd38fbb9 100644 --- a/configs/sites/tier1/nas-toss5/packages.yaml +++ b/configs/sites/tier1/nas-toss5/packages.yaml @@ -9,6 +9,13 @@ packages: variants: +python +grib2 +graphics +lidar2nc +modis zlib-api: buildable: False + # mimalloc is vendored and downloaded via FetchContent; fails on air-gapped compute nodes + arrow: + require: + - '~mimalloc' + gdal: + require: + - '~arrow' ### All other external packages listed alphabetically automake: externals: @@ -52,8 +59,8 @@ packages: prefix: /usr gh: externals: - - spec: gh@2.83.1 - prefix: /nobackup/gmao_SIteam/gh/2.83.1 + - spec: gh@2.96.0 + prefix: /nobackup/gmao_SIteam/gh/2.96.0 git: buildable: False externals: diff --git a/configs/sites/tier1/nas-toss5/packages_gcc-14.2.1.yaml b/configs/sites/tier1/nas-toss5/packages_gcc-14.2.1.yaml index c67a29de2..f85a848dc 100644 --- a/configs/sites/tier1/nas-toss5/packages_gcc-14.2.1.yaml +++ b/configs/sites/tier1/nas-toss5/packages_gcc-14.2.1.yaml @@ -16,7 +16,6 @@ packages: - cray-libpals/1.7.1 - cray-pals/1.7.1 - craype-x86-turin - - libfabric/12.0 - craype-network-ofi - cray-dsmml/0.3.1 - PrgEnv-gnu/8.6.0 @@ -35,9 +34,12 @@ packages: - cray-libpals/1.7.1 - cray-pals/1.7.1 - craype-x86-turin - - libfabric/12.0 - craype-network-ofi - cray-dsmml/0.3.1 + # This site module sets the MPICH wrapper compilers and requires PrgEnv-gnu. + # Spack may activate this MPI external without first activating the GCC external. + - PrgEnv-gnu/8.6.0 + - comp-gcc/14.2.1 - cray-mpich/8.1.33 - mpi/mpich-gfortran extra_attributes: diff --git a/configs/sites/tier1/nas-toss5/packages_gcc-15.3.0.yaml b/configs/sites/tier1/nas-toss5/packages_gcc-15.3.0.yaml new file mode 100644 index 000000000..c669314a3 --- /dev/null +++ b/configs/sites/tier1/nas-toss5/packages_gcc-15.3.0.yaml @@ -0,0 +1,64 @@ +packages: + all: + require: + - any_of: ['%gcc@=15.3.0'] + when: '%gcc' + mpi: + buildable: false + require: + - "cray-mpich@8.1.33" + gcc: + buildable: false + externals: + - spec: gcc@15.3.0 languages:='c,c++,fortran' + prefix: /nobackup/gmao_SIteam/gcc/gcc-15.3.0-TOSS5 + modules: + - cray-libpals/1.7.1 + - cray-pals/1.7.1 + - craype-x86-turin + - craype-network-ofi + - cray-dsmml/0.3.1 + - PrgEnv-gnu/8.6.0 + - comp-gcc/15.3.0 + extra_attributes: + compilers: + c: /nobackup/gmao_SIteam/gcc/gcc-15.3.0-TOSS5/bin/gcc + cxx: /nobackup/gmao_SIteam/gcc/gcc-15.3.0-TOSS5/bin/g++ + fortran: /nobackup/gmao_SIteam/gcc/gcc-15.3.0-TOSS5/bin/gfortran + cray-mpich: + buildable: false + externals: + - spec: cray-mpich@8.1.33 + prefix: /opt/cray/pe/mpich/8.1.33/ofi/gnu/11.2 + modules: + - cray-libpals/1.7.1 + - cray-pals/1.7.1 + - craype-x86-turin + - craype-network-ofi + - cray-dsmml/0.3.1 + # This site module sets the MPICH wrapper compilers and requires PrgEnv-gnu. + # Spack may activate this MPI external without first activating the GCC external. + - PrgEnv-gnu/8.6.0 + - comp-gcc/15.3.0 + - cray-mpich/8.1.33 + - mpi/mpich-gfortran + extra_attributes: + environment: + set: + MPICH_CC: gcc + MPICH_CXX: g++ + MPICH_FC: gfortran + CC: gcc + CXX: g++ + FC: gfortran + + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2024.2 + prefix: /nasa/intel/oneapi-test + + # Testing shows we now need to pass in -lm for GCC + p4est: + require: + - 'ldflags="-lm"' diff --git a/configs/sites/tier1/nas-toss5/packages_oneapi-2024.2.0.yaml b/configs/sites/tier1/nas-toss5/packages_oneapi-2024.2.0.yaml index 7b53a10d5..b06ce774b 100644 --- a/configs/sites/tier1/nas-toss5/packages_oneapi-2024.2.0.yaml +++ b/configs/sites/tier1/nas-toss5/packages_oneapi-2024.2.0.yaml @@ -11,13 +11,12 @@ packages: - "cray-mpich@8.1.33" intel-oneapi-compilers: externals: - - spec: intel-oneapi-compilers@2024.2.0 + - spec: intel-oneapi-compilers@2024.2.0 +fix_rt_linkage prefix: /nasa/intel/oneapi-test modules: - cray-libpals/1.7.1 - cray-pals/1.7.1 - craype-x86-turin - - libfabric/12.0 - craype-network-ofi - cray-dsmml/0.3.1 - PrgEnv-intel/8.6.0 @@ -29,24 +28,24 @@ packages: fortran: /nasa/intel/oneapi-test/compiler/2024.2/bin/ifort environment: prepend_path: - PATH: /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5/bin - CPATH: /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5/include - LD_LIBRARY_PATH: '/nasa/intel/oneapi-test/compiler/2024.2/lib/:/nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5/lib64' + PATH: /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5-rebuild/bin + CPATH: /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5-rebuild/include + LD_LIBRARY_PATH: '/nasa/intel/oneapi-test/compiler/2024.2/lib/:/nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5-rebuild/lib64' extra_rpaths: - - /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5/lib64 - - /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5/lib + - /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5-rebuild/lib64 + - /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5-rebuild/lib gcc: buildable: false externals: - spec: gcc@12.5.0 languages:='c,c++' - prefix: /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5 + prefix: /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5-rebuild modules: - - comp-gcc/12.5.0-TOSS5 + - comp-gcc/12.5.0-rebuild extra_attributes: compilers: - c: /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5/bin/gcc - cxx: /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5/bin/g++ - fortran: /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5/bin/gfortran + c: /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5-rebuild/bin/gcc + cxx: /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5-rebuild/bin/g++ + fortran: /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5-rebuild/bin/gfortran cray-mpich: buildable: false externals: @@ -56,7 +55,6 @@ packages: - cray-libpals/1.7.1 - cray-pals/1.7.1 - craype-x86-turin - - libfabric/12.0 - craype-network-ofi - cray-dsmml/0.3.1 - cray-mpich/8.1.33 diff --git a/configs/sites/tier1/nas-toss5/packages_oneapi-2025.3.0.yaml b/configs/sites/tier1/nas-toss5/packages_oneapi-2025.3.0.yaml index f843a28b4..5023c1e5c 100644 --- a/configs/sites/tier1/nas-toss5/packages_oneapi-2025.3.0.yaml +++ b/configs/sites/tier1/nas-toss5/packages_oneapi-2025.3.0.yaml @@ -11,13 +11,12 @@ packages: - "cray-mpich@8.1.33" intel-oneapi-compilers: externals: - - spec: intel-oneapi-compilers@2025.3.0 + - spec: intel-oneapi-compilers@2025.3.0 +fix_rt_linkage prefix: /nobackup/gmao_SIteam/intel/oneapi modules: - cray-libpals/1.7.1 - cray-pals/1.7.1 - craype-x86-turin - - libfabric/12.0 - craype-network-ofi - cray-dsmml/0.3.1 - PrgEnv-intel/8.6.0 @@ -29,24 +28,24 @@ packages: fortran: /nobackup/gmao_SIteam/intel/oneapi/compiler/2025.3/bin/ifx environment: prepend_path: - PATH: /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5/bin - CPATH: /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5/include - LD_LIBRARY_PATH: '/nobackup/gmao_SIteam/intel/oneapi/compiler/2025.3/lib/:/nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5/lib64' + PATH: /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5-rebuild/bin + CPATH: /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5-rebuild/include + LD_LIBRARY_PATH: '/nobackup/gmao_SIteam/intel/oneapi/compiler/2025.3/lib/:/nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5-rebuild/lib64' extra_rpaths: - - /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5/lib64 - - /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5/lib + - /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5-rebuild/lib64 + - /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5-rebuild/lib gcc: buildable: false externals: - spec: gcc@12.5.0 languages:='c,c++' - prefix: /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5 + prefix: /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5-rebuild modules: - - comp-gcc/12.5.0-TOSS5 + - comp-gcc/12.5.0-rebuild extra_attributes: compilers: - c: /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5/bin/gcc - cxx: /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5/bin/g++ - fortran: /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5/bin/gfortran + c: /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5-rebuild/bin/gcc + cxx: /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5-rebuild/bin/g++ + fortran: /nobackup/gmao_SIteam/gcc/gcc-12.5.0-TOSS5-rebuild/bin/gfortran cray-mpich: buildable: false externals: @@ -56,7 +55,6 @@ packages: - cray-libpals/1.7.1 - cray-pals/1.7.1 - craype-x86-turin - - libfabric/12.0 - craype-network-ofi - cray-dsmml/0.3.1 - cray-mpich/8.1.33 diff --git a/configs/sites/tier1/nas/README.md b/configs/sites/tier1/nas/README.md index 7e1e99367..ddfc1cfb6 100644 --- a/configs/sites/tier1/nas/README.md +++ b/configs/sites/tier1/nas/README.md @@ -1,6 +1,6 @@ # How to Build **spack-stack** at NAS -This guide documents how to build **spack-stack** on NASA NAS systems. The previous install process used three stages across compute and login nodes, but recent testing shows installation can now be done directly on a compute node. +This guide documents how to build **spack-stack** on NASA NAS systems. Use `util/gmao/batch_install.sh` for normal full-stack builds: it performs the internet-facing preparation on the login node and submits the offline install to PBS. --- @@ -11,6 +11,7 @@ This guide documents how to build **spack-stack** on NASA NAS systems. The previ - [Clone spack-stack](#clone-spack-stack) - [Obtain an Interactive Compute Node](#obtain-an-interactive-compute-node) - [Setup spack-stack](#setup-spack-stack) +- [Using batch_install.sh (recommended)](#using-batch_installsh-recommended) - [Create Environments](#create-environments) - [oneAPI - ifx Environment](#oneapi---ifx-environment) - [oneAPI - ifort Environment](#oneapi---ifort-environment) @@ -72,7 +73,7 @@ git clone --recurse-submodules https://github.com/JCSDA/spack-stack.git -b spack NAS login nodes allow only **2 processes**, so use: ```bash -qsub -I -V -X -l select=1:ncpus=128:mpiprocs=128:model=mil_ait -l walltime=12:00:00 -W group_list=s1873 -m b -N Interactive +qsub -I -V -X -l select=1:ncpus=128:mpiprocs=128:model=mil_ait -q normal -l walltime=12:00:00 -W group_list=s1873 -m b -N Interactive ``` This gives a **Milan** compute node for up to 12 hours. @@ -92,6 +93,48 @@ cd spack-stack-2.1.0 --- +## Using batch_install.sh (recommended) + +Run the script from a NAS login node. It creates and concretizes the environments, populates required mirrors on the login node, and—with `-s`—submits the install to a PBS compute node. The generated PBS job exports the shared Cargo mirror and sets `CARGO_NET_OFFLINE=true`, so Rust builds do not try to access the network. + +Start by checking the command that would run: + +```bash +./util/gmao/batch_install.sh -r dev -m build -H nas -s -n +``` + +For a new environment, use `-u` once to populate the bootstrap, source, and Cargo mirrors before the offline build job starts: + +```bash +./util/gmao/batch_install.sh -r dev -m build -H nas -s -u +``` + +For a failed build or a later retry, retain the environment with `-e` and normally omit `-u`: + +```bash +./util/gmao/batch_install.sh -r dev -m build -H nas -s -e +``` + +After the build cache is populated, generate usable Tcl module files with the install mode: + +```bash +./util/gmao/batch_install.sh -r dev -m install -H nas -s -e +``` + +Use `-C oneapi@=2024.2.0` or `-C oneapi@=2025.3.0` to work on only one compiler environment. `-o` (or `--concretize-only`) stops after concretization, and `--help` displays all accepted options. + +With `-s`, NAS requests one 120-task PBS node for eight hours; the script caps each package build at 24 jobs and normally permits two independent package builds. If the filesystem reports `OSError: [Errno 37] No locks available`, an exclusive recovery job may use `-L` (or `--disable-locks`): it creates a job-local Spack configuration with locking disabled and serializes package builds. Do not use `-L` while any other process can modify the same install tree. + +The `-p`, `-q`, and `--constraint` options are Discover-specific and do not apply to NAS PBS jobs. + +--- + +## Manual workflow (fallback) + +The sections below describe the older manual workflow. Prefer `batch_install.sh` above for full-stack builds, especially when Rust packages are present. + +--- + ## Create Environments You only need to create each environment once. @@ -113,8 +156,8 @@ cd envs/ue-oneapi-2024.2.0 ### GCC Environment ```bash -spack stack create env --name ue-gcc-14.2.0 --template unified-dev --site nas --compiler=gcc-14.2.0 -cd envs/ue-gcc-14.2.0 +spack stack create env --name ue-gcc-15.2.0 --template unified-dev --site nas --compiler=gcc-15.2.0 +cd envs/ue-gcc-15.2.0 ``` --- @@ -163,11 +206,11 @@ spack mirror create -a -d /swbuild/gmao_SIteam/spack-stack/source-cache Rust packages frequently require network access during build. Pre-fetch their dependencies: ```bash -export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror +export SPACK_CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror ../../util/fetch_cargo_deps.py ``` -> ⚠️ **Set `CARGO_HOME` on compute nodes** before running `spack install`. +> ⚠️ **Set `SPACK_CARGO_HOME` and `CARGO_NET_OFFLINE=true` on compute nodes** before running `spack install`. --- @@ -176,7 +219,8 @@ export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror Run installation on a **compute node**: ```bash -export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror +export SPACK_CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror +export CARGO_NET_OFFLINE=true spack install -j 16 --verbose --fail-fast --show-log-on-error --no-check-signature 2>&1 | tee log.install ; bell ``` @@ -231,17 +275,19 @@ Typical commands were: ```bash # Step 1 (compute node) -export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror +export SPACK_CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror +export CARGO_NET_OFFLINE=true spack install -j 16 --verbose --fail-fast --show-log-on-error --no-check-signature \ --only dependencies py-cryptography py-maturin py-rpds-py ecflow 2>&1 | tee log.install.deps-for-rust-and-ecflow ; bell # Step 2 (afe login node) -export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror +export SPACK_CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror spack install -j 2 -p 1 --verbose --fail-fast --show-log-on-error --no-check-signature \ py-cryptography py-maturin py-rpds-py ecflow 2>&1 | tee log.install.rust-and-ecflow ; bell # Step 3 (compute node) -export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror +export SPACK_CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror +export CARGO_NET_OFFLINE=true spack install -j 16 --verbose --fail-fast --show-log-on-error --no-check-signature 2>&1 | tee log.install.after-cargo ; bell ``` diff --git a/configs/sites/tier1/nas/config.yaml b/configs/sites/tier1/nas/config.yaml index 714b3c10e..55a6d7304 100644 --- a/configs/sites/tier1/nas/config.yaml +++ b/configs/sites/tier1/nas/config.yaml @@ -3,7 +3,7 @@ config: # Overrides for spack build and staging areas to speed up builds # and avoid errors with Lustre file locking and xattr issues - build_stage: /swbuild/gmao_SIteam/spack-stack/cache/build_stage - test_stage: /swbuild/gmao_SIteam/spack-stack/cache/test_stage - source_cache: /swbuild/gmao_SIteam/spack-stack/cache/source_cache - misc_cache: /swbuild/gmao_SIteam/spack-stack/cache/misc_cache + build_stage: /swbuild/gmao_SIteam/spack-stack/cache-toss4/build_stage + test_stage: /swbuild/gmao_SIteam/spack-stack/cache-toss4/test_stage + source_cache: /swbuild/gmao_SIteam/spack-stack/cache-toss4/source_cache + misc_cache: /swbuild/gmao_SIteam/spack-stack/cache-toss4/misc_cache diff --git a/configs/sites/tier1/nas/modules.yaml b/configs/sites/tier1/nas/modules.yaml index e4c2e5300..d98104ae0 100644 --- a/configs/sites/tier1/nas/modules.yaml +++ b/configs/sites/tier1/nas/modules.yaml @@ -2,3 +2,21 @@ modules: default: enable:: - tcl + # RPATH is sufficient for stack libraries. Avoid making every generated + # package module append lib/lib64 to LD_LIBRARY_PATH: a long path adds + # substantial process-launch overhead, especially for MPI launchers. + prefix_inspections:: + bin: + - PATH + man: + - MANPATH + share/man: + - MANPATH + share/aclocal: + - ACLOCAL_PATH + include: + - CPATH + lib/pkgconfig: + - PKG_CONFIG_PATH + lib64/pkgconfig: + - PKG_CONFIG_PATH diff --git a/configs/sites/tier1/nas/packages.yaml b/configs/sites/tier1/nas/packages.yaml index fe53f7030..42f4a4f6a 100644 --- a/configs/sites/tier1/nas/packages.yaml +++ b/configs/sites/tier1/nas/packages.yaml @@ -9,6 +9,13 @@ packages: variants: +python +grib2 +graphics +lidar2nc +modis zlib-api: buildable: False + # mimalloc is vendored and downloaded via FetchContent; fails on air-gapped compute nodes + arrow: + require: + - '~mimalloc' + gdal: + require: + - '~arrow' ### All other external packages listed alphabetically automake: externals: @@ -52,8 +59,8 @@ packages: prefix: /usr gh: externals: - - spec: gh@2.78.0 - prefix: /nobackup/gmao_SIteam/gh/2.78.0 + - spec: gh@2.96.0 + prefix: /nobackup/gmao_SIteam/gh/2.96.0 git: buildable: False externals: diff --git a/configs/sites/tier1/nas/packages_gcc-14.2.0.yaml b/configs/sites/tier1/nas/packages_gcc-15.2.0.yaml similarity index 69% rename from configs/sites/tier1/nas/packages_gcc-14.2.0.yaml rename to configs/sites/tier1/nas/packages_gcc-15.2.0.yaml index 9dea27286..c3264cbf9 100644 --- a/configs/sites/tier1/nas/packages_gcc-14.2.0.yaml +++ b/configs/sites/tier1/nas/packages_gcc-15.2.0.yaml @@ -1,7 +1,7 @@ packages: all: require: - - any_of: ['%gcc@=14.2.0'] + - any_of: ['%gcc@=15.2.0'] when: '%gcc' mpi: buildable: false @@ -10,26 +10,26 @@ packages: gcc: buildable: false externals: - - spec: gcc@14.2.0 languages:='c,c++,fortran' - prefix: /nobackup/gmao_SIteam/gcc/gcc-14.2.0 + - spec: gcc@15.2.0 languages:='c,c++,fortran' + prefix: /nobackup/gmao_SIteam/gcc/gcc-15.2.0 modules: - - comp-gcc/14.2.0 + - comp-gcc/15.2.0 extra_attributes: compilers: - c: /nobackup/gmao_SIteam/gcc/gcc-14.2.0/bin/gcc - cxx: /nobackup/gmao_SIteam/gcc/gcc-14.2.0/bin/g++ - fortran: /nobackup/gmao_SIteam/gcc/gcc-14.2.0/bin/gfortran + c: /nobackup/gmao_SIteam/gcc/gcc-15.2.0/bin/gcc + cxx: /nobackup/gmao_SIteam/gcc/gcc-15.2.0/bin/g++ + fortran: /nobackup/gmao_SIteam/gcc/gcc-15.2.0/bin/gfortran openmpi: buildable: false externals: - - spec: openmpi@4.1.8 ~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath+lustre~internal-hwloc+two_level_namespace fabrics=ucx schedulers=tm %gcc@14.2.0 - prefix: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-14.2.0 + - spec: openmpi@4.1.8 ~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath+lustre~internal-hwloc+two_level_namespace fabrics=ucx schedulers=tm %gcc@15.2.0 + prefix: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-15.2.0 modules: - - mpi-openmpi/4.1.8/gcc/14.2.0 + - mpi-openmpi/4.1.8/gcc/15.2.0 extra_attributes: environment: set: - OPAL_PREFIX: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-14.2.0 + OPAL_PREFIX: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-15.2.0 UCX_MAX_EAGER_LANES: 1 UCX_MAX_RNDV_LANES: 1 OMPI_MCA_pml: ucx @@ -40,14 +40,14 @@ packages: OMPI_MCA_opal_warn_on_missing_libcuda: 0 UCX_IB_SUBNET_PREFIX: fec0:0000:0000:0008 OMPI_MCA_shmem_mmap_enable_nfs_warning: 0 - OPENMPI: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-14.2.0 + OPENMPI: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-15.2.0 prepend_path: - PATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-14.2.0/bin - LIBRARY_PATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-14.2.0/lib - LD_LIBRARY_PATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-14.2.0/lib - CPATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-14.2.0/include - FPATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-14.2.0/include - MANPATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-14.2.0/share/man + PATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-15.2.0/bin + LIBRARY_PATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-15.2.0/lib + LD_LIBRARY_PATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-15.2.0/lib + CPATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-15.2.0/include + FPATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-15.2.0/include + MANPATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-15.2.0/share/man # Testing shows we now need to pass in -lm for GCC p4est: diff --git a/configs/sites/tier1/nas/packages_oneapi-2024.2.0.yaml b/configs/sites/tier1/nas/packages_oneapi-2024.2.0.yaml index 9746c2af2..bdb63b2bb 100644 --- a/configs/sites/tier1/nas/packages_oneapi-2024.2.0.yaml +++ b/configs/sites/tier1/nas/packages_oneapi-2024.2.0.yaml @@ -8,11 +8,12 @@ packages: mpi: buildable: false require: - - "mpt@2.30" + # Select mpiifort for the ifort compiler stack instead of the default mpiifx. + - "intel-oneapi-mpi@2021.13 +classic-names" intel-oneapi-compilers: buildable: false externals: - - spec: intel-oneapi-compilers@2024.2.0 + - spec: intel-oneapi-compilers@2024.2.0 +fix_rt_linkage prefix: /nobackup/gmao_SIteam/intel/oneapi modules: - comp-intel/2024.2.0-ifort @@ -41,13 +42,13 @@ packages: c: /nobackup/gmao_SIteam/gcc/gcc-12.3.0-TOSS4/bin/gcc cxx: /nobackup/gmao_SIteam/gcc/gcc-12.3.0-TOSS4/bin/g++ fortran: /nobackup/gmao_SIteam/gcc/gcc-12.3.0-TOSS4/bin/gfortran - mpt: + intel-oneapi-mpi: buildable: false externals: - - spec: mpt@2.30 - prefix: /nasa/hpe/mpt/2.30_rhel810 + - spec: intel-oneapi-mpi@2021.13 +classic-names + prefix: /nobackup/gmao_SIteam/intel/oneapi modules: - - mpi-hpe/mpt.2.30 + - mpi-impi/2021.13 intel-oneapi-mkl: buildable: false externals: diff --git a/configs/sites/tier1/nas/packages_oneapi-2025.3.0.yaml b/configs/sites/tier1/nas/packages_oneapi-2025.3.0.yaml index f6e0e34f3..0caa0a775 100644 --- a/configs/sites/tier1/nas/packages_oneapi-2025.3.0.yaml +++ b/configs/sites/tier1/nas/packages_oneapi-2025.3.0.yaml @@ -12,10 +12,10 @@ packages: intel-oneapi-compilers: buildable: false externals: - - spec: intel-oneapi-compilers@2025.3.0 + - spec: intel-oneapi-compilers@2025.3.0 +fix_rt_linkage prefix: /nobackup/gmao_SIteam/intel/oneapi modules: - - comp-intel/2025.3.0 + - comp-intel/2025.3 extra_attributes: compilers: c: /nobackup/gmao_SIteam/intel/oneapi/compiler/2025.3/bin/icx diff --git a/configs/sites/tier1/wcoss2/setup.sh b/configs/sites/tier1/wcoss2/setup.sh index 966263370..217c0ac69 100644 --- a/configs/sites/tier1/wcoss2/setup.sh +++ b/configs/sites/tier1/wcoss2/setup.sh @@ -2,4 +2,4 @@ module load cray-python/3.11.7 ## Go, Rust repo setup for fetcher scripts export GOMODCACHE=${SPACK_STACK_DIR}/cache/go -export CARGO_HOME=${SPACK_STACK_DIR}/cache/cargo +export SPACK_CARGO_HOME=${SPACK_STACK_DIR}/cache/cargo diff --git a/configs/sites/tier2/bucy/README.md b/configs/sites/tier2/bucy/README.md new file mode 100644 index 000000000..a670cd4ad --- /dev/null +++ b/configs/sites/tier2/bucy/README.md @@ -0,0 +1,34 @@ +# Bucy + +Bucy is an RHEL 8.10 x86_64 Tier 2 site using Lmod. The first configuration +builds `geos-dev` for GCC, Intel oneAPI, and LLVM Flang, and `geos-dev-nag` +for NAG. Compiler and MPI installations are external site software; Spack +builds the scientific stack and generates the user-facing Lmod modules. + +The initial compiler matrix is: + +- `gcc@=15.2.0` with OpenMPI 5.0.4 +- `gcc@=16.1.0` with OpenMPI 5.0.10 +- `oneapi@=2024.2.0` with Intel MPI 2021.13 +- `oneapi@=2025.3.0` with Intel MPI 2021.17 +- `nag@=7.2.43` with the patched OpenMPI 5.0.3 +- `llvm@=22.1.0` with OpenMPI 5.0.10 + +The NAG license is not stored in this repository. Before building +`geos-dev-nag`, ensure `NAG_KUSARI_FILE` points to the site license, or load +the site NAG module that sets it: + +```bash +export NAG_KUSARI_FILE=/ford1/share/gmao_SIteam/nag/nag.key +``` + +For a direct local build, use: + +```bash +./util/gmao/batch_install.sh -m local -H bucy -u +``` + +The shared installation, source, bootstrap, and cache locations are under +`/ford1/share/gmao_SIteam/spack-stack`. Tcl module generation is intentionally +not enabled in this first pass; it can be added after the Lmod hierarchy is +validated. diff --git a/configs/sites/tier2/bucy/compilers.yaml b/configs/sites/tier2/bucy/compilers.yaml new file mode 100644 index 000000000..06beaebe3 --- /dev/null +++ b/configs/sites/tier2/bucy/compilers.yaml @@ -0,0 +1,79 @@ +compilers: +- compiler: + spec: gcc@=15.2.0 + paths: + cc: /ford1/share/gmao_SIteam/gcc/gcc-15.2.0/bin/gcc + cxx: /ford1/share/gmao_SIteam/gcc/gcc-15.2.0/bin/g++ + f77: /ford1/share/gmao_SIteam/gcc/gcc-15.2.0/bin/gfortran + fc: /ford1/share/gmao_SIteam/gcc/gcc-15.2.0/bin/gfortran + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] +- compiler: + spec: gcc@=16.1.0 + paths: + cc: /ford1/share/gmao_SIteam/gcc/gcc-16.1.0/bin/gcc + cxx: /ford1/share/gmao_SIteam/gcc/gcc-16.1.0/bin/g++ + f77: /ford1/share/gmao_SIteam/gcc/gcc-16.1.0/bin/gfortran + fc: /ford1/share/gmao_SIteam/gcc/gcc-16.1.0/bin/gfortran + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] +- compiler: + spec: oneapi@=2024.2.0 + paths: + cc: /ford1/share/gmao_SIteam/intel/oneapi/compiler/2024.2/bin/icx + cxx: /ford1/share/gmao_SIteam/intel/oneapi/compiler/2024.2/bin/icpx + f77: /ford1/share/gmao_SIteam/intel/oneapi/compiler/2024.2/bin/ifort + fc: /ford1/share/gmao_SIteam/intel/oneapi/compiler/2024.2/bin/ifort + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: + - /ford1/local/gcc/gcc-12.1.0/lib64 +- compiler: + spec: oneapi@=2025.3.0 + paths: + cc: /ford1/share/gmao_SIteam/intel/oneapi/2025/compiler/2025.3/bin/icx + cxx: /ford1/share/gmao_SIteam/intel/oneapi/2025/compiler/2025.3/bin/icpx + f77: /ford1/share/gmao_SIteam/intel/oneapi/2025/compiler/2025.3/bin/ifx + fc: /ford1/share/gmao_SIteam/intel/oneapi/2025/compiler/2025.3/bin/ifx + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: + - /ford1/share/gmao_SIteam/gcc/gcc-13.3.0/lib64 +- compiler: + spec: nag@=7.2.43 + paths: + cc: /ford1/local/gcc/gcc-12.1.0/bin/gcc + cxx: /ford1/local/gcc/gcc-12.1.0/bin/g++ + f77: /ford1/share/gmao_SIteam/nag/7.2.43/bin/nagfor + fc: /ford1/share/gmao_SIteam/nag/7.2.43/bin/nagfor + operating_system: rhel8 + target: x86_64 + modules: [] + environment: + set: + NAG_KUSARI_FILE: /ford1/share/gmao_SIteam/nag/nag.key + extra_rpaths: + - /ford1/local/gcc/gcc-12.1.0/lib64 + - /ford1/share/gmao_SIteam/nag/7.2.43/lib +- compiler: + spec: llvm@=22.1.0 + paths: + cc: /ford1/share/gmao_SIteam/llvm-flang/22.1.0/bin/clang + cxx: /ford1/share/gmao_SIteam/llvm-flang/22.1.0/bin/clang++ + f77: /ford1/share/gmao_SIteam/llvm-flang/22.1.0/bin/flang + fc: /ford1/share/gmao_SIteam/llvm-flang/22.1.0/bin/flang + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] diff --git a/configs/sites/tier2/bucy/config.yaml b/configs/sites/tier2/bucy/config.yaml new file mode 100644 index 000000000..55e461c74 --- /dev/null +++ b/configs/sites/tier2/bucy/config.yaml @@ -0,0 +1,6 @@ +config: + build_jobs: 6 + build_stage: /ford1/share/gmao_SIteam/spack-stack/cache/build_stage + test_stage: /ford1/share/gmao_SIteam/spack-stack/cache/test_stage + source_cache: /ford1/share/gmao_SIteam/spack-stack/cache/source_cache + misc_cache: /ford1/share/gmao_SIteam/spack-stack/cache/misc_cache diff --git a/configs/sites/tier2/bucy/mirrors.yaml b/configs/sites/tier2/bucy/mirrors.yaml new file mode 100644 index 000000000..6c8f27a82 --- /dev/null +++ b/configs/sites/tier2/bucy/mirrors.yaml @@ -0,0 +1,7 @@ +mirrors: + local-source: + url: file:///ford1/share/gmao_SIteam/spack-stack/source-mirror + binary: false + local-binary: + url: file:///ford1/share/gmao_SIteam/spack-stack/build-mirror + binary: true diff --git a/configs/sites/tier2/bucy/modules.yaml b/configs/sites/tier2/bucy/modules.yaml new file mode 100644 index 000000000..4e9771f38 --- /dev/null +++ b/configs/sites/tier2/bucy/modules.yaml @@ -0,0 +1,4 @@ +modules: + default: + enable:: + - lmod diff --git a/configs/sites/tier2/bucy/packages.yaml b/configs/sites/tier2/bucy/packages.yaml new file mode 100644 index 000000000..dce71047e --- /dev/null +++ b/configs/sites/tier2/bucy/packages.yaml @@ -0,0 +1,8 @@ +packages: + zlib-api: + require: [zlib] + zlib: + buildable: false + externals: + - spec: zlib@1.2.11 + prefix: /usr diff --git a/configs/sites/tier2/bucy/packages_gcc-15.2.0.yaml b/configs/sites/tier2/bucy/packages_gcc-15.2.0.yaml new file mode 100644 index 000000000..21a855f8c --- /dev/null +++ b/configs/sites/tier2/bucy/packages_gcc-15.2.0.yaml @@ -0,0 +1,13 @@ +packages: + all: + require: + - any_of: ['%gcc@=15.2.0'] + when: '%gcc' + mpi: + buildable: false + require: [openmpi@=5.0.4] + openmpi: + buildable: false + externals: + - spec: openmpi@=5.0.4 + prefix: /ford1/share/gmao_SIteam/MPI/openmpi/5.0.4/gcc-15.2.0 diff --git a/configs/sites/tier2/bucy/packages_gcc-16.1.0.yaml b/configs/sites/tier2/bucy/packages_gcc-16.1.0.yaml new file mode 100644 index 000000000..a21cdba13 --- /dev/null +++ b/configs/sites/tier2/bucy/packages_gcc-16.1.0.yaml @@ -0,0 +1,13 @@ +packages: + all: + require: + - any_of: ['%gcc@=16.1.0'] + when: '%gcc' + mpi: + buildable: false + require: [openmpi@=5.0.10] + openmpi: + buildable: false + externals: + - spec: openmpi@=5.0.10 + prefix: /ford1/share/gmao_SIteam/MPI/openmpi/5.0.10/gcc-16.1.0 diff --git a/configs/sites/tier2/bucy/packages_llvm-22.1.0.yaml b/configs/sites/tier2/bucy/packages_llvm-22.1.0.yaml new file mode 100644 index 000000000..1c94cb925 --- /dev/null +++ b/configs/sites/tier2/bucy/packages_llvm-22.1.0.yaml @@ -0,0 +1,13 @@ +packages: + all: + require: + - any_of: ['%llvm@=22.1.0'] + when: '%llvm' + mpi: + buildable: false + require: [openmpi@=5.0.10] + openmpi: + buildable: false + externals: + - spec: openmpi@=5.0.10 + prefix: /ford1/share/gmao_SIteam/MPI/openmpi/5.0.10/llvm-flang-22.1.0 diff --git a/configs/sites/tier2/bucy/packages_nag-7.2.43.yaml b/configs/sites/tier2/bucy/packages_nag-7.2.43.yaml new file mode 100644 index 000000000..f20c07df2 --- /dev/null +++ b/configs/sites/tier2/bucy/packages_nag-7.2.43.yaml @@ -0,0 +1,25 @@ +packages: + c: + require: [gcc@=12.1.0] + cxx: + require: [gcc@=12.1.0] + fortran: + require: [nag@=7.2.43] + mpi: + buildable: false + require: [openmpi@=5.0.3] + gcc: + buildable: false + externals: + - spec: gcc@=12.1.0 languages:='c,c++' + prefix: /ford1/local/gcc/gcc-12.1.0 + nag: + buildable: false + externals: + - spec: nag@=7.2.43 languages:='fortran' + prefix: /ford1/share/gmao_SIteam/nag/7.2.43 + openmpi: + buildable: false + externals: + - spec: openmpi@=5.0.3 + prefix: /ford1/share/gmao_SIteam/MPI/openmpi/5.0.3-GillesNAGPatch/nag-7.2.43 diff --git a/configs/sites/tier2/bucy/packages_oneapi-2024.2.0.yaml b/configs/sites/tier2/bucy/packages_oneapi-2024.2.0.yaml new file mode 100644 index 000000000..2eee0cba9 --- /dev/null +++ b/configs/sites/tier2/bucy/packages_oneapi-2024.2.0.yaml @@ -0,0 +1,13 @@ +packages: + all: + require: + - any_of: ['%oneapi@=2024.2.0'] + when: '%oneapi' + mpi: + buildable: false + require: [intel-oneapi-mpi@=2021.13] + intel-oneapi-mpi: + buildable: false + externals: + - spec: intel-oneapi-mpi@=2021.13 + prefix: /ford1/share/gmao_SIteam/intel/oneapi/mpi/2021.13 diff --git a/configs/sites/tier2/bucy/packages_oneapi-2025.3.0.yaml b/configs/sites/tier2/bucy/packages_oneapi-2025.3.0.yaml new file mode 100644 index 000000000..6c33f6a20 --- /dev/null +++ b/configs/sites/tier2/bucy/packages_oneapi-2025.3.0.yaml @@ -0,0 +1,13 @@ +packages: + all: + require: + - any_of: ['%oneapi@=2025.3.0'] + when: '%oneapi' + mpi: + buildable: false + require: [intel-oneapi-mpi@=2021.17] + intel-oneapi-mpi: + buildable: false + externals: + - spec: intel-oneapi-mpi@=2021.17 + prefix: /ford1/share/gmao_SIteam/intel/oneapi/2025/mpi/2021.17 diff --git a/configs/sites/tier2/discover-gmao/README.md b/configs/sites/tier2/discover-gmao/README.md index ed8b466be..df8d1f344 100644 --- a/configs/sites/tier2/discover-gmao/README.md +++ b/configs/sites/tier2/discover-gmao/README.md @@ -12,6 +12,7 @@ Discover GMAO - [Roles and modes](#roles-and-modes) - [The -H flag](#the--h-flag) - [The -e flag](#the--e-flag) + - [The -L / --disable-locks flag](#the--l---disable-locks-flag) - [The -s flag](#the--s-flag) - [The -a flag](#the--a-flag) - [The -p, -q, and --constraint flags](#the--p--q-and---constraint-flags) @@ -38,6 +39,8 @@ The site builds the `geos-dev` environment template with the following compilers - `oneapi@=2024.2.0` - `oneapi@=2025.3.0` +It also builds the NAG-specific `geos-dev-nag` template with `nag@=7.2.7238`. + Module files are managed with `lmod`. Shared caches are located under `/discover/nobackup/projects/gmao/SIteam/spack-stack/`. --- @@ -136,12 +139,29 @@ The `-e` flag tells the script to continue even if the environment directories a --- +### The -L / --disable-locks flag + +The `-L` / `--disable-locks` flag creates a job-local Spack user configuration containing `config:locks:false`, so Spack's package workers inherit the setting without changing the environment YAML. It also sets `--concurrent-packages=1`. Per-package build parallelism is unchanged. Use it only to work around a filesystem lock failure such as `OSError: [Errno 37] No locks available`, and only when this is the sole process modifying the environment's install tree. + +For example, an exclusive retry can use: + +```bash +./util/gmao/batch_install.sh -r dev -m build -H discover-gmao -s \ + -C oneapi@=2024.2.0 -e -L +``` + +Do not use `-L` for normal shared-stack builds or when another Spack process might install, uninstall, or otherwise modify the same environment. Locks remain enabled by default. + +--- + ### The -s flag -The `-s` flag submits the `spack install` step to the SLURM scheduler via `salloc` rather than running it directly in your current shell. This is strongly recommended for full stack builds on Discover: +The `-s` flag submits the `spack install` step to the SLURM scheduler via `sbatch` rather than running it directly in your current shell. This is strongly recommended for full stack builds on Discover: - Allocates a single node with 120 tasks -- Targets Milan nodes (`--constraint=mil`) +- Limits each Spack package build to 24 jobs; the full-node allocation provides isolation rather than maximum build parallelism +- Does not use `$TSE_TMPDIR`: its inode quota is insufficient for large packages such as Rust; build stages remain in the site-configured project-space cache +- Targets Milan nodes (`--constraint=mil`) by default - Walltime: 8 hours - Output is redirected to `spack...log` in the current directory @@ -229,8 +249,8 @@ The script creates one environment per compiler under `envs/` (relative to the r | Mode | Environment names created | |---|---| -| `build` | `ge-gcc-15.2.0-build`, `ge-oneapi-2024.2.0-build`, `ge-oneapi-2025.3.0-build` | -| `install` | `ge-gcc-15.2.0`, `ge-oneapi-2024.2.0`, `ge-oneapi-2025.3.0` | +| `build` | `ge-gcc-15.2.0-build`, `ge-oneapi-2024.2.0-build`, `ge-oneapi-2025.3.0-build`, `ge-nag-7.2.7238-build` | +| `install` | `ge-gcc-15.2.0`, `ge-oneapi-2024.2.0`, `ge-oneapi-2025.3.0`, `ge-nag-7.2.7238` | --- @@ -411,6 +431,8 @@ The following flags are less commonly used. The descriptions below reflect our b | `-p` / `--partition` | Override the SLURM partition. Defaults to `preops` when account is `s1873`, otherwise unset. | | `-q` / `--qos` | Override the SLURM QOS. Defaults to `benchmark` when account is `s1873`, otherwise unset. | | `--constraint` | Override the SLURM node constraint. Defaults to `mil` (Milan nodes). No short form (`-c`/`-C` are taken). | +| `-L` / `--disable-locks` | Disable Spack install locks and serialize package installs. Use only for an exclusive recovery job after a filesystem lock failure. | +| `-o` / `--concretize-only` | Concretize environments and stop before installation. | | `-t` | Run tests for specific third-party dependencies after installation. The list of packages to test is hardcoded in `batch_install.sh`. | | `-u` | Populate bootstrap, source, and cargo mirrors. Requires `-r dev -m build`. Only needed on first run or when refreshing caches. | diff --git a/configs/sites/tier2/discover-gmao/modules.yaml b/configs/sites/tier2/discover-gmao/modules.yaml index 4e9771f38..348c1b215 100644 --- a/configs/sites/tier2/discover-gmao/modules.yaml +++ b/configs/sites/tier2/discover-gmao/modules.yaml @@ -2,3 +2,21 @@ modules: default: enable:: - lmod + # RPATH is sufficient for stack libraries. Avoid making every generated + # package module append lib/lib64 to LD_LIBRARY_PATH: a long path adds + # substantial process-launch overhead, especially for Intel MPI. + prefix_inspections:: + bin: + - PATH + man: + - MANPATH + share/man: + - MANPATH + share/aclocal: + - ACLOCAL_PATH + include: + - CPATH + lib/pkgconfig: + - PKG_CONFIG_PATH + lib64/pkgconfig: + - PKG_CONFIG_PATH diff --git a/configs/sites/tier2/discover-gmao/packages.yaml b/configs/sites/tier2/discover-gmao/packages.yaml index 0f2f94601..1f4cd897a 100644 --- a/configs/sites/tier2/discover-gmao/packages.yaml +++ b/configs/sites/tier2/discover-gmao/packages.yaml @@ -3,6 +3,10 @@ packages: ewok-env: require: - '+ecflow' + # Completion-index generation can hang during isolated offline builds. + awscli-v2: + require: + - '~autocomplete' # Problems building shared hdf-eos2 with Intel, not needed hdf-eos2: variants: ~shared @@ -79,6 +83,11 @@ packages: externals: - spec: git-lfs@3.4.0 prefix: /usr/local/other/git-lfs/3.4.0 + gh: + buildable: False + externals: + - spec: gh@2.96.0 + prefix: /discover/swdev/gmao_SIteam/gh gmake: externals: - spec: gmake@4.2.1 @@ -99,6 +108,10 @@ packages: externals: - spec: m4@1.4.18 prefix: /usr + ncurses: + externals: + - spec: ncurses@6.1 +termlib abi=6 + prefix: /usr openssh: externals: - spec: openssh@8.4p1 diff --git a/configs/sites/tier2/discover-gmao/packages_gcc-15.2.0.yaml b/configs/sites/tier2/discover-gmao/packages_gcc-15.2.0.yaml index cc82d25e6..6bb650a8e 100644 --- a/configs/sites/tier2/discover-gmao/packages_gcc-15.2.0.yaml +++ b/configs/sites/tier2/discover-gmao/packages_gcc-15.2.0.yaml @@ -25,15 +25,16 @@ packages: - spec: openmpi@5.0.10 ~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath+lustre~internal-hwloc+two_level_namespace fabrics=ucx schedulers=slurm %gcc@15.2.0 prefix: /discover/swdev/gmao_SIteam/MPI/openmpi/5.0.10/gcc-15.2.0 modules: + - comp/gcc/15.2.0 - mpi/openmpi/5.0.10/gcc-15.2.0 extra_attributes: environment: set: OMPI_MCA_orte_tmpdir_base: /tmp OMP_STACKSIZE: 1G - OMPI_MCA_mca_base_component_show_load_errors: 0 - PMIX_MCA_mca_base_component_show_load_errors: 0 - OMPI_MCA_coll_hcoll_enable: 0 + OMPI_MCA_mca_base_component_show_load_errors: "0" + PMIX_MCA_mca_base_component_show_load_errors: "0" + OMPI_MCA_coll_hcoll_enable: "0" OPENMPI: /discover/swdev/gmao_SIteam/MPI/openmpi/5.0.10/gcc-15.2.0 prepend_path: PATH: /discover/swdev/gmao_SIteam/MPI/openmpi/5.0.10/gcc-15.2.0/bin diff --git a/configs/sites/tier2/discover-gmao/packages_nag-7.2.7238.yaml b/configs/sites/tier2/discover-gmao/packages_nag-7.2.7238.yaml new file mode 100644 index 000000000..0f6cbd843 --- /dev/null +++ b/configs/sites/tier2/discover-gmao/packages_nag-7.2.7238.yaml @@ -0,0 +1,83 @@ +packages: + # Discover's NAG programming environment uses GCC for C/C++ and NAG for + # Fortran. Keep the language requirements explicit rather than attaching a + # single compiler constraint to every package. + c: + require: [gcc@=12.3.0] + cxx: + require: [gcc@=12.3.0] + fortran: + require: [nag@=7.2.7238] + mpi: + buildable: false + require: + - openmpi@=4.1.6 + + gcc: + buildable: false + externals: + - spec: gcc@=12.3.0 languages:='c,c++' + prefix: /usr/local/other/gcc/12.3.0 + modules: + - comp/gcc/12.3.0 + extra_attributes: + compilers: + c: /usr/local/other/gcc/12.3.0/bin/gcc + cxx: /usr/local/other/gcc/12.3.0/bin/g++ + + nag: + buildable: false + externals: + - spec: nag@=7.2.7238 languages:='fortran' + prefix: /discover/nobackup/projects/gmao/SIteam/comp/SLES-15/nag/7.2.38 + modules: + - comp/gcc/12.3.0 + - comp/nag/7.2-7238 + extra_attributes: + compilers: + fortran: /discover/nobackup/projects/gmao/SIteam/comp/SLES-15/nag/7.2.38/bin/nagfor + + openmpi: + buildable: false + externals: + # This OpenMPI was built with GCC 12.3 C/C++ compilers and NAG Fortran. + # Do not constrain it to either compiler in the spec: the language + # requirements above describe the mixed toolchain accurately. + - spec: openmpi@=4.1.6+fortran~cxx~cxx_exceptions~cuda~java~memchecker+pmi~static~wrapper-rpath+two_level_namespace schedulers=slurm + prefix: /discover/swdev/gmao_SIteam/MPI/openmpi/4.1.6/nag_7.2.7238-gcc_12.3.0 + modules: + - comp/gcc/12.3.0 + - comp/nag/7.2-7238 + - mpi/openmpi/4.1.6/nag_7.2.7238-gcc_12.3.0 + extra_attributes: + environment: + set: + OMPI_MCA_orte_tmpdir_base: /tmp + OMP_STACKSIZE: 1G + OMPI_MCA_mca_base_component_show_load_errors: 0 + PMIX_MCA_mca_base_component_show_load_errors: 0 + OPENMPI: /discover/swdev/gmao_SIteam/MPI/openmpi/4.1.6/nag_7.2.7238-gcc_12.3.0 + prepend_path: + PATH: /discover/swdev/gmao_SIteam/MPI/openmpi/4.1.6/nag_7.2.7238-gcc_12.3.0/bin + LIBRARY_PATH: /discover/swdev/gmao_SIteam/MPI/openmpi/4.1.6/nag_7.2.7238-gcc_12.3.0/lib + LD_LIBRARY_PATH: /discover/swdev/gmao_SIteam/MPI/openmpi/4.1.6/nag_7.2.7238-gcc_12.3.0/lib + INCLUDE: /discover/swdev/gmao_SIteam/MPI/openmpi/4.1.6/nag_7.2.7238-gcc_12.3.0/include + MANPATH: /discover/swdev/gmao_SIteam/MPI/openmpi/4.1.6/nag_7.2.7238-gcc_12.3.0/share/man + + # NAG needs explicit preprocessing and type-safety flags for netCDF-Fortran. + netcdf-fortran: + require: + - 'fflags="-fpp -mismatch_all"' + + # The shared-library link path in Parallel-NetCDF 1.12.3 inherits + # -pthread through libtool. Its NAG handling forwards one copy directly to + # ld, which rejects it. The GEOS NAG environment only needs static PNetCDF. + parallel-netcdf: + require: + - '~shared' + + # The OpenBLAS shared-library CMake path passes GCC-specific flags that NAG + # does not support. Static-only avoids that path. + openblas: + require: + - '~shared' diff --git a/configs/sites/tier2/discover-gmao/packages_oneapi-2024.2.0.yaml b/configs/sites/tier2/discover-gmao/packages_oneapi-2024.2.0.yaml index 68ef40e65..3e2157f4a 100644 --- a/configs/sites/tier2/discover-gmao/packages_oneapi-2024.2.0.yaml +++ b/configs/sites/tier2/discover-gmao/packages_oneapi-2024.2.0.yaml @@ -8,11 +8,12 @@ packages: mpi: buildable: false require: - - "intel-oneapi-mpi@2021.13" + # Select mpiifort for the ifort compiler stack instead of the default mpiifx. + - "intel-oneapi-mpi@2021.13 +classic-names" intel-oneapi-compilers: buildable: false externals: - - spec: intel-oneapi-compilers@2024.2.0 + - spec: intel-oneapi-compilers@2024.2.0 +fix_rt_linkage prefix: /usr/local/intel/oneapi/2024 modules: - comp/intel/2024.2.0 @@ -44,7 +45,7 @@ packages: intel-oneapi-mpi: buildable: false externals: - - spec: intel-oneapi-mpi@2021.13 + - spec: intel-oneapi-mpi@2021.13 +classic-names prefix: /usr/local/intel/oneapi/2024 modules: - mpi/impi/2021.13 @@ -53,3 +54,11 @@ packages: externals: - spec: intel-oneapi-mkl@2024.2 prefix: /usr/local/intel/oneapi/2024 + + llvm: + buildable: false + externals: + - spec: llvm@22.1.7 + prefix: /discover/nobackup/projects/gmao/SIteam/comp/SLES-15/llvm-flang/22.1.7 + modules: + - comp/llvm-flang/22.1.7 diff --git a/configs/sites/tier2/discover-gmao/packages_oneapi-2025.3.0.yaml b/configs/sites/tier2/discover-gmao/packages_oneapi-2025.3.0.yaml index c0fdbedfe..3013a07b0 100644 --- a/configs/sites/tier2/discover-gmao/packages_oneapi-2025.3.0.yaml +++ b/configs/sites/tier2/discover-gmao/packages_oneapi-2025.3.0.yaml @@ -12,7 +12,7 @@ packages: intel-oneapi-compilers: buildable: false externals: - - spec: intel-oneapi-compilers@2025.3.0 + - spec: intel-oneapi-compilers@2025.3.0 +fix_rt_linkage prefix: /usr/local/intel/oneapi/2025 modules: - comp/intel/2025.3.0 @@ -47,6 +47,8 @@ packages: - spec: intel-oneapi-mpi@2021.17 prefix: /usr/local/intel/oneapi/2025 modules: + # mpi/impi/2021.17 requires this compiler module to be loaded first. + - comp/intel/2025.3.0 - mpi/impi/2021.17 intel-oneapi-mkl: buildable: false diff --git a/configs/sites/tier2/macos.gmao/README.md b/configs/sites/tier2/macos.gmao/README.md index 78f3661b3..9e107362a 100644 --- a/configs/sites/tier2/macos.gmao/README.md +++ b/configs/sites/tier2/macos.gmao/README.md @@ -10,8 +10,7 @@ Default macOS for GMAO - [Setup spack-stack](#setup-spack-stack) - [Using batch\_install.sh script](#using-batch_installsh-script) - [First run](#first-run) - - [Subsequent builds](#subsequent-builds) - - [Installing from build caches](#installing-from-build-caches) + - [Subsequent local installs](#subsequent-local-installs) - [Loading the stack](#loading-the-stack) - [Building the stack by hand](#building-the-stack-by-hand) - [Create Environments](#create-environments) @@ -89,7 +88,7 @@ cd spack-stack-dev ## Using batch_install.sh script -The `batch_install.sh` script automates the creation of environments (e.g., `gcc` and `clang` builds), populating your Spack bootstrap caches, and running the Spack installation pipeline. +The `batch_install.sh` script automates environment creation, local mirror setup, package installation, and module generation. On macOS, use its one-step `local` mode: it installs directly into the final environment and publishes the resulting binaries to your private local buildcache for reuse by later installs. ### Usage help @@ -106,44 +105,37 @@ The script will automatically detect your currently active version of Apple Clan If your NAG compiler is installed in a non-standard location and not in your `PATH`, you can explicitly provide its path using the `-N` flag: ```bash -./util/gmao/batch_install.sh -N /path/to/your/nag/bin/nagfor -r dev -m build -H macos.gmao -e +./util/gmao/batch_install.sh -N /path/to/your/nag/bin/nagfor -m local -H macos.gmao -e ``` If you want to explicitly override the compilers built by the script entirely, use the `-C` flag with a comma-separated list of Spack compiler specs: ```bash -./util/gmao/batch_install.sh -C "gcc@=15.3.0,nag@=7.2.7243" -r dev -m build -H macos.gmao -e +./util/gmao/batch_install.sh -C "gcc@=15.3.0,nag@=7.2.7243" -m local -H macos.gmao -e ``` *Note: For every compiler you specify, you must have a corresponding `packages_-.yaml.template` file in this site directory.* -### First run (Building Bootstrap and Source Caches) +### First run -When you first run the script on a new machine, you need to use the `-u` option. This tells Spack to build the bootstrap mirror and source caches locally before attempting to register them. +On a new machine, use `-u` to populate private bootstrap, source, and Cargo mirrors before installation. ```bash -./util/gmao/batch_install.sh -r dev -m build -H macos.gmao -u -e +./util/gmao/batch_install.sh -m local -H macos.gmao -u ``` -- `-r dev -m build`: Sets the developer mode and tells Spack to build the environments. +- `-m local`: Performs a one-step workstation install. It reuses matching packages from the local cache, builds missing packages in the final environment, publishes the resulting binaries to the local cache, and generates modules and meta-modules when complete. - `-H macos.gmao`: Overrides hostname detection. This is required to force the script to use this specific generic macOS site, avoiding issues where VPNs or routers mask the real hostname. -- `-u`: Updates and populates the bootstrap/source caches. -- `-e`: Allows continuing builds in existing environments (prevents the script from failing if the environment directories were just created). +- `-u`: Updates and populates the private bootstrap, source, and Cargo mirrors. -### Subsequent builds +Use `-e` only to resume an existing environment after a failed or interrupted installation. -Once the initial caches are set up, you can run builds without the `-u` flag: +### Subsequent local installs -```bash -./util/gmao/batch_install.sh -r dev -m build -H macos.gmao -e -``` - -### Installing from build caches - -If you are just installing environments using already populated build caches (the typical workflow for users after the initial maintainer setup), use `-m install`: +Once the local mirrors are set up, omit `-u`: ```bash -./util/gmao/batch_install.sh -r dev -m install -H macos.gmao -e +./util/gmao/batch_install.sh -m local -H macos.gmao -e ``` ### Generating `.yaml.generated` files @@ -154,21 +146,21 @@ To prevent cluttering the site configuration directory, the script writes these Once the installation and module generation are complete, you can point your shell to the newly built modules. OpenMPI is built specifically with `~two_level_namespace` to support flat namespace linking required by GEOS. -To ensure executables (like MAPL tests) running under macOS System Integrity Protection (SIP) can properly resolve runtime paths (RPATHs) when using compilers like NAG, the `macos.gmao` site-level `modules.yaml` configuration explicitly permits exporting `DYLD_LIBRARY_PATH` and `DYLD_FALLBACK_LIBRARY_PATH` variables into your environment when loading modules. +macOS System Integrity Protection (SIP) removes `DYLD_*` variables from child processes. The module configuration therefore also preserves module-derived library paths in `SPACK_DYLD_FALLBACK_LIBRARY_PATH`. GEOS run scripts can restore that value to `DYLD_FALLBACK_LIBRARY_PATH` immediately before launching the model. -Depending on which compiler you built, you will need to add the correct `install/modulefiles/Core` directory to your `module use` path. +Depending on which compiler you built, add its `modules/Core` directory to your `module use` path. #### Loading the GCC Stack ```bash -module use -a /path/to/spack-stack/envs/ge-gcc-15.3.0-build/install/modulefiles/Core +module use -a /path/to/spack-stack/envs/ge-gcc-15.3.0/modules/Core module load stack-gcc stack-openmpi geos-gcm-env ``` #### Loading the NAG Stack ```bash -module use -a /path/to/spack-stack/envs/ge-nag-7.2.7243-build/install/modulefiles/Core +module use -a /path/to/spack-stack/envs/ge-nag-7.2.7243/modules/Core module load stack-nag stack-openmpi geos-gcm-env ``` @@ -193,7 +185,7 @@ spack stack create env --name ge-gcc-15.3.0 --template geos-dev --site macos.gma #### NAG Environment ```bash -spack stack create env --name genag-nag-7.2.7243 --template geos-dev-nag --site macos.gmao --compiler=nag-7.2.7243 +spack stack create env --name ge-nag-7.2.7243 --template geos-dev-nag --site macos.gmao --compiler=nag-7.2.7243 ``` --- @@ -203,11 +195,11 @@ spack stack create env --name genag-nag-7.2.7243 --template geos-dev-nag --site Navigate to the environment directory you wish to work on. For example, if you built the GCC stack: ```bash -cd envs/ge-gcc-15.3.0-build +cd envs/ge-gcc-15.3.0 spack env activate -p . ``` -> **Note:** If you are building or debugging a different compiler stack, be sure to `cd` into that specific environment directory instead (e.g., `envs/ge-nag-7.2.7243-build` or `envs/ge-clang-22.1.8-build`). +> **Note:** If you are building or debugging a different compiler stack, be sure to `cd` into that specific environment directory instead (e.g., `envs/ge-nag-7.2.7243` or `envs/ge-clang-22.1.8`). > **Important:** Run this in *every* terminal where you plan to run Spack commands for this environment. @@ -241,18 +233,18 @@ spack mirror create -a -d $HOME/spack-stack-mirrors/spack-source-mirror Rust packages frequently require network access during the build. Pre-fetch their dependencies to the cargo mirror: ```bash -export CARGO_HOME=$HOME/spack-stack-mirrors/spack-cargo-mirror +export SPACK_CARGO_HOME=$HOME/spack-stack-mirrors/spack-cargo-mirror ../../util/fetch_cargo_deps.py ``` -> ⚠️ **Set `CARGO_HOME` in your environment before running `spack install`.** +> ⚠️ **Set `SPACK_CARGO_HOME` in your environment before running `spack install`.** --- ### Install Packages ```bash -export CARGO_HOME=$HOME/spack-stack-mirrors/spack-cargo-mirror +export SPACK_CARGO_HOME=$HOME/spack-stack-mirrors/spack-cargo-mirror spack install -j 6 --verbose --fail-fast --show-log-on-error --no-check-signature 2>&1 | tee log.install ; bell ``` @@ -291,4 +283,3 @@ spack build-env -- bash --norc --noprofile ``` This drops you into a clean bash shell with the exact environment variables, compiler wrappers, and dependencies loaded that Spack uses during the build. - diff --git a/configs/sites/tier2/macos.gmao/modules.yaml b/configs/sites/tier2/macos.gmao/modules.yaml index 320999839..3d94dfbee 100644 --- a/configs/sites/tier2/macos.gmao/modules.yaml +++ b/configs/sites/tier2/macos.gmao/modules.yaml @@ -1,4 +1,13 @@ modules: + # DYLD_* is removed when a SIP-protected macOS executable starts. Preserve + # the module-derived loader search path in an ordinary variable as well. + prefix_inspections: + ./lib: + - DYLD_FALLBACK_LIBRARY_PATH + - SPACK_DYLD_FALLBACK_LIBRARY_PATH + ./lib64: + - DYLD_FALLBACK_LIBRARY_PATH + - SPACK_DYLD_FALLBACK_LIBRARY_PATH default: enable:: - lmod @@ -27,6 +36,25 @@ modules: set: 'TMPDIR': '/tmp' 'OMPI_MCA_btl': '^tcp' + # Homebrew-backed modules add the entire Homebrew bin directory to PATH. + # Keep the external compiler and tool settings, but do not let them + # override the Spack Python selected by the loaded Python modules. + gcc: + filter: + exclude_env_vars: + - PATH + cmake: + filter: + exclude_env_vars: + - PATH + ninja: + filter: + exclude_env_vars: + - PATH + wget: + filter: + exclude_env_vars: + - PATH all: filter: exclude_env_vars: diff --git a/configs/sites/tier2/macos.gmao/packages.yaml b/configs/sites/tier2/macos.gmao/packages.yaml index 57307fcb5..73069e11a 100644 --- a/configs/sites/tier2/macos.gmao/packages.yaml +++ b/configs/sites/tier2/macos.gmao/packages.yaml @@ -3,6 +3,27 @@ packages: netcdf-c: require: [+shared] + # On macOS use the native backend to avoid the gtk3agg dependency chain + # (gtk3agg pulls in gtkplus -> wayland -> graphviz -> poppler -> gnupg -> + # pinentry -> emacs -> tree-sitter -> node-js@:20.15.1 -> python@:3.12) + py-matplotlib: + require: + - 'backend=macosx' + + # node-js@20.11.0:20.15.1 and @21.2.0:22.2.0 cap python@:3.12. + # @20.16.0:20.20.2 supports python@:3.14; skip the problematic 21.x range. + node-js: + require: + - '@20.16.0:20.20.2' + + # ninja@:1.11 depends on python@:3.12 (build dep). The homebrew external is + # @1.13.2 but buildable:false means the solver must pick from externals only. + # Without this floor the ASP solver sometimes picks @:1.11 as the "only" + # candidate and then fails because no external satisfies that version range. + ninja: + require: + - '@1.12:' + openblas: require: - '~dynamic_dispatch' @@ -24,3 +45,8 @@ packages: externals: - spec: perl@5.34.1~cpanm+opcode+open+shared+threads prefix: /usr + wget: + externals: + - spec: wget@1.25.0 + prefix: /Users/mathomp4/.homebrew/brew + buildable: false diff --git a/configs/sites/tier2/macos.gmao/packages_gcc-15.3.0.yaml.template b/configs/sites/tier2/macos.gmao/packages_gcc-15.3.0.yaml.template index cb8e98040..1dac4841b 100644 --- a/configs/sites/tier2/macos.gmao/packages_gcc-15.3.0.yaml.template +++ b/configs/sites/tier2/macos.gmao/packages_gcc-15.3.0.yaml.template @@ -22,6 +22,8 @@ packages: prefix: @BREW_PREFIX@ extra_attributes: compilers: + c: /usr/bin/clang + cxx: /usr/bin/clang++ fortran: @BREW_PREFIX@/bin/gfortran-15 openmpi: diff --git a/configs/sites/tier2/macos.gmao/packages_gcc-16.2.0.yaml.template b/configs/sites/tier2/macos.gmao/packages_gcc-16.2.0.yaml.template new file mode 100644 index 000000000..95ca6a6e7 --- /dev/null +++ b/configs/sites/tier2/macos.gmao/packages_gcc-16.2.0.yaml.template @@ -0,0 +1,51 @@ +packages: + c: + require: [apple-clang@@APPLE_CLANG_VERSION@] + cxx: + require: [apple-clang@@APPLE_CLANG_VERSION@] + fortran: + require: [gcc@16.2.0] + mpi: + require: [openmpi@=5.0.10] + apple-clang: + externals: + - spec: apple-clang@@APPLE_CLANG_VERSION@ languages:='c,c++' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/clang + cxx: /usr/bin/clang++ + gcc: + buildable: False + externals: + - spec: gcc@16.2.0 languages:='fortran' + prefix: @BREW_PREFIX@ + extra_attributes: + compilers: + c: /usr/bin/clang + cxx: /usr/bin/clang++ + fortran: @BREW_PREFIX@/bin/gfortran-16 + + openmpi: + require:: + - '+internal-hwloc' + - '+internal-libevent' + - '+internal-pmix' + - '+fortran' + # GEOS requires a flat namespace to link properly on macOS + - '~two_level_namespace' + + # ESMF by default for Darwin.gfortranclang turns off OpenMP, so + # we do the same here to avoid build failures. + # We also build as +debug only because esmf ~debug does not work with + # gfortranclang/GEOS on macOS. The geos-gcm-env ~debug spec is removed + # from the environment spack.yaml by batch_install.sh on macOS. + esmf: + require: + - '~openmp' + - '+debug' + + gdbm: + externals: + - spec: gdbm@1.26 + prefix: @BREW_PREFIX@/opt/gdbm diff --git a/configs/templates/geos-dev/spack.yaml b/configs/templates/geos-dev/spack.yaml index 0cd32011a..423552eb1 100644 --- a/configs/templates/geos-dev/spack.yaml +++ b/configs/templates/geos-dev/spack.yaml @@ -13,11 +13,14 @@ spack: # (see geos_gcm_env/package.py). On platforms where only +debug is supported # (e.g., macOS), the site packages.yaml requires geos-gcm-env +debug, which # causes both specs to resolve to +debug (only one unique build). - - geos-gcm-env ~debug +cdo - - geos-gcm-env +debug +cdo + - geos-gcm-env@11.10.0 ~debug +cdo +fms + - geos-gcm-env@11.10.0 +debug +cdo +fms + - geos-gcm-env@12.1.0 ~debug +cdo +fms + - geos-gcm-env@12.1.0 +debug +cdo +fms # Various fms builds - fms@=2024.03 ~gfs_phys constants=GEOS + - fms@=2026.01.01 ~gfs_phys constants=GEOS packages: # Turn on python variant for esmf diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml index 3d0012eb0..2e0e9167d 100644 --- a/configs/templates/unified-dev/spack.yaml +++ b/configs/templates/unified-dev/spack.yaml @@ -8,10 +8,11 @@ spack: include: [] specs: + - ai-env - dev-utils-env - ewok-env - - geos-gcm-env ^esmf@=9.0.0b11 - - global-workflow-env ^esmf@=8.6.1 ^crtm@=3.1.3 + - geos-gcm-env +cdo +fms ^esmf@=9.0.0b11 + - global-workflow-env ^esmf@=8.6.1 ^crtm@=3.1.3 # Is GMAO ready to move to crtm@3.1.3? - gmao-swell-env ^crtm@=v2.4.1-jedi.2 - gsi-env ^crtm@=3.1.3 diff --git a/repos/builtin b/repos/builtin index 6950df944..508ce12c9 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit 6950df94424f67cbc3d6b1090aee0c30228a1233 +Subproject commit 508ce12c90415064f1d48728f5287810565830be diff --git a/repos/spack_stack/spack_repo/spack_stack/packages/ai_env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/ai_env/package.py index 902d6ddfb..9d9cc50ee 100644 --- a/repos/spack_stack/spack_repo/spack_stack/packages/ai_env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/ai_env/package.py @@ -12,7 +12,7 @@ class AiEnv(BundlePackage): # DH* TODO UPDATE FROM INTERNAL TO PUBLIC homepage = "https://github.com/JCSDA/spack-stack" - git = "https://github.comJCSDA/spack-stack.git" + git = "https://github.com/JCSDA/spack-stack.git" maintainers("climbfuji", "srherbener") diff --git a/repos/spack_stack/spack_repo/spack_stack/packages/geos_gcm_env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/geos_gcm_env/package.py index b1a98f0a1..e1e77480c 100644 --- a/repos/spack_stack/spack_repo/spack_stack/packages/geos_gcm_env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/geos_gcm_env/package.py @@ -16,11 +16,13 @@ class GeosGcmEnv(BundlePackage): maintainers("climbfuji", "mathomp4", "Dooruk") # Current version - version("11.10.0") + version("12.1.0") + version("11.10.0", preferred=True) variant("debug", default=False, description="Build debug version of selected dependencies") variant("fms", default=True, description="Build FMS as part of the environment") variant("cdo", default=True, description="Build CDO as part of the environment") + variant("grads", default=False, description="Build GrADS as part of the environment") depends_on("base-env", type="run") @@ -34,7 +36,13 @@ class GeosGcmEnv(BundlePackage): depends_on( "fms@2024.03 precision=32,64 ~gfs_phys +openmp +pic constants=GEOS +deprecated_io +yaml build_type=Release", #noqa: E501 type="run", - when="@11.10: +fms", + when="@11.10:12.0 +fms", + ) + + depends_on( + "fms@2026.01.01 precision=mixed ~gfs_phys +openmp +pic constants=GEOS +yaml build_type=Release", #noqa: E501 + type="run", + when="@12.1: +fms", ) # mapl is built as part of GEOS, don't load; @@ -50,12 +58,23 @@ class GeosGcmEnv(BundlePackage): depends_on("py-ruamel-yaml", type="run") depends_on("udunits", type="run") + # Needed for f2py + depends_on("meson", type="run") + depends_on("ninja", type="run") + depends_on("perl", type="run") depends_on("py-cmocean", type="run") + depends_on("py-matplotlib", type="run") + depends_on("py-netcdf4", type="run") + depends_on("py-pandas", type="run") + depends_on("py-xarray", type="run") + depends_on("py-cartopy", type="run") depends_on("nco", type="run") depends_on("nccmp", type="run") depends_on("cdo", type="run", when="+cdo") + depends_on("grads", type="run", when="+grads") + # There is no need for install() since there is no code. diff --git a/spack b/spack index 0a2c2f21d..515fa33f6 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 0a2c2f21d52d70692f52ed48c3ab4b5bbdcb2bde +Subproject commit 515fa33f6fb04f45d77bfabf14f0e01bc513ad58 diff --git a/util/epic/batch_install.sh b/util/epic/batch_install.sh index e5bb0cc57..c62d7dbe9 100755 --- a/util/epic/batch_install.sh +++ b/util/epic/batch_install.sh @@ -340,7 +340,7 @@ host=${SPACK_STACK_BATCH_HOST} module_choice=${SPACK_STACK_MODULE_CHOICE} bootstrap_mirror_path=${SPACK_STACK_BOOTSTRAP_MIRROR} cargo_mirror_path=${SPACK_STACK_CARGO_MIRROR} -export CARGO_HOME=${cargo_mirror_path} +export SPACK_CARGO_HOME=${cargo_mirror_path} if [[ -z ${SPACK_STACK_ENVIRONMENT_DIRS} ]]; then environment_dirs=${PWD}/envs diff --git a/util/fetch_cargo_deps.py b/util/fetch_cargo_deps.py index bee194ce9..fcb6ee88b 100755 --- a/util/fetch_cargo_deps.py +++ b/util/fetch_cargo_deps.py @@ -1,8 +1,8 @@ #!/usr/bin/env spack-python # # Run this script in an active, concretized Spack environment to fetch Rust -# dependencies and store them in $CARGO_HOME. You must either run it with -# 'spack-python' or have 'spack-python' in your $PATH. Ensure $CARGO_HOME has +# dependencies and store them in $SPACK_CARGO_HOME. You must either run it with +# 'spack-python' or have 'spack-python' in your $PATH. Ensure $SPACK_CARGO_HOME has # the same value when 'spack install' is run. # # For each spec that is a CargoPackage or a PythonPackage with a rust dependency, @@ -41,14 +41,14 @@ if not spack_stack_dir: raise SpackError("SPACK_STACK_DIR must be set") -cargo_home = os.getenv("CARGO_HOME") +cargo_home = os.getenv("SPACK_CARGO_HOME") if not cargo_home: - raise SpackError("CARGO_HOME must be set") + raise SpackError("SPACK_CARGO_HOME must be set") if not os.path.isdir(cargo_home): os.makedirs(cargo_home) # Find each spec that is a CargoPackage or a PythonPackage with a -# rust/rust-bootstrap dependency and fetch its dependencies to $CARGO_HOME +# rust/rust-bootstrap dependency and fetch its dependencies to $SPACK_CARGO_HOME for spec in env.all_specs(): if not spec.concrete: continue @@ -120,7 +120,10 @@ env.set("CARGO_HOME", cargo_home) if cargo_bin: env.prepend_path("PATH", cargo_bin) - for root, dirs, files in os.walk(pkg.stage.source_path): - if "Cargo.toml" in files: - with working_dir(root): - cargo_exe("fetch", env=env) + # Only run 'cargo fetch' at the top-level source directory. + # Walking all subdirectories causes failures when sub-crates have their own + # Cargo.toml but are not listed in the workspace manifest (e.g. py-cryptography + # docs/development sub-crates). Cargo resolves transitive deps from the root. + if os.path.exists(os.path.join(pkg.stage.source_path, "Cargo.toml")): + with working_dir(pkg.stage.source_path): + cargo_exe("fetch", env=env) diff --git a/util/gmao/README.md b/util/gmao/README.md index 0f4d6b45c..61ee93058 100644 --- a/util/gmao/README.md +++ b/util/gmao/README.md @@ -43,6 +43,7 @@ Run with `-h` for full usage: | `-s` | Submit the install step to the batch scheduler (SLURM/PBS) instead of running interactively. | | `-e` | Continue if environment directories already exist. Required on all runs after the first. | | `-u` | Populate bootstrap, source, and cargo mirrors. Requires `-r dev -m build`. First run only. | +| `-o` / `--concretize-only` | Stop after concretization; do not proceed to install. Useful for inspecting solver output before committing to a full build. | | `-a ACCOUNT` | Override the scheduler account (default: `s1873`). | | `-n` | Dry-run: print what would be executed without running anything. | | `-p` / `--partition` | Override SLURM partition. **NCCS hosts only** (discover, discover-gmao). | @@ -65,6 +66,7 @@ The script detects the current site from `$HOSTNAME` (stripping digits and domai | `nas` | NASA NAS Pleiades (TOSS 4) | | `nas-toss5` | NASA NAS Pleiades (TOSS 5) | | `macos.gmao` | macOS with Homebrew (GMAO SI Team) | +| `bucy` | RHEL 8.10 host with shared GMAO SI Team software | ### Site-specific READMEs @@ -72,6 +74,7 @@ For detailed workflow instructions for each site, see: - `configs/sites/tier2/discover-gmao/README.md` — Discover GMAO workflow - `configs/sites/tier2/macos.gmao/README.md` — macOS GMAO workflow +- `configs/sites/tier2/bucy/README.md` — Bucy workflow --- diff --git a/util/gmao/batch_install.sh b/util/gmao/batch_install.sh index ab61af130..048bb2b57 100755 --- a/util/gmao/batch_install.sh +++ b/util/gmao/batch_install.sh @@ -52,18 +52,19 @@ check_macos_prerequisites() { usage() { set +x echo - echo "Usage: $0 -r -m [-d ] [-c ] [-H ]" + echo "Usage: $0 [-r ] -m [-d ] [-c ] [-H ]" echo - echo " -r Set role, can be 'ops' or 'dev'" - echo " -m Set mode, can be 'build' or 'install';" + echo " -r Set role, can be 'ops' or 'dev'; required except with -m local" + echo " -m Set mode, can be 'build', 'install', or 'local';" echo " build: build environments and update build caches;" - echo " install: install environments using build caches" + echo " install: install environments using build caches;" + echo " local: one-step, per-machine install and cache update (macos.gmao and bucy)" echo " -d Build or install environments in ENV_DIRS;" echo " if not set, the default location is used" echo " -c Provide location of build caches as BUILDCACHE_DIR;" echo " if not set, authoritative build caches are used" echo " -u Flag to update bootstrap and source caches;" - echo " requires role 'dev' and mode 'build'" + echo " requires role 'dev' and mode 'build', or mode 'local'" echo " -e Continue builds/install in existing environments;" echo " by default, exit with an error if already exist" echo " -C Set a comma-separated list of compilers to use (e.g. gcc@=15.3.0,nag@=7.2.7243);" @@ -86,8 +87,12 @@ usage() { echo " no short form available (-c and -C are already used);" echo " NOTE: only applies to NCCS hosts (discover, discover-gmao)" echo " -s Submit 'spack install' to batch scheduler" + echo " -L Disable Spack install locks and serialize package installs; use only when this is the sole process modifying the install tree;" + echo " also accepted as --disable-locks" echo " -t Run tests for specific thirdparty dependencies;" echo " these are currently hardcoded in batch_install.sh" + echo " -o Concretize-only: stop after concretization (do not install);" + echo " also accepted as --concretize-only" echo " -n Dry-run: print what would be executed without running anything" echo " -H Provide hostname manually (overrides autodetection);" echo " useful when VPN/etc masks the real hostname" @@ -95,7 +100,66 @@ usage() { echo } -while getopts r:m:d:c:C:N:H:a:p:q:nuesth flag +# Normalize long options before getopts processes the short options. +normalized_args=() +while [[ $# -gt 0 ]]; do + case "$1" in + --concretize-only) + normalized_args+=("-o") + ;; + --disable-locks) + normalized_args+=("-L") + ;; + --partition=*) + normalized_args+=("-p" "${1#*=}") + ;; + --partition) + if [[ -z "${2:-}" ]]; then + echo "ERROR, --partition requires a value" + exit 1 + fi + normalized_args+=("-p" "$2") + shift + ;; + --qos=*) + normalized_args+=("-q" "${1#*=}") + ;; + --qos) + if [[ -z "${2:-}" ]]; then + echo "ERROR, --qos requires a value" + exit 1 + fi + normalized_args+=("-q" "$2") + shift + ;; + --constraint=*) + SPACK_STACK_SLURM_CONSTRAINT="${1#*=}" + ;; + --constraint) + if [[ -z "${2:-}" ]]; then + echo "ERROR, --constraint requires a value" + exit 1 + fi + SPACK_STACK_SLURM_CONSTRAINT="$2" + shift + ;; + --help) + normalized_args+=("-h") + ;; + --*) + echo "ERROR, unknown argument: $1" + usage + exit 1 + ;; + *) + normalized_args+=("$1") + ;; + esac + shift +done +set -- "${normalized_args[@]}" + +while getopts r:m:d:c:C:N:H:a:p:q:nouestLh flag do case "${flag}" in r) @@ -131,6 +195,9 @@ do n) SPACK_STACK_DRY_RUN="true" ;; + o) + SPACK_STACK_CONCRETIZE_ONLY="true" + ;; u) SPACK_STACK_UPDATE_DEV_CACHES="true" ;; @@ -140,6 +207,9 @@ do s) SPACK_STACK_SUBMIT_TO_SCHEDULER="true" ;; + L) + SPACK_STACK_DISABLE_LOCKS="true" + ;; t) SPACK_STACK_RUN_TESTS="true" ;; @@ -151,39 +221,6 @@ do done shift $((OPTIND - 1)) -# Parse any remaining long options (e.g. --constraint, --partition, --qos) -while [[ $# -gt 0 ]]; do - case $1 in - --constraint=*) - SPACK_STACK_SLURM_CONSTRAINT="${1#*=}" - ;; - --constraint) - SPACK_STACK_SLURM_CONSTRAINT="$2" - shift - ;; - --partition=*) - SPACK_STACK_SLURM_PARTITION="${1#*=}" - ;; - --partition) - SPACK_STACK_SLURM_PARTITION="$2" - shift - ;; - --qos=*) - SPACK_STACK_SLURM_QOS="${1#*=}" - ;; - --qos) - SPACK_STACK_SLURM_QOS="$2" - shift - ;; - *) - echo "ERROR, unknown argument: $1" - usage - exit 1 - ;; - esac - shift -done - echo "INFO: $0 options:" echo " SPACK_STACK_ROLE: ${SPACK_STACK_ROLE:-not set}" echo " SPACK_STACK_MODE: ${SPACK_STACK_MODE:-not set}" @@ -194,6 +231,7 @@ echo " SPACK_STACK_DRY_RUN: ${SPACK_STACK_DRY_RUN:-fals echo " SPACK_STACK_UPDATE_DEV_CACHES: ${SPACK_STACK_UPDATE_DEV_CACHES:-false}" echo " SPACK_STACK_IGNORE_ENV_EXIST: ${SPACK_STACK_IGNORE_ENV_EXIST:-false}" echo " SPACK_STACK_SUBMIT_TO_SCHEDULER: ${SPACK_STACK_SUBMIT_TO_SCHEDULER:-false}" +echo " SPACK_STACK_DISABLE_LOCKS: ${SPACK_STACK_DISABLE_LOCKS:-false}" echo " SPACK_STACK_RUN_TESTS: ${SPACK_STACK_RUN_TESTS:-false}" echo " ACCOUNT: ${ACCOUNT:-s1873 (default)}" echo " SPACK_STACK_SLURM_PARTITION: ${SPACK_STACK_SLURM_PARTITION:-use site default}" @@ -203,19 +241,19 @@ echo " SPACK_STACK_SLURM_CONSTRAINT: ${SPACK_STACK_SLURM_CONSTRA # Set default account if not provided via -a or environment ACCOUNT=${ACCOUNT:-s1873} -if [[ -z ${SPACK_STACK_ROLE} ]]; then - echo "ERROR, SPACK_STACK_ROLE not defined. Provide -r ROLE as argument" +if [[ -z ${SPACK_STACK_MODE} ]]; then + echo "ERROR, SPACK_STACK_MODE not defined. Provide -m MODE as argument" exit 1 -elif [[ ! ${SPACK_STACK_ROLE} == "dev" && ! ${SPACK_STACK_ROLE} == "ops" ]]; then - echo "ERROR, invalid role '${SPACK_STACK_ROLE}'" +elif [[ ! ${SPACK_STACK_MODE} == "build" && ! ${SPACK_STACK_MODE} == "install" && ! ${SPACK_STACK_MODE} == "local" ]]; then + echo "ERROR, invalid mode '${SPACK_STACK_MODE}'" exit 1 fi -if [[ -z ${SPACK_STACK_MODE} ]]; then - echo "ERROR, SPACK_STACK_MODE not defined. Provide -m MODE as argument" +if [[ "${SPACK_STACK_MODE}" != "local" && -z ${SPACK_STACK_ROLE} ]]; then + echo "ERROR, SPACK_STACK_ROLE not defined. Provide -r ROLE as argument" exit 1 -elif [[ ! ${SPACK_STACK_MODE} == "build" && ! ${SPACK_STACK_MODE} == "install" ]]; then - echo "ERROR, invalid mode '${SPACK_STACK_MODE}'" +elif [[ -n ${SPACK_STACK_ROLE} && ! ${SPACK_STACK_ROLE} == "dev" && ! ${SPACK_STACK_ROLE} == "ops" ]]; then + echo "ERROR, invalid role '${SPACK_STACK_ROLE}'" exit 1 fi @@ -226,10 +264,12 @@ if [[ ${SPACK_STACK_ROLE} == "ops" && ${SPACK_STACK_MODE} == "build" && -z ${SPA exit 1 fi -# Updating bootstrap and source caches requires role dev and mode build +# Updating shared-site bootstrap and source caches requires role dev and mode build. +# Local mirrors on macOS and bucy can be updated without a role. if [[ ${SPACK_STACK_UPDATE_DEV_CACHES} == "true" ]]; then - if [[ ! ${SPACK_STACK_ROLE} == "dev" || ! ${SPACK_STACK_MODE} == "build" ]]; then - echo "ERROR, SPACK_STACK_UPDATE_DEV_CACHES requires role 'dev' and mode 'build'" + if [[ "${SPACK_STACK_MODE}" != "local" && \ + ( ! ${SPACK_STACK_ROLE} == "dev" || ! ${SPACK_STACK_MODE} == "build" ) ]]; then + echo "ERROR, SPACK_STACK_UPDATE_DEV_CACHES requires role 'dev' and mode 'build', or mode 'local'" exit 1 fi fi @@ -246,7 +286,7 @@ fi case ${SPACK_STACK_BATCH_HOST} in nas) - SPACK_STACK_BATCH_COMPILERS=("oneapi@=2024.2.0" "oneapi@=2025.3.0" "gcc@=13.2.0") + SPACK_STACK_BATCH_COMPILERS=("oneapi@=2024.2.0" "oneapi@=2025.3.0" "gcc@=15.2.0") SPACK_STACK_BATCH_TEMPLATES=("unified-dev") SPACK_STACK_MODULE_CHOICE="tcl" SPACK_STACK_BOOTSTRAP_MIRROR="/swbuild/gmao_SIteam/spack-stack/bootstrap-mirror-toss4" @@ -254,7 +294,7 @@ case ${SPACK_STACK_BATCH_HOST} in SPACK_STACK_ENVIRONMENT_DIRS=${SPACK_STACK_ENVIRONMENT_DIRS:-${PWD}/envs/toss4} ;; nas-toss5) - SPACK_STACK_BATCH_COMPILERS=("oneapi@=2024.2.0" "oneapi@=2025.3.0" "gcc@=14.2.1") + SPACK_STACK_BATCH_COMPILERS=("oneapi@=2024.2.0" "oneapi@=2025.3.0" "gcc@=15.3.0") SPACK_STACK_BATCH_TEMPLATES=("unified-dev") SPACK_STACK_MODULE_CHOICE="tcl" SPACK_STACK_BOOTSTRAP_MIRROR="/swbuild/gmao_SIteam/spack-stack/bootstrap-mirror-toss5" @@ -262,20 +302,27 @@ case ${SPACK_STACK_BATCH_HOST} in SPACK_STACK_ENVIRONMENT_DIRS=${SPACK_STACK_ENVIRONMENT_DIRS:-${PWD}/envs/toss5} ;; discover) - SPACK_STACK_BATCH_COMPILERS=("oneapi@=2024.2.0" "oneapi@=2025.3.0" "gcc@=14.2.0") + SPACK_STACK_BATCH_COMPILERS=("oneapi@=2024.2.0" "oneapi@=2025.3.0" "gcc@=14.2.0" "gcc@=15.2.0") SPACK_STACK_BATCH_TEMPLATES=("unified-dev") SPACK_STACK_MODULE_CHOICE="lmod" - SPACK_STACK_BOOTSTRAP_MIRROR="/discover/swdev/jcsda/spack-stack/bootstrap-mirror" - SPACK_STACK_CARGO_MIRROR="/discover/swdev/jcsda/spack-stack/cargo-mirror" + SPACK_STACK_BOOTSTRAP_MIRROR="/discover/nobackup/projects/gmao/SIteam/spack-stack/bootstrap-mirror" + SPACK_STACK_CARGO_MIRROR="/discover/nobackup/projects/gmao/SIteam/spack-stack/cargo-mirror" ;; discover-gmao) - SPACK_STACK_BATCH_COMPILERS=("oneapi@=2024.2.0" "oneapi@=2025.3.0" "gcc@=15.2.0") - #SPACK_STACK_BATCH_TEMPLATES=("unified-dev") - SPACK_STACK_BATCH_TEMPLATES=("geos-dev") + SPACK_STACK_BATCH_COMPILERS=("oneapi@=2024.2.0" "oneapi@=2025.3.0" "gcc@=15.2.0" "nag@=7.2.7238") + SPACK_STACK_BATCH_TEMPLATES=("geos-dev" "geos-dev-nag") SPACK_STACK_MODULE_CHOICE="lmod" SPACK_STACK_BOOTSTRAP_MIRROR="/discover/nobackup/projects/gmao/SIteam/spack-stack/bootstrap-mirror" SPACK_STACK_CARGO_MIRROR="/discover/nobackup/projects/gmao/SIteam/spack-stack/cargo-mirror" ;; + bucy) + SPACK_STACK_BATCH_COMPILERS=("gcc@=15.2.0" "gcc@=16.1.0" "oneapi@=2024.2.0" "oneapi@=2025.3.0" "nag@=7.2.43" "llvm@=22.1.0") + SPACK_STACK_BATCH_TEMPLATES=("geos-dev" "geos-dev-nag") + SPACK_STACK_MODULE_CHOICE="lmod" + SPACK_STACK_BOOTSTRAP_MIRROR="/ford1/share/gmao_SIteam/spack-stack/bootstrap-mirror" + SPACK_STACK_CARGO_MIRROR="/ford1/share/gmao_SIteam/spack-stack/cargo-mirror" + SPACK_STACK_ENVIRONMENT_DIRS=${SPACK_STACK_ENVIRONMENT_DIRS:-${PWD}/envs/bucy} + ;; macos.gmao) # Detect NAG Fortran Compiler nag_path_tmp="" @@ -294,8 +341,8 @@ case ${SPACK_STACK_BATCH_HOST} in # Note: clang (aka flang) is on hold for macOS until we move to # 1. FMS 2025 (for GEOS purposes) # 2. ESMF PR https://github.com/esmf-org/esmf/pull/558 is merged and released/tagged - #SPACK_STACK_BATCH_COMPILERS=("gcc@=15.3.0" "clang@=22.1.8") - SPACK_STACK_BATCH_COMPILERS=("gcc@=15.3.0") + SPACK_STACK_BATCH_COMPILERS=("gcc@=15.3.0" "gcc@=16.2.0" "clang@=22.1.8") + #SPACK_STACK_BATCH_COMPILERS=("gcc@=15.3.0" "gcc@=16.2.0" ) if [[ -n "${MAC_GMAO_NAG_VERSION}" ]]; then SPACK_STACK_BATCH_COMPILERS+=("nag@=${MAC_GMAO_NAG_VERSION}") fi @@ -318,6 +365,18 @@ case ${SPACK_STACK_BATCH_HOST} in ;; esac +if [[ "${SPACK_STACK_MODE}" == "local" && + "${SPACK_STACK_BATCH_HOST}" != "macos.gmao" && + "${SPACK_STACK_BATCH_HOST}" != "bucy" ]]; then + echo "ERROR, mode 'local' is supported only for hosts macos.gmao and bucy" + exit 1 +fi + +if [[ "${SPACK_STACK_MODE}" == "local" && "${SPACK_STACK_SUBMIT_TO_SCHEDULER}" == "true" ]]; then + echo "ERROR, mode 'local' cannot be submitted to a batch scheduler" + exit 1 +fi + # Apply -C compiler override for all hosts (not just macos.gmao) if [[ -n "${SPACK_STACK_COMPILER_OPT}" ]]; then IFS=',' read -r -a SPACK_STACK_BATCH_COMPILERS <<< "${SPACK_STACK_COMPILER_OPT}" @@ -360,6 +419,13 @@ function fix_permissions() { fi nice -n 19 find ${dir} -type f -print0 | xargs --null chmod a+r ;; + bucy) + nice -n 19 find ${dir} -type d -print0 | xargs --null chmod a+rx + if [[ ${executables} -eq 1 ]]; then + nice -n 19 find ${dir} -type f -executable -print0 | xargs --null chmod a+rx + fi + nice -n 19 find ${dir} -type f -print0 | xargs --null chmod a+r + ;; macos.gmao) ;; *) @@ -426,6 +492,7 @@ function run_interactive_job() { echo " Login node: ${login_node}, PBS model: ${pbs_model}" qsub -V \ -l select=1:ncpus=${tpn}:mpiprocs=${tpn}:model=${pbs_model} \ + -q normal \ -l walltime=${walltime} \ -l site=needed=/home3+/nobackupp18+/nobackupp28+/vast_swbuild/swbuild4 \ -W group_list=${ACCOUNT} \ @@ -471,13 +538,15 @@ function run_interactive_job() { slurm_qos="" fi slurm_log="${job_name}.log" - echo "INFO: salloc output redirected to ${slurm_log}" - salloc --nodes=1 --ntasks-per-node=${tpn} --time=${walltime} \ + echo "INFO: sbatch output redirected to ${slurm_log}" + sbatch --wait \ + --nodes=1 --ntasks-per-node=${tpn} --time=${walltime} \ ${slurm_constraint} ${slurm_partition} ${slurm_qos} \ --job-name=${job_name} \ - --account=${ACCOUNT} bash ${script} \ - > "${slurm_log}" 2>&1 - echo "INFO: salloc job complete, log: ${slurm_log}" + --account=${ACCOUNT} \ + --output="${slurm_log}" --error="${slurm_log}" \ + ${script} + echo "INFO: sbatch job complete, log: ${slurm_log}" ;; discover-gmao) slurm_constraint="${SPACK_STACK_SLURM_CONSTRAINT:+--constraint=${SPACK_STACK_SLURM_CONSTRAINT}}" @@ -497,13 +566,15 @@ function run_interactive_job() { slurm_qos="" fi slurm_log="${job_name}.log" - echo "INFO: salloc output redirected to ${slurm_log}" - salloc --nodes=1 --ntasks-per-node=${tpn} --time=${walltime} \ + echo "INFO: sbatch output redirected to ${slurm_log}" + sbatch --wait \ + --nodes=1 --ntasks-per-node=${tpn} --time=${walltime} \ ${slurm_constraint} ${slurm_partition} ${slurm_qos} \ --job-name=${job_name} \ - --account=${ACCOUNT} bash ${script} \ - > "${slurm_log}" 2>&1 - echo "INFO: salloc job complete, log: ${slurm_log}" + --account=${ACCOUNT} \ + --output="${slurm_log}" --error="${slurm_log}" \ + ${script} + echo "INFO: sbatch job complete, log: ${slurm_log}" ;; *) echo "ERROR, run_interactive_job command not configured for ${host}" @@ -531,7 +602,7 @@ host=${SPACK_STACK_BATCH_HOST} module_choice=${SPACK_STACK_MODULE_CHOICE} bootstrap_mirror_path=${SPACK_STACK_BOOTSTRAP_MIRROR} cargo_mirror_path=${SPACK_STACK_CARGO_MIRROR} -export CARGO_HOME=${cargo_mirror_path} +export SPACK_CARGO_HOME=${cargo_mirror_path} if [[ -z ${SPACK_STACK_ENVIRONMENT_DIRS} ]]; then environment_dirs=${PWD}/envs @@ -556,6 +627,7 @@ if [[ "${SPACK_STACK_MODE}" == "install" ]]; then update_cargo_mirror="false" update_source_cache="false" update_build_cache="false" + publish_build_cache="false" reuse_build_cache="true" elif [[ "${SPACK_STACK_MODE}" == "build" ]]; then if [[ "${SPACK_STACK_ROLE}" == "ops" ]]; then @@ -577,6 +649,23 @@ elif [[ "${SPACK_STACK_MODE}" == "build" ]]; then exit 1 fi update_build_cache="true" + publish_build_cache="true" + reuse_build_cache="true" +elif [[ "${SPACK_STACK_MODE}" == "local" ]]; then + # A macOS stack is private to one workstation. Prepare its local source, + # bootstrap, and Cargo mirrors when requested. Install directly into the + # final environment, generate modules, and publish a private buildcache. + if [[ ${SPACK_STACK_UPDATE_DEV_CACHES} == "true" ]]; then + update_bootstrap_mirror="true" + update_cargo_mirror="true" + update_source_cache="true" + else + update_bootstrap_mirror="false" + update_cargo_mirror="false" + update_source_cache="false" + fi + update_build_cache="false" + publish_build_cache="true" reuse_build_cache="true" else echo "ERROR, invalid mode ${SPACK_STACK_MODE}" @@ -597,6 +686,16 @@ else test_packages=() fi +if [[ "${SPACK_STACK_DISABLE_LOCKS}" == "true" ]]; then + spack_install_lock_flag="--disable-locks" +else + spack_install_lock_flag="" +fi + +# A full node is allocated for isolation, but large package builds can slow down +# when every hardware thread is used. Cap Spack's per-package build parallelism. +spack_build_jobs=24 + # Loop through all compilers and templates for this host first_pass="true" for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do @@ -644,6 +743,10 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do env_name=${env_name_prefix}-${compiler_name}-${compiler_version} [[ "${update_build_cache}" == "true" ]] && env_name=${env_name}-build env_dir=${environment_dirs}/${env_name} + # Different sites can legitimately build an identically named environment + # at the same time (for example NAS TOSS4 and TOSS5). Keep the generated + # job scripts distinct as well, since they share this working directory. + install_script_name=spack-install.${host}.${env_name}.sh # Reset env_exists for this specific environment target env_exists="false" @@ -683,10 +786,16 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do echo " --site=${host} --compiler=${compiler_name}-${compiler_version} \\" echo " --template=${template} --dir=${environment_dirs} --treat-warnings-as-errors" if [[ "${host}" == "macos.gmao" ]]; then - echo "[DRY-RUN] grep -v 'geos-gcm-env ~debug' ${env_dir}/spack.yaml # remove ~debug spec (esmf ~debug unsupported on macOS)" + echo "[DRY-RUN] grep -vE 'geos-gcm-env([^[:space:]]*)?[[:space:]]+~debug' ${env_dir}/spack.yaml # remove ~debug spec (esmf ~debug unsupported on macOS)" fi fi echo "[DRY-RUN] spack env activate -p ${env_dir}" + if [[ "${SPACK_STACK_DISABLE_LOCKS}" == "true" ]]; then + echo "[DRY-RUN] generated batch script exports a job-local SPACK_USER_CONFIG_PATH with config:locks:false" + fi + if [[ "${submit_to_scheduler}" == "true" && ( "${host}" == "discover" || "${host}" == "discover-gmao" ) ]]; then + echo "[DRY-RUN] generated batch script uses project-space paths from site config.yaml for build, test, and cache stages" + fi if [[ "${host}" == "macos.gmao" && ! ${env_exists} == "true" ]]; then echo "[DRY-RUN] spack external find --not-buildable autoconf automake bash cmake cvs doxygen gawk git-lfs groff libtool ninja npm subversion swig texinfo" echo "[DRY-RUN] generating spack-macos-externals.yaml and applying with 'spack config add -f'" @@ -697,7 +806,7 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do fi echo "[DRY-RUN] spack bootstrap now" echo "[DRY-RUN] spack concretize --force --fresh" - echo "[DRY-RUN] ./util/show_duplicate_packages.py -i crtm -i crtm-fix -i esmf -i mapl -i neptune-env -i py-cython -i ip -i fms -i geos-gcm-env" + echo "[DRY-RUN] ./util/show_duplicate_packages.py -i crtm -i crtm-fix -i esmf -i mapl -i neptune-env -i py-cython -i ip -i fms -i geos-gcm-env -i py-versioneer" if [[ "${update_source_cache}" == "true"* ]]; then echo "[DRY-RUN] spack mirror create -a -d " @@ -711,13 +820,32 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do [[ "${compiler_name}" == "oneapi" ]]; then echo "[DRY-RUN] # ectrans/ecbuild workaround (NAS oneapi only):" echo "[DRY-RUN] # See: https://github.com/JCSDA/spack-stack/issues/1775#issuecomment-3898802720" - echo "[DRY-RUN] spack install ecbuild" + echo "[DRY-RUN] spack ${spack_install_lock_flag} install ecbuild" echo "[DRY-RUN] ./util/gmao/patch_ecbuild_ectrans.py --patch \$(spack location -i ecbuild)/.../ecbuild_add_lang_flags.cmake" - echo "[DRY-RUN] spack install ectrans" + echo "[DRY-RUN] spack ${spack_install_lock_flag} install ectrans" echo "[DRY-RUN] ./util/gmao/patch_ecbuild_ectrans.py --revert \$(spack location -i ecbuild)/.../ecbuild_add_lang_flags.cmake" fi - echo "[DRY-RUN] Generating spack-install.${env_name}.sh and executing via:" + echo "[DRY-RUN] Generating ${install_script_name} and executing via:" + echo "[DRY-RUN] Generated install commands: spack ${spack_install_lock_flag} install ..." + if [[ "${submit_to_scheduler}" == "true" ]]; then + echo "[DRY-RUN] Scheduled package builds use --jobs=${spack_build_jobs} (the SLURM allocation remains a full node)" + fi + if [[ "${SPACK_STACK_DISABLE_LOCKS}" == "true" && "${submit_to_scheduler}" == "true" ]]; then + echo "[DRY-RUN] -L also adds --concurrent-packages=1" + fi + if [[ "${host}" == "discover" || "${host}" == "discover-gmao" ]] && \ + [[ "${compiler}" == "oneapi@=2025.3.0" ]]; then + echo "[DRY-RUN] Job preflight: module load comp/intel/2025.3.0 mpi/impi/2021.17" + fi + if [[ "${host}" == "discover" || "${host}" == "discover-gmao" ]] && \ + [[ "${compiler}" == "gcc@=15.2.0" ]]; then + echo "[DRY-RUN] Job preflight: module load comp/gcc/15.2.0 mpi/openmpi/5.0.10/gcc-15.2.0" + fi + if [[ "${host}" == "discover-gmao" ]] && \ + [[ "${compiler}" == "nag@=7.2.7238" ]]; then + echo "[DRY-RUN] Job preflight: module load comp/gcc/12.3.0 comp/nag/7.2-7238 mpi/openmpi/4.1.6/nag_7.2.7238-gcc_12.3.0" + fi if [[ "${submit_to_scheduler}" == "true" ]]; then tpn_dry=$(tasks_per_node ${host}) case ${host} in @@ -730,11 +858,11 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do esac echo "[DRY-RUN] qsub -V \\" echo "[DRY-RUN] -l select=1:ncpus=${tpn_dry}:mpiprocs=${tpn_dry}:model=${pbs_model_dry} \\" - echo "[DRY-RUN] -l walltime=08:00:00 \\" + echo "[DRY-RUN] -q normal -l walltime=08:00:00 \\" echo "[DRY-RUN] -l site=needed=/home3+/nobackupp18+/nobackupp28+/vast_swbuild/swbuild4 \\" echo "[DRY-RUN] -W group_list=${ACCOUNT} -W block=true -W umask=0022 \\" echo "[DRY-RUN] -j oe -k oed -N spack.${host}.${env_name} \\" - echo "[DRY-RUN] spack-install.${env_name}.sh" + echo "[DRY-RUN] ${install_script_name}" ;; nas-toss5) echo "[DRY-RUN] qsub -V \\" @@ -743,7 +871,7 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do echo "[DRY-RUN] -l site=needed=/home3+/nobackupp18+/nobackupp28+/vast_swbuild/swbuild4 \\" echo "[DRY-RUN] -W group_list=${ACCOUNT} -W block=true -W umask=0022 \\" echo "[DRY-RUN] -j oe -k oed -N spack.${host}.${env_name} \\" - echo "[DRY-RUN] spack-install.${env_name}.sh" + echo "[DRY-RUN] ${install_script_name}" ;; discover) if [[ "${ACCOUNT}" == "s1873" ]]; then @@ -751,10 +879,13 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do else slurm_extra_dry="(default partition/qos)" fi - echo "[DRY-RUN] salloc --nodes=1 --ntasks-per-node=${tpn_dry} --time=08:00:00 \\" + echo "[DRY-RUN] sbatch --wait \\" + echo "[DRY-RUN] --nodes=1 --ntasks-per-node=${tpn_dry} --time=08:00:00 \\" echo "[DRY-RUN] --constraint=mil ${slurm_extra_dry} \\" echo "[DRY-RUN] --job-name=spack.${host}.${env_name} \\" - echo "[DRY-RUN] --account=${ACCOUNT} bash spack-install.${env_name}.sh" + echo "[DRY-RUN] --account=${ACCOUNT} \\" + echo "[DRY-RUN] --output=spack.${host}.${env_name}.log --error=spack.${host}.${env_name}.log \\" + echo "[DRY-RUN] ${install_script_name}" ;; discover-gmao) if [[ "${ACCOUNT}" == "s1873" ]]; then @@ -762,23 +893,30 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do else slurm_extra_dry="(default partition/qos)" fi - echo "[DRY-RUN] salloc --nodes=1 --ntasks-per-node=${tpn_dry} --time=08:00:00 \\" + echo "[DRY-RUN] sbatch --wait \\" + echo "[DRY-RUN] --nodes=1 --ntasks-per-node=${tpn_dry} --time=08:00:00 \\" echo "[DRY-RUN] --constraint=mil ${slurm_extra_dry} \\" echo "[DRY-RUN] --job-name=spack.${host}.${env_name} \\" - echo "[DRY-RUN] --account=${ACCOUNT} bash spack-install.${env_name}.sh" + echo "[DRY-RUN] --account=${ACCOUNT} \\" + echo "[DRY-RUN] --output=spack.${host}.${env_name}.log --error=spack.${host}.${env_name}.log \\" + echo "[DRY-RUN] ${install_script_name}" ;; *) - echo "[DRY-RUN] run_interactive_job ${host} spack-install.${env_name}.sh ${reuse_build_cache}" + echo "[DRY-RUN] run_interactive_job ${host} ${install_script_name} ${reuse_build_cache}" ;; esac else - echo "[DRY-RUN] bash spack-install.${env_name}.sh" + echo "[DRY-RUN] bash ${install_script_name}" fi if [[ "${update_build_cache}" == "true" ]]; then echo "[DRY-RUN] spack buildcache push -u " echo "[DRY-RUN] spack buildcache update-index local-binary" echo "[DRY-RUN] fix_permissions ${host} 0" + elif [[ "${publish_build_cache}" == "true" ]]; then + echo "[DRY-RUN] spack buildcache push -u " + echo "[DRY-RUN] spack buildcache update-index local-binary" + echo "[DRY-RUN] fix_permissions ${host} 0" else echo "[DRY-RUN] spack module ${module_choice} refresh --yes --upstream-modules" echo "[DRY-RUN] spack stack setup-meta-modules" @@ -800,33 +938,22 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do # Reset environment echo "Resetting environment ..." case ${host} in - atlantis) - umask 0022 - module purge - case ${compiler} in - clang@=22.1.0) - module use /gpfs/neptune/spack-stack/llvm-22.1.0/modulefiles - module use /gpfs/neptune/spack-stack/openmpi-4.1.8/llvm-22.1.0/modulefiles - ;; - gcc@=13.4.0) - module use /gpfs/neptune/spack-stack/gcc-13.4.0/modulefiles - module use /gpfs/neptune/spack-stack/openmpi-4.1.8/gcc-13.4.0/modulefiles - ;; - oneapi@=2025.3.0) - module use /gpfs/neptune/spack-stack/oneapi-2025.3.0/modulefiles - ;; - esac - ;; nas) umask 0022 set +e module purge + module use /nasa/modulefiles/toss4 + module use /nobackup/gmao_SIteam/modulefiles + module use /nasa/modulefiles/testing set -e ;; nas-toss5) umask 0022 set +e module purge + module use /opt/cray/pe/modulefiles + module use /nasa/modulefiles/toss5_crayex + module use /nobackup/gmao_SIteam/modulefiles-TOSS5 set -e ;; discover) @@ -841,6 +968,12 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do module purge set -e ;; + bucy) + umask 0022 + set +e + module purge + set -e + ;; macos.gmao) set +e ulimit -s unlimited 2>/dev/null || ulimit -s hard 2>/dev/null || ulimit -s 65532 2>/dev/null || true @@ -938,12 +1071,20 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do # the geos-gcm-env ~debug spec from the environment spack.yaml if present. if [[ "${host}" == "macos.gmao" ]]; then env_spack_yaml="${env_dir}/spack.yaml" - if grep -q "geos-gcm-env ~debug" "${env_spack_yaml}" 2>/dev/null; then - echo "INFO: macOS: removing 'geos-gcm-env ~debug' spec from ${env_spack_yaml}" - grep -v "geos-gcm-env ~debug" "${env_spack_yaml}" > "${env_spack_yaml}.tmp" && mv "${env_spack_yaml}.tmp" "${env_spack_yaml}" + if grep -qE 'geos-gcm-env([^[:space:]]*)?[[:space:]]+~debug' "${env_spack_yaml}" 2>/dev/null; then + echo "INFO: macOS: removing GEOS-GCM '~debug' specs from ${env_spack_yaml}" + grep -vE 'geos-gcm-env([^[:space:]]*)?[[:space:]]+~debug' "${env_spack_yaml}" > "${env_spack_yaml}.tmp" && mv "${env_spack_yaml}.tmp" "${env_spack_yaml}" fi fi + # GrADS is needed by unified GEOS environments on these production hosts, + # but should not add its GUI dependency chain elsewhere. + if [[ "${template}" == "unified-dev" && + ( "${host}" == "discover" || "${host}" == "nas" || "${host}" == "nas-toss5" ) ]]; then + env_spack_yaml="${env_dir}/spack.yaml" + sed -i -E '/geos-gcm-env/ s/(geos-gcm-env[^[:space:]]*)/\1 +grads/' "${env_spack_yaml}" + fi + # Clean up the generated yamls in the site configuration now that the env is created if [[ "${host}" == "macos.gmao" && ! ${env_exists} == "true" ]]; then rm -f "${SPACK_STACK_DIR}/configs/sites/tier2/${host}/mirrors.yaml" @@ -952,17 +1093,35 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do fi spack env activate -p ${env_dir} + # Environment configuration can select a different miscellaneous-cache + # location. Clear it after activation so package and patch indexes reflect + # local recipe changes made since the environment was created. + spack clean -m + if [[ "${host}" == "macos.gmao" && ! ${env_exists} == "true" ]]; then echo "Running spack external find for macOS generic packages..." - spack external find --not-buildable autoconf automake bash cmake cvs doxygen gawk git-lfs groff libtool ninja npm subversion swig texinfo + # cmake is excluded here and injected manually below with ~doc to prevent + # the solver from considering cmake+doc (which pulls in py-sphinx@:6.1 -> python@:3.12) + spack external find --not-buildable autoconf automake bash cvs doxygen gawk git-lfs groff libtool ninja npm subversion swig texinfo brew_prefix=$(brew --prefix) tcsh_version=$(${brew_prefix}/bin/tcsh --version | awk '{print $2}') rust_version=$(${brew_prefix}/bin/rustc --version | awk '{print $2}') + cmake_version=$(${brew_prefix}/bin/cmake --version | head -1 | awk '{print $3}') + + if [[ -z "${rust_version}" ]]; then + echo "ERROR: Unable to determine the Homebrew Rust version from ${brew_prefix}/bin/rustc" + exit 1 + fi echo "Manually injecting tricky macOS packages into Spack configuration..." cat << EOF > spack-macos-externals.yaml packages: + cmake: + externals: + - spec: cmake@${cmake_version}+ownlibs+doc + prefix: ${brew_prefix} + buildable: false tcsh: externals: - spec: tcsh@${tcsh_version} @@ -1028,8 +1187,17 @@ EOF exit 1 fi - # Check for duplicate packages - ./util/show_duplicate_packages.py -i crtm -i crtm-fix -i esmf -i mapl -i neptune-env -i py-cython -i ip -i fms -i geos-gcm-env + # Check for duplicate packages. py-versioneer is intentionally duplicated: + # py-pyogrio requires 0.28 while py-partd (required by modern py-dask) + # requires 0.29; both are build-only dependencies. + ./util/show_duplicate_packages.py -i crtm -i crtm-fix -i esmf -i mapl -i neptune-env -i py-cython -i ip -i fms -i geos-gcm-env -i py-versioneer + + # Stop here if --concretize-only / -o was requested + if [[ "${SPACK_STACK_CONCRETIZE_ONLY}" == "true"* ]]; then + echo "INFO: --concretize-only requested; stopping after concretization." + echo "INFO: Concretization log: ${SPACK_STACK_DIR}/logs/log.concretize.${env_name}.${LOG_TIMESTAMP}" + exit 0 + fi # Update local source cache if requested if [[ "${update_source_cache}" == "true"* ]]; then @@ -1067,8 +1235,14 @@ EOF case ${submit_to_scheduler} in "true") - jobs=$(tasks_per_node ${host}) - parallel_install_flags="--concurrent-packages=2 --jobs=${jobs}" + if [[ "${SPACK_STACK_DISABLE_LOCKS}" == "true" ]]; then + # --disable-locks is a command-line setting and is not inherited by + # the worker processes used for concurrent package installation. + # Keep installs serial when locks are disabled. + parallel_install_flags="--concurrent-packages=1 --jobs=${spack_build_jobs}" + else + parallel_install_flags="--concurrent-packages=2 --jobs=${spack_build_jobs}" + fi ;; "false") parallel_install_flags="" @@ -1079,7 +1253,7 @@ EOF ;; esac - install_script=${PWD}/spack-install.${env_name}.sh + install_script=${PWD}/${install_script_name} # Locally ignore the generated install script in git without changing global .gitignore if [[ -d "${SPACK_STACK_DIR}/.git" ]] && ! grep -q "^spack-install\.\*\.sh$" "${SPACK_STACK_DIR}/.git/info/exclude" 2>/dev/null; then @@ -1091,6 +1265,59 @@ EOF set -e +# Initialize the module system (needed when running as a batch job via sbatch, +# where lmod shell functions are not automatically available). +if [[ -f /usr/share/lmod/lmod/init/bash ]]; then + source /usr/share/lmod/lmod/init/bash +fi +set +e +module purge +set -e + +# Point cargo at the local mirror so rust builds don't try to reach the internet. +export SPACK_CARGO_HOME=${cargo_mirror_path} +export CARGO_NET_OFFLINE=true + +if [[ "${SPACK_STACK_DISABLE_LOCKS}" == "true" ]]; then + # Spack's package workers do not inherit command-line configuration or the + # environment's config scopes. Give every worker a private user config scope + # with locking disabled for this isolated recovery job only. + spack_lock_config_dir=\$(mktemp -d "\${TMPDIR:-/tmp}/spack-stack-lock-config.XXXXXX") + cat > "\${spack_lock_config_dir}/config.yaml" <<'LOCK_CONFIG_EOF' +config: + locks: false +LOCK_CONFIG_EOF + export SPACK_USER_CONFIG_PATH="\${spack_lock_config_dir}" + # setup.sh normally disables local configuration for reproducibility. The + # private scope above is intentionally the sole local configuration for -L. + unset SPACK_DISABLE_LOCAL_CONFIG + echo "INFO: -L using job-local config: \${spack_lock_config_dir}/config.yaml" +fi + +# Build stages use the paths configured in the site config.yaml +# (under /discover/nobackup/projects/gmao/SIteam/spack-stack/cache/). +# TSE_TMPDIR is not used: its 200k inode quota is insufficient for large +# packages like rustc which extract tens of thousands of files. + +# On Discover, the 2025.3 Intel MPI module is visible only after its compiler +# module is loaded. Preload the pair in the batch shell so every Spack package +# worker inherits a valid module hierarchy. +if [[ "${host}" == "discover" || "${host}" == "discover-gmao" ]] && \ + [[ "${compiler}" == "oneapi@=2025.3.0" ]]; then + echo "INFO: preloading comp/intel/2025.3.0 and mpi/impi/2021.17" + module load comp/intel/2025.3.0 mpi/impi/2021.17 +elif [[ "${host}" == "discover" || "${host}" == "discover-gmao" ]] && \ + [[ "${compiler}" == "gcc@=15.2.0" ]]; then + echo "INFO: preloading comp/gcc/15.2.0 and mpi/openmpi/5.0.10/gcc-15.2.0" + module load comp/gcc/15.2.0 mpi/openmpi/5.0.10/gcc-15.2.0 +elif [[ "${host}" == "discover-gmao" ]] && \ + [[ "${compiler}" == "nag@=7.2.7238" ]]; then + # This external OpenMPI was built with GCC C/C++ and NAG Fortran. Lmod + # requires the two compiler modules before the compiler-specific MPI module. + echo "INFO: preloading comp/gcc/12.3.0, comp/nag/7.2-7238, and mpi/openmpi/4.1.6/nag_7.2.7238-gcc_12.3.0" + module load comp/gcc/12.3.0 comp/nag/7.2-7238 mpi/openmpi/4.1.6/nag_7.2.7238-gcc_12.3.0 +fi + $(declare -p test_packages) # Workaround for ectrans build failure with oneapi at NAS (nas/nas-toss5). @@ -1104,11 +1331,11 @@ if [[ "${host}" == "nas" || "${host}" == "nas-toss5" ]] && \ [[ "${compiler_name}" == "oneapi" ]]; then set -o pipefail echo "Installing ecbuild before ectrans workaround ..." - spack install --verbose ${buildcache_install_flags} ecbuild 2>&1 | tee ${SPACK_STACK_DIR}/logs/log.install.${env_name}.${LOG_TIMESTAMP}.ecbuild + spack ${spack_install_lock_flag} install --verbose ${buildcache_install_flags} ecbuild 2>&1 | tee ${SPACK_STACK_DIR}/logs/log.install.${env_name}.${LOG_TIMESTAMP}.ecbuild ecbuild_flags_cmake=\$(spack location -i ecbuild)/share/ecbuild/cmake/ecbuild_add_lang_flags.cmake echo "Applying ectrans/ecbuild workaround to \${ecbuild_flags_cmake} ..." ${SPACK_STACK_DIR}/util/gmao/patch_ecbuild_ectrans.py --patch \${ecbuild_flags_cmake} - spack install --verbose ${buildcache_install_flags} ectrans 2>&1 | tee ${SPACK_STACK_DIR}/logs/log.install.${env_name}.${LOG_TIMESTAMP}.ectrans + spack ${spack_install_lock_flag} install --verbose ${buildcache_install_flags} ectrans 2>&1 | tee ${SPACK_STACK_DIR}/logs/log.install.${env_name}.${LOG_TIMESTAMP}.ectrans set +o pipefail echo "Reverting ectrans/ecbuild workaround ..." ${SPACK_STACK_DIR}/util/gmao/patch_ecbuild_ectrans.py --revert \${ecbuild_flags_cmake} @@ -1117,7 +1344,7 @@ fi # If no tests are required, install everything if [[ \${#test_packages[@]} -eq 0 ]]; then set -o pipefail - spack install --verbose ${buildcache_install_flags} ${parallel_install_flags} 2>&1 | tee ${SPACK_STACK_DIR}/logs/log.install.${env_name}.${LOG_TIMESTAMP} + spack ${spack_install_lock_flag} install --verbose ${buildcache_install_flags} ${parallel_install_flags} 2>&1 | tee ${SPACK_STACK_DIR}/logs/log.install.${env_name}.${LOG_TIMESTAMP} set +o pipefail else for (( idx=0; idx<\${#test_packages[@]}; idx++ )); do @@ -1128,15 +1355,15 @@ else set -e idx_padded=\$(printf "%03d" "\$((idx+1))") set -o pipefail - spack install --verbose ${buildcache_install_flags} ${parallel_install_flags} --only=dependencies \${test_package} \\ + spack ${spack_install_lock_flag} install --verbose ${buildcache_install_flags} ${parallel_install_flags} --only=dependencies \${test_package} \\ 2>&1 | tee ${SPACK_STACK_DIR}/logs/log.install.${env_name}.${LOG_TIMESTAMP}.\${idx_padded}.\${test_package}-dependencies - spack install --verbose --no-cache --test=root \${test_package} 2>&1 | tee ${SPACK_STACK_DIR}/logs/log.install.${env_name}.${LOG_TIMESTAMP}.\${idx_padded}.\${test_package} + spack ${spack_install_lock_flag} install --verbose --no-cache --test=root \${test_package} 2>&1 | tee ${SPACK_STACK_DIR}/logs/log.install.${env_name}.${LOG_TIMESTAMP}.\${idx_padded}.\${test_package} set +o pipefail done # idx now equals the length of the array; install the rest idx_padded=\$(printf "%03d" "\$((idx+1))") set -o pipefail - spack install --verbose ${buildcache_install_flags} ${parallel_install_flags} 2>&1 | tee ${SPACK_STACK_DIR}/logs/log.install.${env_name}.${LOG_TIMESTAMP}.\${idx_padded} + spack ${spack_install_lock_flag} install --verbose ${buildcache_install_flags} ${parallel_install_flags} 2>&1 | tee ${SPACK_STACK_DIR}/logs/log.install.${env_name}.${LOG_TIMESTAMP}.\${idx_padded} set +o pipefail fi EOF @@ -1147,8 +1374,9 @@ EOF bash ${install_script} fi - # In build mode, update local binary cache - if [[ "${update_build_cache}" == "true" ]]; then + # Publish binaries for build mode and for the private local mode. Local + # mode keeps the final environment name and still generates its modules. + if [[ "${publish_build_cache}" == "true" ]]; then spack buildcache push -u ${binary_mirror_path} spack buildcache update-index local-binary fi @@ -1164,7 +1392,7 @@ EOF if [[ "${update_source_cache}" == "true" ]]; then fix_permissions ${host} ${source_mirror_path} 0 fi - if [[ "${update_build_cache}" == "true" ]]; then + if [[ "${publish_build_cache}" == "true" ]]; then fix_permissions ${host} ${binary_mirror_path} 0 fi if [[ "${update_cargo_mirror}" == "true" ]]; then diff --git a/util/install_rust.sh b/util/install_rust.sh index 6c5a92dfb..839e120f9 100755 --- a/util/install_rust.sh +++ b/util/install_rust.sh @@ -5,6 +5,7 @@ set -ex WORKDIR=${1} +export CARGO_HOME=${SPACK_CARGO_HOME:-${CARGO_HOME}} export RUSTUP_HOME=${CARGO_HOME}/rustup mkdir -p ${WORKDIR} @@ -13,5 +14,5 @@ curl https://sh.rustup.rs -sSf > install.sh chmod u+x install.sh ./install.sh --no-modify-path -y --default-toolchain stable --quiet -export PATH=${CARGO_HOME}/bin +export PATH=${CARGO_HOME}/bin:${PATH} rustup default stable