diff --git a/CHANGELOG.md b/CHANGELOG.md index eef5278..9f6043b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## Unreleased + +### CI/CD +- Add `workflows/base/spack/deployment/use.sh` with reusable + `kessel_detect_system`, `kessel_activate_deployment` and `kessel_alloc` + helpers, so projects no longer copy the system-detection, clone-or-activate and + node-allocation logic into their `.gitlab/kessel.sh`. The spack-project + template now sources these from the deployment's kessel, and the deployment's + `activate.sh` sources them too so they are available in any activated + environment. +- Add `workflows/base/spack/deployment/alloc-init.sh`, the wrapper `kessel_alloc` + hands to the scheduler to activate the deployment on an allocated node. +- Add a `--persist PATH` option (via `kessel_parse_persist`) to the deployment + entry-point scripts, so users can create a reusable writable deployment without + having to know about the `KESSEL_WORKFLOW_DEPLOYMENT` environment variable. + +### Workflows +- Add an optional `alloc` parameter to `default_ci_message` (and + `BuildEnvironment.ci_message`) to override the allocation line. + ## 0.3.0 ### Spack Integration diff --git a/lib/kessel/workflows/__init__.py b/lib/kessel/workflows/__init__.py index e5117ca..80d309a 100644 --- a/lib/kessel/workflows/__init__.py +++ b/lib/kessel/workflows/__init__.py @@ -134,9 +134,9 @@ def default_ci_message(project: str, workflow: str = "default", args: list[str] = sys.argv[1:], pre_alloc_init: str = "", - post_alloc_init: str = "") -> str: + post_alloc_init: str = "", + alloc: str | None = None) -> str: system_change = "" - alloc = "" workflow_change = "" kessel_cmd = "kessel " + " ".join(args) @@ -149,10 +149,16 @@ def default_ci_message(project: str, if post_alloc_init: post_alloc_init += "\n" - if "LLNL_FLUX_SCHEDULER_PARAMETERS" in os.environ: - alloc = f"flux alloc {os.environ['LLNL_FLUX_SCHEDULER_PARAMETERS']}\n" - elif "SCHEDULER_PARAMETERS" in os.environ: - alloc = f"salloc {os.environ['SCHEDULER_PARAMETERS']}\n" + if alloc is None: + if "LLNL_FLUX_SCHEDULER_PARAMETERS" in os.environ: + alloc = f"flux alloc {os.environ['LLNL_FLUX_SCHEDULER_PARAMETERS']}" + elif "SCHEDULER_PARAMETERS" in os.environ: + alloc = f"salloc {os.environ['SCHEDULER_PARAMETERS']}" + else: + alloc = "" + + if alloc: + alloc += "\n" if workflow != "default": workflow_change = f"kessel activate {workflow}\n" @@ -273,10 +279,9 @@ def load_workflow(name: str) -> Workflow: def git(cmd, cwd=None, check=True) -> None | str: """Run git command and return output, suppressing normal output.""" env = os.environ.copy() - env["GIT_ADVICE_DETACHED_HEAD"] = "false" try: result = subprocess.run( - ["git"] + cmd, + ["git", "-c", "advice.detachedHead=false"] + cmd, cwd=cwd, check=check, stdout=subprocess.PIPE, diff --git a/lib/kessel/workflows/base/cmake/cmake/build.sh b/lib/kessel/workflows/base/cmake/cmake/build.sh index 79e08fe..c737c43 100644 --- a/lib/kessel/workflows/base/cmake/cmake/build.sh +++ b/lib/kessel/workflows/base/cmake/cmake/build.sh @@ -15,5 +15,9 @@ if [ -f "$KESSEL_BUILD_ENV" ]; then source "$KESSEL_BUILD_ENV" fi cmake "$@" "$KESSEL_BUILD_DIR" -cmake --build "$KESSEL_BUILD_DIR" --parallel ${KESSEL_CMAKE_TARGETS} +if [ -n "${CMAKE_BUILD_PARALLEL_LEVEL}" ]; then + cmake --build "$KESSEL_BUILD_DIR" ${KESSEL_CMAKE_TARGETS} +else + cmake --build "$KESSEL_BUILD_DIR" --parallel ${KESSEL_CMAKE_TARGETS} +fi ) diff --git a/lib/kessel/workflows/base/spack/__init__.py b/lib/kessel/workflows/base/spack/__init__.py index bbc1e52..13c44bb 100644 --- a/lib/kessel/workflows/base/spack/__init__.py +++ b/lib/kessel/workflows/base/spack/__init__.py @@ -70,7 +70,8 @@ def init(self) -> None: os.environ["KESSEL_DEPLOYMENT"]).exists()) and "SPACK_ROOT" not in os.environ: raise Exception("No active Spack installation!") - def ci_message(self, parsed_args, pre_alloc_init: str = "", post_alloc_init: str = "") -> str: + def ci_message(self, parsed_args, pre_alloc_init: str = "", post_alloc_init: str = "", + alloc: str | None = None) -> str: if parsed_args.project_spec: project = get_project_name_from_spec(" ".join(parsed_args.project_spec)) else: @@ -81,7 +82,8 @@ def ci_message(self, parsed_args, pre_alloc_init: str = "", post_alloc_init: str system=system, workflow=self.workflow, pre_alloc_init=pre_alloc_init, - post_alloc_init=post_alloc_init) + post_alloc_init=post_alloc_init, + alloc=alloc) def prepare_env(self, args: argparse.Namespace) -> None: self.environ["KESSEL_ENABLE_VIEW"] = "true" if self.view else "false" diff --git a/lib/kessel/workflows/base/spack/deployment/__init__.py b/lib/kessel/workflows/base/spack/deployment/__init__.py new file mode 100644 index 0000000..b4e03ee --- /dev/null +++ b/lib/kessel/workflows/base/spack/deployment/__init__.py @@ -0,0 +1,16 @@ +# © 2026. Triad National Security, LLC. All rights reserved. +# This program was produced under U.S. Government contract 89233218CNA000001 +# for Los Alamos National Laboratory (LANL), which is operated by Triad +# National Security, LLC for the U.S. Department of Energy/National Nuclear +# Security Administration. All rights in the program are reserved by Triad +# National Security, LLC, and the U.S. Department of Energy/National Nuclear +# Security Administration. The Government is granted for itself and others +# acting on its behalf a nonexclusive, paid-up, irrevocable worldwide license +# in this material to reproduce, prepare derivative works, distribute copies to +# the public, perform publicly and display publicly, and to permit others to do +# so. +"""Helpers for using an existing Spack deployment from a project checkout. + +Project scripts like `.gitlab/alloc` import this before kessel or Spack are +activated, so nothing here may depend on an activated environment. +""" diff --git a/lib/kessel/workflows/base/spack/deployment/activate.sh b/lib/kessel/workflows/base/spack/deployment/activate.sh index ef225c9..2012fbb 100644 --- a/lib/kessel/workflows/base/spack/deployment/activate.sh +++ b/lib/kessel/workflows/base/spack/deployment/activate.sh @@ -46,3 +46,7 @@ export SPACK_USER_CONFIG_PATH="$KESSEL_DEPLOYMENT/.spack" export SPACK_SYSTEM_CONFIG_PATH="$KESSEL_CONFIG_DIR" source "$KESSEL_DEPLOYMENT/spack/share/spack/setup-env.sh" + +# Make the deployment-use helpers (kessel_detect_system, kessel_alloc, ...) +# available in any activated environment. +source "$KESSEL_ROOT/lib/kessel/workflows/base/spack/deployment/use.sh" diff --git a/lib/kessel/workflows/base/spack/deployment/alloc-init.sh b/lib/kessel/workflows/base/spack/deployment/alloc-init.sh new file mode 100755 index 0000000..a6183f0 --- /dev/null +++ b/lib/kessel/workflows/base/spack/deployment/alloc-init.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# © 2026. Triad National Security, LLC. All rights reserved. +# This program was produced under U.S. Government contract 89233218CNA000001 +# for Los Alamos National Laboratory (LANL), which is operated by Triad +# National Security, LLC for the U.S. Department of Energy/National Nuclear +# Security Administration. All rights in the program are reserved by Triad +# National Security, LLC, and the U.S. Department of Energy/National Nuclear +# Security Administration. The Government is granted for itself and others +# acting on its behalf a nonexclusive, paid-up, irrevocable worldwide license +# in this material to reproduce, prepare derivative works, distribute copies to +# the public, perform publicly and display publicly, and to permit others to do +# so. +# +# Wrapper handed to salloc/flux alloc as $SHELL by kessel_alloc. It runs inside +# the allocation to activate the workflow deployment (creating a writable copy +# only if one doesn't already exist) using KESSEL_DEPLOYMENT inherited from the +# allocating shell, then hands off to a login bash. + +source "$KESSEL_DEPLOYMENT/kessel/lib/kessel/workflows/base/spack/deployment/use.sh" +kessel_activate_deployment || exit $? + +export SHELL=/bin/bash +export -f kessel + +KESSEL_VERSION=$(cat "$KESSEL_WORKFLOW_DEPLOYMENT/.kessel_version" 2>/dev/null) + +BOLD='\033[1m' +RESET='\033[0m' + +echo +echo -e "${BOLD}Spack deployment and Kessel${KESSEL_VERSION:+ ($KESSEL_VERSION)} are now available${RESET}" +echo +echo "Common commands:" +echo +echo -e "${BOLD}spack env list${RESET} show available environments" +echo -e "${BOLD}kessel run${RESET} run workflow with $KESSEL_SPACK_ENV Spack environment" +echo -e "${BOLD}kessel status${RESET} show current workflow state" +echo -e "${BOLD}kessel build-env${RESET} enter build and run environment (requires completed configure step)" +echo + +exec /bin/bash -l diff --git a/lib/kessel/workflows/base/spack/deployment/use.sh b/lib/kessel/workflows/base/spack/deployment/use.sh new file mode 100644 index 0000000..6abb76a --- /dev/null +++ b/lib/kessel/workflows/base/spack/deployment/use.sh @@ -0,0 +1,66 @@ +# © 2026. Triad National Security, LLC. All rights reserved. +# This program was produced under U.S. Government contract 89233218CNA000001 +# for Los Alamos National Laboratory (LANL), which is operated by Triad +# National Security, LLC for the U.S. Department of Energy/National Nuclear +# Security Administration. All rights in the program are reserved by Triad +# National Security, LLC, and the U.S. Department of Energy/National Nuclear +# Security Administration. The Government is granted for itself and others +# acting on its behalf a nonexclusive, paid-up, irrevocable worldwide license +# in this material to reproduce, prepare derivative works, distribute copies to +# the public, perform publicly and display publicly, and to permit others to do +# so. +# +# Activate the writable workflow deployment (KESSEL_WORKFLOW_DEPLOYMENT), +# cloning it from the cluster deployment (KESSEL_DEPLOYMENT) if needed. Defaults +# to a per-user temp copy; pass "--persist PATH" (see kessel_parse_persist) or +# set KESSEL_WORKFLOW_DEPLOYMENT to a path to persist it, or to "upstream" to use +# the (read-only) cluster deployment directly. +kessel_activate_deployment() { + _KESSEL_WORKFLOW_DEPLOYMENT="$KESSEL_WORKFLOW_DEPLOYMENT" + export KESSEL_WORKFLOW_DEPLOYMENT=${KESSEL_WORKFLOW_DEPLOYMENT:-${TMPDIR:-/tmp}/$USER-ci-envs} + + if [ "$KESSEL_WORKFLOW_DEPLOYMENT" = "upstream" ] && [ -d "$KESSEL_DEPLOYMENT" ]; then + source "$KESSEL_DEPLOYMENT/activate.sh" + else + if [ -d "$KESSEL_DEPLOYMENT" ] && [ ! -d "$KESSEL_WORKFLOW_DEPLOYMENT" ]; then + source "$KESSEL_DEPLOYMENT/activate.sh" + echo "Creating writable deployment copy at $KESSEL_WORKFLOW_DEPLOYMENT" + clone-deployment "$KESSEL_WORKFLOW_DEPLOYMENT" > /dev/null + fi + if [ ! -d "$KESSEL_WORKFLOW_DEPLOYMENT" ]; then + echo "ERROR: $KESSEL_WORKFLOW_DEPLOYMENT does not exist!" >&2 + return 1 + elif [ -z "$_KESSEL_WORKFLOW_DEPLOYMENT" ] && [ ! -O "$KESSEL_WORKFLOW_DEPLOYMENT" ]; then + echo "ERROR: $KESSEL_WORKFLOW_DEPLOYMENT not owned by $USER!" >&2 + return 1 + else + source "$KESSEL_WORKFLOW_DEPLOYMENT/activate.sh" + fi + fi + + unset _KESSEL_WORKFLOW_DEPLOYMENT +} + +# Allocate an interactive compute node and drop into the deployment on it. All +# arguments are forwarded to the scheduler as already-split allocation +# parameters. Requires kessel_detect_system to have set KESSEL_SYSTEM_SCHEDULER. +kessel_alloc() { + _kessel_alloc_init="$KESSEL_DEPLOYMENT/kessel/lib/kessel/workflows/base/spack/deployment/alloc-init.sh" + case "$KESSEL_SYSTEM_SCHEDULER" in + slurm) + echo "salloc $*" >&2 + export SHELL="$_kessel_alloc_init" + salloc "$@" + ;; + flux) + echo "flux alloc $* $_kessel_alloc_init" >&2 + flux alloc "$@" "$_kessel_alloc_init" + ;; + *) + echo "ERROR: unknown scheduler '$KESSEL_SYSTEM_SCHEDULER'!" >&2 + unset _kessel_alloc_init + return 1 + ;; + esac + unset _kessel_alloc_init +} diff --git a/lib/kessel/workflows/base/spack/deployment/util.py b/lib/kessel/workflows/base/spack/deployment/util.py new file mode 100644 index 0000000..8e4c168 --- /dev/null +++ b/lib/kessel/workflows/base/spack/deployment/util.py @@ -0,0 +1,78 @@ +# © 2026. Triad National Security, LLC. All rights reserved. +# This program was produced under U.S. Government contract 89233218CNA000001 +# for Los Alamos National Laboratory (LANL), which is operated by Triad +# National Security, LLC for the U.S. Department of Energy/National Nuclear +# Security Administration. All rights in the program are reserved by Triad +# National Security, LLC, and the U.S. Department of Energy/National Nuclear +# Security Administration. The Government is granted for itself and others +# acting on its behalf a nonexclusive, paid-up, irrevocable worldwide license +# in this material to reproduce, prepare derivative works, distribute copies to +# the public, perform publicly and display publicly, and to permit others to do +# so. + +import argparse +import json +import re +import shutil +import socket +import subprocess + +from pathlib import Path + + +def detect_system_name(): + """Detect the current cluster name""" + if shutil.which("sacctmgr"): + return json.loads(subprocess.check_output(["sacctmgr", "list", "--json", "clusters"]))['clusters'][0]['name'] + if shutil.which("flux"): + return re.sub(r'[0-9]+', "", socket.gethostname()) + return None + + +def detect_batch_system(): + """Detect batch system on a cluster""" + if shutil.which("sacctmgr"): + return "slurm" + if shutil.which("flux"): + return "flux" + return None + + +def deployment_environments(deployment_dir : Path | str) -> List[str]: + """Return all available Spack environment in a given deployment""" + env_dir = Path(deployment_dir) / "environments" + return [str(p.parent.relative_to(env_dir)) for p in env_dir.rglob("spack.yaml") if p.is_file()] + + +def alloc_main(prog_name, exports, params_callback): + parser = argparse.ArgumentParser(prog=prog_name, description=("Allocate a compute node for a given Spack environment via the job scheduler.")) + + parser.add_argument( + "--persist", + metavar="PATH", + help=( + "keep a reusable writable deployment at PATH instead of a " + "temporary per-user copy (sets KESSEL_WORKFLOW_DEPLOYMENT)" + ), + ) + parser.add_argument( + "--print-params", + action="store_true", + help=( + "print the resolved scheduler parameters for SPACK_ENV_NAME and exit " + "without allocating" + ), + ) + parser.add_argument( + "spack_env_name", + metavar="SPACK_ENV_NAME", + help="Spack environment to allocate for", + ) + parser.add_argument( + "scheduler_args", + metavar="extra scheduler args", + nargs=argparse.REMAINDER, + help="additional arguments passed to the scheduler", + ) + args = parser.parse_args() + return 0 diff --git a/share/kessel/cmake/ctest_driver.cmake b/share/kessel/cmake/ctest_driver.cmake index e0e8cdb..2130f89 100644 --- a/share/kessel/cmake/ctest_driver.cmake +++ b/share/kessel/cmake/ctest_driver.cmake @@ -22,7 +22,6 @@ set(CTEST_BUILD_CONFIGURATION "RelWithDebInfo") set(CTEST_MODE "$ENV{CTEST_MODE}") set(CTEST_OUTPUT_ON_FAILURE ON) -set(CTEST_USE_LAUNCHERS TRUE) cmake_host_system_information(RESULT NUM_PHYSICAL_CORES QUERY NUMBER_OF_PHYSICAL_CORES) diff --git a/share/kessel/templates/spack-project/.gitlab/kessel.sh b/share/kessel/templates/spack-project/.gitlab/kessel.sh index 194eda0..d292e62 100644 --- a/share/kessel/templates/spack-project/.gitlab/kessel.sh +++ b/share/kessel/templates/spack-project/.gitlab/kessel.sh @@ -1,30 +1,36 @@ +# Pick the KESSEL_DEPLOYMENT for this cluster, then source the shared helpers +# that ship inside the deployment's kessel. +# Intended to work on sh, zsh, and bash. DEPLOYMENT_VERSION="2025-10-21" SCRIPT_PATH=${BASH_SOURCE[0]:-${(%):-%x}} PARENT_DIR="$( cd "$( dirname "${SCRIPT_PATH}" )" &>/dev/null && pwd )" -export FLECSI_CHECKOUT=$(realpath $PARENT_DIR/..) +export PROJECT_CHECKOUT=$(realpath $PARENT_DIR/..) +# Just enough detection to choose the deployment path; kessel_detect_system does +# the rest. if command -v jq >/dev/null 2>&1 && command -v sacctmgr >/dev/null 2>&1; then - SYSTEM_NAME=$(sacctmgr list --json clusters | jq -r '.clusters[0].name') + KESSEL_SYSTEM=$(sacctmgr list --json clusters | jq -r '.clusters[0].name') elif command -v flux >/dev/null 2>&1; then - SYSTEM_NAME=$(hostname | sed 's/[0-9]//g') + KESSEL_SYSTEM=$(hostname | sed 's/[0-9]//g') fi -# During a regular CI run the workflow deployment is always a temporary copy of -# the cluster deployment to allow customization. This can be overwritten by -# setting KESSEL_WORKFLOW_DEPLOYMENT to another persistent location. -export KESSEL_WORKFLOW_DEPLOYMENT="${KESSEL_WORKFLOW_DEPLOYMENT:-${TMPDIR:-/tmp}/$USER-ci-envs}" - -if [ "$SYSTEM_NAME" = "darwin" ]; then +if [ "$KESSEL_SYSTEM" = "darwin" ]; then export KESSEL_DEPLOYMENT=${KESSEL_DEPLOYMENT:-/usr/projects/YOUR_PROJECT/deployments/${DEPLOYMENT_VERSION}} else echo "ERROR: Unknown system!" >&2 return 1 fi -if [ -d "${KESSEL_DEPLOYMENT}" ] && [ ! -d "${KESSEL_WORKFLOW_DEPLOYMENT}" ]; then - source "${KESSEL_DEPLOYMENT}/activate.sh" - clone-deployment "${KESSEL_WORKFLOW_DEPLOYMENT}" +# Source the shared helpers (kessel_detect_system, kessel_activate_deployment). +_KESSEL_DEPLOYMENT_USE="$KESSEL_DEPLOYMENT/kessel/lib/kessel/workflows/base/spack/deployment/use.sh" +if [ ! -r "$_KESSEL_DEPLOYMENT_USE" ]; then + echo "ERROR: $_KESSEL_DEPLOYMENT_USE not found (deployment missing or too old)!" >&2 + return 1 fi +source "$_KESSEL_DEPLOYMENT_USE" +unset _KESSEL_DEPLOYMENT_USE -source "${KESSEL_WORKFLOW_DEPLOYMENT}/activate.sh" +kessel_parse_persist "$@" || return $? +kessel_detect_system +kessel_activate_deployment