Skip to content
Open
Show file tree
Hide file tree
Changes from 11 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,25 @@
# Changelog

## Unreleased

### CI/CD
- Add `workflows/base/spack/deployment/use.sh` with reusable
`kessel_detect_system`, `kessel_activate_deployment` and `kessel_alloc`
helpers, so projects no longer copy the system-detection, clone-or-activate and
node-allocation logic into their `.gitlab/kessel.sh`. The spack-project
template now sources these from the deployment's kessel, and the deployment's
`activate.sh` sources them too so they are available in any activated
environment.
- Add `workflows/base/spack/deployment/alloc-init.sh`, the wrapper `kessel_alloc`
hands to the scheduler to activate the deployment on an allocated node.
- Add a `--persist PATH` option (via `kessel_parse_persist`) to the deployment
entry-point scripts, so users can create a reusable writable deployment without
having to know about the `KESSEL_WORKFLOW_DEPLOYMENT` environment variable.

### Workflows
- Add an optional `alloc` parameter to `default_ci_message` (and
`BuildEnvironment.ci_message`) to override the allocation line.

## 0.3.0

### Spack Integration
Expand Down
21 changes: 13 additions & 8 deletions lib/kessel/workflows/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -134,9 +134,9 @@ def default_ci_message(project: str,
workflow: str = "default",
args: list[str] = sys.argv[1:],
pre_alloc_init: str = "",
post_alloc_init: str = "") -> str:
post_alloc_init: str = "",
alloc: str | None = None) -> str:
system_change = ""
alloc = ""
workflow_change = ""
kessel_cmd = "kessel " + " ".join(args)

Expand All @@ -149,10 +149,16 @@ def default_ci_message(project: str,
if post_alloc_init:
post_alloc_init += "\n"

if "LLNL_FLUX_SCHEDULER_PARAMETERS" in os.environ:
alloc = f"flux alloc {os.environ['LLNL_FLUX_SCHEDULER_PARAMETERS']}\n"
elif "SCHEDULER_PARAMETERS" in os.environ:
alloc = f"salloc {os.environ['SCHEDULER_PARAMETERS']}\n"
if alloc is None:
if "LLNL_FLUX_SCHEDULER_PARAMETERS" in os.environ:
alloc = f"flux alloc {os.environ['LLNL_FLUX_SCHEDULER_PARAMETERS']}"
elif "SCHEDULER_PARAMETERS" in os.environ:
alloc = f"salloc {os.environ['SCHEDULER_PARAMETERS']}"
else:
alloc = ""

if alloc:
alloc += "\n"

if workflow != "default":
workflow_change = f"kessel activate {workflow}\n"
Expand Down Expand Up @@ -273,10 +279,9 @@ def load_workflow(name: str) -> Workflow:
def git(cmd, cwd=None, check=True) -> None | str:
"""Run git command and return output, suppressing normal output."""
env = os.environ.copy()
env["GIT_ADVICE_DETACHED_HEAD"] = "false"
try:
result = subprocess.run(
["git"] + cmd,
["git", "-c", "advice.detachedHead=false"] + cmd,
cwd=cwd,
check=check,
stdout=subprocess.PIPE,
Expand Down
6 changes: 5 additions & 1 deletion lib/kessel/workflows/base/cmake/cmake/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -15,5 +15,9 @@ if [ -f "$KESSEL_BUILD_ENV" ]; then
source "$KESSEL_BUILD_ENV"
fi
cmake "$@" "$KESSEL_BUILD_DIR"
cmake --build "$KESSEL_BUILD_DIR" --parallel ${KESSEL_CMAKE_TARGETS}
if [ -n "${CMAKE_BUILD_PARALLEL_LEVEL}" ]; then
cmake --build "$KESSEL_BUILD_DIR" ${KESSEL_CMAKE_TARGETS}
else
cmake --build "$KESSEL_BUILD_DIR" --parallel ${KESSEL_CMAKE_TARGETS}
fi
Comment thread
AndresYague marked this conversation as resolved.
)
6 changes: 4 additions & 2 deletions lib/kessel/workflows/base/spack/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,8 @@ def init(self) -> None:
os.environ["KESSEL_DEPLOYMENT"]).exists()) and "SPACK_ROOT" not in os.environ:
raise Exception("No active Spack installation!")

def ci_message(self, parsed_args, pre_alloc_init: str = "", post_alloc_init: str = "") -> str:
def ci_message(self, parsed_args, pre_alloc_init: str = "", post_alloc_init: str = "",
alloc: str | None = None) -> str:
if parsed_args.project_spec:
project = get_project_name_from_spec(" ".join(parsed_args.project_spec))
else:
Expand All @@ -81,7 +82,8 @@ def ci_message(self, parsed_args, pre_alloc_init: str = "", post_alloc_init: str
system=system,
workflow=self.workflow,
pre_alloc_init=pre_alloc_init,
post_alloc_init=post_alloc_init)
post_alloc_init=post_alloc_init,
alloc=alloc)

def prepare_env(self, args: argparse.Namespace) -> None:
self.environ["KESSEL_ENABLE_VIEW"] = "true" if self.view else "false"
Expand Down
4 changes: 4 additions & 0 deletions lib/kessel/workflows/base/spack/deployment/activate.sh
Original file line number Diff line number Diff line change
Expand Up @@ -46,3 +46,7 @@ export SPACK_USER_CONFIG_PATH="$KESSEL_DEPLOYMENT/.spack"
export SPACK_SYSTEM_CONFIG_PATH="$KESSEL_CONFIG_DIR"

source "$KESSEL_DEPLOYMENT/spack/share/spack/setup-env.sh"

# Make the deployment-use helpers (kessel_detect_system, kessel_alloc, ...)
# available in any activated environment.
source "$KESSEL_ROOT/lib/kessel/workflows/base/spack/deployment/use.sh"
41 changes: 41 additions & 0 deletions lib/kessel/workflows/base/spack/deployment/alloc-init.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
#!/bin/bash
# © 2026. Triad National Security, LLC. All rights reserved.
# This program was produced under U.S. Government contract 89233218CNA000001
# for Los Alamos National Laboratory (LANL), which is operated by Triad
# National Security, LLC for the U.S. Department of Energy/National Nuclear
# Security Administration. All rights in the program are reserved by Triad
# National Security, LLC, and the U.S. Department of Energy/National Nuclear
# Security Administration. The Government is granted for itself and others
# acting on its behalf a nonexclusive, paid-up, irrevocable worldwide license
# in this material to reproduce, prepare derivative works, distribute copies to
# the public, perform publicly and display publicly, and to permit others to do
# so.
#
# Wrapper handed to salloc/flux alloc as $SHELL by kessel_alloc. It runs inside
# the allocation to activate the workflow deployment (creating a writable copy
# only if one doesn't already exist) using KESSEL_DEPLOYMENT inherited from the
# allocating shell, then hands off to a login bash.

source "$KESSEL_DEPLOYMENT/kessel/lib/kessel/workflows/base/spack/deployment/use.sh"
kessel_activate_deployment || exit $?

export SHELL=/bin/bash
export -f kessel

KESSEL_VERSION=$(cat "$KESSEL_WORKFLOW_DEPLOYMENT/.kessel_version" 2>/dev/null)

BOLD='\033[1m'
RESET='\033[0m'

echo
echo -e "${BOLD}Spack deployment and Kessel${KESSEL_VERSION:+ ($KESSEL_VERSION)} are now available${RESET}"
echo
echo "Common commands:"
echo
echo -e "${BOLD}spack env list${RESET} show available environments"
echo -e "${BOLD}kessel run${RESET} run workflow with $KESSEL_SPACK_ENV Spack environment"
echo -e "${BOLD}kessel status${RESET} show current workflow state"
echo -e "${BOLD}kessel build-env${RESET} enter build and run environment (requires completed configure step)"
echo

exec /bin/bash -l
120 changes: 120 additions & 0 deletions lib/kessel/workflows/base/spack/deployment/use.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
# © 2026. Triad National Security, LLC. All rights reserved.
# This program was produced under U.S. Government contract 89233218CNA000001
# for Los Alamos National Laboratory (LANL), which is operated by Triad
# National Security, LLC for the U.S. Department of Energy/National Nuclear
# Security Administration. All rights in the program are reserved by Triad
# National Security, LLC, and the U.S. Department of Energy/National Nuclear
# Security Administration. The Government is granted for itself and others
# acting on its behalf a nonexclusive, paid-up, irrevocable worldwide license
# in this material to reproduce, prepare derivative works, distribute copies to
# the public, perform publicly and display publicly, and to permit others to do
# so.
#
# Helpers for using an existing Spack deployment: detect the current system,
# activate the (possibly cloned) deployment, and allocate an interactive node.
# Intended to work on sh, zsh, and bash.

# Detect the current cluster (KESSEL_SYSTEM) and its scheduler
# (KESSEL_SYSTEM_SCHEDULER).
kessel_detect_system() {
if command -v sacctmgr >/dev/null 2>&1 && command -v jq >/dev/null 2>&1; then
KESSEL_SYSTEM=$(sacctmgr list --json clusters | jq -r '.clusters[0].name')
KESSEL_SYSTEM_SCHEDULER="slurm"
elif command -v flux >/dev/null 2>&1; then
KESSEL_SYSTEM=$(hostname | sed 's/[0-9]//g')
KESSEL_SYSTEM_SCHEDULER="flux"
fi
export KESSEL_SYSTEM KESSEL_SYSTEM_SCHEDULER
}

# Parse a "--persist PATH" option out of the arguments, exporting
# KESSEL_WORKFLOW_DEPLOYMENT=PATH so the deployment is created (if needed) at a
# persistent, writable location instead of the default per-user temp copy. When
# --persist is not given, any externally-set KESSEL_WORKFLOW_DEPLOYMENT is left
# untouched. The remaining arguments are placed, shell-quoted, in
# KESSEL_ARGS_REST for the caller to re-apply, e.g.:
#
# kessel_parse_persist "$@" || return $?
# eval "set -- $KESSEL_ARGS_REST"
kessel_parse_persist() {
KESSEL_ARGS_REST=""
while [ "$#" -gt 0 ]; do
case "$1" in
--persist)
if [ "$#" -lt 2 ]; then
echo "ERROR: --persist requires a PATH argument!" >&2
return 2
fi
export KESSEL_WORKFLOW_DEPLOYMENT="$2"
shift 2
;;
--persist=*)
export KESSEL_WORKFLOW_DEPLOYMENT="${1#--persist=}"
shift
;;
*)
# Single-quote each remaining arg (escaping embedded quotes) so
# constraint strings survive the eval unscathed.
_kessel_q=$(printf '%s' "$1" | sed "s/'/'\\\\''/g")
KESSEL_ARGS_REST="${KESSEL_ARGS_REST:+$KESSEL_ARGS_REST }'$_kessel_q'"
shift
;;
esac
done
unset _kessel_q
}

# Activate the writable workflow deployment (KESSEL_WORKFLOW_DEPLOYMENT),
# cloning it from the cluster deployment (KESSEL_DEPLOYMENT) if needed. Defaults
# to a per-user temp copy; pass "--persist PATH" (see kessel_parse_persist) or
# set KESSEL_WORKFLOW_DEPLOYMENT to a path to persist it, or to "upstream" to use
# the (read-only) cluster deployment directly.
kessel_activate_deployment() {
_KESSEL_WORKFLOW_DEPLOYMENT="$KESSEL_WORKFLOW_DEPLOYMENT"
export KESSEL_WORKFLOW_DEPLOYMENT=${KESSEL_WORKFLOW_DEPLOYMENT:-${TMPDIR:-/tmp}/$USER-ci-envs}

if [ "$KESSEL_WORKFLOW_DEPLOYMENT" = "upstream" ] && [ -d "$KESSEL_DEPLOYMENT" ]; then
source "$KESSEL_DEPLOYMENT/activate.sh"
else
if [ -d "$KESSEL_DEPLOYMENT" ] && [ ! -d "$KESSEL_WORKFLOW_DEPLOYMENT" ]; then
source "$KESSEL_DEPLOYMENT/activate.sh"
echo "Creating writable deployment copy at $KESSEL_WORKFLOW_DEPLOYMENT"
clone-deployment "$KESSEL_WORKFLOW_DEPLOYMENT" > /dev/null
fi
if [ ! -d "$KESSEL_WORKFLOW_DEPLOYMENT" ]; then
echo "ERROR: $KESSEL_WORKFLOW_DEPLOYMENT does not exist!" >&2
return 1
elif [ -z "$_KESSEL_WORKFLOW_DEPLOYMENT" ] && [ ! -O "$KESSEL_WORKFLOW_DEPLOYMENT" ]; then
echo "ERROR: $KESSEL_WORKFLOW_DEPLOYMENT not owned by $USER!" >&2
return 1
else
source "$KESSEL_WORKFLOW_DEPLOYMENT/activate.sh"
fi
fi

unset _KESSEL_WORKFLOW_DEPLOYMENT
}

# Allocate an interactive compute node and drop into the deployment on it. All
# arguments are forwarded to the scheduler as already-split allocation
# parameters. Requires kessel_detect_system to have set KESSEL_SYSTEM_SCHEDULER.
kessel_alloc() {
_kessel_alloc_init="$KESSEL_DEPLOYMENT/kessel/lib/kessel/workflows/base/spack/deployment/alloc-init.sh"
case "$KESSEL_SYSTEM_SCHEDULER" in
slurm)
echo "salloc $*" >&2
export SHELL="$_kessel_alloc_init"
salloc "$@"
;;
flux)
echo "flux alloc $* $_kessel_alloc_init" >&2
flux alloc "$@" "$_kessel_alloc_init"
;;
*)
echo "ERROR: unknown scheduler '$KESSEL_SYSTEM_SCHEDULER'!" >&2
unset _kessel_alloc_init
return 1
;;
esac
unset _kessel_alloc_init
}
1 change: 0 additions & 1 deletion share/kessel/cmake/ctest_driver.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ set(CTEST_BUILD_CONFIGURATION "RelWithDebInfo")
set(CTEST_MODE "$ENV{CTEST_MODE}")

set(CTEST_OUTPUT_ON_FAILURE ON)
set(CTEST_USE_LAUNCHERS TRUE)

cmake_host_system_information(RESULT NUM_PHYSICAL_CORES QUERY NUMBER_OF_PHYSICAL_CORES)

Expand Down
32 changes: 19 additions & 13 deletions share/kessel/templates/spack-project/.gitlab/kessel.sh
Original file line number Diff line number Diff line change
@@ -1,30 +1,36 @@
# Pick the KESSEL_DEPLOYMENT for this cluster, then source the shared helpers
# that ship inside the deployment's kessel.
# Intended to work on sh, zsh, and bash.
DEPLOYMENT_VERSION="2025-10-21"
SCRIPT_PATH=${BASH_SOURCE[0]:-${(%):-%x}}
PARENT_DIR="$( cd "$( dirname "${SCRIPT_PATH}" )" &>/dev/null && pwd )"

export FLECSI_CHECKOUT=$(realpath $PARENT_DIR/..)
export PROJECT_CHECKOUT=$(realpath $PARENT_DIR/..)

# Just enough detection to choose the deployment path; kessel_detect_system does
# the rest.
if command -v jq >/dev/null 2>&1 && command -v sacctmgr >/dev/null 2>&1; then
SYSTEM_NAME=$(sacctmgr list --json clusters | jq -r '.clusters[0].name')
KESSEL_SYSTEM=$(sacctmgr list --json clusters | jq -r '.clusters[0].name')
elif command -v flux >/dev/null 2>&1; then
SYSTEM_NAME=$(hostname | sed 's/[0-9]//g')
KESSEL_SYSTEM=$(hostname | sed 's/[0-9]//g')
fi

# During a regular CI run the workflow deployment is always a temporary copy of
# the cluster deployment to allow customization. This can be overwritten by
# setting KESSEL_WORKFLOW_DEPLOYMENT to another persistent location.
export KESSEL_WORKFLOW_DEPLOYMENT="${KESSEL_WORKFLOW_DEPLOYMENT:-${TMPDIR:-/tmp}/$USER-ci-envs}"

if [ "$SYSTEM_NAME" = "darwin" ]; then
if [ "$KESSEL_SYSTEM" = "darwin" ]; then
export KESSEL_DEPLOYMENT=${KESSEL_DEPLOYMENT:-/usr/projects/YOUR_PROJECT/deployments/${DEPLOYMENT_VERSION}}
else
echo "ERROR: Unknown system!" >&2
return 1
fi

if [ -d "${KESSEL_DEPLOYMENT}" ] && [ ! -d "${KESSEL_WORKFLOW_DEPLOYMENT}" ]; then
source "${KESSEL_DEPLOYMENT}/activate.sh"
clone-deployment "${KESSEL_WORKFLOW_DEPLOYMENT}"
# Source the shared helpers (kessel_detect_system, kessel_activate_deployment).
_KESSEL_DEPLOYMENT_USE="$KESSEL_DEPLOYMENT/kessel/lib/kessel/workflows/base/spack/deployment/use.sh"
if [ ! -r "$_KESSEL_DEPLOYMENT_USE" ]; then
echo "ERROR: $_KESSEL_DEPLOYMENT_USE not found (deployment missing or too old)!" >&2
return 1
fi
source "$_KESSEL_DEPLOYMENT_USE"
unset _KESSEL_DEPLOYMENT_USE

source "${KESSEL_WORKFLOW_DEPLOYMENT}/activate.sh"
kessel_parse_persist "$@" || return $?
kessel_detect_system
kessel_activate_deployment
Loading