@@ -10,6 +10,6 @@ One file per fact, YAML frontmatter (`type`, `title`, `area`, `section`,
 holds its overview and lists its entries.
 
 - [Code registration](codereg/index.md) — 58 entries
-- [Services](services/index.md) — 9 entries
+- [Services](services/index.md) — 10 entries
 - [MO correlation diagram](mo-diagram/index.md) — 8 entries
 - [wx, viewer and C++](wx-viewer/index.md) — 23 entries
@@ -46,3 +46,4 @@ what the deployment mode needs (GETTING_STARTED, "Deployment modes"):
 - [A calculation can live in any local folder, not only in a data store](calculations-in-any-local-folder.md)
 - [Waiting for login: a lost monitor parks the job, session start catches it up](waiting-for-login.md)
 - [Test suites get a per-run state directory, ports and X display](test-suites-isolate-per-run.md)
+- [Full builds and long suites go to radium/tellurium via tools/offload](long-suites-go-to-offload-hosts.md)
@@ -0,0 +1,17 @@
+---
+type: rule
+title: "Full builds and long suites go to radium/tellurium via tools/offload"
+area: services
+section: "Rules"
+paths: ["tools/offload/run-offload.sh", "tools/offload/remote-run.sh", "tools/offload/Containerfile"]
+issues: []
+---
+**Do not run full builds, `tests/teaching` (about 26 min on niobium) or other
+long suites on niobium; use `tools/offload/run-offload.sh <pushed ref>
+<build|ctest|teaching|apps|run>`.** The ref must be pushed first; the host
+fetches from GitHub. It prints a summary and a PASS/FAIL line, and keeps the
+full log on the host. radium (container, 2 slots) and tellurium (native, 3
+slots, work root `/mnt/games/ecce` only, off after 20:30) are picked
+automatically; exit status 3 means none is reachable, so run locally. Inside
+radium's container `$USER` must be set or the test data server creates no
+account. See `tools/offload/README.md`.
@@ -0,0 +1,60 @@
+# Offloading long builds and test suites
+
+Full builds and `tests/teaching` take long enough to overload the developer
+machine. `run-offload.sh` runs them on one of two other machines instead.
+
+    tools/offload/run-offload.sh [--host radium|tellurium|auto] <git ref> <command> [args]
+
+`<git ref>` is a branch, `origin/<branch>` or a commit that is **already
+pushed to GitHub**: the host fetches from GitHub, not from this machine.
+
+| command | does |
+| --- | --- |
+| `build` | configure (first time only) and `ninja -k 0` |
+| `ctest [-R re ...]` | build, then `ctest -j --output-on-failure` |
+| `teaching [args]` | build, then `tests/teaching/run_tests.py --jobs 4 [args]` |
+| `apps [args]` | build and install into the build dir, then `tests/apps/run_tests.py` |
+| `run <cmd...>` | build, then any command in the checkout (`$SRC`; build tree `$BLD/b`) |
+
+The full log stays on the host (`<host>:.../logs/<time>-<ref>-<pid>.log`);
+only the last 25 lines and one `PASS:`/`FAIL:`/`SKIP:` line come back. The
+exit status is the command's (77 is SKIP). Examples:
+
+    tools/offload/run-offload.sh origin/main build
+    tools/offload/run-offload.sh wip/foo ctest -R 'parsers|tls_'
+    tools/offload/run-offload.sh --host radium origin/main teaching --group B
+
+`run-on-radium.sh` is `run-offload.sh --host radium`.
+
+## Hosts
+
+| | radium | tellurium |
+| --- | --- | --- |
+| role | long test suites | builds |
+| system | Debian 12, 6c/12t, 15 GB; build and tests in the `ecce-trixie-build` rootless podman image (`Containerfile`, `--memory 12g`) | Debian 13 native, 4c/8t, 31 GB |
+| work root | `~/ecce-offload` | `/mnt/games/ecce` only (never the root disk) |
+| concurrency | 2 jobs, `ninja -j6` | 3 jobs, `ninja -j3` |
+
+`--host auto` (the default) prefers tellurium for `build`/`ctest` and radium
+for `teaching`/`apps`, takes the other if the first has no free slot, and
+queues on the first reachable host if both are full. **tellurium is off at
+night (after 20:30)**: a host that does not answer within 5 s is skipped. If
+no host is reachable the script exits 3 and says so; run the command locally
+then. Slots are `flock` files in `<work root>/locks`, so waiting jobs queue
+instead of failing. One job per ref at a time (the build dir is per ref).
+
+## Layout on a host
+
+`<root>/src` is a shallow clone (`--depth 50`, fetching only `main` and the
+requested branch); `wt/<ref>` a detached worktree per ref; `build/<ref>` the
+persistent build tree (`b` normal, `bi` + `install` for `apps`), so a second
+run of a ref is incremental; `home/<ref>` is `$HOME` for the job (tellurium
+also sets `XDG_CACHE_HOME`, `TMPDIR` under the root); `logs/`.
+
+## Rebuilding radium's image
+
+Only when the CI package list changes: copy `Containerfile` to
+`~/ecce-offload/image` on radium and `podman build -t ecce-trixie-build .`
+there (about 10 min, ~700 MB of downloads, so not casually).
+Inside the container `$USER` is set explicitly; without it the data server
+creates no account and every teaching case fails with "no such parent".
@@ -3,21 +3,23 @@
 set -u
 ref=$1; shift
 cmd=$1; shift
-OFF=$HOME/ecce-offload
+# Set by run-offload.sh: OFF (work root), MODE (container|native), SLOTS, JOBS.
+OFF=${OFF:-$HOME/ecce-offload}
+MODE=${MODE:-container}
 REPO_URL=https://github.com/FriendsofECCE/ECCE.git
 IMAGE=ecce-trixie-build
-SLOTS=2
-JOBS=6
-mkdir -p "$OFF"/{src,wt,build,logs,locks,home}
+SLOTS=${SLOTS:-2}
+JOBS=${JOBS:-6}
+mkdir -p "$OFF"/{src,wt,build,logs,locks,home,cache,tmp}
 
-safe=$(printf %s "$ref" | tr -c 'A-Za-z0-9._-\n' '_')
+safe=$(printf %s "$ref" | tr -c 'A-Za-z0-9._-' '_')
 stamp=$(date +%Y%m%d-%H%M%S)
-log=$OFF/logs/$stamp-$safe.log
+log=$OFF/logs/$stamp-$safe-$$.log
 exec 3>&1
 # Everything below goes to the log; the summary is printed on fd 3.
 exec >"$log" 2>&1
 say() { echo "$@" >&3; }
-fail() { say "FAIL: $*  (log: radium:$log)"; exit 1; }
+fail() { say "FAIL: $*  (log: $(hostname):$log)"; exit 1; }
 
 # Per-ref lock: two jobs on one ref would share one build directory.
 exec 8>"$OFF/locks/ref-$safe"
@@ -59,41 +61,50 @@ wt=$OFF/wt/$safe
 bdir=$OFF/build/$safe
 mkdir -p "$bdir" "$OFF/home/$safe"
 echo "ref $ref = $sha, command: $cmd $*"
-say "radium: $ref = ${sha:0:10}, $cmd $* (slot $got)"
+say "$(hostname): $ref = ${sha:0:10}, $cmd $* (slot $got)"
 
-# Commands run inside the container; /work/src is the checkout, /work/build
-# the persistent per-ref build directory.
+# $SRC is the checkout, $BLD the persistent per-ref build directory.
 case $cmd in
-  build)    inner='configure; ninja -C /work/build/b -j'$JOBS' -k 0' ;;
-  ctest)    inner='configure; ninja -C /work/build/b -j'$JOBS' -k 0 && cd /work/build/b && ctest -j2 --output-on-failure "$@"' ;;
-  teaching) inner='configure; ninja -C /work/build/b -j'$JOBS' -k 0 && cd /work/src && tests/teaching/run_tests.py --build /work/build/b --jobs 4 "$@"' ;;
-  apps)     inner='install_tree; cd /work/src && ECCE_TEST_HOME=/work/build/install ECCE_TEST_WRAPPERS=/work/build/install-bin tests/apps/run_tests.py "$@"' ;;
-  run)      inner='configure; cd /work/src && "$@"' ;;
+  build)    inner='configure; ninja -C $BLD/b -j'$JOBS' -k 0' ;;
+  ctest)    inner='configure; ninja -C $BLD/b -j'$JOBS' -k 0 && cd $BLD/b && ctest -j${CTEST_JOBS:-2} --output-on-failure "$@"' ;;
+  teaching) inner='configure; ninja -C $BLD/b -j'$JOBS' -k 0 && cd $SRC && tests/teaching/run_tests.py --build $BLD/b --jobs 4 "$@"' ;;
+  apps)     inner='install_tree; cd $SRC && ECCE_TEST_HOME=$BLD/install ECCE_TEST_WRAPPERS=$BLD/install-bin tests/apps/run_tests.py "$@"' ;;
+  run)      inner='configure; cd $SRC && "$@"' ;;
   *) fail "unknown command '$cmd' (build|ctest|teaching|apps|run)" ;;
 esac
 prelude='
 configure() {
-  [ -f /work/build/b/build.ninja ] || cmake -G Ninja -S /work/src -B /work/build/b -DECCE_REQUIRE_LIBSSH=ON || return 1
+  [ -f $BLD/b/build.ninja ] || cmake -G Ninja -S $SRC -B $BLD/b -DECCE_REQUIRE_LIBSSH=ON || return 1
 }
 install_tree() {
-  [ -f /work/build/bi/build.ninja ] || cmake -G Ninja -S /work/src -B /work/build/bi \
-    -DCMAKE_INSTALL_PREFIX=/work/build/install -DECCE_HOME_DIR=/work/build/install \
-    -DECCE_WRAPPER_DESTINATION=/work/build/install-bin || return 1
-  ninja -C /work/build/bi -j'$JOBS' -k 0 && ninja -C /work/build/bi install
+  [ -f $BLD/bi/build.ninja ] || cmake -G Ninja -S $SRC -B $BLD/bi \
+    -DCMAKE_INSTALL_PREFIX=$BLD/install -DECCE_HOME_DIR=$BLD/install \
+    -DECCE_WRAPPER_DESTINATION=$BLD/install-bin || return 1
+  ninja -C $BLD/bi -j'$JOBS' -k 0 && ninja -C $BLD/bi install
 }
 set -e'
 
 start=$(date +%s)
-podman run --rm --memory 12g --memory-swap 12g --userns=keep-id --shm-size 1g \
-    -e HOME=/work/home -e TERM=dumb -e CTEST_OUTPUT_ON_FAILURE=1 \
-    -v "$wt":/work/src -v "$bdir":/work/build -v "$OFF/home/$safe":/work/home \
-    -w /work/src "$IMAGE" bash -c "$prelude
+# Inside the container the checkout, build dir and home are /work/*; native
+# hosts use the real paths.  Only the work root is ever written to.
+if [ "$MODE" = container ]; then
+    podman run --rm --memory 12g --memory-swap 12g --userns=keep-id --shm-size 1g \
+        -e HOME=/work/home -e USER="$(id -un)" -e LOGNAME="$(id -un)" -e SRC=/work/src -e BLD=/work/build -e TERM=dumb \
+        -e CTEST_OUTPUT_ON_FAILURE=1 \
+        -v "$wt":/work/src -v "$bdir":/work/build -v "$OFF/home/$safe":/work/home \
+        -w /work/src "$IMAGE" bash -c "$prelude
 $inner" bash "$@" </dev/null
+else
+    PATH=$PATH:/usr/sbin:/sbin HOME=$OFF/home/$safe SRC=$wt BLD=$bdir XDG_CACHE_HOME=$OFF/cache TMPDIR=$OFF/tmp \
+        TERM=dumb CTEST_OUTPUT_ON_FAILURE=1 \
+        bash -c "$prelude
+$inner" bash "$@" </dev/null
+fi
 rc=$?
 secs=$(( $(date +%s) - start ))
 
 say "---- last lines of the log ----"
 tail -n 25 "$log" >&3
 if [ $rc -eq 0 ]; then verdict=PASS; elif [ $rc -eq 77 ]; then verdict=SKIP; else verdict=FAIL; fi
-say "$verdict: $cmd $* on $ref (${sha:0:10}) exit $rc, ${secs}s  (log: radium:$log)"
+say "$verdict: $cmd $* on $ref (${sha:0:10}) exit $rc, ${secs}s  (log: $(hostname):$log)"
 exit $rc
@@ -0,0 +1,49 @@
+#!/bin/bash
+# Run a long ECCE build or test suite on an offload machine.
+#   run-offload.sh [--host radium|tellurium|auto] <git ref> <build|ctest [-R re]|teaching|apps|run ...> [args]
+# The ref must already be pushed to GitHub.  See tools/offload/README.md.
+set -eu
+host=auto
+if [ "${1:-}" = --host ]; then host=$2; shift 2; fi
+if [ $# -lt 2 ]; then
+    sed -n '2,4p' "$0" | sed 's/^# \{0,1\}//' >&2
+    exit 2
+fi
+here=$(cd "$(dirname "$0")" && pwd)
+
+# Per-host settings: work root, container or native, parallel jobs, ninja -j.
+conf() {
+    case $1 in
+      radium)    echo 'OFF=$HOME/ecce-offload MODE=container SLOTS=2 JOBS=6' ;;
+      tellurium) echo 'OFF=/mnt/games/ecce MODE=native SLOTS=3 JOBS=3 CTEST_JOBS=3' ;;
+      *) echo "unknown host $1" >&2; return 1 ;;
+    esac
+}
+# Free slots on a host (-1 if unreachable, within 5 s).
+free() {
+    ssh -o BatchMode=yes -o ConnectTimeout=5 "$1" "$(conf "$1"); n=0; mkdir -p \$OFF/locks;
+      for i in \$(seq 1 \$SLOTS); do ( exec 9>\$OFF/locks/slot\$i; flock -n 9 ) && n=\$((n+1)); done; echo \$n" 2>/dev/null || echo -1
+}
+if [ "$host" = auto ]; then
+    # Builds prefer tellurium, test suites radium.  A host that does not
+    # answer (tellurium is off at night) is skipped; if every reachable host
+    # is full the job queues on the first reachable one.
+    case $2 in build|ctest) order="tellurium radium" ;; *) order="radium tellurium" ;; esac
+    host= queue=
+    for h in $order; do
+        n=$(free "$h")
+        [ "$n" -ge 0 ] || { echo "offload: $h unreachable, skipped" >&2; continue; }
+        [ -n "$queue" ] || queue=$h
+        if [ "$n" -gt 0 ]; then host=$h; break; fi
+    done
+    host=${host:-$queue}
+    if [ -z "$host" ]; then
+        echo "offload: no host reachable; run the command locally (niobium)" >&2
+        exit 3
+    fi
+    echo "offload host: $host" >&2
+fi
+envs=$(conf "$host")
+args=$(printf '%q ' "$@")
+# remote-run.sh is sent on stdin, so the host needs no checkout to start from.
+exec ssh -o BatchMode=yes -o ConnectTimeout=5 -o ServerAliveInterval=30 "$host" "env $envs bash -s -- $args" < "$here/remote-run.sh"
@@ -1,14 +1,3 @@
-#!/bin/sh
-# Run a long ECCE build or test suite on radium instead of this machine.
-#   run-on-radium.sh <git ref> <build|ctest [-R re]|teaching|apps|run ...> [args]
-# The ref must already be pushed to GitHub.  See tools/offload/README.md.
-set -eu
-if [ $# -lt 2 ]; then
-    sed -n '2,4p' "$0" | sed 's/^# \{0,1\}//' >&2
-    exit 2
-fi
-here=$(cd "$(dirname "$0")" && pwd)
-host=${RADIUM_HOST:-radium}
-args=$(printf '%q ' "$@")
-# remote-run.sh is sent on stdin, so radium needs no checkout to start from.
-exec ssh -o BatchMode=yes -o ServerAliveInterval=30 "$host" "bash -s -- $args" < "$here/remote-run.sh"
+#!/bin/bash
+# Thin wrapper kept for existing callers: run-offload.sh pinned to radium.
+exec "$(dirname "$0")/run-offload.sh" --host radium "$@"