From 8ccc683e9a59491a3a1d792477302a9f7ae3944f Mon Sep 17 00:00:00 2001 From: Jea-Eok-Kim Date: Thu, 3 Sep 2026 22:49:41 +0900 Subject: [PATCH 1/2] slurm: make run-parts.sh exclusive detection work with custom prefix and recent Slurm The exclusive-job check in run-parts.sh had two independent failures: 1. It called scontrol/squeue through PATH. slurmd's environment does not include a custom slurm_install_prefix, so both commands produced empty output, numcpus_sys and numcpus_job were both "", the comparison was true, and every job ran the *-exclusive-* prolog/epilog scripts. On a shared node this reset power limits and clocks on all GPUs and dropped page caches for every job. 2. It parsed "scontrol show job" with grep -Eio "TRES=cpu=[0-9]+". On recent Slurm the output has both ReqTRES= and AllocTRES= lines, so the pattern matched twice and numcpus_job became a multi-line value that never compared equal. With scontrol on PATH, exclusive jobs were therefore never detected. Use {{ slurm_install_prefix }}/bin/squeue by absolute path (the file is already deployed via the template module) and read allocated CPUs and node count with -o %C / -o %D instead of parsing scontrol. Guard against an empty result so a lookup failure means "not exclusive" rather than "exclusive". Observed on DGX OS 7.5.0, Slurm 26.05.1, slurm_install_prefix=/raid/slurm/usr/local: - before the binaries were symlinked into /usr/local/bin, every srun --gres=gpu:1 job logged "Running .../50-exclusive-gpu" and prolog took 6-8 s (srun: Prolog hung on node); - after symlinking, a bash -x run of the script showed numcpus_job='156'. Signed-off-by: Jea-Eok-Kim --- .../etc/slurm/shared/bin/run-parts.sh | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/roles/slurm/templates/etc/slurm/shared/bin/run-parts.sh b/roles/slurm/templates/etc/slurm/shared/bin/run-parts.sh index f748196ca..f4d3cb09c 100755 --- a/roles/slurm/templates/etc/slurm/shared/bin/run-parts.sh +++ b/roles/slurm/templates/etc/slurm/shared/bin/run-parts.sh @@ -13,17 +13,27 @@ log () { logger -s -t slurm "$@" } -# Find out if we are running in exclusive mode +# Use an absolute path: slurmd's PATH does not include a custom +# slurm_install_prefix, and with an empty command result both sides of the +# comparison below were "" and every job was treated as exclusive. +squeue_bin="{{ slurm_install_prefix }}/bin/squeue" + +# Find out if we are running in exclusive mode. +# Ask squeue for allocated CPUs and node count directly instead of parsing +# "scontrol show job": on recent Slurm the pattern TRES=cpu= matched both the +# ReqTRES= and AllocTRES= lines, yielding a multi-line value that never +# compared equal, so exclusive jobs were never detected. exclusive=0 -numcpus_sys=$(( $(grep -c ^processor /proc/cpuinfo) * $(scontrol show job "$SLURM_JOBID" | grep -Eio "TRES=.*node=[0-9]+" | cut -d= -f5) )) -numcpus_job=$(scontrol show job "$SLURM_JOBID" | grep -Eio "TRES=cpu=[0-9]+" | cut -d= -f3) -if [ "$numcpus_sys" == "$numcpus_job" ] ; then +numcpus_job=$("$squeue_bin" -h -j "$SLURM_JOBID" -o %C 2>/dev/null) +numnodes_job=$("$squeue_bin" -h -j "$SLURM_JOBID" -o %D 2>/dev/null) +numcpus_sys=$(( $(grep -c ^processor /proc/cpuinfo) * ${numnodes_job:-1} )) +if [ -n "$numcpus_job" ] && [ "$numcpus_sys" -eq "$numcpus_job" ] 2>/dev/null ; then exclusive=1 fi # Find out if there are any more jobs on this node for this user last_user_job=0 -num_jobs=$(squeue -h -u "$SLURM_JOB_USER" -w "$HOSTNAME" -t running | wc -l) +num_jobs=$("$squeue_bin" -h -u "$SLURM_JOB_USER" -w "$HOSTNAME" -t running | wc -l) if [ "$num_jobs" -eq 0 ]; then last_user_job=1 fi From ebb66b833055c4eabdbd47905550cfbc3e1e5003 Mon Sep 17 00:00:00 2001 From: 100milliongold Date: Mon, 7 Sep 2026 09:55:52 +0900 Subject: [PATCH 2/2] fix: keep run-parts.sh alive when the squeue lookups fail This script runs with "set -e", so a bare `numcpus_job=$(squeue ...)` assignment aborts the entire prolog/epilog run when squeue exits nonzero. Redirecting stderr does not suppress the exit status. The result is that a transient squeue failure skips every part script, not just the exclusive ones, and the job fails. Both allocation fields are now fetched by a single `-o "%C %D"` call inside an `if` condition, so the failure is visible and handled. A failed, empty or non-numeric lookup leaves exclusive=0 and logs a warning. The same reasoning is applied to the last-user-job count, with one difference: piping squeue into `wc -l` hides its exit status, and a failed lookup would be read as "no other jobs" and run the *-lastuserjob-* cleanup scripts while another job of the same user is still on the node. A failed lookup now leaves last_user_job=0. Decision matrix, verified on a DGX B300 (Ubuntu 24.04, bash 5.2, 256 CPUs) by substituting a squeue stub that honours the -o format and the -j flag: case before after ---------- ------------------------------ ------------------------------ exclusive rc=0 all three parts ran rc=0 all three parts ran shared rc=0 exclusive part skipped rc=0 exclusive part skipped otherjobs rc=0 lastuserjob part skipped rc=0 lastuserjob part skipped empty rc=0 silently non-exclusive rc=0 non-exclusive + 1 warning nonnumeric rc=0 silently non-exclusive rc=0 non-exclusive + 1 warning fail rc=1 NO part ran at all rc=0 normal part ran + 2 warnings The three normal cases are unchanged, so there is no regression; the failure cases stop taking the whole run down and stop deciding silently. --- .../etc/slurm/shared/bin/run-parts.sh | 38 ++++++++++++++----- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/roles/slurm/templates/etc/slurm/shared/bin/run-parts.sh b/roles/slurm/templates/etc/slurm/shared/bin/run-parts.sh index f4d3cb09c..fb2c4579d 100755 --- a/roles/slurm/templates/etc/slurm/shared/bin/run-parts.sh +++ b/roles/slurm/templates/etc/slurm/shared/bin/run-parts.sh @@ -23,19 +23,39 @@ squeue_bin="{{ slurm_install_prefix }}/bin/squeue" # "scontrol show job": on recent Slurm the pattern TRES=cpu= matched both the # ReqTRES= and AllocTRES= lines, yielding a multi-line value that never # compared equal, so exclusive jobs were never detected. +# +# The lookup runs inside a conditional on purpose. This script has "set -e", so +# a bare assignment from a failing squeue aborts the whole prolog/epilog run and +# fails the job; redirecting stderr does not suppress the exit status. A failed, +# empty or non-numeric lookup must instead leave exclusive=0, which only skips +# the *-exclusive-* scripts and lets every other script run. exclusive=0 -numcpus_job=$("$squeue_bin" -h -j "$SLURM_JOBID" -o %C 2>/dev/null) -numnodes_job=$("$squeue_bin" -h -j "$SLURM_JOBID" -o %D 2>/dev/null) -numcpus_sys=$(( $(grep -c ^processor /proc/cpuinfo) * ${numnodes_job:-1} )) -if [ -n "$numcpus_job" ] && [ "$numcpus_sys" -eq "$numcpus_job" ] 2>/dev/null ; then - exclusive=1 +if job_alloc=$("$squeue_bin" -h -j "$SLURM_JOBID" -o "%C %D" 2>/dev/null); then + read -r numcpus_job numnodes_job <<<"$job_alloc" || true + if [[ "$numcpus_job" =~ ^[0-9]+$ ]] && [[ "$numnodes_job" =~ ^[0-9]+$ ]]; then + numcpus_sys=$(( $(grep -c ^processor /proc/cpuinfo) * numnodes_job )) + if [ "$numcpus_sys" -eq "$numcpus_job" ]; then + exclusive=1 + fi + else + log "[WARN] squeue returned no usable allocation for job ${SLURM_JOBID} ('${job_alloc}'); treating the job as non-exclusive." + fi +else + log "[WARN] squeue failed for job ${SLURM_JOBID}; treating the job as non-exclusive." fi -# Find out if there are any more jobs on this node for this user +# Find out if there are any more jobs on this node for this user. +# Same reasoning as above with one difference: a failed lookup must not be read +# as "no other jobs", because that would run the *-lastuserjob-* cleanup scripts +# while another job of the same user is still on the node. Piping squeue into +# "wc -l" also hides its exit status, so the status is captured explicitly. last_user_job=0 -num_jobs=$("$squeue_bin" -h -u "$SLURM_JOB_USER" -w "$HOSTNAME" -t running | wc -l) -if [ "$num_jobs" -eq 0 ]; then - last_user_job=1 +if user_jobs=$("$squeue_bin" -h -u "$SLURM_JOB_USER" -w "$HOSTNAME" -t running 2>/dev/null); then + if [ -z "$user_jobs" ]; then + last_user_job=1 + fi +else + log "[WARN] squeue failed while counting jobs for ${SLURM_JOB_USER} on ${HOSTNAME}; not running the last-user-job scripts." fi # Re-implement run-parts since on centos it is just a bash script with no useful flags.