eval-sweep.yml

name: eval-sweep

# Multi-runner parallel sweep across method × budget × test_set.
#
# Two modes (select at dispatch):
#   smoke — GitHub-hosted ubuntu-latest, 1 cell (ppr×64000×swebench_verified),
#            5 instances, no Hetzner provisioning. Quick sanity check.
#   full  — Hetzner CCX63 (4 × self-hosted runners), complete 75-cell matrix,
#            baked repo cache. Production sweep.
#
# Trigger via workflow_dispatch. Results are committed to eval-results/sweep.

'on':
  workflow_dispatch:
    inputs:
      mode:
        description: 'smoke = GH Actions, 5 instances | full = Hetzner CCX63, complete sweep'
        required: true
        default: smoke
        type: choice
        options: [smoke, full]
      methods:
        description: 'full mode: comma-separated methods to run (ppr,ego,bm25,internal-bm25,rrf,pit,aider) or all'
        required: false
        default: all
        type: string

concurrency:
  group: eval-sweep
  cancel-in-progress: false

env:
  REGISTRY: ghcr.io
  IMAGE_REF: ghcr.io/${{ github.repository_owner }}/diffctx-eval:latest
  MANIFESTS_SUBDIR: v1
  TIMEOUT_PER_INSTANCE: '600'
  RESULTS_BRANCH: eval-results/sweep
  SERVER_TAG: eval-${{ github.run_id }}-${{ github.run_attempt }}

jobs:
  # ============================================================================
  # 1. Compute a stable sweep_id used as the directory name for raw artifacts
  # ============================================================================
  prep:
    runs-on: ubuntu-latest
    permissions:
      contents: read
    outputs:
      sweep_id: ${{ steps.id.outputs.sweep_id }}
      ts: ${{ steps.id.outputs.ts }}
      cells: ${{ steps.cells.outputs.cells }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
      - id: id
        run: |
          ts=$(date -u +%Y%m%dT%H%M%SZ)
          sha=$(git rev-parse --short HEAD)
          echo "sweep_id=sweep-${ts}-${sha}" >> "$GITHUB_OUTPUT"
          echo "ts=${ts}" >> "$GITHUB_OUTPUT"
      # The full matrix, filtered by `methods`, so one arm can be rerun
      # without paying for the other 60 cells (the aider rerun after its
      # token accounting was fixed is the case that asked for it).
      - id: cells
        env:
          METHODS: ${{ inputs.methods }}
        run: python3 .github/scripts/sweep_cells.py "${METHODS:-all}" >> "$GITHUB_OUTPUT"

  # ============================================================================
  # 2. Provision Hetzner CCX63 — full mode only
  # ============================================================================
  provision:
    needs: prep
    if: inputs.mode == 'full'
    runs-on: ubuntu-latest
    permissions:
      actions: write
    outputs:
      server_id: ${{ steps.create.outputs.server_id }}
      server_ip: ${{ steps.create.outputs.server_ip }}
    steps:
      # Fail fast BEFORE paying for a server: a missing image otherwise dies
      # silently inside cloud-init and surfaces only as "runners never came
      # online" 15 minutes later (observed after the treemapper->diffctx
      # rebrand renamed the reference without republishing the image).
      - name: Verify evaluation image exists in ghcr
        env:
          GH_TOKEN: ${{ secrets.GH_PAT }}
        run: |
          set -euo pipefail
          TOKEN=$(curl -sf -u "nikolay-e:${GH_TOKEN}" \
            "https://ghcr.io/token?scope=repository:nikolay-e/diffctx-eval:pull" | jq -r '.token')
          ACCEPT="application/vnd.oci.image.index.v1+json"
          ACCEPT="${ACCEPT}, application/vnd.docker.distribution.manifest.list.v2+json"
          ACCEPT="${ACCEPT}, application/vnd.docker.distribution.manifest.v2+json"
          if ! curl -sf -o /dev/null \
            -H "Authorization: Bearer ${TOKEN}" \
            -H "Accept: ${ACCEPT}" \
            "https://ghcr.io/v2/nikolay-e/diffctx-eval/manifests/latest"; then
            echo "::error::ghcr.io/nikolay-e/diffctx-eval:latest not found — build via" \
              "the eval-image workflow or PUSH=1 scripts/build_eval_image.sh" >&2
            exit 1
          fi
          echo "Eval image present."

      - name: Create Hetzner CCX63 with 4 self-hosted runners
        id: create
        env:
          HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
          GH_TOKEN: ${{ secrets.GH_PAT }}
          REPO: ${{ github.repository }}
          SERVER_NAME: ${{ env.SERVER_TAG }}
          BENCH_SSH_PUBLIC_KEY: ${{ secrets.BENCH_SSH_PUBLIC_KEY }}
        run: |
          set -euo pipefail

          REG_TOKEN=$(curl -sf -X POST \
            -H "Authorization: Bearer ${GH_TOKEN}" \
            -H "Accept: application/vnd.github+json" \
            "https://api.github.com/repos/${REPO}/actions/runners/registration-token" \
            | jq -r '.token')

          RUNNER_VER=$(curl -sf \
            -H "Authorization: Bearer ${GH_TOKEN}" \
            "https://api.github.com/repos/actions/runner/releases/latest" \
            | jq -r '.tag_name[1:]')

          REPO_URL="https://github.com/${REPO}"

          cat > /tmp/cloud-init.sh << ENDINIT
          #!/bin/bash
          set -euo pipefail
          export DEBIAN_FRONTEND=noninteractive
          apt-get update -y
          apt-get install -y docker.io curl jq
          systemctl enable docker && systemctl start docker
          mkdir -p /root/.ssh && chmod 700 /root/.ssh
          echo "${BENCH_SSH_PUBLIC_KEY}" >> /root/.ssh/authorized_keys
          chmod 600 /root/.ssh/authorized_keys
          # Hetzner marks the root password expired when the server is created
          # without an API-level ssh_keys ref; -M -1 alone does not clear the
          # forced-change flag, which blocks non-TTY SSH ("password expired").
          chage -M -1 -d "\$(date +%Y-%m-%d)" root
          useradd -m -s /bin/bash github
          usermod -aG docker github
          echo "${GH_TOKEN}" | docker login ghcr.io -u nikolay-e --password-stdin
          docker pull ghcr.io/nikolay-e/diffctx-eval:latest
          nohup bash -c '
            mkdir -p /data/bench_repos
            BAKE_LOG=/data/bench_repos/bake.log
            if docker run --rm \
              --entrypoint python3 \
              -v /data/bench_repos:/cache/contextbench_repos \
              -e BAKE_PARALLELISM=16 \
              ghcr.io/nikolay-e/diffctx-eval:latest \
              /app/scripts/bake_eval_cache.py /cache/contextbench_repos \
              >> "\${BAKE_LOG}" 2>&1; then
              touch /data/bench_repos/.ready
              echo "[bake] OK at \$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "\${BAKE_LOG}"
            else
              rc=\$?
              touch /data/bench_repos/.bake_failed
              echo "[bake] FAILED rc=\${rc} at \$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "\${BAKE_LOG}"
            fi
          ' &
          curl -sL -o /tmp/runner.tar.gz \
            "https://github.com/actions/runner/releases/download/v${RUNNER_VER}/actions-runner-linux-x64-${RUNNER_VER}.tar.gz"
          for N in 1 2 3 4; do
            mkdir -p "/home/github/runner-\${N}"
            tar xzf /tmp/runner.tar.gz -C "/home/github/runner-\${N}"
            chown -R github:github "/home/github/runner-\${N}"
            sudo -u github bash -c "
              cd /home/github/runner-\${N} &&
              ./config.sh \
                --url '${REPO_URL}' \
                --token '${REG_TOKEN}' \
                --name '${SERVER_NAME}-\${N}' \
                --labels 'self-hosted,large' \
                --unattended \
                --replace"
            # Installed as a systemd service (not nohup'd) so a host reboot
            # (transient Hetzner-side infra event, not app-caused - see #98)
            # brings the runner back instead of orphaning queued cells forever.
            cd "/home/github/runner-\${N}"
            ./svc.sh install github
            ./svc.sh start
          done
          ENDINIT

          RESPONSE=$(curl -sf -X POST "https://api.hetzner.cloud/v1/servers" \
            -H "Authorization: Bearer ${HCLOUD_TOKEN}" \
            -H "Content-Type: application/json" \
            -d "{
              \"name\": \"${SERVER_NAME}\",
              \"server_type\": \"ccx63\",
              \"image\": \"ubuntu-24.04\",
              \"location\": \"nbg1\",
              \"user_data\": $(python3 -c "import json,sys; print(json.dumps(open('/tmp/cloud-init.sh').read()))")
            }")

          SERVER_ID=$(echo "${RESPONSE}" | jq -r '.server.id')
          SERVER_IP=$(echo "${RESPONSE}" | jq -r '.server.public_net.ipv4.ip')
          echo "server_id=${SERVER_ID}" >> "${GITHUB_OUTPUT}"
          echo "server_ip=${SERVER_IP}" >> "${GITHUB_OUTPUT}"
          echo "${SERVER_ID}" > server_id.txt
          echo "Provisioned Hetzner CCX63 server_id=${SERVER_ID} ip=${SERVER_IP} name=${SERVER_NAME}"
          SSHCMD="ssh root@${SERVER_IP} -i <(security find-generic-password -s diffctx-eval-ssh-key -a \$USER -w)"
          echo "SSH: ${SSHCMD}"

      - name: Upload server_id for cleanup workflow
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9  # v7.0.2
        with:
          name: hetzner-server-id
          path: server_id.txt
          retention-days: 7

      - name: Wait for all 4 runners to come online
        env:
          GH_TOKEN: ${{ secrets.GH_PAT }}
          REPO: ${{ github.repository }}
          SERVER_NAME: ${{ env.SERVER_TAG }}
        run: |
          set -euo pipefail
          echo "Polling for 4 runners prefixed '${SERVER_NAME}'..."
          DEADLINE=$((SECONDS + 900))
          while [[ ${SECONDS} -lt ${DEADLINE} ]]; do
            ONLINE=$(curl -sf \
              -H "Authorization: Bearer ${GH_TOKEN}" \
              "https://api.github.com/repos/${REPO}/actions/runners?per_page=100" \
              | jq --arg pfx "${SERVER_NAME}" \
                '[.runners[] | select(.name | startswith($pfx)) | select(.status == "online")] | length')
            echo "  Online runners: ${ONLINE}/4"
            [[ "${ONLINE}" -ge 4 ]] && echo "All 4 runners ready." && exit 0
            sleep 30
          done
          echo "ERROR: runners did not come online within 15 minutes." >&2
          exit 1

  # ============================================================================
  # 3a. Smoke sweep — ubuntu-latest, minimal matrix, 5 instances, no bake cache
  # ============================================================================
  sweep-smoke:
    needs: [prep]
    if: inputs.mode == 'smoke'
    permissions:
      contents: read
      packages: read
    strategy:
      fail-fast: false
      matrix:
        method: [ppr, ego, bm25, internal-bm25, rrf, pit, aider]
        budget: [64000]
        test_set: [swebench_verified]
    runs-on: ubuntu-latest
    timeout-minutes: 60
    container:
      image: ghcr.io/${{ github.repository_owner }}/diffctx-eval:latest
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
      options: >-
        --cpus 2
        --memory 6g
        --memory-swap 6g
    env:
      CELL_TAG: ${{ matrix.method }}_b${{ matrix.budget }}_${{ matrix.test_set }}
      # One number for both the run and its assertion below.
      SMOKE_LIMIT: 5
      CELL_DIR: results/sweep/${{ matrix.method }}_b${{ matrix.budget }}_${{ matrix.test_set }}
      MANIFESTS_DIR: datasets/eval-splits/v1
      INPUT_TIMEOUT: '120'
      INPUT_MANIFESTS_SUBDIR: v1
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1

      - name: Cell start metadata
        run: |
          mkdir -p "${CELL_DIR}"
          cat > "${CELL_DIR}/metadata.json" <<JSON
          {
            "sweep_id": "${{ needs.prep.outputs.sweep_id }}",
            "mode": "smoke",
            "cell": {
              "method": "${{ matrix.method }}",
              "budget": ${{ matrix.budget }},
              "test_set": "${{ matrix.test_set }}",
              "operating_point": "winner.json"
            },
            "git": {
              "sha": "${{ github.sha }}",
              "ref": "${{ github.ref }}",
              "actor": "${{ github.actor }}"
            },
            "started_at_utc": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
            "runner": {
              "os": "${RUNNER_OS}",
              "arch": "${RUNNER_ARCH}",
              "name": "${RUNNER_NAME}",
              "image": "${IMAGE_REF}"
            },
            "stage": "started"
          }
          JSON

      - name: Build per-cell manifest dir
        run: |
          mkdir -p /tmp/manifest_one
          cp "${MANIFESTS_DIR}/test_${{ matrix.test_set }}.txt" /tmp/manifest_one/

      - name: Resolve baseline + scoring
        id: cfg
        run: |
          method="${{ matrix.method }}"
          case "$method" in
            ppr|ego)
              echo "baseline=diffctx" >> "$GITHUB_OUTPUT"
              echo "scoring=$method" >> "$GITHUB_OUTPUT"
              ;;
            bm25)
              echo "baseline=bm25" >> "$GITHUB_OUTPUT"
              echo "scoring=ego" >> "$GITHUB_OUTPUT"
              ;;
            rrf)
              echo "baseline=diffctx" >> "$GITHUB_OUTPUT"
              echo "scoring=rrf" >> "$GITHUB_OUTPUT"
              ;;
            pit)
              # Percentile fusion: the same two signals as rrf, blended on
              # empirical-CDF position instead of rank. It shares the arm so the
              # #125 gate reads "did fusion fail, or did the RANK FORM of fusion
              # fail" from one matrix rather than two runs.
              echo "baseline=diffctx" >> "$GITHUB_OUTPUT"
              echo "scoring=pit" >> "$GITHUB_OUTPUT"
              ;;
            internal-bm25)
              # BM25 as diffctx's own scoring, NOT the external baseline above:
              # the #125 gate needs the lexical signal inside the same candidate
              # universe, otherwise the comparison is against a different
              # discovery and the fusion ceiling is unreadable.
              echo "baseline=diffctx" >> "$GITHUB_OUTPUT"
              echo "scoring=bm25" >> "$GITHUB_OUTPUT"
              ;;
            aider)
              echo "baseline=aider_fair" >> "$GITHUB_OUTPUT"
              echo "scoring=ego" >> "$GITHUB_OUTPUT"
              ;;
            *)
              # Without this a new method silently emits empty outputs and the
              # cell runs as whatever the downstream defaults happen to be.
              echo "::error::unknown sweep method '$method'" >&2
              exit 1
              ;;
          esac

      # The operating point is whatever the engine ships, read from the
      # engine — never a literal here. This file carried tau=0.12/cbf=0.5 for
      # weeks after the v5 calibration moved the default to 0.05/0.4, so a
      # full sweep measured a configuration nobody was running (#175 again).
      - name: Resolve the shipped operating point
        run: |
          python - >> "$GITHUB_ENV" <<'PY'
          from diffctx._diffctx import DEFAULT_CORE_BUDGET_FRACTION, DEFAULT_TAU
          print(f"INPUT_TAU={DEFAULT_TAU}")
          print(f"INPUT_CBF={DEFAULT_CORE_BUDGET_FRACTION}")
          PY

      - name: Build winner.json
        run: |
          cat > /tmp/winner.json <<JSON
          {
            "winner": {
              "tau": ${INPUT_TAU},
              "core_budget_fraction": ${INPUT_CBF},
              "budget": ${{ matrix.budget }},
              "scoring": "${SCORING}",
              "extra_env": {}
            }
          }
          JSON
          cp /tmp/winner.json "${CELL_DIR}/winner.json"
        env:
          SCORING: ${{ steps.cfg.outputs.scoring }}

      - name: Install uv (required by aider baseline)
        run: 'pip install --only-binary :all: uv'

      - name: Run cell (smoke — 5 instances)
        timeout-minutes: 50
        shell: bash
        run: |
          set -o pipefail
          python -m eval run-final \
            --baseline "${BASELINE}" \
            --winner /tmp/winner.json \
            --manifests-dir /tmp/manifest_one \
            --workers 1 \
            --limit "${SMOKE_LIMIT}" \
            --timeout-per-instance "${INPUT_TIMEOUT}" \
            --min-memory-gb 2 \
            --min-disk-gb 5 \
            --out "${CELL_DIR}" 2>&1 | tee "${CELL_DIR}/run.log"
        env:
          BASELINE: ${{ steps.cfg.outputs.baseline }}
          # Unauthenticated Hub reads are rate-limited; the first thing that
          # breaks on a 500-instance cell (#233).
          HF_TOKEN: ${{ secrets.HF_TOKEN }}

      - name: Per-instance metric summary (always)
        if: always()
        run: |
          # Baselines prefix the file with their name (bm25__<set>.checkpoint.jsonl),
          # hence the leading wildcard; the bare name alone failed the bm25 smoke
          # cell with a checkpoint sitting right beside the error.
          found=0
          for ckpt in "${CELL_DIR}"/*"${{ matrix.test_set }}.checkpoint.jsonl"; do
            [ -f "${ckpt}" ] || continue
            found=1
            python -m eval cell-metrics --ckpt "${ckpt}" --out "${CELL_DIR}/cell_summary.json"
            cat "${CELL_DIR}/cell_summary.json"
          done
          if [ "${found}" = "0" ]; then
            printf '{"error": "no checkpoint produced", "expected_path": "%s"}\n' \
              "${CELL_DIR}/*${{ matrix.test_set }}.checkpoint.jsonl" > "${CELL_DIR}/cell_summary.json"
            ls -la "${CELL_DIR}" || true
          fi

      # A smoke run that reports "no checkpoint produced" and still goes green
      # is a check that cannot fail — it passed for months while nothing
      # asserted anything (#233). Five instances that all apply and produce a
      # selection is the floor this path exists to defend.
      - name: Assert the smoke cell actually ran
        run: |
          python - "${CELL_DIR}/cell_summary.json" "${SMOKE_LIMIT}" <<'PY'
          import json, sys
          summary = json.load(open(sys.argv[1]))
          limit = int(sys.argv[2])
          if "error" in summary:
              sys.exit(f"smoke cell produced no checkpoint: {summary}")
          ok = int(summary.get("ok") or 0)
          if ok < limit:
              sys.exit(f"smoke cell resolved {ok}/{limit} instances: {summary}")
          print(f"smoke cell ok: {summary}")
          PY

      - name: Append cell-end metadata
        if: always()
        run: |
          end_ts="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
          if command -v jq >/dev/null 2>&1; then
            tmp=$(mktemp)
            jq --arg end "${end_ts}" --arg status "${{ job.status }}" \
              '. + {ended_at_utc: $end, exit_status: $status}' \
              "${CELL_DIR}/metadata.json" > "${tmp}" \
            && mv "${tmp}" "${CELL_DIR}/metadata.json"
          fi

      - name: Upload cell artifact
        if: always()
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9  # v7.0.2
        with:
          name: cell-${{ matrix.method }}-b${{ matrix.budget }}-${{ matrix.test_set }}
          path: ${{ env.CELL_DIR }}/
          retention-days: 30
          if-no-files-found: warn
          overwrite: true

  # ============================================================================
  # 3b. Full sweep — Hetzner CCX63, 75 cells (see the exclude block), baked repo cache
  # ============================================================================
  sweep-full:
    needs: [prep, provision]
    if: inputs.mode == 'full'
    permissions:
      contents: read
      packages: read
    strategy:
      fail-fast: false
      # 4 self-hosted runners on the CCX63 host; the queue can only drain at
      # that rate anyway, so cap the matrix-level parallelism to match.
      max-parallel: 4
      # Budget axis collapsed (#52): diffctx methods and bm25 sweep every
      # budget inside ONE cell via `run_final_eval --budgets ...`; aider keeps
      # one budget per cell. The cell list, its excludes and the reasons for
      # them live in .github/scripts/sweep_cells.py.
      matrix:
        include: ${{ fromJSON(needs.prep.outputs.cells) }}
    runs-on: [self-hosted, large]
    timeout-minutes: 350
    container:
      image: ghcr.io/${{ github.repository_owner }}/diffctx-eval:latest
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
      options: >-
        --cpus 12
        --memory 44g
        --memory-swap 44g
        --shm-size 8g
        --pids-limit -1
        --ulimit nofile=1048576:1048576
        --ulimit nproc=65535:65535
        -v /data/bench_repos:/cache/contextbench_repos
    env:
      # `matrix.budget` exists only on aider include-cells; multi-budget
      # cells (ppr/ego/bm25, #52) are labeled bALL and enumerate their
      # budgets in the checkpoint filenames.
      CELL_TAG: ${{ matrix.method }}_b${{ matrix.budget || 'ALL' }}_L${{ matrix.depth }}_${{ matrix.test_set }}
      CELL_DIR: >-
        results/sweep/${{ matrix.method }}_b${{ matrix.budget || 'ALL' }}_L${{ matrix.depth }}_${{ matrix.test_set }}
      MANIFESTS_DIR: datasets/eval-splits/v1
      INPUT_TIMEOUT: '600'
      INPUT_MANIFESTS_SUBDIR: v1
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1

      - name: Cell start metadata (write FIRST, before any failure point)
        run: |
          mkdir -p "${CELL_DIR}"
          cat > "${CELL_DIR}/metadata.json" <<JSON
          {
            "sweep_id": "${{ needs.prep.outputs.sweep_id }}",
            "mode": "full",
            "cell": {
              "method": "${{ matrix.method }}",
              "budget": "${{ matrix.budget || 'ALL' }}",
              "depth": ${{ matrix.depth }},
              "test_set": "${{ matrix.test_set }}",
              "operating_point": "winner.json"
            },
            "git": {
              "sha": "${{ github.sha }}",
              "ref": "${{ github.ref }}",
              "actor": "${{ github.actor }}"
            },
            "started_at_utc": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
            "runner": {
              "os": "${RUNNER_OS}",
              "arch": "${RUNNER_ARCH}",
              "name": "${RUNNER_NAME}",
              "image": "${IMAGE_REF}"
            },
            "stage": "started"
          }
          JSON

      - name: Wait for shared repo cache
        run: |
          while true; do
            if [ -f /cache/contextbench_repos/.bake_failed ]; then
              echo "=== BAKE FAILED — full log ==="
              cat /cache/contextbench_repos/bake.log || echo "(bake.log not found)"
              echo "=== end of bake.log ==="
              exit 1
            fi
            if [ -f /cache/contextbench_repos/.ready ]; then
              echo "Repo cache ready."
              break
            fi
            echo "Waiting for repo cache extraction to complete..."
            sleep 15
          done

      - name: Reclaim disk — purge own runner's worktrees, prune bare caches
        shell: bash
        run: |
          set -euo pipefail
          base=/cache/contextbench_repos
          # `_ensure_worker_state` names worktrees `worktrees/w<runner>_<pid>/`
          # using `RUNNER_NAME` (with `-` and space mapped to `_`). Four
          # self-hosted runners share `/data/bench_repos` on the CCX63 host,
          # so a blanket `rm -rf worktrees/` would corrupt concurrent cells
          # on the other runners. Restrict to dirs prefixed with our own
          # runner name so cleanup is single-runner-scoped.
          runner_slug="$(printf '%s' "${RUNNER_NAME:-unknown}" | tr ' -' '__')"
          echo "--- Disk before cleanup ---"
          df -h "${base}" || true
          du -sh "${base}/worktrees" 2>/dev/null || echo "  (no worktrees dir)"
          echo "  runner_slug=${runner_slug}"
          # 1. Drop only THIS runner's accumulated worktree dirs from prior
          #    cells. Concurrent cells on other runners (different PIDs and
          #    different RUNNER_NAME prefixes) are untouched.
          if [ -d "${base}/worktrees" ]; then
            find "${base}/worktrees" -mindepth 1 -maxdepth 1 -type d \
              -name "w${runner_slug}_*" -prune -exec rm -rf {} + 2>/dev/null || true
          fi
          # 2. Prune dangling worktree registrations on every bare cache.
          #    Idempotent and safe even when other runners have live
          #    registrations: `git worktree prune` only drops entries whose
          #    target directory is missing.
          for repo_cache in "${base}"/*/; do
            if [ -d "${repo_cache}/.git" ] || [ -f "${repo_cache}/HEAD" ]; then
              git -C "${repo_cache}" worktree prune --expire=now 2>/dev/null || true
            fi
          done
          # 3. Sanity check: a single cell on a large repo (angular, AutoGPT,
          #    transformers) can grow worktrees to 50-100 GB. With 4 cells
          #    running concurrently on the same physical disk we want at
          #    least ~150 GB headroom (4 cells x ~35 GB peak typical, plus
          #    swap and other infrastructure). Run #49 hit ENOSPC at ~310 GB
          #    free at cell start, so 150 GB is the conservative floor.
          avail_kb=$(df --output=avail "${base}" | tail -1 | tr -d ' ')
          avail_gb=$((avail_kb / 1024 / 1024))
          echo "--- Disk after cleanup: ${avail_gb} GB available ---"
          df -h "${base}"
          if [ "${avail_gb}" -lt 150 ]; then
            msg="Less than 150 GB free on /cache/contextbench_repos after"
            msg="${msg} cleanup. Other runners may be holding live worktrees;"
            msg="${msg} retrying in 60s in case they are about to release."
            echo "::warning::${msg}" >&2
            sleep 60
            avail_kb=$(df --output=avail "${base}" | tail -1 | tr -d ' ')
            avail_gb=$((avail_kb / 1024 / 1024))
            echo "--- Disk after retry wait: ${avail_gb} GB ---"
            if [ "${avail_gb}" -lt 100 ]; then
              echo "::error::Disk still tight (${avail_gb} GB); aborting cell." >&2
              exit 1
            fi
          fi

      - name: Validate bake manifest and purge known-bad repos
        shell: bash
        run: |
          set -euo pipefail
          MANIFEST=/cache/contextbench_repos/.bake_manifest.json
          if [ ! -f "${MANIFEST}" ]; then
            echo "WARN: no .bake_manifest.json found — proceeding without preflight purge"
            exit 0
          fi
          python3 - <<'PY'
          import json, shutil
          from pathlib import Path
          base = Path("/cache/contextbench_repos")
          manifest = json.loads((base / ".bake_manifest.json").read_text())
          purged = []
          for entry in manifest.get("failed_clones", []):
              repo = entry["repo"]
              cache_dir = base / repo.replace("/", "__")
              if cache_dir.exists():
                  shutil.rmtree(cache_dir, ignore_errors=True)
                  purged.append(repo)
          print(f"Purged {len(purged)} bake-failed repos: {purged[:10]}{'...' if len(purged) > 10 else ''}")
          fc = len(manifest.get("failed_clones", []))
          ff = len(manifest.get("failed_fetches", []))
          print(f"Bake manifest: failed_clones={fc} failed_fetches={ff}")
          PY

      - name: System info dump
        run: |
          {
            echo "=== uname -a ==="; uname -a
            echo; echo "=== /proc/cpuinfo (1 core) ==="; awk '/^processor/{p++} p<2' /proc/cpuinfo
            echo; echo "=== /proc/meminfo (head) ==="; head -5 /proc/meminfo
            echo; echo "=== df -h ==="; df -h
            echo; echo "=== python ==="; python --version
            echo; echo "=== pip freeze (head 30) ==="; pip freeze 2>/dev/null | head -30 || true
            echo; echo "=== rustc ==="; rustc --version 2>/dev/null || echo "no rustc"
            echo; echo "=== cargo ==="; cargo --version 2>/dev/null || echo "no cargo"
            echo; echo "=== git ==="; git --version
          } > "${CELL_DIR}/system_info.log" 2>&1 || true

      - name: Build per-cell single-test-set manifest dir
        run: |
          mkdir -p /tmp/manifest_one
          cp "${MANIFESTS_DIR}/test_${{ matrix.test_set }}.txt" /tmp/manifest_one/

      - name: Resolve baseline + scoring + budgets
        id: cfg
        run: |
          method="${{ matrix.method }}"
          # Budget lists (#52): B=0 is the recall-floor sanity bound, B=-1 the
          # unlimited-budget ceiling; {8k..128k} form the budget curve from
          # STATS_PLAN.md. diffctx methods sweep all of them in one cell via
          # compute_scored_state reuse; bm25 loops per budget in-process
          # (no reuse but same artifact layout); aider gets its budget from
          # the matrix include entry.
          case "$method" in
            ppr|ego)
              echo "baseline=diffctx" >> "$GITHUB_OUTPUT"
              echo "scoring=$method" >> "$GITHUB_OUTPUT"
              echo "budgets=-1,0,8000,16000,32000,64000,128000" >> "$GITHUB_OUTPUT"
              ;;
            bm25)
              echo "baseline=bm25" >> "$GITHUB_OUTPUT"
              echo "scoring=ego" >> "$GITHUB_OUTPUT"
              echo "budgets=0,8000,16000,32000,64000,128000" >> "$GITHUB_OUTPUT"
              ;;
            rrf)
              echo "baseline=diffctx" >> "$GITHUB_OUTPUT"
              echo "scoring=rrf" >> "$GITHUB_OUTPUT"
              echo "budgets=-1,0,8000,16000,32000,64000,128000" >> "$GITHUB_OUTPUT"
              ;;
            pit)
              echo "baseline=diffctx" >> "$GITHUB_OUTPUT"
              echo "scoring=pit" >> "$GITHUB_OUTPUT"
              echo "budgets=-1,0,8000,16000,32000,64000,128000" >> "$GITHUB_OUTPUT"
              ;;
            internal-bm25)
              # The lexical signal as diffctx's OWN scoring, inside the same
              # candidate universe. The `bm25` arm above is the external
              # baseline with its own discovery; comparing fusion against it
              # measures discovery as much as ranking, which is why the #125
              # ceiling was unreadable.
              echo "baseline=diffctx" >> "$GITHUB_OUTPUT"
              echo "scoring=bm25" >> "$GITHUB_OUTPUT"
              echo "budgets=-1,0,8000,16000,32000,64000,128000" >> "$GITHUB_OUTPUT"
              ;;
            aider)
              echo "baseline=aider_fair" >> "$GITHUB_OUTPUT"
              echo "scoring=ego" >> "$GITHUB_OUTPUT"
              echo "budgets=" >> "$GITHUB_OUTPUT"
              ;;
            *)
              echo "::error::unknown sweep method '$method'" >&2
              exit 1
              ;;
          esac

      # The operating point is whatever the engine ships, read from the
      # engine — never a literal here. This file carried tau=0.12/cbf=0.5 for
      # weeks after the v5 calibration moved the default to 0.05/0.4, so a
      # full sweep measured a configuration nobody was running (#175 again).
      - name: Resolve the shipped operating point
        run: |
          python - >> "$GITHUB_ENV" <<'PY'
          from diffctx._diffctx import DEFAULT_CORE_BUDGET_FRACTION, DEFAULT_TAU
          print(f"INPUT_TAU={DEFAULT_TAU}")
          print(f"INPUT_CBF={DEFAULT_CORE_BUDGET_FRACTION}")
          PY

      - name: Build winner.json
        run: |
          # Multi-budget cells still need a syntactically valid headline
          # budget; 8000 is the calibrated default and is present in every
          # budget list, so headline selection is always resolvable.
          cat > /tmp/winner.json <<JSON
          {
            "winner": {
              "tau": ${INPUT_TAU},
              "core_budget_fraction": ${INPUT_CBF},
              "budget": ${{ matrix.budget || 8000 }},
              "scoring": "${SCORING}",
              "extra_env": {}
            }
          }
          JSON
          cp /tmp/winner.json "${CELL_DIR}/winner.json"
        env:
          SCORING: ${{ steps.cfg.outputs.scoring }}

      - name: Install uv (required by aider baseline; safe no-op for others)
        run: 'pip install --only-binary :all: --no-cache-dir uv'

      # Resume across "Re-run failed jobs": artifacts persist across attempts
      # of the same run, so a retried cell restores its previous attempt's
      # checkpoints and run_final_eval's resume_from skips completed instances
      # instead of re-running the whole cell from scratch.
      - name: Restore previous attempt's checkpoints (resume)
        # A first attempt has no earlier artifact to restore; running the
        # download anyway put one "Artifact not found" error per cell on the
        # run page (74 on the first full run) and buried the real failures.
        if: github.run_attempt > 1
        continue-on-error: true
        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
        with:
          name: cell-${{ matrix.method }}-b${{ matrix.budget || 'ALL' }}-L${{ matrix.depth }}-${{ matrix.test_set }}
          path: /tmp/prev_cell

      - name: Seed checkpoints from previous attempt
        shell: bash
        run: |
          if [ -d /tmp/prev_cell ]; then
            restored=$(find /tmp/prev_cell -name '*.checkpoint.jsonl' | wc -l)
            if [ "${restored}" -gt 0 ]; then
              cd /tmp/prev_cell
              find . -name '*.checkpoint.jsonl' \
                -exec cp --parents {} "${GITHUB_WORKSPACE}/${CELL_DIR}/" \;
              cd "${GITHUB_WORKSPACE}"
              echo "Restored ${restored} checkpoint file(s) from previous attempt"
            else
              echo "Previous artifact present but no checkpoints inside"
            fi
          else
            echo "No previous attempt artifact — fresh cell"
          fi

      - name: Run cell (logged to run.log)
        timeout-minutes: 330
        shell: bash
        run: |
          set -o pipefail
          # Apply EGO graph depth as an env override only when matrix.depth >= 0
          # (the sentinel -1 means "depth doesn't apply to this method"; in that
          # case we leave DIFFCTX_OP_GRAPH_DEPTH unset and the default kicks in).
          if [ "${MATRIX_DEPTH}" != "-1" ]; then
            export DIFFCTX_OP_GRAPH_DEPTH="${MATRIX_DEPTH}"
          fi
          BUDGETS_ARGS=()
          if [ -n "${BUDGETS}" ]; then
            BUDGETS_ARGS=(--budgets="${BUDGETS}")
          fi
          python -m eval run-final \
            --baseline "${BASELINE}" \
            --winner /tmp/winner.json \
            --manifests-dir /tmp/manifest_one \
            "${BUDGETS_ARGS[@]}" \
            --workers 10 \
            --timeout-per-instance "${INPUT_TIMEOUT}" \
            --min-memory-gb 32 \
            --min-disk-gb 50 \
            --out "${CELL_DIR}" 2>&1 | tee "${CELL_DIR}/run.log"
        env:
          BASELINE: ${{ steps.cfg.outputs.baseline }}
          # Unauthenticated Hub reads are rate-limited; the first thing that
          # breaks on a 500-instance cell (#233).
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
          BUDGETS: ${{ steps.cfg.outputs.budgets }}
          MATRIX_DEPTH: ${{ matrix.depth }}

      - name: Per-instance metric summary (always)
        if: always()
        run: |
          # Multi-budget cells (#52): one summary per b<budget>.checkpoint.jsonl.
          # Baselines prefix both layouts with their name (bm25__<set>_budget_sweep,
          # aider_fair__<set>.checkpoint.jsonl), hence the leading wildcard.
          found=0
          for ckpt in "${CELL_DIR}"/*"${{ matrix.test_set }}_budget_sweep"/b*.checkpoint.jsonl; do
            [ -f "${ckpt}" ] || continue
            found=1
            b=$(basename "${ckpt}" .checkpoint.jsonl)
            python -m eval cell-metrics --ckpt "${ckpt}" --out "${CELL_DIR}/cell_summary_${b}.json"
            echo "=== ${b} ==="
            head -40 "${CELL_DIR}/cell_summary_${b}.json"
          done
          for ckpt in "${CELL_DIR}"/*"${{ matrix.test_set }}.checkpoint.jsonl"; do
            [ -f "${ckpt}" ] || continue
            found=1
            python -m eval cell-metrics --ckpt "${ckpt}" --out "${CELL_DIR}/cell_summary.json"
            cat "${CELL_DIR}/cell_summary.json"
          done
          ckpt="${CELL_DIR}/*${{ matrix.test_set }}.checkpoint.jsonl"
          if [ "${found}" = "0" ]; then
            printf '{"error": "no checkpoint produced", "expected_path": "%s"}\n' "${ckpt}" \
              > "${CELL_DIR}/cell_summary.json"
            ls -la "${CELL_DIR}" || true
          fi

      - name: Append cell-end metadata
        if: always()
        run: |
          end_ts="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
          if command -v jq >/dev/null 2>&1; then
            tmp=$(mktemp)
            jq --arg end "${end_ts}" --arg status "${{ job.status }}" --arg disk "$(df -h / | tail -1)" \
              '. + {ended_at_utc: $end, exit_status: $status, runner_disk_at_end: $disk}' \
              "${CELL_DIR}/metadata.json" > "${tmp}" \
            && mv "${tmp}" "${CELL_DIR}/metadata.json"
          fi
          cat "${CELL_DIR}/metadata.json"

      - name: Upload cell artifact (90-day retention)
        if: always()
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9  # v7.0.2
        with:
          name: cell-${{ matrix.method }}-b${{ matrix.budget || 'ALL' }}-L${{ matrix.depth }}-${{ matrix.test_set }}
          path: ${{ env.CELL_DIR }}/
          retention-days: 90
          if-no-files-found: warn
          # A retried job must be able to replace its previous attempt's
          # artifact (v4+ artifacts are immutable per name by default).
          overwrite: true

      - name: Reclaim disk — drop this cell's per-worker worktrees
        if: always()
        shell: bash
        run: |
          set -e
          base=/cache/contextbench_repos
          runner_slug="$(printf '%s' "${RUNNER_NAME:-unknown}" | tr ' -' '__')"
          # Defense-in-depth: clean up at the END of each cell so a crash
          # on the NEXT cell's preflight doesn't leave the runner ENOSPC'd.
          # Scoped to this runner only — the other 3 self-hosted runners
          # on this CCX63 share the disk and may have live worktrees.
          if [ -d "${base}/worktrees" ]; then
            find "${base}/worktrees" -mindepth 1 -maxdepth 1 -type d \
              -name "w${runner_slug}_*" -prune -exec rm -rf {} + 2>/dev/null || true
          fi
          for repo_cache in "${base}"/*/; do
            if [ -d "${repo_cache}/.git" ] || [ -f "${repo_cache}/HEAD" ]; then
              git -C "${repo_cache}" worktree prune --expire=now 2>/dev/null || true
            fi
          done
          df -h "${base}" || true

  # ============================================================================
  # 4. Aggregate all cells into per-cell summary table + raw artifact archive.
  # ============================================================================
  aggregate:
    needs: [prep, sweep-smoke, sweep-full]
    if: always()
    runs-on: ubuntu-latest
    permissions:
      contents: write
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1

      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
        with:
          python-version: '3.12'

      - name: Install deps
        run: 'pip install --only-binary :all: numpy scipy'

      - name: Download all cell artifacts
        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
        with:
          path: all_cells
          pattern: cell-*

      - name: Aggregate per-cell summaries
        run: |
          mkdir -p aggregated
          python -m eval aggregate-sweep \
            --cells-dir all_cells \
            --sweep-id "${{ needs.prep.outputs.sweep_id }}" \
            --out aggregated/

      - name: Show aggregate summary
        run: |
          echo "=== aggregated/ ==="
          ls -la aggregated/
          echo
          echo "=== aggregated/SWEEP_TABLE.md ==="
          cat aggregated/SWEEP_TABLE.md || true
          echo
          echo "=== aggregated/grand_summary.json (head 80) ==="
          head -80 aggregated/grand_summary.json || true

      - name: Upload aggregated tables (1-year retention)
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9  # v7.0.2
        with:
          name: sweep-aggregated-${{ needs.prep.outputs.sweep_id }}
          path: aggregated/
          retention-days: 365
          overwrite: true

      - name: Commit raw artifacts to results branch
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          BR: ${{ env.RESULTS_BRANCH }}
          REPO: ${{ github.repository }}
          SWEEP_ID: ${{ needs.prep.outputs.sweep_id }}
          TS: ${{ needs.prep.outputs.ts }}
          INPUT_TIMEOUT: '600'
          INPUT_MANIFESTS_SUBDIR: v1
        run: |
          set -euo pipefail
          git clone --depth 1 --branch main \
            "https://x-access-token:${GITHUB_TOKEN}@github.com/${REPO}.git" \
            sidecar
          cd sidecar
          # The identity belongs to the clone that commits. Set on the outer
          # checkout, it left this one anonymous: the first full sweep's
          # commit failed, "|| echo nothing to commit" hid it, and three
          # pushes of a branch that was never created followed.
          git config user.name 'github-actions[bot]'
          git config user.email 'github-actions[bot]@users.noreply.github.com'
          if git ls-remote --heads origin "${BR}" | grep -q "${BR}"; then
            git fetch origin "${BR}":"${BR}"
            git checkout "${BR}"
          else
            git checkout --orphan "${BR}"
            git rm -rf . 2>/dev/null || true
          fi
          mkdir -p "${SWEEP_ID}"
          cp -r ../aggregated "${SWEEP_ID}/aggregated"
          cp -r ../all_cells "${SWEEP_ID}/cells"
          cat > "${SWEEP_ID}/README.md" <<EOF
          # Sweep ${SWEEP_ID}
          - Triggered: ${TS}
          - Source SHA: ${GITHUB_SHA}
          - Manifests: ${INPUT_MANIFESTS_SUBDIR}
          - Operating point: each cell's \`winner.json\` (resolved from the engine at run time)
          - Per-instance timeout: ${INPUT_TIMEOUT}s

          See \`aggregated/SWEEP_TABLE.md\` and \`cells/\` for raw per-instance JSONL checkpoints.
          EOF
          git add -A
          if git diff --cached --quiet; then
            echo "nothing to commit"
          else
            git commit -m "sweep ${SWEEP_ID} (source ${GITHUB_SHA})"
          fi
          pushed=0
          for attempt in 1 2 3; do
            echo "Push attempt ${attempt}/3..."
            git pull --rebase origin "${BR}" 2>/dev/null || true
            if git push -u origin "${BR}"; then
              pushed=1
              break
            fi
            sleep 10
          done
          # `sleep` was the loop's last command, so three failed pushes exited 0
          # and the results branch silently missed the sweep.
          [ "${pushed}" = 1 ] || { echo "::error::results branch push failed after 3 attempts"; exit 1; }

  # ============================================================================
  # 5. Deregister runners and destroy Hetzner server — full mode only
  # ============================================================================
  cleanup:
    needs: [sweep-full]
    if: always() && inputs.mode == 'full'
    runs-on: ubuntu-latest
    permissions:
      actions: write
    steps:
      - name: Deregister self-hosted runners
        env:
          GH_TOKEN: ${{ secrets.GH_PAT }}
          REPO: ${{ github.repository }}
          SERVER_NAME: ${{ env.SERVER_TAG }}
        run: |
          set -euo pipefail
          RUNNER_IDS=$(curl -sf \
            -H "Authorization: Bearer ${GH_TOKEN}" \
            "https://api.github.com/repos/${REPO}/actions/runners?per_page=100" \
            | jq --arg pfx "${SERVER_NAME}" \
              '[.runners[] | select(.name | startswith($pfx)) | .id]')
          echo "Runner IDs to remove: ${RUNNER_IDS}"
          echo "${RUNNER_IDS}" | jq -r '.[]' | while read -r rid; do
            echo "  Deleting runner ${rid}..."
            curl -sf -X DELETE \
              -H "Authorization: Bearer ${GH_TOKEN}" \
              "https://api.github.com/repos/${REPO}/actions/runners/${rid}" || true
          done

      # continue-on-error: when provision fails before creating the server the
      # artifact was never uploaded; that must not fail cleanup (observed on
      # run 28648441942 — verify-image failed fast, no server existed, cleanup
      # went red on a missing artifact and masked the real failure).
      - name: Download server_id artifact
        continue-on-error: true
        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
        with:
          name: hetzner-server-id
          path: /tmp/hetzner

      - name: Delete Hetzner server
        env:
          HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
        run: |
          set -euo pipefail
          if [ ! -f /tmp/hetzner/server_id.txt ]; then
            echo "No server_id artifact — server was never provisioned, nothing to delete."
            exit 0
          fi
          SERVER_ID=$(cat /tmp/hetzner/server_id.txt)
          if [ -z "${SERVER_ID}" ] || [ "${SERVER_ID}" = "null" ]; then
            echo "No server ID found — nothing to delete."
            exit 0
          fi
          echo "Deleting Hetzner server ${SERVER_ID}..."
          curl -sf -X DELETE "https://api.hetzner.cloud/v1/servers/${SERVER_ID}" \
            -H "Authorization: Bearer ${HCLOUD_TOKEN}" || true
          echo "Server ${SERVER_ID} deletion requested."