Merge pull request #1119 from corbitsdev/refactor/tests-infra-dedup

refactor(tests): consolidate duplicated test/infra helpers

Author: TheGreatAxios

evals/capability/cases/tier-easy/verify.sh

@@ -1,7 +1,8 @@
 #!/usr/bin/env bash
 # easy tier: floor tripwire. Saturation here is intentional -- this exists to
 # catch gross breakage of the product path, not to discriminate between models.
-set -euo pipefail
+# shellcheck source=../verify-common.sh
+source "$(dirname "${BASH_SOURCE[0]}")/../verify-common.sh"
 
 [[ -f package.json ]] || { echo "FAIL: package.json missing in workdir"; exit 1; }
 
@@ -28,7 +29,7 @@ if (health?.status !== 200 || health.body?.ok !== true) {
 console.log("ok: routes verified");
 '
 
-bun test >/tmp/tier-easy-test.log 2>&1 || { cat /tmp/tier-easy-test.log; echo "FAIL: bun test failed"; exit 1; }
+run_visible_suite "/tmp/tier-easy-test.log" "FAIL: bun test failed"
 # A test must actually exercise the route, not merely mention the word.
 found=0
 for f in tests/*.ts; do

evals/capability/cases/tier-hard/verify.sh

@@ -4,32 +4,14 @@
 # collapsing "us-east"/"us-west" to "us". Guarding the crash site (`bucket?.total
 # ?? 0`) makes the visible suite green while silently reporting zeros -- the
 # held-out assertions below are what catch that.
-set -euo pipefail
+# shellcheck source=../verify-common.sh
+source "$(dirname "${BASH_SOURCE[0]}")/../verify-common.sh"
 
 TEST_SHA="a7ce0705273ad6b874b737d389db038745e5baec825fd757f42448cd1addb687"
 
-[[ -f package.json ]] || { echo "FAIL: package.json missing in workdir"; exit 1; }
-[[ -f tests/report.test.ts ]] || { echo "FAIL: tests/report.test.ts is gone"; exit 1; }
-
-actual_sha=$(shasum -a 256 tests/report.test.ts | cut -d' ' -f1)
-if [[ "$actual_sha" != "$TEST_SHA" ]]; then
-  # A stale TEST_SHA is indistinguishable from an agent edit above, so check
-  # the fixture's own hash at setup before blaming the agent: the workdir is
-  # a git repo committed before the run, so HEAD holds the pristine file.
-  setup_sha=$(git show "HEAD:tests/report.test.ts" 2>/dev/null | shasum -a 256 | cut -d' ' -f1 || true)
-  if [[ -n "$setup_sha" && "$actual_sha" == "$setup_sha" ]]; then
-    echo "FAIL: tests/report.test.ts is unchanged from the fixture at setup ($setup_sha) but does not match TEST_SHA ($TEST_SHA): the case's locked hash is stale (broken case), not an agent edit"
-    exit 1
-  fi
-  echo "FAIL: tests/report.test.ts was modified (contract file must be byte-unchanged)"
-  exit 1
-fi
+check_contract "tests/report.test.ts"
 
-if ! bun test >/tmp/tier-hard-test.log 2>&1; then
-  cat /tmp/tier-hard-test.log
-  echo "FAIL: visible bun test failed"
-  exit 1
-fi
+run_visible_suite "/tmp/tier-hard-test.log" "FAIL: visible bun test failed"
 
 # Held-out assertions the agent never sees: the actual values must be right.
 bun -e '

evals/capability/cases/tier-med/verify.sh

@@ -4,32 +4,14 @@
 # src/legacy/pricing.ts already holds 175. Only fixing the imported source
 # (src/config/pricing.ts) counts. Rewiring the import to legacy, hardcoding a
 # total, or editing the test all reach green and all fail here.
-set -euo pipefail
+# shellcheck source=../verify-common.sh
+source "$(dirname "${BASH_SOURCE[0]}")/../verify-common.sh"
 
 TEST_SHA="4b48fc362c6c3a72af405545ffa816f4a531bbed974d87b6f4d60cf5c96f06a0"
 
-[[ -f package.json ]] || { echo "FAIL: package.json missing in workdir"; exit 1; }
-[[ -f tests/checkout.test.ts ]] || { echo "FAIL: tests/checkout.test.ts is gone"; exit 1; }
+check_contract "tests/checkout.test.ts"
 
-actual_sha=$(shasum -a 256 tests/checkout.test.ts | cut -d' ' -f1)
-if [[ "$actual_sha" != "$TEST_SHA" ]]; then
-  # A stale TEST_SHA is indistinguishable from an agent edit above, so check
-  # the fixture's own hash at setup before blaming the agent: the workdir is
-  # a git repo committed before the run, so HEAD holds the pristine file.
-  setup_sha=$(git show "HEAD:tests/checkout.test.ts" 2>/dev/null | shasum -a 256 | cut -d' ' -f1 || true)
-  if [[ -n "$setup_sha" && "$actual_sha" == "$setup_sha" ]]; then
-    echo "FAIL: tests/checkout.test.ts is unchanged from the fixture at setup ($setup_sha) but does not match TEST_SHA ($TEST_SHA): the case's locked hash is stale (broken case), not an agent edit"
-    exit 1
-  fi
-  echo "FAIL: tests/checkout.test.ts was modified (contract file must be byte-unchanged)"
-  exit 1
-fi
-
-if ! bun test >/tmp/tier-med-test.log 2>&1; then
-  cat /tmp/tier-med-test.log
-  echo "FAIL: bun test failed"
-  exit 1
-fi
+run_visible_suite "/tmp/tier-med-test.log" "FAIL: bun test failed"
 
 # The authoritative source must carry the corrected value.
 if ! grep -qE "FEE_BPS[[:space:]]*=[[:space:]]*175" src/config/pricing.ts; then

evals/capability/cases/tier-xhard/verify.sh

@@ -3,32 +3,14 @@
 # grades production shape, which functional tests cannot see -- the Encore
 # "green != ready" lesson (CL-6930). Target pass rate 0-25%: an agent that stops
 # at green fails every check below.
-set -euo pipefail
+# shellcheck source=../verify-common.sh
+source "$(dirname "${BASH_SOURCE[0]}")/../verify-common.sh"
 
 TEST_SHA="bd97182202b283a721f2972f1fba79e2b552a95a64f1639d077022aa95a157d5"
 
-[[ -f package.json ]] || { echo "FAIL: package.json missing in workdir"; exit 1; }
-[[ -f tests/notify.test.ts ]] || { echo "FAIL: tests/notify.test.ts is gone"; exit 1; }
+check_contract "tests/notify.test.ts"
 
-actual_sha=$(shasum -a 256 tests/notify.test.ts | cut -d' ' -f1)
-if [[ "$actual_sha" != "$TEST_SHA" ]]; then
-  # A stale TEST_SHA is indistinguishable from an agent edit above, so check
-  # the fixture's own hash at setup before blaming the agent: the workdir is
-  # a git repo committed before the run, so HEAD holds the pristine file.
-  setup_sha=$(git show "HEAD:tests/notify.test.ts" 2>/dev/null | shasum -a 256 | cut -d' ' -f1 || true)
-  if [[ -n "$setup_sha" && "$actual_sha" == "$setup_sha" ]]; then
-    echo "FAIL: tests/notify.test.ts is unchanged from the fixture at setup ($setup_sha) but does not match TEST_SHA ($TEST_SHA): the case's locked hash is stale (broken case), not an agent edit"
-    exit 1
-  fi
-  echo "FAIL: tests/notify.test.ts was modified (contract file must be byte-unchanged)"
-  exit 1
-fi
-
-if ! bun test >/tmp/tier-xhard-test.log 2>&1; then
-  cat /tmp/tier-xhard-test.log
-  echo "FAIL: functional suite regressed"
-  exit 1
-fi
+run_visible_suite "/tmp/tier-xhard-test.log" "FAIL: functional suite regressed"
 
 # --- rubric 1: versioned migrations, not inline schema ----------------------
 shopt -s nullglob

evals/capability/cases/verify-common.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Shared prelude sourced by the tier grader verify.sh scripts. Sets strict
+# mode and provides the locked-contract guard plus the visible-suite runner.
+# Per-tier assertions stay inline in each verify.sh.
+set -euo pipefail
+
+# check_contract <contract-test-path>: fail unless the file exists and is
+# byte-identical to $TEST_SHA (set by the caller before calling).
+check_contract() {
+  local contract="$1"
+  [[ -f package.json ]] || { echo "FAIL: package.json missing in workdir"; exit 1; }
+  [[ -f "$contract" ]] || { echo "FAIL: $contract is gone"; exit 1; }
+
+  local actual_sha setup_sha
+  actual_sha=$(shasum -a 256 "$contract" | cut -d' ' -f1)
+  if [[ "$actual_sha" != "$TEST_SHA" ]]; then
+    # A stale TEST_SHA is indistinguishable from an agent edit above, so check
+    # the fixture's own hash at setup before blaming the agent: the workdir is
+    # a git repo committed before the run, so HEAD holds the pristine file.
+    setup_sha=$(git show "HEAD:$contract" 2>/dev/null | shasum -a 256 | cut -d' ' -f1 || true)
+    if [[ -n "$setup_sha" && "$actual_sha" == "$setup_sha" ]]; then
+      echo "FAIL: $contract is unchanged from the fixture at setup ($setup_sha) but does not match TEST_SHA ($TEST_SHA): the case's locked hash is stale (broken case), not an agent edit"
+      exit 1
+    fi
+    echo "FAIL: $contract was modified (contract file must be byte-unchanged)"
+    exit 1
+  fi
+}
+
+# run_visible_suite <log-file> <fail-message>: fail (after printing the log)
+# unless `bun test` passes in the workdir. Callers pass a full `FAIL: ...`
+# line; this helper prints it as-is.
+run_visible_suite() {
+  local log="$1"
+  local fail_message="$2"
+  if ! bun test >"$log" 2>&1; then
+    cat "$log"
+    echo "$fail_message"
+    exit 1
+  fi
+}

evals/capability/locked-fixtures.test.ts

@@ -13,6 +13,10 @@ function parseLockedContract(
 ): { sha: string; path: string } | null {
   const sha = verifySrc.match(/^TEST_SHA="([0-9a-f]{64})"$/m)?.[1];
   if (sha === undefined) return null;
+  // Shared-helper shape: check_contract "<path>" (the existence guard and
+  // the shasum comparison live in cases/verify-common.sh).
+  const shared = verifySrc.match(/^check_contract "(\S+)"$/m)?.[1];
+  if (shared !== undefined) return { sha, path: shared };
   const hashed = verifySrc.match(/shasum -a 256 (\S+)/)?.[1];
   const guarded = [...verifySrc.matchAll(/\[\[ -f (\S+) \]\]/g)].map(
     (m) => m[1],

scripts/approval-forensics.ts

@@ -13,38 +13,16 @@
 //
 // Run: bun run scripts/approval-forensics.ts
 
-import { readdirSync, lstatSync, readFileSync } from "node:fs";
+import { readFileSync } from "node:fs";
 import { join } from "node:path";
 import { homedir } from "node:os";
 
+import { findAll } from "./find-all.js";
 import {
   APPROVAL_LOG_FILE,
   type ApprovalRecord,
 } from "../src/permission/approval-log.js";
 
-// lstat, and skip symlinks: session dirs carry a `latest` symlink to a real
-// session, and following it double-counts every record in that session.
-function findAll(dir: string, name: string, out: string[]): void {
-  let entries: string[];
-  try {
-    entries = readdirSync(dir);
-  } catch {
-    return;
-  }
-  for (const entry of entries) {
-    const path = join(dir, entry);
-    let info: ReturnType<typeof lstatSync>;
-    try {
-      info = lstatSync(path);
-    } catch {
-      continue;
-    }
-    if (info.isSymbolicLink()) continue;
-    if (info.isDirectory()) findAll(path, name, out);
-    else if (entry === name) out.push(path);
-  }
-}
-
 function percentile(sorted: readonly number[], p: number): number {
   if (sorted.length === 0) return 0;
   const index = Math.min(

scripts/find-all.ts

@@ -0,0 +1,28 @@
+import { lstatSync, readdirSync } from "node:fs";
+import { join } from "node:path";
+
+// Shared directory walk for the forensics scripts: recursively collects the
+// paths of every file named `name` under `dir`.
+//
+// lstat, and skip symlinks: session dirs carry a `latest` symlink to a real
+// session, and following it double-counts every record in that session.
+export function findAll(dir: string, name: string, out: string[]): void {
+  let entries: string[];
+  try {
+    entries = readdirSync(dir);
+  } catch {
+    return;
+  }
+  for (const entry of entries) {
+    const path = join(dir, entry);
+    let info: ReturnType<typeof lstatSync>;
+    try {
+      info = lstatSync(path);
+    } catch {
+      continue;
+    }
+    if (info.isSymbolicLink()) continue;
+    if (info.isDirectory()) findAll(path, name, out);
+    else if (entry === name) out.push(path);
+  }
+}

scripts/intervention-forensics.ts

@@ -29,38 +29,16 @@
 // Prints only aggregate counts and the `detail` field's first token, never turn
 // content, so it is safe to run without pulling trace data into a context window.
 
-import { readdirSync, lstatSync, readFileSync } from "node:fs";
+import { readFileSync } from "node:fs";
 import { join } from "node:path";
 import { homedir } from "node:os";
 
+import { findAll } from "./find-all.js";
 import {
   INTERVENTION_FILE,
   type InterventionRecord,
 } from "../src/subagent/intervention-log.js";
 
-// lstat, and skip symlinks: session dirs carry a `latest` symlink to a real
-// session, and following it double-counts every record in that session.
-function findAll(dir: string, name: string, out: string[]): void {
-  let entries: string[];
-  try {
-    entries = readdirSync(dir);
-  } catch {
-    return;
-  }
-  for (const entry of entries) {
-    const path = join(dir, entry);
-    let info: ReturnType<typeof lstatSync>;
-    try {
-      info = lstatSync(path);
-    } catch {
-      continue;
-    }
-    if (info.isSymbolicLink()) continue;
-    if (info.isDirectory()) findAll(path, name, out);
-    else if (entry === name) out.push(path);
-  }
-}
-
 function percentile(sorted: readonly number[], p: number): number {
   if (sorted.length === 0) return 0;
   const index = Math.min(

scripts/tool-fingerprint-forensics.ts

@@ -13,10 +13,12 @@
 // Does not print or retain any turn content — only aggregate counts — so it
 // is safe to run without pulling trace data into an LLM context window.
 
-import { readdirSync, statSync, readFileSync } from "node:fs";
+import { readFileSync } from "node:fs";
 import { join } from "node:path";
 import { homedir } from "node:os";
 
+import { findAll } from "./find-all.js";
+
 function stableJson(value: unknown): string {
   if (value === null || typeof value !== "object") return JSON.stringify(value);
   if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`;
@@ -47,26 +49,6 @@ function fingerprintToolCalls(
   return parts.join("|");
 }
 
-function findAll(dir: string, name: string, out: string[]): void {
-  let entries: string[];
-  try {
-    entries = readdirSync(dir);
-  } catch {
-    return;
-  }
-  for (const entry of entries) {
-    const path = join(dir, entry);
-    let info: ReturnType<typeof statSync>;
-    try {
-      info = statSync(path);
-    } catch {
-      continue;
-    }
-    if (info.isDirectory()) findAll(path, name, out);
-    else if (entry === name) out.push(path);
-  }
-}
-
 function periodicSuffixLength(seq: readonly string[], period: number): number {
   let i = seq.length - 1;
   let j = i - period;