test(chain): wait on states with short timings and 100 ms polling

The chain starts its hub with every Triage wait short and polls every
100 ms. No scenario sleeps for a fixed time or asserts a duration: woken
waits for its manager to poll again, unset reads the hub's answer to the
delivery, unrouted keeps the reconcile pass out of reach instead of
timing the retry, and local-sidecars asserts each cause in its own test.
A lint rule under tests/chain fails on a literal real sleep of a second
or more, and AGENTS.md says a failed chain shard is read, never rerun.

Author: TheGreatAxios

AGENTS.md

@@ -57,6 +57,7 @@ Check these for a stock route or primitive before touching the hub, sidecars or
 - Tests beside the code are fast and offline; anything that runs real services lives in `e2e/`.
 - The one exception is `tests/chain/`: it starts the real hub and local sidecars but reaches no outside service (a fake GitHub and a stub decision model), and CI runs it as its own job, `bun run test:chain`.
 - Never read `process.env` in tests.
+- A failed chain shard is read, never rerun.
 - A change to how a workflow judges or reviews states its package's eval result before and after.
 
 ## Docs

docs/DEV.md

@@ -89,6 +89,8 @@ Hub tests connect to `postgres://postgres:postgres@localhost:5432/interchange` a
 
 `bun run test:chain` (`tests/chain/`) runs its files two at a time, one scenario each, each isolated from the others. Each file starts a hub on a database of its own, serving a portal build, sets up a workspace through the portal's routes, waits for the hub to start its listener, and sends GitHub's webhooks to it. GitHub and the decision model are local stand-ins, so it needs no account or key; each hub's sidecars are local processes, and it installs the workflows' native modules from the npm registry as a deploy does. Code review needs an isolated sidecar, which a local process is not, so it is not part of the chain test.
 
+The chain starts its hub with every Triage wait short (`TIMINGS` in `tests/chain/stack.ts`, set through the `TRIAGE_*_MS` variables in [SELF_HOST.md](SELF_HOST.md#triage-tuning)); the hub publishes the listener's and manager's share in their release, since a workflow's process gets none of the hub's environment. A scenario polls every 100 ms for the state it names and never sleeps for a fixed time, which `bun run lint` enforces under `tests/chain/`. The one real wait left is stock's one-minute ping timeout in the scenarios that freeze a listener's sidecar. A failed chain shard is read, never rerun.
+
 In CI the chain runs in five shards, each with its own list of files in `.github/workflows/ci.yml`; a new chain file must be added to one. A pull request that changes none of `apps/hub`, `apps/sidecar`, `apps/web/scripts`, `apps/web/src/lib`, `tooling/`, `packages/`, `tests/chain/`, `vendor/`, the lockfile, the root `package.json` or the CI workflow and setup actions skips the chain, so a change limited to the portal's pages and components, docs, e2e or research does not run it; every push to `main` runs it.
 
 Scripts are listed in [IMPLEMENTATION.md](IMPLEMENTATION.md#scripts).

tests/chain/.oxlintrc.json

@@ -0,0 +1,4 @@
+{
+  "jsPlugins": ["./lint.mjs"],
+  "rules": { "chain/no-real-sleep": "error" }
+}

tests/chain/chain.ts

@@ -12,11 +12,11 @@ import { parseTriageState, triageStateName, type PrTriageRow } from "@corbits/tr
 import { saveAutomationMode } from "../../apps/web/src/lib/hub-api.ts";
 import { fakeGithub, OWNER, type Pull, type Write } from "./github.ts";
 import { stubModel, type Answer, type Question } from "./model.ts";
-import { buildPortal, createDatabase, deliver, dropDatabase, listenerStarted, setUpWorkspace, signUp, startHub, type Database, type Hub, type Workspace } from "./stack.ts";
-
+import { buildPortal, createDatabase, deliver, dropDatabase, listenerStarted, POLL_MS, setUpWorkspace, signUp, startHub, type Database, type Hub, type Workspace } from "./stack.ts";
 
 const MANAGER = "triage-manager-github";
-// Every deployment starts a sidecar process of its own, and a judged pull request starts two in turn.
+// The deadline on a manager settling: every deployment starts a sidecar process of its own, and a judged pull request
+// starts two in turn.
 export const SETTLE_MS = 150_000;
 
 export const DETERMINISTIC = `${OWNER}/deterministic`;
@@ -77,6 +77,7 @@ export function chainStack(env: Record<string, string> = {}) {
   const model = stubModel(decide);
   const portalDir = mkdtempSync(join(tmpdir(), "corbits-chain-portal-"));
   let database: Database;
+  let handle: ReturnType<typeof createDB>;
   let hub: Hub;
   let workspace: Workspace;
 
@@ -95,6 +96,7 @@ export function chainStack(env: Record<string, string> = {}) {
     github.addRepo(UNROUTED, policy([MAX_FILES]));
     const [created] = await Promise.all([createDatabase(), buildPortal(portalDir)]);
     database = created;
+    handle = createDB(database.config);
     hub = await startHub(database, github, portalDir, env);
     workspace = await signUp(hub, database);
     // Until a scenario switches it to policy, only posting runs on its own, so a Do in a policy is held.
@@ -105,16 +107,17 @@ export function chainStack(env: Record<string, string> = {}) {
 
   afterAll(async () => {
     await hub?.stop();
+    await handle?.close();
     if (database) await dropDatabase(database);
     github.stop();
     model.stop();
     rmSync(portalDir, { recursive: true, force: true });
   });
 
-  /** GitHub's webhook for a pull request opened or pushed to, after the fake shows that head. */
-  async function pushed(repo: string, pull: Pull, action: "opened" | "synchronize" = "opened"): Promise<void> {
+  /** GitHub's webhook for a pull request opened or pushed to, after the fake shows that head; the hub's answer to it. */
+  async function pushed(repo: string, pull: Pull, action: "opened" | "synchronize" = "opened"): Promise<string> {
     github.open(repo, pull);
-    await deliver(hub, workspace, "pull_request", {
+    return deliver(hub, workspace, "pull_request", {
       action, installation: { id: 1 }, repository: { full_name: repo },
       pull_request: { number: pull.number, state: "open", draft: false, head: { sha: pull.head }, user: { login: pull.author }, title: pull.title, body: pull.body },
     });
@@ -159,18 +162,13 @@ export function chainStack(env: Record<string, string> = {}) {
     while (Date.now() < deadline) {
       const value = await read();
       if (value !== undefined) return value;
-      await Bun.sleep(500);
+      await Bun.sleep(POLL_MS);
     }
     throw new Error(`Timed out waiting for ${what}.\nRuns:\n${await runEvents()}\nGitHub misses: ${github.misses.join(", ")}\nHub:\n${hub.logs.slice(-80).join("\n")}`);
   }
 
   async function rows(query: string): Promise<Record<string, unknown>[]> {
-    const { db, close } = createDB(database.config);
-    try {
-      return [...await db.execute(sql.raw(query))] as Record<string, unknown>[];
-    } finally {
-      await close();
-    }
+    return [...await handle.db.execute(sql.raw(query))] as Record<string, unknown>[];
   }
 
   async function claimOf(repo: string, number: number): Promise<Claim | undefined> {

tests/chain/disconnected.test.ts

@@ -1,14 +1,15 @@
 // A listener whose sidecar stops answering.
 import { expect, test } from "bun:test";
 import { TRIAGE_LISTENERS, VERDICT_LABELS } from "@corbits/triage-contracts";
-import { RECONCILE_INTERVAL_MS } from "./stack.ts";
+import { TIMINGS } from "./stack.ts";
 import { chainStack, SETTLE_MS, DISCONNECTED, isLiveDeployment, sha, pullOf, comments, labelled, once } from "./chain.ts";
 
 const chain = chainStack();
 const { github, pushed, until, rows, settled } = chain;
 
 // The hub drops a silent sidecar's socket after stock's one-minute ping timeout, and the next reconcile pass checks it.
-const DISCONNECT_MS = 60_000 + 2 * RECONCILE_INTERVAL_MS + SETTLE_MS;
+// That timeout is the one real wait left: a sidecar pings on stock's own 30-second interval, which nothing here shortens.
+const DISCONNECT_MS = 60_000 + 2 * TIMINGS.reconcileIntervalMs + SETTLE_MS;
 
 /** Every GitHub listener deployment's anchor run, oldest first, and the pid of the local sidecar it runs on. */
 async function listeners(): Promise<{ id: string; status: string; pid: number }[]> {

tests/chain/lint.mjs

@@ -0,0 +1,29 @@
+// The chain's lint rule: a scenario waits on the state it names, never on a fixed real duration, so a sleep or timer of
+// a second or more written as a literal fails `bun run lint`.
+const LIMIT_MS = 1_000;
+
+function delayOf(node) {
+  const { callee } = node;
+  if (callee.type === "MemberExpression" && callee.object.name === "Bun" && callee.property.name === "sleep") return node.arguments[0];
+  if (callee.type === "Identifier" && callee.name === "sleep") return node.arguments[0];
+  if (callee.type === "Identifier" && callee.name === "setTimeout") return node.arguments[1];
+  return undefined;
+}
+
+export default {
+  meta: { name: "chain" },
+  rules: {
+    "no-real-sleep": {
+      create(context) {
+        return {
+          CallExpression(node) {
+            const delay = delayOf(node);
+            if (delay?.type === "Literal" && typeof delay.value === "number" && delay.value >= LIMIT_MS) {
+              context.report({ node, message: `Wait on the state the scenario names, not ${delay.value} ms of real time.` });
+            }
+          },
+        };
+      },
+    },
+  },
+};

tests/chain/local-sidecars.test.ts

@@ -8,7 +8,8 @@ import { sql } from "drizzle-orm";
 import { createDB } from "@intx/db";
 import { fakeGithub, OWNER, type Pull } from "./github.ts";
 import { stubModel } from "./model.ts";
-import { buildPortal, createDatabase, deliver, dropDatabase, listenerStarted, setUpWorkspace, signUp, startHub, type Database, type Hub, type Workspace } from "./stack.ts";
+import { once } from "./chain.ts";
+import { buildPortal, createDatabase, deliver, dropDatabase, listenerStarted, POLL_MS, setUpWorkspace, signUp, startHub, type Database, type Hub, type Workspace } from "./stack.ts";
 
 setDefaultTimeout(10 * 60_000);
 
@@ -79,7 +80,7 @@ async function until<T>(what: string, read: () => Promise<T | undefined>, ms = 1
   while (Date.now() < deadline) {
     const value = await read();
     if (value !== undefined) return value;
-    await Bun.sleep(500);
+    await Bun.sleep(POLL_MS);
   }
   throw new Error(`Timed out waiting for ${what}.\n${hub.logs.slice(-80).join("\n")}`);
 }
@@ -95,7 +96,7 @@ async function claimOf(repo: string): Promise<Claim | undefined> {
   return row?.["state"] as Claim | undefined;
 }
 
-test("the listener and two live managers each run in a sidecar of their own, and a released manager's sidecar is gone", async () => {
+const managed = once(async function managed() {
   const releaseFirst = github.hold(FIRST, 1);
   const releaseSecond = github.hold(SECOND, 1);
   await pushed(FIRST, sha("first-1"));
@@ -108,12 +109,8 @@ test("the listener and two live managers each run in a sidecar of their own, and
     const claim = await claimOf(SECOND);
     return claim?.status === "live" ? claim : undefined;
   });
-  expect(first.runId).not.toBe(second.runId);
-
   const live = await rows(`select a.id, a.sidecar_id, r.status, coalesce(s.frozen_approval_bundle -> 'projection' ->> 'id', d.name) as workflow from sidecar_allocation a join workflow_run r on r.id = a.anchor_run_id join workflow_definition d on d.id = r.definition_id left join workflow_run_launch_spec s on s.anchor_run_id = r.id where a.status = 'allocated'`);
-  expect(live.filter((row) => row["status"] === "running").map((row) => row["workflow"]).sort()).toEqual(["triage-listener-github", "triage-manager-github", "triage-manager-github"]);
-  expect(new Set(live.map((row) => row["sidecar_id"])).size).toBe(live.length);
-  expect(manifests().sort()).toEqual(live.map((row) => `${String(row["id"])}.json`).sort());
+  const liveManifests = manifests();
 
   await pushed(FIRST, sha("first-2"), "synchronize");
   await until("the first push's second head to be recorded", async () => {
@@ -130,21 +127,38 @@ test("the listener and two live managers each run in a sidecar of their own, and
     const claim = await claimOf(SECOND);
     return claim !== undefined && claim.history.length >= 1 ? claim : undefined;
   });
-  expect(firstDone.history.map((pass) => pass.runId)).toEqual([first.runId!, first.runId!]);
-  expect(firstDone.history.at(-1)!.head).toBe(sha("first-2"));
-  expect(secondDone.history).toEqual([expect.objectContaining({ head: sha("second-1"), runId: second.runId })]);
-  expect(github.pull(FIRST, 1).comments.map((comment) => comment.body)).toContainEqual(expect.stringContaining("**Triage: Pass**"));
-  expect(github.pull(SECOND, 1).comments.map((comment) => comment.body)).toEqual([expect.stringContaining("**Triage: Pass**")]);
   const managerAllocations = live.filter((row) => row["workflow"] === "triage-manager-github").map((row) => `'${String(row["id"])}'`).join(",");
   const released = await until("both managers' allocations to be released", async () => {
     const found = await rows(`select status, deployment_cleanup_confirmed from sidecar_allocation where id in (${managerAllocations})`);
     return found.every((row) => row["status"] === "released") ? found : undefined;
   });
-  expect(released).toEqual([{ status: "released", deployment_cleanup_confirmed: true }, { status: "released", deployment_cleanup_confirmed: true }]);
-  const listenerRow = live.find((row) => row["workflow"] === "triage-listener-github")!;
-  expect(manifests()).toEqual([`${String(listenerRow["id"])}.json`]);
+  const listenerId = String(live.find((row) => row["workflow"] === "triage-listener-github")!["id"]);
+  const listener = await rows(`select a.status, r.status as run from sidecar_allocation a join workflow_run r on r.id = a.anchor_run_id where a.id = '${listenerId}'`);
   const dataDirs = readdirSync(join(hub.dataDir, "local-sidecars")).filter((name) => name !== "manifests");
-  expect(dataDirs).toEqual([expect.stringMatching(new RegExp(`^${String(listenerRow["id"])}-`))]);
-  const listener = await rows(`select a.status, r.status as run from sidecar_allocation a join workflow_run r on r.id = a.anchor_run_id where a.id = '${String(listenerRow["id"])}'`);
+  return { first, second, live, liveManifests, firstDone, secondDone, released, listenerId, listener, manifests: manifests(), dataDirs };
+});
+
+test("the listener and two live managers each run in a sidecar of their own", async () => {
+  const { first, second, live, liveManifests } = await managed();
+  expect(first.runId).not.toBe(second.runId);
+  expect(live.filter((row) => row["status"] === "running").map((row) => row["workflow"]).sort()).toEqual(["triage-listener-github", "triage-manager-github", "triage-manager-github"]);
+  expect(new Set(live.map((row) => row["sidecar_id"])).size).toBe(live.length);
+  expect(liveManifests.sort()).toEqual(live.map((row) => `${String(row["id"])}.json`).sort());
+});
+
+test("each manager receives only its own pull request's mail", async () => {
+  const { first, second, firstDone, secondDone } = await managed();
+  expect(firstDone.history.map((pass) => pass.runId)).toEqual([first.runId!, first.runId!]);
+  expect(firstDone.history.at(-1)!.head).toBe(sha("first-2"));
+  expect(secondDone.history).toEqual([expect.objectContaining({ head: sha("second-1"), runId: second.runId })]);
+  expect(github.pull(FIRST, 1).comments.map((comment) => comment.body)).toContainEqual(expect.stringContaining("**Triage: Pass**"));
+  expect(github.pull(SECOND, 1).comments.map((comment) => comment.body)).toEqual([expect.stringContaining("**Triage: Pass**")]);
+});
+
+test("a released manager's sidecar is stopped and removed, and the listener's runs on", async () => {
+  const { released, listenerId, listener, manifests: left, dataDirs } = await managed();
+  expect(released).toEqual([{ status: "released", deployment_cleanup_confirmed: true }, { status: "released", deployment_cleanup_confirmed: true }]);
+  expect(left).toEqual([`${listenerId}.json`]);
+  expect(dataDirs).toEqual([expect.stringMatching(new RegExp(`^${listenerId}-`))]);
   expect(listener).toEqual([{ status: "allocated", run: "running" }]);
 });

tests/chain/reread.test.ts

@@ -1,7 +1,7 @@
 // A head whose CI finishes with no event: its manager ended while it waited, and the hub's reconcile pass wakes the next.
 import { expect, test } from "bun:test";
 import { VERDICT_LABELS } from "@corbits/triage-contracts";
-import { RECONCILE_INTERVAL_MS } from "./stack.ts";
+import { TIMINGS } from "./stack.ts";
 import { chainStack, REREAD, SETTLE_MS, isLiveDeployment, sha, pullOf, once } from "./chain.ts";
 
 const { github, pushed, until, managers, settled, verdictsOf } = chainStack();
@@ -16,7 +16,7 @@ const reread = once(async function reread() {
   const whileWaiting = { comments: github.pull(REREAD, 1).comments.map((comment) => comment.body), claim: ended, managers: (await managers()).slice(before) };
   // CI finishes with no check_run, as on a host whose builds send no webhook.
   github.setChecks(REREAD, head, [{ name: "build", status: "completed", conclusion: "success" }]);
-  const claim = await settled(REREAD, 1, ended.history.length + 1, 2 * RECONCILE_INTERVAL_MS + SETTLE_MS);
+  const claim = await settled(REREAD, 1, ended.history.length + 1, 2 * TIMINGS.reconcileIntervalMs + SETTLE_MS);
   return { head, whileWaiting, claim, deployed: (await managers()).slice(before), pull: github.pull(REREAD, 1) };
 });
 

tests/chain/sidecar-cap.test.ts

@@ -6,9 +6,10 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { sql } from "drizzle-orm";
 import { createDB } from "@intx/db";
+import { DEFAULT_TIMINGS, TIMING_VARIABLES } from "@corbits/triage-contracts";
 import { fakeGithub, OWNER, type Pull } from "./github.ts";
 import { stubModel } from "./model.ts";
-import { buildPortal, createDatabase, deliver, dropDatabase, listenerStarted, setUpWorkspace, signUp, startHub, type Database, type Hub, type Workspace } from "./stack.ts";
+import { buildPortal, createDatabase, deliver, dropDatabase, listenerStarted, POLL_MS, setUpWorkspace, signUp, startHub, type Database, type Hub, type Workspace } from "./stack.ts";
 
 setDefaultTimeout(10 * 60_000);
 
@@ -30,7 +31,8 @@ beforeAll(async () => {
   github.addRepo(SECOND, POLICY);
   const [created] = await Promise.all([createDatabase(), buildPortal(portalDir)]);
   database = created;
-  hub = await startHub(database, github, portalDir, { TRIAGE_MAX_LOCAL_SIDECARS: "2" });
+  // The second manager's wake waits out the first manager's whole pass for a slot, so it keeps the production window.
+  hub = await startHub(database, github, portalDir, { TRIAGE_MAX_LOCAL_SIDECARS: "2", [TIMING_VARIABLES.managerReadyMs]: String(DEFAULT_TIMINGS.managerReadyMs) });
   workspace = await signUp(hub, database);
   await setUpWorkspace(workspace, github, model.origin);
   await listenerStarted(workspace);
@@ -75,7 +77,7 @@ async function until<T>(what: string, read: () => Promise<T | undefined>, ms = 1
   while (Date.now() < deadline) {
     const value = await read();
     if (value !== undefined) return value;
-    await Bun.sleep(500);
+    await Bun.sleep(POLL_MS);
   }
   throw new Error(`Timed out waiting for ${what}.\n${hub.logs.slice(-80).join("\n")}`);
 }