feat(triage-contracts): make Triage's waits overridable and publish them in the release

Every wait the hub, listener and manager share is one named timing in
triage-contracts with its production value as default and a TRIAGE_*_MS
override. The hub reads them once as it starts and writes the listener's
and manager's share into their release.json, since a workflow's process
gets none of the hub's environment; a tree without them runs on the
defaults. The hub's deploy hold stays derived from the same ready window
the listener waits.

Author: TheGreatAxios

.env.example

@@ -50,6 +50,16 @@ HUB_DATA_DIR=./tmp/hub-data
 
 # Milliseconds between passes that re-queue open pull requests whose head is not triaged. Default 300000.
 # TRIAGE_RECONCILE_INTERVAL_MS=
+# Milliseconds before a pass that could not reach a listener tries again, doubling to two minutes. Default 15000.
+# TRIAGE_RECONCILE_RETRY_MS=
+# Milliseconds a listener waits for a just-deployed manager to reach a sidecar before it gives up on it. Default 120000.
+# TRIAGE_MANAGER_READY_MS=
+# Milliseconds a just-deployed manager may have no sidecar and still hold its pull request. Default 120000.
+# TRIAGE_MANAGER_START_GRACE_MS=
+# Milliseconds a manager waiting on CI or a reviewer waits between checks. Default 30000.
+# TRIAGE_MANAGER_POLL_MS=
+# Milliseconds between a listener's checks on whether its manager reached a sidecar. Default 2000.
+# TRIAGE_WAKE_POLL_MS=
 
 # Pull request heads the reconciler lets a tenant have queued or running at once. Default 5.
 # TRIAGE_MAX_IN_FLIGHT=

apps/hub/src/env.ts

@@ -73,13 +73,9 @@ const HubEnvSchema = type({
   "RAILWAY_GIT_REPO_OWNER?": "string",
   "RAILWAY_GIT_REPO_NAME?": "string",
   "RAILWAY_ENVIRONMENT_ID?": "string",
-  "TRIAGE_RECONCILE_INTERVAL_MS?": positiveInteger,
   "TRIAGE_MAX_IN_FLIGHT?": positiveInteger,
   "TRIAGE_CAPPED_RETRY_MS?": positiveInteger,
-  "TRIAGE_COALESCE_QUIET_MS?": positiveInteger,
   "TRIAGE_MAX_LOCAL_SIDECARS?": positiveInteger,
-  "TRIAGE_STALLED_AFTER_MS?": positiveInteger,
-  "TRIAGE_LIVENESS_INTERVAL_MS?": positiveInteger,
   "HUB_AGENT_GC_PACK_THRESHOLD?": positiveInteger,
   "HUB_AGENT_GC_LOOSE_THRESHOLD?": positiveInteger,
   "HUB_AGENT_GC_WARN_BYTES?": positiveInteger,
@@ -268,20 +264,6 @@ export function migrationEnv(db: DatabaseConfig): Record<string, string> {
   };
 }
 
-export function triageReconcileIntervalMs(env: HubEnv): number {
-  return env.TRIAGE_RECONCILE_INTERVAL_MS === undefined ? 5 * 60_000 : Number(env.TRIAGE_RECONCILE_INTERVAL_MS);
-}
-
-/** How long a workspace's pull request may wait for a manager before the hub raises its alarm. */
-export function triageStalledAfterMs(env: Pick<HubEnv, "TRIAGE_STALLED_AFTER_MS">): number {
-  return numberOr(env.TRIAGE_STALLED_AFTER_MS, 5 * 60_000);
-}
-
-/** How often the hub checks whether each workspace's pull requests still reach a manager. */
-export function triageLivenessIntervalMs(env: Pick<HubEnv, "TRIAGE_LIVENESS_INTERVAL_MS">): number {
-  return numberOr(env.TRIAGE_LIVENESS_INTERVAL_MS, 60_000);
-}
-
 /** Heads the reconciler lets a tenant have queued or running at once. */
 export function triageReconcilePolicy(env: Pick<HubEnv, "TRIAGE_MAX_IN_FLIGHT" | "TRIAGE_CAPPED_RETRY_MS">): ReconcilePolicy {
   return {
@@ -302,8 +284,7 @@ const COALESCE_BURST_MS = 3_000;
  * A pull request's webhook events go out as one mail once none arrived for the quiet period, or at most four quiet periods after the first; the window must close before the reconciler gives up on its queued head.
  * A new commit or a newly opened pull request waits only for the burst, long enough to take in the review requests, labels and assignees GitHub delivers with it within a second or so.
  */
-export function triageCoalesceWindow(env: HubEnv, policy: ReconcilePolicy): CoalesceWindow {
-  const quietMs = numberOr(env.TRIAGE_COALESCE_QUIET_MS, 30_000);
+export function triageCoalesceWindow(quietMs: number, policy: ReconcilePolicy): CoalesceWindow {
   if (4 * quietMs >= policy.unstartedAfterMs) {
     throw new Error(`TRIAGE_COALESCE_QUIET_MS must be below ${policy.unstartedAfterMs / 4}: four times it must stay under the ${policy.unstartedAfterMs} ms a queued head may wait to start`);
   }

apps/hub/src/manager-state.ts

@@ -7,9 +7,10 @@ import { jsonb, primaryKey, text } from "drizzle-orm/pg-core";
 import { type } from "arktype";
 import { canExecuteWorkflowRun, schema, type DB } from "@intx/db";
 import { asRecord } from "@corbits/common";
-import { APPROVE_DO_KIND, AWAITED_STAGES, MANAGER_READY_MS, REPO_SOURCES, TRIAGE_JUDGES, TRIAGE_LISTENERS, TRIAGE_MANAGERS, type AwaitedStage, type RepoSource } from "@corbits/triage-contracts";
+import { APPROVE_DO_KIND, AWAITED_STAGES, REPO_SOURCES, TRIAGE_JUDGES, TRIAGE_LISTENERS, TRIAGE_MANAGERS, type AwaitedStage, type RepoSource } from "@corbits/triage-contracts";
 import type { Tx } from "./github/admin-github-grant.js";
 import { resolveLiveDeployments, workflowName, type LiveDeployment } from "./github/deployment.js";
+import { TIMINGS } from "./timings.js";
 import { triageSchema } from "./triage-db.js";
 import { START_DEADLINE_MS } from "./workflow-release/rollout.js";
 
@@ -194,7 +195,7 @@ export async function recordResult(db: DB["db"], tenantId: string, pr: PullReque
  * How long a just-deployed manager may have no sidecar and still hold the pull request, and a just-deployed listener
  * still count as live. The listener waits this long for a manager to start.
  */
-const MANAGER_START_GRACE_MS = 2 * 60_000;
+const MANAGER_START_GRACE_MS = TIMINGS.managerStartGraceMs;
 
 /**
  * How long a manager or judge may sit on its sidecar without its first mail starting it. Both are deployed with that
@@ -596,9 +597,9 @@ async function endedWaiting(tx: Tx, tenantId: string, pr: PullRequestRef, histor
 
 /**
  * How long an ask holds off the next: the listener's wake of the manager it deployed and of the one it deploys in its
- * place when that wake fails, each waiting `MANAGER_READY_MS`, and stock's 30-second cancel grace between them.
+ * place when that wake fails, each waiting `managerReadyMs`, and stock's 30-second cancel grace between them.
  */
-const DEPLOY_ASK_HOLD_MS = 2 * MANAGER_READY_MS + 30_000;
+const DEPLOY_ASK_HOLD_MS = 2 * TIMINGS.managerReadyMs + 30_000;
 
 /** Whether a manager the hub asked the listener for has not taken the claim yet and may still be starting. */
 function deploying(claim: Claim, now: Date): boolean {

apps/hub/src/server.ts

@@ -40,6 +40,7 @@ import { createManagerIntake, createManagerWake, createOverdueWake, createSupers
 import { MANAGER_RUN_TOKEN_ROUTES } from "./manager-routes.js";
 import { deploymentUp, endNeverRan, liveListener, notePlaced, passRequest } from "./manager-state.js";
 import { createTakeLedger, createTriageLiveness } from "./triage-liveness.js";
+import { TIMINGS } from "./timings.js";
 import { mountTenantRoutes } from "./tenant-routes.js";
 import { archiveStoredPolicies } from "./stored-policies.js";
 import { connectDockerSidecars, dockerPlaces, sweepDockerSidecarsOf } from "./sidecar/docker.js";
@@ -73,7 +74,7 @@ import { createListenerWatch, releaseWriteOf } from "./workflow-release/triggers
 import { setUpPublisher, TRIAGE_PUBLISHER } from "./workflow-release/publisher.js";
 import { catalogModels, createWorkspaceLock, deployedAllocation, deploymentState, liveDeployments } from "./workflow-release/reads.js";
 import { createRolloutQueue, createWorkflowRollout, stopGivenUp, type DeployRequest } from "./workflow-release/rollout.js";
-import { bitbucketEgressAllowlist, childProcessEnv, clientIpHeaders, databaseConfig, dockerSettings, e2bSettings, emailSettings, interchangeSettings, githubApiOrigin, loadHubEnv, inviteLinkOrigin, maxLocalSidecars, railwaySandboxSettings, signInSettings, triageCoalesceWindow, triageReconcileIntervalMs, triageReconcilePolicy, triageLivenessIntervalMs, triageStalledAfterMs } from "./env.js";
+import { bitbucketEgressAllowlist, childProcessEnv, clientIpHeaders, databaseConfig, dockerSettings, e2bSettings, emailSettings, interchangeSettings, githubApiOrigin, loadHubEnv, inviteLinkOrigin, maxLocalSidecars, railwaySandboxSettings, signInSettings, triageCoalesceWindow, triageReconcilePolicy } from "./env.js";
 import { HOOK_MOUNT_PATH } from "./hooks.js";
 import { assertMigrated } from "./migrations.js";
 import { createBridgeHandler, logJson, withAppLogin } from "./github/bridge.js";
@@ -132,7 +133,6 @@ const EXPECTED_DEPLOYMENT_SET = {
   "bitbucket-datacenter": { workflows: hostWorkflows("bitbucket-datacenter"), tools: [bitbucketRead.id] },
 };
 
-const TRIAGE_RETRY_MS = 15_000;
 const TRIAGE_RETRY_MAX_MS = 120_000;
 /** A repository is listed again once its listing is this old, and every pass once its webhook (the App's, on GitHub) heard nothing this long. */
 const TRIAGE_RELIST_QUIET_MS = 30 * 60_000;
@@ -652,7 +652,7 @@ const triageLiveness = createTriageLiveness({
   db: composition.db,
   store: triageStateStore,
   takes,
-  stalledAfterMs: triageStalledAfterMs(env),
+  stalledAfterMs: TIMINGS.stalledAfterMs,
   mail: smtp && async function mailStall(notice) {
     await smtp.triageStall({ ...notice, link: triageSettingsLink });
   },
@@ -699,7 +699,7 @@ async function reconcileAll(): Promise<ReconcileOutcome> {
   const outcome: ReconcileOutcome = { retry, ...(unrouted && { unrouted: true }) };
   return limits.length === 0 ? outcome : { ...outcome, retryAt: Math.min(...limits) };
 }
-triageLoop = createReconcileLoop(reconcileAll, { intervalMs: triageReconcileIntervalMs(env), retryMs: TRIAGE_RETRY_MS, retryMaxMs: TRIAGE_RETRY_MAX_MS }, logJson);
+triageLoop = createReconcileLoop(reconcileAll, { intervalMs: TIMINGS.reconcileIntervalMs, retryMs: TIMINGS.reconcileRetryMs, retryMaxMs: TRIAGE_RETRY_MAX_MS }, logJson);
 const kickTriage = triageLoop.kick;
 const githubPrTriage = createGithubPrTriage({
   ...credentialDeps,
@@ -778,7 +778,7 @@ coalescer = createCoalescer({
   sendMail: sendTriageMail,
   retry: kickTriage,
   openHeadFor: tenantOpenHead,
-  window: triageCoalesceWindow(env, reconcilePolicy),
+  window: triageCoalesceWindow(TIMINGS.coalesceQuietMs, reconcilePolicy),
   setTimer,
   now,
   log: logJson,
@@ -1087,7 +1087,7 @@ await takes.seed(composition.db);
 function checkLiveness(): void {
   void triageLiveness.check(reconcileTenants);
 }
-livenessTimer = setInterval(checkLiveness, triageLivenessIntervalMs(env));
+livenessTimer = setInterval(checkLiveness, TIMINGS.livenessIntervalMs);
 
 /** Rolls every workspace onto a new build's release, as many at once as the queue allows. */
 async function rollWorkspaces(): Promise<void> {

apps/hub/src/timings.ts

@@ -0,0 +1,4 @@
+import { timingsFrom } from "@corbits/triage-contracts";
+
+/** The hub's waits, read from its environment once as it starts; the release carries the workflows' share. */
+export const TIMINGS = timingsFrom(process.env);

apps/hub/src/workflow-release/publish.ts

@@ -6,8 +6,9 @@ import { and, eq } from "drizzle-orm";
 import git from "isomorphic-git";
 import { schema, type DB } from "@intx/db";
 import { AssetServiceError, DEFAULT_ASSET_REF, type AssetService, type RepoStore } from "@intx/hub-sessions";
-import { deployedByChain, publishable, withRelease, type ModelInfo, type PublishedSource, type SourceIndexEntry, type SourceTree } from "@corbits/triage-contracts";
+import { deployedByChain, publishable, withRelease, workflowTimings, type ModelInfo, type PublishedSource, type SourceIndexEntry, type SourceTree } from "@corbits/triage-contracts";
 import { allowCreator } from "../github/admin-github-grant.js";
+import { TIMINGS } from "../timings.js";
 
 /** The workflow sources the portal build writes under `packages/sources/`, in the order they publish. */
 export type Bundle = {
@@ -132,7 +133,7 @@ async function publishSource(deps: PublishDeps, publisher: Publisher, workflow:
 export async function publishRelease(deps: PublishDeps, publisher: Publisher, bundle: Bundle, models: readonly ModelInfo[]): Promise<PublishedSource[]> {
   const published: PublishedSource[] = [];
   for (const entry of (await bundle.index()).filter((row) => publishable(models, row.workflow))) {
-    const { hash, tree } = await withRelease(models, entry, await bundle.tree(entry), published);
+    const { hash, tree } = await withRelease(models, entry, await bundle.tree(entry), published, workflowTimings(TIMINGS));
     published.push(await publishSource(deps, publisher, entry.workflow, assetName(entry.workflow, publisher, hash), tree));
   }
   return published;

docs/SELF_HOST.md

@@ -291,6 +291,11 @@ All optional; the defaults suit most installs.
 | Variable | Default | Effect |
 | --- | --- | --- |
 | `TRIAGE_RECONCILE_INTERVAL_MS` | 300000 | How often the hub re-queues open pull requests whose latest commit has no verdict. |
+| `TRIAGE_RECONCILE_RETRY_MS` | 15000 | How soon a pass that could not reach a host's listener tries again, doubling up to two minutes. |
+| `TRIAGE_MANAGER_READY_MS` | 120000 | How long Triage waits for a pull request's just-started manager to come up before it starts another in its place. |
+| `TRIAGE_MANAGER_START_GRACE_MS` | 120000 | How long a just-started manager may take to come up and still keep its pull request. |
+| `TRIAGE_MANAGER_POLL_MS` | 30000 | How often a pull request's manager checks again while it waits for CI, the decision model or code review. |
+| `TRIAGE_WAKE_POLL_MS` | 2000 | How often Triage checks whether a just-started manager has come up. |
 | `TRIAGE_MAX_IN_FLIGHT` | 5 | Pull request heads a workspace may have queued or running at once. |
 | `TRIAGE_CAPPED_RETRY_MS` | 21600000 | How long a head that ran out of attempts waits before a fresh set. |
 | `TRIAGE_STALLED_AFTER_MS` | 300000 | How long a pull request may wait for Triage to start before the hub raises the alarm for its host in that workspace: the `triage_stalled` log line, a notice in the portal, and, with [email](#email) set up, one email to the workspace's owners and admins; `triage_stall_cleared` and a second email follow once Triage starts a pull request on that host again. The alarm fires within this time plus one `TRIAGE_LIVENESS_INTERVAL_MS`. |

packages/triage-contracts/src/index.ts

@@ -31,3 +31,4 @@ export * from "./policy-patch.js";
 export * from "./code-patterns.js";
 export * from "./visible-text.js";
 export * from "./workflow-release.js";
+export * from "./timings.js";

packages/triage-contracts/src/timings.ts

@@ -0,0 +1,61 @@
+import { type } from "arktype";
+
+/**
+ * Every wait the hub, the listener and the manager share, each its production value unless its variable overrides it.
+ * The hub reads them once as it starts and publishes the workflows' share in each release, since a workflow's process
+ * gets none of the hub's environment.
+ */
+export const TIMING_VARIABLES = {
+  /** How long a listener's wake mail waits for a just-deployed manager to reach a sidecar before it fails. */
+  managerReadyMs: "TRIAGE_MANAGER_READY_MS",
+  /** How long a just-deployed manager may have no sidecar and still hold its pull request. */
+  managerStartGraceMs: "TRIAGE_MANAGER_START_GRACE_MS",
+  /** How long a manager whose head awaits CI or a reviewer waits between passes. */
+  managerPollMs: "TRIAGE_MANAGER_POLL_MS",
+  /** How often a wake mail checks whether the manager it wakes has reached its sidecar. */
+  wakePollMs: "TRIAGE_WAKE_POLL_MS",
+  /** How often the hub re-queues open pull requests whose head has no verdict. */
+  reconcileIntervalMs: "TRIAGE_RECONCILE_INTERVAL_MS",
+  /** The first retry of a reconcile pass that could not mail a listener, doubling up to its ceiling. */
+  reconcileRetryMs: "TRIAGE_RECONCILE_RETRY_MS",
+  /** Quiet time before a pull request's webhook events go out as one mail. */
+  coalesceQuietMs: "TRIAGE_COALESCE_QUIET_MS",
+  /** How often the hub checks whether each workspace's pull requests still reach a manager. */
+  livenessIntervalMs: "TRIAGE_LIVENESS_INTERVAL_MS",
+  /** How long a pull request may wait for a manager before the hub raises its alarm. */
+  stalledAfterMs: "TRIAGE_STALLED_AFTER_MS",
+} as const;
+
+export type Timings = Record<keyof typeof TIMING_VARIABLES, number>;
+
+export const DEFAULT_TIMINGS: Timings = {
+  managerReadyMs: 2 * 60_000,
+  managerStartGraceMs: 2 * 60_000,
+  managerPollMs: 30_000,
+  wakePollMs: 2_000,
+  reconcileIntervalMs: 5 * 60_000,
+  reconcileRetryMs: 15_000,
+  coalesceQuietMs: 30_000,
+  livenessIntervalMs: 60_000,
+  stalledAfterMs: 5 * 60_000,
+};
+
+/** The waits the listener and manager run on, which the hub writes into their release. */
+export const WorkflowTimings = type({ managerReadyMs: "number.integer > 0", managerPollMs: "number.integer > 0", wakePollMs: "number.integer > 0" });
+export type WorkflowTimings = typeof WorkflowTimings.infer;
+
+export function workflowTimings({ managerReadyMs, managerPollMs, wakePollMs }: Timings): WorkflowTimings {
+  return { managerReadyMs, managerPollMs, wakePollMs };
+}
+
+/** The timings `env` sets, each unset one its default; a value that is not a positive integer throws. */
+export function timingsFrom(env: Readonly<Record<string, string | undefined>>): Timings {
+  const timings = { ...DEFAULT_TIMINGS };
+  for (const [key, variable] of Object.entries(TIMING_VARIABLES) as [keyof Timings, string][]) {
+    const value = env[variable];
+    if (value === undefined) continue;
+    if (!/^[1-9]\d*$/.test(value)) throw new Error(`${variable} must be a positive integer of milliseconds, not ${JSON.stringify(value)}`);
+    timings[key] = Number(value);
+  }
+  return timings;
+}

packages/triage-contracts/src/triage-state.ts

@@ -12,9 +12,6 @@ export const CI_WAIT_MS = 60 * 60_000;
 /** How long a head with no checks yet waits for its CI to start before it is taken as having none. */
 export const CI_START_MS = 2 * 60_000;
 
-/** How long a listener's wake mail waits for a just-deployed manager to reach a sidecar before it fails. */
-export const MANAGER_READY_MS = 2 * 60_000;
-
 /** `stale`: the base branch moved after the head's verdict, so it is due again. */
 export const PR_TRIAGE_STATUSES = ["new", "queued", "running", "triaged", "failed", "stale"] as const;
 export type PrTriageStatus = (typeof PR_TRIAGE_STATUSES)[number];