feat(jev): escalate unsure decisions to an agent

README.md

@@ -80,6 +80,18 @@ probability is under `sure` (0.6). Retune them per judge with
 Thresholds are tuned to a judge version, so pin `model` — e.g.
 `jev({ model: "jev-1.13.0" })` — when they must stay stable.
 
+Escalation: `sure` doubles as the escalation gate. Give the judge an
+agent — `jev({ escalate: agent })`, anything with
+`send(prompt)` returning `{ type: "reply", reply }` or
+`{ type: "suspended" }`, e.g. an `@intx/agent` agent from a `@corbits/*`
+adapter — and every unsure decision is re-asked with the exact answer
+space. The reply goes through the same pass rules minus the confidence
+gate, so a primary unsure can still fail after escalation. A thrown,
+suspended, or off-answer-space reply fails the case closed. Validate the
+escalated model per workload too: Jev's unsure says nothing about the
+fallback's accuracy, and reasoning models may answer in prose the answer
+space cannot absorb — those fail closed by design.
+
 ## Runs
 
 - **Function**: `(input, signal) => output`. `signal` aborts on `timeout`.

e2e/jev.e2e.test.ts

@@ -10,6 +10,7 @@ import {
   evaluate,
   jev,
   type Check,
+  type Escalation,
   type Judge,
   type Outcome,
   type Report,
@@ -359,6 +360,149 @@ describe("ask", () => {
   });
 });
 
+describe("escalation", () => {
+  const question = "Is this reply ready to send to the customer?";
+  const unsure = { type: "noul", noul: 0.5 } as const;
+
+  function escalating(primary: unknown, send: Escalation["send"]) {
+    const { judge } = stubJev(primary);
+    return { ...judge, escalate: { send } };
+  }
+
+  test("an unsure primary decision escalates and the reply follows the pass rules", async () => {
+    const prompts: string[] = [];
+    const judge = escalating(unsure, async (prompt) => {
+      prompts.push(prompt);
+      return { type: "reply", reply: "yes" };
+    });
+    const { outcome } = await judged(question, judge);
+    expect(outcome).toEqual({
+      pass: true,
+      value: 1,
+      label: "yes",
+      explanation: 'escalated to "yes"',
+    });
+    expect(prompts[0]).toContain(question);
+    expect(prompts[0]).toContain('"yes", "no"');
+    expect(prompts[0]).toContain("refund?");
+  });
+
+  test("an escalated no fails the case", async () => {
+    const judge = escalating(unsure, async () => ({ type: "reply", reply: "no" }));
+    const { outcome } = await judged(question, judge);
+    expect(outcome).toEqual({
+      pass: false,
+      value: 0,
+      label: "no",
+      explanation: 'escalated to "no"',
+    });
+  });
+
+  test("a confident primary decision never escalates", async () => {
+    let calls = 0;
+    const judge = escalating({ type: "noul", noul: 0.9 }, async () => {
+      calls += 1;
+      return { type: "reply", reply: "yes" };
+    });
+    const { outcome } = await judged(question, judge);
+    expect(outcome?.pass).toBe(true);
+    expect(calls).toBe(0);
+  });
+
+  test("escalation trouble fails closed", async () => {
+    const throws = escalating(unsure, async () => {
+      throw new Error("boom");
+    });
+    expect((await judged(question, throws)).outcome).toMatchObject({
+      pass: false,
+      explanation: "unsure, and escalation threw: boom",
+    });
+    const suspends = escalating(unsure, async () => ({ type: "suspended" }));
+    expect((await judged(question, suspends)).outcome).toMatchObject({
+      pass: false,
+      explanation: "unsure, and escalation did not reply",
+    });
+    const offSpace = escalating(unsure, async () => ({
+      type: "reply",
+      reply: "maybe, could go either way",
+    }));
+    expect((await judged(question, offSpace)).outcome).toMatchObject({
+      pass: false,
+      explanation:
+        'unsure, and escalation replied outside the answer space ("maybe, could go either way")',
+    });
+  });
+
+  test("an ask check escalates in its own answer space", async () => {
+    const check = ask("Where does the reply fall short?", {
+      options: { ready: "complete", revise: "needs work", escalate: "beyond repair" },
+      pass: "ready",
+    });
+    const judge = escalating(
+      {
+        type: "choice",
+        choice: "revise",
+        probabilities: { ready: 0.3, revise: 0.35, escalate: 0.35 },
+        confidence: 0.35,
+      },
+      async () => ({ type: "reply", reply: "I would say: revise" }),
+    );
+    const { outcome } = await judged(check, judge);
+    expect(outcome).toEqual({
+      pass: false,
+      value: 0,
+      label: "revise",
+      explanation: 'escalated to "revise"',
+    });
+  });
+
+  test("credit rules survive escalation", async () => {
+    const check = ask("How complete is the reply?", {
+      options: { full: 1, partial: 0.75, hedged: 0.5 },
+      passAt: 0.5,
+    });
+    const judge = escalating(
+      {
+        type: "choice",
+        choice: "hedged",
+        probabilities: { full: 0.2, partial: 0.3, hedged: 0.5 },
+        confidence: 0.5,
+      },
+      async () => ({ type: "reply", reply: "hedged" }),
+    );
+    const { outcome } = await judged(check, judge);
+    expect(outcome).toEqual({
+      pass: true,
+      value: 0.5,
+      label: "hedged",
+      explanation: 'escalated to "hedged" (credit 0.50)',
+    });
+  });
+
+  test("answers with non-word edges still match", async () => {
+    const check = ask("Is the task complete?", {
+      options: { "@done": "finished", open: "not finished" },
+      pass: "@done",
+    });
+    const judge = escalating(
+      {
+        type: "choice",
+        choice: "open",
+        probabilities: { "@done": 0.45, open: 0.55 },
+        confidence: 0.55,
+      },
+      async () => ({ type: "reply", reply: "I think: @done" }),
+    );
+    const { outcome } = await judged(check, judge);
+    expect(outcome).toEqual({
+      pass: true,
+      value: 1,
+      label: "@done",
+      explanation: 'escalated to "@done"',
+    });
+  });
+});
+
 describe("stored reports", () => {
   test("a saved run keeps the distribution", async () => {
     const dir = await mkdtemp(join(tmpdir(), "jev-e2e-"));

src/index.ts

@@ -37,6 +37,7 @@ export {
   jev,
   DEFAULTS,
   type Ask,
+  type Escalation,
   type JevOptions,
   type Judge,
   type Sentence,

src/jev.ts

@@ -20,6 +20,8 @@ export type JevOptions = {
   timeout?: Duration;
   /** Pass bars for every Jev check through this judge. */
   thresholds?: Thresholds;
+  /** Consulted when a decision lands unsure. `sure` doubles as the escalation gate. */
+  escalate?: Escalation;
   /** Replaces `fetch` and the scheduler, e.g. to stub Jev in tests. */
   deps?: EvaluateDeps;
 };
@@ -35,9 +37,22 @@ export const DEFAULTS: Resolved = { yes: 0.7, no: 0.3, sure: 0.6 };
 export type Judge = {
   readonly config: EvaluateConfig;
   readonly thresholds?: Resolved;
+  readonly escalate?: Escalation;
   readonly deps?: EvaluateDeps;
 };
 
+/**
+ * An agent consulted when the primary Jev decision lands unsure — e.g. an
+ * `@intx/agent` agent from a `@corbits/*` adapter. It gets the same question
+ * with the exact answer space; its reply must stay inside that space.
+ */
+export type Escalation = {
+  readonly send: (
+    content: string,
+    opts?: { signal?: AbortSignal },
+  ) => Promise<{ type: "reply"; reply: string } | { type: "suspended" }>;
+};
+
 function resolve(label: string, raw: Thresholds): Resolved {
   const out: Resolved = {
     yes: raw.yes ?? DEFAULTS.yes,
@@ -72,6 +87,7 @@ export function jev(options: JevOptions = {}): Judge {
   return {
     config,
     ...(thresholds === undefined ? {} : { thresholds }),
+    ...(options.escalate === undefined ? {} : { escalate: options.escalate }),
     ...(options.deps === undefined ? {} : { deps: options.deps }),
   };
 }
@@ -343,16 +359,21 @@ export async function askJevBatch(
     return failedAll(names, `jev fallback (${out.reason}): ${out.detail ?? "no detail"}`);
   }
   const decisions = new Map(out.decisions.map((decision) => [decision.id, decision]));
-  return Object.fromEntries(
-    names.map((name) => {
+  const entries = await Promise.all(
+    names.map(async (name) => {
       const decision = decisions.get(name);
+      const check = checks[name] as Ask | Sentence | string;
       const outcome =
         decision === undefined
           ? failed("jev returned no decision")
-          : toOutcome(checks[name] as Ask | Sentence | string, decision, judge);
-      return [name, outcome];
+          : toOutcome(check, decision, judge);
+      if (judge.escalate !== undefined && isUnsure(outcome)) {
+        return [name, await escalateUnsure(judge, check, run)] as const;
+      }
+      return [name, outcome] as const;
     }),
   );
+  return Object.fromEntries(entries);
 }
 
 /**
@@ -381,3 +402,103 @@ export async function decideBatch(
 function failedAll(names: readonly string[], explanation: string): Record<string, Outcome> {
   return Object.fromEntries(names.map((name) => [name, failed(explanation)]));
 }
+
+/** Unsure outcomes are exactly the failed ones whose explanation starts "unsure (". */
+function isUnsure(outcome: Outcome): boolean {
+  return !outcome.pass && (outcome.explanation?.startsWith("unsure (") ?? false);
+}
+
+function allowedAnswers(check: Ask | Sentence | string): string[] {
+  if (typeof check === "string" || "yes" in check) return ["yes", "no"];
+  const a = check as Ask;
+  if (a.levels !== undefined) return [...a.levels];
+  if (a.options !== undefined) return Object.keys(a.options);
+  return [...(a.pass ?? [])];
+}
+
+function escalationPrompt(check: Ask | Sentence | string, run: Run, answers: string[]): string {
+  const question = typeof check === "string" ? check : check.question;
+  return [
+    "A primary judge could not decide the following case confidently. Decide it.",
+    `Question: ${question}`,
+    `Reply with exactly one of: ${answers.map((answer) => `"${answer}"`).join(", ")}.`,
+    `Case: ${JSON.stringify(toState(run))}`,
+  ].join("\n\n");
+}
+
+/**
+ * Longest match wins, so overlapping names resolve to the more specific one.
+ * Boundary assertions apply only at word-character edges: `\b` sits between
+ * two non-word characters where no boundary exists, so answers like "@done"
+ * or "50%" would otherwise never match an exact reply.
+ */
+function pickAnswer(reply: string, answers: string[]): string | undefined {
+  const escaped = (answer: string) => answer.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  const edge = (answer: string, at: "head" | "tail") => {
+    const c = at === "head" ? answer.charAt(0) : answer.charAt(answer.length - 1);
+    return /[A-Za-z0-9_]/.test(c) ? "\\b" : "";
+  };
+  for (const answer of [...answers].sort((a, b) => b.length - a.length)) {
+    const pattern = `${edge(answer, "head")}${escaped(answer)}${edge(answer, "tail")}`;
+    if (new RegExp(pattern, "i").test(reply)) return answer;
+  }
+  return undefined;
+}
+
+/** The escalated answer as a Decision, so `toOutcome`'s own rules apply. */
+function escalatedDecision(check: Ask | Sentence | string, picked: string): Decision {
+  if (typeof check === "string" || "yes" in check)
+    return { id: "escalated", type: "noul", noul: picked.toLowerCase() === "yes" ? 1 : 0 };
+  const a = check as Ask;
+  if (a.levels !== undefined)
+    return {
+      id: "escalated",
+      type: "score",
+      score: a.levels.indexOf(picked),
+      legend: Object.fromEntries(a.levels.map((level) => [level, level])),
+      probabilities: { [picked]: 1 },
+      confidence: 1,
+    };
+  return {
+    id: "escalated",
+    type: "choice",
+    choice: picked,
+    probabilities: { [picked]: 1 },
+    confidence: 1,
+  };
+}
+
+/**
+ * Consult `judge.escalate` for an unsure decision. The reply goes through the
+ * same pass rules as a primary decision, minus the confidence gate; a thrown,
+ * suspended, or off-space reply fails the case closed.
+ */
+async function escalateUnsure(
+  judge: Judge,
+  check: Ask | Sentence | string,
+  run: Run,
+): Promise<Outcome> {
+  const escalation = judge.escalate as Escalation;
+  let reply: string;
+  try {
+    const result = await escalation.send(escalationPrompt(check, run, allowedAnswers(check)));
+    if (result.type !== "reply") return failed(`unsure, and escalation did not reply`);
+    reply = result.reply;
+  } catch (cause) {
+    return failed(
+      `unsure, and escalation threw: ${cause instanceof Error ? cause.message : String(cause)}`,
+    );
+  }
+  const picked = pickAnswer(reply, allowedAnswers(check));
+  if (picked === undefined)
+    return failed(
+      `unsure, and escalation replied outside the answer space ("${reply.trim().slice(0, 40)}")`,
+    );
+  const outcome = toOutcome(check, escalatedDecision(check, picked), judge);
+  delete outcome.probabilities;
+  delete outcome.confidence;
+  const raw = (check as Ask).options?.[picked];
+  const credit = typeof raw === "number" ? raw : undefined;
+  outcome.explanation = `escalated to "${picked}"${credit === undefined ? "" : ` (credit ${p(credit)})`}`;
+  return outcome;
+}