feat!: run the judge as one workflow step

README.md

@@ -69,20 +69,22 @@ A check that throws, hangs past `timeout`, or returns anything else fails
 itself closed; its siblings still run. Jev transport failures fail closed.
 
 Jev sees `{ input, output, expectedOutput, steps }`, and a case's string and
-`ask` checks answer in one call. Configure it with
-`judge: jev({ endpoint, model, apiKey, timeout })`; `endpoint` is
-`"official"` (default), `"gateway"` or a URL. Without an `apiKey` the
-key comes from the environment as `@corbits/system-one` reads it.
+`ask` checks answer in one call. The model is not configured on `jev()`.
+Build it with `judgeModel({ model, invokeStep, thresholds, escalate, timeout })`
+from `@corbits/evals/workflow`. `model` is one workflow step (or a one-step
+workflow). `invokeStep` runs that step; its input is `{ state, questions }`
+and its output is `{ decisions }` or `{ fallback: true, reason }`.
+`jev({ thresholds, escalate })` only sets bars and an escalation agent.
 
 Thresholds: a yes/no check passes at p(yes) >= `yes` and fails at
 p(yes) <= `no` (0.7 / 0.3); `ask` fails as unsure when its top
 probability is under `sure` (0.6). Retune them per judge with
-`jev({ thresholds: { yes, no, sure } })` or per check as above.
-Thresholds are tuned to a judge version, so pin `model` — e.g.
-`jev({ model: "jev-1.13.0" })` — when they must stay stable.
+`judgeModel({ thresholds: { yes, no, sure }, ... })` or per check as above.
+Thresholds are tuned to a judge version, so pin the step agent's model
+when they must stay stable.
 
 Escalation: `sure` doubles as the escalation gate. Give the judge an
-agent — `jev({ escalate: agent })`, anything with
+agent — `escalate` on `jev` or `judgeModel`, anything with
 `send(prompt)` returning `{ type: "reply", reply }` or
 `{ type: "suspended" }`, e.g. an `@intx/agent` agent from a `@corbits/*`
 adapter — and every unsure decision is re-asked with the exact answer
@@ -104,7 +106,8 @@ space cannot absorb — those fail closed by design.
   definition is refused. Each step lands in `r.steps`. `output` picks the
   result from the finished run (default: the last listed step's output),
   `trigger` maps the input to the trigger payload. Without them, type the
-  run as `workflow<I, O>(...)`.
+  run as `workflow<I, O>(...)`. The same entry exports `judgeModel` for
+  string and `ask` scorers. A code-only eval needs no judge.
 
 `repeats` runs each case several times; a check passes only if every
 attempt passes. `concurrency` (default 4) bounds cases in flight;

e2e/calibrate.e2e.test.ts

@@ -1,48 +1,43 @@
 import { describe, expect, test } from "bun:test";
-import type { EvaluateDeps } from "@corbits/system-one";
+import { defineAgent } from "@intx/agent";
+import { step, type StepInvoker } from "@intx/workflow";
 
-import { ask, calibrate, jev } from "../src/index.js";
-
-function stubDeps(fetch: EvaluateDeps["fetch"]): EvaluateDeps {
-  return {
-    fetch,
-    scheduler: {
-      setTimeout: ((fn: () => void, ms: number) => {
-        const t = setTimeout(fn, ms);
-        return () => clearTimeout(t);
-      }) as EvaluateDeps["scheduler"]["setTimeout"],
-      now: () => Date.now(),
-    },
-  };
-}
+import { ask, calibrate } from "../src/index.js";
+import { judgeModel } from "../src/workflow.js";
 
 /** Answers yes/no questions with the input's probability; choice questions pick ready/revise by it. */
 function stubCalibration(byInput: Record<string, number>) {
-  const fetch = (async (_url: string, init: RequestInit) => {
-    const body = JSON.parse(String(init.body)) as {
+  const invokeStep: StepInvoker = async (request) => {
+    const body = request.input as {
       state: { input: string };
       questions: Record<string, { type: string }>;
     };
     const p = byInput[body.state.input] ?? 0.5;
-    const answers = Object.fromEntries(
-      Object.entries(body.questions).map(([id, q]) => [
-        id,
-        q.type === "choice"
-          ? {
-              type: "choice",
-              choice: p >= 0.5 ? "ready" : "revise",
-              probabilities: { ready: p, revise: Math.round((1 - p) * 100) / 100 },
-              confidence: p,
-            }
-          : { type: "noul", noul: p },
-      ]),
+    const decisions = Object.entries(body.questions).map(([id, q]) =>
+      q.type === "choice"
+        ? {
+            id,
+            type: "choice",
+            choice: p >= 0.5 ? "ready" : "revise",
+            probabilities: { ready: p, revise: Math.round((1 - p) * 100) / 100 },
+            confidence: p,
+          }
+        : { id, type: "noul", noul: p },
     );
-    return new Response(JSON.stringify({ model: "jev-test", answers }), {
-      status: 200,
-      headers: { "content-type": "application/json" },
-    });
-  }) as unknown as EvaluateDeps["fetch"];
-  return jev({ apiKey: "test-key", deps: stubDeps(fetch) });
+    return { output: { decisions } };
+  };
+  return judgeModel({
+    model: step({
+      agent: defineAgent({
+        id: "jev",
+        systemPrompt: "judge",
+        tools: [],
+        capabilities: [],
+        inference: { sources: [{ provider: "stub", model: "stub" }] },
+      }),
+    }),
+    invokeStep,
+  });
 }
 
 describe("calibrate", () => {