Skip to content

Commit ea825cc

Browse files
TheGreatAxiosTheGreatAxios
authored
feat(jev): calibrate thresholds on labeled samples (#15)
* feat(jev): calibrate thresholds on labeled samples * fix(calibrate): sweep ignores a check's own threshold overrides
1 parent 5a777a1 commit ea825cc

7 files changed

Lines changed: 421 additions & 8 deletions

File tree

‎README.md‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,26 @@ eval, with no file: `evaluate(name, { ...options, compare: before })`.
142142
Flips print as `fixed` / `broke`, new and gone cases as `added` /
143143
`removed`. `quiet: true` skips the summary.
144144

145+
## Calibrating
146+
147+
Thresholds are per-workload starting points, not constants. `calibrate`
148+
runs a labeled sample through Jev once and prints, per check, how many
149+
verdicts agree, land unsure, or confidently disagree at each threshold:
150+
151+
```ts
152+
const table = await calibrate("refund-agent", {
153+
run: refundAgent,
154+
cases: labeledCases, // { input, expect?, labels: { readyToSend: true } }
155+
checks: { readyToSend: "Is this reply ready to send to the customer?" },
156+
judge,
157+
});
158+
console.log(table.markdown());
159+
```
160+
161+
Yes/no checks sweep `yes` with `no` mirrored; `ask` sweeps `sure`. Paste
162+
the recommended override into the check or judge. Zero confident
163+
disagreements is the goal; if none exists, the fewest wins.
164+
145165
## Peers
146166

147167
- `@corbits/evals`: `@intx/inference` and `@intx/types`, peers of

‎e2e/calibrate.e2e.test.ts‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
import { describe, expect, test } from "bun:test";
2+
import type { EvaluateDeps } from "@corbits/system-one";
3+
4+
import { ask, calibrate, jev } from "../src/index.js";
5+
6+
function stubDeps(fetch: EvaluateDeps["fetch"]): EvaluateDeps {
7+
return {
8+
fetch,
9+
scheduler: {
10+
setTimeout: ((fn: () => void, ms: number) => {
11+
const t = setTimeout(fn, ms);
12+
return () => clearTimeout(t);
13+
}) as EvaluateDeps["scheduler"]["setTimeout"],
14+
now: () => Date.now(),
15+
},
16+
};
17+
}
18+
19+
/** Answers yes/no questions with the input's probability; choice questions pick ready/revise by it. */
20+
function stubCalibration(byInput: Record<string, number>) {
21+
const fetch = (async (_url: string, init: RequestInit) => {
22+
const body = JSON.parse(String(init.body)) as {
23+
state: { input: string };
24+
questions: Record<string, { type: string }>;
25+
};
26+
const p = byInput[body.state.input] ?? 0.5;
27+
const answers = Object.fromEntries(
28+
Object.entries(body.questions).map(([id, q]) => [
29+
id,
30+
q.type === "choice"
31+
? {
32+
type: "choice",
33+
choice: p >= 0.5 ? "ready" : "revise",
34+
probabilities: { ready: p, revise: Math.round((1 - p) * 100) / 100 },
35+
confidence: p,
36+
}
37+
: { type: "noul", noul: p },
38+
]),
39+
);
40+
return new Response(JSON.stringify({ model: "jev-test", answers }), {
41+
status: 200,
42+
headers: { "content-type": "application/json" },
43+
});
44+
}) as unknown as EvaluateDeps["fetch"];
45+
return jev({ apiKey: "test-key", deps: stubDeps(fetch) });
46+
}
47+
48+
describe("calibrate", () => {
49+
const question = "Is this reply ready to send to the customer?";
50+
51+
test("sweeps thresholds and recommends the most permissive clean bar", async () => {
52+
// a: confidently yes (label yes), b: leans no (label no), c: confidently no (label no).
53+
const judge = stubCalibration({ a: 0.9, b: 0.4, c: 0.1 });
54+
const result = await calibrate("cal", {
55+
run: (input: string) => `reply to ${input}`,
56+
cases: [
57+
{ input: "a", labels: { ready: true } },
58+
{ input: "b", labels: { ready: false } },
59+
{ input: "c", labels: { ready: false } },
60+
],
61+
checks: { ready: question },
62+
judge,
63+
});
64+
expect(result.total).toBe(3);
65+
const t = result.checks["ready"] as {
66+
rows: { threshold: number; agrees: number; unsure: number; disagrees: number }[];
67+
recommended: number;
68+
setting: string;
69+
};
70+
// Every verdict agrees at 0.5: yes stays yes, 0.4 and 0.1 fall in "no".
71+
expect(t.rows[0]).toEqual({ threshold: 0.5, agrees: 3, unsure: 0, disagrees: 0 });
72+
// At 0.95 the mirrored no bar is 0.05, so every case lands unsure — high bars
73+
// trade disagreements for unsure, never for silent agreement.
74+
const top = t.rows[t.rows.length - 1] as {
75+
threshold: number;
76+
agrees: number;
77+
unsure: number;
78+
disagrees: number;
79+
};
80+
expect(top).toEqual({ threshold: 0.95, agrees: 0, unsure: 3, disagrees: 0 });
81+
expect(t.recommended).toBe(0.5);
82+
expect(t.setting).toBe("{ yes: 0.5, no: 0.5 }");
83+
expect(result.markdown()).toContain("| 0.5 | 3 | 0 | 0 |");
84+
expect(result.markdown()).toContain("recommended: `{ yes: 0.5, no: 0.5 }`");
85+
});
86+
87+
test("an ask check sweeps the sure bar instead", async () => {
88+
const judge = stubCalibration({ a: 0.9, b: 0.9, c: 0.9 });
89+
const result = await calibrate("cal", {
90+
run: (input: string) => `reply to ${input}`,
91+
cases: [
92+
{ input: "a", labels: { ready: true } },
93+
{ input: "b", labels: { ready: true } },
94+
{ input: "c", labels: { ready: false } },
95+
],
96+
checks: {
97+
ready: ask("Ready to send?", {
98+
options: { ready: "send", revise: "work" },
99+
pass: "ready",
100+
}),
101+
},
102+
judge,
103+
});
104+
const t = result.checks["ready"] as {
105+
rows: { threshold: number; agrees: number; unsure: number; disagrees: number }[];
106+
recommended: number;
107+
};
108+
// Top probability 0.9 clears every sure bar up to 0.9, so case c confidently disagrees there;
109+
// at 0.95 all three land unsure and nothing disagrees.
110+
expect(t.rows[0]).toEqual({ threshold: 0.5, agrees: 2, unsure: 0, disagrees: 1 });
111+
expect(t.rows[t.rows.length - 1]).toEqual({
112+
threshold: 0.95,
113+
agrees: 0,
114+
unsure: 3,
115+
disagrees: 0,
116+
});
117+
expect(t.recommended).toBe(0.95);
118+
});
119+
120+
test("a check's own bars never pin the sweep", async () => {
121+
// The check carries sure: 0.6 — without stripping it, every grid row would
122+
// evaluate at 0.6 and the sweep would show no sensitivity at all.
123+
const judge = stubCalibration({ a: 0.9, b: 0.9 });
124+
const result = await calibrate("cal", {
125+
run: (input: string) => `reply to ${input}`,
126+
cases: [
127+
{ input: "a", labels: { ready: true } },
128+
{ input: "b", labels: { ready: false } },
129+
],
130+
checks: {
131+
ready: ask("Ready to send?", {
132+
options: { ready: "send", revise: "work" },
133+
pass: "ready",
134+
sure: 0.6,
135+
}),
136+
},
137+
judge,
138+
});
139+
const t = result.checks["ready"] as {
140+
rows: { threshold: number; agrees: number; unsure: number; disagrees: number }[];
141+
};
142+
// Case b (label no) confidently disagrees at low bars, turns unsure at 0.95.
143+
expect(t.rows[0]).toEqual({ threshold: 0.5, agrees: 1, unsure: 0, disagrees: 1 });
144+
expect(t.rows[t.rows.length - 1]).toEqual({
145+
threshold: 0.95,
146+
agrees: 0,
147+
unsure: 2,
148+
disagrees: 0,
149+
});
150+
});
151+
152+
test("every check needs a label on every case", async () => {
153+
const judge = stubCalibration({ a: 0.9 });
154+
await expect(
155+
calibrate("cal", {
156+
run: (input: string) => input,
157+
cases: [{ input: "a", labels: {} }],
158+
checks: { ready: question },
159+
judge,
160+
}),
161+
).rejects.toThrow('case "a" has no label for check "ready"');
162+
});
163+
});

‎src/calibrate.ts‎

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
import type { Decision } from "@corbits/system-one";
2+
3+
import { caseName } from "./evaluate.js";
4+
import { DEFAULTS, decideBatch, toOutcome, type Ask, type Judge, type Sentence } from "./jev.js";
5+
import { mapPool, runOnce, type RunSpec } from "./runner.js";
6+
import { toTraced, type Runnable } from "./runs.js";
7+
import { toMs, type Duration } from "./timeout.js";
8+
import type { Outcome } from "./types.js";
9+
10+
/** A case with a known verdict per check, e.g. hand-labeled or from `must`. */
11+
export type Labeled<I = unknown, E = unknown> = {
12+
name?: string;
13+
input: I;
14+
expect?: E;
15+
/** The expected verdict per check name. */
16+
labels: Readonly<Record<string, boolean>>;
17+
};
18+
19+
export type CalibrateOptions<I, O, E> = {
20+
run: Runnable<I, O>;
21+
cases: readonly Labeled<I, E>[];
22+
checks: Record<string, Ask | Sentence | string>;
23+
judge: Judge;
24+
concurrency?: number;
25+
/** Bounds each case run. The Jev call is bounded by `judge.timeout`. */
26+
timeout?: Duration;
27+
};
28+
29+
export type CalibrationRow = {
30+
threshold: number;
31+
agrees: number;
32+
unsure: number;
33+
disagrees: number;
34+
};
35+
36+
export type CalibrationCheck = {
37+
rows: CalibrationRow[];
38+
/** The most permissive threshold with no confident disagreements. */
39+
recommended: number;
40+
/** The override to paste into the check or judge. */
41+
setting: string;
42+
};
43+
44+
export type Calibration = {
45+
name: string;
46+
total: number;
47+
checks: Record<string, CalibrationCheck>;
48+
markdown(): string;
49+
};
50+
51+
/** The sweep. 0.5 is chance for a binary; 0.95 leaves almost nothing unsure. */
52+
const GRID = [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95];
53+
54+
function r3(n: number): number {
55+
return Math.round(n * 1000) / 1000;
56+
}
57+
58+
function isUnsure(o: Outcome): boolean {
59+
return o.explanation?.startsWith("unsure (") === true;
60+
}
61+
62+
/**
63+
* The sweep owns the bars: a check's own `sure`/`yes`/`no` overrides would
64+
* outrank every candidate judge (check-level wins over judge-level in
65+
* `thresholdsFor`), making every row identical and the recommendation a no-op.
66+
*/
67+
function sweepable(check: Ask | Sentence | string): Ask | Sentence | string {
68+
if (typeof check === "string") return check;
69+
const copy: Record<string, unknown> = { ...check };
70+
delete copy["sure"];
71+
delete copy["yes"];
72+
delete copy["no"];
73+
return copy as Ask | Sentence;
74+
}
75+
76+
/**
77+
* Run a labeled sample through Jev once and sweep the pass bars, so
78+
* thresholds become a per-workload choice instead of a constant. Yes/no
79+
* checks sweep `yes` with `no` mirrored; `ask` sweeps `sure`.
80+
*/
81+
export async function calibrate<I, O, E = O>(
82+
name: string,
83+
options: CalibrateOptions<I, O, E>,
84+
): Promise<Calibration> {
85+
if (name.trim() === "") throw new Error("calibrate needs a name");
86+
const names = Object.keys(options.checks);
87+
if (names.length === 0) throw new Error(`calibrate "${name}" has no checks`);
88+
const run = toTraced(options.run);
89+
const timeoutMs = options.timeout === undefined ? undefined : toMs(options.timeout, "timeout");
90+
const concurrency = options.concurrency ?? 4;
91+
if (!Number.isInteger(concurrency) || concurrency < 1) {
92+
throw new Error(`calibrate "${name}": concurrency must be a whole number >= 1`);
93+
}
94+
95+
type Sample = { decision: Decision; label: boolean };
96+
const samples: Record<string, Sample[]> = {};
97+
for (const checkName of names) samples[checkName] = [];
98+
99+
const spec: RunSpec<I, O, E> = {
100+
cases: [],
101+
run,
102+
checks: {},
103+
repeats: 1,
104+
concurrency: 1,
105+
...(timeoutMs === undefined ? {} : { timeoutMs }),
106+
};
107+
108+
await mapPool(options.cases, concurrency, async (labeled) => {
109+
const key = caseName(labeled);
110+
for (const checkName of names) {
111+
if (labeled.labels[checkName] === undefined) {
112+
throw new Error(`calibrate "${name}": case "${key}" has no label for check "${checkName}"`);
113+
}
114+
}
115+
const evalCase = {
116+
name: key,
117+
input: labeled.input,
118+
...(labeled.expect === undefined ? {} : { expect: labeled.expect }),
119+
};
120+
const runResult = await runOnce(spec, evalCase);
121+
const decisions = await decideBatch(options.judge, options.checks, runResult);
122+
for (const checkName of names) {
123+
const decision = decisions[checkName];
124+
if (decision === undefined) {
125+
throw new Error(`calibrate "${name}": Jev returned no decision for "${checkName}"`);
126+
}
127+
samples[checkName]?.push({ decision, label: labeled.labels[checkName] as boolean });
128+
}
129+
});
130+
131+
const checks: Record<string, CalibrationCheck> = {};
132+
for (const checkName of names) {
133+
const sample = samples[checkName] ?? [];
134+
const check = options.checks[checkName] as Ask | Sentence | string;
135+
const isQuestion =
136+
typeof check !== "string" &&
137+
(("options" in check && check.options !== undefined) ||
138+
("levels" in check && check.levels !== undefined));
139+
const base = options.judge.thresholds ?? DEFAULTS;
140+
const rows: CalibrationRow[] = [];
141+
for (const threshold of GRID) {
142+
const judge: Judge = {
143+
config: {},
144+
thresholds: isQuestion
145+
? { yes: base.yes, no: base.no, sure: threshold }
146+
: { yes: threshold, no: r3(1 - threshold), sure: base.sure },
147+
};
148+
let agrees = 0;
149+
let unsure = 0;
150+
let disagrees = 0;
151+
for (const s of sample) {
152+
const o = toOutcome(sweepable(check), s.decision, judge);
153+
if (isUnsure(o)) unsure += 1;
154+
else if (o.pass === s.label) agrees += 1;
155+
else disagrees += 1;
156+
}
157+
rows.push({ threshold, agrees, unsure, disagrees });
158+
}
159+
const clean = rows.find((row) => row.disagrees === 0);
160+
const recommended =
161+
clean?.threshold ?? rows.reduce((a, b) => (b.disagrees <= a.disagrees ? b : a)).threshold;
162+
const setting = isQuestion
163+
? `{ sure: ${recommended} }`
164+
: `{ yes: ${recommended}, no: ${r3(1 - recommended)} }`;
165+
checks[checkName] = { rows, recommended, setting };
166+
}
167+
168+
const data = { name, total: options.cases.length, checks };
169+
return { ...data, markdown: () => toMarkdown(data) };
170+
}
171+
172+
function toMarkdown(data: {
173+
name: string;
174+
total: number;
175+
checks: Record<string, CalibrationCheck>;
176+
}): string {
177+
const lines = [`# calibrate: ${data.name}`, ``, `- cases: ${data.total}`, ``];
178+
for (const [checkName, t] of Object.entries(data.checks)) {
179+
lines.push(
180+
`## ${checkName}`,
181+
``,
182+
`| threshold | agrees | unsure | disagrees |`,
183+
`| --- | --- | --- | --- |`,
184+
);
185+
for (const row of t.rows) {
186+
lines.push(`| ${row.threshold} | ${row.agrees} | ${row.unsure} | ${row.disagrees} |`);
187+
}
188+
lines.push(``, `recommended: \`${t.setting}\``, ``);
189+
}
190+
return lines.join("\n");
191+
}

‎src/evaluate.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ function toAllowances(
111111
);
112112
}
113113

114-
function caseName(evalCase: Case<unknown, unknown>): string {
114+
export function caseName(evalCase: Case<unknown, unknown>): string {
115115
if (evalCase.name !== undefined) return evalCase.name;
116116
return typeof evalCase.input === "string"
117117
? evalCase.input

0 commit comments

Comments
 (0)