From 2f4662200f14730d00931a4136cefb1c8290fbb1 Mon Sep 17 00:00:00 2001 From: "user.email" Date: Mon, 26 Oct 2026 02:05:06 -0700 Subject: [PATCH] feat: add string includes and prebuilt judge scorers --- README.md | 25 ++-- e2e/prebuilt.e2e.test.ts | 154 ++++++++++++++++++++++++ src/checks.ts | 26 +++++ src/index.ts | 1 + src/prebuilt.ts | 245 +++++++++++++++++++++++++++++++++++++++ src/workflow.ts | 11 ++ 6 files changed, 450 insertions(+), 12 deletions(-) create mode 100644 e2e/prebuilt.e2e.test.ts create mode 100644 src/prebuilt.ts diff --git a/README.md b/README.md index de6bfb8..50da473 100644 --- a/README.md +++ b/README.md @@ -47,18 +47,19 @@ returned. `unsure (p=…)` in between. `{ question, yes, no }` sets its own bars. - **Helpers**: -| helper | passes when | -| ----------------------------- | ------------------------------------------------------------------------------ | -| `matches(field?)` | output deep-equals expectedOutput, or the same field (key or accessor) of each | -| `fitsSchema(schema)` | output satisfies a Standard Schema (arktype, zod, valibot) | -| `mentions(words)` | output contains every word, case-insensitive; value is the fraction found | -| `avoids(words)` | output contains none of the words | -| `ranSteps(names)` | `tool` and `step` names match `names`, order ignored; value is their F1 | -| `stepsAtMost(n)` | the run took at most `n` steps | -| `under({ ms })` | the run finished within `ms` | -| `ask(q, { options, pass })` | Jev picks an option named in `pass` | -| `ask(q, { options, passAt })` | Jev picks an option; its credit (0..1) is the value and passes at `passAt` | -| `ask(q, { levels, pass })` | Jev grades at level `pass` or above (levels lowest first) | +| helper | passes when | +| ----------------------------- | ----------------------------------------------------------------------------------------------------------- | +| `matches(field?)` | output deep-equals expectedOutput, or the same field (key or accessor) of each | +| `includes()` | both strings, and output contains expectedOutput after NFKC, trim, and whitespace collapse; value is 1 or 0 | +| `fitsSchema(schema)` | output satisfies a Standard Schema (arktype, zod, valibot) | +| `mentions(words)` | output contains every word, case-insensitive; value is the fraction found | +| `avoids(words)` | output contains none of the words | +| `ranSteps(names)` | `tool` and `step` names match `names`, order ignored; value is their F1 | +| `stepsAtMost(n)` | the run took at most `n` steps | +| `under({ ms })` | the run finished within `ms` | +| `ask(q, { options, pass })` | Jev picks an option named in `pass` | +| `ask(q, { options, passAt })` | Jev picks an option; its credit (0..1) is the value and passes at `passAt` | +| `ask(q, { levels, pass })` | Jev grades at level `pass` or above (levels lowest first) | `mentions` and `avoids` also take `(r) => words` for per-case words. `mentions` and `ranSteps` take `{ atLeast }` to pass on partial credit. diff --git a/e2e/prebuilt.e2e.test.ts b/e2e/prebuilt.e2e.test.ts new file mode 100644 index 0000000..b1f8bcf --- /dev/null +++ b/e2e/prebuilt.e2e.test.ts @@ -0,0 +1,154 @@ +import { describe, expect, test } from "bun:test"; + +import { includes, matches, runEvaluation } from "../src/index.js"; +import { + answerRelevancy, + correctness, + faithfulness, + parsePrebuilt, + trajectoryAccuracy, + type PrebuiltId, +} from "../src/workflow.js"; + +const source = { provider: "ollama", model: "pinned" }; + +function judgeOf( + id: PrebuiltId, + definition: { steps: Record; stepOrder: readonly string[] }, +) { + expect(definition.stepOrder).toEqual(["judge"]); + const judge = definition.steps["judge"]; + expect(judge).toMatchObject({ + kind: "step", + input: { from: "trigger.payload" }, + agent: { + id: `${id}-judge`, + capabilities: [], + inference: { sources: [source] }, + }, + }); + const agent = (judge as { agent: { toolFactories: unknown[]; systemPrompt: string } }).agent; + expect(agent.toolFactories).toEqual([]); + expect(agent.systemPrompt.length).toBeGreaterThan(0); +} + +describe("includes", () => { + test("matches and includes share one string expectedOutput", async () => { + const report = await runEvaluation({ + name: "includes-mix", + task: (input: string) => `Answer: ${input}`, + data: [{ input: "Refund", expectedOutput: " reFUND " }], + scorers: { exact: matches(), has: includes() }, + allow: { exact: 1 }, + }); + expect(report.ok).toBe(true); + expect(report.cases[0]?.checks["exact"]?.pass).toBe(false); + expect(report.cases[0]?.checks["has"]).toEqual({ pass: true, value: 1 }); + }); + + test("fails closed on a non-string, a missing expected, or an empty needle", () => { + const check = includes(); + const run = { input: "", steps: [], ms: 0 }; + expect(check({ ...run, output: { a: 1 }, expectedOutput: '{"a":1}' })).toMatchObject({ + pass: false, + value: 0, + }); + expect(check({ ...run, output: "hello" })).toMatchObject({ pass: false, value: 0 }); + expect(check({ ...run, output: "hello", expectedOutput: " \n\t " })).toMatchObject({ + pass: false, + value: 0, + }); + expect(check({ ...run, output: "see the file now", expectedOutput: "file now" })).toEqual({ + pass: true, + value: 1, + }); + }); +}); + +describe("prebuilt judges", () => { + test("each factory is one judge step pinned to the source", () => { + const run = { + input: "q", + output: "a", + expectedOutput: "e", + steps: [{ name: "lookup", kind: "tool" as const }], + ms: 1, + }; + const checks = [ + faithfulness(source), + answerRelevancy(source), + correctness(source), + trajectoryAccuracy(source, ["lookup"]), + ]; + for (const check of checks) judgeOf(check.id, check.definition); + expect(faithfulness(source).payload(run)).toEqual({ context: "e", answer: "a" }); + expect(faithfulness(source, { context: () => "ctx" }).payload(run)).toEqual({ + context: "ctx", + answer: "a", + }); + expect(answerRelevancy(source).payload({ ...run, expectedOutput: "ignored" })).toEqual({ + question: "q", + answer: "a", + }); + expect(correctness(source).payload(run)).toEqual({ input: "q", expected: "e", answer: "a" }); + expect(trajectoryAccuracy(source, (r) => r.steps.map((s) => s.name)).payload(run)).toEqual({ + expected: ["lookup"], + actual: run.steps, + }); + expect(faithfulness(source).payload({ ...run, output: 1 })).toMatchObject({ + pass: false, + value: 0, + }); + expect(() => faithfulness({ provider: "", model: "m" })).toThrow("non-empty"); + }); + + test("parsePrebuilt accepts one object and rejects fences and a score past 1", () => { + expect(parsePrebuilt("faithfulness", { category: "supported" })).toEqual({ + pass: true, + value: 1, + label: "supported", + }); + expect(parsePrebuilt("faithfulness", '{"category":"unverifiable"}')).toMatchObject({ + pass: false, + value: 0.5, + label: "unverifiable", + }); + expect( + parsePrebuilt("correctness", { category: "incorrect", explanation: "no" }), + ).toMatchObject({ + pass: false, + value: 0, + label: "incorrect", + }); + expect(parsePrebuilt("answerRelevancy", { score: 1, explanation: "on topic" })).toEqual({ + pass: true, + value: 1, + explanation: "on topic", + }); + expect( + parsePrebuilt("trajectoryAccuracy", { score: 0.5, explanation: "missed a step" }), + ).toEqual({ + pass: false, + value: 0.5, + explanation: "missed a step", + }); + const fenced = '```json\n{"score":1,"explanation":"ok"}\n```'; + expect(parsePrebuilt("answerRelevancy", fenced)).toEqual({ + pass: false, + value: 0, + explanation: "judge output failed closed", + }); + expect(parsePrebuilt("answerRelevancy", { score: 1.5, explanation: "too high" })).toEqual({ + pass: false, + value: 0, + explanation: "judge output failed closed", + }); + expect( + parsePrebuilt("correctness", '{"category":"correct"}{"category":"incorrect"}'), + ).toMatchObject({ pass: false, value: 0 }); + expect(parsePrebuilt("faithfulness", "the answer is supported")).toMatchObject({ + pass: false, + value: 0, + }); + }); +}); diff --git a/src/checks.ts b/src/checks.ts index c13bc41..613a289 100644 --- a/src/checks.ts +++ b/src/checks.ts @@ -27,6 +27,32 @@ export function toOutcome(name: string, raw: unknown): Outcome { }; } +/** Case-insensitive substring. Value is 1 or 0. Strings only; no coercion. */ +export function includes(): CheckFn { + return ({ output, expectedOutput }) => { + if (typeof output !== "string" || typeof expectedOutput !== "string") { + return { + pass: false, + value: 0, + explanation: "includes needs string output and expectedOutput", + }; + } + const needle = collapsed(expectedOutput); + if (needle.length === 0) { + return { pass: false, value: 0, explanation: "includes needle is empty" }; + } + const pass = collapsed(output).includes(needle); + return pass + ? { pass: true, value: 1 } + : { pass: false, value: 0, explanation: "output does not include the expected text" }; + }; +} + +/** NFKC, trim, collapse internal whitespace, then lowercase. */ +function collapsed(value: string): string { + return value.normalize("NFKC").trim().replace(/\s+/g, " ").toLowerCase(); +} + /** Output deep-equals expectedOutput, or one field of each (a key or an accessor). */ export function matches( field?: string | ((value: T) => unknown), diff --git a/src/index.ts b/src/index.ts index efe1428..5ab8218 100644 --- a/src/index.ts +++ b/src/index.ts @@ -13,6 +13,7 @@ export type { export { avoids, fitsSchema, + includes, matches, mentions, ranSteps, diff --git a/src/prebuilt.ts b/src/prebuilt.ts new file mode 100644 index 0000000..14a0916 --- /dev/null +++ b/src/prebuilt.ts @@ -0,0 +1,245 @@ +import { defineAgent } from "@intx/agent"; +import { defineWorkflow, step, type WorkflowDefinition } from "@intx/workflow"; + +import type { Outcome, Run, Step } from "./types.js"; + +/** Pinned inference source for a prebuilt judge. Both fields are required. */ +export type ModelSource = { provider: string; model: string }; + +export type PrebuiltId = "faithfulness" | "answerRelevancy" | "correctness" | "trajectoryAccuracy"; + +/** + * A one-step judge definition plus the payload for its trigger and the + * outcome used when the run cannot be judged. This does not call a model. + */ +export type Prebuilt

= { + id: PrebuiltId; + definition: WorkflowDefinition; + payload: (run: Run) => P | Outcome; + failure: Outcome; +}; + +const CLOSED: Outcome = { + pass: false, + value: 0, + explanation: "judge output failed closed", +}; + +const FAITH = ["contradicted", "unverifiable", "supported"] as const; +const CORRECT = ["incorrect", "partial", "correct"] as const; + +const RUBRIC = { + faithfulness: [ + "You judge whether every claim in the answer is faithful to the context.", + "Reply with one JSON object and nothing else. No prose. No markdown fences.", + '{"category":"contradicted"|"unverifiable"|"supported","explanation":"..."}', + "Order, lowest first: contradicted, unverifiable, supported.", + "contradicted: the answer conflicts with the context.", + "unverifiable: the answer adds claims the context neither supports nor contradicts.", + "supported: every claim in the answer is supported by the context.", + "Pick the lowest category that applies.", + ].join("\n"), + answerRelevancy: [ + "You judge how completely the answer addresses the question.", + "Reply with one JSON object and nothing else. No prose. No markdown fences.", + '{"score":0,"explanation":"..."}', + "score is a finite number from 0 to 1. 1 means the answer is fully relevant.", + ].join("\n"), + correctness: [ + "You judge whether the answer is correct for the input, given the expected answer.", + "Reply with one JSON object and nothing else. No prose. No markdown fences.", + '{"category":"incorrect"|"partial"|"correct","explanation":"..."}', + "Order, lowest first: incorrect, partial, correct.", + "incorrect: the answer is wrong. partial: some of it is right. correct: it matches.", + "Pick the lowest category that applies.", + ].join("\n"), + trajectoryAccuracy: [ + "You judge how closely the actual steps follow the expected trajectory.", + "Reply with one JSON object and nothing else. No prose. No markdown fences.", + '{"score":0,"explanation":"..."}', + "score is a finite number from 0 to 1. 1 means the trajectory is accurate.", + "expected is the names that should have run. actual is the steps that did.", + ].join("\n"), +} as const; + +function requireSource(source: ModelSource): ModelSource { + if (source.provider.trim() === "" || source.model.trim() === "") { + throw new Error("prebuilt judge needs a non-empty provider and model"); + } + return { provider: source.provider, model: source.model }; +} + +function judgeWorkflow(id: PrebuiltId, source: ModelSource): WorkflowDefinition { + const pinned = requireSource(source); + return defineWorkflow({ + id, + trigger: { type: "manual" }, + steps: { + judge: step({ + agent: defineAgent({ + id: `${id}-judge`, + systemPrompt: RUBRIC[id], + tools: [], + capabilities: [], + inference: { sources: [pinned] }, + }), + input: { from: "trigger.payload" }, + }), + }, + }); +} + +function closed(explanation: string): Outcome { + return { pass: false, value: 0, explanation }; +} + +function isOutcome(value: object): value is Outcome { + return "pass" in value && "value" in value; +} + +export function faithfulness( + source: ModelSource, + options: { context?: (run: Run) => unknown } = {}, +): Prebuilt<{ context: string; answer: string }> { + const contextOf = options.context; + return { + id: "faithfulness", + definition: judgeWorkflow("faithfulness", source), + failure: CLOSED, + payload: (run) => { + if (typeof run.output !== "string") return closed("faithfulness needs a string answer"); + const context = + contextOf !== undefined + ? contextOf(run) + : typeof run.expectedOutput === "string" + ? run.expectedOutput + : run.input; + if (typeof context !== "string") return closed("faithfulness needs a string context"); + return { context, answer: run.output }; + }, + }; +} + +export function answerRelevancy( + source: ModelSource, +): Prebuilt<{ question: string; answer: string }> { + return { + id: "answerRelevancy", + definition: judgeWorkflow("answerRelevancy", source), + failure: CLOSED, + payload: (run) => { + if (typeof run.input !== "string" || typeof run.output !== "string") { + return closed("answerRelevancy needs a string question and answer"); + } + return { question: run.input, answer: run.output }; + }, + }; +} + +export function correctness( + source: ModelSource, +): Prebuilt<{ input: string; expected: string; answer: string }> { + return { + id: "correctness", + definition: judgeWorkflow("correctness", source), + failure: CLOSED, + payload: (run) => { + if ( + typeof run.input !== "string" || + typeof run.expectedOutput !== "string" || + typeof run.output !== "string" + ) { + return closed("correctness needs string input, expected, and answer"); + } + return { input: run.input, expected: run.expectedOutput, answer: run.output }; + }, + }; +} + +export function trajectoryAccuracy( + source: ModelSource, + expected: readonly string[] | ((run: Run) => readonly string[]), +): Prebuilt<{ expected: readonly string[]; actual: readonly Step[] }> { + return { + id: "trajectoryAccuracy", + definition: judgeWorkflow("trajectoryAccuracy", source), + failure: CLOSED, + payload: (run) => { + const names = typeof expected === "function" ? expected(run) : expected; + if (!Array.isArray(names) || names.some((name) => typeof name !== "string")) { + return closed("trajectoryAccuracy needs a string array of expected step names"); + } + return { expected: names, actual: run.steps }; + }, + }; +} + +function asObject(output: unknown): Record | undefined { + if (typeof output === "string") { + const text = output.trim(); + if (text.length === 0 || text.includes("```")) return undefined; + try { + const value: unknown = JSON.parse(text); + if (typeof value !== "object" || value === null || Array.isArray(value)) return undefined; + return value as Record; + } catch { + return undefined; + } + } + if (typeof output !== "object" || output === null || Array.isArray(output)) return undefined; + if (isOutcome(output)) return undefined; + return output as Record; +} + +function explanationOf(raw: Record, keys: readonly string[]): string | undefined { + for (const key of Object.keys(raw)) { + if (!keys.includes(key)) return undefined; + } + if (!("explanation" in raw)) return ""; + return typeof raw["explanation"] === "string" ? raw["explanation"] : undefined; +} + +function categoryOutcome( + raw: Record, + scale: readonly string[], + passAt: string, +): Outcome | undefined { + const explanation = explanationOf(raw, ["category", "explanation"]); + const category = raw["category"]; + if (explanation === undefined || typeof category !== "string") return undefined; + const index = scale.indexOf(category); + if (index < 0) return undefined; + const outcome: Outcome = { + pass: category === passAt, + value: index / 2, + label: category, + }; + if (explanation.length > 0) outcome.explanation = explanation; + return outcome; +} + +function scoreOutcome(raw: Record): Outcome | undefined { + const explanation = explanationOf(raw, ["score", "explanation"]); + const score = raw["score"]; + if (explanation === undefined || explanation.length === 0) return undefined; + if (typeof score !== "number" || !Number.isFinite(score) || score < 0 || score > 1) { + return undefined; + } + return { pass: score === 1, value: score, explanation }; +} + +/** Parse one judge object. Fences, prose, a second object, or a bad score fail closed. */ +export function parsePrebuilt(id: PrebuiltId, output: unknown): Outcome { + const raw = asObject(output); + if (raw === undefined) return CLOSED; + if (id === "faithfulness") { + return categoryOutcome(raw, FAITH, "supported") ?? CLOSED; + } + if (id === "correctness") { + return categoryOutcome(raw, CORRECT, "correct") ?? CLOSED; + } + if (id === "answerRelevancy" || id === "trajectoryAccuracy") { + return scoreOutcome(raw) ?? CLOSED; + } + return CLOSED; +} diff --git a/src/workflow.ts b/src/workflow.ts index 5d9d915..4a186c9 100644 --- a/src/workflow.ts +++ b/src/workflow.ts @@ -201,3 +201,14 @@ export function judgeModel(options: JudgeModelOptions): Judge { }, }; } + +export { + answerRelevancy, + correctness, + faithfulness, + parsePrebuilt, + trajectoryAccuracy, + type ModelSource, + type Prebuilt, + type PrebuiltId, +} from "./prebuilt.js";