feat: add string includes and prebuilt judge scorers

README.md

@@ -47,18 +47,19 @@ returned.
   `unsure (p=…)` in between. `{ question, yes, no }` sets its own bars.
 - **Helpers**:
 
-| helper                        | passes when                                                                    |
-| ----------------------------- | ------------------------------------------------------------------------------ |
-| `matches(field?)`             | output deep-equals expectedOutput, or the same field (key or accessor) of each |
-| `fitsSchema(schema)`          | output satisfies a Standard Schema (arktype, zod, valibot)                     |
-| `mentions(words)`             | output contains every word, case-insensitive; value is the fraction found      |
-| `avoids(words)`               | output contains none of the words                                              |
-| `ranSteps(names)`             | `tool` and `step` names match `names`, order ignored; value is their F1        |
-| `stepsAtMost(n)`              | the run took at most `n` steps                                                 |
-| `under({ ms })`               | the run finished within `ms`                                                   |
-| `ask(q, { options, pass })`   | Jev picks an option named in `pass`                                            |
-| `ask(q, { options, passAt })` | Jev picks an option; its credit (0..1) is the value and passes at `passAt`     |
-| `ask(q, { levels, pass })`    | Jev grades at level `pass` or above (levels lowest first)                      |
+| helper                        | passes when                                                                                                 |
+| ----------------------------- | ----------------------------------------------------------------------------------------------------------- |
+| `matches(field?)`             | output deep-equals expectedOutput, or the same field (key or accessor) of each                              |
+| `includes()`                  | both strings, and output contains expectedOutput after NFKC, trim, and whitespace collapse; value is 1 or 0 |
+| `fitsSchema(schema)`          | output satisfies a Standard Schema (arktype, zod, valibot)                                                  |
+| `mentions(words)`             | output contains every word, case-insensitive; value is the fraction found                                   |
+| `avoids(words)`               | output contains none of the words                                                                           |
+| `ranSteps(names)`             | `tool` and `step` names match `names`, order ignored; value is their F1                                     |
+| `stepsAtMost(n)`              | the run took at most `n` steps                                                                              |
+| `under({ ms })`               | the run finished within `ms`                                                                                |
+| `ask(q, { options, pass })`   | Jev picks an option named in `pass`                                                                         |
+| `ask(q, { options, passAt })` | Jev picks an option; its credit (0..1) is the value and passes at `passAt`                                  |
+| `ask(q, { levels, pass })`    | Jev grades at level `pass` or above (levels lowest first)                                                   |
 
 `mentions` and `avoids` also take `(r) => words` for per-case words.
 `mentions` and `ranSteps` take `{ atLeast }` to pass on partial credit.

e2e/prebuilt.e2e.test.ts

@@ -0,0 +1,154 @@
+import { describe, expect, test } from "bun:test";
+
+import { includes, matches, runEvaluation } from "../src/index.js";
+import {
+  answerRelevancy,
+  correctness,
+  faithfulness,
+  parsePrebuilt,
+  trajectoryAccuracy,
+  type PrebuiltId,
+} from "../src/workflow.js";
+
+const source = { provider: "ollama", model: "pinned" };
+
+function judgeOf(
+  id: PrebuiltId,
+  definition: { steps: Record<string, unknown>; stepOrder: readonly string[] },
+) {
+  expect(definition.stepOrder).toEqual(["judge"]);
+  const judge = definition.steps["judge"];
+  expect(judge).toMatchObject({
+    kind: "step",
+    input: { from: "trigger.payload" },
+    agent: {
+      id: `${id}-judge`,
+      capabilities: [],
+      inference: { sources: [source] },
+    },
+  });
+  const agent = (judge as { agent: { toolFactories: unknown[]; systemPrompt: string } }).agent;
+  expect(agent.toolFactories).toEqual([]);
+  expect(agent.systemPrompt.length).toBeGreaterThan(0);
+}
+
+describe("includes", () => {
+  test("matches and includes share one string expectedOutput", async () => {
+    const report = await runEvaluation({
+      name: "includes-mix",
+      task: (input: string) => `Answer: ${input}`,
+      data: [{ input: "Refund", expectedOutput: "  reFUND  " }],
+      scorers: { exact: matches(), has: includes() },
+      allow: { exact: 1 },
+    });
+    expect(report.ok).toBe(true);
+    expect(report.cases[0]?.checks["exact"]?.pass).toBe(false);
+    expect(report.cases[0]?.checks["has"]).toEqual({ pass: true, value: 1 });
+  });
+
+  test("fails closed on a non-string, a missing expected, or an empty needle", () => {
+    const check = includes();
+    const run = { input: "", steps: [], ms: 0 };
+    expect(check({ ...run, output: { a: 1 }, expectedOutput: '{"a":1}' })).toMatchObject({
+      pass: false,
+      value: 0,
+    });
+    expect(check({ ...run, output: "hello" })).toMatchObject({ pass: false, value: 0 });
+    expect(check({ ...run, output: "hello", expectedOutput: " \n\t " })).toMatchObject({
+      pass: false,
+      value: 0,
+    });
+    expect(check({ ...run, output: "see the file  now", expectedOutput: "file now" })).toEqual({
+      pass: true,
+      value: 1,
+    });
+  });
+});
+
+describe("prebuilt judges", () => {
+  test("each factory is one judge step pinned to the source", () => {
+    const run = {
+      input: "q",
+      output: "a",
+      expectedOutput: "e",
+      steps: [{ name: "lookup", kind: "tool" as const }],
+      ms: 1,
+    };
+    const checks = [
+      faithfulness(source),
+      answerRelevancy(source),
+      correctness(source),
+      trajectoryAccuracy(source, ["lookup"]),
+    ];
+    for (const check of checks) judgeOf(check.id, check.definition);
+    expect(faithfulness(source).payload(run)).toEqual({ context: "e", answer: "a" });
+    expect(faithfulness(source, { context: () => "ctx" }).payload(run)).toEqual({
+      context: "ctx",
+      answer: "a",
+    });
+    expect(answerRelevancy(source).payload({ ...run, expectedOutput: "ignored" })).toEqual({
+      question: "q",
+      answer: "a",
+    });
+    expect(correctness(source).payload(run)).toEqual({ input: "q", expected: "e", answer: "a" });
+    expect(trajectoryAccuracy(source, (r) => r.steps.map((s) => s.name)).payload(run)).toEqual({
+      expected: ["lookup"],
+      actual: run.steps,
+    });
+    expect(faithfulness(source).payload({ ...run, output: 1 })).toMatchObject({
+      pass: false,
+      value: 0,
+    });
+    expect(() => faithfulness({ provider: "", model: "m" })).toThrow("non-empty");
+  });
+
+  test("parsePrebuilt accepts one object and rejects fences and a score past 1", () => {
+    expect(parsePrebuilt("faithfulness", { category: "supported" })).toEqual({
+      pass: true,
+      value: 1,
+      label: "supported",
+    });
+    expect(parsePrebuilt("faithfulness", '{"category":"unverifiable"}')).toMatchObject({
+      pass: false,
+      value: 0.5,
+      label: "unverifiable",
+    });
+    expect(
+      parsePrebuilt("correctness", { category: "incorrect", explanation: "no" }),
+    ).toMatchObject({
+      pass: false,
+      value: 0,
+      label: "incorrect",
+    });
+    expect(parsePrebuilt("answerRelevancy", { score: 1, explanation: "on topic" })).toEqual({
+      pass: true,
+      value: 1,
+      explanation: "on topic",
+    });
+    expect(
+      parsePrebuilt("trajectoryAccuracy", { score: 0.5, explanation: "missed a step" }),
+    ).toEqual({
+      pass: false,
+      value: 0.5,
+      explanation: "missed a step",
+    });
+    const fenced = '```json\n{"score":1,"explanation":"ok"}\n```';
+    expect(parsePrebuilt("answerRelevancy", fenced)).toEqual({
+      pass: false,
+      value: 0,
+      explanation: "judge output failed closed",
+    });
+    expect(parsePrebuilt("answerRelevancy", { score: 1.5, explanation: "too high" })).toEqual({
+      pass: false,
+      value: 0,
+      explanation: "judge output failed closed",
+    });
+    expect(
+      parsePrebuilt("correctness", '{"category":"correct"}{"category":"incorrect"}'),
+    ).toMatchObject({ pass: false, value: 0 });
+    expect(parsePrebuilt("faithfulness", "the answer is supported")).toMatchObject({
+      pass: false,
+      value: 0,
+    });
+  });
+});

src/checks.ts

@@ -27,6 +27,32 @@ export function toOutcome(name: string, raw: unknown): Outcome {
   };
 }
 
+/** Case-insensitive substring. Value is 1 or 0. Strings only; no coercion. */
+export function includes(): CheckFn<unknown, unknown, unknown> {
+  return ({ output, expectedOutput }) => {
+    if (typeof output !== "string" || typeof expectedOutput !== "string") {
+      return {
+        pass: false,
+        value: 0,
+        explanation: "includes needs string output and expectedOutput",
+      };
+    }
+    const needle = collapsed(expectedOutput);
+    if (needle.length === 0) {
+      return { pass: false, value: 0, explanation: "includes needle is empty" };
+    }
+    const pass = collapsed(output).includes(needle);
+    return pass
+      ? { pass: true, value: 1 }
+      : { pass: false, value: 0, explanation: "output does not include the expected text" };
+  };
+}
+
+/** NFKC, trim, collapse internal whitespace, then lowercase. */
+function collapsed(value: string): string {
+  return value.normalize("NFKC").trim().replace(/\s+/g, " ").toLowerCase();
+}
+
 /** Output deep-equals expectedOutput, or one field of each (a key or an accessor). */
 export function matches<T = unknown>(
   field?: string | ((value: T) => unknown),

src/index.ts

@@ -13,6 +13,7 @@ export type {
 export {
   avoids,
   fitsSchema,
+  includes,
   matches,
   mentions,
   ranSteps,