vein: meta/validate-workflow; lab: promote eval scoring steps to eval/*

mcp/src/lab/AGENTS.md

@@ -458,12 +458,13 @@ completed ingestions, and skip re-merging requirements.
   all-pass ⇒ `EvalSet.recursion=false`), and `harvey-judge-criterion` /
   `harvey-dispute-criterion` (per-criterion agent-in-schema-mode subflows —
   they exist because foreach bodies get no onError; a judge crash scores as
-  an honest FAIL via `harvey/aggregate-scores`' zip, never a pass).
+  an honest FAIL via `eval/aggregate-scores`' zip, never a pass).
 - Steps: `harvey/normalize-documents`, `harvey/ingest-state`,
   `harvey/drafter-plan`, `harvey/validate-deliverables`,
-  `harvey/filter-contested`, `harvey/aggregate-scores`,
-  `harvey/merge-disputes`, `harvey/build-eval-chain`,
-  `harvey/criterion-refs` (pure/plumbing), `harvey/generate-docx` +
+  `harvey/filter-contested`, `harvey/merge-disputes` (pure/plumbing;
+  the generic scoring trio `eval/aggregate-scores`, `eval/build-eval-chain`,
+  `eval/criterion-refs` is shared with other rubric harnesses — see
+  `eval/`), `harvey/generate-docx` +
   `harvey/generate-xlsx` (pandoc / openpyxl deliverable generators,
   granted as agent tools so the production prompts' generate calls work),
   `harvey/graph-sub-agent` — the PINNED read-only graph research sub-agent
@@ -654,6 +655,14 @@ Domain-agnostic eval substrate, shared by every experiment. See
   tasks (byte-identical wrong answer ×≥3 = bias — immune to redundancy and
   prompt nudges; distinct wrong answers = variance). Verdict channel only —
   gold never enters. Smoke: `npx tsx src/lab/eval/matrix-smoke.ts`.
+- `eval/aggregate-scores`, `eval/build-eval-chain`, `eval/criterion-refs` —
+  rubric-judged SCORING plumbing (promoted from `harvey/*`, exercised by
+  `harvey/deliver-smoke.ts`): zip a rubric with per-criterion judge verdicts
+  (same order; null/error = honest FAIL; length mismatch refuses) into
+  scores_json; build the idempotent EvalSet→EvalTrigger→EvalTriggerOutput→
+  CriterionResult(+EvalRequirement) batch-triplet payload for
+  `graph/create-batch-triplet` (`workflow` names the scored workflow);
+  recover the persisted CriterionResult ref_ids from the batch results.
 
 **Naming rule:** `eval/*` = generic. The eval *workflows* that wire these with
 a rubric/task/dataset belong to the experiment and are named `<experiment>-…`.

mcp/src/lab/eval/seed.ts

@@ -24,6 +24,11 @@ import { SEED_OPTS } from "../seed-opts.js";
  *     bands (floor/movable/ceiling), the empirical noise floor from
  *     same-version re-runs, and bias-vs-variance tags for never-correct
  *     tasks (plans/evolve-scoreboard-and-task-matrix.md, Phase 1).
+ *   - `eval/aggregate-scores`, `eval/build-eval-chain`, `eval/criterion-refs`
+ *     — rubric-judged scoring plumbing (promoted from harvey/*): zip judge
+ *     verdicts into scores_json, build the EvalSet→EvalTrigger→
+ *     EvalTriggerOutput→CriterionResult batch-triplet payload, recover the
+ *     persisted CriterionResult ref_ids. Used by harvey-score and wfbench.
  */
 
 const SEED_STEPS: Array<{ file: string; type: string }> = [
@@ -32,6 +37,9 @@ const SEED_STEPS: Array<{ file: string; type: string }> = [
   { file: "optimize.ts", type: "eval/optimize" },
   { file: "evolve-loop.ts", type: "eval/evolve-loop" },
   { file: "matrix.ts", type: "eval/matrix" },
+  { file: "aggregate-scores.ts", type: "eval/aggregate-scores" },
+  { file: "build-eval-chain.ts", type: "eval/build-eval-chain" },
+  { file: "criterion-refs.ts", type: "eval/criterion-refs" },
 ];
 
 const HERE = dirname(fileURLToPath(import.meta.url));

mcp/src/lab/harvey/steps/aggregate-scores.ts → mcp/src/lab/eval/steps/aggregate-scores.ts

@@ -2,8 +2,9 @@ import { z, defineStep } from "vein";
 
 /**
  * Fold the per-criterion judge verdicts into the run's scores object (the
- * format_results / scores_json equivalent, shaped like the harvey-labs
- * harness: score = number of passes, all-pass gating).
+ * format_results / scores_json equivalent, shaped like the harvey-labs /
+ * stakwork rubric harnesses: score = number of passes, all-pass gating).
+ * Domain-agnostic: any rubric-judged eval (harvey-score, wfbench) uses it.
  *
  * `results` is the judge foreach's output array, IN THE SAME ORDER as
  * `rubric` (vein's foreach preserves input order), so criterion identity
@@ -13,7 +14,7 @@ import { z, defineStep } from "vein";
  * counts as an honest FAIL with the error recorded, never as a pass.
  */
 export default defineStep({
-  type: "harvey/aggregate-scores",
+  type: "eval/aggregate-scores",
   description:
     "Zip rubric criteria with their judge verdicts (same order) into scores_json: { score, max_score, " +
     "n_passed, n_total, pass_rate, all_pass, judge_model, criteria_results, failed, judgeCost }. " +
@@ -30,7 +31,7 @@ export default defineStep({
       // A length mismatch means the zip is unsafe — misattributed verdicts
       // are worse than a loud failure.
       throw new Error(
-        `harvey/aggregate-scores: ${cfg.results.length} results for ${cfg.rubric.length} criteria — refusing to zip`,
+        `eval/aggregate-scores: ${cfg.results.length} results for ${cfg.rubric.length} criteria — refusing to zip`,
       );
     }
     const criteria_results = cfg.rubric.map((c, i) => {

mcp/src/lab/harvey/steps/build-eval-chain.ts → mcp/src/lab/eval/steps/build-eval-chain.ts

@@ -9,6 +9,8 @@ import { z, defineStep, type StepContext } from "vein";
  *     -HAS_CRITERION_RESULT-> CriterionResult (one per judged criterion)
  *   EvalRequirement -HAS_TRIGGER-> EvalTrigger (one per judged criterion)
  *
+ * Domain-agnostic (harvey-score, wfbench): the domain passes its EvalSet id
+ * and the workflow name recorded as the trigger's agent.
  * Pure payload construction — the write itself is graph/create-batch-triplet
  * (its inline-node dedupe collapses the repeated EvalTriggerOutput side).
  * Ids are derived from ctx.runId, so a retried write MERGES instead of
@@ -17,21 +19,21 @@ import { z, defineStep, type StepContext } from "vein";
  * criteria_results are passed.
  */
 export default defineStep({
-  type: "harvey/build-eval-chain",
+  type: "eval/build-eval-chain",
   description:
     "Construct the create-batch-triplet payload persisting a scored attempt: EvalSet→EvalTrigger→" +
     "EvalTriggerOutput→CriterionResult(+EvalRequirement links), ids derived from the runId (idempotent " +
     "rewrites). Output: { triplets, trigger_id, output_id }.",
   input: z.object({
     evalsetId: z.string().describe("EvalSet id (the task namespace slug)."),
-    task: z.string().describe("The harvey task id (recorded as the trigger's workflow_input)."),
-    scores: z.any().describe("harvey/aggregate-scores output (criteria_results may carry dispute annotations)."),
+    task: z.string().describe("The task id (recorded as the trigger's workflow_input)."),
+    scores: z.any().describe("eval/aggregate-scores output (criteria_results may carry dispute annotations)."),
     criteria_results: z
       .array(z.any())
       .optional()
-      .describe("Override for scores.criteria_results — pass harvey/merge-disputes' ANNOTATED list."),
+      .describe("Override for scores.criteria_results — e.g. harvey/merge-disputes' ANNOTATED list."),
     judge_model: z.string().optional(),
-    workflow: z.string().default("harvey-deliver").describe("Recorded as EvalTrigger.workflow_id/agent."),
+    workflow: z.string().describe("The scored workflow's name — recorded as EvalTrigger.workflow_id/agent."),
   }),
   output: z.any(),
   async run(cfg, ctx) {
@@ -64,7 +66,7 @@ export default defineStep({
     };
 
     // Position of each criterion's HAS_CRITERION_RESULT triplet in the array
-    // below — harvey/criterion-refs zips these with the batch write's results
+    // below — eval/criterion-refs zips these with the batch write's results
     // to recover the created CriterionResult ref_ids.
     const criterionSlots: Array<{ criterion_id: string; index: number }> = [];
     const triplets: Array<Record<string, any>> = [

mcp/src/lab/harvey/steps/criterion-refs.ts → mcp/src/lab/eval/steps/criterion-refs.ts

@@ -1,7 +1,7 @@
 import { z, defineStep } from "vein";
 
 /**
- * Zip harvey/build-eval-chain's criterion slots with graph/create-batch-
+ * Zip eval/build-eval-chain's criterion slots with graph/create-batch-
  * triplet's per-triplet results to recover each persisted CriterionResult
  * node's ref_id: slots name the triplet-array INDEX of each criterion's
  * HAS_CRITERION_RESULT write, and the batch step returns results in input
@@ -14,13 +14,13 @@ import { z, defineStep } from "vein";
  * then simply run without graph anchoring, never blocking scoring.
  */
 export default defineStep({
-  type: "harvey/criterion-refs",
+  type: "eval/criterion-refs",
   description:
     "Recover persisted CriterionResult ref_ids: zip build-eval-chain's criterionSlots with " +
     "create-batch-triplet's results. Fail-soft ([] on any shape surprise). Output: " +
     "[{ criterion_id, ref_id }].",
   input: z.object({
-    slots: z.any().optional().describe("harvey/build-eval-chain's criterionSlots: [{ criterion_id, index }]."),
+    slots: z.any().optional().describe("eval/build-eval-chain's criterionSlots: [{ criterion_id, index }]."),
     record: z.any().optional().describe("graph/create-batch-triplet's output ({ results: [...] })."),
   }),
   output: z.any(),

mcp/src/lab/harvey/deliver-smoke.ts

@@ -16,15 +16,17 @@ import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { WorkspaceManager, buildRegistry, type StepContext } from "vein";
 import { seedHarveySteps, seedHarveyWorkflows } from "./seed.js";
+import { seedEvalSteps } from "../eval/seed.js";
 
 async function main() {
   const dir = mkdtempSync(join(process.cwd(), ".harvey-deliver-smoke-"));
   try {
     // ── 1. seed + discover ───────────────────────────────────────────────
     const workspace = new WorkspaceManager(dir);
+    await seedEvalSteps(workspace);
     await seedHarveySteps(workspace);
     await seedHarveyWorkflows(workspace);
-    const { registry } = await buildRegistry(workspace.path);
+    const { registry } = await buildRegistry(await workspace.materializeCustomSteps());
 
     const expectedSteps = [
       "harvey/normalize-documents",
@@ -33,10 +35,10 @@ async function main() {
       "harvey/drafter-plan",
       "harvey/validate-deliverables",
       "harvey/filter-contested",
-      "harvey/aggregate-scores",
+      "eval/aggregate-scores",
       "harvey/merge-disputes",
-      "harvey/build-eval-chain",
-      "harvey/criterion-refs",
+      "eval/build-eval-chain",
+      "eval/criterion-refs",
       "harvey/generate-docx",
       "harvey/generate-xlsx",
       // graph/* are vein LIB steps — discovered from the engine, not seeded.
@@ -176,7 +178,7 @@ async function main() {
     console.log("✔ filter-contested (drop + fail-open)");
 
     // ── 7. aggregate-scores (zip; null = honest fail) ────────────────────
-    out = await run("harvey/aggregate-scores", {
+    out = await run("eval/aggregate-scores", {
       rubric,
       results: [
         { object: { verdict: "pass", reasoning: "ok" }, cost: 0.1 },
@@ -196,7 +198,7 @@ async function main() {
     assert.equal(out.failed[0].match_criteria, "...");
     assert.equal(out.judgeCost, 0.1);
     await assert.rejects(
-      () => run("harvey/aggregate-scores", { rubric, results: [null] }),
+      () => run("eval/aggregate-scores", { rubric, results: [null] }),
       /refusing to zip/,
     );
     console.log("✔ aggregate-scores (zip / honest fail / length guard)");
@@ -245,7 +247,7 @@ async function main() {
     console.log("✔ merge-disputes (left-join + refs + fail-soft + contested)");
 
     // ── 8b. criterion-refs (slot × batch-result zip, fail-soft) ──────────
-    let refs: any = await run("harvey/criterion-refs", {
+    let refs: any = await run("eval/criterion-refs", {
       slots: [
         { criterion_id: "c1", index: 2 },
         { criterion_id: "c2", index: 4 },
@@ -261,17 +263,18 @@ async function main() {
       },
     });
     assert.deepEqual(refs, [{ criterion_id: "c1", ref_id: "cr1" }]); // errored slot dropped
-    refs = await run("harvey/criterion-refs", { slots: "garbage", record: { error: "record failed" } });
+    refs = await run("eval/criterion-refs", { slots: "garbage", record: { error: "record failed" } });
     assert.deepEqual(refs, []);
     console.log("✔ criterion-refs (zip + fail-soft)");
 
     // ── 9. build-eval-chain (ontology shape) ─────────────────────────────
     const ctx = { runId: "run42" } as StepContext;
     const chain: any = await run(
-      "harvey/build-eval-chain",
+      "eval/build-eval-chain",
       {
         evalsetId: "slug",
         task: "a/b",
+        workflow: "harvey-deliver",
         scores: out,
         criteria_results: [
           { id: "c1", criterion_id: "c1", title: "A", verdict: "pass", reasoning: "ok" },

mcp/src/lab/harvey/seed.ts

@@ -31,17 +31,16 @@ const SEED_STEPS: Array<{ file: string; type: string }> = [
   { file: "digest-results.ts", type: "harvey/digest-results" },
   // harvey-deliver pipeline steps (standalone production-style pipeline —
   // rubric as input; NOT part of the benchmark harness): intake, drafting
-  // plan, scoring plumbing, and the pinned read-only graph sub-agent.
+  // plan, scoring plumbing, and the pinned read-only graph sub-agent. The
+  // generic scoring steps (aggregate-scores, build-eval-chain,
+  // criterion-refs) live in eval/* now — seeded by eval/seed.ts.
   { file: "normalize-documents.ts", type: "harvey/normalize-documents" },
   { file: "graph-sub-agent.ts", type: "harvey/graph-sub-agent" },
   { file: "ingest-state.ts", type: "harvey/ingest-state" },
   { file: "drafter-plan.ts", type: "harvey/drafter-plan" },
   { file: "validate-deliverables.ts", type: "harvey/validate-deliverables" },
   { file: "filter-contested.ts", type: "harvey/filter-contested" },
-  { file: "aggregate-scores.ts", type: "harvey/aggregate-scores" },
   { file: "merge-disputes.ts", type: "harvey/merge-disputes" },
-  { file: "build-eval-chain.ts", type: "harvey/build-eval-chain" },
-  { file: "criterion-refs.ts", type: "harvey/criterion-refs" },
   // deliverable generation (pandoc / openpyxl) — grantable agent tools so the
   // production prompts' harvey_generate_docx/_xlsx calls work verbatim.
   { file: "generate-docx.ts", type: "harvey/generate-docx" },

mcp/src/lab/harvey/steps/merge-disputes.ts

@@ -33,7 +33,7 @@ export default defineStep({
     "annotates nothing. Output: { criteria_results, annotations, flagged_count, contested_count, " +
     "contested_requirements }.",
   input: z.object({
-    criteria_results: z.array(z.any()).describe("Full criteria_results from harvey/aggregate-scores."),
+    criteria_results: z.array(z.any()).describe("Full criteria_results from eval/aggregate-scores."),
     failed: z.array(z.any()).default([]).describe("The failed subset, in the order disputes ran."),
     disputes: z
       .any()
@@ -42,7 +42,7 @@ export default defineStep({
     criterionRefs: z
       .any()
       .optional()
-      .describe("[{ criterion_id, ref_id }] for the persisted CriterionResult nodes (harvey/criterion-refs output)."),
+      .describe("[{ criterion_id, ref_id }] for the persisted CriterionResult nodes (eval/criterion-refs output)."),
     requirements: z
       .any()
       .optional()

mcp/src/lab/harvey/workflows/harvey-judge-criterion.yaml

@@ -5,7 +5,7 @@ name: harvey-judge-criterion
 # verdict. Run per criterion by harvey-score's foreach (this subflow exists
 # because foreach bodies get no onError of their own — the fall-soft lives on
 # the agent step here: a judge crash returns { error } and is scored as an
-# honest FAIL by harvey/aggregate-scores, never a pass).
+# honest FAIL by eval/aggregate-scores, never a pass).
 #
 # The judge is the core agent step in SCHEMA mode with bash-only tools: it
 # reads the deliverables itself (pandoc for .docx — mirrors the harness's

mcp/src/lab/harvey/workflows/harvey-score.yaml

@@ -86,7 +86,7 @@ steps:
             taskDescription: "{{ input.task_description }}"
 
   - id: scores
-    type: harvey/aggregate-scores
+    type: eval/aggregate-scores
     depends: judged
     config:
       rubric: "{{ scorable.rubric }}"
@@ -95,7 +95,7 @@ steps:
       dropped: "{{ scorable.dropped }}"
 
   - id: chain
-    type: harvey/build-eval-chain
+    type: eval/build-eval-chain
     depends: scores
     config:
       evalsetId: "{{ input.evalset_id }}"
@@ -121,7 +121,7 @@ steps:
   # record write failed) — the dispute agents anchor Cause triplets to them
   # and the annotation write-back targets them.
   - id: critrefs
-    type: harvey/criterion-refs
+    type: eval/criterion-refs
     depends: record
     config:
       slots: "{{ chain.criterionSlots }}"