From 4834771f0aef115cce79ad08ca3da9015ef12ae0 Mon Sep 17 00:00:00 2001 From: Evan Feenstra Date: Fri, 4 Sep 2026 12:51:41 -0700 Subject: [PATCH] =?UTF-8?q?lab:=20wfbench=20=E2=80=94=20Workflow=20Editor?= =?UTF-8?q?=20Agent=20Benchmark=20harness=20(stakwork=2058313=20twin)=20(#?= =?UTF-8?q?1650)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * lab: wfbench — Workflow Editor Agent Benchmark harness (stakwork 58313's twin) wfbench-run: graph roster (EvalSet / EvalRequirement / EvalTrigger with 58313's ids, HAS_BASELINE_TRIGGER on the first trigger) in parallel with the meta/* author agent -> resolve the shipped version -> input-key gate -> rerun via meta/run-workflow -> per-criterion judge subflow -> eval/aggregate-scores -> record EvalTriggerOutput + CriterionResult per 58312 -> one Hive callback body (POSTed and returned). Ten pure steps, two workflows, seeded unstamped; wfbench/* is never an agent tool. Offline smoke (src/lab/wfbench/smoke.ts): seeds, discovers, static- validates both workflows through the authoring capability, drives every pure step, and checks every graph payload's attributes and edges against JARVIS_ONTOLOGY. * vein: core `pack` step; lab: drop the three per-experiment pack-result copies A workflow's output is its last step's output, so returning fields from several steps needs a passthrough step — harvey, gaia and wfbench each carried an identical one, and the wfbench author reached for gaia/pack-result because it was the first it found. `pack` is that primitive in core (config = output), discoverable everywhere. Every harvey/gaia/wfbench workflow, seed, smoke and doc is re-pointed; the copies are deleted. (cherry picked from commit dbc98d53e9191c1ba82df96c7562a30f02424b29) --- mcp/src/lab/AGENTS.md | 51 +- mcp/src/lab/createLabVein.ts | 6 + mcp/src/lab/eval/seed.ts | 4 +- mcp/src/lab/gaia/evolve-smoke.ts | 2 +- mcp/src/lab/gaia/seed.ts | 4 +- mcp/src/lab/gaia/steps/pack-result.ts | 18 - .../gaia/workflows/gaia-candidate-run.yaml | 4 +- .../lab/gaia/workflows/gaia-evolve-gen.yaml | 4 +- mcp/src/lab/gaia/workflows/gaia-evolve.yaml | 2 +- mcp/src/lab/gaia/workflows/gaia-produce.yaml | 4 +- mcp/src/lab/gaia/workflows/gaia-run.yaml | 2 +- mcp/src/lab/harvey/evolve-smoke.ts | 2 +- mcp/src/lab/harvey/seed.ts | 4 +- mcp/src/lab/harvey/steps/pack-result.ts | 19 - .../workflows/harvey-candidate-run.yaml | 4 +- .../lab/harvey/workflows/harvey-deliver.yaml | 2 +- .../workflows/harvey-dispute-criterion.yaml | 2 +- .../lab/harvey/workflows/harvey-draft.yaml | 10 +- .../harvey/workflows/harvey-evolve-gen.yaml | 4 +- .../lab/harvey/workflows/harvey-evolve.yaml | 2 +- .../harvey/workflows/harvey-ingest-doc.yaml | 4 +- .../workflows/harvey-judge-criterion.yaml | 2 +- .../harvey/workflows/harvey-knowledge.yaml | 6 +- .../lab/harvey/workflows/harvey-produce.yaml | 4 +- mcp/src/lab/harvey/workflows/harvey-run.yaml | 4 +- .../lab/harvey/workflows/harvey-score.yaml | 10 +- mcp/src/lab/plans/wfbench-harness.md | 66 ++- mcp/src/lab/wfbench/seed.ts | 82 +++ mcp/src/lab/wfbench/smoke.ts | 301 +++++++++++ .../lab/wfbench/steps/build-eval-output.ts | 89 +++ mcp/src/lab/wfbench/steps/build-materials.ts | 93 ++++ mcp/src/lab/wfbench/steps/build-roster.ts | 78 +++ mcp/src/lab/wfbench/steps/check-input-keys.ts | 67 +++ mcp/src/lab/wfbench/steps/classify-run.ts | 47 ++ mcp/src/lab/wfbench/steps/normalize-task.ts | 88 +++ .../lab/wfbench/steps/resolve-candidate.ts | 54 ++ mcp/src/lab/wfbench/steps/trigger-edge.ts | 35 ++ mcp/src/lab/wfbench/steps/webhook-body.ts | 65 +++ .../workflows/wfbench-judge-criterion.yaml | 79 +++ .../lab/wfbench/workflows/wfbench-run.yaml | 510 ++++++++++++++++++ vein/AGENTS.md | 2 +- vein/SPEC.md | 17 + vein/package.json | 2 +- vein/src/steps/core/pack.test.ts | 35 ++ vein/src/steps/core/pack.ts | 28 + vein/src/steps/registry.ts | 2 + 46 files changed, 1814 insertions(+), 106 deletions(-) delete mode 100644 mcp/src/lab/gaia/steps/pack-result.ts delete mode 100644 mcp/src/lab/harvey/steps/pack-result.ts create mode 100644 mcp/src/lab/wfbench/seed.ts create mode 100644 mcp/src/lab/wfbench/smoke.ts create mode 100644 mcp/src/lab/wfbench/steps/build-eval-output.ts create mode 100644 mcp/src/lab/wfbench/steps/build-materials.ts create mode 100644 mcp/src/lab/wfbench/steps/build-roster.ts create mode 100644 mcp/src/lab/wfbench/steps/check-input-keys.ts create mode 100644 mcp/src/lab/wfbench/steps/classify-run.ts create mode 100644 mcp/src/lab/wfbench/steps/normalize-task.ts create mode 100644 mcp/src/lab/wfbench/steps/resolve-candidate.ts create mode 100644 mcp/src/lab/wfbench/steps/trigger-edge.ts create mode 100644 mcp/src/lab/wfbench/steps/webhook-body.ts create mode 100644 mcp/src/lab/wfbench/workflows/wfbench-judge-criterion.yaml create mode 100644 mcp/src/lab/wfbench/workflows/wfbench-run.yaml create mode 100644 vein/src/steps/core/pack.test.ts create mode 100644 vein/src/steps/core/pack.ts diff --git a/mcp/src/lab/AGENTS.md b/mcp/src/lab/AGENTS.md index dbc01053a..4556e86e8 100644 --- a/mcp/src/lab/AGENTS.md +++ b/mcp/src/lab/AGENTS.md @@ -519,7 +519,7 @@ are thin plumbing over `ctx.services.gaia.*`. - **Committed harness** (`gaia/seed.ts`, seeded at boot like harvey's): steps `gaia/list-tasks`, `gaia/get-task` (stages a task's attached file into the run's artifacts dir), `gaia/evaluate` (HARNESS-ONLY), and the - combiners `gaia/pack-result` + `gaia/summarize-batch`; workflows + combiner `gaia/summarize-batch` (+ vein's core `pack`); workflows `gaia-produce` (agent step; the produce system prompt, model, maxSteps: 50 and agentTools live in `params`; an `onError` fallback scores a blown-up agent as an empty wrong answer instead of killing the batch), `gaia-run` @@ -554,7 +554,7 @@ are thin plumbing over `ctx.services.gaia.*`. unmeasured (`baselineSamples: 1`, the pre-Phase-1 behavior). Candidate contract: input `{ taskId }`, last step outputs `taskId`, `answer` (bare string), `cost`, `steps`; candidates may use - `gaia/get-task` / `gaia/pack-result` as steps but NEVER `gaia/evaluate` + `gaia/get-task` / `pack` as steps but NEVER `gaia/evaluate` (produce-time oracle) and never gaia/*, eval/*, meta/* as agentTools. Scores are TRAIN scores — validate the best version on a held-out `gaia-batch` slice before promoting. Offline checks: @@ -602,6 +602,53 @@ are thin plumbing over `ctx.services.gaia.*`. scorer-hash mismatch / LFS pointer stubs / idempotent re-entry / half-written checkout) with `exec` and `fetchText` faked (`npx tsx src/lab/gaia/smoke.ts`). +### `wfbench/` — Workflow Editor Agent Benchmark (stakwork 58313's twin) + +The benchmark harness for the workflow-AUTHORING agent, rebuilt as a vein +workflow (design + step map: `plans/wfbench-harness.md`). One task in +(`{ task_slug, task_title?, instructions, criteria, workflow_input_json?, +rerun_expected_output?, webhook_url? }` — Hive's payload shape), one callback +out. `wfbench-run`: graph roster (EvalSet → EvalRequirement×N, EvalTrigger, +HAS_BASELINE_TRIGGER on the EvalSet's first trigger else HAS_TRIGGER — 58313's +ids, written with vein's `graph/*` steps under `params.namespace`) ‖ the +author (core `agent` + `agentTools: ["meta/*"]`, editor tool only — 54419's +twin) builds `wfbench-` → resolve the version it actually shipped +(`vpin || vactive`, `published` vs `vbefore` — the gaia-evolve-gen guard) → +input-key gate (`wfbench/check-input-keys`: the `input.` references in +the produced YAML vs the launch payload; a mismatch or empty body is a harness +error, never launched) → rerun via `meta/run-workflow` (57425's twin; own runId += 58313's project_id) → `wfbench/classify-run` (a runtime-FAILED rerun is still +judged; no runId is a harness error) → `wfbench/build-materials` (workflow YAML ++ custom step sources + launch payload + run output + expected output, inlined +as one markdown block) → per-criterion judge (`wfbench-judge-criterion`: agent +schema mode, nothing to read — a crash packs `{ error }` = honest FAIL) → +`eval/aggregate-scores` → record `EvalTrigger -HAS_OUTPUT-> EvalTriggerOutput +-HAS_CRITERION_RESULT-> CriterionResult <-HAS_CRITERION_RESULT- EvalRequirement` +(`wfbench/build-eval-output`, 58312's ids: `-`, +`--`) → `wfbench/webhook-body` (58313's 4-way +`resolve_webhook_payload`: success `{ task_slug, task_title, n_passed, n_total, +all_pass, pass_rate, judge_model, criteria_results }` or `{ harness_error: +true, error_type, error }` — no fake 0/N) → POST `webhook_url` → the run's +output IS that body (+ diagnostics), fixing 58313's set_output divergence. + +- Graph writes use only ontology-declared attributes (vein's backend rejects + the rest): 58313's `EvalSet.project_id` (an int there) and the `name` on + EvalTrigger / EvalTriggerOutput are omitted; 58312's `CriterionResult + -HAS_CAUSE-> Workflow_version` is not written (no such relationship in + the ontology, and a vein workflow is not a Workflow_version node). +- Grant discipline: `wfbench/*` is never an agentTool. The author gets + `meta/*` (+ `meta/validate-workflow`) and the editor; the judge gets + nothing useful; the produced workflow is necessarily publisher `ai`. +- The author's `params.authorSystem` carries a stakwork→vein translation + table, so PORTING a stakwork workflow is just a task whose instructions + are the stakwork body (plus a real project's input/output as + `workflow_input_json` / `rerun_expected_output`). +- v1 is create-new only (58313 v1 too). Needs `ANTHROPIC_API_KEY` + the + graph; no stakwork credentials. +- Smoke (offline — seeds, discovers, static-validates both workflows + through the authoring capability, drives every pure step, and checks the + graph payloads against `JARVIS_ONTOLOGY`): `npx tsx src/lab/wfbench/smoke.ts`. + ### `eval/` — generic, reusable eval primitives (NOT an experiment) Domain-agnostic eval substrate, shared by every experiment. See diff --git a/mcp/src/lab/createLabVein.ts b/mcp/src/lab/createLabVein.ts index b23a2dbee..abad91ea4 100644 --- a/mcp/src/lab/createLabVein.ts +++ b/mcp/src/lab/createLabVein.ts @@ -23,6 +23,7 @@ import { seedSheetsSteps } from "./sheets/seed.js"; import { seedHarveySteps, seedHarveyWorkflows } from "./harvey/seed.js"; import { seedGaiaSteps, seedGaiaWorkflows } from "./gaia/seed.js"; import { seedArtifactSteps } from "./artifacts/seed.js"; +import { seedWfbenchSteps, seedWfbenchWorkflows } from "./wfbench/seed.js"; import { buildHarveyServices, type HarveyServices } from "./harvey/service.js"; import { buildGaiaServices, type GaiaServices } from "./gaia/service.js"; import { buildGitseeServices, type GitseeServices } from "./gitsee/services/index.js"; @@ -184,6 +185,11 @@ export async function createLabVein( await seedGaiaWorkflows(workspace); // generic artifact plumbing (artifacts/dir — bridge runId → path for cwd). await seedArtifactSteps(workspace); + // wfbench — the Workflow Editor Agent Benchmark harness (stakwork 58313's + // twin; plans/wfbench-harness.md). Pure plumbing steps; never granted to + // an agent (the author gets meta/* only). + await seedWfbenchSteps(workspace); + await seedWfbenchWorkflows(workspace); const vein = await createVein({ workspace, diff --git a/mcp/src/lab/eval/seed.ts b/mcp/src/lab/eval/seed.ts index 59488c511..fb23f8b86 100644 --- a/mcp/src/lab/eval/seed.ts +++ b/mcp/src/lab/eval/seed.ts @@ -28,7 +28,9 @@ import { SEED_OPTS } from "../seed-opts.js"; * — rubric-judged scoring plumbing (promoted from harvey/*): zip judge * verdicts into scores_json, build the EvalSet→EvalTrigger→ * EvalTriggerOutput→CriterionResult batch-triplet payload, recover the - * persisted CriterionResult ref_ids. Used by harvey-score and wfbench. + * persisted CriterionResult ref_ids. harvey-score uses all three; + * wfbench uses aggregate-scores + criterion-refs (its record payload, + * wfbench/build-eval-output, follows stakwork 58312's id conventions). */ const SEED_STEPS: Array<{ file: string; type: string }> = [ diff --git a/mcp/src/lab/gaia/evolve-smoke.ts b/mcp/src/lab/gaia/evolve-smoke.ts index 6e4a95754..8fae3a752 100644 --- a/mcp/src/lab/gaia/evolve-smoke.ts +++ b/mcp/src/lab/gaia/evolve-smoke.ts @@ -37,7 +37,7 @@ async function main() { // step types referenced by the workflows all exist in the registry const { registry } = await buildRegistry(await workspace.materializeCustomSteps()); const wanted = [ - "gaia/list-tasks", "gaia/get-task", "gaia/evaluate", "gaia/pack-result", + "gaia/list-tasks", "gaia/get-task", "gaia/evaluate", "pack", "gaia/summarize-batch", "gaia/digest-results", "eval/evolve-loop", "eval/matrix", "artifacts/dir", "meta/run-workflow", "agent", "subflow", "foreach", diff --git a/mcp/src/lab/gaia/seed.ts b/mcp/src/lab/gaia/seed.ts index 4da6a13b8..de7641291 100644 --- a/mcp/src/lab/gaia/seed.ts +++ b/mcp/src/lab/gaia/seed.ts @@ -18,7 +18,8 @@ import { SEED_OPTS } from "../seed-opts.js"; * artifacts dir so agent steps (cwd = artifacts dir) can read it. * - `gaia/evaluate` — the real leaderboard scorer. HARNESS-ONLY: grant only * to harness workflows, never to a producing agent's `agentTools`. - * - `gaia/pack-result`, `gaia/summarize-batch` — pure combiners. + * - `gaia/summarize-batch` — pure combiner (the echo combiner is vein's core + * `pack` step now). * - `gaia/digest-results` — aggregate graded results into the evolve loop's * propose digest (verdict channel only; accuracy as `fitness`). * @@ -35,7 +36,6 @@ const SEED_STEPS: Array<{ file: string; type: string }> = [ { file: "list-tasks.ts", type: "gaia/list-tasks" }, { file: "get-task.ts", type: "gaia/get-task" }, { file: "evaluate.ts", type: "gaia/evaluate" }, - { file: "pack-result.ts", type: "gaia/pack-result" }, { file: "summarize-batch.ts", type: "gaia/summarize-batch" }, { file: "digest-results.ts", type: "gaia/digest-results" }, ]; diff --git a/mcp/src/lab/gaia/steps/pack-result.ts b/mcp/src/lab/gaia/steps/pack-result.ts deleted file mode 100644 index b2ddc26d2..000000000 --- a/mcp/src/lab/gaia/steps/pack-result.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { z, defineStep } from "vein"; - -/** - * Trivial combiner: echoes its (already-template-resolved) config back as - * the step output. Used as the final step of a small workflow that needs - * to assemble fields from several earlier steps (task + agent output) into - * one object, since workflow output = the last step's output. - */ -export default defineStep({ - type: "gaia/pack-result", - description: - "Echo the resolved config object back as output — a combiner for assembling fields from earlier steps into one object (workflow output = last step's output). Config: any JSON object. Output: the same object.", - input: z.record(z.string(), z.any()), - output: z.any(), - async run(cfg) { - return cfg; - }, -}); diff --git a/mcp/src/lab/gaia/workflows/gaia-candidate-run.yaml b/mcp/src/lab/gaia/workflows/gaia-candidate-run.yaml index 15f9d89d5..387b7003a 100644 --- a/mcp/src/lab/gaia/workflows/gaia-candidate-run.yaml +++ b/mcp/src/lab/gaia/workflows/gaia-candidate-run.yaml @@ -54,7 +54,7 @@ steps: # deep must exist here (§5.3.5). onError: id: grade_failed - type: gaia/pack-result + type: pack config: taskId: "{{ input.taskId }}" isCorrect: false @@ -68,7 +68,7 @@ steps: run: "{{ run }}" - id: result - type: gaia/pack-result + type: pack depends: grade config: taskId: "{{ input.taskId }}" diff --git a/mcp/src/lab/gaia/workflows/gaia-evolve-gen.yaml b/mcp/src/lab/gaia/workflows/gaia-evolve-gen.yaml index 0a2aa03db..28dafdbee 100644 --- a/mcp/src/lab/gaia/workflows/gaia-evolve-gen.yaml +++ b/mcp/src/lab/gaia/workflows/gaia-evolve-gen.yaml @@ -174,7 +174,7 @@ steps: # always runs. On the no-op branch `canddigest` is skipped, so `digest` is # undefined and `noop` tells the loop to score nothing. - id: result - type: gaia/pack-result + type: pack depends: [canddigest, published] config: candidate: "{{ input.candidateName }}" @@ -310,7 +310,7 @@ params: the question/level and stage the task's attached file into the run's artifacts dir; give agent steps that dir as cwd via artifacts/dir, exactly as the base workflow does. - - the LAST step (use gaia/pack-result) must output: taskId, question, + - the LAST step (use pack) must output: taskId, question, level, answer (the BARE final-answer string), cost, steps. - NEVER use gaia/evaluate anywhere in the candidate — self-grading at produce time is oracle access, and a candidate that embeds the diff --git a/mcp/src/lab/gaia/workflows/gaia-evolve.yaml b/mcp/src/lab/gaia/workflows/gaia-evolve.yaml index 42f1c0ae6..7e798f074 100644 --- a/mcp/src/lab/gaia/workflows/gaia-evolve.yaml +++ b/mcp/src/lab/gaia/workflows/gaia-evolve.yaml @@ -96,7 +96,7 @@ steps: # ── promote: the reviewable report ───────────────────────────────────── - id: report - type: gaia/pack-result + type: pack depends: evolve config: mission: "{{ input.mission }}" diff --git a/mcp/src/lab/gaia/workflows/gaia-produce.yaml b/mcp/src/lab/gaia/workflows/gaia-produce.yaml index d9ae90b99..9a1bd5ba7 100644 --- a/mcp/src/lab/gaia/workflows/gaia-produce.yaml +++ b/mcp/src/lab/gaia/workflows/gaia-produce.yaml @@ -18,7 +18,7 @@ steps: # task is simply scored incorrect instead of aborting every other task. onError: id: produce_failed - type: gaia/pack-result + type: pack config: object: { answer: "" } cost: 0 @@ -50,7 +50,7 @@ steps: additionalProperties: false - id: result - type: gaia/pack-result + type: pack config: taskId: "{{ task.taskId }}" question: "{{ task.question }}" diff --git a/mcp/src/lab/gaia/workflows/gaia-run.yaml b/mcp/src/lab/gaia/workflows/gaia-run.yaml index f1573d180..00ca4b37b 100644 --- a/mcp/src/lab/gaia/workflows/gaia-run.yaml +++ b/mcp/src/lab/gaia/workflows/gaia-run.yaml @@ -24,7 +24,7 @@ steps: answer: "{{ produced.answer }}" - id: result - type: gaia/pack-result + type: pack config: accuracy: "{{ score.accuracy }}" correct: "{{ score.correct }}" diff --git a/mcp/src/lab/harvey/evolve-smoke.ts b/mcp/src/lab/harvey/evolve-smoke.ts index b4386b337..3fa4167af 100644 --- a/mcp/src/lab/harvey/evolve-smoke.ts +++ b/mcp/src/lab/harvey/evolve-smoke.ts @@ -36,7 +36,7 @@ async function main() { // step types referenced by the workflows all exist in the registry const { registry } = await buildRegistry(await workspace.materializeCustomSteps()); const wanted = [ - "harvey/get-task", "harvey/evaluate", "harvey/pack-result", "harvey/digest-results", + "harvey/get-task", "harvey/evaluate", "pack", "harvey/digest-results", "eval/evolve-loop", "artifacts/dir", "meta/run-workflow", "agent", "subflow", "foreach", ]; for (const t of wanted) assert.ok(registry[t], `registry has ${t}`); diff --git a/mcp/src/lab/harvey/seed.ts b/mcp/src/lab/harvey/seed.ts index 97169648b..de6b0323b 100644 --- a/mcp/src/lab/harvey/seed.ts +++ b/mcp/src/lab/harvey/seed.ts @@ -15,8 +15,7 @@ import { SEED_OPTS } from "../seed-opts.js"; * producing agents. * - `harvey/evaluate` — stage deliverables + run the real eval. Grant ONLY * to harness workflows, never to the producing agent. - * - `harvey/pack-result` — echo combiner (assemble workflow outputs; onError - * fallbacks). + * - (the echo combiner is vein's core `pack` step now — not seeded here) * - `harvey/digest-results` — aggregate graded results into the propose * digest (verdict channel only; see the step header). * @@ -27,7 +26,6 @@ import { SEED_OPTS } from "../seed-opts.js"; const SEED_STEPS: Array<{ file: string; type: string }> = [ { file: "get-task.ts", type: "harvey/get-task" }, { file: "evaluate.ts", type: "harvey/evaluate" }, - { file: "pack-result.ts", type: "harvey/pack-result" }, { file: "digest-results.ts", type: "harvey/digest-results" }, // harvey-deliver pipeline steps (standalone production-style pipeline — // rubric as input; NOT part of the benchmark harness): intake, drafting diff --git a/mcp/src/lab/harvey/steps/pack-result.ts b/mcp/src/lab/harvey/steps/pack-result.ts deleted file mode 100644 index 8d555851a..000000000 --- a/mcp/src/lab/harvey/steps/pack-result.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { z, defineStep } from "vein"; - -/** - * Trivial combiner: echoes its (already-template-resolved) config back as - * the step output. Used as the final step of a workflow that needs to - * assemble fields from several earlier steps into one object (workflow - * output = the last step's output), and as an `onError` fallback that packs - * an explicit failure result instead of killing a batch run. - */ -export default defineStep({ - type: "harvey/pack-result", - description: - "Echo the resolved config object back as output — a combiner for assembling fields from earlier steps into one object (workflow output = last step's output). Config: any JSON object. Output: the same object.", - input: z.record(z.string(), z.any()), - output: z.any(), - async run(cfg) { - return cfg; - }, -}); diff --git a/mcp/src/lab/harvey/workflows/harvey-candidate-run.yaml b/mcp/src/lab/harvey/workflows/harvey-candidate-run.yaml index 7004cfdca..90dbb1eb5 100644 --- a/mcp/src/lab/harvey/workflows/harvey-candidate-run.yaml +++ b/mcp/src/lab/harvey/workflows/harvey-candidate-run.yaml @@ -49,7 +49,7 @@ steps: retry: { max: 1, delayMs: 15000 } onError: id: grade_failed - type: harvey/pack-result + type: pack config: score: 0 all_pass: false @@ -61,7 +61,7 @@ steps: timeoutMs: "{{ params.evalTimeoutMs }}" - id: result - type: harvey/pack-result + type: pack depends: grade config: task: "{{ input.task }}" diff --git a/mcp/src/lab/harvey/workflows/harvey-deliver.yaml b/mcp/src/lab/harvey/workflows/harvey-deliver.yaml index 525e1a66d..995c33d62 100644 --- a/mcp/src/lab/harvey/workflows/harvey-deliver.yaml +++ b/mcp/src/lab/harvey/workflows/harvey-deliver.yaml @@ -206,7 +206,7 @@ steps: webhookUrl: "{{ input.webhookUrl }}" - id: result - type: harvey/pack-result + type: pack depends: score config: task: "{{ input.task }}" diff --git a/mcp/src/lab/harvey/workflows/harvey-dispute-criterion.yaml b/mcp/src/lab/harvey/workflows/harvey-dispute-criterion.yaml index 92662ec8c..3617b6898 100644 --- a/mcp/src/lab/harvey/workflows/harvey-dispute-criterion.yaml +++ b/mcp/src/lab/harvey/workflows/harvey-dispute-criterion.yaml @@ -28,7 +28,7 @@ steps: retry: { max: 1, delayMs: 10000 } onError: id: dispute_failed - type: harvey/pack-result + type: pack config: error: "{{ $error.cause ? $error.message + ' (cause: ' + $error.cause + ')' : $error.message }}" config: diff --git a/mcp/src/lab/harvey/workflows/harvey-draft.yaml b/mcp/src/lab/harvey/workflows/harvey-draft.yaml index 986db00a5..122a6174e 100644 --- a/mcp/src/lab/harvey/workflows/harvey-draft.yaml +++ b/mcp/src/lab/harvey/workflows/harvey-draft.yaml @@ -82,7 +82,7 @@ steps: options: onError: id: verify_completeness_failed - type: harvey/pack-result + type: pack config: { error: "{{ $error.cause ? $error.message + ' (cause: ' + $error.cause + ')' : $error.message }}" } config: cwd: "{{ dir.path }}" @@ -107,7 +107,7 @@ steps: options: onError: id: verify_correctness_failed - type: harvey/pack-result + type: pack config: { error: "{{ $error.cause ? $error.message + ' (cause: ' + $error.cause + ')' : $error.message }}" } config: cwd: "{{ dir.path }}" @@ -132,7 +132,7 @@ steps: options: onError: id: verify_arithmetic_failed - type: harvey/pack-result + type: pack config: { error: "{{ $error.cause ? $error.message + ' (cause: ' + $error.cause + ')' : $error.message }}" } config: cwd: "{{ dir.path }}" @@ -157,7 +157,7 @@ steps: options: onError: id: verify_doctrine_failed - type: harvey/pack-result + type: pack config: { error: "{{ $error.cause ? $error.message + ' (cause: ' + $error.cause + ')' : $error.message }}" } config: cwd: "{{ dir.path }}" @@ -208,7 +208,7 @@ steps: why). - id: result - type: harvey/pack-result + type: pack depends: aggregate config: outputDir: "{{ dir.path }}/output" diff --git a/mcp/src/lab/harvey/workflows/harvey-evolve-gen.yaml b/mcp/src/lab/harvey/workflows/harvey-evolve-gen.yaml index c2c1ca56e..114ff5947 100644 --- a/mcp/src/lab/harvey/workflows/harvey-evolve-gen.yaml +++ b/mcp/src/lab/harvey/workflows/harvey-evolve-gen.yaml @@ -168,7 +168,7 @@ steps: # always runs. On the no-op branch `canddigest` is skipped, so `digest` is # undefined and `noop` tells the loop to score nothing. - id: result - type: harvey/pack-result + type: pack depends: [canddigest, published] config: candidate: "{{ input.candidateName }}" @@ -291,7 +291,7 @@ params: HARD CONTRACT for the candidate (the harness runs it as-is): - input { task, workdir? }; thread workdir into an artifacts/dir step (config sub) exactly as the base workflow does. - - the LAST step (use harvey/pack-result) must output: task, outputDir + - the LAST step (use pack) must output: task, outputDir (ABSOLUTE path containing the deliverable files), usage, cost, steps. - deliverables are written under outputDir; the grader stages exactly that directory. diff --git a/mcp/src/lab/harvey/workflows/harvey-evolve.yaml b/mcp/src/lab/harvey/workflows/harvey-evolve.yaml index fa96aab45..86572f1d5 100644 --- a/mcp/src/lab/harvey/workflows/harvey-evolve.yaml +++ b/mcp/src/lab/harvey/workflows/harvey-evolve.yaml @@ -97,7 +97,7 @@ steps: # ── promote: the reviewable report ───────────────────────────────────── - id: report - type: harvey/pack-result + type: pack depends: evolve config: mission: "{{ input.mission }}" diff --git a/mcp/src/lab/harvey/workflows/harvey-ingest-doc.yaml b/mcp/src/lab/harvey/workflows/harvey-ingest-doc.yaml index 007f40cd0..c0515a441 100644 --- a/mcp/src/lab/harvey/workflows/harvey-ingest-doc.yaml +++ b/mcp/src/lab/harvey/workflows/harvey-ingest-doc.yaml @@ -64,7 +64,7 @@ steps: retry: { max: 1, delayMs: 15000 } onError: id: ingest_failed - type: harvey/pack-result + type: pack config: error: "{{ $error.cause ? $error.message + ' (cause: ' + $error.cause + ')' : $error.message }}" config: @@ -121,7 +121,7 @@ steps: # skip-propagation never swallows this. Refs into a skipped branch resolve # undefined-safe in pack-result. - id: result - type: harvey/pack-result + type: pack depends: [mark, needed, docnode] config: file: "{{ input.file }}" diff --git a/mcp/src/lab/harvey/workflows/harvey-judge-criterion.yaml b/mcp/src/lab/harvey/workflows/harvey-judge-criterion.yaml index cfea13a88..a8d9c0e3c 100644 --- a/mcp/src/lab/harvey/workflows/harvey-judge-criterion.yaml +++ b/mcp/src/lab/harvey/workflows/harvey-judge-criterion.yaml @@ -22,7 +22,7 @@ steps: retry: { max: 1, delayMs: 10000 } onError: id: judge_failed - type: harvey/pack-result + type: pack config: error: "{{ $error.cause ? $error.message + ' (cause: ' + $error.cause + ')' : $error.message }}" config: diff --git a/mcp/src/lab/harvey/workflows/harvey-knowledge.yaml b/mcp/src/lab/harvey/workflows/harvey-knowledge.yaml index 2fe0be000..67a03d7cb 100644 --- a/mcp/src/lab/harvey/workflows/harvey-knowledge.yaml +++ b/mcp/src/lab/harvey/workflows/harvey-knowledge.yaml @@ -64,7 +64,7 @@ steps: options: onError: id: cross_check_failed - type: harvey/pack-result + type: pack config: error: "{{ $error.cause ? $error.message + ' (cause: ' + $error.cause + ')' : $error.message }}" config: @@ -109,7 +109,7 @@ steps: options: onError: id: case_law_failed - type: harvey/pack-result + type: pack config: error: "{{ $error.cause ? $error.message + ' (cause: ' + $error.cause + ')' : $error.message }}" config: @@ -180,7 +180,7 @@ steps: appended, edited, or resolved (or the legitimate no-op conditions). - id: result - type: harvey/pack-result + type: pack depends: [tailor] config: checklistFile: checklist.md diff --git a/mcp/src/lab/harvey/workflows/harvey-produce.yaml b/mcp/src/lab/harvey/workflows/harvey-produce.yaml index 3b1b4d6a7..70da40e3b 100644 --- a/mcp/src/lab/harvey/workflows/harvey-produce.yaml +++ b/mcp/src/lab/harvey/workflows/harvey-produce.yaml @@ -55,7 +55,7 @@ steps: retry: { max: 1, delayMs: 15000 } onError: id: produce_failed - type: harvey/pack-result + type: pack config: # usage: {} (not omitted) — the template evaluator does NOT # short-circuit, so downstream `produced.usage.inputTokens` must @@ -92,7 +92,7 @@ steps: agentTools: "{{ params.agentTools }}" - id: result - type: harvey/pack-result + type: pack depends: produce config: task: "{{ input.task }}" diff --git a/mcp/src/lab/harvey/workflows/harvey-run.yaml b/mcp/src/lab/harvey/workflows/harvey-run.yaml index 41c48ce1f..3d079988f 100644 --- a/mcp/src/lab/harvey/workflows/harvey-run.yaml +++ b/mcp/src/lab/harvey/workflows/harvey-run.yaml @@ -48,7 +48,7 @@ steps: retry: { max: 1, delayMs: 15000 } onError: id: grade_failed - type: harvey/pack-result + type: pack config: score: 0 all_pass: false @@ -68,7 +68,7 @@ steps: timeoutMs: "{{ params.evalTimeoutMs }}" - id: result - type: harvey/pack-result + type: pack depends: grade config: task: "{{ input.task }}" diff --git a/mcp/src/lab/harvey/workflows/harvey-score.yaml b/mcp/src/lab/harvey/workflows/harvey-score.yaml index 1dd520904..2377f0b61 100644 --- a/mcp/src/lab/harvey/workflows/harvey-score.yaml +++ b/mcp/src/lab/harvey/workflows/harvey-score.yaml @@ -54,7 +54,7 @@ steps: options: onError: id: reqdetails_failed - type: harvey/pack-result + type: pack config: error: "{{ $error.cause ? $error.message + ' (cause: ' + $error.cause + ')' : $error.message }}" config: @@ -110,7 +110,7 @@ steps: options: onError: id: record_failed - type: harvey/pack-result + type: pack config: error: "{{ $error.cause ? $error.message + ' (cause: ' + $error.cause + ')' : $error.message }}" config: @@ -204,7 +204,7 @@ steps: # The final JSON (build_webhook_body equivalent) — also this workflow's # output via the trailing `result` echo. - id: body - type: harvey/pack-result + type: pack depends: [merged, chain, record] config: task: "{{ input.task }}" @@ -239,7 +239,7 @@ steps: options: onError: id: post_failed - type: harvey/pack-result + type: pack config: error: "{{ $error.cause ? $error.message + ' (cause: ' + $error.cause + ')' : $error.message }}" config: @@ -265,7 +265,7 @@ steps: recursion: false - id: result - type: harvey/pack-result + type: pack depends: [body, post, have_webhook, all_pass, norec, persist_annotations, contested_writeback] config: task: "{{ input.task }}" diff --git a/mcp/src/lab/plans/wfbench-harness.md b/mcp/src/lab/plans/wfbench-harness.md index 5521f39d4..6eb7a9f5d 100644 --- a/mcp/src/lab/plans/wfbench-harness.md +++ b/mcp/src/lab/plans/wfbench-harness.md @@ -44,47 +44,58 @@ Not a "chat assistant as a step". Reasons: | skill `harvey_lab_score_rubric` (all criteria, one call) | `foreach` criteria → subflow `wfbench-judge-criterion` (clone of `harvey-judge-criterion`: agent schema mode, materials in prompt / artifacts dir, NO tools) → `eval/aggregate-scores`. Judge crash = `{ error }` = honest FAIL | | `guard_judge_ran`, `guard_valid_score` | `if` on aggregate output, else `judge_failed` | | 58312 record (EvalTriggerOutput + CriterionResult + edges) | `eval/build-eval-chain` → `graph/create-batch-triplet` (idempotent on runId) | -| `resolve_webhook_payload` (4-way) + `post_result` | one `wfbench/pack-result` depending on every branch (`||` chain) → `if webhookUrl` → `http POST`. Output == what was posted (fixes the 58313 `set_output` divergence) | +| `resolve_webhook_payload` (4-way) + `post_result` | one `pack` depending on every branch (`||` chain) → `if webhookUrl` → `http POST`. Output == what was posted (fixes the 58313 `set_output` divergence) | Webhook body: byte-compatible with what Hive parses today (`RunnerScoreSchema`): success `{ task_slug, task_title, n_passed, n_total, all_pass, pass_rate, judge_model, criteria_results }`, failure `{ harness_error: true, error_type }` with no score fields. -## Files +## Files (v1 — BUILT; offline smoke green) ``` mcp/src/lab/wfbench/ seed.ts # seedWfbenchSteps / seedWfbenchWorkflows (SEED_OPTS) - steps/ - check-input-keys.ts # pure - classify-run.ts # pure - build-materials.ts # pure (reads meta/* outputs passed in) - pack-result.ts # pure passthrough (like harvey/pack-result) - stakwork-fetch.ts # GET jobs.stakwork.com workflow body + one project's - # input/output; STAK_CUSTOMER_TOKEN via ctx.services.secrets + smoke.ts # offline: seed, discover, validate YAML, drive every pure step, + # check graph payloads against JARVIS_ONTOLOGY + steps/ # all pure — no services, LLM, or graph + normalize-task.ts # Hive/58313 payload → canonical task (hard-fails early) + build-roster.ts # EvalSet / EvalRequirement×N / EvalTrigger payloads (58313 ids) + trigger-edge.ts # guard_first_run: HAS_BASELINE_TRIGGER vs HAS_TRIGGER + resolve-candidate.ts # vpin || vactive, published vs vbefore (never trust the echo) + check-input-keys.ts # input. refs in the YAML vs the launch payload + classify-run.ts # launch_ok / completed / failed / harness error + build-materials.ts # judge materials → one markdown block (judge needs no tools) + build-eval-output.ts # EvalTriggerOutput + CriterionResult triplets (58312 ids) + webhook-body.ts # resolve_webhook_payload (exact Hive keys) + pack-result.ts # passthrough / onError pack workflows/ - wfbench-run.yaml # the 58313 twin (input below) - wfbench-judge-criterion.yaml # per-criterion judge subflow - wfbench-port-task.yaml # stakwork workflow id → task object (+ derived rubric) - wfbench-batch.yaml # foreach tasks → wfbench-run + wfbench-run.yaml # the 58313 twin + wfbench-judge-criterion.yaml # per-criterion judge subflow (agent schema mode) ``` -`wfbench-run` input: `{ task_slug, task_title, instructions, criteria, -workflow_input_json, rerun_expected_output?, webhookUrl?, namespace?, -baseline_workflow? }`. Params: `authorSystem`, `authorModel`, -`authorMaxSteps`, `judgeSystem`, `judgeModel`, `judgeMaxSteps`. +Not built (deferred): `wfbench-batch` (foreach tasks), `wfbench-port-task` +(stakwork id → task; needs `STAK_CUSTOMER_TOKEN`) and the edit-existing +(`baseline_workflow`) path. The bench itself needs NO stakwork credentials: +tasks arrive as input. -Registered in `createLabVein.ts` next to gaia. Needs the graph (mcp's -Neo4j), `ANTHROPIC_API_KEY`, and `STAK_CUSTOMER_TOKEN` in the secret store -for the port path. +`wfbench-run` input: `{ task_slug, task_title?, instructions, criteria, +workflow_input_json?, rerun_expected_output?, webhook_url? }`. Params: +`namespace`, `authorSystem`, `authorGuidance`, `authorModel`, +`authorMaxSteps`, `judgeModel`, `judgeConcurrency` (+ `judgeSystem` / +`judgePrompt` / `judgeMaxSteps` on the judge subflow). + +Registered in `createLabVein.ts` after the artifact steps. Needs the graph +(mcp's Neo4j) and `ANTHROPIC_API_KEY`. ## Grant discipline - author: `meta/*` + editor tool only. Never bash, never `graph/*`, never `wfbench/*`. -- judge: no tools (materials are pre-resolved into the prompt). Cheap and - non-gameable. +- judge: nothing useful to call (materials are pre-resolved into the + prompt; the core `llm` step takes a Zod schema, not JSON Schema, so the + judge is the `agent` step in schema mode with the editor tool over an + empty dir). Cheap and non-gameable. - produced workflow: runs via `meta/run-workflow` so it is necessarily publisher `ai`; the seeded harness surface can never be graded as a candidate. @@ -126,6 +137,17 @@ one; 58313 v1 skipped it. and the smoke were re-pointed in the same change; `workflow` is now a required input of build-eval-chain). +## Graph writes vs 58313 / 58312 (what differs, and why) + +vein's graph backend rejects attributes the ontology does not declare, so: +`EvalSet.project_id` (an int in stakwork; vein runIds are strings) and the +`name` on EvalTrigger / EvalTriggerOutput are omitted (EvalTrigger gets +`agent: wfbench-run` as its title). 58312's `CriterionResult -HAS_CAUSE-> +Workflow_version(material_ref_id)` is not written: the ontology has no +such relationship and the produced vein workflow is not a Workflow_version +node. Everything else — ids, edge types, EvalTriggerOutput's score fields, +CriterionResult's verdict/reasoning — matches, and the smoke asserts it. + ## Not ported on purpose - 58414's polling loop and the `api_fallback_fetch` (no async publish). diff --git a/mcp/src/lab/wfbench/seed.ts b/mcp/src/lab/wfbench/seed.ts new file mode 100644 index 000000000..e5c679928 --- /dev/null +++ b/mcp/src/lab/wfbench/seed.ts @@ -0,0 +1,82 @@ +import { readFile } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; +import type { WorkspaceStore } from "vein"; +import { SEED_OPTS } from "../seed-opts.js"; + +/** + * wfbench — the Workflow Editor Agent Benchmark harness (the vein port of + * stakwork workflow 58313; see plans/wfbench-harness.md). Pure plumbing + * steps + the two workflows. Everything here is seeded UNSTAMPED (no + * publisher arg → not "ai"), so the meta surface — i.e. the author agent + * inside a benchmark run — can read but never edit, run, or overwrite the + * harness. Content-hash reconciled (SEED_OPTS): a changed committed copy + * wins at boot, an unchanged one leaves a workspace-side edit active. + * + * Steps (all pure — no services, no LLM, no graph; the graph writes are + * vein's graph/* lib steps and the judge is the core agent step): + * wfbench/normalize-task the Hive/58313 task payload → canonical task + * wfbench/build-roster EvalSet / EvalRequirement / EvalTrigger payloads (58313 ids) + * wfbench/trigger-edge HAS_BASELINE_TRIGGER vs HAS_TRIGGER (guard_first_run) + * wfbench/resolve-candidate pin what the author actually shipped (never its echo) + * wfbench/check-input-keys input-key contract gate (wfbench_check_input_keys.py) + * wfbench/classify-run launch_ok / completed / failed (wfbench_classify_run_result.py) + * wfbench/build-materials judge materials (wfbench_build_produced_materials.py) + * wfbench/build-eval-output EvalTriggerOutput + CriterionResult triplets (58312) + * wfbench/webhook-body the one Hive callback body (resolve_webhook_payload) + * + * GRANT DISCIPLINE: none of wfbench/* is ever granted to an agent's + * agentTools — the author gets meta/* only, the judge gets nothing. + */ +const SEED_STEPS: Array<{ file: string; type: string }> = [ + { file: "normalize-task.ts", type: "wfbench/normalize-task" }, + { file: "build-roster.ts", type: "wfbench/build-roster" }, + { file: "trigger-edge.ts", type: "wfbench/trigger-edge" }, + { file: "resolve-candidate.ts", type: "wfbench/resolve-candidate" }, + { file: "check-input-keys.ts", type: "wfbench/check-input-keys" }, + { file: "classify-run.ts", type: "wfbench/classify-run" }, + { file: "build-materials.ts", type: "wfbench/build-materials" }, + { file: "build-eval-output.ts", type: "wfbench/build-eval-output" }, + { file: "webhook-body.ts", type: "wfbench/webhook-body" }, +]; + +const SEED_WORKFLOWS: Array<{ name: string; description: string }> = [ + { + name: "wfbench-judge-criterion", + description: + "wfbench: LLM-as-judge for ONE rubric criterion over the produced workflow's materials (agent schema mode, no tools). Run per criterion by wfbench-run; a crash packs { error } = honest FAIL. Input: { criterion, task_desc, materials_text }.", + }, + { + name: "wfbench-run", + description: + "Workflow Editor Agent Benchmark — Task Runner (stakwork 58313's twin): graph roster (EvalSet/EvalRequirement/EvalTrigger) -> meta/* author builds wfbench- -> input-key gate -> rerun via meta/run-workflow -> per-criterion judge -> record EvalTriggerOutput/CriterionResult -> POST the Hive callback. Input: { task_slug, task_title?, instructions, criteria, workflow_input_json?, rerun_expected_output?, webhook_url? }.", + }, +]; + +const HERE = dirname(fileURLToPath(import.meta.url)); + +export async function seedWfbenchSteps(workspace: WorkspaceStore): Promise { + const dir = join(HERE, "steps"); + for (const { file, type } of SEED_STEPS) { + try { + const code = await readFile(join(dir, file), "utf-8"); + const { version, changed } = await workspace.publishStep(type, code, undefined, "wfbench-seed", SEED_OPTS); + if (changed) console.log(`[wfbench] seeded step: ${type} @ ${version}`); + } catch (err) { + console.warn(`[wfbench] could not seed step "${type}":`, err instanceof Error ? err.message : err); + } + } +} + +export async function seedWfbenchWorkflows(workspace: WorkspaceStore): Promise { + const dir = join(HERE, "workflows"); + for (const { name, description } of SEED_WORKFLOWS) { + try { + const yaml = await readFile(join(dir, `${name}.yaml`), "utf-8"); + const { version, changed } = await workspace.publishWorkflowByContent(name, yaml, description, "wfbench", undefined, SEED_OPTS); + if (changed) console.log(`[wfbench] seeded workflow: ${name} @ ${version}`); + } catch (err) { + console.warn(`[wfbench] could not seed workflow "${name}":`, err instanceof Error ? err.message : err); + } + } +} diff --git a/mcp/src/lab/wfbench/smoke.ts b/mcp/src/lab/wfbench/smoke.ts new file mode 100644 index 000000000..a789c7aaa --- /dev/null +++ b/mcp/src/lab/wfbench/smoke.ts @@ -0,0 +1,301 @@ +/** + * wfbench OFFLINE smoke — no LLM, no graph, no network. Seeds the harness + * into a throwaway workspace, discovers the steps, static-validates both + * workflows through the authoring capability (the same check the author + * agent gets via meta/validate-workflow), then drives every pure step with + * fixtures and asserts the graph payloads match stakwork 58313 / 58312's id + * conventions and only use attributes the jarvis ontology declares (vein's + * graph backend rejects undeclared ones). + * + * npx tsx src/lab/wfbench/smoke.ts + */ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { WorkspaceManager, buildRegistry, createVein, JARVIS_ONTOLOGY, type StepContext } from "vein"; +import { seedWfbenchSteps, seedWfbenchWorkflows } from "./seed.js"; +import { seedEvalSteps } from "../eval/seed.js"; +import { seedArtifactSteps } from "../artifacts/seed.js"; + +const declared = (type: string): Set => { + const schema = JARVIS_ONTOLOGY.schemas.find((s) => s["type"] === type); + assert.ok(schema, `ontology has no ${type}`); + const meta = new Set(["type", "ref_id", "node_key", "parent", "domain", "type_description", "index", "icon", "primary_color", "secondary_color", "shape", "description_key", "title_key", "paid_properties", "is_system"]); + return new Set(Object.keys(schema!).filter((k) => !meta.has(k))); +}; +const assertDeclared = (type: string, data: Record) => { + const ok = declared(type); + for (const k of Object.keys(data)) assert.ok(ok.has(k), `${type}.${k} is not declared on the ontology`); +}; +const edgeAllowed = (source: string, edge: string, target: string) => + JARVIS_ONTOLOGY.edge_schemas.some((e) => e.source === source && e.edge === edge && e.target === target); + +async function main() { + const dir = mkdtempSync(join(process.cwd(), ".wfbench-smoke-")); + try { + // ── 1. seed + discover ─────────────────────────────────────────────── + const workspace = new WorkspaceManager(dir); + await seedEvalSteps(workspace); + await seedArtifactSteps(workspace); + await seedWfbenchSteps(workspace); + await seedWfbenchWorkflows(workspace); + const { registry } = await buildRegistry(await workspace.materializeCustomSteps()); + const expectedSteps = [ + "wfbench/normalize-task", + "wfbench/build-roster", + "wfbench/trigger-edge", + "wfbench/resolve-candidate", + "wfbench/check-input-keys", + "wfbench/classify-run", + "wfbench/build-materials", + "wfbench/build-eval-output", + "wfbench/webhook-body", + "pack", + "eval/aggregate-scores", + "eval/criterion-refs", + "artifacts/dir", + // vein lib/core steps the workflows lean on + "graph/create-node", + "graph/create-batch-triplet", + "graph/create-triplet", + "graph/graph-neighbors", + "graph/register-namespace", + "meta/get-workflow", + "meta/run-workflow", + "meta/get-step", + "agent", + ]; + for (const t of expectedSteps) assert.ok(registry[t], `registry missing ${t}`); + console.log(`✔ seeded + discovered ${expectedSteps.length} steps`); + + // ── 2. static validation (what meta/validate-workflow runs) ────────── + const vein = await createVein({ workspace, serveUi: false }); + const authoring = (vein.services as any).authoring; + for (const name of ["wfbench-judge-criterion", "wfbench-run"]) { + const entry = (await workspace.listWorkflows()).find((w) => w.name === name); + assert.ok(entry, `workflow ${name} not seeded`); + const yaml = await workspace.getWorkflowSource(name, entry!.activeVersion); + const v = await authoring.validateWorkflow(yaml, name); + assert.equal(v.ok, true, `${name}: ${JSON.stringify(v.errors, null, 2)}`); + if (v.warnings.length) console.log(` ${name} warnings:`, v.warnings.map((w: any) => `${w.path}: ${w.message}`)); + console.log(`✔ ${name} validates (${v.summary.steps} steps)`); + } + + const run = (type: string, input: unknown, ctx?: StepContext) => + registry[type]!.run(registry[type]!.input.parse(input), ctx ?? ({} as StepContext)); + const ctx = { runId: "run42" } as StepContext; + + // ── 3. normalize-task ──────────────────────────────────────────────── + let task: any = await run("wfbench/normalize-task", { + task_slug: "Fetch PR Titles!", + instructions: "Given owner/repo, list the titles of open PRs.", + criteria: JSON.stringify([ + { id: "c1", title: "Uses input keys", match_criteria: "reads owner and repo" }, + { id: "c1", title: "dup id", match_criteria: "…" }, + { title: "no id", match_criteria: "…", deliverables: ["out.json"] }, + ]), + workflow_input_json: '{"owner":"stakwork","repo":"hive"}', + rerun_expected_output: '{"titles":["a"]}', + }); + assert.equal(task.task_slug, "fetch-pr-titles"); + assert.equal(task.task_title, "Fetch PR Titles!"); + assert.deepEqual(task.criteria.map((c: any) => c.id), ["c1", "c1-2", "c3"]); + assert.deepEqual(task.criteria[2].deliverables, ["out.json"]); + assert.deepEqual(task.workflow_input_keys, ["owner", "repo"]); + assert.deepEqual(task.rerun_expected_output, { titles: ["a"] }); + await assert.rejects(() => run("wfbench/normalize-task", { task_slug: "x", instructions: "y", criteria: "[]" }), /non-empty/); + await assert.rejects( + () => run("wfbench/normalize-task", { task_slug: "x", instructions: "y", criteria: [{ id: "a" }], workflow_input_json: "[1]" }), + /JSON object/, + ); + console.log("✔ normalize-task (slug / criteria ids / JSON strings / hard fails)"); + + // ── 4. build-roster (58313 ids, ontology-declared attrs only) ──────── + const roster: any = await run( + "wfbench/build-roster", + { task_slug: task.task_slug, task_title: task.task_title, instructions: task.instructions, criteria: task.criteria, workflow: "wfbench-run", workflow_version: "v3" }, + ctx, + ); + assert.equal(roster.run_id, "run42"); + assert.deepEqual(roster.evalset, { node_type: "EvalSet", node_data: { id: "fetch-pr-titles", name: "Fetch PR Titles!" } }); + assertDeclared("EvalSet", roster.evalset.node_data); + assert.equal(roster.requirement_triplets.length, 3); + assert.deepEqual(roster.requirement_ids, ["fetch-pr-titles-c1", "fetch-pr-titles-c1-2", "fetch-pr-titles-c3"]); + for (const [i, t] of roster.requirement_triplets.entries()) { + assert.equal(t.edge_type, "HAS_REQUIREMENT"); + assert.deepEqual(t.source_data, { id: "fetch-pr-titles" }); + assert.equal(t.edge_data.order, i); + assertDeclared("EvalRequirement", t.target_data); + assert.ok(edgeAllowed("EvalSet", "HAS_REQUIREMENT", "EvalRequirement")); + } + assert.equal(roster.requirement_triplets[0].target_data.description, "reads owner and repo"); + assert.equal(roster.trigger_id, "fetch-pr-titles-run42"); + assert.equal(roster.trigger.node_type, "EvalTrigger"); + const trig = roster.trigger.node_data; + assert.equal(trig.id, "fetch-pr-titles-run42"); + assert.equal(trig.project_id, "run42"); + assert.equal(trig.workflow_id, "wfbench-run"); + assert.equal(trig.workflow_version_id, "v3"); + assert.equal(trig.workflow_input, task.instructions); + assertDeclared("EvalTrigger", trig); + assert.ok(edgeAllowed("EvalSet", "HAS_TRIGGER", "EvalTrigger") && edgeAllowed("EvalSet", "HAS_BASELINE_TRIGGER", "EvalTrigger")); + console.log("✔ build-roster (EvalSet / EvalRequirement×3 / EvalTrigger — 58313 ids, declared attrs)"); + + // ── 5. trigger-edge (guard_first_run) ──────────────────────────────── + let e: any = await run("wfbench/trigger-edge", { neighbors: [], trigger_ref_id: "t1" }); + assert.deepEqual([e.edge_type, e.is_baseline, e.prior_triggers], ["HAS_BASELINE_TRIGGER", true, 0]); + e = await run("wfbench/trigger-edge", { neighbors: [{ ref_id: "t1", node_type: "EvalTrigger" }], trigger_ref_id: "t1" }); + assert.equal(e.edge_type, "HAS_BASELINE_TRIGGER"); // own trigger excluded + e = await run("wfbench/trigger-edge", { neighbors: [{ ref_id: "t0", node_type: "EvalTrigger" }, { ref_id: "r1", node_type: "EvalRequirement" }], trigger_ref_id: "t1" }); + assert.deepEqual([e.edge_type, e.prior_triggers], ["HAS_TRIGGER", 1]); + e = await run("wfbench/trigger-edge", { neighbors: "graph/graph-neighbors: HTTP 500" }); + assert.deepEqual([e.edge_type, e.is_baseline, e.readable], ["HAS_TRIGGER", false, false]); + console.log("✔ trigger-edge (baseline / own-excluded / prior / unreadable)"); + + // ── 6. resolve-candidate (never trust the echo) ────────────────────── + const author = { object: { workflow: "wfbench-fetch-pr-titles", version: "v2", summary: "s", changes: ["a"], customSteps: ["cand/x", 3] }, cost: 0.5, steps: 12 }; + let cand: any = await run("wfbench/resolve-candidate", { + author, candidate: "wfbench-fetch-pr-titles", + vbefore: { error: "not found" }, + vpin: { version: "v2", yaml: "name: x\nsteps: []\n" }, + vactive: { version: "v2", yaml: "name: x\nsteps: []\n" }, + }); + assert.deepEqual([cand.version, cand.published, cand.yaml, cand.customSteps, cand.authorCost], ["v2", true, "name: x\nsteps: []\n", ["cand/x"], 0.5]); + cand = await run("wfbench/resolve-candidate", { + author: { object: { version: "placeholder" } }, candidate: "c", + vbefore: { version: "v1" }, vpin: { error: "Version placeholder not found" }, vactive: { version: "v2", yaml: "y" }, + }); + assert.deepEqual([cand.version, cand.published, cand.yaml], ["v2", true, "y"]); // echo bogus → active + cand = await run("wfbench/resolve-candidate", { + author: { error: "agent blew up" }, candidate: "c", + vbefore: { version: "v1" }, vpin: { error: "x" }, vactive: { version: "v1", yaml: "y" }, + }); + assert.deepEqual([cand.version, cand.published, cand.yaml, cand.authorError], ["v1", false, "", "agent blew up"]); // nothing shipped + console.log("✔ resolve-candidate (pin / active fallback / no-op publish / author error)"); + + // ── 7. check-input-keys ────────────────────────────────────────────── + const yaml = 'name: x\nsteps:\n - id: a\n type: http\n config:\n url: "https://api/{{ input.owner }}/{{ input["repo"] }}"\n'; + let keys: any = await run("wfbench/check-input-keys", { workflow_yaml: yaml, workflow_input: { owner: "o", repo: "r", extra: 1 } }); + assert.deepEqual([keys.keys_match, keys.referenced_keys, keys.missing, keys.unused, keys.error_type], [true, ["owner", "repo"], [], ["extra"], null]); + assert.deepEqual(keys.launch_payload, { owner: "o", repo: "r", extra: 1 }); + keys = await run("wfbench/check-input-keys", { workflow_yaml: yaml, workflow_input: { owner: "o" } }); + assert.deepEqual([keys.keys_match, keys.missing, keys.error_type], [false, ["repo"], "input_keys_mismatch"]); + keys = await run("wfbench/check-input-keys", { workflow_yaml: "", workflow_input: { owner: "o" } }); + assert.deepEqual([keys.keys_match, keys.error_type], [false, "no_workflow_produced"]); + console.log("✔ check-input-keys (match+unused / missing / empty body)"); + + // ── 8. classify-run ────────────────────────────────────────────────── + let cls: any = await run("wfbench/classify-run", { gate_error_type: "input_keys_mismatch", gate_error: "m" }); + assert.deepEqual([cls.launch_ok, cls.execution_status, cls.error_type, cls.error], [false, "none", "input_keys_mismatch", "m"]); + cls = await run("wfbench/classify-run", { run: { error: "not agent-authored" } }); + assert.deepEqual([cls.launch_ok, cls.error_type], [false, "launch_refused"]); + cls = await run("wfbench/classify-run", { run: { runId: "child1", status: "success", output: { titles: ["a"] } } }); + assert.deepEqual([cls.launch_ok, cls.execution_status, cls.project_id, cls.error_type], [true, "completed", "child1", null]); + assert.deepEqual(cls.run_output, { titles: ["a"] }); + const failedCls: any = await run("wfbench/classify-run", { run: { runId: "child2", status: "error", error: { message: "boom" } } }); + assert.deepEqual([failedCls.launch_ok, failedCls.execution_status, failedCls.error_type, failedCls.error], [true, "failed", "produced_workflow_failed", "boom"]); + console.log("✔ classify-run (not launched / refused / completed / failed-still-launched)"); + + // ── 9. build-materials ─────────────────────────────────────────────── + let mats: any = await run("wfbench/build-materials", { + workflow: "wfbench-fetch-pr-titles", version: "v2", workflow_yaml: yaml, + custom_steps: [{ type: "cand/x", code: "export default 1" }, { type: "cand/y", error: "not found" }], + run_output: { titles: ["a"] }, execution_status: "completed", project_id: "child1", + rerun_expected_output: { titles: ["a"] }, launch_payload: { owner: "o" }, instructions: "do it", + }); + assert.equal(mats.n_materials, 2); + assert.deepEqual(mats.materials.map((m: any) => m.type), ["WORKFLOW", "STEP", "LAUNCH_PAYLOAD", "RUN_OUTPUT", "EXPECTED_OUTPUT"]); + assert.equal(mats.warnings.length, 1); + assert.match(mats.materials_text, /### WORKFLOW: wfbench-fetch-pr-titles@v2\n\n```yaml/); + assert.match(mats.materials_text, /### STEP: cand\/x/); + assert.equal(mats.task_desc, "do it"); + mats = await run("wfbench/build-materials", { workflow: "w", workflow_yaml: "", instructions: "do it" }); + assert.equal(mats.n_materials, 0); + assert.deepEqual(mats.materials.map((m: any) => m.type), ["LAUNCH_PAYLOAD", "RUN_OUTPUT"]); + console.log("✔ build-materials (produced vs context / warnings / none)"); + + // ── 10. judge zip → build-eval-output (58312 / 58115 chain) ────────── + const scores: any = await run("eval/aggregate-scores", { + rubric: task.criteria, + results: [ + { object: { verdict: "pass", reasoning: "ok" }, cost: 0.1 }, + { error: "judge blew up" }, + { object: { verdict: "pass", reasoning: "fine" }, cost: 0.1 }, + ], + judge_model: "claude-sonnet-5", + }); + assert.deepEqual([scores.n_passed, scores.n_total, scores.all_pass], [2, 3, false]); + const chain: any = await run("wfbench/build-eval-output", { task_slug: task.task_slug, scores, trigger_ref_id: "trig-ref", judge_model: "claude-sonnet-5" }, ctx); + assert.equal(chain.scored, true); + assert.equal(chain.output_id, "fetch-pr-titles-run42"); + assert.equal(chain.trigger_id, "fetch-pr-titles-run42"); + // 1 spine + 2 per criterion + assert.equal(chain.triplets.length, 1 + 2 * 3); + const spine = chain.triplets[0]; + assert.deepEqual([spine.source_ref_id, spine.target_type, spine.edge_type], ["trig-ref", "EvalTriggerOutput", "HAS_OUTPUT"]); + assert.deepEqual(spine.target_data, { + id: "fetch-pr-titles-run42", result: "fail", verdict: "fail", score: 2, max_score: 3, n_passed: 2, n_total: 3, judge_model: "claude-sonnet-5", + }); + assertDeclared("EvalTriggerOutput", spine.target_data); + assert.ok(edgeAllowed("EvalTrigger", "HAS_OUTPUT", "EvalTriggerOutput")); + assert.deepEqual(chain.criterionSlots, [{ criterion_id: "c1", index: 1 }, { criterion_id: "c1-2", index: 3 }, { criterion_id: "c3", index: 5 }]); + const fromOutput = chain.triplets[1]; + const fromReq = chain.triplets[2]; + assert.deepEqual([fromOutput.source_type, fromOutput.edge_type, fromOutput.target_type], ["EvalTriggerOutput", "HAS_CRITERION_RESULT", "CriterionResult"]); + assert.deepEqual(fromOutput.source_data, spine.target_data); // identical object → batch dedupe hits + assert.deepEqual([fromReq.source_type, fromReq.edge_type, fromReq.target_type], ["EvalRequirement", "HAS_CRITERION_RESULT", "CriterionResult"]); + assert.deepEqual(fromReq.source_data, { id: "fetch-pr-titles-c1" }); + assert.deepEqual(fromReq.target_data, fromOutput.target_data); + assert.deepEqual(fromOutput.target_data, { id: "fetch-pr-titles-run42-c1", criterion_id: "c1", title: "Uses input keys", verdict: "pass", reasoning: "ok" }); + assertDeclared("CriterionResult", fromOutput.target_data); + assert.ok(edgeAllowed("EvalTriggerOutput", "HAS_CRITERION_RESULT", "CriterionResult") && edgeAllowed("EvalRequirement", "HAS_CRITERION_RESULT", "CriterionResult")); + assert.equal(chain.triplets[3].target_data.verdict, "fail"); + assert.match(chain.triplets[3].target_data.reasoning, /judge error/); + // inline trigger fallback + unscored + const chain2: any = await run("wfbench/build-eval-output", { task_slug: "t", scores }, ctx); + assert.deepEqual([chain2.triplets[0].source_type, chain2.triplets[0].source_data], ["EvalTrigger", { id: "t-run42" }]); + const chain3: any = await run("wfbench/build-eval-output", { task_slug: "t", scores: { error: "refusing to zip" } }, ctx); + assert.deepEqual([chain3.scored, chain3.triplets], [false, []]); + // criterion-refs recovers the persisted CriterionResult refs by slot + const refs: any = await run("eval/criterion-refs", { + slots: chain.criterionSlots, + record: { results: chain.triplets.map((_: any, i: number) => ({ target_ref_id: `ref${i}` })) }, + }); + assert.deepEqual(refs, [{ criterion_id: "c1", ref_id: "ref1" }, { criterion_id: "c1-2", ref_id: "ref3" }, { criterion_id: "c3", ref_id: "ref5" }]); + console.log("✔ build-eval-output (EvalTriggerOutput + CriterionResult×3, 58312 ids, declared attrs, slots→refs)"); + + // ── 11. webhook-body (resolve_webhook_payload) ─────────────────────── + const base = { task_slug: "fetch-pr-titles", task_title: "Fetch PR Titles!", judge_model: "claude-sonnet-5" }; + let body: any = await run("wfbench/webhook-body", { ...base, keys: { keys_match: false, error_type: "input_keys_mismatch", error: "m" }, cls: { launch_ok: false } }); + assert.deepEqual(body, { task_slug: "fetch-pr-titles", task_title: "Fetch PR Titles!", harness_error: true, error_type: "input_keys_mismatch", error: "m" }); + body = await run("wfbench/webhook-body", { ...base, keys: { keys_match: true }, cls: { launch_ok: false, error_type: "launch_refused", error: "r" } }); + assert.deepEqual([body.harness_error, body.error_type, body.n_passed], [true, "launch_refused", undefined]); + body = await run("wfbench/webhook-body", { ...base, keys: { keys_match: true }, cls: { launch_ok: true }, mats: { n_materials: 0 } }); + assert.equal(body.error_type, "no_materials_produced"); + body = await run("wfbench/webhook-body", { ...base, keys: { keys_match: true }, cls: { launch_ok: true }, mats: { n_materials: 1 }, scores: { error: "refusing to zip" } }); + assert.deepEqual([body.error_type, body.error], ["judge_failed", "refusing to zip"]); + body = await run("wfbench/webhook-body", { ...base, keys: { keys_match: true }, cls: { launch_ok: true }, mats: { n_materials: 1 } }); + assert.equal(body.error_type, "judge_failed"); // judge never ran + body = await run("wfbench/webhook-body", { ...base, keys: { keys_match: true }, cls: { launch_ok: true, execution_status: "failed" }, mats: { n_materials: 1 }, scores }); + assert.deepEqual(Object.keys(body).sort(), ["all_pass", "criteria_results", "judge_model", "n_passed", "n_total", "pass_rate", "task_slug", "task_title"]); + assert.deepEqual([body.n_passed, body.n_total, body.all_pass, body.judge_model, body.criteria_results.length], [2, 3, false, "claude-sonnet-5", 3]); + assert.equal(body.harness_error, undefined); + console.log("✔ webhook-body (keys / launch / no materials / judge failed ×2 / success — exact Hive keys)"); + + // ── 12. pack-result ────────────────────────────────────────────────── + assert.deepEqual(await run("pack", { a: 1, b: { c: 2 } }), { a: 1, b: { c: 2 } }); + console.log("✔ pack-result"); + + console.log("\nALL WFBENCH SMOKE CHECKS PASSED"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +main().then( + () => process.exit(0), + (err) => { + console.error(err); + process.exit(1); + }, +); diff --git a/mcp/src/lab/wfbench/steps/build-eval-output.ts b/mcp/src/lab/wfbench/steps/build-eval-output.ts new file mode 100644 index 000000000..7a158741c --- /dev/null +++ b/mcp/src/lab/wfbench/steps/build-eval-output.ts @@ -0,0 +1,89 @@ +import { z, defineStep, type StepContext } from "vein"; + +/** + * 58312 "Record Eval Trigger Output and Criterion Results" as a batch- + * triplet payload (the write is graph/create-batch-triplet). Ids and edges + * follow 58312 / 58115 exactly, on the ontology's declared relationships: + * + * EvalTrigger(-) -HAS_OUTPUT-> EvalTriggerOutput(-) + * EvalTriggerOutput -HAS_CRITERION_RESULT-> CriterionResult (one per criterion) + * EvalRequirement(-) -HAS_CRITERION_RESULT-> CriterionResult + * + * EvalTriggerOutput carries 58312's fields: result, verdict, score, + * max_score, n_passed, n_total, judge_model (its `name` is not declared on + * the ontology and is omitted). 58312's CriterionResult -HAS_CAUSE-> + * Workflow_version edge is NOT written: the ontology has no such + * relationship and vein's produced workflow is not a Workflow_version node. + * Ids derive from the runId, so a retried write merges instead of + * duplicating. `criterionSlots` index each criterion's first + * HAS_CRITERION_RESULT triplet for eval/criterion-refs. + */ +export default defineStep({ + type: "wfbench/build-eval-output", + description: + "Build the create-batch-triplet payload recording a judged run per 58312: EvalTrigger -HAS_OUTPUT-> EvalTriggerOutput { id: -, result, verdict, score, max_score, n_passed, n_total, judge_model } and, per criterion, EvalTriggerOutput/EvalRequirement -HAS_CRITERION_RESULT-> CriterionResult { id: --, criterion_id, title, verdict, reasoning }. Output: { scored, triplets, criterionSlots, trigger_id, output_id }.", + input: z.object({ + task_slug: z.string().min(1), + scores: z.any().optional().describe("eval/aggregate-scores output (absent/errored → scored=false, no triplets)."), + trigger_ref_id: z.any().optional().describe("The EvalTrigger's ref_id (preferred); falls back to an inline { id } source."), + trigger_id: z.string().optional().describe("The EvalTrigger id (default -)."), + judge_model: z.string().optional(), + }), + output: z.any(), + async run(cfg, ctx) { + const runId = (ctx as StepContext)?.runId || "no-run"; + const slug = cfg.task_slug; + const trigger_id = cfg.trigger_id || `${slug}-${runId}`; + const output_id = `${slug}-${runId}`; + const s = cfg.scores && typeof cfg.scores === "object" && !cfg.scores.error ? (cfg.scores as Record) : null; + if (!s || typeof s.all_pass !== "boolean" || !Array.isArray(s.criteria_results)) { + return { scored: false, triplets: [], criterionSlots: [], trigger_id, output_id }; + } + const outputNodeData = { + id: output_id, + result: s.all_pass ? "pass" : "fail", + verdict: s.all_pass ? "pass" : "fail", + score: typeof s.score === "number" ? s.score : 0, + max_score: typeof s.max_score === "number" ? s.max_score : 0, + n_passed: typeof s.n_passed === "number" ? s.n_passed : 0, + n_total: typeof s.n_total === "number" ? s.n_total : 0, + ...(cfg.judge_model || s.judge_model ? { judge_model: String(cfg.judge_model || s.judge_model) } : {}), + }; + const triggerSide = + typeof cfg.trigger_ref_id === "string" && cfg.trigger_ref_id + ? { source_ref_id: cfg.trigger_ref_id } + : { source_type: "EvalTrigger", source_data: { id: trigger_id } }; + + const criterionSlots: Array<{ criterion_id: string; index: number }> = []; + const triplets: Array> = [ + { ...triggerSide, target_type: "EvalTriggerOutput", target_data: outputNodeData, edge_type: "HAS_OUTPUT" }, + ]; + for (const c of s.criteria_results as Array>) { + const cid = String(c?.criterion_id ?? c?.id ?? ""); + if (!cid) continue; + const critData = { + id: `${slug}-${runId}-${cid}`, + criterion_id: cid, + title: typeof c?.title === "string" ? c.title : "", + verdict: c?.verdict === "pass" ? "pass" : "fail", + reasoning: typeof c?.reasoning === "string" ? c.reasoning : "", + }; + criterionSlots.push({ criterion_id: cid, index: triplets.length }); + triplets.push({ + source_type: "EvalTriggerOutput", + source_data: outputNodeData, + target_type: "CriterionResult", + target_data: critData, + edge_type: "HAS_CRITERION_RESULT", + }); + triplets.push({ + source_type: "EvalRequirement", + source_data: { id: `${slug}-${cid}` }, + target_type: "CriterionResult", + target_data: critData, + edge_type: "HAS_CRITERION_RESULT", + }); + } + return { scored: true, triplets, criterionSlots, trigger_id, output_id }; + }, +}); diff --git a/mcp/src/lab/wfbench/steps/build-materials.ts b/mcp/src/lab/wfbench/steps/build-materials.ts new file mode 100644 index 000000000..ee29c646b --- /dev/null +++ b/mcp/src/lab/wfbench/steps/build-materials.ts @@ -0,0 +1,93 @@ +import { z, defineStep } from "vein"; + +/** + * 58313's wfbench_build_produced_materials.py: turn the produced artifacts + * into judge-ready materials. PRODUCED materials (what n_materials counts — + * zero means the judge has nothing to grade and the harness reports + * no_materials_produced instead of a fake 0/N): the workflow body and every + * custom step the author wrote. CONTEXT materials (always attached): the + * launch payload, the rerun's output + status, and the expected output when + * the task carries one. `materials_text` is the single markdown block the + * judge reads — the judge needs no tools. + */ +const clip = (s: string, max: number) => (s.length > max ? `${s.slice(0, max)}\n…[truncated ${s.length - max} chars]` : s); +const json = (v: unknown) => { + try { + return JSON.stringify(v, null, 2) ?? "null"; + } catch { + return String(v); + } +}; + +export default defineStep({ + type: "wfbench/build-materials", + description: + "Assemble judge materials from the produced workflow YAML, its custom step sources (meta/get-step outputs), the rerun output/status, the launch payload and the expected output. Output: { n_materials (produced only), materials: [{ type, name, content }], materials_text, task_desc, warnings }.", + input: z.object({ + workflow: z.string().describe("Produced workflow name."), + version: z.any().optional(), + workflow_yaml: z.string().describe("Produced workflow YAML ('' when nothing was published)."), + custom_steps: z.array(z.any()).optional().describe("meta/get-step outputs for the steps the author created."), + run_output: z.any().optional(), + execution_status: z.string().optional(), + project_id: z.any().optional(), + rerun_expected_output: z.any().optional(), + launch_payload: z.any().optional(), + instructions: z.string().describe("The task instructions (becomes task_desc)."), + maxChars: z.number().int().positive().default(60_000).describe("Per-material content cap."), + }), + output: z.any(), + async run(cfg) { + const warnings: string[] = []; + const produced: Array<{ type: string; name: string; content: string }> = []; + const context: Array<{ type: string; name: string; content: string }> = []; + + if (cfg.workflow_yaml.trim()) { + produced.push({ + type: "WORKFLOW", + name: `${cfg.workflow}${cfg.version ? `@${cfg.version}` : ""}`, + content: clip(cfg.workflow_yaml, cfg.maxChars), + }); + } else { + warnings.push("no workflow body"); + } + for (const s of cfg.custom_steps ?? []) { + const o = s && typeof s === "object" ? (s as Record) : {}; + const src = [o.source, o.code, o.sourceCode].find((v) => typeof v === "string" && v.trim()); + const name = String(o.type ?? o.name ?? "step"); + if (!src) { + warnings.push(`custom step ${name}: no source (${typeof o.error === "string" ? o.error : "unreadable"})`); + continue; + } + produced.push({ type: "STEP", name, content: clip(src, cfg.maxChars) }); + } + + context.push({ type: "LAUNCH_PAYLOAD", name: "workflow_input", content: json(cfg.launch_payload ?? {}) }); + context.push({ + type: "RUN_OUTPUT", + name: `run ${cfg.project_id ?? "none"} (${cfg.execution_status ?? "none"})`, + content: clip(json(cfg.run_output ?? null), cfg.maxChars), + }); + if (cfg.rerun_expected_output != null && cfg.rerun_expected_output !== "") { + context.push({ + type: "EXPECTED_OUTPUT", + name: "rerun_expected_output", + content: clip(typeof cfg.rerun_expected_output === "string" ? cfg.rerun_expected_output : json(cfg.rerun_expected_output), cfg.maxChars), + }); + } + + const materials = [...produced, ...context]; + const fence = (type: string) => (type === "WORKFLOW" ? "yaml" : type === "STEP" ? "ts" : "json"); + const materials_text = materials + .map((m) => `### ${m.type}: ${m.name}\n\n\`\`\`${fence(m.type)}\n${m.content}\n\`\`\``) + .join("\n\n"); + + return { + n_materials: produced.length, + materials, + materials_text, + task_desc: cfg.instructions, + warnings, + }; + }, +}); diff --git a/mcp/src/lab/wfbench/steps/build-roster.ts b/mcp/src/lab/wfbench/steps/build-roster.ts new file mode 100644 index 000000000..08b9e7ef6 --- /dev/null +++ b/mcp/src/lab/wfbench/steps/build-roster.ts @@ -0,0 +1,78 @@ +import { z, defineStep, type StepContext } from "vein"; + +/** + * The graph ROSTER for one benchmark run (stakwork 58313 steps 2–6), as + * write-ready payloads — the writes themselves are graph/create-node and + * graph/create-batch-triplet. Node ids follow 58313 exactly: + * + * EvalSet id = task_slug (55741) + * EvalRequirement id = - (58114, one per criterion, + * EvalSet -HAS_REQUIREMENT-> it) + * EvalTrigger id = - (55741; project_id = this runId) + * + * Properties are limited to what the jarvis ontology declares for each + * type (vein's graph backend rejects undeclared attributes): 58313's + * EvalSet.project_id (an int there) and EvalTrigger.name are therefore + * omitted; the trigger's `agent` carries the harness name so the node has + * a title. The EvalSet→EvalTrigger edge (HAS_TRIGGER vs + * HAS_BASELINE_TRIGGER) is decided later by wfbench/trigger-edge. + */ +export default defineStep({ + type: "wfbench/build-roster", + description: + "Build the eval roster payloads for one run with 58313's id conventions: EvalSet { id: task_slug }, EvalRequirement { id: - } triplets (HAS_REQUIREMENT, edge order), EvalTrigger { id: -, workflow_id, workflow_version_id, workflow_input, project_id }. Output: { run_id, evalset_id, trigger_id, evalset: { node_type, node_data }, requirement_triplets, requirement_ids, trigger: { node_type, node_data } }.", + input: z.object({ + task_slug: z.string().min(1), + task_title: z.string(), + instructions: z.string(), + criteria: z.array(z.any()).describe("wfbench/normalize-task's criteria."), + workflow: z.string().describe("The harness workflow name (recorded as EvalTrigger.workflow_id / agent)."), + workflow_version: z.any().optional().describe("The harness workflow's version (EvalTrigger.workflow_version_id)."), + }), + output: z.any(), + async run(cfg, ctx) { + const runId = (ctx as StepContext)?.runId || "no-run"; + const slug = cfg.task_slug; + const evalset = { node_type: "EvalSet", node_data: { id: slug, name: cfg.task_title } }; + + const requirement_triplets = cfg.criteria.map((c: any, i: number) => ({ + source_type: "EvalSet", + source_data: { id: slug }, + target_type: "EvalRequirement", + target_data: { + id: `${slug}-${c.id}`, + name: String(c.title ?? ""), + description: String(c.match_criteria ?? ""), + deliverables: Array.isArray(c.deliverables) ? c.deliverables : [], + }, + edge_type: "HAS_REQUIREMENT", + edge_data: { order: i }, + })); + + const trigger_id = `${slug}-${runId}`; + const trigger = { + node_type: "EvalTrigger", + node_data: { + id: trigger_id, + agent: cfg.workflow, + source: "vein", + environment: "vein-lab", + workflow_id: cfg.workflow, + workflow_version_id: cfg.workflow_version == null ? "" : String(cfg.workflow_version), + workflow_input: cfg.instructions, + project_id: runId, + run_count: 1, + }, + }; + + return { + run_id: runId, + evalset_id: slug, + trigger_id, + evalset, + requirement_triplets, + requirement_ids: requirement_triplets.map((t) => t.target_data.id), + trigger, + }; + }, +}); diff --git a/mcp/src/lab/wfbench/steps/check-input-keys.ts b/mcp/src/lab/wfbench/steps/check-input-keys.ts new file mode 100644 index 000000000..b8c7901af --- /dev/null +++ b/mcp/src/lab/wfbench/steps/check-input-keys.ts @@ -0,0 +1,67 @@ +import { z, defineStep } from "vein"; + +/** + * 58313's wfbench_check_input_keys.py, for vein: a vein workflow declares + * no input schema in YAML — its input contract is the set of `input.` + * references inside its {{ }} templates. Compare those against the task's + * workflow_input keys: + * missing = referenced by the workflow but absent from the payload → the + * run would resolve undefined → HARNESS ERROR, do not launch. + * unused = payload keys the workflow never reads → warning only. + * An empty body (nothing published) is also a harness error. + */ +const TEMPLATE_RE = /\{\{([\s\S]*?)\}\}/g; +const DOT_RE = /\binput\.([A-Za-z_$][\w$]*)/g; +const BRACKET_RE = /\binput\[\s*["']([^"']+)["']\s*\]/g; + +export function referencedInputKeys(yaml: string): string[] { + const keys = new Set(); + for (const m of yaml.matchAll(TEMPLATE_RE)) { + const body = m[1] ?? ""; + for (const k of body.matchAll(DOT_RE)) keys.add(k[1]!); + for (const k of body.matchAll(BRACKET_RE)) keys.add(k[1]!); + } + return [...keys].sort(); +} + +export default defineStep({ + type: "wfbench/check-input-keys", + description: + "Gate before launching the produced workflow: the `input.` references in its YAML templates vs the task's workflow_input keys. keys_match=false (harness error, no launch) when the body is empty or it references keys the payload lacks. Output: { keys_match, referenced_keys, payload_keys, missing, unused, launch_payload, error_type, error }.", + input: z.object({ + workflow_yaml: z.string().describe("The produced workflow's YAML ('' when nothing was published)."), + workflow_input: z.record(z.string(), z.any()).describe("The launch payload (normalize-task's workflow_input)."), + }), + output: z.any(), + async run(cfg) { + const payload_keys = Object.keys(cfg.workflow_input).sort(); + if (!cfg.workflow_yaml.trim()) { + return { + keys_match: false, + referenced_keys: [], + payload_keys, + missing: [], + unused: payload_keys, + launch_payload: cfg.workflow_input, + error_type: "no_workflow_produced", + error: "the author published no workflow body", + }; + } + const referenced_keys = referencedInputKeys(cfg.workflow_yaml); + const missing = referenced_keys.filter((k) => !payload_keys.includes(k)); + const unused = payload_keys.filter((k) => !referenced_keys.includes(k)); + const keys_match = missing.length === 0; + return { + keys_match, + referenced_keys, + payload_keys, + missing, + unused, + launch_payload: cfg.workflow_input, + error_type: keys_match ? null : "input_keys_mismatch", + error: keys_match + ? null + : `produced workflow reads input key(s) the task payload lacks: ${missing.join(", ")} (payload keys: ${payload_keys.join(", ") || "none"})`, + }; + }, +}); diff --git a/mcp/src/lab/wfbench/steps/classify-run.ts b/mcp/src/lab/wfbench/steps/classify-run.ts new file mode 100644 index 000000000..4d8d51eec --- /dev/null +++ b/mcp/src/lab/wfbench/steps/classify-run.ts @@ -0,0 +1,47 @@ +import { z, defineStep } from "vein"; + +/** + * 58313's wfbench_classify_run_result.py: three-way classification of the + * produced workflow's launch (meta/run-workflow's { runId, status, output, + * error } — or { error } when the meta surface refused, or nothing at all + * when the launch gate skipped it): + * launch_ok + completed → score it + * launch_ok + failed → still score (partial credit: the artifact exists) + * no runId → harness error, never score + */ +export default defineStep({ + type: "wfbench/classify-run", + description: + "Classify the produced workflow's run: { launch_ok, execution_status: completed|failed|none, project_id (the child runId), error_type, error, run_output }. No runId (skipped launch, refused, or crashed before starting) → launch_ok=false with the gate's error_type.", + input: z.object({ + run: z.any().optional().describe("meta/run-workflow output (undefined when the launch was skipped)."), + gate_error_type: z.any().optional().describe("Why the launch was skipped (check-input-keys' error_type)."), + gate_error: z.any().optional(), + }), + output: z.any(), + async run(cfg) { + const r = cfg.run && typeof cfg.run === "object" ? cfg.run : null; + const runId = typeof r?.runId === "string" && r.runId ? r.runId : null; + if (!runId) { + const refused = typeof r?.error === "string" ? r.error : typeof cfg.gate_error === "string" ? cfg.gate_error : null; + return { + launch_ok: false, + execution_status: "none", + project_id: null, + error_type: typeof r?.error === "string" ? "launch_refused" : (cfg.gate_error_type ?? "not_launched"), + error: refused ?? "the produced workflow was not launched", + run_output: null, + }; + } + const completed = r.status === "success"; + const err = r.error && typeof r.error === "object" ? r.error.message : r.error; + return { + launch_ok: true, + execution_status: completed ? "completed" : "failed", + project_id: runId, + error_type: completed ? null : "produced_workflow_failed", + error: completed ? null : (typeof err === "string" ? err : `run status ${String(r.status)}`), + run_output: r.output ?? null, + }; + }, +}); diff --git a/mcp/src/lab/wfbench/steps/normalize-task.ts b/mcp/src/lab/wfbench/steps/normalize-task.ts new file mode 100644 index 000000000..d636ba5e4 --- /dev/null +++ b/mcp/src/lab/wfbench/steps/normalize-task.ts @@ -0,0 +1,88 @@ +import { z, defineStep } from "vein"; + +/** + * Normalize one benchmark task (the Hive payload / stakwork 58313 set_var + * shape) into the harness's canonical form. Fails LOUD on a malformed task + * — bad criteria or an unparseable workflow_input_json is a harness error + * before any agent budget is spent, exactly where 58313's set_var would + * have choked. + * + * Output: { task_slug, task_title, instructions, criteria, n_criteria, + * workflow_input, workflow_input_keys, rerun_expected_output } + * criteria: [{ id, title, match_criteria, deliverables }] (rubric order kept) + */ +const slugify = (s: string) => + s + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, ""); + +const parseMaybeJson = (v: unknown): unknown => { + if (typeof v !== "string") return v; + const t = v.trim(); + if (!t) return undefined; + if (!/^[\[{]/.test(t)) return v; + try { + return JSON.parse(t); + } catch { + return v; + } +}; + +export default defineStep({ + type: "wfbench/normalize-task", + description: + "Normalize a benchmark task (task_slug, task_title, instructions, criteria as array or JSON string, workflow_input_json as object or JSON string, rerun_expected_output) into the harness shape. Throws on empty/invalid criteria or a non-object workflow input. Output: { task_slug, task_title, instructions, criteria: [{ id, title, match_criteria, deliverables }], n_criteria, workflow_input, workflow_input_keys, rerun_expected_output }.", + input: z.object({ + task_slug: z.string().min(1).describe("Task id — slugified; becomes the EvalSet id."), + task_title: z.string().optional().describe("Human title (defaults to the slug)."), + instructions: z.string().min(1).describe("The one-line (or longer) English task the author builds a workflow from."), + criteria: z.any().describe("Rubric: JSON array (or JSON string) of { id, title, match_criteria, deliverables? }."), + workflow_input_json: z + .any() + .optional() + .describe("The input the PRODUCED workflow is launched with — object or JSON string. Its keys are the input-key contract."), + rerun_expected_output: z.any().optional().describe("Optional expected output of the rerun, handed to the judge."), + }), + output: z.any(), + async run(cfg) { + const task_slug = slugify(cfg.task_slug); + if (!task_slug) throw new Error(`wfbench/normalize-task: task_slug "${cfg.task_slug}" slugifies to nothing`); + const task_title = (cfg.task_title ?? "").trim() || cfg.task_slug; + + const rawCriteria = parseMaybeJson(cfg.criteria); + if (!Array.isArray(rawCriteria) || rawCriteria.length === 0) { + throw new Error("wfbench/normalize-task: criteria must be a non-empty JSON array of rubric criteria"); + } + const seen = new Set(); + const criteria = rawCriteria.map((c: any, i: number) => { + const obj = c && typeof c === "object" ? c : {}; + let id = String(obj.id ?? obj.criterion_id ?? `c${i + 1}`); + if (seen.has(id)) id = `${id}-${i + 1}`; + seen.add(id); + return { + id, + title: String(obj.title ?? obj.name ?? id), + match_criteria: String(obj.match_criteria ?? obj.description ?? ""), + deliverables: Array.isArray(obj.deliverables) ? obj.deliverables : [], + }; + }); + + const wi = parseMaybeJson(cfg.workflow_input_json) ?? {}; + if (typeof wi !== "object" || wi === null || Array.isArray(wi)) { + throw new Error("wfbench/normalize-task: workflow_input_json must be a JSON object (the produced workflow's input)"); + } + const workflow_input = wi as Record; + + return { + task_slug, + task_title, + instructions: cfg.instructions, + criteria, + n_criteria: criteria.length, + workflow_input, + workflow_input_keys: Object.keys(workflow_input).sort(), + rerun_expected_output: parseMaybeJson(cfg.rerun_expected_output) ?? null, + }; + }, +}); diff --git a/mcp/src/lab/wfbench/steps/resolve-candidate.ts b/mcp/src/lab/wfbench/steps/resolve-candidate.ts new file mode 100644 index 000000000..4a7899599 --- /dev/null +++ b/mcp/src/lab/wfbench/steps/resolve-candidate.ts @@ -0,0 +1,54 @@ +import { z, defineStep } from "vein"; + +/** + * Resolve WHAT the author actually shipped — never trust its echo (the + * gaia-evolve-gen lesson: in schema mode a no-tool text turn ends the loop + * and IS the structured output, so "placeholder" versions happen). + * + * version = vpin (the echoed version, if it resolves) || vactive (the + * candidate's active version — the author's own last publish) + * published = a version resolved AND it differs from vbefore (the active + * version BEFORE the author ran) — at first run vbefore is + * { error } and any publish trips it. + * + * meta/get-workflow returns { error } instead of throwing, so a bad pin + * simply has no `version`. + */ +const ok = (w: any) => w && typeof w === "object" && !w.error && typeof w.version === "string" && w.version; + +export default defineStep({ + type: "wfbench/resolve-candidate", + description: + "Resolve the produced workflow from the author's structured output plus meta/get-workflow reads: version = vpin || vactive, published = version differs from vbefore. Output: { workflow, version, published, yaml, summary, changes, customSteps, missingSecrets, authorCost, authorSteps, authorError }.", + input: z.object({ + author: z.any().describe("The author agent step's output ({ object, cost, steps } or an onError pack { error })."), + candidate: z.string().describe("The harness-pinned candidate workflow name."), + vbefore: z.any().optional().describe("meta/get-workflow of the candidate BEFORE the author ran."), + vpin: z.any().optional().describe("meta/get-workflow of the candidate at the author's echoed version."), + vactive: z.any().optional().describe("meta/get-workflow of the candidate's active version AFTER the author ran."), + }), + output: z.any(), + async run(cfg) { + const a = cfg.author && typeof cfg.author === "object" ? cfg.author : {}; + const obj = a.object && typeof a.object === "object" ? a.object : {}; + const src = ok(cfg.vpin) ? cfg.vpin : ok(cfg.vactive) ? cfg.vactive : null; + const version: string | null = src ? src.version : null; + const before: string | null = ok(cfg.vbefore) ? cfg.vbefore.version : null; + const published = !!version && version !== before; + const strs = (v: unknown) => (Array.isArray(v) ? v.filter((x) => typeof x === "string") : []); + return { + workflow: cfg.candidate, + version, + published, + version_before: before, + yaml: published && typeof src?.yaml === "string" ? src.yaml : "", + summary: typeof obj.summary === "string" ? obj.summary : "", + changes: strs(obj.changes), + customSteps: strs(obj.customSteps), + missingSecrets: Array.isArray(obj.missingSecrets) ? obj.missingSecrets : [], + authorCost: typeof a.cost === "number" ? a.cost : null, + authorSteps: typeof a.steps === "number" ? a.steps : null, + authorError: typeof a.error === "string" ? a.error : null, + }; + }, +}); diff --git a/mcp/src/lab/wfbench/steps/trigger-edge.ts b/mcp/src/lab/wfbench/steps/trigger-edge.ts new file mode 100644 index 000000000..fb89f2ee7 --- /dev/null +++ b/mcp/src/lab/wfbench/steps/trigger-edge.ts @@ -0,0 +1,35 @@ +import { z, defineStep } from "vein"; + +/** + * 58313's guard_first_run: the first EvalTrigger ever linked to an EvalSet + * hangs off HAS_BASELINE_TRIGGER; every later one off HAS_TRIGGER. Input is + * graph/graph-neighbors' output for the EvalSet (edge_type filtered to the + * two trigger edges). This run's own trigger is excluded by ref_id in case + * it is already linked (a retried write). An unreadable hop (error string) + * fails toward HAS_TRIGGER — never claim a baseline on no evidence. + */ +export default defineStep({ + type: "wfbench/trigger-edge", + description: + "Decide the EvalSet→EvalTrigger edge: HAS_BASELINE_TRIGGER when the EvalSet has no prior EvalTrigger neighbors (excluding this run's trigger_ref_id), else HAS_TRIGGER; an unreadable hop yields HAS_TRIGGER. Output: { edge_type, is_baseline, prior_triggers, readable }.", + input: z.object({ + neighbors: z.any().describe("graph/graph-neighbors output for the EvalSet (array, or an error string)."), + trigger_ref_id: z.string().optional().describe("This run's EvalTrigger ref_id, excluded from the prior count."), + }), + output: z.any(), + async run(cfg) { + const readable = Array.isArray(cfg.neighbors); + const prior = readable + ? (cfg.neighbors as any[]).filter( + (n) => n && n.node_type === "EvalTrigger" && (!cfg.trigger_ref_id || n.ref_id !== cfg.trigger_ref_id), + ).length + : 0; + const is_baseline = readable && prior === 0; + return { + edge_type: is_baseline ? "HAS_BASELINE_TRIGGER" : "HAS_TRIGGER", + is_baseline, + prior_triggers: prior, + readable, + }; + }, +}); diff --git a/mcp/src/lab/wfbench/steps/webhook-body.ts b/mcp/src/lab/wfbench/steps/webhook-body.ts new file mode 100644 index 000000000..5d7b33848 --- /dev/null +++ b/mcp/src/lab/wfbench/steps/webhook-body.ts @@ -0,0 +1,65 @@ +import { z, defineStep } from "vein"; + +/** + * 58313's resolve_webhook_payload (4-way, priority order) as one pure step + * — and the workflow's own result is built from the SAME object, so what + * the run returns is what was POSTed (58313's set_output diverged on error + * paths). Byte-compatible with what Hive parses (RunnerScoreSchema): + * + * success: { task_slug, task_title, n_passed, n_total, all_pass, + * pass_rate, judge_model, criteria_results } + * failure: { task_slug, task_title, harness_error: true, error_type, error } + * error_type ∈ no_workflow_produced | input_keys_mismatch | + * launch_refused | not_launched | no_materials_produced | + * judge_failed + * (no score fields — Hive must never read a fake 0/N) + * + * Priority: launch/exec harness error → judge error → success → no materials. + * A produced workflow that FAILED at runtime is not a harness error — it is + * still judged (partial credit), exactly as 58313 classifies it. + */ +export default defineStep({ + type: "wfbench/webhook-body", + description: + "Resolve the single Hive callback body from the harness branches: launch gate (check-input-keys) → run classification → materials → judge scores. Success: { task_slug, task_title, n_passed, n_total, all_pass, pass_rate, judge_model, criteria_results }; failure: { task_slug, task_title, harness_error: true, error_type, error }.", + input: z.object({ + task_slug: z.string(), + task_title: z.string(), + judge_model: z.string().optional(), + keys: z.any().optional().describe("wfbench/check-input-keys output."), + cls: z.any().optional().describe("wfbench/classify-run output."), + mats: z.any().optional().describe("wfbench/build-materials output."), + scores: z.any().optional().describe("eval/aggregate-scores output (absent when the judge never ran)."), + }), + output: z.any(), + async run(cfg) { + const base = { task_slug: cfg.task_slug, task_title: cfg.task_title }; + const fail = (error_type: string, error: unknown) => ({ + ...base, + harness_error: true, + error_type, + error: typeof error === "string" && error ? error : error_type, + }); + const keys = cfg.keys && typeof cfg.keys === "object" ? cfg.keys : {}; + const cls = cfg.cls && typeof cfg.cls === "object" ? cfg.cls : {}; + const mats = cfg.mats && typeof cfg.mats === "object" ? cfg.mats : {}; + const scores = cfg.scores && typeof cfg.scores === "object" ? cfg.scores : null; + + if (keys.keys_match === false) return fail(keys.error_type ?? "input_keys_mismatch", keys.error); + if (cls.launch_ok === false) return fail(cls.error_type ?? "not_launched", cls.error); + const n_materials = typeof mats.n_materials === "number" ? mats.n_materials : 0; + if (n_materials === 0) return fail("no_materials_produced", "the produced artifact resolved to no judge materials"); + if (!scores || scores.error || typeof scores.all_pass !== "boolean" || !Array.isArray(scores.criteria_results)) { + return fail("judge_failed", scores?.error ?? "the judge produced no valid score"); + } + return { + ...base, + n_passed: scores.n_passed, + n_total: scores.n_total, + all_pass: scores.all_pass, + pass_rate: scores.pass_rate, + judge_model: cfg.judge_model ?? scores.judge_model ?? null, + criteria_results: scores.criteria_results, + }; + }, +}); diff --git a/mcp/src/lab/wfbench/workflows/wfbench-judge-criterion.yaml b/mcp/src/lab/wfbench/workflows/wfbench-judge-criterion.yaml new file mode 100644 index 000000000..7aca87fd6 --- /dev/null +++ b/mcp/src/lab/wfbench/workflows/wfbench-judge-criterion.yaml @@ -0,0 +1,79 @@ +name: wfbench-judge-criterion +# LLM-as-judge for ONE rubric criterion (the harvey_lab_score_rubric shape +# 58313 calls once for all criteria — here one call per criterion, run by +# wfbench-run's foreach in rubric order; eval/aggregate-scores zips the +# verdicts back onto the rubric by position). This subflow exists because +# foreach bodies get no onError of their own: a judge crash returns +# { error } and is scored as an honest FAIL, never a pass. +# +# The judge is the core agent step in SCHEMA mode with NO useful tools — +# every material is already inlined in the prompt by wfbench/build-materials +# (the editor tool is the smallest built-in subset; its cwd is empty). +# +# Input: { criterion: { id, title, match_criteria }, task_desc, materials_text } +# Output: agent schema-mode result { object: { verdict, reasoning }, cost, +# usage, steps } — or { error } from the onError pack. +steps: + - id: dir + type: artifacts/dir + config: + sub: judge + + - id: judge + type: agent + depends: dir + options: + retry: { max: 1, delayMs: 10000 } + onError: + id: judge_failed + type: pack + config: + error: "{{ $error.cause ? $error.message + ' (cause: ' + $error.cause + ')' : $error.message }}" + config: + cwd: "{{ dir.path }}" + system: "{{ params.judgeSystem }}" + prompt: | + {{ params.judgePrompt }} + + ## Task the workflow author was given + {{ input.task_desc }} + + ## Materials (the produced workflow, its custom steps, the launch + ## payload, the rerun's output and status, and the expected output + ## when the task has one) + {{ input.materials_text }} + + ## Criterion + **{{ input.criterion.title }}** + + {{ input.criterion.match_criteria }} + model: "{{ params.judgeModel }}" + maxSteps: "{{ params.judgeMaxSteps }}" + toolFilter: ["str_replace_based_edit_tool"] + schema: + type: object + properties: + verdict: + type: string + enum: [pass, fail] + reasoning: + type: string + description: Brief explanation of the verdict, grounded in the materials. + required: [verdict, reasoning] + additionalProperties: false + +params: + judgeModel: claude-sonnet-5 + judgeMaxSteps: 5 + judgeSystem: | + You are evaluating a workflow that an AI workflow-editor agent authored + from English instructions, against ONE specific quality criterion. Read + every material fully before judging: the workflow YAML, any custom step + source, the input it was launched with, what the run produced (and + whether it completed), and the expected output when one is given. PASS + means the materials satisfy the criterion as described; FAIL means they + do not. Judge only the stated criterion — not general quality — and + ground your reasoning in what the materials actually contain. Answer + directly with the structured verdict; you have no files to read. + judgePrompt: | + Evaluate the produced workflow and its rerun against the criterion below. diff --git a/mcp/src/lab/wfbench/workflows/wfbench-run.yaml b/mcp/src/lab/wfbench/workflows/wfbench-run.yaml new file mode 100644 index 000000000..9aed2f0a8 --- /dev/null +++ b/mcp/src/lab/wfbench/workflows/wfbench-run.yaml @@ -0,0 +1,510 @@ +name: wfbench-run +# The Workflow Editor Agent Benchmark — Task Runner (the vein twin of +# stakwork workflow 58313 + its nested 54419 / 55741 / 55740 / 58114 / 58414 +# / 57425 / 58312). One task in, one Hive callback out: +# +# roster (graph) EvalSet → EvalRequirement×N, EvalTrigger, HAS_TRIGGER | +# HAS_BASELINE_TRIGGER — 58313's ids, on graph/* steps +# author the meta/* authoring agent (54419's twin) builds a NEW +# workflow named wfbench- from the instructions +# resolve pin the version it actually shipped (never its echo) +# gate + rerun input-key contract (check-input-keys) → meta/run-workflow +# judge per-criterion LLM judge (wfbench-judge-criterion) → +# eval/aggregate-scores; a runtime-failed rerun is still +# judged, a harness error never is +# record EvalTriggerOutput + CriterionResult per 58312 +# callback one body (wfbench/webhook-body) POSTed to webhook_url AND +# returned as this run's output (no set_output divergence) +# +# Grant discipline: the author gets meta/* + the editor tool ONLY (never +# bash, never graph/*, never wfbench/*); the judge gets nothing useful; the +# produced workflow runs via meta/run-workflow, so it is necessarily +# publisher "ai" — the seeded harness surface can never be graded. +# +# Input: { task_slug, task_title?, instructions, criteria, workflow_input_json?, +# rerun_expected_output?, webhook_url? } +# criteria: [{ id, title, match_criteria, deliverables? }] (or a JSON string) +# workflow_input_json: the payload the produced workflow is launched with +# Output: the callback body + diagnostics (see the `result` step). +steps: + # ── task ───────────────────────────────────────────────────────────── + - id: task + type: wfbench/normalize-task + config: + task_slug: "{{ input.task_slug }}" + task_title: "{{ input.task_title }}" + instructions: "{{ input.instructions }}" + criteria: "{{ input.criteria }}" + workflow_input_json: "{{ input.workflow_input_json }}" + rerun_expected_output: "{{ input.rerun_expected_output }}" + + # This harness's own version → EvalTrigger.workflow_version_id (58313 used + # the engine's @WORKFLOW_VERSION magic var). Reads are not ownership-gated. + - id: self + type: meta/get-workflow + depends: [] + config: + name: wfbench-run + + - id: roster + type: wfbench/build-roster + depends: [task, self] + config: + task_slug: "{{ task.task_slug }}" + task_title: "{{ task.task_title }}" + instructions: "{{ task.instructions }}" + criteria: "{{ task.criteria }}" + workflow: wfbench-run + workflow_version: "{{ self.version }}" + + # ── roster: graph writes (58313 steps 2–9), in parallel with the author ── + - id: ns + type: graph/register-namespace + depends: roster + config: + namespace: "{{ params.namespace }}" + + - id: evalset + type: graph/create-node + depends: ns + config: + node_type: EvalSet + node_data: "{{ roster.evalset.node_data }}" + namespace: "{{ params.namespace }}" + + # One EvalRequirement per criterion (58114), EvalSet -HAS_REQUIREMENT-> it. + # Inline EvalSet source merges onto the node above (same node_key). + - id: reqs + type: graph/create-batch-triplet + depends: evalset + config: + triplets: "{{ roster.requirement_triplets }}" + namespace: "{{ params.namespace }}" + + # After the requirements (58313's order) so every roster write is upstream + # of the record + result steps. + - id: trig + type: graph/create-node + depends: reqs + config: + node_type: EvalTrigger + node_data: "{{ roster.trigger.node_data }}" + namespace: "{{ params.namespace }}" + + # hop_check_trigger_exists + guard_first_run: prior triggers ⇒ HAS_TRIGGER, + # none ⇒ HAS_BASELINE_TRIGGER. + - id: hop + type: graph/graph-neighbors + depends: trig + config: + ref_id: "{{ evalset.ref_id }}" + edge_type: [HAS_TRIGGER, HAS_BASELINE_TRIGGER] + node_type: [EvalTrigger] + namespace: "{{ params.namespace }}" + + - id: edge + type: wfbench/trigger-edge + depends: hop + config: + neighbors: "{{ hop }}" + trigger_ref_id: "{{ trig.ref_id }}" + + - id: link + type: graph/create-triplet + depends: edge + config: + source_ref_id: "{{ evalset.ref_id }}" + target_ref_id: "{{ trig.ref_id }}" + edge_type: "{{ edge.edge_type }}" + namespace: "{{ params.namespace }}" + + # ── author (54419's twin) ───────────────────────────────────────────── + - id: dir + type: artifacts/dir + depends: task + config: + sub: author + + # The candidate's active version BEFORE the author runs — the `published` + # gate compares against it (an author can burn its budget and ship + # nothing; the fallback below would then resolve to this very version). + - id: vbefore + type: meta/get-workflow + depends: task + config: + name: "wfbench-{{ task.task_slug }}" + + - id: author + type: agent + depends: [dir, vbefore] + options: + retry: { max: 1, delayMs: 15000 } + onError: + id: author_failed + type: pack + config: + error: "{{ $error.cause ? $error.message + ' (cause: ' + $error.cause + ')' : $error.message }}" + config: + cwd: "{{ dir.path }}" + system: "{{ params.authorSystem }}" + prompt: | + TASK: + {{ task.instructions }} + + Build a NEW vein workflow that does exactly this, and publish it under + the EXACT name "wfbench-{{ task.task_slug }}" (meta/publish-workflow; + republishing the same name creates a new version — iterate that way). + + INPUT CONTRACT: the harness will launch your workflow ONCE with this + input object — reference ONLY these keys (as input. templates): + {{ task.workflow_input }} + Reading any other input key is a harness error and your workflow will + not be launched or graded. + + {{ params.authorGuidance }} + + When done, return the workflow name, the EXACT version string of your + final publish, a summary of the design, the list of custom step types + you created (if any), and any secrets you needed that do not exist. + model: "{{ params.authorModel }}" + maxSteps: "{{ params.authorMaxSteps }}" + # meta/* + the editor tool only (EVOLVE_SPEC §5.3.2): no bash, no + # graders, no graph. The editor lets it draft YAML in its scratch dir. + toolFilter: ["str_replace_based_edit_tool"] + agentTools: ["meta/*"] + schema: + type: object + properties: + workflow: + type: string + description: "the published workflow name (must be the exact name you were given)" + version: + type: string + description: "EXACT version string of the final healthy publish (from meta/publish-workflow)" + summary: + type: string + description: "how the workflow is built and why — steps, data flow, what it produces" + changes: + type: array + items: { type: string } + description: "one line per notable design decision" + customSteps: + type: array + items: { type: string } + description: "step TYPES you created with meta/create-step (e.g. 'cand/parse-feed'); [] if none" + missingSecrets: + type: array + description: "credentials the workflow needs that meta/list-secrets does not show" + items: + type: object + properties: + name: { type: string } + why: { type: string } + required: [name, why] + additionalProperties: false + required: [workflow, version, summary] + additionalProperties: false + + # ── resolve what actually shipped (never trust the echo) ───────────── + - id: vactive + type: meta/get-workflow + depends: author + config: + name: "wfbench-{{ task.task_slug }}" + + - id: vpin + type: meta/get-workflow + depends: author + config: + name: "wfbench-{{ task.task_slug }}" + version: "{{ author.object.version }}" + + - id: cand + type: wfbench/resolve-candidate + depends: [vpin, vactive, vbefore] + config: + author: "{{ author }}" + candidate: "wfbench-{{ task.task_slug }}" + vbefore: "{{ vbefore }}" + vpin: "{{ vpin }}" + vactive: "{{ vactive }}" + + # Sources of the custom steps the author says it created — judge materials. + - id: stepsrc + type: foreach + depends: cand + config: + items: "{{ cand.customSteps }}" + concurrency: 3 + body: + id: one + type: meta/get-step + config: + type: "{{ $current }}" + + # ── gate + rerun (57425's twin) ─────────────────────────────────────── + - id: keys + type: wfbench/check-input-keys + depends: cand + config: + workflow_yaml: "{{ cand.yaml }}" + workflow_input: "{{ task.workflow_input }}" + + - id: launchable + type: if + depends: keys + config: + cond: "{{ keys.keys_match && cand.published }}" + + # Runs the candidate as its OWN run (own runId = 58313's project_id), under + # this run's controller. No webhook wait: meta/run-workflow awaits. + - id: run + type: meta/run-workflow + depends: launchable + when: true + config: + name: "{{ cand.workflow }}" + version: "{{ cand.version }}" + input: "{{ keys.launch_payload }}" + + # Depends on the gate too so it runs on BOTH branches (`run` is skipped on + # the false branch and resolves undefined here). + - id: cls + type: wfbench/classify-run + depends: [run, launchable] + config: + run: "{{ run }}" + gate_error_type: "{{ keys.error_type }}" + gate_error: "{{ keys.error }}" + + # ── judge (harvey_lab_score_rubric's twin) ──────────────────────────── + - id: mats + type: wfbench/build-materials + depends: [cls, stepsrc] + config: + workflow: "{{ cand.workflow }}" + version: "{{ cand.version }}" + workflow_yaml: "{{ cand.yaml }}" + custom_steps: "{{ stepsrc }}" + run_output: "{{ cls.run_output }}" + execution_status: "{{ cls.execution_status }}" + project_id: "{{ cls.project_id }}" + rerun_expected_output: "{{ task.rerun_expected_output }}" + launch_payload: "{{ keys.launch_payload }}" + instructions: "{{ task.instructions }}" + + # guard_materials_present + if_launch_ok: judge only a launched run with + # at least one produced material (a runtime-FAILED rerun is still judged). + - id: judgeable + type: if + depends: mats + config: + cond: "{{ cls.launch_ok && mats.n_materials > 0 }}" + + - id: judged + type: foreach + depends: judgeable + when: true + config: + items: "{{ task.criteria }}" + concurrency: "{{ params.judgeConcurrency }}" + body: + id: one + type: subflow + config: + workflow: wfbench-judge-criterion + input: + criterion: "{{ $current }}" + task_desc: "{{ mats.task_desc }}" + materials_text: "{{ mats.materials_text }}" + + # guard_judge_ran / guard_valid_score: a zip refusal packs { error } and + # the callback reports judge_failed instead of a fake score. + - id: scores + type: eval/aggregate-scores + depends: judged + options: + onError: + id: scores_failed + type: pack + config: + error: "{{ $error.message }}" + config: + rubric: "{{ task.criteria }}" + results: "{{ judged }}" + judge_model: "{{ params.judgeModel }}" + + # ── record (58312's twin) ───────────────────────────────────────────── + - id: chain + type: wfbench/build-eval-output + depends: [scores, link] + config: + task_slug: "{{ task.task_slug }}" + scores: "{{ scores }}" + trigger_ref_id: "{{ trig.ref_id }}" + trigger_id: "{{ roster.trigger_id }}" + judge_model: "{{ params.judgeModel }}" + + - id: recordable + type: if + depends: chain + config: + cond: "{{ chain.scored }}" + + - id: record + type: graph/create-batch-triplet + depends: recordable + when: true + config: + triplets: "{{ chain.triplets }}" + namespace: "{{ params.namespace }}" + + - id: critrefs + type: eval/criterion-refs + depends: [record, recordable] + config: + slots: "{{ chain.criterionSlots }}" + record: "{{ record }}" + + # ── callback (resolve_webhook_payload + post_result) ────────────────── + - id: body + type: wfbench/webhook-body + depends: [scores, judgeable, cls, keys] + config: + task_slug: "{{ task.task_slug }}" + task_title: "{{ task.task_title }}" + judge_model: "{{ params.judgeModel }}" + keys: "{{ keys }}" + cls: "{{ cls }}" + mats: "{{ mats }}" + scores: "{{ scores }}" + + - id: have_webhook + type: if + depends: body + config: + cond: "{{ input.webhook_url }}" + + - id: post + type: http + depends: have_webhook + when: true + options: + onError: + id: post_failed + type: pack + config: + error: "{{ $error.cause ? $error.message + ' (cause: ' + $error.cause + ')' : $error.message }}" + config: + url: "{{ input.webhook_url }}" + method: POST + headers: { "content-type": "application/json" } + body: "{{ body }}" + + # The run's output IS the callback body, plus diagnostics for humans. + - id: result + type: pack + depends: [post, have_webhook, body, critrefs] + config: + callback: "{{ body }}" + posted: "{{ input.webhook_url ? true : false }}" + post: "{{ post }}" + produced: "{{ cand }}" + launch: "{{ keys }}" + run: "{{ cls }}" + n_materials: "{{ mats.n_materials }}" + material_warnings: "{{ mats.warnings }}" + graph: + evalset_ref_id: "{{ evalset.ref_id }}" + trigger_ref_id: "{{ trig.ref_id }}" + trigger_id: "{{ roster.trigger_id }}" + trigger_edge: "{{ edge.edge_type }}" + requirements: "{{ reqs }}" + output_id: "{{ chain.output_id }}" + record: "{{ record }}" + criterion_refs: "{{ critrefs }}" + +params: + # Graph partition every roster/record write lands in (registered on each run). + namespace: wfbench + authorModel: claude-sonnet-5 + # An author that reads step docs, drafts, publishes, smoke-tests and fixes + # burns calls fast; the publish-early rule below protects the budget. + authorMaxSteps: 150 + # Extra per-run steering appended to the author's task prompt. + authorGuidance: "" + judgeModel: claude-sonnet-5 + judgeConcurrency: 4 + # The author's persona + method (layer-1 tunable). The translation + # cheat-sheet at the end is what makes this harness usable for PORTING + # stakwork workflows: hand the stakwork body as the instructions. + authorSystem: | + You are a WORKFLOW EDITOR agent inside a benchmark harness. You build a + NEW vein workflow from English instructions. After you finish, the + harness launches your workflow once with a fixed input object and an + LLM judge grades the workflow body, your custom steps, and the run's + output against a rubric you cannot see. You never run the task + yourself — you build the workflow that does. + + Your only levers are the meta/* authoring tools (the workspace surface): + meta/list-steps, meta/search-steps, meta/get-step to discover step + types and read their input schemas; meta/list-workflows and + meta/get-workflow to read existing YAML for patterns; meta/create-step / + meta/edit-step to author custom steps (TypeScript: `import { z, + defineStep } from "vein"`, export default defineStep({ type, description, + input, output, run })); meta/run-step to test ONE step cheaply; + meta/validate-workflow to static-check YAML BEFORE publishing; + meta/publish-workflow to publish; meta/run-workflow to run it; + meta/list-runs, meta/get-run, meta/search-runs to inspect runs; + meta/list-secrets for available credential NAMES. + + METHOD + 1. Read the task. List the steps that exist (meta/list-steps) and read + the ones you will use (meta/get-step) — never guess a config shape. + 2. Draft the YAML. Every template is quoted: `url: "{{ input.url }}"` + (an unquoted `{{` is a YAML mapping and silently corrupts the value). + Reference only the input keys you were given, as {{ input. }}. + Put prompts and thresholds in `params:`; keep `steps:` sequential + unless `depends:` parallelism is clearly useful. + 3. VALIDATE (meta/validate-workflow), fix every error, then PUBLISH + EARLY (meta/publish-workflow) under the exact name you were given. + A rough published workflow beats a perfect unpublished one: if you + run out of steps with nothing published, this task scores nothing. + 4. Smoke-test with meta/run-workflow using the input you were given, + inspect with meta/get-run, fix and republish. Runs cost real money: + at most three smoke runs. + 5. BUDGET YOUR CALLS. You have a hard tool-call cap. Reserve the LAST + ~5 calls for: a final meta/publish-workflow, one verification read + (meta/get-workflow), and your structured answer. + + HARD CONTRACT (the harness runs it as-is): + - publish under the EXACT workflow name you were given; the version you + return must be a real version string from meta/publish-workflow. + - the workflow's output is its LAST step's output — make that the + task's deliverable (structured, not prose). + - secrets: reference NAMES from meta/list-secrets only (steps read them + via ctx.services.secrets); never paste credential values. A missing + credential goes in missingSecrets and the workflow degrades gracefully. + - no `agent` step may be granted meta/*, eval/*, graph/* or wfbench/* + tools; the authoring surface and graders are off-limits to producers. + + PORTING A STAKWORK WORKFLOW (when the instructions contain a stakwork + workflow body): reproduce its behavior, step for step where it matters, + with this translation table — + set_var / SetVar → `input` keys (the launch payload) + `params:` + JSONBuilder / JSONPathParser + / IfElseValue → {{ }} template expressions on step outputs + Request → `http` step (url, method, headers, body) + IfElseCondition + goTo → `if` step + `when: true|false` on branches + WhileLoop → `loop` step (`until`, `maxIterations`) + ForEachCondition → `foreach` step (`items`, `body`, `concurrency`) + WorkflowRunner (nested wf) → `subflow` (publish the nested workflow first) + Script (python) → a custom step via meta/create-step + Skill / LLM / agent steps → `llm` (structured output) or `agent` step + [#(step).output.x] / $(item) → {{ step.x }} / {{ $current }} + %%SECRET%% → a secret NAME via ctx.services.secrets + Keep the produced output shape identical to the original's set_output. + + Your structured answer is harvested from your FINAL message — the turn + where you stop calling tools. Fill every field with REAL values: the + exact name, the exact version, a truthful summary of what you shipped. diff --git a/vein/AGENTS.md b/vein/AGENTS.md index 324c8180a..9faafc994 100644 --- a/vein/AGENTS.md +++ b/vein/AGENTS.md @@ -46,7 +46,7 @@ vein/ │ ├── secret-store.ts # SecretStore iface + FileSecretStore (AES-256-GCM, VEIN_SECRET_KEY) + MemorySecretStore — backs ctx.services.secrets + /secrets endpoints │ ├── index.ts # barrel export — createVein (primary entry), createRegistry, coreRegistry, all types │ ├── steps/ -│ │ ├── core/ # 9 built-in steps: http, log, if, loop, foreach, subflow, llm, agent, wait (static import) +│ │ ├── core/ # 10 built-in steps: http, log, if, loop, foreach, subflow, llm, agent, wait, pack (static import) │ │ ├── lib/ # built-in domain integrations (github/fetch-pr, ...) — file dynamic-imported at build; heavy SDKs lazy-imported in run() (see "Lib step dependency convention") │ │ │ └── graph/ # graph/* knowledge-graph steps over src/graph (the vein-native twins of the mcp lab's jarvis/* steps — same names, inputs, outputs — plus two vein-only ones: create-schema registers/extends a node type, edit-edge patches an edge's properties); _shared.ts lazy-imports the backend; graph-steps.test.ts is a live end-to-end test │ │ └── registry.ts # auto-discovery: buildRegistry() core (static) + lib (dynamic) + workspace custom/ (dynamic); createRegistry() for in-code steps diff --git a/vein/SPEC.md b/vein/SPEC.md index 7d824291b..710e77f42 100644 --- a/vein/SPEC.md +++ b/vein/SPEC.md @@ -263,6 +263,23 @@ Config: - Calls an LLM with the resolved prompt. If `schema` is provided, uses structured output (e.g. `generateObject`). - Output: `{ text: string }` (no schema) or the parsed object (with schema). +#### 4.1.7 `pack` + +Config: any object. Output: that object, template-resolved. The workflow's +output is its last step's output, so `pack` is how a workflow returns fields +from several steps — and how an `onError` fallback packs an explicit failure +shape instead of killing the run. + +```yaml +- id: result + type: pack + depends: [fetch, score] + config: + repo: "{{ input.repo }}" + stars: "{{ fetch.body.stargazers_count }}" + verdict: "{{ score.all_pass ? 'pass' : 'fail' }}" +``` + ### 4.2 Lib Steps Lib steps live in `src/steps/lib//` inside the engine itself. They are reusable domain-specific integrations (GitHub, Neo4j, Slack, …) that ship with vein but are kept out of the static dependency graph. Each file is a `defineStep(...)` export, same as core steps. diff --git a/vein/package.json b/vein/package.json index a1581f3b5..ab2942633 100644 --- a/vein/package.json +++ b/vein/package.json @@ -22,7 +22,7 @@ "build:web": "npm --prefix web run build", "dev": "npm run build:web && tsx --env-file=.env src/server.ts", "start": "node build/server.js", - "test": "tsx --test src/expr.test.ts src/core.test.ts src/runner.test.ts src/run-control.test.ts src/control-flow.test.ts src/store.test.ts src/workspace.test.ts src/storage-conformance.test.ts src/integration.test.ts src/services.test.ts src/cassette.test.ts src/run-step.test.ts src/createVein.test.ts src/authoring.test.ts src/ai-integration.test.ts src/chat-store.test.ts src/chat-endpoints.test.ts src/steps/registry.test.ts src/steps/core/agent.test.ts src/auth.test.ts src/secret-store.test.ts src/artifacts.test.ts src/slack.test.ts src/gdrive.test.ts src/html-extract.test.ts src/shell.test.ts src/validate.test.ts", + "test": "tsx --test src/expr.test.ts src/core.test.ts src/runner.test.ts src/run-control.test.ts src/control-flow.test.ts src/store.test.ts src/workspace.test.ts src/storage-conformance.test.ts src/integration.test.ts src/services.test.ts src/cassette.test.ts src/run-step.test.ts src/createVein.test.ts src/authoring.test.ts src/ai-integration.test.ts src/chat-store.test.ts src/chat-endpoints.test.ts src/steps/registry.test.ts src/steps/core/agent.test.ts src/steps/core/pack.test.ts src/auth.test.ts src/secret-store.test.ts src/artifacts.test.ts src/slack.test.ts src/gdrive.test.ts src/html-extract.test.ts src/shell.test.ts src/validate.test.ts", "test:graph": "tsx --test --test-concurrency=1 \"src/graph/*.test.ts\" \"src/steps/lib/graph/*.test.ts\"" }, "dependencies": { diff --git a/vein/src/steps/core/pack.test.ts b/vein/src/steps/core/pack.test.ts new file mode 100644 index 000000000..847026312 --- /dev/null +++ b/vein/src/steps/core/pack.test.ts @@ -0,0 +1,35 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { z } from "zod"; +import { flow, step } from "../../core.js"; +import { runWorkflow } from "../../runner.js"; +import { coreRegistry } from "../registry.js"; +import { MemoryRunStore } from "../../store.js"; + +describe("pack (core)", () => { + it("returns the resolved config as the step output — the workflow's return value", async () => { + const wf = flow("pack-test", { + input: z.object({ repo: z.string() }), + steps: [ + step("a", "log", { message: "one" }), + step("b", "log", { message: "two" }, { depends: [] }), + step("result", "pack", { + repo: "{{ input.repo }}", + first: "{{ a }}", + second: "{{ b }}", + n: "{{ 1 + 2 }}", + nested: { ok: true, list: ["{{ input.repo }}", "x"] }, + }, { depends: ["a", "b"] }), + ], + }); + const result = await runWorkflow(wf, { repo: "stakgraph" }, coreRegistry(), { store: new MemoryRunStore() }); + assert.equal(result.status, "success", JSON.stringify(result.error)); + assert.deepEqual(result.output, { + repo: "stakgraph", + first: "one", + second: "two", + n: 3, + nested: { ok: true, list: ["stakgraph", "x"] }, + }); + }); +}); diff --git a/vein/src/steps/core/pack.ts b/vein/src/steps/core/pack.ts new file mode 100644 index 000000000..55b1ef177 --- /dev/null +++ b/vein/src/steps/core/pack.ts @@ -0,0 +1,28 @@ +import { z } from "zod"; +import { defineStep } from "../../core.js"; + +const EXAMPLE = `- id: result + type: pack + depends: [fetch, score] + config: + repo: "{{ input.repo }}" + stars: "{{ fetch.body.stargazers_count }}" + verdict: "{{ score.all_pass ? 'pass' : 'fail' }}"`; + +/** + * Assemble an object from other steps' outputs. A workflow's output is its + * LAST step's output, so every workflow that must return more than one + * step's result needs a step whose only job is to pack fields together — + * and an `onError` fallback that packs an explicit failure shape instead + * of killing the run wants the same primitive. The config IS the output: + * every field is template-resolved by the runner before `run` sees it. + */ +export default defineStep({ + type: "pack", + description: `Assemble an object from templates: the resolved config object IS the output. Use it as a workflow's last step to return fields from several earlier steps, or as an onError fallback that packs an explicit failure shape. Output: the config object.\n\n${EXAMPLE}`, + input: z.record(z.string(), z.any()), + output: z.any(), + async run(cfg) { + return cfg; + }, +}); diff --git a/vein/src/steps/registry.ts b/vein/src/steps/registry.ts index e90f2caba..6ecbf2eaa 100644 --- a/vein/src/steps/registry.ts +++ b/vein/src/steps/registry.ts @@ -27,6 +27,7 @@ import log from "./core/log.js"; import llm from "./core/llm.js"; import agent from "./core/agent.js"; import wait from "./core/wait.js"; +import pack from "./core/pack.js"; const CORE_STEPS: StepRegistry = { http, @@ -38,6 +39,7 @@ const CORE_STEPS: StepRegistry = { llm, agent, wait, + pack, }; export const CORE_STEP_TYPES = Object.freeze(Object.keys(CORE_STEPS));