Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions mcp/src/lab/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,11 @@ workflows/steps inside it — **not** separate servers. Adding an experiment

- `createLabVein.ts` — the single instance: registry (vein core+lib + all
experiment steps), merged `services` bag, seeded workflow templates.
**Seeding is additive**: dropping a step from a seeder's `SEED_STEPS` does
NOT remove it from existing workspaces (graph-backed ones persist, and
the author agent keeps discovering it). When you remove or rename a
seeded step, add the old type to that seeder's `RETIRED_STEPS` list —
`retireSteps` (`seed-opts.ts`) soft-deletes it at boot.
- `mount.ts` — bridges the vein (Hono) app into Express under `/lab`
(API + run-streaming SSE). Registered before `express.json()` to keep
raw request streams. Lazy-initialized so mcp boot isn't coupled to
Expand Down
6 changes: 5 additions & 1 deletion mcp/src/lab/gaia/seed.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { readFile } from "node:fs/promises";
import { fileURLToPath } from "node:url";
import { dirname, join } from "node:path";
import type { WorkspaceStore } from "vein";
import { SEED_OPTS } from "../seed-opts.js";
import { SEED_OPTS, retireSteps } from "../seed-opts.js";

/**
* GAIA LAB steps + workflows — the harness that scored 5/5 on the first
Expand Down Expand Up @@ -40,6 +40,9 @@ const SEED_STEPS: Array<{ file: string; type: string }> = [
{ file: "digest-results.ts", type: "gaia/digest-results" },
];

// Types this seeder USED to publish (seeding is additive — see retireSteps).
const RETIRED_STEPS = ["gaia/pack-result"]; // → vein core `pack`

const HERE = dirname(fileURLToPath(import.meta.url));

const SEED_WORKFLOWS: Array<{ name: string; description: string }> = [
Expand Down Expand Up @@ -102,4 +105,5 @@ export async function seedGaiaSteps(workspace: WorkspaceStore): Promise<void> {
);
}
}
await retireSteps(workspace, RETIRED_STEPS, "gaia");
}
12 changes: 11 additions & 1 deletion mcp/src/lab/harvey/seed.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { readFile } from "node:fs/promises";
import { fileURLToPath } from "node:url";
import { dirname, join } from "node:path";
import type { WorkspaceStore } from "vein";
import { SEED_OPTS } from "../seed-opts.js";
import { SEED_OPTS, retireSteps } from "../seed-opts.js";

/**
* Harvey LAB verification steps — THIN plumbing only. The actual grader is
Expand Down Expand Up @@ -45,6 +45,15 @@ const SEED_STEPS: Array<{ file: string; type: string }> = [
{ file: "generate-xlsx.ts", type: "harvey/generate-xlsx" },
];

// Types this seeder USED to publish. Seeding is additive, so without this a
// dropped step lingers in every existing workspace (see retireSteps).
const RETIRED_STEPS = [
"harvey/pack-result", // → vein core `pack`
"harvey/aggregate-scores", // → eval/aggregate-scores
"harvey/build-eval-chain", // → eval/build-eval-chain
"harvey/criterion-refs", // → eval/criterion-refs
];

const HERE = dirname(fileURLToPath(import.meta.url));

// Order matters only for readability — all four are plain publishes. The
Expand Down Expand Up @@ -141,4 +150,5 @@ export async function seedHarveySteps(workspace: WorkspaceStore): Promise<void>
);
}
}
await retireSteps(workspace, RETIRED_STEPS, "harvey");
}
23 changes: 22 additions & 1 deletion mcp/src/lab/seed-opts.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import type { PublishByContentOptions } from "vein";
import type { PublishByContentOptions, WorkspaceStore } from "vein";

/**
* How every lab seeder reconciles its committed templates into the workspace
Expand All @@ -11,3 +11,24 @@ import type { PublishByContentOptions } from "vein";
* still the only way to propagate it to other instances.
*/
export const SEED_OPTS: PublishByContentOptions = { reactivateKnown: false };

/**
* Retire steps a seeder no longer ships. Seeding is ADDITIVE — a step dropped
* from a `SEED_STEPS` list stays live in every existing workspace (the graph
* workspace is persistent, and the file one keeps its materialized file), so
* an author agent keeps discovering and using it. Each seeder keeps a
* `RETIRED_STEPS` list of the types it used to publish and calls this at the
* end of its step seeding; `deleteStep` is a soft delete on the graph store
* (restorable by a later publish under the same name) and an unlink on the
* file store. Missing types are a silent no-op, so a fresh workspace pays
* nothing.
*/
export async function retireSteps(workspace: WorkspaceStore, types: readonly string[], tag: string): Promise<void> {
for (const type of types) {
try {
if (await workspace.deleteStep(type)) console.log(`[${tag}] retired step: ${type}`);
} catch (err) {
console.warn(`[${tag}] could not retire step "${type}":`, err instanceof Error ? err.message : err);
}
}
}
6 changes: 5 additions & 1 deletion mcp/src/lab/wfbench/seed.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { readFile } from "node:fs/promises";
import { fileURLToPath } from "node:url";
import { dirname, join } from "node:path";
import type { WorkspaceStore } from "vein";
import { SEED_OPTS } from "../seed-opts.js";
import { SEED_OPTS, retireSteps } from "../seed-opts.js";

/**
* wfbench — the Workflow Editor Agent Benchmark harness (the vein port of
Expand Down Expand Up @@ -53,6 +53,9 @@ const SEED_WORKFLOWS: Array<{ name: string; description: string }> = [
},
];

// Types this seeder USED to publish (seeding is additive — see retireSteps).
const RETIRED_STEPS = ["wfbench/pack-result"]; // → vein core `pack`

const HERE = dirname(fileURLToPath(import.meta.url));

export async function seedWfbenchSteps(workspace: WorkspaceStore): Promise<void> {
Expand All @@ -66,6 +69,7 @@ export async function seedWfbenchSteps(workspace: WorkspaceStore): Promise<void>
console.warn(`[wfbench] could not seed step "${type}":`, err instanceof Error ? err.message : err);
}
}
await retireSteps(workspace, RETIRED_STEPS, "wfbench");
}

export async function seedWfbenchWorkflows(workspace: WorkspaceStore): Promise<void> {
Expand Down
12 changes: 11 additions & 1 deletion mcp/src/lab/wfbench/smoke.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,14 @@ async function main() {
try {
// ── 1. seed + discover ───────────────────────────────────────────────
const workspace = new WorkspaceManager(dir);
// A step this seeder USED to ship, left over from an earlier deploy —
// reseeding must retire it (seeding is additive otherwise).
await workspace.publishStep(
"wfbench/pack-result",
'import { z, defineStep } from "vein";\nexport default defineStep({ type: "wfbench/pack-result", input: z.any(), output: z.any(), async run(cfg) { return cfg; } });\n',
undefined,
"old-deploy",
);
await seedEvalSteps(workspace);
await seedArtifactSteps(workspace);
await seedWfbenchSteps(workspace);
Expand Down Expand Up @@ -66,7 +74,9 @@ async function main() {
"agent",
];
for (const t of expectedSteps) assert.ok(registry[t], `registry missing ${t}`);
console.log(`✔ seeded + discovered ${expectedSteps.length} steps`);
assert.equal(registry["wfbench/pack-result"], undefined, "retired step must not be discoverable after reseeding");
assert.ok(registry["pack"], "core pack step");
console.log(`✔ seeded + discovered ${expectedSteps.length} steps; stale wfbench/pack-result retired`);

// ── 2. static validation (what meta/validate-workflow runs) ──────────
const vein = await createVein({ workspace, serveUi: false });
Expand Down
Loading