diff --git a/.github/workflows/ci_test_lanes.yml b/.github/workflows/ci_test_lanes.yml index 4df9c66c4..05d06b92e 100644 --- a/.github/workflows/ci_test_lanes.yml +++ b/.github/workflows/ci_test_lanes.yml @@ -198,6 +198,42 @@ jobs: - name: Run eval-harness scenarios (replay) run: npm run eval:scenarios + eval_combined: + # COMBINED layered-matrix lane (WRIT state layer + Tier-2 agent layer). + # Runs packages/eval-combined against a real isolated Neotoma server and + # emits the per-category WRIT×Tier-2 coverage matrix — the qa-gate surface + # for the QA-evals reframe (QE2): a regression in EITHER layer fails the + # lane. Needs the writ submodule checked out + built; runs without an API + # key (WRIT degrades to a substring judge; OPENAI_API_KEY/WRIT_JUDGE_API_KEY + # gives higher-fidelity llm_judge scoring). The runner fails closed if a + # requested WRIT run errors (neotoma#1738/#1739). + runs-on: ubuntu-latest + steps: + - name: Checkout (with writ submodule) + uses: actions/checkout@v4 + # NOT submodules:recursive — other submodules (e.g. foundation) use SSH + # URLs that fail in CI. writ is public HTTPS, so init ONLY it explicitly. + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: "20" + cache: "npm" + + - name: Init writ submodule (public HTTPS) + run: git submodule update --init writ + + - name: Install dependencies + run: npm ci + + - name: Build writ (eval-combined imports writ/dist) + run: | + npm ci --prefix writ + npm run build --prefix writ + + - name: Run combined WRIT + Tier-2 matrix + run: npm run eval:combined + python_sdk: runs-on: ubuntu-latest steps: diff --git a/package.json b/package.json index 4cb2cd6cd..e4cc5c1e3 100644 --- a/package.json +++ b/package.json @@ -85,6 +85,7 @@ "eval:tier1": "vitest run tests/integration/agentic_eval_matrix.test.ts", "eval:tier1:update": "cross-env UPDATE_AGENTIC_EVAL_SNAPSHOTS=1 vitest run tests/integration/agentic_eval_matrix.test.ts", "eval:scenarios": "tsx packages/eval-harness/src/cli.ts run --mode replay --provider stub --reporter tty", + "eval:combined": "tsx packages/eval-combined/src/cli.ts run --output tty", "test:remote:critical": "cross-env RUN_REMOTE_TESTS=1 vitest run tests/integration/mcp_actions_matrix.test.ts tests/integration/mcp_resources.test.ts tests/integration/mcp_store_unstructured.test.ts tests/integration/mcp_store_parquet.test.ts tests/integration/mcp_schema_actions.test.ts tests/integration/schema_recommendation_integration.test.ts tests/integration/observation_ingestion.test.ts tests/integration/relationship_snapshots.test.ts tests/services/auto_enhancement_converter_detection.test.ts tests/services/auto_enhancement_processor.test.ts", "test:contract": "vitest run tests/contract", "test:bench": "cross-env RUN_BENCH=1 vitest run tests/performance", diff --git a/packages/eval-combined/src/runner.ts b/packages/eval-combined/src/runner.ts index da9c58b99..375752b5e 100644 --- a/packages/eval-combined/src/runner.ts +++ b/packages/eval-combined/src/runner.ts @@ -183,7 +183,11 @@ export async function runCombined(opts: CombinedOptions): Promise) — must await, // else `.filter` is called on a Promise and throws "scenarios.filter is // not a function" (the WRIT integration was silently broken; #1738). @@ -201,7 +205,14 @@ export async function runCombined(opts: CombinedOptions): Promise