ci(eval-combined): bump writ + add combined WRIT+Tier2 qa-gate lane (#1737)

.github/workflows/ci_test_lanes.yml

@@ -198,6 +198,42 @@ jobs:
       - name: Run eval-harness scenarios (replay)
         run: npm run eval:scenarios
 
+  eval_combined:
+    # COMBINED layered-matrix lane (WRIT state layer + Tier-2 agent layer).
+    # Runs packages/eval-combined against a real isolated Neotoma server and
+    # emits the per-category WRIT×Tier-2 coverage matrix — the qa-gate surface
+    # for the QA-evals reframe (QE2): a regression in EITHER layer fails the
+    # lane. Needs the writ submodule checked out + built; runs without an API
+    # key (WRIT degrades to a substring judge; OPENAI_API_KEY/WRIT_JUDGE_API_KEY
+    # gives higher-fidelity llm_judge scoring). The runner fails closed if a
+    # requested WRIT run errors (neotoma#1738/#1739).
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout (with writ submodule)
+        uses: actions/checkout@v4
+        # NOT submodules:recursive — other submodules (e.g. foundation) use SSH
+        # URLs that fail in CI. writ is public HTTPS, so init ONLY it explicitly.
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: "npm"
+
+      - name: Init writ submodule (public HTTPS)
+        run: git submodule update --init writ
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Build writ (eval-combined imports writ/dist)
+        run: |
+          npm ci --prefix writ
+          npm run build --prefix writ
+
+      - name: Run combined WRIT + Tier-2 matrix
+        run: npm run eval:combined
+
   python_sdk:
     runs-on: ubuntu-latest
     steps:

package.json

@@ -85,6 +85,7 @@
     "eval:tier1": "vitest run tests/integration/agentic_eval_matrix.test.ts",
     "eval:tier1:update": "cross-env UPDATE_AGENTIC_EVAL_SNAPSHOTS=1 vitest run tests/integration/agentic_eval_matrix.test.ts",
     "eval:scenarios": "tsx packages/eval-harness/src/cli.ts run --mode replay --provider stub --reporter tty",
+    "eval:combined": "tsx packages/eval-combined/src/cli.ts run --output tty",
     "test:remote:critical": "cross-env RUN_REMOTE_TESTS=1 vitest run tests/integration/mcp_actions_matrix.test.ts tests/integration/mcp_resources.test.ts tests/integration/mcp_store_unstructured.test.ts tests/integration/mcp_store_parquet.test.ts tests/integration/mcp_schema_actions.test.ts tests/integration/schema_recommendation_integration.test.ts tests/integration/observation_ingestion.test.ts tests/integration/relationship_snapshots.test.ts tests/services/auto_enhancement_converter_detection.test.ts tests/services/auto_enhancement_processor.test.ts",
     "test:contract": "vitest run tests/contract",
     "test:bench": "cross-env RUN_BENCH=1 vitest run tests/performance",

packages/eval-combined/src/runner.ts

@@ -183,7 +183,11 @@ export async function runCombined(opts: CombinedOptions): Promise<CombinedResult
   if (!opts.tier2Only) {
     log("[eval-combined] running WRIT benchmark...");
     try {
-      const writ = await import(join(writPath, "src", "index.js"));
+      // Import WRIT's BUILT entry (dist/, the package "main"), not src/*.js —
+      // those are stale untracked CommonJS artifacts that throw "exports is not
+      // defined in ES module scope" under writ's "type":"module" (#1738
+      // follow-up). The CI lane runs `npm run build --prefix writ` first.
+      const writ = await import(join(writPath, "dist", "index.js"));
       // loadAllScenarios is async (returns Promise<Scenario[]>) — must await,
       // else `.filter` is called on a Promise and throws "scenarios.filter is
       // not a function" (the WRIT integration was silently broken; #1738).
@@ -201,7 +205,14 @@ export async function runCombined(opts: CombinedOptions): Promise<CombinedResult
         const { startIsolatedNeotomaServer } = await import(
           join(evalHarnessPath, "src", "isolated_server.js")
         );
-        const server = await startIsolatedNeotomaServer({ hooksEnabled: true });
+        // WRIT replays 5-20 sessions/scenario across 77 scenarios = bursty
+        // back-to-back writes that trip the default write rate limit (HTTP 429),
+        // which silently drops facts and tanks recall. Raise the limit on the
+        // isolated server (this is a benchmark harness, not production traffic).
+        const server = await startIsolatedNeotomaServer({
+          hooksEnabled: true,
+          env: { NEOTOMA_WRITE_RATE_LIMIT_PER_MIN: "1000000" },
+        });
         try {
           const adapter = new writ.NeotomaAdapter(server.baseUrl, {
             token: server.token,