Require provider and model flags for all eval runners

Local settings must not pick the model. Capability eval and SWE dry-run
now require --provider/--model the same as a real SWE run.

Author: TheGreatAxios

CHANGELOG.md

@@ -78,6 +78,10 @@ parallel copies under `docs/` or `scripts/notes/`. At cut time: rename
 
 ### Evals
 
+- **Eval runners require an explicit model pair.** `eval:capability` and
+  `eval:public-swe-one` take `--provider` / `--model` (capability also
+  accepts `--matrix` with complete cells) so local `.corbits/settings.json`
+  is not the implicit target.
 - **Capability eval records `task` tool calls.** `taskToolCallCount` is derived
   from the turn stream (informational). Older result files without the field
   default from `toolCallsByName.task` so the frozen baseline still parses.

evals/capability/README.md

@@ -91,35 +91,40 @@ shell parser.
 
 ## Prerequisites
 
-- Configured provider (same as interactive `corbits`)
+- A configured provider matching the CLI `--provider` / `--model` (or `--matrix` cells)
 - Network access for inference
 - Bun
 
 Evals default to `--dangerously-skip-permissions` so the agent can write without a human at the gate. Override with `--ask-permissions` if you want the non-interactive deny path.
 
+## Provider/model
+
+CLI `--provider <name>` and `--model <id>` are required for every run, including `--dry-run`. Alternatively, pass `--matrix` with complete `provider:model` cells. Local `.corbits/settings.json` is never the implicit eval target.
+
 ## Run
 
 ```bash
-# All cases with the configured default provider/model
-bun run eval:capability
+# All cases — explicit provider/model required
+bun run eval:capability -- --provider <name> --model <id>
 
 # One case, explicit model
-bun run eval:capability -- --case simple-health --provider xai/thegreataxios --model grok-4.5
+bun run eval:capability -- --case simple-health --provider xai --model grok-4.5
 
 # Multi-model matrix (cases × variants)
 bun run eval:capability -- \
-  --matrix "xai/thegreataxios:grok-4.5,openai:gpt-4.1" \
+  --matrix "xai:grok-4.5,openai:gpt-4.1" \
   --out evals/capability/results/matrix.json
 
 # Labeled variants
 bun run eval:capability -- --matrix "fast=xai:grok-4.5,strong=openai:gpt-4.1"
 
 # Baseline improve/regress (keys by variantId::caseId)
-bun run eval:capability -- --out evals/capability/results/run2.json \
+bun run eval:capability -- --provider <name> --model <id> \
+  --out evals/capability/results/run2.json \
   --baseline evals/capability/results/run1.json
 
 # Gate run: 5 repeats per cell against the frozen baseline
-bun run eval:capability -- --repeats 5 \
+bun run eval:capability -- --provider <name> --model <id> --repeats 5 \
   --out evals/capability/results/candidate.json \
   --baseline evals/capability/results/baseline-0286.json
 ```
@@ -129,8 +134,8 @@ bun run eval:capability -- --repeats 5 \
 Any change intended to shift agent behavior (prompts, directors, tools) is
 confirmed here, not by anecdote:
 
-1. Run the suite with `--repeats 5` (repeats smooth model variance; a single
-   run of a bait case proves nothing).
+1. Run the suite with `--provider` / `--model` (or `--matrix`) and `--repeats 5`
+   (repeats smooth model variance; a single run of a bait case proves nothing).
 2. Compare against the frozen baseline
    (`evals/capability/results/baseline-0286.json`) with `--baseline`.
 3. Read the verdicts: any pass-rate change per cell is significant; behavior
@@ -148,8 +153,8 @@ Flags:
 | Flag | Meaning |
 |------|---------|
 | `--case <id\|all>` | Case id or `all` (default) |
-| `--provider` / `--model` | Single-variant override via `loadConfig` |
-| `--matrix <cells>` | Multi-variant: `p:m,p2:m2` or `label=p:m` (comma-separated) |
+| `--provider <name>` / `--model <id>` | Required unless `--matrix`. Single-variant via `loadConfig`. Not inferred from local settings |
+| `--matrix <cells>` | Alternative to `--provider`/`--model`. Multi-variant: `p:m,p2:m2` or `label=p:m` (comma-separated). Every cell must include both sides |
 | `--config <path>` | Settings file override (CI injection) |
 | `--out <path>` | Write machine-readable results JSON |
 | `--baseline <path>` | Compare this run to a prior results file (improve/regress + metric deltas) |
@@ -158,7 +163,7 @@ Flags:
 | `--agent-timeout-ms <n>` | Wall-clock limit for `runExec` (default `600000`, env `CORBITS_EVAL_AGENT_TIMEOUT_MS`) |
 | `--verify-timeout-ms <n>` | Wall-clock limit for `verify.sh` (default `120000`, env `CORBITS_EVAL_VERIFY_TIMEOUT_MS`) |
 | `--repeats <n>` | Runs per case×variant cell (default `1`; gate runs use `5`, baseline freezes `3`). Results record every repeat plus per-cell aggregates |
-| `--dry-run` | Load cases × variants and print plan; no inference |
+| `--dry-run` | Load cases × variants and print plan; no inference. Still requires `--provider`/`--model` or `--matrix` |
 
 ## Case format
 

evals/capability/lib.test.ts

@@ -309,6 +309,16 @@ describe("parseMatrix", () => {
     expect(v[0]!.provider).toBe("xai");
     expect(v[0]!.model).toBe("thegreataxios/grok-4.5");
   });
+
+  test("rejects incomplete cells", () => {
+    expect(() => parseMatrix("xai:", {})).toThrow(/both provider and model/);
+    expect(() => parseMatrix(":grok-4.5", {})).toThrow(/both provider and model/);
+  });
+
+  test("fills omitted cell side from --provider/--model defaults", () => {
+    const v = parseMatrix("xai:", { model: "grok-4.5" });
+    expect(v[0]).toEqual({ id: "xai:grok-4.5", provider: "xai", model: "grok-4.5" });
+  });
 });
 
 describe("expandMatrix", () => {

evals/capability/lib.ts

@@ -527,6 +527,8 @@ export function defaultVariantId(provider?: string, model?: string): string {
  *   - `provider/model` (slash only when no colon)
  *   - `label=provider:model`
  * Empty / omitted → single default variant (caller provider/model flags).
+ * Each expanded cell must have both provider and model (after applying
+ * `--provider`/`--model` as cell defaults when a side is omitted).
  */
 export function parseMatrix(
   matrix: string | undefined,
@@ -549,10 +551,14 @@ export function parseMatrix(
   if (cells.length === 0) {
     throw new Error("--matrix has no variants");
   }
-  return cells.map((cell, index) => parseMatrixCell(cell, index));
+  return cells.map((cell, index) => parseMatrixCell(cell, index, fallback));
 }
 
-function parseMatrixCell(cell: string, index: number): EvalVariant {
+function parseMatrixCell(
+  cell: string,
+  index: number,
+  fallback: { provider?: string; model?: string },
+): EvalVariant {
   let label: string | undefined;
   let rest = cell;
   const eq = cell.indexOf("=");
@@ -575,15 +581,15 @@ function parseMatrixCell(cell: string, index: number): EvalVariant {
       `matrix cell ${index + 1} "${cell}" must be provider:model or label=provider:model`,
     );
   }
-  if (provider === undefined && model === undefined) {
-    throw new Error(`matrix cell ${index + 1} "${cell}" is empty`);
+  provider = provider ?? fallback.provider;
+  model = model ?? fallback.model;
+  if (provider === undefined || model === undefined) {
+    throw new Error(
+      `matrix cell ${index + 1} "${cell}" must specify both provider and model`,
+    );
   }
   const id = label ?? defaultVariantId(provider, model);
-  return {
-    id,
-    ...(provider !== undefined ? { provider } : {}),
-    ...(model !== undefined ? { model } : {}),
-  };
+  return { id, provider, model };
 }
 
 /** Cartesian product of cases × variants (cases outer for stable progress). */

evals/public/README.md

@@ -17,7 +17,7 @@ without vendoring a full leaderboard runner into product CI.
 
 ```bash
 # Dry plan (loads HF row, prints prompt)
-bun scripts/eval-public-swe-one.ts --dry-run
+bun scripts/eval-public-swe-one.ts --dry-run --provider <name> --model <id>
 
 # Default instance: psf__requests-3362 (small repo, single failing test)
 bun scripts/eval-public-swe-one.ts \

scripts/eval-capability.test.ts

@@ -0,0 +1,57 @@
+import { describe, expect, test } from "bun:test";
+
+import { parseArgs } from "./eval-capability.ts";
+
+describe("parseArgs", () => {
+  test("--help does not require provider or model", () => {
+    const opts = parseArgs(["--help"]);
+    expect(opts.help).toBe(true);
+    expect(opts.provider).not.toBe("xai/thegreataxios");
+    expect(opts.model).not.toBe("xai/thegreataxios");
+  });
+
+  test("no flags throws", () => {
+    expect(() => parseArgs([])).toThrow(/--provider/);
+    expect(() => parseArgs([])).toThrow(/--model/);
+  });
+
+  test("--provider without --model throws", () => {
+    expect(() => parseArgs(["--provider", "foo"])).toThrow(/--model/);
+  });
+
+  test("--model without --provider throws", () => {
+    expect(() => parseArgs(["--model", "bar"])).toThrow(/--provider/);
+  });
+
+  test("--provider foo --model bar parses those values", () => {
+    const opts = parseArgs(["--provider", "foo", "--model", "bar"]);
+    expect(opts.provider).toBe("foo");
+    expect(opts.model).toBe("bar");
+  });
+
+  test("--dry-run without pair throws", () => {
+    expect(() => parseArgs(["--dry-run"])).toThrow(/--provider/);
+    expect(() => parseArgs(["--dry-run"])).toThrow(/--model/);
+  });
+
+  test("--matrix xai:grok-4.5 is enough without top-level flags", () => {
+    const opts = parseArgs(["--matrix", "xai:grok-4.5"]);
+    expect(opts.matrix).toBe("xai:grok-4.5");
+  });
+
+  test("incomplete matrix cell throws", () => {
+    expect(() => parseArgs(["--matrix", "xai:"])).toThrow(/both provider and model/);
+    expect(() => parseArgs(["--matrix", ":grok-4.5"])).toThrow(/both provider and model/);
+  });
+
+  test("parsed defaults never equal xai/thegreataxios", () => {
+    const help = parseArgs(["--help"]);
+    const pair = parseArgs(["--provider", "foo", "--model", "bar"]);
+    expect(help.provider).not.toBe("xai/thegreataxios");
+    expect(help.model).not.toBe("xai/thegreataxios");
+    expect(pair.provider).not.toBe("xai/thegreataxios");
+    expect(pair.model).not.toBe("xai/thegreataxios");
+    expect(pair.provider).toBe("foo");
+    expect(pair.model).toBe("bar");
+  });
+});

scripts/eval-capability.ts

@@ -69,6 +69,7 @@ type CliOptions = {
   /** Runs per case×variant cell (gate runs use 5; freeze runs use 3). */
   repeats: number;
   dryRun: boolean;
+  help: boolean;
   /**
    * Allow a run/comparison to proceed when the resolved provider/model
    * differs from what was requested, instead of hard-failing.
@@ -77,12 +78,13 @@ type CliOptions = {
 };
 
 function printUsage(): void {
-  console.log(`Usage: bun scripts/eval-capability.ts [options]
+  console.log(`Usage: bun scripts/eval-capability.ts --provider <name> --model <id> [options]
+       bun scripts/eval-capability.ts --matrix <cells> [options]
 
   --case <id|all>       Case id (default: all)
-  --provider <name>     Provider override (single-variant run)
-  --model <id>          Model override (single-variant run)
-  --matrix <cells>      Multi-variant: "p1:m1,p2:m2" or "label=p:m,..."
+  --provider <name>     Provider name (required except --help, or --matrix with complete cells)
+  --model <id>          Model id (required except --help, or --matrix with complete cells)
+  --matrix <cells>      Multi-variant: "p1:m1,p2:m2" or "label=p:m,..."; each cell needs both sides
   --config <path>       Settings file override
   --out <path>          Write results JSON
   --baseline <path>     Compare to prior results JSON
@@ -91,19 +93,20 @@ function printUsage(): void {
   --agent-timeout-ms <n> Wall-clock limit for runExec (default 1200000)
   --verify-timeout-ms <n> Wall-clock limit for verify.sh (default 120000)
   --repeats <n>         Runs per case×variant cell (default 1; gate runs use 5)
-  --dry-run             List cases × variants only
+  --dry-run             List cases × variants only (still requires --provider/--model or --matrix)
   --allow-provider-fallback  Allow resolved provider/model to differ from
                              what was requested (default: hard-fail)
   -h, --help            Show help
 `);
 }
 
-function parseArgs(argv: readonly string[]): CliOptions {
+export function parseArgs(argv: readonly string[]): CliOptions {
   const opts: CliOptions = {
     caseSelector: "all",
     skipPermissions: true,
     repeats: 1,
     dryRun: false,
+    help: false,
     allowProviderFallback: false,
     agentTimeoutMs: Number(process.env.CORBITS_EVAL_AGENT_TIMEOUT_MS ?? 1_200_000),
     verifyTimeoutMs: Number(process.env.CORBITS_EVAL_VERIFY_TIMEOUT_MS ?? 120_000),
@@ -118,8 +121,7 @@ function parseArgs(argv: readonly string[]): CliOptions {
     switch (a) {
       case "-h":
       case "--help":
-        printUsage();
-        process.exit(0);
+        opts.help = true;
         break;
       case "--case":
         opts.caseSelector = next();
@@ -183,9 +185,32 @@ function parseArgs(argv: readonly string[]): CliOptions {
         throw new Error(`Unknown argument: ${a}`);
     }
   }
+  if (!opts.help) {
+    requireExplicitModelPair(opts);
+  }
   return opts;
 }
 
+function requireExplicitModelPair(opts: CliOptions): void {
+  const matrix = opts.matrix?.trim();
+  if (matrix !== undefined && matrix.length > 0) {
+    parseMatrix(matrix, {
+      provider: opts.provider,
+      model: opts.model,
+    });
+    return;
+  }
+  if (!opts.provider && !opts.model) {
+    throw new Error("missing required --provider and --model (or --matrix)");
+  }
+  if (!opts.provider) {
+    throw new Error("missing required --provider");
+  }
+  if (!opts.model) {
+    throw new Error("missing required --model");
+  }
+}
+
 function runCommand(
   command: string,
   args: readonly string[],
@@ -510,7 +535,9 @@ async function runCase(
 
     const config = await loadConfig(argv, { allowUnconfigured: false });
     if (!config.configured) {
-      throw new Error("Provider not configured for eval run");
+      throw new Error(
+        "Provider not configured. Pass --provider <name> --model <id> (or --matrix) matching a configured provider.",
+      );
     }
 
     const agentStarted = Date.now();
@@ -684,6 +711,10 @@ function formatMetricsLine(r: CaseResult): string {
 
 async function main(): Promise<number> {
   const opts = parseArgs(process.argv.slice(2));
+  if (opts.help) {
+    printUsage();
+    return 0;
+  }
   const all = await loadEvalCases(CASES_ROOT);
   const selected = filterCases(all, opts.caseSelector);
   const variants = parseMatrix(opts.matrix, {

scripts/eval-public-swe-one.test.ts

@@ -10,11 +10,16 @@ describe("parseArgs", () => {
     expect(opts.model).not.toBe("xai/thegreataxios");
   });
 
-  test("--dry-run does not require provider or model", () => {
-    const opts = parseArgs(["--dry-run"]);
+  test("--dry-run alone throws", () => {
+    expect(() => parseArgs(["--dry-run"])).toThrow(/--provider/);
+    expect(() => parseArgs(["--dry-run"])).toThrow(/--model/);
+  });
+
+  test("--dry-run with provider and model parses", () => {
+    const opts = parseArgs(["--dry-run", "--provider", "foo", "--model", "bar"]);
     expect(opts.dryRun).toBe(true);
-    expect(opts.provider).not.toBe("xai/thegreataxios");
-    expect(opts.model).not.toBe("xai/thegreataxios");
+    expect(opts.provider).toBe("foo");
+    expect(opts.model).toBe("bar");
   });
 
   test("agent run without --provider throws", () => {
@@ -38,10 +43,7 @@ describe("parseArgs", () => {
 
   test("parsed defaults never equal xai/thegreataxios", () => {
     const help = parseArgs(["--help"]);
-    const dry = parseArgs(["--dry-run"]);
     expect(help.provider).not.toBe("xai/thegreataxios");
     expect(help.model).not.toBe("xai/thegreataxios");
-    expect(dry.provider).not.toBe("xai/thegreataxios");
-    expect(dry.model).not.toBe("xai/thegreataxios");
   });
 });

scripts/eval-public-swe-one.ts

@@ -10,7 +10,7 @@
  * Usage:
  *   bun scripts/eval-public-swe-one.ts --provider <name> --model <id>
  *   bun scripts/eval-public-swe-one.ts --instance psf__requests-3362 --provider <name> --model <id>
- *   bun scripts/eval-public-swe-one.ts --dry-run
+ *   bun scripts/eval-public-swe-one.ts --dry-run --provider <name> --model <id>
  *
  * Optional official grading (heavy; needs Docker resources):
  *   bun scripts/eval-public-swe-one.ts --instance … --provider <name> --model <id> --evaluate
@@ -62,8 +62,8 @@ One public SWE-bench Lite instance via Corbits product exec.
 
 Options:
   --instance <id>     SWE-bench instance_id (default: ${DEFAULT_INSTANCE})
-  --provider <name>   Provider name (required except --help / --dry-run)
-  --model <id>        Model id (required except --help / --dry-run)
+  --provider <name>   Provider name (required except --help)
+  --model <id>        Model id (required except --help)
   --subset <hf>       HF dataset id (default: ${DEFAULT_SUBSET})
   --split <name>      Dataset split (default: ${DEFAULT_SPLIT})
   --timeout-ms <n>    Agent wall-clock timeout (default: ${DEFAULT_AGENT_TIMEOUT_MS})
@@ -133,7 +133,7 @@ export function parseArgs(argv: string[]): CliOptions {
         throw new Error(`unknown arg: ${a}`);
     }
   }
-  if (!opts.help && !opts.dryRun) {
+  if (!opts.help) {
     if (!opts.provider && !opts.model) {
       throw new Error("missing required --provider and --model");
     }