fix(evals): a pairwise judge that errored is not a tie

LifeOS/install/skills/Evals/Graders/ModelBased/PairwiseComparison.test.ts

@@ -0,0 +1,144 @@
+import { afterAll, beforeEach, describe, expect, mock, test } from 'bun:test';
+import { aggregatePairwise, PairwiseComparisonGrader, type ComparisonResult } from './PairwiseComparison.ts';
+import type { GraderContext } from '../Base.ts';
+
+const comparison = (
+  position: string,
+  winner: 'A' | 'B' | 'tie',
+  errored = false,
+): ComparisonResult => ({ position, winner, reasoning: 'because', errored });
+
+describe('aggregatePairwise', () => {
+  test('output winning both positions scores 1', () => {
+    const { score } = aggregatePairwise(
+      [comparison('output_first', 'A'), comparison('reference_first', 'A')],
+      true,
+    );
+    expect(score).toBe(1);
+  });
+
+  test('reference winning both positions scores 0', () => {
+    const { score } = aggregatePairwise(
+      [comparison('output_first', 'B'), comparison('reference_first', 'B')],
+      true,
+    );
+    expect(score).toBe(0);
+  });
+
+  test('a genuine split still scores 0.5', () => {
+    const { score } = aggregatePairwise(
+      [comparison('output_first', 'A'), comparison('reference_first', 'B')],
+      true,
+    );
+    expect(score).toBe(0.5);
+  });
+
+  test('a genuine tie from a judge that answered still scores 0.5', () => {
+    const { score, winner } = aggregatePairwise(
+      [comparison('output_first', 'tie'), comparison('reference_first', 'tie')],
+      true,
+    );
+    expect(score).toBe(0.5);
+    expect(winner).toBe('tie');
+  });
+
+  // The defect: a judge that never answered used to be indistinguishable from
+  // one that answered "tie", so a total judge outage scored 0.5 and PASSED.
+  test('a judge that errored is not a tie', () => {
+    const { score, winner } = aggregatePairwise(
+      [comparison('output_first', 'tie', true), comparison('reference_first', 'tie', true)],
+      true,
+    );
+    expect(score).toBe(0);
+    expect(winner).toBe('error');
+  });
+
+  test('one errored comparison fails the whole grade', () => {
+    // The swap exists to cancel position bias; scoring on the surviving half
+    // would report a debiased result that was never debiased.
+    const { score } = aggregatePairwise(
+      [comparison('output_first', 'A'), comparison('reference_first', 'tie', true)],
+      true,
+    );
+    expect(score).toBe(0);
+  });
+
+  test('errors fail closed without position swap too', () => {
+    const { score } = aggregatePairwise([comparison('output_first', 'tie', true)], false);
+    expect(score).toBe(0);
+  });
+});
+
+/**
+ * The aggregator above is a pure function fed hand-built inputs. These drive the
+ * real grader instead, because the defect this file exists to prevent lived in the
+ * WIRING — whether compare() actually marks a failed judge — not in the scoring.
+ * Without them, deleting `errored: true` from compare()'s catch restores the
+ * original fail-open bug with the whole suite still green.
+ */
+describe('PairwiseComparisonGrader.grade with an unreachable judge', () => {
+  const INFERENCE = '../../../../LIFEOS/TOOLS/Inference.ts';
+
+  type JudgeReply = { success: boolean; output?: string; error?: string };
+  const DOWN: JudgeReply = { success: false, error: 'judge unreachable' };
+  const WINS_A: JudgeReply = { success: true, output: 'REASONING: better\nWINNER: A' };
+
+  // A judge outage as the code actually sees it: compare() throws on !result.success.
+  // Indirected through `replies` so a case can fail one call and not the other.
+  let replies: JudgeReply[] = [];
+  mock.module(INFERENCE, () => ({
+    inference: async () => replies.shift() ?? DOWN,
+  }));
+
+  beforeEach(() => { replies = []; });
+  afterAll(() => { mock.restore(); });
+
+  const context = (): GraderContext => ({
+    task_id: 'task-1',
+    trial_id: 'trial-1',
+    output: 'the output under evaluation',
+    transcript: {
+      task_id: 'task-1',
+      trial_id: 'trial-1',
+      started_at: new Date(0).toISOString(),
+      turns: [],
+      tool_calls: [],
+      metrics: {} as never,
+    },
+  });
+
+  const grade = (position_swap: boolean) => new PairwiseComparisonGrader({
+    type: 'pairwise_comparison',
+    params: { reference: 'a reference answer', position_swap },
+  }).grade(context());
+
+  test('fails closed instead of scoring a tie', async () => {
+    const result = await grade(true);
+    expect(result.score).toBe(0);
+    expect(result.passed).toBe(false);
+  });
+
+  test('says the judge errored rather than reporting a verdict', async () => {
+    const result = await grade(true);
+    expect(result.reasoning).toContain('judge error');
+    expect(result.reasoning).not.toContain('tie wins');
+  });
+
+  test('fails closed without position swap too', async () => {
+    const result = await grade(false);
+    expect(result.score).toBe(0);
+    expect(result.passed).toBe(false);
+  });
+
+  // Covers the swap arm specifically. The all-calls-fail cases above short-circuit
+  // on the FIRST comparison, so they pass even if the second arm drops its flag —
+  // only a half-outage reaches that line.
+  test('a judge that dies after the first comparison also fails closed', async () => {
+    replies = [WINS_A, DOWN];
+    const result = await grade(true);
+
+    // Without the second arm's flag this scores (1 + 0.5)/2 = 0.75 and PASSES.
+    expect(result.score).toBe(0);
+    expect(result.passed).toBe(false);
+  });
+});

LifeOS/install/skills/Evals/Graders/ModelBased/PairwiseComparison.ts

@@ -9,6 +9,61 @@ import { inference, type InferenceLevel } from '../../../../LIFEOS/TOOLS/Inferen
 import { judgeLevelForModel } from './JudgeLevel.ts';
 import { readFileSync, existsSync } from 'fs';
 
+export interface ComparisonResult {
+  position: string;
+  winner: 'A' | 'B' | 'tie';
+  reasoning: string;
+  /** The judge call itself failed; the winner field carries no verdict. */
+  errored?: boolean;
+}
+
+/**
+ * Turn per-position verdicts into a score.
+ *
+ * A judge that never answered is not a tie. Both are recorded as `tie` for the
+ * winner tally, so without the `errored` flag an outage scored (0 + 2*0.5)/2 =
+ * 0.5, and `passed = score >= 0.5` made that a PASS — every task in the suite
+ * passing on the strength of a judge that was down. The sibling model-based
+ * graders already return 0 when their judge throws; this makes pairwise agree.
+ *
+ * One failed comparison fails the grade rather than scoring on the survivor:
+ * the swap exists to cancel position bias, so half of it is not a debiased
+ * result, it is a biased one wearing a debiased result's score.
+ */
+export function aggregatePairwise(
+  results: ComparisonResult[],
+  positionSwap: boolean,
+): { score: number; winner: string; outputWins: number; referenceWins: number; ties: number } {
+  const outputWins = results.filter(r => r.winner === 'A').length;
+  const referenceWins = results.filter(r => r.winner === 'B').length;
+  const ties = results.filter(r => r.winner === 'tie').length;
+
+  if (results.some(r => r.errored)) {
+    return { score: 0, winner: 'error', outputWins, referenceWins, ties };
+  }
+
+  let score: number;
+  let winner: string;
+
+  if (outputWins > referenceWins) {
+    score = 1.0;
+    winner = 'output';
+  } else if (referenceWins > outputWins) {
+    score = 0.0;
+    winner = 'reference';
+  } else {
+    score = 0.5;
+    winner = 'tie';
+  }
+
+  // For the score, also consider partial wins
+  if (positionSwap && results.length === 2) {
+    score = (outputWins + ties * 0.5) / 2;
+  }
+
+  return { score, winner, outputWins, referenceWins, ties };
+}
+
 export class PairwiseComparisonGrader extends BaseGrader {
   type = 'pairwise_comparison' as const;
   category = 'model_based' as const;
@@ -36,7 +91,7 @@ export class PairwiseComparisonGrader extends BaseGrader {
     const positionSwap = params.position_swap ?? true;
 
     // Run comparison(s)
-    const results: { position: string; winner: 'A' | 'B' | 'tie'; reasoning: string }[] = [];
+    const results: ComparisonResult[] = [];
 
     // First comparison: Output = A, Reference = B
     const result1 = await this.compare(context.output, reference, level, params.criteria);
@@ -51,37 +106,19 @@ export class PairwiseComparisonGrader extends BaseGrader {
         position: 'reference_first',
         winner: flippedWinner as 'A' | 'B' | 'tie',
         reasoning: result2.reasoning,
+        errored: result2.errored,
       });
     }
 
-    // Aggregate results
-    const outputWins = results.filter(r => r.winner === 'A').length;
-    const referenceWins = results.filter(r => r.winner === 'B').length;
-    const ties = results.filter(r => r.winner === 'tie').length;
-
-    let score: number;
-    let aggregateWinner: string;
-
-    if (outputWins > referenceWins) {
-      score = 1.0;
-      aggregateWinner = 'output';
-    } else if (referenceWins > outputWins) {
-      score = 0.0;
-      aggregateWinner = 'reference';
-    } else {
-      score = 0.5;
-      aggregateWinner = 'tie';
-    }
-
-    // For the score, also consider partial wins
-    if (positionSwap && results.length === 2) {
-      score = (outputWins + ties * 0.5) / 2;
-    }
+    const { score, winner: aggregateWinner, outputWins, referenceWins, ties } =
+      aggregatePairwise(results, positionSwap);
 
     const passed = score >= 0.5;
 
     return this.createResult(score, passed, performance.now() - start, {
-      reasoning: `${aggregateWinner} wins (output: ${outputWins}, reference: ${referenceWins}, ties: ${ties})`,
+      reasoning: aggregateWinner === 'error'
+        ? `judge error (output: ${outputWins}, reference: ${referenceWins}, ties: ${ties})`
+        : `${aggregateWinner} wins (output: ${outputWins}, reference: ${referenceWins}, ties: ${ties})`,
       details: {
         results,
         position_swap: positionSwap,
@@ -96,7 +133,7 @@ export class PairwiseComparisonGrader extends BaseGrader {
     outputB: string,
     level: InferenceLevel,
     criteria?: string[]
-  ): Promise<{ winner: 'A' | 'B' | 'tie'; reasoning: string }> {
+  ): Promise<Omit<ComparisonResult, 'position'>> {
     const criteriaText = criteria?.length
       ? `Focus on these criteria:\n${criteria.map(c => `- ${c}`).join('\n')}`
       : 'Consider overall quality, accuracy, clarity, and helpfulness.';
@@ -147,9 +184,12 @@ Compare these outputs and determine which is better.`;
         reasoning: reasoningMatch?.[1]?.trim() ?? text,
       };
     } catch (e) {
+      // `tie` keeps the winner field a valid verdict shape; `errored` is what
+      // tells the aggregator this comparison produced no verdict at all.
       return {
         winner: 'tie',
         reasoning: `Comparison error: ${e}`,
+        errored: true,
       };
     }
   }

LifeOS/install/skills/Evals/package.json

@@ -4,7 +4,8 @@
   "type": "module",
   "description": "AI agent evaluation framework for LifeOS — graders, pass@k scoring, multi-turn scenarios",
   "scripts": {
-    "scenario": "bun run Tools/ScenarioRunner.ts"
+    "scenario": "bun run Tools/ScenarioRunner.ts",
+    "test": "bun test"
   },
   "dependencies": {
     "@ai-sdk/anthropic": "2.0.74",