Add pluggable eval metric registry architecture

.github/workflows/unit_tests.yml

@@ -21,7 +21,7 @@ jobs:
         run: |
           echo "::set-output name=version::$(node -v)"
       - name: Get node_modules cache
-        uses: actions/cache@v3.0.2
+        uses: actions/cache@v4
         id: node_modules
         with:
           path: |

features/eval/services/metricRegistry.test.ts

@@ -0,0 +1,60 @@
+import { registerMetric, unregisterMetric, getMetric, listMetrics, runMetric } from '@/features/eval/services/metricRegistry';
+import { type EvalMetric, type EvalMetricContext } from '@/features/eval/types/metric';
+
+const baseContext: EvalMetricContext = {
+  targetType: 'prompt',
+  targetId: 'prompt-1',
+  targetVersion: '1',
+  provider: 'anthropic',
+  model: 'claude-sonnet',
+  input: 'question',
+  output: 'answer',
+};
+
+function buildMetric(name: string, score = 100): EvalMetric {
+  return {
+    name,
+    evaluate: jest.fn().mockResolvedValue({ score }),
+  };
+}
+
+describe('metricRegistry', () => {
+  afterEach(() => {
+    listMetrics().forEach((metric) => unregisterMetric(metric.name));
+  });
+
+  it('registers and retrieves a metric by name', () => {
+    const metric = buildMetric('schema-validation');
+    registerMetric(metric);
+
+    expect(getMetric('schema-validation')).toBe(metric);
+    expect(listMetrics()).toContain(metric);
+  });
+
+  it('throws when registering a duplicate metric name', () => {
+    registerMetric(buildMetric('llm-judge'));
+
+    expect(() => registerMetric(buildMetric('llm-judge'))).toThrow('Eval metric already registered: llm-judge');
+  });
+
+  it('runs a registered metric against a context and returns its result', async () => {
+    const metric = buildMetric('citation-accuracy', 87);
+    registerMetric(metric);
+
+    const result = await runMetric('citation-accuracy', baseContext);
+
+    expect(metric.evaluate).toHaveBeenCalledWith(baseContext);
+    expect(result).toEqual({ score: 87 });
+  });
+
+  it('throws when running an unknown metric', async () => {
+    await expect(runMetric('does-not-exist', baseContext)).rejects.toThrow('Unknown eval metric: does-not-exist');
+  });
+
+  it('unregisters a metric so it is no longer retrievable', () => {
+    registerMetric(buildMetric('temp-metric'));
+    unregisterMetric('temp-metric');
+
+    expect(getMetric('temp-metric')).toBeUndefined();
+  });
+});

features/eval/services/metricRegistry.ts

@@ -0,0 +1,31 @@
+import { type EvalMetric, type EvalMetricContext, type EvalMetricResult } from '@/features/eval/types/metric';
+
+const metrics = new Map<string, EvalMetric>();
+
+/** Registers a metric so it can be run by name. Throws on duplicate names to catch registration bugs early. */
+export function registerMetric(metric: EvalMetric): void {
+  if (metrics.has(metric.name)) {
+    throw new Error(`Eval metric already registered: ${metric.name}`);
+  }
+  metrics.set(metric.name, metric);
+}
+
+export function unregisterMetric(name: string): void {
+  metrics.delete(name);
+}
+
+export function getMetric(name: string): EvalMetric | undefined {
+  return metrics.get(name);
+}
+
+export function listMetrics(): EvalMetric[] {
+  return Array.from(metrics.values());
+}
+
+export async function runMetric(name: string, context: EvalMetricContext): Promise<EvalMetricResult> {
+  const metric = metrics.get(name);
+  if (!metric) {
+    throw new Error(`Unknown eval metric: ${name}`);
+  }
+  return metric.evaluate(context);
+}

features/eval/types/metric.ts

@@ -0,0 +1,28 @@
+export type EvalTargetType = 'prompt' | 'agent';
+
+/** Normalized inputs a metric may need. Individual metrics read only the fields they require. */
+export type EvalMetricContext = {
+  targetType: EvalTargetType;
+  targetId: string;
+  targetVersion: string;
+  provider: string;
+  model: string;
+  input: unknown;
+  output: unknown;
+  expectedOutputSchema?: unknown;
+  kbContextChunks?: { sourceId: string; text: string }[];
+  citations?: { sourceId: string }[];
+};
+
+export type EvalMetricResult = {
+  score: number;
+  pass?: boolean;
+  rationale?: string;
+  metadata?: Record<string, unknown>;
+};
+
+/** Implement this interface and register the instance with the metric registry to add a new eval metric. */
+export type EvalMetric = {
+  name: string;
+  evaluate(context: EvalMetricContext): Promise<EvalMetricResult>;
+};