diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 99375f78..a4aebd5e 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -21,7 +21,7 @@ jobs: run: | echo "::set-output name=version::$(node -v)" - name: Get node_modules cache - uses: actions/cache@v3.0.2 + uses: actions/cache@v4 id: node_modules with: path: | diff --git a/features/eval/services/metricRegistry.test.ts b/features/eval/services/metricRegistry.test.ts new file mode 100644 index 00000000..91b38fc1 --- /dev/null +++ b/features/eval/services/metricRegistry.test.ts @@ -0,0 +1,60 @@ +import { registerMetric, unregisterMetric, getMetric, listMetrics, runMetric } from '@/features/eval/services/metricRegistry'; +import { type EvalMetric, type EvalMetricContext } from '@/features/eval/types/metric'; + +const baseContext: EvalMetricContext = { + targetType: 'prompt', + targetId: 'prompt-1', + targetVersion: '1', + provider: 'anthropic', + model: 'claude-sonnet', + input: 'question', + output: 'answer', +}; + +function buildMetric(name: string, score = 100): EvalMetric { + return { + name, + evaluate: jest.fn().mockResolvedValue({ score }), + }; +} + +describe('metricRegistry', () => { + afterEach(() => { + listMetrics().forEach((metric) => unregisterMetric(metric.name)); + }); + + it('registers and retrieves a metric by name', () => { + const metric = buildMetric('schema-validation'); + registerMetric(metric); + + expect(getMetric('schema-validation')).toBe(metric); + expect(listMetrics()).toContain(metric); + }); + + it('throws when registering a duplicate metric name', () => { + registerMetric(buildMetric('llm-judge')); + + expect(() => registerMetric(buildMetric('llm-judge'))).toThrow('Eval metric already registered: llm-judge'); + }); + + it('runs a registered metric against a context and returns its result', async () => { + const metric = buildMetric('citation-accuracy', 87); + registerMetric(metric); + + const result = await runMetric('citation-accuracy', baseContext); + + expect(metric.evaluate).toHaveBeenCalledWith(baseContext); + expect(result).toEqual({ score: 87 }); + }); + + it('throws when running an unknown metric', async () => { + await expect(runMetric('does-not-exist', baseContext)).rejects.toThrow('Unknown eval metric: does-not-exist'); + }); + + it('unregisters a metric so it is no longer retrievable', () => { + registerMetric(buildMetric('temp-metric')); + unregisterMetric('temp-metric'); + + expect(getMetric('temp-metric')).toBeUndefined(); + }); +}); diff --git a/features/eval/services/metricRegistry.ts b/features/eval/services/metricRegistry.ts new file mode 100644 index 00000000..810ca471 --- /dev/null +++ b/features/eval/services/metricRegistry.ts @@ -0,0 +1,31 @@ +import { type EvalMetric, type EvalMetricContext, type EvalMetricResult } from '@/features/eval/types/metric'; + +const metrics = new Map(); + +/** Registers a metric so it can be run by name. Throws on duplicate names to catch registration bugs early. */ +export function registerMetric(metric: EvalMetric): void { + if (metrics.has(metric.name)) { + throw new Error(`Eval metric already registered: ${metric.name}`); + } + metrics.set(metric.name, metric); +} + +export function unregisterMetric(name: string): void { + metrics.delete(name); +} + +export function getMetric(name: string): EvalMetric | undefined { + return metrics.get(name); +} + +export function listMetrics(): EvalMetric[] { + return Array.from(metrics.values()); +} + +export async function runMetric(name: string, context: EvalMetricContext): Promise { + const metric = metrics.get(name); + if (!metric) { + throw new Error(`Unknown eval metric: ${name}`); + } + return metric.evaluate(context); +} diff --git a/features/eval/types/metric.ts b/features/eval/types/metric.ts new file mode 100644 index 00000000..8d93ce98 --- /dev/null +++ b/features/eval/types/metric.ts @@ -0,0 +1,28 @@ +export type EvalTargetType = 'prompt' | 'agent'; + +/** Normalized inputs a metric may need. Individual metrics read only the fields they require. */ +export type EvalMetricContext = { + targetType: EvalTargetType; + targetId: string; + targetVersion: string; + provider: string; + model: string; + input: unknown; + output: unknown; + expectedOutputSchema?: unknown; + kbContextChunks?: { sourceId: string; text: string }[]; + citations?: { sourceId: string }[]; +}; + +export type EvalMetricResult = { + score: number; + pass?: boolean; + rationale?: string; + metadata?: Record; +}; + +/** Implement this interface and register the instance with the metric registry to add a new eval metric. */ +export type EvalMetric = { + name: string; + evaluate(context: EvalMetricContext): Promise; +};