From d562171a7ec750d2246ab31bb03b6382c0ac11ca Mon Sep 17 00:00:00 2001 From: lyyzka <3495484503@qq.com> Date: Wed, 26 Aug 2026 16:40:15 +0800 Subject: [PATCH 1/5] feat: add visual Agent Eval pipeline --- README.md | 4 + docs/agent-eval.md | 102 +++++ server/src/__tests__/eval-evaluator.test.ts | 127 ++++++ server/src/agent-os/runtime.ts | 16 + server/src/api/admin-router.ts | 66 ++- server/src/db/migrate.ts | 67 ++++ server/src/eval/contracts.ts | 354 +++++++++++++++++ server/src/eval/evaluator.ts | 420 ++++++++++++++++++++ server/src/eval/service.ts | 411 +++++++++++++++++++ src/admin/AdminApp.tsx | 14 +- src/admin/EvalPage.tsx | 396 ++++++++++++++++++ src/admin/admin.css | 222 +++++++++++ src/admin/api.ts | 110 +++++ 13 files changed, 2298 insertions(+), 11 deletions(-) create mode 100644 docs/agent-eval.md create mode 100644 server/src/__tests__/eval-evaluator.test.ts create mode 100644 server/src/eval/contracts.ts create mode 100644 server/src/eval/evaluator.ts create mode 100644 server/src/eval/service.ts create mode 100644 src/admin/EvalPage.tsx diff --git a/README.md b/README.md index 7dd2579f..9721649e 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,11 @@ does not reactivate the retired runtime. | `server/agent-os/` | Persistent IPython kernel runner | | `server/src/im/` | WuKongIM bootstrap, webhook, routing and payload contracts | | `server/src/agents/` | Typed learning-domain services used by the Host Bridge | +| `server/src/eval/` | Deterministic answer, RAG, tool, and multi-Agent evaluation pipeline | | `src/lib/im/` | Browser-side WuKongIM SDK wrapper | +| `src/admin/EvalPage.tsx` | Eval run pipeline, failure drill-down, and version comparison dashboard | | `scripts/guard-agent-os.mjs` | CI guard for the independent runtime boundary | +The Eval request contract and scoring rules are documented in [`docs/agent-eval.md`](docs/agent-eval.md). + Licensed under [MIT](LICENSE). diff --git a/docs/agent-eval.md b/docs/agent-eval.md new file mode 100644 index 00000000..cee84a41 --- /dev/null +++ b/docs/agent-eval.md @@ -0,0 +1,102 @@ +# Agent Eval + +LingxiLoop Agent Eval is an admin-only, deterministic regression system for four product capabilities: + +1. Agent answer quality; +2. RAG retrieval and citation traceability; +3. tool selection, arguments, order, and execution result; +4. multi-Agent participation, handoffs, completion, and parallelism. + +Each case flows through `ingest → answer → RAG → tools → collaboration → aggregate`. A missing optional stage is reported as `skipped`; a stage named in `requiredStages` fails when it has no observable evidence. Reports are immutable and grouped by `suiteKey`, which makes scores comparable across `version` values. + +## Run an evaluation + +Admins can paste the same payload into **Admin → Agent Eval → 运行评测**, or call the API: + +```http +POST /api/admin/eval/runs +Authorization: Bearer +Content-Type: application/json +``` + +```json +{ + "schemaVersion": "lingxiloop.eval.v1", + "suiteKey": "agent-regression", + "suiteName": "Agent Regression", + "version": "2026.08.26", + "passThreshold": 0.8, + "cases": [ + { + "caseId": "grounded-answer", + "sourceAgentRunId": "", + "expectations": { + "requiredStages": ["answer", "rag", "tools"], + "answer": { + "requiredKeywords": ["conclusion"], + "forbiddenPatterns": ["I am guessing"], + "maxLatencyMs": 15000, + "maxTokens": 4000 + }, + "rag": { + "requiredSourceIds": ["source-123"], + "requireCitations": true, + "minRetrievalRecall": 1, + "minCitationPrecision": 1 + }, + "tools": { + "calls": [ + { "name": "knowledge.search", "argsSubset": { "query": "evaluation" } } + ], + "requireSuccess": true, + "allowUnexpected": false + } + } + } + ] +} +``` + +`sourceAgentRunId` automatically hydrates the answer, latency, token use, Host Bridge actions, legacy tool calls, RAG evidence identities, and Canvas collaboration assignments. An optional `observation` object overrides individual hydrated fields, which is useful for a controlled fixture. A case without a run ID must supply `observation` directly. + +## Inline observation + +```json +{ + "caseId": "parallel-research", + "observation": { + "answer": "The conclusion is grounded in the supplied evidence. [S1]", + "retrievedSourceIds": ["source-123"], + "citations": [{ "sourceId": "source-123", "chunkId": "chunk-7", "marker": "S1" }], + "toolCalls": [{ "name": "knowledge.search", "args": { "query": "evaluation" }, "status": "ok" }], + "agentTurns": [ + { "agentId": "sage", "status": "completed", "startedAt": "2026-08-26T10:00:00Z", "finishedAt": "2026-08-26T10:00:05Z" }, + { "agentId": "forge", "status": "completed", "startedAt": "2026-08-26T10:00:01Z", "finishedAt": "2026-08-26T10:00:06Z" } + ], + "latencyMs": 6000, + "tokenCount": 1800 + }, + "expectations": { + "answer": { "requiredKeywords": ["conclusion"] }, + "collaboration": { + "requiredAgentIds": ["sage", "forge"], + "minAgents": 2, + "requireAllCompleted": true, + "requireParallelism": true + } + } +} +``` + +## Scoring and gates + +Only observed stages contribute to a case's weighted score. Default weights are answer `35%`, RAG `25%`, tools `20%`, and collaboration `20%`; a case can override them with `expectations.weights`. Stage gates default to answer `0.75`, RAG `0.75`, tools `1.0`, and collaboration `0.8`. A failed stage gate fails the case even when the weighted total is high. + +Answer reference similarity is deterministic lexical F1 (including CJK unigram/bigram features), not an LLM-as-judge call. Expected answers, keywords, and source IDs stay in the evaluator and are never sent to Agent OS. + +## Read reports + +- `GET /api/admin/eval/runs?sinceDays=90&suiteKey=agent-regression` returns dashboard KPIs, recent runs, stage averages, previous-version scores, and deltas. +- `GET /api/admin/eval/runs/:id` returns cases, stage results, findings, metrics, and failure reasons. + +RAG trace events persist source, chunk, marker, and title metadata only. They deliberately exclude retrieved excerpts so the observability ledger does not duplicate source content. diff --git a/server/src/__tests__/eval-evaluator.test.ts b/server/src/__tests__/eval-evaluator.test.ts new file mode 100644 index 00000000..3921aee0 --- /dev/null +++ b/server/src/__tests__/eval-evaluator.test.ts @@ -0,0 +1,127 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { EvalInputError, validateEvalRunInput } from '../eval/contracts.js' +import { answerSimilarity, evaluateCase, evaluateRun } from '../eval/evaluator.js' + +test('answer similarity supports CJK phrases and normalized Latin tokens', () => { + assert.ok(answerSimilarity('RAG 会检索相关知识并生成回答。', 'RAG 检索知识后生成可靠回答') > 0.5) + assert.equal(answerSimilarity(' Agent-OS VERSION_1 ', 'agent-os version_1'), 1) + assert.equal(answerSimilarity('完全无关', 'tool calling'), 0) +}) + +test('Eval pipeline passes answer, RAG, tool, and parallel collaboration gates', () => { + const report = evaluateCase({ + caseId: 'grounded-research', + sourceAgentRunId: 'run-1', + expectations: { + requiredStages: ['answer', 'rag', 'tools', 'collaboration'], + answer: { requiredKeywords: ['可追溯'], forbiddenPatterns: ['我猜'], maxLatencyMs: 5_000, maxTokens: 500 }, + rag: { requiredSourceIds: ['source-a'], requireCitations: true }, + tools: { calls: [{ name: 'knowledge.search', argsSubset: { query: '评测' } }], requireSuccess: true, allowUnexpected: false }, + collaboration: { requiredAgentIds: ['sage', 'forge'], minAgents: 2, requireAllCompleted: true, requireParallelism: true }, + }, + }, { + answer: '结论可追溯到给定证据。[S1]', + retrievedSourceIds: ['source-a'], + citations: [{ sourceId: 'source-a', marker: 'S1' }], + toolCalls: [{ name: 'knowledge.search', args: { query: '评测', limit: 5 }, status: 'ok' }], + agentTurns: [ + { agentId: 'sage', status: 'completed', startedAt: '2026-01-01T00:00:00Z', finishedAt: '2026-01-01T00:00:03Z' }, + { agentId: 'forge', status: 'completed', startedAt: '2026-01-01T00:00:01Z', finishedAt: '2026-01-01T00:00:04Z' }, + ], + latencyMs: 1_200, + tokenCount: 240, + }) + assert.equal(report.status, 'pass') + assert.equal(report.stages.length, 6) + assert.ok(report.stages.every((stage) => stage.status === 'pass')) + assert.equal(report.failureReasons.length, 0) +}) + +test('Eval pipeline preserves root causes across RAG, tools, and collaboration failures', () => { + const report = evaluateCase({ + caseId: 'bad-trace', + expectations: { + answer: { requiredKeywords: ['证据'] }, + rag: { requiredSourceIds: ['source-required'], requireCitations: true }, + tools: { calls: [{ name: 'calendar.create' }], forbiddenToolNames: ['email.send'], requireSuccess: true }, + collaboration: { minAgents: 2, maxFailedAgents: 0 }, + }, + }, { + answer: '这是一个猜测。[S9]', + retrievedSourceIds: ['source-other'], + citations: [{ sourceId: 'source-other', marker: 'S1' }], + toolCalls: [{ name: 'email.send', status: 'error' }], + agentTurns: [{ agentId: 'sage', status: 'failed', error: 'timeout' }], + }) + assert.equal(report.status, 'fail') + assert.ok(report.failureReasons.some((reason) => reason.includes('缺少关键点'))) + assert.ok(report.failureReasons.some((reason) => reason.includes('召回率'))) + assert.ok(report.failureReasons.some((reason) => reason.includes('禁止工具'))) + assert.ok(report.failureReasons.some((reason) => reason.includes('Agent 参与'))) + assert.equal(report.stages.find((stage) => stage.stage === 'aggregate')?.status, 'fail') +}) + +test('unconfigured optional dimensions are skipped without inflating the score', () => { + const report = evaluateCase({ + caseId: 'answer-only', + expectations: { answer: { requiredKeywords: ['42'] } }, + }, { answer: '答案是 42。' }) + assert.equal(report.status, 'pass') + assert.equal(report.score, 1) + assert.equal(report.stages.find((stage) => stage.stage === 'rag')?.status, 'skipped') + assert.equal(report.stages.find((stage) => stage.stage === 'tools')?.score, null) +}) + +test('a required but unconfigured stage fails as missing coverage', () => { + const report = evaluateCase({ + caseId: 'requires-rag', + expectations: { answer: {}, requiredStages: ['rag'] }, + }, { answer: '有回答' }) + const rag = report.stages.find((stage) => stage.stage === 'rag') + assert.equal(report.status, 'fail') + assert.equal(rag?.status, 'fail') + assert.ok(rag?.findings.some((item) => item.checkId === 'coverage.required_stage')) +}) + +test('run aggregation exposes comparable per-stage scores and case counts', () => { + const report = evaluateRun({ + suiteKey: 'agent-regression', + suiteName: 'Agent Regression', + version: 'v2', + passThreshold: 0.8, + cases: [ + { caseId: 'one', expectations: { answer: { requiredKeywords: ['ok'] } }, observation: { answer: 'ok' } }, + { caseId: 'two', expectations: { answer: { requiredKeywords: ['ok'] } }, observation: { answer: 'missing' } }, + ], + }, new Map()) + assert.equal(report.status, 'fail') + assert.equal(report.summary.caseCount, 2) + assert.equal(report.summary.passedCases, 1) + assert.equal(report.summary.failedCases, 1) + assert.equal(report.summary.stageScores.answer, 0.8334) + assert.equal(report.summary.stageScores.rag, null) +}) + +test('Eval request validation accepts a trace-backed case and rejects malformed nested evidence', () => { + const valid = validateEvalRunInput({ + schemaVersion: 'lingxiloop.eval.v1', + suiteKey: 'regression-v1', + version: 'abc123', + cases: [{ + caseId: 'grounded', + sourceAgentRunId: 'run-1', + expectations: { requiredStages: ['answer', 'rag'], rag: { requiredSourceIds: ['source-1'] } }, + }], + }) + assert.equal(valid.cases[0].sourceAgentRunId, 'run-1') + assert.throws(() => validateEvalRunInput({ + suiteKey: 'regression-v1', + version: 'abc123', + cases: [{ + caseId: 'broken', + observation: { citations: [{ marker: 'S1' }] }, + expectations: { requiredStages: ['unknown'] }, + }], + }), EvalInputError) +}) diff --git a/server/src/agent-os/runtime.ts b/server/src/agent-os/runtime.ts index a6fde7d0..baad6582 100644 --- a/server/src/agent-os/runtime.ts +++ b/server/src/agent-os/runtime.ts @@ -170,6 +170,22 @@ export class AgentOSRuntime { return } const context = await this.host.loadContext(work) + // Persist only evidence identity/traceability metadata — never excerpts — + // so an Eval run can score RAG recall and citation validity later without + // copying potentially sensitive source text into the observability ledger. + await this.event(work, runId, { + kind: 'knowledge.context.loaded', stage: 'completed', visibility: 'internal', + data: { + sourceCount: context.knowledgeSourceCount ?? 0, + citations: (context.knowledgeContext ?? []).map((citation) => ({ + sourceId: citation.sourceId, + chunkId: citation.chunkId, + marker: citation.marker, + title: citation.sourceTitle, + })), + ...(context.knowledgeIngestionFailure ? { ingestionFailure: context.knowledgeIngestionFailure } : {}), + }, + }) const dynamicKnowledgeItems = knowledgeItems(context) const key = sessionKey(work) const stored = await this.host.loadSession(key) diff --git a/server/src/api/admin-router.ts b/server/src/api/admin-router.ts index 4ddff6f1..19d4ec71 100644 --- a/server/src/api/admin-router.ts +++ b/server/src/api/admin-router.ts @@ -14,16 +14,19 @@ * - All handlers wrapped in safe() so HttpError → status code; the * parent router's errorHandler catches everything else. */ -import { Router, type Request, type Response, type NextFunction } from 'express' -import { pool } from '../db/pool.js' -import { gravatarUrlForEmail, type AuthedRequest } from '../auth.js' +import { type NextFunction, type Request, type Response, Router } from 'express' import { - requireAdmin, HttpError, - getSettings, setSetting, type AppSettings, - listWaitlist, approveWaitlist, rejectWaitlist, + type AppSettings, approveWaitlist, changeUserTier, + getSettings, HttpError, + listWaitlist, rejectWaitlist, + requireAdmin, setSetting, suspendUser, unsuspendUser, } from '../admin.js' +import { type AuthedRequest, gravatarUrlForEmail } from '../auth.js' +import { pool } from '../db/pool.js' +import { EvalInputError, validateEvalRunInput } from '../eval/contracts.js' +import { createEvalRun, getEvalDashboard, getEvalRunDetail } from '../eval/service.js' export const adminRouter = Router() @@ -351,6 +354,57 @@ adminRouter.get('/stats', safe(async (req, res) => { }) })) +/* ============== Agent Eval — deterministic pipeline + history ========= */ + +function rethrowEvalError(error: unknown): never { + if (error instanceof EvalInputError) throw new HttpError(400, error.message) + const status = Number((error as { status?: unknown } | null)?.status) + if (status >= 400 && status <= 599) { + throw new HttpError(status, error instanceof Error ? error.message : String(error)) + } + throw error +} + +/** Evaluate one immutable suite run. Cases may contain an inline observation, + * an Agent OS run id to hydrate, or both (inline fields override hydrated + * fields for controlled regression fixtures). Evaluation is synchronous and + * deterministic; a successful response means the full report was committed. */ +adminRouter.post('/eval/runs', safe(async (req, res) => { + const adminId = await requireAdmin(req) + try { + const input = validateEvalRunInput(req.body) + const result = await createEvalRun(input, adminId) + res.status(201).json(result) + } catch (error) { + rethrowEvalError(error) + } +})) + +/** Compact board payload: summary KPIs, stage averages, version deltas and the + * recent immutable run list. Detail/findings are loaded only on selection. */ +adminRouter.get('/eval/runs', safe(async (req, res) => { + await requireAdmin(req) + const suiteKey = typeof req.query.suiteKey === 'string' && req.query.suiteKey.trim() + ? req.query.suiteKey.trim() + : undefined + const rawLimit = Number(req.query.limit ?? 80) + const rawDays = Number(req.query.sinceDays ?? 90) + res.json(await getEvalDashboard({ + suiteKey, + limit: Number.isFinite(rawLimit) ? rawLimit : 80, + sinceDays: Number.isFinite(rawDays) ? rawDays : 90, + })) +})) + +adminRouter.get('/eval/runs/:id', safe(async (req, res) => { + await requireAdmin(req) + try { + res.json(await getEvalRunDetail(String(req.params.id))) + } catch (error) { + rethrowEvalError(error) + } +})) + /* ============== Observability — per-purpose sub2api spend =========== */ /** Hero KPIs + per-purpose rollup + daily trend + top-spenders, in ONE diff --git a/server/src/db/migrate.ts b/server/src/db/migrate.ts index e63b89f8..c81de477 100644 --- a/server/src/db/migrate.ts +++ b/server/src/db/migrate.ts @@ -2503,6 +2503,70 @@ CREATE TABLE IF NOT EXISTS course_schema_cutovers ( detail JSONB NOT NULL DEFAULT '{}'::jsonb ); +-- ============== Agent Eval runs ======================================== +-- Deterministic, offline evaluation reports. Raw Agent OS traces remain in +-- their authoritative ledgers; these tables store the immutable observation +-- snapshot and per-stage findings used by the version comparison dashboard. +CREATE TABLE IF NOT EXISTS eval_runs ( + id TEXT PRIMARY KEY, + suite_key TEXT NOT NULL, + suite_name TEXT NOT NULL, + version TEXT NOT NULL, + baseline_run_id TEXT REFERENCES eval_runs(id) ON DELETE SET NULL, + status TEXT NOT NULL CHECK (status IN ('pass','fail','error')), + score DOUBLE PRECISION NOT NULL CHECK (score BETWEEN 0 AND 1), + pass_threshold DOUBLE PRECISION NOT NULL CHECK (pass_threshold BETWEEN 0 AND 1), + case_count INTEGER NOT NULL, + passed_cases INTEGER NOT NULL, + failed_cases INTEGER NOT NULL, + error_cases INTEGER NOT NULL DEFAULT 0, + source TEXT NOT NULL CHECK (source IN ('inline','agent-os','mixed')), + summary JSONB NOT NULL DEFAULT '{}'::jsonb, + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + created_by TEXT NOT NULL, + started_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + finished_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() +); +CREATE INDEX IF NOT EXISTS idx_eval_runs_suite_created ON eval_runs(suite_key, created_at DESC); +CREATE INDEX IF NOT EXISTS idx_eval_runs_status_created ON eval_runs(status, created_at DESC); + +CREATE TABLE IF NOT EXISTS eval_cases ( + id TEXT PRIMARY KEY, + eval_run_id TEXT NOT NULL REFERENCES eval_runs(id) ON DELETE CASCADE, + case_key TEXT NOT NULL, + name TEXT NOT NULL, + position INTEGER NOT NULL, + source_agent_run_id TEXT, + status TEXT NOT NULL CHECK (status IN ('pass','fail','error')), + score DOUBLE PRECISION NOT NULL CHECK (score BETWEEN 0 AND 1), + observation JSONB NOT NULL, + expectations JSONB NOT NULL, + failure_reasons JSONB NOT NULL DEFAULT '[]'::jsonb, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + UNIQUE(eval_run_id, case_key) +); +CREATE INDEX IF NOT EXISTS idx_eval_cases_run_position ON eval_cases(eval_run_id, position); +CREATE INDEX IF NOT EXISTS idx_eval_cases_source_run ON eval_cases(source_agent_run_id) WHERE source_agent_run_id IS NOT NULL; + +CREATE TABLE IF NOT EXISTS eval_stage_results ( + id TEXT PRIMARY KEY, + eval_run_id TEXT NOT NULL REFERENCES eval_runs(id) ON DELETE CASCADE, + eval_case_id TEXT NOT NULL REFERENCES eval_cases(id) ON DELETE CASCADE, + stage TEXT NOT NULL CHECK (stage IN ('ingest','answer','rag','tools','collaboration','aggregate')), + position INTEGER NOT NULL, + status TEXT NOT NULL CHECK (status IN ('pass','fail','skipped','error')), + score DOUBLE PRECISION CHECK (score BETWEEN 0 AND 1), + duration_ms INTEGER NOT NULL DEFAULT 0, + findings JSONB NOT NULL DEFAULT '[]'::jsonb, + metrics JSONB NOT NULL DEFAULT '{}'::jsonb, + failure_reason TEXT, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + UNIQUE(eval_case_id, stage) +); +CREATE INDEX IF NOT EXISTS idx_eval_stages_run ON eval_stage_results(eval_run_id, position); +CREATE INDEX IF NOT EXISTS idx_eval_stages_failures ON eval_stage_results(eval_run_id, status) WHERE status IN ('fail','error'); + CREATE OR REPLACE FUNCTION touch_knowledge_workspace_updated_at() RETURNS trigger AS $$ BEGIN UPDATE projects SET updated_at = NOW() WHERE id = COALESCE(NEW.project_id, OLD.project_id); @@ -3207,6 +3271,9 @@ async function schemaAlreadyCurrent(client: import('pg').PoolClient): Promise 0 AND (SELECT count(*) FROM pg_class WHERE relname = 'course_invitations') > 0 AND (SELECT count(*) FROM pg_class WHERE relname = 'course_schema_cutovers') > 0 + AND (SELECT count(*) FROM pg_class WHERE relname = 'eval_runs') > 0 + AND (SELECT count(*) FROM pg_class WHERE relname = 'eval_cases') > 0 + AND (SELECT count(*) FROM pg_class WHERE relname = 'eval_stage_results') > 0 AS ok `) return rows[0]?.ok === true diff --git a/server/src/eval/contracts.ts b/server/src/eval/contracts.ts new file mode 100644 index 00000000..096dc235 --- /dev/null +++ b/server/src/eval/contracts.ts @@ -0,0 +1,354 @@ +export const EVAL_SCHEMA_VERSION = 'lingxiloop.eval.v1' as const + +export type EvalStage = 'ingest' | 'answer' | 'rag' | 'tools' | 'collaboration' | 'aggregate' +export type EvalStatus = 'pass' | 'fail' | 'error' +export type EvalStageStatus = 'pass' | 'fail' | 'skipped' | 'error' +export type EvalFindingStatus = 'pass' | 'fail' | 'not_observed' + +export interface EvalFinding { + checkId: string + status: EvalFindingStatus + severity: 'info' | 'warning' | 'error' + message: string + expected?: unknown + actual?: unknown +} + +export interface EvalCitationObservation { + sourceId: string + chunkId?: string + marker?: string + title?: string +} + +export interface EvalToolCallObservation { + name: string + args?: unknown + result?: unknown + status?: 'ok' | 'error' | 'pending' + durationMs?: number +} + +export interface EvalAgentTurnObservation { + agentId: string + role?: string + status?: string + handoffTo?: string + startedAt?: string + finishedAt?: string + error?: string +} + +export interface EvalObservation { + answer?: string + retrievedSourceIds?: string[] + citedSourceIds?: string[] + citations?: EvalCitationObservation[] + toolCalls?: EvalToolCallObservation[] + agentTurns?: EvalAgentTurnObservation[] + latencyMs?: number + tokenCount?: number + error?: string + metadata?: Record +} + +export interface AnswerExpectations { + referenceAnswer?: string + requiredKeywords?: string[] + forbiddenPatterns?: string[] + minLength?: number + maxLength?: number + minSimilarity?: number + maxLatencyMs?: number + maxTokens?: number +} + +export interface RagExpectations { + requiredSourceIds?: string[] + requireCitations?: boolean + minRetrievalRecall?: number + minCitationPrecision?: number +} + +export interface ExpectedToolCall { + name: string + argsSubset?: unknown + required?: boolean +} + +export interface ToolExpectations { + calls?: ExpectedToolCall[] + allowedToolNames?: string[] + forbiddenToolNames?: string[] + allowUnexpected?: boolean + enforceOrder?: boolean + requireSuccess?: boolean + maxCalls?: number +} + +export interface CollaborationExpectations { + requiredAgentIds?: string[] + minAgents?: number + maxHandoffs?: number + maxFailedAgents?: number + requireAllCompleted?: boolean + requireParallelism?: boolean +} + +export interface EvalCaseExpectations { + answer?: AnswerExpectations + rag?: RagExpectations + tools?: ToolExpectations + collaboration?: CollaborationExpectations + requiredStages?: Array> + passThreshold?: number + weights?: Partial> +} + +export interface EvalCaseInput { + caseId: string + name?: string + sourceAgentRunId?: string + observation?: EvalObservation + expectations: EvalCaseExpectations + metadata?: Record +} + +export interface EvalRunInput { + schemaVersion?: typeof EVAL_SCHEMA_VERSION + suiteKey: string + suiteName?: string + version: string + baselineRunId?: string + passThreshold?: number + cases: EvalCaseInput[] + metadata?: Record +} + +export interface EvalStageResult { + stage: EvalStage + status: EvalStageStatus + score: number | null + durationMs: number + findings: EvalFinding[] + metrics: Record + failureReason: string | null +} + +export interface EvalCaseReport { + caseId: string + name: string + sourceAgentRunId: string | null + status: EvalStatus + score: number + observation: EvalObservation + expectations: EvalCaseExpectations + stages: EvalStageResult[] + failureReasons: string[] +} + +export interface EvalRunReport { + schemaVersion: typeof EVAL_SCHEMA_VERSION + suiteKey: string + suiteName: string + version: string + baselineRunId: string | null + status: EvalStatus + score: number + passThreshold: number + summary: { + caseCount: number + passedCases: number + failedCases: number + errorCases: number + stageScores: Record<'answer' | 'rag' | 'tools' | 'collaboration', number | null> + } + cases: EvalCaseReport[] +} + +export class EvalInputError extends Error { + constructor(message: string) { + super(message) + this.name = 'EvalInputError' + } +} + +function isObject(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value) +} + +function assertStringArray(record: Record, key: string, path: string): void { + const value = record[key] + if (value !== undefined && (!Array.isArray(value) || value.some((item) => typeof item !== 'string'))) { + throw new EvalInputError(`${path}.${key} must be an array of strings`) + } +} + +function assertOptionalNumber(record: Record, key: string, path: string, options: { integer?: boolean; max?: number } = {}): void { + const value = record[key] + if (value === undefined) return + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0 || + (options.integer && !Number.isInteger(value)) || (options.max !== undefined && value > options.max)) { + throw new EvalInputError(`${path}.${key} must be a ${options.integer ? 'non-negative integer' : 'non-negative number'}${options.max !== undefined ? ` up to ${options.max}` : ''}`) + } +} + +function assertOptionalBoolean(record: Record, key: string, path: string): void { + if (record[key] !== undefined && typeof record[key] !== 'boolean') { + throw new EvalInputError(`${path}.${key} must be a boolean`) + } +} + +function assertOptionalRecord(record: Record, key: string, path: string): void { + if (record[key] !== undefined && !isObject(record[key])) { + throw new EvalInputError(`${path}.${key} must be an object`) + } +} + +function validateObservation(value: unknown, path: string): void { + if (!isObject(value)) throw new EvalInputError(`${path} must be an object`) + for (const key of ['answer', 'error'] as const) { + if (value[key] !== undefined && typeof value[key] !== 'string') throw new EvalInputError(`${path}.${key} must be a string`) + } + assertStringArray(value, 'retrievedSourceIds', path) + assertStringArray(value, 'citedSourceIds', path) + assertOptionalNumber(value, 'latencyMs', path) + assertOptionalNumber(value, 'tokenCount', path, { integer: true }) + for (const [key, identity] of [['citations', 'sourceId'], ['toolCalls', 'name'], ['agentTurns', 'agentId']] as const) { + const items = value[key] + if (items === undefined) continue + if (!Array.isArray(items) || items.some((item) => !isObject(item) || typeof item[identity] !== 'string' || !item[identity])) { + throw new EvalInputError(`${path}.${key} must be an array of objects with ${identity}`) + } + } + for (const item of Array.isArray(value.toolCalls) ? value.toolCalls : []) { + if (!isObject(item)) continue + if (item.status !== undefined && !['ok', 'error', 'pending'].includes(String(item.status))) { + throw new EvalInputError(`${path}.toolCalls[].status is unsupported`) + } + assertOptionalNumber(item, 'durationMs', `${path}.toolCalls[]`) + } + assertOptionalRecord(value, 'metadata', path) +} + +function validateExpectations(value: Record, path: string): void { + const allowedStages = new Set(['answer', 'rag', 'tools', 'collaboration']) + if (value.requiredStages !== undefined && (!Array.isArray(value.requiredStages) || + value.requiredStages.some((item) => typeof item !== 'string' || !allowedStages.has(item)))) { + throw new EvalInputError(`${path}.requiredStages contains an unsupported stage`) + } + assertOptionalNumber(value, 'passThreshold', path, { max: 1 }) + if (value.weights !== undefined) { + if (!isObject(value.weights)) throw new EvalInputError(`${path}.weights must be an object`) + for (const [key, weight] of Object.entries(value.weights)) { + if (!allowedStages.has(key) || typeof weight !== 'number' || !Number.isFinite(weight) || weight < 0) { + throw new EvalInputError(`${path}.weights contains an invalid stage or weight`) + } + } + } + for (const stage of allowedStages) { + if (value[stage] !== undefined && !isObject(value[stage])) throw new EvalInputError(`${path}.${stage} must be an object`) + } + const answer = isObject(value.answer) ? value.answer : null + if (answer) { + assertStringArray(answer, 'requiredKeywords', `${path}.answer`) + assertStringArray(answer, 'forbiddenPatterns', `${path}.answer`) + for (const key of ['minLength', 'maxLength', 'maxLatencyMs', 'maxTokens'] as const) assertOptionalNumber(answer, key, `${path}.answer`, { integer: true }) + assertOptionalNumber(answer, 'minSimilarity', `${path}.answer`, { max: 1 }) + if (answer.referenceAnswer !== undefined && typeof answer.referenceAnswer !== 'string') throw new EvalInputError(`${path}.answer.referenceAnswer must be a string`) + } + const rag = isObject(value.rag) ? value.rag : null + if (rag) { + assertStringArray(rag, 'requiredSourceIds', `${path}.rag`) + assertOptionalNumber(rag, 'minRetrievalRecall', `${path}.rag`, { max: 1 }) + assertOptionalNumber(rag, 'minCitationPrecision', `${path}.rag`, { max: 1 }) + assertOptionalBoolean(rag, 'requireCitations', `${path}.rag`) + } + const tools = isObject(value.tools) ? value.tools : null + if (tools) { + assertStringArray(tools, 'allowedToolNames', `${path}.tools`) + assertStringArray(tools, 'forbiddenToolNames', `${path}.tools`) + assertOptionalNumber(tools, 'maxCalls', `${path}.tools`, { integer: true }) + if (tools.calls !== undefined && (!Array.isArray(tools.calls) || tools.calls.some((item) => !isObject(item) || typeof item.name !== 'string' || !item.name))) { + throw new EvalInputError(`${path}.tools.calls must be an array of objects with name`) + } + for (const item of Array.isArray(tools.calls) ? tools.calls : []) { + if (isObject(item)) assertOptionalBoolean(item, 'required', `${path}.tools.calls[]`) + } + for (const key of ['allowUnexpected', 'enforceOrder', 'requireSuccess'] as const) { + assertOptionalBoolean(tools, key, `${path}.tools`) + } + } + const collaboration = isObject(value.collaboration) ? value.collaboration : null + if (collaboration) { + assertStringArray(collaboration, 'requiredAgentIds', `${path}.collaboration`) + for (const key of ['minAgents', 'maxHandoffs', 'maxFailedAgents'] as const) assertOptionalNumber(collaboration, key, `${path}.collaboration`, { integer: true }) + for (const key of ['requireAllCompleted', 'requireParallelism'] as const) { + assertOptionalBoolean(collaboration, key, `${path}.collaboration`) + } + } +} + +export function validateEvalRunInput(value: unknown): EvalRunInput { + if (!isObject(value)) throw new EvalInputError('request body must be an object') + if (value.schemaVersion !== undefined && value.schemaVersion !== EVAL_SCHEMA_VERSION) { + throw new EvalInputError(`schemaVersion must be ${EVAL_SCHEMA_VERSION}`) + } + const suiteKey = typeof value.suiteKey === 'string' ? value.suiteKey.trim() : '' + const version = typeof value.version === 'string' ? value.version.trim() : '' + if (!/^[a-z0-9][a-z0-9._-]{0,79}$/i.test(suiteKey)) { + throw new EvalInputError('suiteKey must contain 1-80 letters, numbers, dots, underscores, or dashes') + } + if (!version || version.length > 120) throw new EvalInputError('version must contain 1-120 characters') + if (value.suiteName !== undefined && (typeof value.suiteName !== 'string' || !value.suiteName.trim() || value.suiteName.trim().length > 160)) { + throw new EvalInputError('suiteName must contain 1-160 characters') + } + if (value.baselineRunId !== undefined && (typeof value.baselineRunId !== 'string' || !value.baselineRunId.trim())) { + throw new EvalInputError('baselineRunId must be a non-empty string') + } + assertOptionalRecord(value, 'metadata', 'request') + if (!Array.isArray(value.cases) || value.cases.length === 0 || value.cases.length > 100) { + throw new EvalInputError('cases must contain between 1 and 100 items') + } + const seen = new Set() + for (const [index, rawCase] of value.cases.entries()) { + if (!isObject(rawCase)) throw new EvalInputError(`cases[${index}] must be an object`) + const caseId = typeof rawCase.caseId === 'string' ? rawCase.caseId.trim() : '' + if (!caseId || caseId.length > 120) throw new EvalInputError(`cases[${index}].caseId must contain 1-120 characters`) + if (seen.has(caseId)) throw new EvalInputError(`duplicate caseId: ${caseId}`) + seen.add(caseId) + if (!isObject(rawCase.expectations)) throw new EvalInputError(`cases[${index}].expectations must be an object`) + if (rawCase.name !== undefined && (typeof rawCase.name !== 'string' || !rawCase.name.trim() || rawCase.name.trim().length > 160)) { + throw new EvalInputError(`cases[${index}].name must contain 1-160 characters`) + } + assertOptionalRecord(rawCase, 'metadata', `cases[${index}]`) + validateExpectations(rawCase.expectations, `cases[${index}].expectations`) + if (rawCase.sourceAgentRunId !== undefined && (typeof rawCase.sourceAgentRunId !== 'string' || !rawCase.sourceAgentRunId.trim())) { + throw new EvalInputError(`cases[${index}].sourceAgentRunId must be a non-empty string`) + } + if (rawCase.observation !== undefined) validateObservation(rawCase.observation, `cases[${index}].observation`) + if (!rawCase.sourceAgentRunId && !isObject(rawCase.observation)) { + throw new EvalInputError(`cases[${index}] must provide sourceAgentRunId or observation`) + } + } + const threshold = value.passThreshold + if (threshold !== undefined && (typeof threshold !== 'number' || !Number.isFinite(threshold) || threshold < 0 || threshold > 1)) { + throw new EvalInputError('passThreshold must be between 0 and 1') + } + return { + ...value, + suiteKey, + version, + ...(typeof value.suiteName === 'string' ? { suiteName: value.suiteName.trim() } : {}), + ...(typeof value.baselineRunId === 'string' ? { baselineRunId: value.baselineRunId.trim() } : {}), + cases: value.cases.map((rawCase) => { + const item = rawCase as Record + return { + ...item, + caseId: String(item.caseId).trim(), + ...(typeof item.name === 'string' ? { name: item.name.trim() } : {}), + ...(typeof item.sourceAgentRunId === 'string' ? { sourceAgentRunId: item.sourceAgentRunId.trim() } : {}), + } + }), + } as unknown as EvalRunInput +} diff --git a/server/src/eval/evaluator.ts b/server/src/eval/evaluator.ts new file mode 100644 index 00000000..705330f0 --- /dev/null +++ b/server/src/eval/evaluator.ts @@ -0,0 +1,420 @@ +import { + type AnswerExpectations, + type CollaborationExpectations, + EVAL_SCHEMA_VERSION, + type EvalCaseInput, + type EvalCaseReport, + type EvalFinding, + type EvalObservation, + type EvalRunInput, + type EvalRunReport, + type EvalStage, + type EvalStageResult, + type RagExpectations, + type ToolExpectations, +} from './contracts.js' + +const DEFAULT_PASS_THRESHOLD = 0.8 +const STAGE_THRESHOLDS: Record<'answer' | 'rag' | 'tools' | 'collaboration', number> = { + answer: 0.75, + rag: 0.75, + tools: 1, + collaboration: 0.8, +} +const DEFAULT_WEIGHTS = { answer: 0.35, rag: 0.25, tools: 0.2, collaboration: 0.2 } + +function clamp01(value: number): number { + return Math.max(0, Math.min(1, value)) +} + +function round(value: number, places = 4): number { + const factor = 10 ** places + return Math.round(value * factor) / factor +} + +function finding( + checkId: string, + status: EvalFinding['status'], + message: string, + options: { severity?: EvalFinding['severity']; expected?: unknown; actual?: unknown } = {}, +): EvalFinding { + return { + checkId, + status, + severity: status === 'pass' ? 'info' : options.severity ?? (status === 'fail' ? 'error' : 'warning'), + message, + ...(options.expected !== undefined ? { expected: options.expected } : {}), + ...(options.actual !== undefined ? { actual: options.actual } : {}), + } +} + +function stageResult( + stage: EvalStage, + startedAt: number, + findings: EvalFinding[], + metrics: EvalStageResult['metrics'] = {}, + threshold?: number, +): EvalStageResult { + const observed = findings.filter((item) => item.status !== 'not_observed') + const passed = observed.filter((item) => item.status === 'pass').length + const score = observed.length ? round(passed / observed.length) : null + const hardFailure = findings.some((item) => item.status === 'fail' && item.severity === 'error') + const status = score === null ? 'skipped' : hardFailure || score < (threshold ?? 1) ? 'fail' : 'pass' + const failed = findings.find((item) => item.status === 'fail') + return { + stage, + status, + score, + durationMs: Math.max(0, Date.now() - startedAt), + findings, + metrics, + failureReason: failed?.message ?? null, + } +} + +function normalizedText(value: string): string { + return value.normalize('NFKC').toLocaleLowerCase().replace(/\s+/g, ' ').trim() +} + +function textFeatures(value: string): Set { + const normalized = normalizedText(value) + const features = new Set() + for (const token of normalized.match(/[a-z0-9]+(?:[-_.][a-z0-9]+)*/g) ?? []) features.add(token) + const cjkRuns = normalized.match(/[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]+/gu) ?? [] + for (const run of cjkRuns) { + // CJK text has no whitespace word boundary. Unigrams preserve overlap + // across small paraphrases; bigrams reward local phrase agreement. + for (const character of run) features.add(character) + for (let index = 0; index < run.length - 1; index += 1) features.add(run.slice(index, index + 2)) + } + return features +} + +export function answerSimilarity(actual: string, reference: string): number { + const left = textFeatures(actual) + const right = textFeatures(reference) + if (left.size === 0 || right.size === 0) return normalizedText(actual) === normalizedText(reference) ? 1 : 0 + let intersection = 0 + for (const token of left) if (right.has(token)) intersection += 1 + const precision = intersection / left.size + const recall = intersection / right.size + return precision + recall === 0 ? 0 : round((2 * precision * recall) / (precision + recall)) +} + +function evaluateAnswer(observation: EvalObservation, expected: AnswerExpectations): EvalStageResult { + const startedAt = Date.now() + const findings: EvalFinding[] = [] + const answer = observation.answer?.trim() ?? '' + findings.push(finding('answer.response_present', answer ? 'pass' : 'fail', answer ? '回答已生成' : '没有可评测的 Agent 回答')) + if (observation.error) { + findings.push(finding('answer.runtime_error', 'fail', `Agent 运行失败：${observation.error}`, { actual: observation.error })) + } else { + findings.push(finding('answer.runtime_error', 'pass', 'Agent 运行未报告错误')) + } + for (const keyword of expected.requiredKeywords ?? []) { + const found = normalizedText(answer).includes(normalizedText(keyword)) + findings.push(finding('answer.required_keyword', found ? 'pass' : 'fail', found ? `包含关键点“${keyword}”` : `缺少关键点“${keyword}”`, { + expected: keyword, + })) + } + for (const pattern of expected.forbiddenPatterns ?? []) { + let matched = false + try { matched = new RegExp(pattern, 'iu').test(answer) } catch { matched = normalizedText(answer).includes(normalizedText(pattern)) } + findings.push(finding('answer.forbidden_pattern', matched ? 'fail' : 'pass', matched ? `命中禁止内容“${pattern}”` : `未命中禁止内容“${pattern}”`, { + expected: pattern, + })) + } + if (expected.minLength !== undefined) { + findings.push(finding('answer.min_length', answer.length >= expected.minLength ? 'pass' : 'fail', + answer.length >= expected.minLength ? '回答长度达到下限' : `回答过短：${answer.length} < ${expected.minLength}`, + { expected: expected.minLength, actual: answer.length })) + } + if (expected.maxLength !== undefined) { + findings.push(finding('answer.max_length', answer.length <= expected.maxLength ? 'pass' : 'fail', + answer.length <= expected.maxLength ? '回答长度未超过上限' : `回答过长：${answer.length} > ${expected.maxLength}`, + { expected: expected.maxLength, actual: answer.length })) + } + let similarity: number | null = null + if (expected.referenceAnswer) { + similarity = answerSimilarity(answer, expected.referenceAnswer) + const minimum = expected.minSimilarity ?? 0.55 + findings.push(finding('answer.reference_similarity', similarity >= minimum ? 'pass' : 'fail', + similarity >= minimum ? `参考答案相似度 ${similarity}` : `参考答案相似度 ${similarity} 低于 ${minimum}`, + { expected: minimum, actual: similarity })) + } + if (expected.maxLatencyMs !== undefined) { + const actual = observation.latencyMs + findings.push(finding('answer.latency_budget', actual !== undefined && actual <= expected.maxLatencyMs ? 'pass' : 'fail', + actual !== undefined && actual <= expected.maxLatencyMs ? '响应时延在预算内' : `响应时延 ${actual ?? '未观测'}ms 超出预算`, + { expected: expected.maxLatencyMs, actual: actual ?? null })) + } + if (expected.maxTokens !== undefined) { + const actual = observation.tokenCount + findings.push(finding('answer.token_budget', actual !== undefined && actual <= expected.maxTokens ? 'pass' : 'fail', + actual !== undefined && actual <= expected.maxTokens ? 'Token 用量在预算内' : `Token 用量 ${actual ?? '未观测'} 超出预算`, + { expected: expected.maxTokens, actual: actual ?? null })) + } + return stageResult('answer', startedAt, findings, { + answerLength: answer.length, + similarity, + latencyMs: observation.latencyMs ?? null, + tokenCount: observation.tokenCount ?? null, + }, STAGE_THRESHOLDS.answer) +} + +function evaluateRag(observation: EvalObservation, expected: RagExpectations): EvalStageResult { + const startedAt = Date.now() + const findings: EvalFinding[] = [] + const retrieved = new Set(observation.retrievedSourceIds ?? observation.citations?.map((item) => item.sourceId) ?? []) + const answer = observation.answer ?? '' + const markersInAnswer = new Set([...answer.matchAll(/\[(S\d+)\]/gi)].map((match) => match[1].toUpperCase())) + const citationsByMarker = new Map((observation.citations ?? []).filter((item) => item.marker).map((item) => [String(item.marker).toUpperCase(), item.sourceId])) + const cited = new Set(observation.citedSourceIds ?? [...markersInAnswer].flatMap((marker) => citationsByMarker.get(marker) ?? [])) + const required = new Set(expected.requiredSourceIds ?? []) + let recall: number | null = null + if (required.size > 0) { + recall = [...required].filter((sourceId) => retrieved.has(sourceId)).length / required.size + const minimum = expected.minRetrievalRecall ?? 1 + findings.push(finding('rag.retrieval_recall', recall >= minimum ? 'pass' : 'fail', + recall >= minimum ? `检索召回率 ${round(recall)}` : `检索召回率 ${round(recall)} 低于 ${minimum}`, + { expected: minimum, actual: round(recall) })) + } + if (expected.requireCitations) { + findings.push(finding('rag.citations_present', cited.size > 0 ? 'pass' : 'fail', cited.size > 0 ? '回答包含来源引用' : '回答缺少来源引用')) + } + const unknownMarkers = [...markersInAnswer].filter((marker) => !citationsByMarker.has(marker)) + if (markersInAnswer.size > 0 || observation.citations?.length) { + findings.push(finding('rag.marker_validity', unknownMarkers.length === 0 ? 'pass' : 'fail', + unknownMarkers.length === 0 ? '引用标记均可追溯' : `存在无法追溯的引用标记：${unknownMarkers.join(', ')}`, + { actual: unknownMarkers })) + } + let citationPrecision: number | null = null + if (cited.size > 0) { + citationPrecision = [...cited].filter((sourceId) => retrieved.has(sourceId)).length / cited.size + const minimum = expected.minCitationPrecision ?? 1 + findings.push(finding('rag.citation_precision', citationPrecision >= minimum ? 'pass' : 'fail', + citationPrecision >= minimum ? `引用准确率 ${round(citationPrecision)}` : `引用准确率 ${round(citationPrecision)} 低于 ${minimum}`, + { expected: minimum, actual: round(citationPrecision) })) + } else if (!expected.requireCitations) { + findings.push(finding('rag.citation_precision', 'not_observed', '没有引用可用于计算准确率')) + } + if (findings.length === 0) findings.push(finding('rag.evidence', 'not_observed', '未配置 RAG 检查项')) + return stageResult('rag', startedAt, findings, { + retrievedSources: retrieved.size, + citedSources: cited.size, + retrievalRecall: recall === null ? null : round(recall), + citationPrecision: citationPrecision === null ? null : round(citationPrecision), + }, STAGE_THRESHOLDS.rag) +} + +function isSubset(expected: unknown, actual: unknown): boolean { + if (Array.isArray(expected)) { + return Array.isArray(actual) && expected.every((item, index) => isSubset(item, actual[index])) + } + if (expected !== null && typeof expected === 'object') { + if (actual === null || typeof actual !== 'object' || Array.isArray(actual)) return false + return Object.entries(expected as Record).every(([key, value]) => + key in (actual as Record) && isSubset(value, (actual as Record)[key])) + } + return Object.is(expected, actual) +} + +function evaluateTools(observation: EvalObservation, expected: ToolExpectations): EvalStageResult { + const startedAt = Date.now() + const findings: EvalFinding[] = [] + const actual = observation.toolCalls ?? [] + const expectedCalls = expected.calls ?? [] + const matchedIndexes: number[] = [] + const usedIndexes = new Set() + for (const call of expectedCalls) { + const index = actual.findIndex((candidate, candidateIndex) => !usedIndexes.has(candidateIndex) && candidate.name === call.name && + (call.argsSubset === undefined || isSubset(call.argsSubset, candidate.args))) + if (index >= 0) { matchedIndexes.push(index); usedIndexes.add(index) } + if (call.required !== false) { + findings.push(finding('tools.required_call', index >= 0 ? 'pass' : 'fail', + index >= 0 ? `已调用必需工具 ${call.name}` : `缺少必需工具调用 ${call.name}`, + { expected: call })) + } + } + if (expected.enforceOrder && matchedIndexes.length > 1) { + const ordered = matchedIndexes.every((value, index) => index === 0 || value > matchedIndexes[index - 1]) + findings.push(finding('tools.call_order', ordered ? 'pass' : 'fail', ordered ? '工具调用顺序符合预期' : '工具调用顺序与预期不符')) + } + const forbidden = new Set(expected.forbiddenToolNames ?? []) + for (const call of actual.filter((item) => forbidden.has(item.name))) { + findings.push(finding('tools.forbidden_call', 'fail', `调用了禁止工具 ${call.name}`, { actual: call.name })) + } + const allowed = new Set(expected.allowedToolNames ?? expectedCalls.map((item) => item.name)) + if (expected.allowUnexpected === false && allowed.size > 0) { + const unexpected = actual.filter((call) => !allowed.has(call.name)).map((call) => call.name) + findings.push(finding('tools.unexpected_calls', unexpected.length === 0 ? 'pass' : 'fail', + unexpected.length === 0 ? '未发现意外工具调用' : `存在意外工具调用：${unexpected.join(', ')}`, { actual: unexpected })) + } + if (expected.requireSuccess) { + const failed = actual.filter((call) => call.status === 'error') + findings.push(finding('tools.execution_success', failed.length === 0 ? 'pass' : 'fail', + failed.length === 0 ? '工具调用均成功' : `${failed.length} 次工具调用失败`, { actual: failed.map((call) => call.name) })) + } + if (expected.maxCalls !== undefined) { + findings.push(finding('tools.call_budget', actual.length <= expected.maxCalls ? 'pass' : 'fail', + actual.length <= expected.maxCalls ? '工具调用次数在预算内' : `工具调用次数 ${actual.length} 超过 ${expected.maxCalls}`, + { expected: expected.maxCalls, actual: actual.length })) + } + if (findings.length === 0) findings.push(finding('tools.trace', 'not_observed', '未配置工具调用检查项')) + return stageResult('tools', startedAt, findings, { + callCount: actual.length, + failedCalls: actual.filter((call) => call.status === 'error').length, + uniqueTools: new Set(actual.map((call) => call.name)).size, + }, STAGE_THRESHOLDS.tools) +} + +function turnsOverlap(left: NonNullable[number], right: NonNullable[number]): boolean { + if (!left.startedAt || !left.finishedAt || !right.startedAt || !right.finishedAt) return false + const a0 = Date.parse(left.startedAt); const a1 = Date.parse(left.finishedAt) + const b0 = Date.parse(right.startedAt); const b1 = Date.parse(right.finishedAt) + return [a0, a1, b0, b1].every(Number.isFinite) && a0 < b1 && b0 < a1 +} + +function evaluateCollaboration(observation: EvalObservation, expected: CollaborationExpectations): EvalStageResult { + const startedAt = Date.now() + const findings: EvalFinding[] = [] + const turns = observation.agentTurns ?? [] + const agents = new Set(turns.map((turn) => turn.agentId)) + const failed = turns.filter((turn) => turn.status === 'failed' || turn.error) + const handoffs = turns.filter((turn) => turn.handoffTo) + for (const agentId of expected.requiredAgentIds ?? []) { + findings.push(finding('collaboration.required_agent', agents.has(agentId) ? 'pass' : 'fail', + agents.has(agentId) ? `Agent ${agentId} 已参与` : `缺少必需 Agent ${agentId}`, { expected: agentId })) + } + if (expected.minAgents !== undefined) { + findings.push(finding('collaboration.agent_count', agents.size >= expected.minAgents ? 'pass' : 'fail', + agents.size >= expected.minAgents ? `${agents.size} 个 Agent 参与协作` : `仅 ${agents.size} 个 Agent 参与，少于 ${expected.minAgents}`, + { expected: expected.minAgents, actual: agents.size })) + } + if (expected.maxHandoffs !== undefined) { + findings.push(finding('collaboration.handoff_budget', handoffs.length <= expected.maxHandoffs ? 'pass' : 'fail', + handoffs.length <= expected.maxHandoffs ? '交接次数在预算内' : `交接次数 ${handoffs.length} 超过 ${expected.maxHandoffs}`, + { expected: expected.maxHandoffs, actual: handoffs.length })) + } + if (expected.maxFailedAgents !== undefined) { + findings.push(finding('collaboration.failed_agents', failed.length <= expected.maxFailedAgents ? 'pass' : 'fail', + failed.length <= expected.maxFailedAgents ? '失败 Agent 数量在阈值内' : `${failed.length} 个 Agent 失败`, + { expected: expected.maxFailedAgents, actual: failed.length })) + } + if (expected.requireAllCompleted) { + const incomplete = turns.filter((turn) => turn.status !== 'completed') + findings.push(finding('collaboration.all_completed', turns.length > 0 && incomplete.length === 0 ? 'pass' : 'fail', + turns.length > 0 && incomplete.length === 0 ? '所有协作任务均已完成' : `${incomplete.length || turns.length} 个协作任务未完成`, + { actual: incomplete.map((turn) => ({ agentId: turn.agentId, status: turn.status })) })) + } + let parallelPairs = 0 + for (let left = 0; left < turns.length; left += 1) { + for (let right = left + 1; right < turns.length; right += 1) { + if (turns[left].agentId !== turns[right].agentId && turnsOverlap(turns[left], turns[right])) parallelPairs += 1 + } + } + if (expected.requireParallelism) { + findings.push(finding('collaboration.parallelism', parallelPairs > 0 ? 'pass' : 'fail', + parallelPairs > 0 ? '观测到并行 Agent 执行' : '未观测到并行 Agent 执行')) + } + if (findings.length === 0) findings.push(finding('collaboration.trace', 'not_observed', '未配置多 Agent 协作检查项')) + return stageResult('collaboration', startedAt, findings, { + agentCount: agents.size, + handoffCount: handoffs.length, + failedAgents: failed.length, + parallelPairs, + }, STAGE_THRESHOLDS.collaboration) +} + +function requiredStageFailure(stage: EvalStageResult, required: boolean): EvalStageResult { + if (!required || stage.status !== 'skipped') return stage + const findings = [...stage.findings, finding('coverage.required_stage', 'fail', `必需阶段 ${stage.stage} 缺少可评测证据`)] + return stageResult(stage.stage, Date.now(), findings, stage.metrics, STAGE_THRESHOLDS[stage.stage as keyof typeof STAGE_THRESHOLDS]) +} + +export function evaluateCase(input: EvalCaseInput, observation: EvalObservation, runThreshold = DEFAULT_PASS_THRESHOLD): EvalCaseReport { + const ingestStartedAt = Date.now() + const ingestFindings = [ + finding('ingest.observation', 'pass', input.sourceAgentRunId ? `已载入 Agent OS 运行 ${input.sourceAgentRunId}` : '已载入内联观测数据'), + finding('ingest.expectations', 'pass', '评测期望已校验'), + ] + const stages: EvalStageResult[] = [stageResult('ingest', ingestStartedAt, ingestFindings, {}, 1)] + const required = new Set(input.expectations.requiredStages ?? []) + const dimensionResults: EvalStageResult[] = [] + if (input.expectations.answer || required.has('answer')) { + dimensionResults.push(evaluateAnswer(observation, input.expectations.answer ?? {})) + } else { + dimensionResults.push(stageResult('answer', Date.now(), [finding('answer.configured', 'not_observed', '此用例未配置回答评测')])) + } + if (input.expectations.rag || required.has('rag')) { + dimensionResults.push(evaluateRag(observation, input.expectations.rag ?? {})) + } else { + dimensionResults.push(stageResult('rag', Date.now(), [finding('rag.configured', 'not_observed', '此用例未配置 RAG 评测')])) + } + if (input.expectations.tools || required.has('tools')) { + dimensionResults.push(evaluateTools(observation, input.expectations.tools ?? {})) + } else { + dimensionResults.push(stageResult('tools', Date.now(), [finding('tools.configured', 'not_observed', '此用例未配置工具评测')])) + } + if (input.expectations.collaboration || required.has('collaboration')) { + dimensionResults.push(evaluateCollaboration(observation, input.expectations.collaboration ?? {})) + } else { + dimensionResults.push(stageResult('collaboration', Date.now(), [finding('collaboration.configured', 'not_observed', '此用例未配置协作评测')])) + } + const gated = dimensionResults.map((stage) => requiredStageFailure(stage, required.has(stage.stage as never))) + stages.push(...gated) + const weights = { ...DEFAULT_WEIGHTS, ...(input.expectations.weights ?? {}) } + const observed = gated.filter((stage) => stage.score !== null) + const weightSum = observed.reduce((sum, stage) => sum + Math.max(0, weights[stage.stage as keyof typeof weights] ?? 0), 0) + const score = weightSum > 0 + ? round(observed.reduce((sum, stage) => sum + (stage.score ?? 0) * Math.max(0, weights[stage.stage as keyof typeof weights] ?? 0), 0) / weightSum) + : 0 + const threshold = input.expectations.passThreshold ?? runThreshold + const hardFailure = gated.some((stage) => stage.status === 'fail' || stage.status === 'error') + const status = hardFailure || score < threshold ? 'fail' : 'pass' + const failures = gated.flatMap((stage) => stage.findings.filter((item) => item.status === 'fail').map((item) => item.message)) + stages.push(stageResult('aggregate', Date.now(), [finding('aggregate.threshold', status === 'pass' ? 'pass' : 'fail', + status === 'pass' ? `综合分 ${score} 达到阈值 ${threshold}` : `综合分 ${score} 未通过阈值 ${threshold} 或存在阶段门控失败`, + { expected: threshold, actual: score })], { score, threshold }, 1)) + return { + caseId: input.caseId, + name: input.name?.trim() || input.caseId, + sourceAgentRunId: input.sourceAgentRunId ?? null, + status, + score, + observation, + expectations: input.expectations, + stages, + failureReasons: failures, + } +} + +export function evaluateRun(input: EvalRunInput, observations: Map): EvalRunReport { + const passThreshold = clamp01(input.passThreshold ?? DEFAULT_PASS_THRESHOLD) + const cases = input.cases.map((item) => evaluateCase(item, observations.get(item.caseId) ?? item.observation ?? {}, passThreshold)) + const score = round(cases.reduce((sum, item) => sum + item.score, 0) / cases.length) + const stageScores = Object.fromEntries((['answer', 'rag', 'tools', 'collaboration'] as const).map((stage) => { + const values = cases.flatMap((item) => item.stages.filter((candidate) => candidate.stage === stage && candidate.score !== null).map((candidate) => candidate.score as number)) + return [stage, values.length ? round(values.reduce((sum, value) => sum + value, 0) / values.length) : null] + })) as EvalRunReport['summary']['stageScores'] + const failedCases = cases.filter((item) => item.status === 'fail').length + const errorCases = cases.filter((item) => item.status === 'error').length + return { + schemaVersion: EVAL_SCHEMA_VERSION, + suiteKey: input.suiteKey, + suiteName: input.suiteName?.trim() || input.suiteKey, + version: input.version, + baselineRunId: input.baselineRunId ?? null, + status: failedCases > 0 || errorCases > 0 || score < passThreshold ? 'fail' : 'pass', + score, + passThreshold, + summary: { + caseCount: cases.length, + passedCases: cases.filter((item) => item.status === 'pass').length, + failedCases, + errorCases, + stageScores, + }, + cases, + } +} diff --git a/server/src/eval/service.ts b/server/src/eval/service.ts new file mode 100644 index 00000000..2bace4cd --- /dev/null +++ b/server/src/eval/service.ts @@ -0,0 +1,411 @@ +import { randomUUID } from 'node:crypto' +import type { PoolClient } from 'pg' +import { pool } from '../db/pool.js' +import type { + EvalAgentTurnObservation, + EvalCaseReport, + EvalObservation, + EvalRunInput, + EvalRunReport, + EvalStageResult, + EvalToolCallObservation, +} from './contracts.js' +import { evaluateRun } from './evaluator.js' + +interface AgentRunSourceRow { + id: string + agent_id: string + status: string + run_error: string | null + result_text: string | null + canvas_id: string | null + latency_ms: string | number | null + token_count: number + input_tokens: number + cached_input_tokens: number + cache_creation_tokens: number + output_tokens: number +} + +interface DashboardRunRow { + id: string + suite_key: string + suite_name: string + version: string + baseline_run_id: string | null + status: string + score: number + pass_threshold: number + case_count: number + passed_cases: number + failed_cases: number + error_cases: number + source: string + summary: EvalRunReport['summary'] + metadata: Record + created_by: string + created_at: string + finished_at: string + previous_score: number | null + explicit_baseline_score: number | null +} + +export interface EvalDashboardRun { + id: string + suiteKey: string + suiteName: string + version: string + baselineRunId: string | null + status: string + score: number + passThreshold: number + caseCount: number + passedCases: number + failedCases: number + errorCases: number + source: string + summary: EvalRunReport['summary'] + metadata: Record + createdBy: string + createdAt: string + finishedAt: string + baselineScore: number | null + scoreDelta: number | null +} + +function finiteNumber(value: unknown): number | undefined { + if (value === null || value === undefined || value === '') return undefined + const number = Number(value) + return Number.isFinite(number) ? number : undefined +} + +function jsonRecord(value: unknown): Record { + return value && typeof value === 'object' && !Array.isArray(value) ? value as Record : {} +} + +async function loadAgentRunObservation(runId: string): Promise { + const { rows } = await pool.query( + `SELECT r.id, r.agent_id, COALESCE(w.status, r.status) AS status, COALESCE(w.error, r.error) AS run_error, + w.result_text, w.canvas_id, + EXTRACT(EPOCH FROM (COALESCE(w.finished_at, r.finished_at, r.updated_at) - + COALESCE(w.lease_started_at, r.started_at))) * 1000 AS latency_ms, + r.token_count, r.input_tokens, r.cached_input_tokens, + r.cache_creation_tokens, r.output_tokens + FROM agent_runs r + LEFT JOIN agent_work_items w ON w.id = r.id + WHERE r.id = $1 + LIMIT 1`, + [runId], + ) + const run = rows[0] + if (!run) throw Object.assign(new Error(`Agent OS run not found: ${runId}`), { status: 404 }) + + const [eventsResult, hostActionsResult, legacyToolsResult] = await Promise.all([ + pool.query<{ kind: string; data: Record }>( + `SELECT kind,data FROM agent_events + WHERE run_id=$1 AND kind IN ('knowledge.context.loaded','model.completed') + ORDER BY created_at ASC`, + [runId], + ), + pool.query<{ action: string; args: unknown; result: unknown; status: string; error: string | null; created_at: string }>( + `SELECT action,args,result,status,error,created_at FROM agent_host_actions WHERE run_id=$1 ORDER BY created_at ASC`, + [runId], + ), + pool.query<{ name: string; args: unknown; result: unknown; status: string; error: string | null; duration_ms: number | null; created_at: string }>( + `SELECT name,args,result,status,error,duration_ms,created_at FROM tool_calls WHERE run_id=$1 ORDER BY created_at ASC`, + [runId], + ), + ]) + + const knowledge = jsonRecord(eventsResult.rows.find((row) => row.kind === 'knowledge.context.loaded')?.data) + const eventTokenCount = eventsResult.rows + .filter((row) => row.kind === 'model.completed') + .reduce((sum, row) => { + const usage = jsonRecord(jsonRecord(row.data).usage) + return sum + (finiteNumber(usage.inputTokens) ?? 0) + (finiteNumber(usage.outputTokens) ?? 0) + }, 0) + const citations = Array.isArray(knowledge.citations) + ? knowledge.citations.flatMap((item) => { + const value = jsonRecord(item) + const sourceId = typeof value.sourceId === 'string' ? value.sourceId : '' + return sourceId ? [{ + sourceId, + ...(typeof value.chunkId === 'string' ? { chunkId: value.chunkId } : {}), + ...(typeof value.marker === 'string' ? { marker: value.marker } : {}), + ...(typeof value.title === 'string' ? { title: value.title } : {}), + }] : [] + }) + : [] + const nativeTools: EvalToolCallObservation[] = hostActionsResult.rows.map((row) => ({ + name: row.action, + args: row.args, + result: row.result, + status: row.status === 'succeeded' ? 'ok' : row.status === 'failed' ? 'error' : 'pending', + })) + const legacyTools: EvalToolCallObservation[] = legacyToolsResult.rows.map((row) => ({ + name: row.name, + args: row.args, + result: row.result, + status: row.status === 'ok' ? 'ok' : row.status === 'error' ? 'error' : 'pending', + ...(row.duration_ms !== null ? { durationMs: row.duration_ms } : {}), + })) + + let agentTurns: EvalAgentTurnObservation[] = [{ + agentId: run.agent_id, + status: run.status, + ...(run.run_error ? { error: run.run_error } : {}), + }] + if (run.canvas_id) { + const [assignments, handoffs] = await Promise.all([ + pool.query<{ + agent_id: string; assignment: string; status: string; started_at: string | null + completed_at: string | null; error: string | null + }>( + `SELECT agent_id,assignment,status,started_at,completed_at,error + FROM canvas_agent_assignments WHERE canvas_id=$1 ORDER BY created_at ASC`, + [run.canvas_id], + ), + pool.query<{ actor_id: string; detail: Record }>( + `SELECT actor_id,detail FROM canvas_activity WHERE canvas_id=$1 AND action='handoff' ORDER BY created_at ASC`, + [run.canvas_id], + ), + ]) + const handoffByAgent = new Map() + for (const handoff of handoffs.rows) { + const to = jsonRecord(handoff.detail).toAgentId + if (typeof to === 'string') handoffByAgent.set(handoff.actor_id, to) + } + agentTurns = assignments.rows.map((row) => ({ + agentId: row.agent_id, + role: row.assignment, + status: row.status, + ...(handoffByAgent.has(row.agent_id) ? { handoffTo: handoffByAgent.get(row.agent_id) } : {}), + ...(row.started_at ? { startedAt: row.started_at } : {}), + ...(row.completed_at ? { finishedAt: row.completed_at } : {}), + ...(row.error ? { error: row.error } : {}), + })) + } + + const tokenBreakdown = run.input_tokens + run.cached_input_tokens + run.cache_creation_tokens + run.output_tokens + return { + answer: run.result_text ?? '', + retrievedSourceIds: [...new Set(citations.map((item) => item.sourceId))], + citations, + toolCalls: [...nativeTools, ...legacyTools], + agentTurns, + latencyMs: finiteNumber(run.latency_ms), + tokenCount: tokenBreakdown || run.token_count || eventTokenCount || undefined, + ...(run.run_error ? { error: run.run_error } : {}), + metadata: { sourceAgentRunId: runId, agentId: run.agent_id, agentStatus: run.status, canvasId: run.canvas_id }, + } +} + +async function resolveObservations(input: EvalRunInput): Promise> { + const observations = new Map() + // Keep hydration serial: one suite may contain 100 cases and each historical + // run fans out to several trace queries. Serial reads avoid turning a manual + // admin action into an accidental connection-pool flood. + for (const item of input.cases) { + observations.set(item.caseId, item.sourceAgentRunId + ? { ...(await loadAgentRunObservation(item.sourceAgentRunId)), ...(item.observation ?? {}) } + : item.observation ?? {}) + } + return observations +} + +function runSource(input: EvalRunInput): string { + const historical = input.cases.filter((item) => item.sourceAgentRunId).length + if (historical === 0) return 'inline' + return historical === input.cases.length ? 'agent-os' : 'mixed' +} + +async function persistCase(client: PoolClient, runId: string, item: EvalCaseReport, position: number): Promise { + const caseId = `eval-case-${randomUUID()}` + await client.query( + `INSERT INTO eval_cases + (id,eval_run_id,case_key,name,position,source_agent_run_id,status,score,observation,expectations,failure_reasons) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9::jsonb,$10::jsonb,$11::jsonb)`, + [caseId, runId, item.caseId, item.name, position, item.sourceAgentRunId, item.status, item.score, + JSON.stringify(item.observation), JSON.stringify(item.expectations), JSON.stringify(item.failureReasons)], + ) + for (const [stagePosition, stage] of item.stages.entries()) { + await client.query( + `INSERT INTO eval_stage_results + (id,eval_run_id,eval_case_id,stage,position,status,score,duration_ms,findings,metrics,failure_reason) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9::jsonb,$10::jsonb,$11)`, + [`eval-stage-${randomUUID()}`, runId, caseId, stage.stage, stagePosition, stage.status, stage.score, + stage.durationMs, JSON.stringify(stage.findings), JSON.stringify(stage.metrics), stage.failureReason], + ) + } +} + +export async function createEvalRun(input: EvalRunInput, createdBy: string): Promise<{ id: string; report: EvalRunReport }> { + if (input.baselineRunId) { + const baseline = await pool.query<{ suite_key: string }>(`SELECT suite_key FROM eval_runs WHERE id=$1`, [input.baselineRunId]) + if (!baseline.rows[0]) throw Object.assign(new Error('baseline eval run not found'), { status: 404 }) + if (baseline.rows[0].suite_key !== input.suiteKey) { + throw Object.assign(new Error('baseline eval run must belong to the same suiteKey'), { status: 409 }) + } + } + const observations = await resolveObservations(input) + const report = evaluateRun(input, observations) + const id = `eval-${randomUUID()}` + const client = await pool.connect() + try { + await client.query('BEGIN') + await client.query( + `INSERT INTO eval_runs + (id,suite_key,suite_name,version,baseline_run_id,status,score,pass_threshold,case_count, + passed_cases,failed_cases,error_cases,source,summary,metadata,created_by,started_at,finished_at) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14::jsonb,$15::jsonb,$16,NOW(),NOW())`, + [id, report.suiteKey, report.suiteName, report.version, report.baselineRunId, report.status, report.score, + report.passThreshold, report.summary.caseCount, report.summary.passedCases, report.summary.failedCases, + report.summary.errorCases, runSource(input), JSON.stringify(report.summary), JSON.stringify(input.metadata ?? {}), createdBy], + ) + for (const [position, item] of report.cases.entries()) await persistCase(client, id, item, position) + await client.query('COMMIT') + } catch (error) { + await client.query('ROLLBACK').catch(() => undefined) + throw error + } finally { + client.release() + } + return { id, report } +} + +function toDashboardRun(row: DashboardRunRow): EvalDashboardRun { + const baselineScore = row.explicit_baseline_score ?? row.previous_score + return { + id: row.id, + suiteKey: row.suite_key, + suiteName: row.suite_name, + version: row.version, + baselineRunId: row.baseline_run_id, + status: row.status, + score: Number(row.score), + passThreshold: Number(row.pass_threshold), + caseCount: row.case_count, + passedCases: row.passed_cases, + failedCases: row.failed_cases, + errorCases: row.error_cases, + source: row.source, + summary: row.summary, + metadata: row.metadata ?? {}, + createdBy: row.created_by, + createdAt: row.created_at, + finishedAt: row.finished_at, + baselineScore: baselineScore === null ? null : Number(baselineScore), + scoreDelta: baselineScore === null ? null : Number((Number(row.score) - Number(baselineScore)).toFixed(4)), + } +} + +export async function getEvalDashboard(args: { suiteKey?: string; limit?: number; sinceDays?: number } = {}): Promise<{ + summary: { totalRuns: number; passRate: number; averageScore: number; failedRuns: number; suites: number } + runs: EvalDashboardRun[] + stageAverages: EvalRunReport['summary']['stageScores'] +}> { + const limit = Math.min(200, Math.max(1, args.limit ?? 80)) + const sinceDays = Math.min(365, Math.max(1, args.sinceDays ?? 90)) + const params: unknown[] = [sinceDays] + let suiteWhere = '' + if (args.suiteKey) { + params.push(args.suiteKey) + suiteWhere = `AND r.suite_key=$${params.length}` + } + params.push(limit) + const { rows } = await pool.query( + `WITH scored AS ( + SELECT r.*, + LAG(r.score) OVER (PARTITION BY r.suite_key ORDER BY r.created_at,r.id) AS previous_score + FROM eval_runs r + WHERE r.created_at >= NOW() - ($1::double precision * INTERVAL '1 day') + ) + SELECT r.*, baseline.score AS explicit_baseline_score + FROM scored r + LEFT JOIN eval_runs baseline ON baseline.id=r.baseline_run_id + WHERE TRUE ${suiteWhere} + ORDER BY r.created_at DESC + LIMIT $${params.length}`, + params, + ) + const runs = rows.map(toDashboardRun) + const totalRuns = runs.length + const stageNames = ['answer', 'rag', 'tools', 'collaboration'] as const + const stageAverages = Object.fromEntries(stageNames.map((stage) => { + const values = runs.flatMap((run) => { + const value = run.summary?.stageScores?.[stage] + return typeof value === 'number' ? [value] : [] + }) + return [stage, values.length ? Number((values.reduce((sum, value) => sum + value, 0) / values.length).toFixed(4)) : null] + })) as EvalRunReport['summary']['stageScores'] + return { + summary: { + totalRuns, + passRate: totalRuns ? runs.filter((run) => run.status === 'pass').length / totalRuns : 0, + averageScore: totalRuns ? runs.reduce((sum, run) => sum + run.score, 0) / totalRuns : 0, + failedRuns: runs.filter((run) => run.status !== 'pass').length, + suites: new Set(runs.map((run) => run.suiteKey)).size, + }, + runs, + stageAverages, + } +} + +export async function getEvalRunDetail(id: string): Promise + failureReasons: string[] + stages: EvalStageResult[] +}> }> { + const { rows } = await pool.query( + `WITH scored AS ( + SELECT r.*, LAG(r.score) OVER (PARTITION BY r.suite_key ORDER BY r.created_at,r.id) AS previous_score + FROM eval_runs r + ) + SELECT r.*, baseline.score AS explicit_baseline_score + FROM scored r LEFT JOIN eval_runs baseline ON baseline.id=r.baseline_run_id + WHERE r.id=$1`, + [id], + ) + if (!rows[0]) throw Object.assign(new Error('eval run not found'), { status: 404 }) + const [caseRows, stageRows] = await Promise.all([ + pool.query<{ + id: string; case_key: string; name: string; position: number; source_agent_run_id: string | null + status: string; score: number; observation: EvalObservation; expectations: Record; failure_reasons: string[] + }>(`SELECT * FROM eval_cases WHERE eval_run_id=$1 ORDER BY position`, [id]), + pool.query<{ + eval_case_id: string; stage: EvalStageResult['stage']; status: EvalStageResult['status']; score: number | null + duration_ms: number; findings: EvalStageResult['findings']; metrics: EvalStageResult['metrics']; failure_reason: string | null; position: number + }>(`SELECT * FROM eval_stage_results WHERE eval_run_id=$1 ORDER BY eval_case_id,position`, [id]), + ]) + const stagesByCase = new Map() + for (const row of stageRows.rows) { + const list = stagesByCase.get(row.eval_case_id) ?? [] + list.push({ stage: row.stage, status: row.status, score: row.score === null ? null : Number(row.score), + durationMs: row.duration_ms, findings: row.findings, metrics: row.metrics, failureReason: row.failure_reason }) + stagesByCase.set(row.eval_case_id, list) + } + return { + ...toDashboardRun(rows[0]), + cases: caseRows.rows.map((row) => ({ + id: row.id, + caseId: row.case_key, + name: row.name, + position: row.position, + sourceAgentRunId: row.source_agent_run_id, + status: row.status, + score: Number(row.score), + observation: row.observation, + expectations: row.expectations, + failureReasons: row.failure_reasons, + stages: stagesByCase.get(row.id) ?? [], + })), + } +} diff --git a/src/admin/AdminApp.tsx b/src/admin/AdminApp.tsx index 8a5a6475..934a3e81 100644 --- a/src/admin/AdminApp.tsx +++ b/src/admin/AdminApp.tsx @@ -15,15 +15,16 @@ * the basePath is `/admin`. */ import { useEffect, useState } from 'react' -import { useAuth } from '@/stores/auth' import { CloudLogo } from '@/components/Avatar' -import { adminApi, type AdminStats } from './api' +import { useAuth } from '@/stores/auth' +import { type AdminStats, adminApi } from './api' +import { EvalPage } from './EvalPage' +import { ObservabilityPage } from './ObservabilityPage' +import { SettingsPage } from './SettingsPage' import { UsersPage } from './UsersPage' import { WaitlistPage } from './WaitlistPage' -import { SettingsPage } from './SettingsPage' -import { ObservabilityPage } from './ObservabilityPage' -type Route = 'users' | 'waitlist' | 'settings' | 'observability' +type Route = 'users' | 'waitlist' | 'settings' | 'observability' | 'eval' /** Empty string on the admin origin, `/admin` on localhost dev. Kept as * a function (not a constant) so tests / SSR don't crash on a missing @@ -42,6 +43,7 @@ function parseRoute(): Route { if (rest.startsWith('waitlist')) return 'waitlist' if (rest.startsWith('settings')) return 'settings' if (rest.startsWith('observability')) return 'observability' + if (rest.startsWith('eval')) return 'eval' return 'users' } @@ -127,6 +129,7 @@ export function AdminApp() { +
@@ -140,6 +143,7 @@ export function AdminApp() { void adminApi.stats().then(setStats).catch(() => {}) }} />} {route === 'observability' && } + {route === 'eval' && } {route === 'settings' && }
diff --git a/src/admin/EvalPage.tsx b/src/admin/EvalPage.tsx new file mode 100644 index 00000000..2b97509b --- /dev/null +++ b/src/admin/EvalPage.tsx @@ -0,0 +1,396 @@ +import { useEffect, useMemo, useState } from 'react' +import { + adminApi, + type EvalCaseDetail, + type EvalCreateRunRequest, + type EvalDashboardPayload, + type EvalDashboardRun, + type EvalRunDetail, + type EvalStageName, + type EvalStageResult, + type EvalStageStatus, +} from './api' + +const STAGES: Array<{ key: EvalStageName; label: string; short: string }> = [ + { key: 'ingest', label: '轨迹采集', short: '采集' }, + { key: 'answer', label: '回答质量', short: '回答' }, + { key: 'rag', label: 'RAG', short: 'RAG' }, + { key: 'tools', label: '工具调用', short: '工具' }, + { key: 'collaboration', label: '多 Agent', short: '协作' }, + { key: 'aggregate', label: '门控汇总', short: '汇总' }, +] + +const STAGE_THRESHOLDS: Partial> = { + answer: 0.75, + rag: 0.75, + tools: 1, + collaboration: 0.8, +} + +const RUN_TEMPLATE: EvalCreateRunRequest = { + schemaVersion: 'lingxiloop.eval.v1', + suiteKey: 'agent-regression', + suiteName: 'Agent 回归套件', + version: 'v1.0.0', + passThreshold: 0.8, + cases: [{ + caseId: 'grounded-answer', + name: '基于知识库回答并调用工具', + sourceAgentRunId: '请替换为 Agent OS runId', + expectations: { + requiredStages: ['answer', 'rag', 'tools'], + answer: { + requiredKeywords: ['结论'], + forbiddenPatterns: ['我不知道但我猜'], + maxLatencyMs: 15000, + maxTokens: 4000, + }, + rag: { + requiredSourceIds: ['请替换为知识源 ID'], + requireCitations: true, + minRetrievalRecall: 1, + minCitationPrecision: 1, + }, + tools: { + calls: [{ name: 'knowledge.search', required: true }], + requireSuccess: true, + allowUnexpected: true, + }, + }, + }], +} + +const fmtPercent = (value: number | null, digits = 0): string => value === null ? '—' : `${(value * 100).toFixed(digits)}%` +const fmtDate = (value: string): string => new Intl.DateTimeFormat('zh-CN', { + month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit', +}).format(new Date(value)) + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} + +function summaryPipeline(run: EvalDashboardRun): Array<{ stage: EvalStageName; status: EvalStageStatus; score: number | null }> { + return STAGES.map(({ key }) => { + if (key === 'ingest') return { stage: key, status: 'pass', score: 1 } + if (key === 'aggregate') return { stage: key, status: run.status, score: run.score } + const score = run.summary.stageScores[key] + if (score === null) return { stage: key, status: 'skipped', score: null } + return { stage: key, status: score >= (STAGE_THRESHOLDS[key] ?? 1) ? 'pass' : 'fail', score } + }) +} + +export function EvalPage() { + const [data, setData] = useState(null) + const [loading, setLoading] = useState(true) + const [refreshing, setRefreshing] = useState(false) + const [error, setError] = useState(null) + const [sinceDays, setSinceDays] = useState(90) + const [suiteFilter, setSuiteFilter] = useState('') + const [refreshKey, setRefreshKey] = useState(0) + const [selectedId, setSelectedId] = useState(null) + const [detail, setDetail] = useState(null) + const [detailError, setDetailError] = useState(null) + const [createOpen, setCreateOpen] = useState(false) + + useEffect(() => { + let cancelled = false + if (!data) setLoading(true) + setError(null) + adminApi.evalDashboard({ sinceDays, suiteKey: suiteFilter || undefined, limit: 120 }) + .then((payload) => { if (!cancelled) setData(payload) }) + .catch((reason) => { if (!cancelled) setError(errorMessage(reason)) }) + .finally(() => { if (!cancelled) { setLoading(false); setRefreshing(false) } }) + return () => { cancelled = true } + }, [sinceDays, suiteFilter, refreshKey]) + + useEffect(() => { + if (!selectedId) { setDetail(null); setDetailError(null); return } + let cancelled = false + setDetail(null); setDetailError(null) + adminApi.evalRun(selectedId) + .then((payload) => { if (!cancelled) setDetail(payload) }) + .catch((reason) => { if (!cancelled) setDetailError(errorMessage(reason)) }) + return () => { cancelled = true } + }, [selectedId]) + + useEffect(() => { + if (!selectedId && !createOpen) return + const onKeyDown = (event: KeyboardEvent) => { + if (event.key !== 'Escape') return + if (createOpen) setCreateOpen(false) + else setSelectedId(null) + } + window.addEventListener('keydown', onKeyDown) + return () => window.removeEventListener('keydown', onKeyDown) + }, [selectedId, createOpen]) + + const suites = useMemo(() => { + const names = new Map() + for (const run of data?.runs ?? []) names.set(run.suiteKey, run.suiteName) + return [...names.entries()].sort((left, right) => left[1].localeCompare(right[1], 'zh-CN')) + }, [data]) + const trendSuite = suiteFilter || data?.runs[0]?.suiteKey || '' + const trendRuns = (data?.runs ?? []).filter((run) => run.suiteKey === trendSuite) + + const refresh = () => { setRefreshing(true); setRefreshKey((value) => value + 1) } + + return ( +
+
+
+
QUALITY CONTROL
+

Agent Eval

+
回答、RAG、工具调用与多 Agent 协作的确定性回归评测
+
+
+ + +
+
+ +
+ + +
结果不可变 · 缺失阶段不会被当作通过
+
+ + {error &&
{error}
} + {loading && !data ? : data && <> +
+ + + + = 0 ? '+' : ''}${(data.runs[0].scoreDelta * 100).toFixed(1)}pp`} + note={data.runs[0] ? `${data.runs[0].suiteName} · ${data.runs[0].version}` : '暂无基线'} + tone={(data.runs[0]?.scoreDelta ?? 0) < 0 ? 'coral' : 'green'} /> +
+ +
+ + +
+ +
+
+
+

运行流水线

+

选择一次运行，查看每个用例的门控、指标与根因。

+
+ {data.runs.length} 次运行 +
+ {data.runs.length === 0 + ? setCreateOpen(true)} /> + :
+ {data.runs.map((run) => setSelectedId(run.id)} />)} +
} +
+ } + + {selectedId && setSelectedId(null)} />} + {createOpen && setCreateOpen(false)} + onCreated={(id) => { setCreateOpen(false); refresh(); setSelectedId(id) }} + />} +
+ ) +} + +function Kpi({ label, value, note, tone }: { label: string; value: string; note: string; tone: 'ink' | 'sky' | 'green' | 'coral' }) { + return
+
{label}
+
{value}
+
{note}
+
+} + +function VersionTrend({ runs, suiteName }: { runs: EvalDashboardRun[]; suiteName: string }) { + const chronological = [...runs].reverse().slice(-24) + const width = 720; const height = 196; const left = 42; const right = 18; const top = 18; const bottom = 36 + const x = (index: number) => chronological.length <= 1 ? width / 2 : left + index * ((width - left - right) / (chronological.length - 1)) + const y = (score: number) => top + (1 - score) * (height - top - bottom) + const path = chronological.map((run, index) => `${index ? 'L' : 'M'}${x(index).toFixed(1)},${y(run.score).toFixed(1)}`).join(' ') + return
+
+

版本趋势

{suiteName}

+ {chronological.at(-1) && {chronological.at(-1)?.status === 'pass' ? '当前通过' : '当前未通过'}} +
+ {chronological.length === 0 ?
运行一次评测后，这里会显示版本变化。
: + + {[0, 0.5, 0.8, 1].map((tick) => + + {Math.round(tick * 100)} + )} + {chronological.length > 1 && } + + {chronological.map((run, index) => + + {(index === 0 || index === chronological.length - 1) && {run.version}} + {run.version}: {fmtPercent(run.score, 1)} + )} + } +
+} + +function StageAverages({ values }: { values: EvalDashboardPayload['stageAverages'] }) { + return
+

能力分布

所选范围的阶段均分

+
+ {(['answer', 'rag', 'tools', 'collaboration'] as const).map((stage) => { + const value = values[stage] + const label = STAGES.find((item) => item.key === stage)?.label ?? stage + return
+
{label}{fmtPercent(value, 1)}
+
+
+ })} +
+
+} + +function MiniPipeline({ stages, compact = false }: { + stages: Array<{ stage: EvalStageName; status: EvalStageStatus; score: number | null }> + compact?: boolean +}) { + return
+ {stages.map((stage, index) => { + const meta = STAGES.find((item) => item.key === stage.stage) ?? { label: stage.stage, short: stage.stage } + return
+ {index > 0 &&
} +
+ {stage.status === 'pass' ? '✓' : stage.status === 'skipped' ? '–' : '!'} + {meta.short} + {!compact && {fmtPercent(stage.score)}} +
+
+ })} +
+} + +function RunCard({ run, onOpen }: { run: EvalDashboardRun; onOpen: () => void }) { + return +} + +function RunDetailDrawer({ detail, error, onClose }: { detail: EvalRunDetail | null; error: string | null; onClose: () => void }) { + const [caseId, setCaseId] = useState(null) + useEffect(() => { setCaseId(detail?.cases[0]?.id ?? null) }, [detail?.id]) + const selected = detail?.cases.find((item) => item.id === caseId) ?? detail?.cases[0] + return
{ if (event.target === event.currentTarget) onClose() }}> + +
+} + +function CaseDetail({ item }: { item: EvalCaseDetail }) { + const [expandedStage, setExpandedStage] = useState(item.stages.find((stage) => stage.status === 'fail' || stage.status === 'error')?.stage ?? null) + useEffect(() => { setExpandedStage(item.stages.find((stage) => stage.status === 'fail' || stage.status === 'error')?.stage ?? null) }, [item.id]) + return
+
CASE{item.caseId}
{item.sourceAgentRunId && {item.sourceAgentRunId}}
+ ({ stage: stage.stage, status: stage.status, score: stage.score }))} /> + {item.failureReasons.length > 0 &&
+

失败原因

+
    {item.failureReasons.map((reason, index) =>
  • {reason}
    • )}
+
} +
+

阶段检查

+ {item.stages.map((stage) => setExpandedStage((current) => current === stage.stage ? null : stage.stage)} />)} +
+
+} + +function StageDisclosure({ stage, open, onToggle }: { stage: EvalStageResult; open: boolean; onToggle: () => void }) { + const meta = STAGES.find((item) => item.key === stage.stage) + return
+ + {open &&
+ {stage.findings.map((item, index) =>
+ {item.status === 'pass' ? '✓' : item.status === 'not_observed' ? '○' : '×'} +
{item.checkId}

{item.message}

+
)} + {Object.keys(stage.metrics).length > 0 &&
+ {Object.entries(stage.metrics).map(([key, value]) =>
{key}{value === null ? '—' : String(value)}
)} +
} +
} +
+} + +function CreateRunDialog({ onClose, onCreated }: { onClose: () => void; onCreated: (id: string) => void }) { + const [value, setValue] = useState(() => JSON.stringify(RUN_TEMPLATE, null, 2)) + const [submitting, setSubmitting] = useState(false) + const [error, setError] = useState(null) + const submit = async () => { + setError(null) + let parsed: EvalCreateRunRequest + try { parsed = JSON.parse(value) as EvalCreateRunRequest } catch (reason) { setError(`JSON 格式错误：${errorMessage(reason)}`); return } + setSubmitting(true) + try { + const created = await adminApi.createEvalRun(parsed) + onCreated(created.id) + } catch (reason) { + setError(errorMessage(reason)) + } finally { setSubmitting(false) } + } + return
{ if (event.target === event.currentTarget && !submitting) onClose() }}> +
+
NEW EVAL RUN

运行评测套件

粘贴观测 JSON，或填写 Agent OS runId 自动回填真实轨迹。

+
支持的层answerragtoolscollaboration期望值只进入评测器，不会发送给 Agent。
+ {error &&
{error}
} +