diff --git a/packages/app/src/app/api/v1/workflow-info/route.ts b/packages/app/src/app/api/v1/workflow-info/route.ts index fc17db31..dfb0234c 100644 --- a/packages/app/src/app/api/v1/workflow-info/route.ts +++ b/packages/app/src/app/api/v1/workflow-info/route.ts @@ -31,7 +31,7 @@ const getCachedWorkflowInfo = cachedQuery(async (date: string) => { getRunConfigsByDate(sql, date), ]); return { runs, changelogs, configs, runConfigs }; -}, 'workflow-info'); +}, 'workflow-info-v2'); export async function GET(request: NextRequest) { const date = request.nextUrl.searchParams.get('date') ?? ''; diff --git a/packages/app/src/components/GlobalFilterContext.tsx b/packages/app/src/components/GlobalFilterContext.tsx index 11a75324..10cd3c01 100644 --- a/packages/app/src/components/GlobalFilterContext.tsx +++ b/packages/app/src/components/GlobalFilterContext.tsx @@ -36,6 +36,8 @@ import { import { computeAutoSwitchDecision } from '@/lib/unofficial-run-auto-switch'; import { countCurvesByPrecision, resolveEffectivePrecisions } from '@/lib/default-precisions'; import { resolveEffectiveSequence } from '@/lib/default-sequence'; +import { scenarioRunIdsForDate } from '@/lib/run-configs'; +import { resolveRunDate } from '@/lib/run-date'; import type { AvailabilityRow, WorkflowInfoResponse } from '@/lib/api'; const RUNDATE_RE = /^\d{4}-\d{2}-\d{2}$/u; @@ -411,14 +413,10 @@ export function GlobalFilterProvider({ setSelectedRunDateRev((v) => v + 1); }, []); - const effectiveRunDate = useMemo(() => { - if (availableDates.length === 0) return selectedRunDate; - const latest = availableDates.at(-1)!; - if (userPickedDateRef.current && selectedRunDate && availableDates.includes(selectedRunDate)) { - return selectedRunDate; - } - return latest; - }, [availableDates, selectedRunDate]); + const effectiveRunDate = useMemo( + () => resolveRunDate(availableDates, selectedRunDate, userPickedDateRef.current), + [availableDates, selectedRunDate], + ); // Sync selectedRunDate state when effectiveRunDate changes useEffect(() => { @@ -437,16 +435,33 @@ export function GlobalFilterProvider({ const workflowError = workflowQueryError ? workflowQueryError.message : null; - const availableRuns = useMemo( - () => (workflowData ? buildRunInfo(workflowData) : {}), - [workflowData], - ); + const dateRuns = useMemo(() => (workflowData ? buildRunInfo(workflowData) : {}), [workflowData]); + + const runConfigs = useMemo(() => workflowData?.runConfigs ?? [], [workflowData]); + + // Keep run selection and g_runid scenario-safe at their source. Filtering + // later in InferenceContext leaves the global single-turn run ID alive. + const availableRuns = useMemo(() => { + const runIds = scenarioRunIdsForDate( + runConfigs, + dbModelKeys, + effectiveSequence, + effectivePrecisions, + ); + return Object.fromEntries(Object.entries(dateRuns).filter(([runId]) => runIds.has(runId))); + }, [dateRuns, runConfigs, dbModelKeys, effectiveSequence, effectivePrecisions]); const workflowInfo = useMemo( () => (Object.keys(availableRuns).length > 0 ? [{ runInfoBySequence: availableRuns }] : null), [availableRuns], ); + const effectiveSelectedRunId = useMemo(() => { + const runIds = Object.keys(availableRuns); + if (runIds.includes(selectedRunId)) return selectedRunId; + return runIds.reduce((latest, runId) => (runId > latest ? runId : latest), ''); + }, [availableRuns, selectedRunId]); + // Auto-select latest run ID when availableRuns change const urlInitRef = useRef({ runIdApplied: false }); @@ -484,8 +499,10 @@ export function GlobalFilterProvider({ } setUrlParams({ g_model: selectedModel, - g_rundate: selectedRunDate, - g_runid: selectedRunId, + // Never write a stale URL date after model/scenario availability has + // already resolved it to a valid date (e.g. Agentic + 2026-07-04). + g_rundate: effectiveRunDate, + g_runid: effectiveSelectedRunId, // Don't pin the sequence to the URL until it's resolved from real // availability — writing the pre-load placeholder (8k/1k) would clobber a // shared `?i_seq=agentic-traces` link before the model's availability @@ -497,8 +514,8 @@ export function GlobalFilterProvider({ }); }, [ selectedModel, - selectedRunDate, - selectedRunId, + effectiveRunDate, + effectiveSelectedRunId, effectiveSequence, sequenceResolved, effectivePrecisions, @@ -520,7 +537,7 @@ export function GlobalFilterProvider({ selectedRunDate: effectiveRunDate, setSelectedRunDate: setSelectedRunDateManual, selectedRunDateRev, - selectedRunId, + selectedRunId: effectiveSelectedRunId, setSelectedRunId, availableModels, availableSequences, @@ -543,7 +560,7 @@ export function GlobalFilterProvider({ effectiveRunDate, setSelectedRunDateManual, selectedRunDateRev, - selectedRunId, + effectiveSelectedRunId, availableModels, availableSequences, availablePrecisions, diff --git a/packages/app/src/components/inference/InferenceContext.tsx b/packages/app/src/components/inference/InferenceContext.tsx index e3738b56..d4c4edcb 100644 --- a/packages/app/src/components/inference/InferenceContext.tsx +++ b/packages/app/src/components/inference/InferenceContext.tsx @@ -374,7 +374,7 @@ export function InferenceProvider({ ); const effectiveSelectedRunId = useMemo(() => { - if (!filteredAvailableRuns) return selectedRunId; + if (!filteredAvailableRuns) return ''; const filteredRunIds = Object.keys(filteredAvailableRuns); if (filteredRunIds.length === 0 || filteredRunIds.includes(selectedRunId)) return selectedRunId; return filteredRunIds.reduce((max, id) => (id > max ? id : max), filteredRunIds[0]); diff --git a/packages/app/src/components/inference/utils/runEnumeration.test.ts b/packages/app/src/components/inference/utils/runEnumeration.test.ts index 155fae66..d1953c73 100644 --- a/packages/app/src/components/inference/utils/runEnumeration.test.ts +++ b/packages/app/src/components/inference/utils/runEnumeration.test.ts @@ -16,6 +16,10 @@ function rc(over: Partial): RunConfigRow { framework: 'vllm', spec_method: 'none', disagg: false, + // Defaults to a single_turn 8k/1k row; agentic tests override these. + benchmark_type: 'single_turn', + isl: 8192, + osl: 1024, ...over, }; } diff --git a/packages/app/src/lib/api.ts b/packages/app/src/lib/api.ts index 27088455..eca84096 100644 --- a/packages/app/src/lib/api.ts +++ b/packages/app/src/lib/api.ts @@ -96,6 +96,12 @@ export interface RunConfigRow { framework: string; spec_method: string; disagg: boolean; + /** Benchmark scenario: `single_turn` (fixed-seq isl/osl) or `agentic_traces`. */ + benchmark_type: string; + /** ISL in tokens — null for agentic_traces. With osl + benchmark_type this maps to a sequence. */ + isl: number | null; + /** OSL in tokens — null for agentic_traces. */ + osl: number | null; } export interface WorkflowInfoResponse { diff --git a/packages/app/src/lib/run-configs.test.ts b/packages/app/src/lib/run-configs.test.ts new file mode 100644 index 00000000..f3444b5f --- /dev/null +++ b/packages/app/src/lib/run-configs.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from 'vitest'; + +import type { RunConfigRow } from './api'; +import { scenarioRunIdsForDate } from './run-configs'; + +function row(overrides: Partial): RunConfigRow { + return { + github_run_id: 1, + run_started_at: '2026-07-10T00:00:00Z', + html_url: null, + head_sha: null, + model: 'dsv4', + precision: 'fp4', + hardware: 'b200', + framework: 'sglang', + spec_method: 'none', + disagg: false, + benchmark_type: 'agentic_traces', + isl: null, + osl: null, + ...overrides, + }; +} + +const ids = (rows: RunConfigRow[], sequence: string) => + [...scenarioRunIdsForDate(rows, ['dsv4'], sequence, ['fp4'])].toSorted(); + +describe('scenarioRunIdsForDate', () => { + it('does not expose July 4 single-turn runs under Agentic Traces', () => { + const july4 = row({ + github_run_id: 28593351944, + benchmark_type: 'single_turn', + isl: 8192, + osl: 1024, + }); + expect(ids([july4], 'agentic-traces')).toEqual([]); + expect(ids([july4], '8k/1k')).toEqual(['28593351944']); + }); + + it('includes a run in each scenario for which it produced data', () => { + const rows = [ + row({ github_run_id: 42 }), + row({ github_run_id: 42, benchmark_type: 'single_turn', isl: 8192, osl: 1024 }), + ]; + expect(ids(rows, 'agentic-traces')).toEqual(['42']); + expect(ids(rows, '8k/1k')).toEqual(['42']); + }); + + it('filters by model and precision and deduplicates run IDs', () => { + const rows = [ + row({ github_run_id: 7 }), + row({ github_run_id: 7, hardware: 'b300' }), + row({ github_run_id: 8, model: 'glm5' }), + row({ github_run_id: 9, precision: 'fp8' }), + ]; + expect(ids(rows, 'agentic-traces')).toEqual(['7']); + }); +}); diff --git a/packages/app/src/lib/run-configs.ts b/packages/app/src/lib/run-configs.ts new file mode 100644 index 00000000..2eac8cc7 --- /dev/null +++ b/packages/app/src/lib/run-configs.ts @@ -0,0 +1,21 @@ +import { rowToSequence } from '@semianalysisai/inferencex-constants'; + +import type { RunConfigRow } from './api'; + +/** Run IDs with benchmark rows for the selected model, scenario, and precision. */ +export function scenarioRunIdsForDate( + runConfigs: RunConfigRow[], + modelDbKeys: string[], + sequence: string, + precisions: string[] = [], +): Set { + const selectedPrecisions = precisions.length > 0 ? new Set(precisions) : null; + const ids = new Set(); + for (const row of runConfigs) { + if (!modelDbKeys.includes(row.model)) continue; + if (selectedPrecisions && !selectedPrecisions.has(row.precision)) continue; + if (rowToSequence(row) !== sequence) continue; + ids.add(String(row.github_run_id)); + } + return ids; +} diff --git a/packages/app/src/lib/run-date.test.ts b/packages/app/src/lib/run-date.test.ts new file mode 100644 index 00000000..2b0ad7a3 --- /dev/null +++ b/packages/app/src/lib/run-date.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from 'vitest'; + +import { resolveRunDate } from './run-date'; + +describe('resolveRunDate', () => { + it('replaces a stale single-turn date with the latest agentic date', () => { + expect(resolveRunDate(['2026-07-09', '2026-07-10'], '2026-07-04', true)).toBe('2026-07-10'); + }); + + it('preserves an available date chosen by the user', () => { + expect(resolveRunDate(['2026-07-09', '2026-07-10'], '2026-07-09', true)).toBe('2026-07-09'); + }); +}); diff --git a/packages/app/src/lib/run-date.ts b/packages/app/src/lib/run-date.ts new file mode 100644 index 00000000..75e5b140 --- /dev/null +++ b/packages/app/src/lib/run-date.ts @@ -0,0 +1,10 @@ +/** Resolve a selected run date against the dates available for the active scenario. */ +export function resolveRunDate( + availableDates: string[], + selectedDate: string, + preserveSelected: boolean, +): string { + if (availableDates.length === 0) return selectedDate; + if (preserveSelected && availableDates.includes(selectedDate)) return selectedDate; + return availableDates.at(-1)!; +} diff --git a/packages/db/src/json-provider.ts b/packages/db/src/json-provider.ts index f6c626f0..5db07952 100644 --- a/packages/db/src/json-provider.ts +++ b/packages/db/src/json-provider.ts @@ -913,7 +913,8 @@ export function getRunConfigsByDate(date: string): RunConfigRow[] { const c = s.configs.get(br.config_id); if (!c) continue; - const key = `${wr.github_run_id}|${c.model}|${c.precision}|${c.hardware}|${c.framework}|${c.spec_method}|${c.disagg}`; + const benchmarkType = br.benchmark_type ?? 'single_turn'; + const key = `${wr.github_run_id}|${c.model}|${c.precision}|${c.hardware}|${c.framework}|${c.spec_method}|${c.disagg}|${benchmarkType}|${br.isl}|${br.osl}`; if (seen.has(key)) continue; seen.add(key); @@ -928,6 +929,9 @@ export function getRunConfigsByDate(date: string): RunConfigRow[] { framework: c.framework, spec_method: c.spec_method, disagg: c.disagg, + benchmark_type: benchmarkType, + isl: br.isl, + osl: br.osl, }); } diff --git a/packages/db/src/queries/workflow-info.ts b/packages/db/src/queries/workflow-info.ts index 01e13dd8..a83b167b 100644 --- a/packages/db/src/queries/workflow-info.ts +++ b/packages/db/src/queries/workflow-info.ts @@ -76,6 +76,12 @@ export interface RunConfigRow { framework: string; spec_method: string; disagg: boolean; + /** Benchmark scenario: `single_turn` (fixed-seq isl/osl) or `agentic_traces`. */ + benchmark_type: string; + /** ISL in tokens — null for agentic_traces. Together with osl + benchmark_type this maps to a sequence. */ + isl: number | null; + /** OSL in tokens — null for agentic_traces. */ + osl: number | null; } /** @@ -96,7 +102,10 @@ export async function getRunConfigsByDate(sql: DbClient, date: string): Promise< c.hardware, c.framework, c.spec_method, - c.disagg + c.disagg, + br.benchmark_type, + br.isl, + br.osl FROM benchmark_results br JOIN configs c ON c.id = br.config_id JOIN latest_workflow_runs wr ON wr.id = br.workflow_run_id