diff --git a/apps/cli/test/commands/results/serve.test.ts b/apps/cli/test/commands/results/serve.test.ts index 6e1eb03f..d3eefb0a 100644 --- a/apps/cli/test/commands/results/serve.test.ts +++ b/apps/cli/test/commands/results/serve.test.ts @@ -850,7 +850,13 @@ describe('serve app', () => { mkdirSync(homeDir, { recursive: true }); writeFileSync( path.join(projectDir, '.agentv', 'config.yaml'), - 'execution:\n verbose: true\n', + `execution: + verbose: true +results: + mode: github + repo: EntityProcess/project-local-results + auto_push: false +`, ); writeFileSync( path.join(homeDir, 'config.yaml'), diff --git a/apps/web/src/content/docs/docs/tools/dashboard.mdx b/apps/web/src/content/docs/docs/tools/dashboard.mdx index 4300420c..fe7eead6 100644 --- a/apps/web/src/content/docs/docs/tools/dashboard.mdx +++ b/apps/web/src/content/docs/docs/tools/dashboard.mdx @@ -271,7 +271,7 @@ results_by_project: auto_push: true ``` -The keys under `results_by_project` must match the project IDs in `$AGENTV_HOME/projects.yaml`. Project-local `.agentv/config.yaml` `results` blocks still take precedence. A top-level global `results` block remains the fallback for projects without a specific mapping. +The keys under `results_by_project` must match the project IDs in `$AGENTV_HOME/projects.yaml`. For registered project flows, a matching machine-local `results_by_project` entry takes precedence over a project-local `.agentv/config.yaml` `results` block. Project-local `results` remains the fallback when no project-specific mapping matches, and a top-level global `results` block remains the final fallback. With `auto_push: true`, every new `agentv eval` or `agentv pipeline bench` pushes results directly to the configured repo's base branch (e.g., `main`). Results appear immediately in Dashboard without requiring PR merges. diff --git a/packages/core/src/evaluation/loaders/config-loader.ts b/packages/core/src/evaluation/loaders/config-loader.ts index f8b95981..3c1c806b 100644 --- a/packages/core/src/evaluation/loaders/config-loader.ts +++ b/packages/core/src/evaluation/loaders/config-loader.ts @@ -70,8 +70,10 @@ export type AgentVConfig = { * falls back to the home/global config at `${AGENTV_HOME:-~/.agentv}/config.yaml`. * The first valid project-local file wins for normal settings. Machine-local * `results_by_project` mappings from the global config are still attached when - * the project-local file has no `results` block, so registered projects can use - * per-machine results repos without editing source-controlled config. + * the project-local file has no project mapping of its own, so registered + * projects can use per-machine results repos without editing source-controlled + * config. `resolveResultsConfigForProject()` then lets a matching project + * mapping override the top-level project-local `results` fallback. */ export async function loadConfig( evalFilePath: string, @@ -89,8 +91,6 @@ export async function loadConfig( const config = await readConfigFile(configPath); if (config) { - if (config.results) return config; - const globalConfig = (await fileExists(globalConfigPath)) ? await readConfigFile(globalConfigPath) : null; diff --git a/packages/core/test/evaluation/loaders/config-loader.test.ts b/packages/core/test/evaluation/loaders/config-loader.test.ts index 63bf83bb..9f898cf7 100644 --- a/packages/core/test/evaluation/loaders/config-loader.test.ts +++ b/packages/core/test/evaluation/loaders/config-loader.test.ts @@ -134,7 +134,7 @@ describe('loadConfig', () => { } }); - it('keeps project-local results ahead of global results_by_project', async () => { + it('lets matching global results_by_project override project-local results', async () => { const tempDir = mkdtempSync(path.join(os.tmpdir(), 'agentv-local-results-precedence-')); try { const projectDir = path.join(tempDir, 'project'); @@ -149,6 +149,7 @@ describe('loadConfig', () => { `results: mode: github repo: EntityProcess/local-results + auto_push: false `, ); writeFileSync( @@ -157,14 +158,61 @@ describe('loadConfig', () => { agentv: mode: github repo: EntityProcess/global-results + path: /tmp/agentv-global-results + auto_push: true +`, + ); + + await withOptionalEnv('AGENTV_HOME', homeDir, async () => { + const config = await loadConfig(path.join(evalDir, 'suite.eval.yaml'), projectDir); + expect(resolveResultsConfigForProject(config, 'agentv')).toEqual({ + mode: 'github', + repo: 'EntityProcess/global-results', + path: '/tmp/agentv-global-results', + auto_push: true, + }); + }); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it('keeps project-local results when no global results_by_project id matches', async () => { + const tempDir = mkdtempSync(path.join(os.tmpdir(), 'agentv-local-results-fallback-')); + try { + const projectDir = path.join(tempDir, 'project'); + const evalDir = path.join(projectDir, 'evals'); + const localConfigDir = path.join(projectDir, '.agentv'); + const homeDir = path.join(tempDir, 'home'); + mkdirSync(evalDir, { recursive: true }); + mkdirSync(localConfigDir, { recursive: true }); + mkdirSync(homeDir, { recursive: true }); + writeFileSync( + path.join(localConfigDir, 'config.yaml'), + `results: + mode: github + repo: EntityProcess/local-results + auto_push: false +`, + ); + writeFileSync( + path.join(homeDir, 'config.yaml'), + `results_by_project: + other-project: + mode: github + repo: EntityProcess/global-results + path: /tmp/agentv-global-results + auto_push: true `, ); await withOptionalEnv('AGENTV_HOME', homeDir, async () => { const config = await loadConfig(path.join(evalDir, 'suite.eval.yaml'), projectDir); - expect(resolveResultsConfigForProject(config, 'agentv')?.repo).toBe( - 'EntityProcess/local-results', - ); + expect(resolveResultsConfigForProject(config, 'agentv')).toEqual({ + mode: 'github', + repo: 'EntityProcess/local-results', + auto_push: false, + }); }); } finally { rmSync(tempDir, { recursive: true, force: true });