Bound Codex instruction fetches and derive eval diagnostics from the real session

Author: TheGreatAxios

scripts/eval-capability.test.ts

@@ -256,21 +256,28 @@ describe("initEvalGitRepo", () => {
 });
 
 describe("buildEvalDiagnostics", () => {
-  test("non-Codex provider gets a null instructions hash and the advertised tool list", () => {
-    const diagnostics = buildEvalDiagnostics(sampleConfig({ providerName: "openai" }));
+  test("non-Codex provider gets a null instructions hash and the default orchestrator tool list", async () => {
+    const diagnostics = await buildEvalDiagnostics(sampleConfig({ providerName: "openai" }));
     expect(diagnostics.codexInstructionsHash).toBeNull();
     expect(diagnostics.advertisedTools).toContain("read_file");
     expect(diagnostics.advertisedTools).toContain("run_shell");
     expect(diagnostics.reasoningEffort).toBeNull();
   });
 
-  test("Codex provider gets a non-null instructions hash", () => {
-    const diagnostics = buildEvalDiagnostics(sampleConfig({ providerName: "codex/default" }));
+  test("Codex provider gets a non-null instructions hash", async () => {
+    const diagnostics = await buildEvalDiagnostics(sampleConfig({ providerName: "codex/default" }));
     expect(diagnostics.codexInstructionsHash).toMatch(/^[0-9a-f]{12}$/);
   });
 
-  test("echoes back the configured reasoning effort", () => {
-    const diagnostics = buildEvalDiagnostics(sampleConfig({ reasoningEffort: "high" }));
+  test("echoes back the configured reasoning effort", async () => {
+    const diagnostics = await buildEvalDiagnostics(sampleConfig({ reasoningEffort: "high" }));
     expect(diagnostics.reasoningEffort).toBe("high");
   });
+
+  test("--director build reports the director's own advertised allowlist", async () => {
+    const diagnostics = await buildEvalDiagnostics(sampleConfig({ director: "build" }));
+    expect(diagnostics.advertisedTools).not.toEqual(
+      (await buildEvalDiagnostics(sampleConfig({}))).advertisedTools,
+    );
+  });
 });

scripts/eval-capability.ts

@@ -15,12 +15,14 @@ import { join, dirname, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { spawn } from "node:child_process";
 import { loadConfig, type Config } from "../src/config/index.js";
-import { runExec } from "../src/exec/runner.js";
+import { runExec, resolveExecDirectorOverlay } from "../src/exec/runner.js";
 import { SETTINGS_DIR_NAME } from "../src/branding.js";
 import { codexProfileFromProviderName } from "../src/config/codex-providers.js";
 import { codexInstructionsHash } from "../src/auth/codex/instructions.js";
 import { advertisedToolNamesForSessionMode } from "../src/agent/tool-search.js";
 import { detectLanguageServerAvailable } from "../src/agent/lsp-availability.js";
+import { resolveSessionMode } from "../src/config/session-mode.js";
+import { loadLocalSettings, localSettingsPath } from "../src/config/settings.js";
 import {
   loadEvalCases,
   filterCases,
@@ -539,14 +541,24 @@ async function resolveVariantLabels(
 /**
  * Per-cell diagnostics for debugging eval failures: which Codex instructions
  * text was pinned, which built-in tools the model was offered, and the
- * requested reasoning effort. Mirrors the exec runner's own session-mode/tool
- * gating (see src/agent/tool-search.ts) rather than re-running toolset setup.
+ * requested reasoning effort. Reuses the exec runner's own resolution
+ * (resolveSessionMode, resolveExecDirectorOverlay) rather than forking the
+ * logic, so a --director overlay or a non-default session mode here reports
+ * the same advertised list exec actually runs with.
+ *
+ * reasoningEffort echoes the configured value, not the provider's internal
+ * default when unset — accepted as-is per review.
  */
-export function buildEvalDiagnostics(config: Config): EvalDiagnostics {
+export async function buildEvalDiagnostics(config: Config): Promise<EvalDiagnostics> {
   const codexProfile = codexProfileFromProviderName(config.providerName);
-  const advertisedTools = advertisedToolNamesForSessionMode("orchestrator", {
-    languageServerAvailable: detectLanguageServerAvailable(config.cwd),
-  });
+  const localSettings = await loadLocalSettings(localSettingsPath(config.cwd)).catch(() => null);
+  const sessionMode = resolveSessionMode(config.settings, localSettings) ?? "orchestrator";
+  const overlay = resolveExecDirectorOverlay(config.director);
+  const advertisedTools =
+    overlay.advertisedAllow ??
+    advertisedToolNamesForSessionMode(sessionMode, {
+      languageServerAvailable: detectLanguageServerAvailable(config.cwd),
+    });
   return {
     codexInstructionsHash: codexProfile !== undefined ? codexInstructionsHash() : null,
     advertisedTools,
@@ -657,7 +669,7 @@ async function runCase(
       );
     }
 
-    const diagnostics = buildEvalDiagnostics(config);
+    const diagnostics = await buildEvalDiagnostics(config);
     const agentStarted = Date.now();
     // runExec runs the agent in-process (no child, unlike verify.sh below), so
     // the fixture origin must reach it via process.env directly for the

src/auth/codex/instructions.test.ts

@@ -89,6 +89,23 @@ describe("refreshCodexInstructions", () => {
     expect(codexInstructions()).toBe(before);
   });
 
+  test("rejects within the timeout when a fetch never resolves (hung connection)", async () => {
+    const before = codexInstructions();
+    global.fetch = ((_input: RequestInfo | URL, init?: RequestInit) => {
+      return new Promise((_resolve, reject) => {
+        const signal = init?.signal;
+        if (signal) {
+          signal.addEventListener("abort", () => reject(signal.reason as Error));
+        }
+      });
+    }) as unknown as typeof fetch;
+
+    const started = Date.now();
+    await expect(refreshCodexInstructions()).rejects.toBeTruthy();
+    expect(Date.now() - started).toBeLessThan(15_000);
+    expect(codexInstructions()).toBe(before);
+  }, 20_000);
+
   test("codexInstructionsHash reflects the currently resolved instructions text", async () => {
     const hashBefore = codexInstructionsHash();
     expect(hashBefore).toMatch(/^[0-9a-f]{12}$/);

src/auth/codex/instructions.ts

@@ -20,6 +20,10 @@ const PROMPT_PATH = "codex-rs/core/gpt_5_codex_prompt.md";
 const PROMPT_SENTINEL = "You are Codex";
 const MIN_PROMPT_LENGTH = 1000;
 
+// Bounds both network calls below so a black-holed connection can never hang
+// exec boot, which awaits refreshCodexInstructions before first inference.
+const CODEX_INSTRUCTIONS_TIMEOUT_MS = 10_000;
+
 export function isValidCodexPrompt(text: string): boolean {
   return text.length >= MIN_PROMPT_LENGTH && text.startsWith(PROMPT_SENTINEL);
 }
@@ -52,7 +56,10 @@ export function codexInstructionsHash(): string {
 }
 
 async function latestReleaseTag(): Promise<string> {
-  const res = await fetch(RELEASES_LATEST, { headers: { accept: "application/vnd.github+json" } });
+  const res = await fetch(RELEASES_LATEST, {
+    headers: { accept: "application/vnd.github+json" },
+    signal: AbortSignal.timeout(CODEX_INSTRUCTIONS_TIMEOUT_MS),
+  });
   if (!res.ok) throw new Error(`Codex release lookup failed (HTTP ${String(res.status)}).`);
   const data = (await res.json()) as { tag_name?: unknown };
   if (typeof data.tag_name !== "string") throw new Error("Codex release lookup returned no tag.");
@@ -61,7 +68,10 @@ async function latestReleaseTag(): Promise<string> {
 
 export async function refreshCodexInstructions(): Promise<void> {
   const tag = await latestReleaseTag();
-  const res = await fetch(`https://raw.githubusercontent.com/openai/codex/${encodeURIComponent(tag)}/${PROMPT_PATH}`);
+  const res = await fetch(
+    `https://raw.githubusercontent.com/openai/codex/${encodeURIComponent(tag)}/${PROMPT_PATH}`,
+    { signal: AbortSignal.timeout(CODEX_INSTRUCTIONS_TIMEOUT_MS) },
+  );
   if (!res.ok) throw new Error(`Codex prompt fetch failed (HTTP ${String(res.status)}).`);
   const text = await res.text();
   if (!isValidCodexPrompt(text)) throw new Error("Codex prompt fetch returned an unexpected body.");