Merge pull request #3584 from huangruiteng/codex/issue3539-provider-credential-boundary

fix(benchmark): isolate provider credentials from agents

Author: huangruiteng

benchmark/deepswe/README.md

@@ -140,11 +140,14 @@ The treatment also needs three independent product-path proofs:
 Prepare the latter two with
 `benchmark_toolkit.native_codex_profile.install_native_codex_profile`. Do not copy
 `SKILL.md` files into a runner image. Generate the first input with
-`render_native_codex_goal_prompt`, build app-server's environment with
-`native_codex_app_server_environment(profile, provider_env_key=..., base_env=...)`,
-then separately exclude that key from agent shell and tool processes. The
-credential-free `native_codex_profile_environment` remains the default for CLI
-and preflight work. Set
+`render_native_codex_goal_prompt`, keep app-server on the credential-free
+`native_codex_profile_environment`, and route its provider through
+`serve_runner_owned_provider_gateway`. The app-server receives only the gateway
+URL and a fixed non-secret env sentinel. A Linux host-side worker must also run
+inside `native_codex_isolation`; filtering child env alone does not prevent a
+danger-full-access agent from reading parent process environments or ambient
+HOME files. `native_codex_app_server_shell_policy_args` remains defense in depth
+for model-created shells, not the credential boundary. Set
 `NativeGoalConfig.required_skill_ids=profile.required_skill_ids`. The runtime then
 uses `skills/list` before thread creation and fails before model work unless Codex
 actually discovers the installed skill set. A filesystem check alone is not

benchmark/native_codex_goal.py

@@ -36,7 +36,7 @@
     compact_native_codex_profile_receipt,
     inspect_native_codex_profile,
     install_native_codex_profile,
-    native_codex_app_server_environment,
+    native_codex_app_server_shell_policy_args,
     native_codex_profile_environment,
     render_native_codex_goal_prompt,
 )
@@ -60,7 +60,7 @@
     "compact_native_goal_receipt",
     "inspect_native_codex_profile",
     "install_native_codex_profile",
-    "native_codex_app_server_environment",
+    "native_codex_app_server_shell_policy_args",
     "native_codex_profile_environment",
     "observe_native_goal_event",
     "probe_native_goal_process",

benchmark/tests/test_native_codex_profile.py

@@ -13,7 +13,7 @@
     compact_native_codex_profile_receipt,
     inspect_native_codex_profile,
     install_native_codex_profile,
-    native_codex_app_server_environment,
+    native_codex_app_server_shell_policy_args,
     native_codex_profile_environment,
     render_native_codex_goal_prompt,
 )
@@ -84,51 +84,47 @@ def test_compact_profile_receipt_excludes_local_paths() -> None:
     assert "/private" not in rendered
 
 
-def test_app_server_environment_explicitly_admits_only_provider_value(
-    tmp_path: Path,
-) -> None:
+def test_profile_environment_excludes_ambient_provider_values(tmp_path: Path) -> None:
     profile = _fake_profile(tmp_path)
     base_env = {
         "PATH": "/usr/bin:/bin",
         "CODEX_GOAL_API_KEY": "provider-value",
         "UNRELATED_PRIVATE_VALUE": "must-not-pass",
     }
-    original_base_env = dict(base_env)
-
     profile_env = native_codex_profile_environment(profile, base_env=base_env)
-    app_server_env = native_codex_app_server_environment(
-        profile,
-        provider_env_key="CODEX_GOAL_API_KEY",
-        base_env=base_env,
-    )
 
     assert "CODEX_GOAL_API_KEY" not in profile_env
     assert "UNRELATED_PRIVATE_VALUE" not in profile_env
-    assert app_server_env["CODEX_GOAL_API_KEY"] == "provider-value"
-    assert "UNRELATED_PRIVATE_VALUE" not in app_server_env
-    assert app_server_env["CODEX_HOME"] == str(profile.codex_home)
-    assert base_env == original_base_env
+    assert profile_env["CODEX_HOME"] == str(profile.codex_home)
 
 
-def test_app_server_environment_rejects_invalid_or_missing_provider_key(
-    tmp_path: Path,
-) -> None:
-    profile = _fake_profile(tmp_path)
+def test_native_codex_app_server_shell_policy_is_explicit_and_fail_closed() -> None:
+    args = native_codex_app_server_shell_policy_args(
+        excluded_env_keys=("SECOND_PROVIDER_KEY", "PRIMARY_PROVIDER_KEY"),
+    )
+    assert args == (
+        "-c",
+        'shell_environment_policy.inherit="core"',
+        "-c",
+        "shell_environment_policy.ignore_default_excludes=false",
+        "-c",
+        (
+            'shell_environment_policy.include_only=["HOME", "LANG", "LC_ALL", '
+            '"LC_CTYPE", "LOGNAME", "PATH", "SHELL", "SSL_CERT_DIR", '
+            '"SSL_CERT_FILE", "TERM", "TMPDIR", "TZ", "USER"]'
+        ),
+        "-c",
+        'shell_environment_policy.exclude=["PRIMARY_PROVIDER_KEY", "SECOND_PROVIDER_KEY"]',
+    )
 
-    with pytest.raises(ValueError, match="provider_env_key"):
-        native_codex_app_server_environment(
-            profile,
-            provider_env_key="invalid-key",
-            base_env={"invalid-key": "value"},
-        )
-    with pytest.raises(
-        NativeCodexProfileError,
-        match="provider_environment_value_missing:CODEX_GOAL_API_KEY",
-    ):
-        native_codex_app_server_environment(
-            profile,
-            provider_env_key="CODEX_GOAL_API_KEY",
-            base_env={"PATH": "/usr/bin:/bin"},
+
+@pytest.mark.parametrize("excluded_env_key", ("", "invalid-key", "WITH SPACE"))
+def test_native_codex_app_server_shell_policy_rejects_unsafe_keys(
+    excluded_env_key: str,
+) -> None:
+    with pytest.raises(ValueError, match="excluded_env_keys"):
+        native_codex_app_server_shell_policy_args(
+            excluded_env_keys=(excluded_env_key,),
         )
 
 

benchmark/widesearch/README.md

@@ -16,16 +16,23 @@ Local WideSearch（公开 web 表格填充 benchmark，200 题）practice：同
 - Verifier：官方 `widesearch_evaluator.py`（SR / 行级 / 项级 F1 + LLM judge，
   judge 复用 ARK 模型）。
 
-## 答案隔离边界（重要）
+## 隔离边界（重要）
 
-本地路径是**过程性隔离**，不是沙盒强隔离：
+native runner 现在默认 fail closed，并组合两层 runner-owned 边界：
 - 每次 run 用独立 fresh workspace，run 前清空，杜绝旧答案误用；
 - answer 必须存在、非空、mtime 晚于 run 开始（否则判定失败）；
 - evaluator 在 agent 结束后独立后置执行，只读 sealed answer + gold；
-- 但 agent（codex）拥有宿主全盘读权限（`workspace-write` 不挡读；macOS
-  `sandbox-exec` 会破坏 codex 运行时，已实测证伪），因此**不能假定 agent 读不到
-  gold**。任何结论都限定为内部能力论证，发布权威结果需真沙盒
-  （pier/colima docker 或云端 sealed-dual 强隔离）。
+- upstream provider key 只在 namespace 外的 runner-owned loopback gateway；
+  app-server 只收到 gateway URL 与固定非秘密 sentinel；
+- app-server 在 Linux user/mount/PID namespace 的合成 root 中运行，只暴露
+  fresh workspace、每次 run 新建的正式 LoopX profile 与只读系统 runtime；
+  runner/祖先进程环境、ambient HOME/CODEX_HOME、gold 和 provider 文件均不可见。
+
+`shell_environment_policy` 只是 defense in depth，不能代替上述 OS authority
+boundary。当前 native runner 需要 Linux unprivileged namespace primitives；macOS
+或禁用 user namespace 的 Linux 会以
+`widesearch_native_isolation_unavailable_use_pier` 失败，不会退回 ambient
+`danger-full-access`。这些平台请使用下文 Pier/Colima 路径。
 
 ## 运行
 
@@ -44,8 +51,9 @@ uv run --python 3.12 --with dateparser==1.2.2 \
   --arm treatment --case ws_en_001 --data-root <data-root>
 ```
 
-模型凭据：`ARK_OPENAI_BASE_URL` / `ARK_OPENAI_API_KEY` / `ARK_OPENAI_MODEL`
-（本地可从 cc-switch `volcengine-ark-deepseek` 读取）。
+runner-owned gateway 凭据：`ARK_OPENAI_BASE_URL` / `ARK_OPENAI_API_KEY`；模型名：
+`ARK_OPENAI_MODEL`。真实 key 不进入 app-server 环境、正式 profile 或 agent
+namespace。不要把这些值写入数据目录、job 模板或命令行。
 
 ## Hosted Responses API 兼容（重要）
 

benchmark/widesearch/pier/README.md

@@ -3,7 +3,8 @@
 Reusable, provider-neutral pier wiring for the WideSearch real-sandbox path.
 Copy these templates into a local pier dataset root and fill the `<PLACEHOLDER>`
 values. Do not commit credentials, local absolute paths, raw runs, or verifier
-output; inject secrets through the job environment at runtime.
+output. Inject judge secrets only into the separate verifier; keep the model
+provider secret in the runner-owned gateway, outside the agent container.
 
 Layout expected by pier (task root):
 
@@ -35,9 +36,10 @@ Key wiring facts:
   used by the pier agent setup; Debian dash does not support it).
 - Gold lives only in the verifier image; the agent image never contains it.
 
-Model credentials: for countable runs the agent image should reach the hosted
-model through a local gateway so the API key is not shell-readable in the
-container (matching `provider_credential_shell_excluded` in the integrity
-attestation). Use `network_access: "permitted_solving"` in the integrity policy
-for web-research benchmarks (see
-`loopx.capabilities.benchmark_toolkit.integrity`).
+Model credentials: countable runs must route the agent image through a
+runner-owned local gateway. The real provider credential stays in the gateway
+process, outside the agent container; the Codex provider config receives only a
+fixed non-secret sentinel. This is the concrete boundary required by
+`provider_credential_shell_excluded` in the integrity attestation. Use
+`network_access: "permitted_solving"` in the integrity policy for web-research
+benchmarks (see `loopx.capabilities.benchmark_toolkit.integrity`).

benchmark/widesearch/pier/job.yaml.template

@@ -1,9 +1,9 @@
 # Pier job for the WideSearch real-sandbox path.
 # - Keep jobs_dir / datasets under $HOME: colima does not bind-mount macOS
 #   /tmp, so verifier bind mounts of /logs/verifier silently fail there.
-# - Inject model credentials through the environment; do not bake secrets into
-#   the job file. For countable runs the agent image should reach the model
-#   through a local gateway so the key is not shell-readable in the container.
+# - Do not inject provider credentials into the agent container. The agent must
+#   reach the hosted model through a runner-owned local gateway. The sentinel
+#   below satisfies Codex's provider config without granting an upstream secret.
 job_name: widesearch-<RUN_LABEL>
 jobs_dir: <JOBS_DIR_UNDER_HOME>
 n_attempts: 1
@@ -24,14 +24,13 @@ agents:
         approval_policy = "never"
         [model_providers.custom]
         name = "hosted-responses"
-        base_url = "<MODEL_BASE_URL>"
-        env_key = "OPENAI_API_KEY"
+        base_url = "<RUNNER_LOCAL_GATEWAY_BASE_URL>"
+        env_key = "LOOPX_MODEL_PROVIDER_SENTINEL"
         wire_api = "responses"
         [tools]
         web_search = true
     env:
-      OPENAI_BASE_URL: "<MODEL_BASE_URL>"
-      OPENAI_API_KEY: "<INJECT_VIA_RUNTIME_ENV>"
+      LOOPX_MODEL_PROVIDER_SENTINEL: "local-gateway-no-provider-secret"
 datasets:
   - path: <DATASET_ROOT_UNDER_HOME>
     task_names: ["<CASE_ID>"]