Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
5f7091b
fix: send chat model selections by model id
vm-zero Jul 8, 2026
4abdab3
fix: remove chat model selection request wrapper
vm-zero Jul 8, 2026
839a56f
chore: merge main into fix/chat-message-model-only
vm-zero Jul 8, 2026
6467b3d
Merge remote-tracking branch 'origin/main' into fix/chat-message-mode…
e7h4n Jul 8, 2026
e786d57
fix: resolve chat model CI failures
e7h4n Jul 8, 2026
e28a60d
Merge remote-tracking branch 'origin/main' into fix/chat-message-mode…
e7h4n Jul 8, 2026
c365d3d
Merge remote-tracking branch 'origin/main' into fix/chat-message-mode…
e7h4n Jul 8, 2026
f9c5520
fix: resolve CI failures on PR #20639
e7h4n Jul 8, 2026
2b4c684
Merge remote-tracking branch 'origin/main' into fix/chat-message-mode…
e7h4n Jul 8, 2026
7fb8b8f
fix: resolve CI failures on PR #20639
e7h4n Jul 8, 2026
83f5570
chore: merge main into fix/chat-message-model-only
e7h4n Jul 8, 2026
4f9dcd9
fix: resolve CI failures on PR #20639
e7h4n Jul 8, 2026
893fc5f
Merge remote-tracking branch 'origin/main' into fix/chat-message-mode…
e7h4n Jul 8, 2026
7872c9e
chore: merge main into fix/chat-message-model-only
e7h4n Jul 8, 2026
b7724b5
chore: merge main into fix/chat-message-model-only
e7h4n Jul 8, 2026
ee39c9d
chore: merge main into fix/chat-message-model-only
e7h4n Jul 8, 2026
adf4505
chore: merge main into fix/chat-message-model-only
e7h4n Jul 8, 2026
fa34548
chore: merge main into fix/chat-message-model-only
e7h4n Jul 8, 2026
de399c4
Merge remote-tracking branch 'origin/main' into fix/chat-message-mode…
e7h4n Jul 8, 2026
e7df76a
chore: resolve merge conflict with main
e7h4n Jul 8, 2026
8ebd743
Merge remote-tracking branch 'origin/main' into fix/chat-message-mode…
e7h4n Jul 8, 2026
f213617
fix: resolve CI failures on PR #20639
e7h4n Jul 8, 2026
d5af64c
Merge remote-tracking branch 'origin/main' into fix/chat-message-mode…
e7h4n Jul 8, 2026
0fde36e
Merge remote-tracking branch 'origin/main' into fix/chat-message-mode…
e7h4n Jul 8, 2026
bbe7eb2
Merge remote-tracking branch 'origin/main' into fix/chat-message-mode…
e7h4n Jul 8, 2026
4820478
Merge remote-tracking branch 'origin/main' into fix/chat-message-mode…
e7h4n Jul 8, 2026
260c249
Merge remote-tracking branch 'origin/main' into fix/chat-message-mode…
e7h4n Jul 8, 2026
4345cfa
fix: align chat model e2e with model-only writes
lancy Jul 9, 2026
ce19d56
Merge remote-tracking branch 'origin/main' into fix/chat-message-mode…
lancy Jul 9, 2026
8a04372
chore: merge main into fix/chat-message-model-only
lancy Jul 9, 2026
4fdcb8b
chore: merge main into fix/chat-message-model-only
lancy Jul 9, 2026
2f16800
Merge remote-tracking branch 'origin/main' into fix/chat-message-mode…
lancy Jul 9, 2026
fb321c6
fix: resolve ci failures on pr 20639
lancy Jul 9, 2026
776ef9d
chore: resolve merge conflict with main
lancy Jul 9, 2026
51f53f5
fix(e2e): replace removed --debug-no-mock-claude flag in t54-0
lancy Jul 9, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 44 additions & 6 deletions e2e/helpers/codex-zero.bash
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,49 @@ disable_codex_beta() {
return 0
}

configure_codex_zero_model_policy() {
local model="$1"
local provider_type="$2"
local provider_id="$3"
local current payload

current=$(_codex_zero_curl "/api/zero/model-policies")
payload=$(printf '%s' "$current" | jq -c \
--arg model "$model" \
--arg providerType "$provider_type" \
--arg providerId "$provider_id" \
'
(.policies // [])
| map({
model,
isDefault,
defaultProviderType,
credentialScope,
modelProviderId
}) as $existing
| ($existing | map(select(.model != $model))) as $others
| (
($existing | map(select(.model == $model)) | first)
// {model: $model, isDefault: false}
) as $target
| {
policies: (
$others
+ [
$target
+ {
defaultProviderType: $providerType,
credentialScope: "org",
modelProviderId: $providerId
}
]
)
}
')

_codex_zero_curl "/api/zero/model-policies" -X PUT -d "$payload" >/dev/null
}

# Poll /api/zero/chat-threads/:id/messages until the current run has a terminal
# assistant lifecycle marker (the paged message API no longer exposes agent run
# status; terminal runs append a null-content lifecycle marker row instead).
Expand Down Expand Up @@ -189,17 +232,12 @@ send_chat_run_message() {
# USE_MOCK_CODEX=true env var causes guest-mock-codex to echo the prompt
# verbatim — see crates/runner/src/executor.rs (insert_codex_env) and
# guest_mock_codex::build_events.
# Model-first selection can carry either the sentinel provider id for an org
# policy route, or a concrete provider id for an explicit model/provider pin.
# BYOK smoke tests use a concrete id so they do not mutate shared org policy.
local selected_model="${CODEX_ZERO_SELECTED_MODEL:-gpt-5.5}"
local model_provider_id="${CODEX_ZERO_MODEL_PROVIDER_ID:-00000000-0000-4000-8000-000000000000}"
payload=$(jq -nc \
--arg agentId "$agent_id" \
--arg prompt "$prompt" \
--arg modelProviderId "$model_provider_id" \
--arg selectedModel "$selected_model" \
'{agentId: $agentId, prompt: $prompt, modelSelection: {modelProviderId: $modelProviderId, selectedModel: $selectedModel}, hasTextContent: true, realAgentInPreview: true}')
'{agentId: $agentId, prompt: $prompt, model: $selectedModel, hasTextContent: true, realAgentInPreview: true}')
body=$(_codex_zero_curl "/api/zero/chat/messages" \
-X POST \
-d "$payload")
Expand Down
24 changes: 24 additions & 0 deletions e2e/helpers/setup.bash
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,30 @@ zero_model_first_selection_provider_id() {
printf '%s' "00000000-0000-4000-8000-000000000000"
}

zero_chat_run_with_model() {
local agent_id="$1"
local prompt="$2"
local selected_model="$3"
local real_agent_in_preview="${4:-false}"
local payload body

payload=$(jq -nc \
--arg agentId "$agent_id" \
--arg prompt "$prompt" \
--arg selectedModel "$selected_model" \
--argjson realAgentInPreview "$real_agent_in_preview" \
'{agentId: $agentId, prompt: $prompt, model: $selectedModel, hasTextContent: true, realAgentInPreview: $realAgentInPreview}')

body=$(zero_curl "/api/zero/chat/messages" -X POST -d "$payload")
LAST_RUN_ID=$(printf '%s' "$body" | jq -r '.runId // ""')
LAST_THREAD_ID=$(printf '%s' "$body" | jq -r '.threadId // ""')
export LAST_RUN_ID LAST_THREAD_ID
[[ -n "$LAST_RUN_ID" && -n "$LAST_THREAD_ID" ]] || {
echo "# zero_chat_run_with_model: bad response: $body" >&2
return 1
}
}

zero_chat_run_with_model_selection() {
local agent_id="$1"
local prompt="$2"
Expand Down
22 changes: 12 additions & 10 deletions e2e/tests/03-runner/t-codex-zero-byok-smoke.bats
Original file line number Diff line number Diff line change
Expand Up @@ -31,18 +31,20 @@ setup_file() {
# 1. Feature switch on (also fails the file early if not yet wired)
enable_codex_beta

# 2. Org-level openai-api-key provider. The selected model is explicitly
# pinned to this provider on the chat message, without provider defaults or
# shared org policy mutation.
# 2. Org-level openai-api-key provider. Chat writes only carry the selected
# model; provider resolution comes from org model policy.
$ZERO_CLI org model-provider setup --type "openai-api-key" --secret "$OPENAI_API_KEY" >/dev/null
export OPENAI_PROVIDER_ID
OPENAI_PROVIDER_ID=$(zero_model_provider_id_by_type "openai-api-key")
export CODEX_ZERO_SELECTED_MODEL="gpt-5.4-mini"
export CODEX_ZERO_MODEL_PROVIDER_ID="$OPENAI_PROVIDER_ID"
configure_codex_zero_model_policy \
"$CODEX_ZERO_SELECTED_MODEL" \
"openai-api-key" \
"$OPENAI_PROVIDER_ID"

# 3. Compose declares framework: codex explicitly. The framework is
# resolved from the explicit model-first provider pin. At secret resolution
# the provider's declared framework wins (Epic #11520) and is propagated
# resolved from the org model policy route. At secret resolution the
# provider's declared framework wins (Epic #11520) and is propagated
# downstream via build-zero-context.ts's resolvedFramework, with no
# compose-vs-provider equality check.
cat > "$TEST_DIR/vm0-basic.yaml" <<EOF
Expand Down Expand Up @@ -103,10 +105,10 @@ teardown_file() {
# survive the subshell boundary.
wait_for_chat_assistant_done "$THREAD_ID"

# Assert: real codex produced the expected sentinel. The org's only
# provider is openai-api-key, so a real (non-mock) codex completion
# already proves the BYOK provider routing chain resolved end-to-end —
# without the key the run cannot produce the sentinel.
# Assert: real codex produced the expected sentinel. The selected model's
# policy routes to openai-api-key, so a real (non-mock) codex completion
# proves the BYOK provider routing chain resolved end-to-end — without the
# key the run cannot produce the sentinel.
[[ "$LAST_MSG_CONTENT" == *"RESULT=579"* ]] \
|| fail "Expected 'RESULT=579' in assistant content, got: $LAST_MSG_CONTENT"
}
44 changes: 23 additions & 21 deletions e2e/tests/03-runner/t30-vm0-model-provider-inject.bats
Original file line number Diff line number Diff line change
Expand Up @@ -2,41 +2,43 @@

# Test model-provider credential injection into container environment
#
# Verifies that an explicit model-first provider pin is honored by zero run and
# Verifies that an explicit runner provider type is honored by vm0 run and
# injected into the container as CLAUDE_CODE_OAUTH_TOKEN.

load '../../helpers/setup'

setup() {
export UNIQUE_ID="$(date +%s%3N)-$RANDOM"
export AGENT_ID=""
export THREAD_ID=""
export TEST_DIR="$(mktemp -d)"
export AGENT_NAME="e2e-mp-inject-${UNIQUE_ID}"

cat > "$TEST_DIR/vm0.yaml" <<EOF
version: "1.0"
agents:
${AGENT_NAME}:
description: "Model provider credential injection test"
framework: claude-code
EOF

$VM0_CLI compose "$TEST_DIR/vm0.yaml" >/dev/null
}

teardown() {
[ -n "$THREAD_ID" ] && zero_curl "/api/zero/chat-threads/$THREAD_ID" -X DELETE >/dev/null 2>&1 || true
[ -n "$AGENT_ID" ] && $ZERO_CLI agent delete "$AGENT_ID" -y 2>/dev/null || true
if [ -n "$TEST_DIR" ] && [ -d "$TEST_DIR" ]; then
rm -rf "$TEST_DIR"
fi
}

@test "model-provider credential is injected into container" {
local provider_id
provider_id=$(zero_model_provider_id_by_type "claude-code-oauth-token")

AGENT_ID=$(create_private_zero_agent "e2e-mp-inject-${UNIQUE_ID}") || {
echo "# Failed to create private zero agent" >&2
return 1
}

zero_chat_run_with_model_selection \
"$AGENT_ID" \
"case \"\$CLAUDE_CODE_OAUTH_TOKEN\" in \"\"|\"***\") marker=MISMATCH ;; *) marker=OK ;; esac; printf 'INJECTED_%s\n' \"\$marker\"" \
"$provider_id" \
"claude-sonnet-4-6" \
false
THREAD_ID="$LAST_THREAD_ID"
zero_model_provider_id_by_type "claude-code-oauth-token" >/dev/null

# GitHub Actions masks sk-ant-like values in logs, so emit a non-secret marker
# only after verifying the injected token is present inside the container.
WAIT_FOR_LOG_TIMEOUT=60 wait_for_log "$LAST_RUN_ID" -- "INJECTED_OK"
run $VM0_CLI run "$AGENT_NAME" \
--model-provider-type "claude-code-oauth-token" \
"case \"\$CLAUDE_CODE_OAUTH_TOKEN\" in \"\"|\"***\") marker=MISMATCH ;; *) marker=OK ;; esac; printf 'INJECTED_%s\n' \"\$marker\""

echo "$output"
assert_success
assert_output --partial "INJECTED_OK"
}
50 changes: 32 additions & 18 deletions e2e/tests/03-runner/t54-vm0-billable-firewall.bats
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,12 @@

# Verify firewall_billable propagation through the full stack.
#
# Uses explicit model-first pins for provider selection. The test never changes
# the shared e2e org's workspace default, so no race with other chunks.
# BYOK selection uses the runner provider-type path so chat writes do not carry
# provider pins. The vm0 case exercises the web chat model-only path and relies
# on org model policy for provider resolution.
#
# t54-0: run pinned to org BYOK anthropic-api-key; "$" marker absent.
# t54-1: run uses the model policy's vm0 route; billableFirewalls covers
# t54-0: direct run with org BYOK anthropic-api-key; "$" marker absent.
# t54-1: chat run uses the model policy's vm0 route; billableFirewalls covers
# the concrete anthropic firewall → "$" marker present.

load '../../helpers/setup'
Expand All @@ -17,13 +18,26 @@ setup_file() {
fi

export UNIQUE_ID="$(date +%s%3N)-$RANDOM"
export TEST_DIR="$(mktemp -d)"
export RUN_AGENT_NAME="e2e-billable-runner-${UNIQUE_ID}"
export THREAD_IDS=""

$ZERO_CLI org model-provider setup \
--type anthropic-api-key \
--secret "$ANTHROPIC_API_KEY" >/dev/null
export ANTHROPIC_PROVIDER_ID
ANTHROPIC_PROVIDER_ID=$(zero_model_provider_id_by_type "anthropic-api-key")
zero_model_provider_id_by_type "anthropic-api-key" >/dev/null

cat > "$TEST_DIR/vm0.yaml" <<EOF
version: "1.0"
agents:
${RUN_AGENT_NAME}:
description: "Billable firewall BYOK e2e"
framework: claude-code
environment:
ANTHROPIC_MODEL: "claude-sonnet-4-6"
EOF

$VM0_CLI compose "$TEST_DIR/vm0.yaml" >/dev/null

# Create a private zero agent for this file so CI does not consume the
# shared org's limited public-agent slots.
Expand All @@ -46,25 +60,26 @@ teardown_file() {
zero_curl "/api/zero/chat-threads/$thread_id" -X DELETE >/dev/null 2>&1 || true
done
[ -n "$AGENT_ID" ] && $ZERO_CLI agent delete "$AGENT_ID" -y 2>/dev/null || true
if [ -n "$TEST_DIR" ] && [ -d "$TEST_DIR" ]; then
rm -rf "$TEST_DIR"
fi
}

@test "t54-0: BYOK provider — firewall not billable" {
zero_chat_run_with_model_selection \
"$AGENT_ID" \
"Reply with exactly: DONE" \
"$ANTHROPIC_PROVIDER_ID" \
"claude-sonnet-4-6" \
true
THREAD_IDS="$THREAD_IDS $LAST_THREAD_ID"
export THREAD_IDS
run $VM0_CLI run "$RUN_AGENT_NAME" \
--model-provider-type "anthropic-api-key" \
--real-agent-in-preview \
"Reply with exactly: DONE"

RUN_ID="$LAST_RUN_ID"
echo "$output"
assert_success

RUN_ID=$(echo "$output" | grep -oP 'Run ID:\s+\K[a-f0-9-]{36}' | head -1)
[ -n "$RUN_ID" ] || {
echo "# Failed to extract Run ID"
return 1
}

wait_for_zero_run_completed "$RUN_ID"
WAIT_FOR_LOG_TIMEOUT=60 wait_for_log "$RUN_ID" --network -- "[model-provider:anthropic-api-key]"
refute_output --partial '[model-provider:anthropic-api-key $]'

Expand All @@ -73,10 +88,9 @@ teardown_file() {
}

@test "t54-1: vm0 meta-provider — firewall billable" {
zero_chat_run_with_model_selection \
zero_chat_run_with_model \
"$AGENT_ID" \
"Reply with exactly: DONE" \
"$(zero_model_first_selection_provider_id)" \
"claude-sonnet-4-6" \
true
THREAD_IDS="$THREAD_IDS $LAST_THREAD_ID"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,6 @@ function goalsClient() {
return setupApp({ context })(zeroGoalsContract);
}

const MODEL_FIRST_SELECTION_PROVIDER_ID =
"00000000-0000-4000-8000-000000000000";
const USER_ARTIFACTS_BUCKET = "test-user-artifacts";
const CHAT_CALLBACK_PRE_CREATE_TIMING_PREFIX =
"api_dispatch_pre_create_zero_chat_callback_";
Expand Down Expand Up @@ -163,13 +161,10 @@ async function startChatRun(
? {}
: { attachFiles: body.attachFiles }),
...(body.selectedModel === undefined
? { modelProvider: "anthropic-api-key" }
: {
modelSelection: {
modelProviderId: MODEL_FIRST_SELECTION_PROVIDER_ID,
selectedModel: body.selectedModel,
},
}),
? body.threadId === undefined
? { model: "claude-sonnet-4-6" }
: {}
: { model: body.selectedModel }),
},
[201],
);
Expand Down Expand Up @@ -2362,10 +2357,11 @@ describe("CHAT-02: auto-send across a model switch", () => {
prompt: "And stringify?",
});
const secondHeaders = await claimChatRun(runnerGroup, second.runId);
await chat.updateThreadModelSelection(actor, first.threadId, {
modelProviderId: MODEL_FIRST_SELECTION_PROVIDER_ID,
selectedModel: "claude-sonnet-4-6",
});
await chat.updateThreadModelSelection(
actor,
first.threadId,
"claude-sonnet-4-6",
);
await queueChatMessage(actor, {
agentId,
threadId: first.threadId,
Expand Down
12 changes: 2 additions & 10 deletions turbo/apps/api/src/signals/routes/__tests__/chat-files.bdd.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,6 @@ const context = testContext();
const bdd = createBddApi(context);
const api = createChatFilesBddApi(context);
const authOrg = createAuthOrgAgentsBddApi(context);
const MODEL_FIRST_SELECTION_PROVIDER_ID =
"00000000-0000-4000-8000-000000000000";

describe("CHAT-01 chat thread lifecycle", () => {
it("creates, mutates, searches, and deletes a thread through visible APIs", async () => {
Expand Down Expand Up @@ -173,10 +171,7 @@ describe("CHAT-01 chat thread lifecycle", () => {
});

await api.renameThread(owner, thread.id, "Pinned launch plan");
await api.updateThreadModelSelection(owner, thread.id, {
modelProviderId: MODEL_FIRST_SELECTION_PROVIDER_ID,
selectedModel: "gpt-5.4-mini",
});
await api.updateThreadModelSelection(owner, thread.id, "gpt-5.4-mini");
await api.pinThread(owner, thread.id);
const readEmpty = await api.markThreadRead(owner, thread.id);

Expand Down Expand Up @@ -558,10 +553,7 @@ describe("CHAT-02 chat messages and visible validation", () => {
{
agentId: agent.agentId,
prompt: "Persist the model selected at send time",
modelSelection: {
modelProviderId: MODEL_FIRST_SELECTION_PROVIDER_ID,
selectedModel: "gpt-5.4-mini",
},
model: "gpt-5.4-mini",
},
[201],
);
Expand Down
Loading
Loading