Skip to content

Commit 7f48d4a

Browse files
authored
fix(chat): continue one verified effect-free transport failure (#6242)
Continue the same accepted Chat request once after complete native readback proves a typed connection failure and no tool or delivery effects. Preserve Session, authorization and configuration; reserve one attempt durably and recheck after backoff. Terminal requests, uncertain effects, policy errors, changed grants and restart ambiguity remain stopped. Maintainer-authorized merge of exact-head review 5483887922 (APPROVE, author-owned COMMENTED). Required local coverage: 377 Chat/Core/Lark tests, Ruff, runtime and semantic smokes, bundle build/integrity, actual Codex app-server recovery and packaged UI; immutable baseline and late-tool negative checks independently verified. Native merge readiness is ready for this unchanged head; CI is not consulted under the configured policy. Existing owner boundaries make further recovery orchestration unnecessary. Private composition with #6239 is validated separately; production Bot adoption and original request acceptance remain open. Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com>
1 parent e0a452b commit 7f48d4a

6 files changed

Lines changed: 481 additions & 13 deletions

File tree

‎loopx/canary/module_metric_baseline.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,9 @@
3131
"dict_any_count": 0
3232
},
3333
"loopx/chat_runtime.py": {
34-
"any_count": 35,
34+
"any_count": 36,
3535
"dict_any_count": 0,
36-
"lines": 2024
36+
"lines": 2128
3737
},
3838
"loopx/chat_server.py": {
3939
"any_count": 25,

‎loopx/chat_agent.py‎

Lines changed: 85 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ def __init__(
3232
super().__init__(message)
3333
self.gate = gate
3434
self.error_code = error_code
35+
self.native_error_info: dict[str, Any] | str | None = None
3536

3637

3738
class CodexChatTimeoutError(CodexChatAgentError):
@@ -1340,6 +1341,84 @@ def _control_items(self, method: str, params: dict[str, Any], deadline: float) -
13401341
raise self._runtime_error("Codex returned an invalid control inventory cursor.")
13411342
seen_cursors.add(cursor)
13421343

1344+
def transport_retry_readback(self, turn_id: str) -> dict[str, Any] | None:
1345+
"""Read native terminal/effect facts; notification history is insufficient.
1346+
1347+
Only a failed, latest Turn with a typed connection error and a complete
1348+
text-only inventory is eligible. Unknown items, tools (including reads),
1349+
delivery, additional user input and unreadable inventories stop recovery.
1350+
This is adapter IO evidence, never permission to reopen a Core Turn.
1351+
"""
1352+
deadline = time.monotonic() + min(self.response_timeout_sec, 5.0)
1353+
1354+
def failed_turn(*, await_terminal: bool = False) -> dict[str, Any] | None:
1355+
while True:
1356+
turns = list(self._control_items("thread/turns/list", {
1357+
"sortDirection": "desc", "itemsView": "summary",
1358+
}, deadline))
1359+
ids = [row.get("id") for row in turns]
1360+
if (not ids or any(not isinstance(value, str) or not value for value in ids)
1361+
or len(set(ids)) != len(ids) or ids[0] != turn_id
1362+
or any(row.get("status") not in {"completed", "interrupted", "failed"}
1363+
for row in turns[1:])):
1364+
return None
1365+
if turns[0].get("status") == "failed":
1366+
break
1367+
# The native error notification can precede its persisted
1368+
# failed Turn. A provisional interrupted snapshot has no
1369+
# completion timestamp or error yet. Wait within the same
1370+
# readback deadline; never dispatch from this interim state.
1371+
if (not await_terminal or turns[0].get("status") not in {"inProgress", "interrupted"}
1372+
or turns[0].get("completedAt") is not None or turns[0].get("error") is not None):
1373+
return None
1374+
self._control_wait(deadline)
1375+
error = turns[0].get("error")
1376+
info = _retry_error_details(error).get("retry", {}).get("codex_error_info")
1377+
if not isinstance(info, dict) or len(info) != 1:
1378+
return None
1379+
variant, detail = next(iter(info.items()))
1380+
if not isinstance(detail, dict) or "httpStatusCode" not in detail:
1381+
return None
1382+
status = detail["httpStatusCode"]
1383+
if status is not None and not (type(status) is int and 500 <= status <= 599):
1384+
return None
1385+
return {"codex_error_info": {variant: {"httpStatusCode": status}}}
1386+
1387+
try:
1388+
terminal = failed_turn(await_terminal=True)
1389+
if terminal is None:
1390+
return None
1391+
with self._command_items_lock:
1392+
if self._command_items.get(turn_id):
1393+
return None
1394+
items = list(self._control_items("thread/items/list", {"turnId": turn_id}, deadline))
1395+
item_ids: set[str] = set()
1396+
user_count = 0
1397+
for entry in items:
1398+
item = entry.get("item")
1399+
if entry.get("turnId") != turn_id or not isinstance(item, dict):
1400+
return None
1401+
item_id = item.get("id")
1402+
if not isinstance(item_id, str) or not item_id or item_id in item_ids:
1403+
return None
1404+
item_ids.add(item_id)
1405+
if item.get("type") not in {"userMessage", "agentMessage", "reasoning", "plan"}:
1406+
return None
1407+
if item.get("type") == "userMessage":
1408+
user_count += 1
1409+
if item.get("type") == "agentMessage" and (
1410+
item.get("delivery") is not None or item.get("questions")
1411+
):
1412+
return None
1413+
# An accepted user item binds the complete inventory to this native
1414+
# request. A second user item can be a correction; never drop it.
1415+
if user_count != 1 or failed_turn() != terminal:
1416+
return None
1417+
return {"thread_id": self.thread_id, "turn_id": turn_id,
1418+
"item_count": len(items), **terminal}
1419+
except (CodexChatAgentError, OSError, TypeError, ValueError):
1420+
return None
1421+
13431422
def _control_wait(self, deadline: float) -> None:
13441423
remaining = deadline - time.monotonic()
13451424
if remaining <= 0:
@@ -1541,10 +1620,12 @@ def send(
15411620
str(turn.get("status") or "") if isinstance(turn, dict) else ""
15421621
)
15431622
if turn_status == "failed":
1544-
raise _terminal_turn_error(
1623+
error = _terminal_turn_error(
15451624
turn.get("error"),
15461625
"Codex app-server reported a terminal turn failure.",
15471626
)
1627+
error.native_error_info = _retry_error_details(turn.get("error")).get("retry", {}).get("codex_error_info")
1628+
raise error
15481629
if turn_status == "interrupted":
15491630
raise CodexChatAgentError(
15501631
"Codex app-server reported an interrupted turn.",
@@ -1567,10 +1648,12 @@ def send(
15671648
},
15681649
)
15691650
continue
1570-
raise _terminal_turn_error(
1651+
error = _terminal_turn_error(
15711652
params.get("error") if isinstance(params, dict) else None,
15721653
"Codex app-server reported a turn error.",
15731654
)
1655+
error.native_error_info = _retry_error_details(params.get("error")).get("retry", {}).get("codex_error_info") if isinstance(params, dict) else None
1656+
raise error
15741657
visible_tail = display_filter.finish()
15751658
if visible_tail and on_event:
15761659
visible_delta_count += 1

‎loopx/chat_runtime.py‎

Lines changed: 113 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
from __future__ import annotations
44

55
from dataclasses import dataclass
6+
from datetime import datetime, timedelta, timezone
7+
import hashlib
68
import json
79
import logging
810
import os
@@ -1515,14 +1517,13 @@ def _run_turn(
15151517
# This caches no Session, Turn or authority facts: each checkpoint
15161518
# still reads current state, including after a long provider wait.
15171519
with effect_runtime_request_scope():
1518-
self._run_started_turn(
1519-
session_id=session_id,
1520-
turn_id=turn_id,
1521-
message=message,
1522-
attachments=attachments,
1523-
adapter=adapter,
1524-
loopx_execution=loopx_execution,
1525-
)
1520+
retry_binding: dict[str, str] = {}
1521+
while self._run_started_turn(
1522+
session_id=session_id, turn_id=turn_id, message=message,
1523+
attachments=attachments, adapter=adapter,
1524+
loopx_execution=loopx_execution, retry_binding=retry_binding,
1525+
):
1526+
pass
15261527
finally:
15271528
done_event.set()
15281529
with self.lock:
@@ -1538,7 +1539,8 @@ def _run_started_turn(
15381539
attachments: list[AttachmentPayload],
15391540
adapter: ChatRuntimeAdapter,
15401541
loopx_execution: bool,
1541-
) -> None:
1542+
retry_binding: dict[str, str],
1543+
) -> bool | None:
15421544
"""The body of a Turn whose `queued -> starting` fact is already durable.
15431545
15441546
`_run_turn` owns the single-flight release, so nothing here needs the
@@ -1652,6 +1654,14 @@ def event_sink(kind: str, payload: dict[str, Any]) -> None:
16521654
adapter.goal_driver = None
16531655
else:
16541656
from .usage_goal import observe_goal_execution
1657+
if isinstance(adapter, CodexAppServerAdapter):
1658+
binding = self._transport_retry_binding(session_id, turn_id, adapter)
1659+
if retry_binding and binding != retry_binding["sha256"]:
1660+
raise CodexChatAgentError(
1661+
"The original request configuration changed; transport recovery stopped.",
1662+
error_code="host_gate", gate=None,
1663+
)
1664+
retry_binding["sha256"] = binding
16551665
# Manager/native/external conversations have different execution
16561666
# and waiting boundaries. Do not invent Goal timing for them.
16571667
observed_goal = str(session.get("goal_id") or "") if scope["kind"] == "owner_goal" else ""
@@ -1718,6 +1728,15 @@ def event_sink(kind: str, payload: dict[str, Any]) -> None:
17181728
self._fail_turn(session_id, turn_id, exc.error_code, str(exc), status="timed_out")
17191729
except CodexChatAgentError as exc:
17201730
event_buffer.close()
1731+
if execution_ended():
1732+
return
1733+
if (not loopx_execution and parse_native_goal_command(message) is None
1734+
and isinstance(adapter, CodexAppServerAdapter)
1735+
and exc.error_code in {"host_gate", "upstream_transport_failure"}
1736+
and self._prepare_transport_retry(
1737+
session_id, turn_id, adapter, retry_binding, exc.native_error_info,
1738+
)):
1739+
return True
17211740
if execution_ended():
17221741
return
17231742
self._fail_turn(session_id, turn_id, exc.error_code, str(exc), status="failed", gate=exc.gate)
@@ -1740,6 +1759,91 @@ def event_sink(kind: str, payload: dict[str, Any]) -> None:
17401759
with self.lock:
17411760
self.turn_event_buffers.pop(key, None)
17421761

1762+
def _transport_retry_binding(
1763+
self, session_id: str, turn_id: str, adapter: CodexAppServerAdapter,
1764+
) -> str:
1765+
"""Compare the original request and current grants/config, not prose."""
1766+
session = self.store.load_session(session_id) or {}
1767+
turn = self.store.load_turn(session_id, turn_id) or {}
1768+
self._check_codex_home(session)
1769+
if session.get("project_context") is not None or session.get("steward_context") is not None:
1770+
self.project_contexts.session_context(session)
1771+
if session.get("project_context") is not None:
1772+
project_context_policy.validate_project_executor_scope(
1773+
str(session["agent_id"]), session["project_context"], self.capabilities,
1774+
)
1775+
manager = (self.manager_runtime_profile(
1776+
str(session.get("channel_id") or "manager"),
1777+
steward_context=session.get("steward_context"),
1778+
) if is_manager_channel(session.get("channel_id")) else None)
1779+
native = adapter.session
1780+
facts = {
1781+
"session": {key: value for key, value in session.items() if key not in {
1782+
"status", "active_turn_id", "last_activity_at", "last_error_code", "updated_at",
1783+
}},
1784+
"request": {key: turn.get(key) for key in (
1785+
"turn_id", "client_turn_id", "host_claim_id", "message", "attachments", "origin",
1786+
)},
1787+
"native": {key: getattr(native, key) for key in (
1788+
"thread_id", "model", "reasoning_effort", "sandbox", "permissions_profile",
1789+
"runtime_profile", "project_context", "execution_mode",
1790+
)},
1791+
"manager": manager,
1792+
"allocation": manager_session_model_allocation(self, session) if manager else None,
1793+
}
1794+
return hashlib.sha256(json.dumps(facts, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
1795+
1796+
def _prepare_transport_retry(
1797+
self, session_id: str, turn_id: str, adapter: CodexAppServerAdapter,
1798+
retry_binding: dict[str, str], native_error_info: dict[str, Any] | str | None,
1799+
) -> bool:
1800+
"""One same-worker retry, reserved durably before backoff/dispatch.
1801+
1802+
The existing queued CAS and Core active claim own execution. The event
1803+
ledger retains the attempt across restart/compaction; a new worker never
1804+
reopens starting/terminal Turns or resets this allowance. Lark delivery
1805+
and other adapters do not decide whether to replay a model request.
1806+
"""
1807+
from .control_plane.chat_turn_acceptance import managed_turn_execution_allowed
1808+
1809+
def current() -> bool:
1810+
with self.lock:
1811+
attached = self.adapters.get(session_id) is adapter
1812+
return (attached and not self.closed.is_set() and adapter.healthcheck()
1813+
and managed_turn_execution_allowed(
1814+
self.store.load_session(session_id), self.store.load_turn(session_id, turn_id),
1815+
) and retry_binding.get("sha256") == self._transport_retry_binding(session_id, turn_id, adapter)
1816+
and not any(row.get("kind") == "turn.steered"
1817+
for row in self.store.events_after(session_id, turn_id, None)))
1818+
1819+
try:
1820+
if not isinstance(native_error_info, dict):
1821+
return False
1822+
if retry_binding.get("attempt") or not current() or any(row.get("kind") == "transport.retry_planned"
1823+
for row in self.store.events_after(session_id, turn_id, None)):
1824+
return False
1825+
original = str((self.store.load_turn(session_id, turn_id) or {}).get("upstream_turn_id") or "")
1826+
proof = adapter.session.transport_retry_readback(original) if original else None
1827+
if proof is None or proof["codex_error_info"] != native_error_info or not current():
1828+
return False
1829+
delay = 2.0
1830+
retry_binding["attempt"] = "1"
1831+
self.store.append_event(session_id, turn_id, kind="transport.retry_planned", payload={
1832+
"attempt": 1, "upstream_turn_id": original,
1833+
"binding_sha256": retry_binding["sha256"], "readback": proof,
1834+
"not_before": (datetime.now(timezone.utc) + timedelta(seconds=delay)).isoformat(),
1835+
})
1836+
self.store.append_event(session_id, turn_id, kind="agent.phase", payload={
1837+
"phase": "retrying", "label": "连接暂时中断,已核验本轮未执行工具;正在原会话继续。",
1838+
})
1839+
if self.closed.wait(delay) or not current():
1840+
return False
1841+
# Stop/config changes during the final native read also win. A tool
1842+
# or new input appearing late cannot borrow the earlier proof.
1843+
return adapter.session.transport_retry_readback(original) == proof and current()
1844+
except (CodexChatAgentError, KeyError, OSError, TypeError, ValueError):
1845+
return False
1846+
17431847
def _fail_turn(
17441848
self,
17451849
session_id: str,

‎tests/test_chat_agent.py‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,77 @@ def write(packet):
114114
return session, state
115115

116116

117+
@pytest.mark.parametrize("fault", [
118+
None, "other", "cyberPolicy", "unauthorized", "usageLimitExceeded", "rateLimitExceeded",
119+
"http401", "http429", "bad-status", "commandExecution", "fileChange", "dynamicToolCall",
120+
"mcpToolCall", "webSearch", "collabAgentToolCall", "futureItem", "delivery", "correction",
121+
"wrong-turn", "missing-data", "cursor-cycle", "new-turn", "late-tool", "changed-error",
122+
"native-error-before-terminal", "pending-terminal", "interrupted", "completed",
123+
])
124+
def test_transport_recovery_requires_complete_native_terminal_and_effect_readback(tmp_path, monkeypatch, fault):
125+
session = chat_agent.CodexChatAgentSession(process=SimpleNamespace(poll=lambda: None),
126+
messages=queue.Queue(), thread_id="recovery-thread", work_dir=tmp_path,
127+
response_timeout_sec=.2)
128+
calls = []
129+
reads = 0
130+
131+
def write(packet):
132+
nonlocal reads
133+
method, params = packet["method"], packet["params"]
134+
calls.append((method, params))
135+
assert params["threadId"] == "recovery-thread"
136+
if method == "thread/turns/list":
137+
reads += 1
138+
info = {"responseStreamDisconnected": {"httpStatusCode": None}}
139+
if fault in {"other", "cyberPolicy", "unauthorized", "usageLimitExceeded", "rateLimitExceeded"}:
140+
info = fault
141+
if fault in {"http401", "http429", "bad-status"}:
142+
info = {"responseStreamDisconnected": {"httpStatusCode": {
143+
"http401": 401, "http429": 429, "bad-status": True,
144+
}[fault]}}
145+
if fault == "changed-error" and reads > 1:
146+
info = "other"
147+
result = {"data": [{"id": "failed-turn", "status": "failed", "error": {
148+
"codexErrorInfo": info, "message": "private fixture detail",
149+
}}]}
150+
if fault == "new-turn":
151+
result["data"].insert(0, {"id": "new-turn", "status": "inProgress"})
152+
if fault == "pending-terminal" or (fault == "native-error-before-terminal" and reads == 1):
153+
result["data"][0].update(status="interrupted", error=None, completedAt=None)
154+
if fault in {"interrupted", "completed"}:
155+
result["data"][0].update(status=fault, error=None, completedAt=1)
156+
else:
157+
assert method == "thread/items/list"
158+
item = {"id": "request", "type": "userMessage"}
159+
if params.get("cursor"):
160+
item = {"id": "answer", "type": "agentMessage", "delivery": None}
161+
if fault == "late-tool":
162+
item["type"] = "commandExecution"
163+
if fault == "correction":
164+
item["type"] = "userMessage"
165+
elif fault in {"commandExecution", "fileChange", "dynamicToolCall", "mcpToolCall",
166+
"webSearch", "collabAgentToolCall", "futureItem"}:
167+
item["type"] = fault
168+
if fault == "delivery":
169+
item.update(type="agentMessage", delivery={"status": "delivered"})
170+
result = {"data": [{"turnId": "wrong" if fault == "wrong-turn" else "failed-turn", "item": item}],
171+
"nextCursor": "second" if not params.get("cursor") or fault == "cursor-cycle" else None}
172+
if fault == "missing-data":
173+
result.pop("data")
174+
session.messages.put({"id": packet["id"], "result": result})
175+
176+
monkeypatch.setattr(session, "_write", write)
177+
proof = session.transport_retry_readback("failed-turn")
178+
if fault in {None, "native-error-before-terminal"}:
179+
assert proof == {"thread_id": "recovery-thread", "turn_id": "failed-turn", "item_count": 2,
180+
"codex_error_info": {"responseStreamDisconnected": {"httpStatusCode": None}}}
181+
assert reads == (3 if fault else 2) and len(calls) == reads + 2
182+
assert "private fixture" not in json.dumps(proof)
183+
else:
184+
assert proof is None
185+
assert all(method in {"thread/turns/list", "thread/items/list"} for method, _ in calls)
186+
187+
117188
def test_stop_cancels_completed_and_late_command_items_but_preserves_older_terminal(tmp_path, monkeypatch):
118189
session, state = _stop_server(tmp_path, monkeypatch)
119190
session.interrupt("current-turn")

0 commit comments

Comments
 (0)