diff --git a/.changeset/command-deliveries.md b/.changeset/command-deliveries.md new file mode 100644 index 0000000000..ba70b8ab83 --- /dev/null +++ b/.changeset/command-deliveries.md @@ -0,0 +1,5 @@ +--- +"eve": minor +--- + +Session commands now return serializable, followable `Delivery` receipts, with request statuses distinct from session termination and explicit retryability on `SessionNotActiveError`. Session handles use `id`/`stream()` with one line-indexed `SessionStreamOptions`, `SessionInfo` matches `ctx.session`, client handles are type-only exports, local resets become `newSession()`, eval/CLI results use `reply`/`data`, and `invokeTool` returns call-shaped outcomes. diff --git a/apps/templates/eve-chat-template/apps/web/app/_components/agent-chat.tsx b/apps/templates/eve-chat-template/apps/web/app/_components/agent-chat.tsx index 66b87ac0f9..116e0e41a8 100644 --- a/apps/templates/eve-chat-template/apps/web/app/_components/agent-chat.tsx +++ b/apps/templates/eve-chat-template/apps/web/app/_components/agent-chat.tsx @@ -270,7 +270,7 @@ export function AgentChatSession({ const toastError = displayError && dismissedError !== displayError ? displayError : null; const resetSession = useCallback(() => { - agent.reset(); + agent.newSession(); setActiveChatId(null); activeChatIdRef.current = null; eventIndexRef.current = 0; diff --git a/apps/templates/eve-llm-council-template/apps/web/app/council-app.tsx b/apps/templates/eve-llm-council-template/apps/web/app/council-app.tsx index 435db8633c..f2cae46fd2 100644 --- a/apps/templates/eve-llm-council-template/apps/web/app/council-app.tsx +++ b/apps/templates/eve-llm-council-template/apps/web/app/council-app.tsx @@ -202,7 +202,7 @@ export function CouncilApp() { setResult(undefined); setResultError(undefined); setSummaryUsage({}); - agent.reset(); + agent.newSession(); void agent.send(message, { outputSchema: councilResultSchema }); }; diff --git a/apps/templates/eve-voice-template/apps/web/components/voice-call.tsx b/apps/templates/eve-voice-template/apps/web/components/voice-call.tsx index 7e5388b2ca..15a5033383 100644 --- a/apps/templates/eve-voice-template/apps/web/components/voice-call.tsx +++ b/apps/templates/eve-voice-template/apps/web/components/voice-call.tsx @@ -108,7 +108,7 @@ export function VoiceCall() { setError(null); setStarting(true); // One call is one eve session. - agent.reset(); + agent.newSession(); setTurns((turnsRef.current = [])); setAnchors([]); delegatedTurnsRef.current = 0; diff --git a/apps/templates/personal-agent-template/apps/web/app/composables/chat/providers/eve/init.ts b/apps/templates/personal-agent-template/apps/web/app/composables/chat/providers/eve/init.ts index 547f429d36..1d470ee105 100644 --- a/apps/templates/personal-agent-template/apps/web/app/composables/chat/providers/eve/init.ts +++ b/apps/templates/personal-agent-template/apps/web/app/composables/chat/providers/eve/init.ts @@ -47,7 +47,7 @@ export function removeEveAgent(chatId: string) { export function resetAllEveAgents() { for (const agent of agentsByChatId.values()) { - agent.reset(); + agent.newSession(); } agentsByChatId.clear(); } diff --git a/apps/templates/personal-agent-template/apps/web/app/composables/chat/providers/eve/session.ts b/apps/templates/personal-agent-template/apps/web/app/composables/chat/providers/eve/session.ts index 03c4dd5d00..1fee1a0772 100644 --- a/apps/templates/personal-agent-template/apps/web/app/composables/chat/providers/eve/session.ts +++ b/apps/templates/personal-agent-template/apps/web/app/composables/chat/providers/eve/session.ts @@ -62,7 +62,7 @@ export function createEveChatSession( } function reset() { - agent.value.reset(); + agent.value.newSession(); } async function retry() { diff --git a/docs/concepts/execution-model-and-durability.mdx b/docs/concepts/execution-model-and-durability.mdx index 5f47cff360..3b28618146 100644 --- a/docs/concepts/execution-model-and-durability.mdx +++ b/docs/concepts/execution-model-and-durability.mdx @@ -119,7 +119,7 @@ The resolver runs before the replacement's first model call, and its result ente The read and the prompt are yours to bound: -- `startIndex: -2000` reads only the newest 2,000 events, and `signal` limits how long the read may take. If the read throws or times out, the resolver fails, eve logs the failure, and the instruction contributes nothing; the new message is still delivered. +- `startIndex: -2000` reads only the newest 2,000 stored lines (each can hold several events), and `signal` limits how long the read may take. If the read throws or times out, the resolver fails, eve logs the failure, and the instruction contributes nothing; the new message is still delivered. - Keep the instruction's role `"user"` and label the text as historical data. Text the earlier user wrote should not reach the model as system instructions. - `sessions` reads any session ID, including a session that set `experimental.workflow.retention: 0`, until its data is deleted. Skip such sessions if your agent sets zero retention. Pass only the ID eve provides in `ctx.session.predecessor`, never one taken from a request or a model. diff --git a/docs/concepts/sessions-runs-and-streaming.md b/docs/concepts/sessions-runs-and-streaming.md index 10ad85fde7..4ab7349bcf 100644 --- a/docs/concepts/sessions-runs-and-streaming.md +++ b/docs/concepts/sessions-runs-and-streaming.md @@ -346,7 +346,7 @@ for await (const event of events) { } ``` -`stream()` accepts `startIndex` (negative values count back from the tail), `follow`, and `signal`. Key stored events on [`meta.position`](#positions), which stays stable across reads. With `follow: false`, the read ends at the durable tail observed when it opens; otherwise it keeps following new events until the stream closes or the signal aborts. +`stream()` takes `SessionStreamOptions` from `eve/client`: `startIndex` (a durable line position, with negative values counting back from the tail), `follow`, and `signal`. Client readers also accept `streamReconnectPolicy`. Key stored events on [`meta.position`](#positions), which stays stable across reads. With `follow: false`, the read ends at the durable tail observed when it opens; otherwise it keeps following new events until the stream closes or the signal aborts. A read reflects events that are already durable. A hook runs after its line is written, so a bounded read there includes the event that triggered it. `sessions` reads any session ID without channel auth, so check that a caller may read a session before passing an ID from a request. It is available only in code that runs inside the eve server. diff --git a/docs/concepts/upgrade-to-v27-events.md b/docs/concepts/upgrade-to-v27-events.md index 668b93c061..35c77193a6 100644 --- a/docs/concepts/upgrade-to-v27-events.md +++ b/docs/concepts/upgrade-to-v27-events.md @@ -261,15 +261,38 @@ in [Instrumentation](/docs/observability/instrumentation#lifecycle-events). The OpenTelemetry trace schema remains version 4, with the same span names and attributes. +## Commands and session handles + +Sends, answers, and session controls return a `Delivery` directly. Its own `sessionId` and +`deliveryId` are serializable without reading the stream. Iterate it or call `result()` to follow +that exact command's settlement. Replace `MessageResponse` and `MessageResult` imports with +`Delivery` and `DeliveryResult`. `ChannelSource.send()` returns a delivery too; use its `sessionId` +to attach a session handle when you need one. + +A refused inactive-session command throws `SessionNotActiveError`; its `retryable` flag distinguishes +an inbox still starting from a session that cannot accept more work. There are no separate +`no_active_turn` or `no_active_session` control results. A cancellation delivery settles when +applied; a reset delivery settles after retirement and address release. + +Session readers expose `id` and `stream()`. `sessions.attach(id)` from `eve/server` returns a +`SessionReader` without I/O, not a `ServerSession`. Channel handles carry the existing commands +through `SessionOperations`; a reader does not acquire command authority. The data type previously +exported as `Session` from `eve/context` is now `SessionInfo`, exactly the metadata in `ctx.session`: +`id`, `auth`, `turn`, optional `parent`, `schedule`, and `predecessor`. + +Every reader takes `SessionStreamOptions` from `eve/client`, replacing `StreamOptions` and +`GetEventStreamOptions`. `startIndex` addresses durable lines, matching `meta.position.line`, +not individual events. The shared options include `follow` and `signal`; only the client uses +`streamReconnectPolicy`. Client handles (`ClientSession`, `ClientAgentSession`, `ClientSessions`) +are type-only exports. Obtain them from `new Client()` and its methods instead of constructing them. + ## Clients and frontends - **Frontend commands resolve on acceptance.** `EveAgentStore.send()` and the frontend hooks' - `send()`/`respond()` return `{ sessionId, deliveryId }` without waiting for a reply or question. - Observe `view` and lifecycle callbacks for execution. `resume()` resolves after bounded catch-up, - without waiting for an active turn to end. The lower-level `MessageResponse.result()` still reads - its response through completion. -- **Read the native tables.** Frontend snapshots expose `view` (also `data.view`) and - `data.previews`. Replace `data.activeTurnId` with `activeTurn(view)?.turnId`, + `send()`/`respond()` return a `Delivery` without waiting for a reply or question. Observe `view` + and callbacks, or await `delivery.result()`, which reads the store's existing stream rather than + opening a second one. `resume()` resolves after bounded catch-up, without waiting for a turn to end. +- **Read the native tables.** Frontend snapshots expose `view` and `previews`. Replace `data.activeTurnId` with `activeTurn(view)?.turnId`, `data.turns[turnId].waiting` with `waiting(view, turnId)`, and `data.tasks[taskId].calls` with calls in `view.calls` whose `taskId` matches. Turns have `status: "running" | "settled"` and a separate `outcome`, just like the event fold. The reduced `ConversationTurn`, @@ -280,8 +303,11 @@ The OpenTelemetry trace schema remains version 4, with the same span names and a Saved cursors from v26 sessions don't apply to v27 sessions. - **Identity is `meta.position`.** `meta.id` is gone. Deduplicate on `(line, index)`. - **Responses end on their delivery.** A message's response ends when its `delivery.settled` - arrives, finishing that line. `result().status` keeps its meaning: `"waiting"` while the session - stays open, `"completed"` once it ended, `"failed"` on failure. + arrives, finishing that line. `result().status` now uses `RequestStatus`: `working`, + `awaiting-input`, `completed`, `failed`, or `cancelled`. **`completed` has changed meaning:** + it means the request completed, not that the session ended. Replace old `completed` checks for + session termination with `sessionEnded`; old `waiting` comparisons no longer typecheck. Read text + from `reply`, not `message`. `requestStatus(view, deliveryId)` from `eve/events` applies the same rule. - **Interactions.** Every interaction a person is asked, sign-ins included, is in one list: `view.interactions`, a response's `interactions`, `openInteractions(view)`. Filter by `request.kind`. An approval or a question about a call shows on its call's tool part @@ -302,10 +328,19 @@ The OpenTelemetry trace schema remains version 4, with the same span names and a client doesn't need its own reducer for session state. `callState(view, callId)` says where a call stands for a display, `callLabel(row)` how to name it, and `waiting(view, { taskId })` or `waiting(view, { callId })` who a task's or call's work waits on. +- **Distinguish local and server resets.** Call `newSession()` to clear local frontend state and + its cursor. `reset()` now retires the server-side session and returns its delivery. - **Follow children with `child.opened`.** `session.agent(opened)` takes a `child.opened` event. ## Evals +- Rename `loadedSkill` to `calledSkill`, `calledSubagent` to `calledAgent`, `noFailedActions` to + `noFailedCalls`, and `parked()` to `awaitingInput()`. `calledTool` and call-count/order assertions stay. +- Replace `await t.target.attachSession(id)` with `const session = t.target.attach(id)` followed by + `await session.nextTurn()`. Attachment is I/O-free. Replace `t.target.watchTurn(id, options)` with + `t.target.attach(id, { startIndex }).watch({ until })`, passing only the options you need. Reads + advance the handle's cursor. Eval session handles use `id`; accepted turns still expose `sessionId`. +- Turn results expose `view`, `reply`, `data`, `sessionEnded`, and the shared request status. - Match v27 facts in `t.event(...)`, `eventOrder(...)`, and `waitForEvent(...)`. Event matchers can match a fact's `scope`. - `turn.waitForToolCall(name, options)` waits for a settled call to a tool, and `toolCallsOf(events)` @@ -317,14 +352,20 @@ The OpenTelemetry trace schema remains version 4, with the same span names and a ## MCP and `eve invoke` -- **MCP channel.** An `input_required` invocation lists its questions and approvals in - `interactions`, keyed by `interactionId`; answer every entry in one `agent_update` call with - `responses: [{ interactionId, optionId?, text? }]`. An `authorization_required` invocation lists - its sign-ins in the same `interactions` shape, each with its `signIn` challenge. An id names one - interaction for the session's life, so the ids of an earlier batch never answer a later one. -- **`eve invoke`.** A run that needs a person ends with one `input-required` status whose - `interactions` include sign-ins; `authorization-required` is gone. A sign-in resumes once it - completes, without a prompt. +- **MCP channel.** `input_required` remains the wire spelling for `awaiting-input`; sign-ins now + use it too, instead of `authorization_required`. Questions, approvals, and sign-ins share + `interactions`, keyed by `interactionId`. Answer all non-sign-in entries in one `agent_update` + call with `responses: [{ interactionId, optionId?, text? }]`. Complete sign-ins externally and + poll again. An id names one interaction for the session's life, so an earlier batch's ids never + answer a later one. `sessionEnded` separately reports session termination. +- **`eve remote invoke`.** JSON results use the shared request statuses, `reply`, `data`, and + `sessionEnded`, without a nested `outcome`. Resume coordinates include the original `deliveryId`. + Deployment authentication failures are CLI errors, not session statuses. An `awaiting-input` + sign-in resumes without a prompt once completed. +- **Direct tools.** `invokeTool` returns a call-shaped `outcome`: `completed` with `output` and + `modelOutput`, `failed` with `error: ErrorInfo`, or `rejected` with a `reason`, optional + `connections`, and optional policy `cause`. Invalid input is a failure with `error.code`; + approval and sign-in requirements are rejections, never conversational pauses. ## Sessions and remote agents diff --git a/docs/evals/assertions.mdx b/docs/evals/assertions.mdx index 2af15f3221..c2d76a016c 100644 --- a/docs/evals/assertions.mdx +++ b/docs/evals/assertions.mdx @@ -12,22 +12,22 @@ Scoped assertions take no explicit value and gate by default. Assertions on `t` | Assertion | Asserts | | ----------------------------------------------------- | ----------------------------------------------------------------------------- | | `t.succeeded()` | The run did not fail and did not park on unanswered HITL input | -| `t.parked()` | The run cleanly parked on HITL input | +| `t.awaitingInput()` | The run cleanly parked on HITL input | | `t.messageIncludes(token)` | Joined assistant text contains `token` (string or RegExp) | | `turn.outputEquals(value)` / `.outputMatches(schema)` | Deep equality or Standard Schema validation of turn/session structured output | | `t.calledTool(name, opts?)` | A matching tool call completed (`input`, `output`, `status`, `count`) | -| `t.loadedSkill(skill, opts?)` | A load of `skill` completed (`output`, `status`, `count`) | +| `t.calledSkill(skill, opts?)` | A load of `skill` completed (`output`, `status`, `count`) | | `t.notCalledTool(name)` | No request for `name` in any lifecycle state | | `t.toolOrder([...names])` | Tool requests appear in order | | `t.usedNoTools()` | No tool calls at all; skill loads don't count | | `t.maxToolCalls(n)` | At most `n` tool calls; skill loads don't count | -| `t.noFailedActions()` | No tool, subagent, or skill action reported a failure | -| `t.calledSubagent(name, opts?)` | A subagent delegation happened (identity, remote, output, status, count) | +| `t.noFailedCalls()` | No tool, subagent, or skill action reported a failure | +| `t.calledAgent(name, opts?)` | A subagent delegation happened (identity, remote, output, status, count) | | `t.event(type, opts?)` / `t.notEvent(type, opts?)` | Typed event presence, data, and count matching | | `t.eventOrder([...matchers])` | Matching event groups occur in order | | `t.eventsSatisfy(label, predicate)` | Escape hatch: any predicate over the typed event stream | -`succeeded()` accepts both a closed session and a healthy session left open for the next user message; it rejects protocol failures and unanswered HITL. `parked()` requires a clean HITL park. Structured output assertions live on turns and independent sessions, where the output is unambiguous (see the [output schema guide](../guides/client/output-schema)). +`succeeded()` accepts both a closed session and a healthy session left open for the next user message; it rejects protocol failures and unanswered HITL. `awaitingInput()` requires a clean HITL park. Structured output assertions live on turns and independent sessions, where the output is unambiguous (see the [output schema guide](../guides/client/output-schema)). ```ts await t.send("What is the weather in Brooklyn?"); @@ -41,7 +41,8 @@ The same vocabulary narrows naturally in multi-turn and externally-created sessi const first = await t.send("Call get_weather for Brooklyn"); first.calledTool("get_weather", { count: 1 }); -const attached = await t.target.attachSession(sessionId); +const attached = t.target.attach(sessionId); +await attached.nextTurn(); attached.succeeded(); attached.messageIncludes("Sunny"); ``` @@ -132,7 +133,7 @@ The same entries land in the [run artifacts](./running#artifacts) under `.eve/ev ## The matcher mini-language -`t.calledTool` and `t.calledSubagent` take matcher objects. Tools accept `{ input, output, status, count }`; subagents accept `{ callId, childSessionId, output, status, count }`. Calls match `status: "completed"` by default. Tool lifecycle checks also accept `"pending"`, `"failed"`, and `"rejected"`. Subagent checks accept `"pending"`, `"completed"`, `"failed"`, `"rejected"`, `"interrupted"`, and `"abandoned"`. A numeric `count` requires an exact number of calls matching every supplied constraint. Use a predicate for ranges or other custom count requirements; it receives the observed number of matching calls. +`t.calledTool` and `t.calledAgent` take matcher objects. Tools accept `{ input, output, status, count }`; subagents accept `{ callId, childSessionId, output, status, count }`. Calls match `status: "completed"` by default. Tool lifecycle checks also accept `"pending"`, `"failed"`, and `"rejected"`. Subagent checks accept `"pending"`, `"completed"`, `"failed"`, `"rejected"`, `"interrupted"`, and `"abandoned"`. A numeric `count` requires an exact number of calls matching every supplied constraint. Use a predicate for ranges or other custom count requirements; it receives the observed number of matching calls. A subagent call is a task call: it stays `pending` until its `call.settled` arrives, then takes that call's outcome and, when completed, its `output`. A completed call leaves the child session available for later calls with the same `taskId`. @@ -150,7 +151,7 @@ t.calledTool("echo", { output: (value) => String(value).includes(marker) }); parked.calledTool("guarded", { status: "pending", count: 1 }); t.calledTool("guarded", { output: /approved/, count: 1 }); -t.calledSubagent("weather", { +t.calledAgent("weather", { remoteUrl: (value) => value === process.env.WEATHER_AGENT_URL, output: /72F/, }); @@ -202,7 +203,7 @@ await t.require( sessionIds, satisfies((ids) => ids.length > 0, "dispatch started a session"), ); -await t.target.attachSession(sessionIds[0]!); +await t.target.attach(sessionIds[0]!).nextTurn(); ``` Use the matching `require*` lookups when dependent code needs protocol data: diff --git a/docs/evals/cases.mdx b/docs/evals/cases.mdx index 68eb246ee9..7ea888e6ab 100644 --- a/docs/evals/cases.mdx +++ b/docs/evals/cases.mdx @@ -105,14 +105,14 @@ export default defineEval({ Every `t.session()` or `t.send()` creates a new independent conversation. Continue an existing conversation through its session handle. Events from all sessions feed the same run-level assertions. -- `t.session(options?)` creates a session without starting a turn or consuming its stream. It resolves on `202 Accepted` with an `EveEvalSession` whose `.sessionId` and `.state` are available. Options accept per-request `headers` and `signal`; creation uses the eval timeout signal by default. +- `t.session(options?)` creates a session without starting a turn or consuming its stream. It resolves on `202 Accepted` with an `EveEvalSession` whose `.id` and `.state` are available. Options accept per-request `headers` and `signal`; creation uses the eval timeout signal by default. - `t.send(message, options?)` creates a new session with its first message in one request and waits for the turn to settle. The returned turn carries `.session`, `.message`, and `.expectOk()`. - `session.send(message, options?)` sends a follow-up on that session and waits for it to settle. - `session.start(message, options?)` starts a text turn and returns as soon as the server accepts it. The live turn exposes `.session`, `.sessionId`, `.waitForEvent(...)`, `.cancel()`, and `.result()`. - `session.cancel()` requests cooperative cancellation of that session's active turn. Both `accepted` and `no_active_turn` are successful outcomes. - `session.compact()` compacts that session's history between turns and waits for it to finish. It resolves to the captured events of the compaction, a context change, so assertions such as `.event("context.settled", { data: { kind: "compaction", outcome: "completed" } })` work on the result. It throws when the session is no longer active. - `session.sendFile(text, path, mediaType?)` attaches a local file as a data URL. -- `session.agent(opened).stream(options?)` follows a delegated child's stream through the parent with the eval client's credentials. Pass a `child.opened` event from that session. It also reaches children on another deployment, which `t.target.watchTurn(...)` cannot address. +- `session.agent(opened).stream(options?)` follows a delegated child's stream through the parent with the eval client's credentials. Pass a `child.opened` event from that session. It also reaches children on another deployment, which `t.target.attach(...).watch()` cannot address. - `session.requireInteraction(filter?)` records a gate, requires exactly one pending request, and returns it. Filters match tool name, action input, prompt, display, and option ids. - `session.respond(responses, options?)` answers specific pending input requests and sends them as the next turn. `session.startRespond(...)` returns a live turn for the same operation. - `session.respondAll(optionId)` answers every pending input request with the same option and sends the responses as the next turn. @@ -120,7 +120,7 @@ Every `t.session()` or `t.send()` creates a new independent conversation. Contin The transcript uses `User:` and `Assistant:` labels separated by blank lines. It excludes reasoning, tool calls, tool results, and messages from other sessions. It updates after each turn settles, so read it after `await session.send(...)`, `await session.respond(...)`, or `await live.result()`. -Each `send` (and `respond`/`respondAll`) resolves to a turn with `.message`, `.data`, `.events`, `.interactions`, `.toolCalls`, `.session`, `.sessionId`, `.status`, and `.expectOk()`. Turn results retain their captured output and events; `.session` references the live conversation handle. `expectOk()` throws only when the turn ended failed; a session left open for a next message is the normal end state of a successful turn. +Each `send` (and `respond`/`respondAll`) resolves to a turn with `.reply`, `.data`, `.events`, `.interactions`, `.toolCalls`, `.session`, `.sessionId`, `.status`, and `.expectOk()`. Turn results retain their captured output and events; `.session` references the live conversation handle. `expectOk()` throws only when the turn ended failed; a session left open for a next message is the normal end state of a successful turn. ## Prewarming a session @@ -134,7 +134,7 @@ export default defineEval({ async test(t) { const session = await t.session(); const turn = await session.send("Greet Alice briefly."); - await t.require(turn.sessionId, equals(session.sessionId)); + await t.require(turn.sessionId, equals(session.id)); turn.event("session.started", { count: 1 }); turn.event("turn.started", { count: 1, data: { turnId: "turn_0" } }); t.succeeded(); diff --git a/docs/evals/reporters.mdx b/docs/evals/reporters.mdx index 989125503c..6ac98b49a2 100644 --- a/docs/evals/reporters.mdx +++ b/docs/evals/reporters.mdx @@ -101,7 +101,11 @@ Each entry in `result.assertions` is an `AssertionResult`. `name` is the display An eval can create several sessions, and a long session can produce several traces, so completion exposes a list instead of a single trace id. `onEvalStart` does not include a trace because the agent session has not started yet. If the target has no tracing configured, `onSessionStart` does not fire and the completed trace list is empty. -`result.result.derived` holds the facts the runner derived from the eval's sessions, such as tool calls and `models`: the distinct ids of the models the steps started with, in session order. Each entry in `result.result.sessions` holds the same facts for one session, in first-use order, so a reporter can tell the primary session's models from those of the other sessions the eval created or attached. A delegated subagent runs in its own session, so its models appear only when the eval attaches that session with `t.target.attachSession`. Compaction and `auto` routing calls are not included. +`result.result.derived` holds the facts the runner derived from the eval's sessions, such as tool calls and `models`: the distinct ids of the models the steps started with, in session order. Each entry in `result.result.sessions` holds the same facts for one session, in first-use order, so a reporter can tell the primary session's models from those of the other sessions the eval created or attached. A delegated subagent runs in its own session, so its models appear only when the eval attaches and reads that session with `t.target.attach(...).nextTurn()` or `watch()`. Compaction summaries are included; provider-internal routing without an eve model lifecycle is not. + +`result.result.reply` is the last observed turn's text; `data` is its structured output, without a text fallback. Its `status` describes the request, while `sessionEnded` and the optional `view` describe that turn's session. The top-level `sessionId` still identifies the primary captured session; inspect `sessions` for each session's events and view. When selecting one value for a reporter, fall back to `reply` only when `data === undefined`, so structured `null` remains an output. + +Braintrust and Datadog metadata use `eveStatus`, `eveAwaitingInput`, and `eveSessionEnded`. Update saved filters that used `eveParked` to use `eveAwaitingInput`, and use `eveSessionEnded` to distinguish request completion from session retirement. Existing records are not rewritten. `derived.usage` is the token usage and cost a session recorded, summed from its `usage.recorded` facts, including what its delegated agents spent; on a turn, it is the session's total so far. The eval's `usage` counts each captured session once and leaves out sessions another captured session opened, because that session already counts their spend. It is absent when a counted session recorded no usage. It includes compaction summaries. diff --git a/docs/evals/targets.mdx b/docs/evals/targets.mdx index 2667b22f97..2379a2f863 100644 --- a/docs/evals/targets.mdx +++ b/docs/evals/targets.mdx @@ -15,7 +15,8 @@ import { defineEval } from "eve/evals"; export default defineEval({ async test(t) { const { sessionIds } = await t.target.dispatchSchedule("heartbeat"); - await t.target.attachSession(sessionIds[0]!); + const session = t.target.attach(sessionIds[0]!); + await session.nextTurn(); t.succeeded(); t.calledTool("send_report"); }, @@ -24,8 +25,9 @@ export default defineEval({ - `t.target.fetch(path, init)` performs an authenticated fetch against the target, useful for channel and webhook ingress. See [Authentication](#authentication) for how the runner authenticates. - `t.target.dispatchSchedule(id)` triggers a [schedule](../schedules) through the dev-only schedule route and returns the session ids it created. It works only against a target with dev routes enabled (the local `eve eval` dev server, or a deployment running in development mode), and throws otherwise. -- `t.target.attachSession(sessionId, { startIndex? })` consumes one turn from a session created outside the eval, by a channel or a schedule, so its events feed the run-level assertions. `startIndex` skips events before that position, so a session already partway through its stream resumes from where you left off rather than replaying from the start. The attached session stays bound to that exact ID, so `session.send(...)` and `session.respond(...)` continue it after the turn settles. -- `t.target.watchTurn(sessionId, { startIndex? })` starts consuming an externally-created turn immediately and returns a live-turn handle. Use `waitForEvent(...)` to coordinate with mid-turn work, `cancel()` to request cancellation, and `result()` to consume through the boundary. The live turn's `.session` is the attached `EveEvalSession` for follow-up sends after settlement. +- `t.target.attach(sessionId, { startIndex? })` returns a handle without I/O. Its `id` is the exact session ID, and `startIndex` sets the initial stream cursor. An unknown or inaccessible session fails when you read it, not when you attach it. +- `session.nextTurn()` reads and records the next turn boundary. Its events feed both the session's assertions and the run-level assertions. Each read advances the same session cursor. +- `session.watch()` starts consuming immediately and returns a live-turn handle. Use `waitForEvent(...)` to coordinate with mid-turn work, `cancel()` to request cancellation, and `result()` to read through the boundary. Pass `{ until }` to stop at the commit containing a matching event, such as `context.settled` for a context change between turns. Sessions attached this way are full `EveEvalSession`s: you can keep driving them and assert directly on that session (`session.succeeded()`, `session.calledTool(...)`). Aggregate assertions on `t` continue to read the whole run, including every attached session. diff --git a/docs/evals/tool-stubs.mdx b/docs/evals/tool-stubs.mdx index 632d5154ec..275eec1209 100644 --- a/docs/evals/tool-stubs.mdx +++ b/docs/evals/tool-stubs.mdx @@ -252,7 +252,7 @@ turn.expectOk(); turn.calledTool("list_tasks", { status: "failed", count: 1 }); ``` -The eval can pass when the agent handles the failure correctly. `noFailedActions()` still rejects the failed call, so use explicit assertions for expected failures. Stub infrastructure and response-processing failures still fail verification. +The eval can pass when the agent handles the failure correctly. `noFailedCalls()` still rejects the failed call, so use explicit assertions for expected failures. Stub infrastructure and response-processing failures still fail verification. This works for ordinary tools, workflow tools, whole-agent replacements, and qualified connection operations. An exception in a persistent workflow tool ends that task; retry with a new call, rather than its failed task ID. Normal approvals apply before an outcome is consumed. diff --git a/docs/guides/client/messages.mdx b/docs/guides/client/messages.mdx index af512e00f7..a74f13283f 100644 --- a/docs/guides/client/messages.mdx +++ b/docs/guides/client/messages.mdx @@ -21,26 +21,29 @@ const { session, response } = await client.sessions.create({ console.log(response.sessionId); const result = await response.result(); -console.log(result.status, result.message); +console.log(result.status, result.reply); ``` -`response.result()` consumes the event stream and returns a `MessageResult`: +`response.result()` consumes the event stream and returns a `DeliveryResult`: -| Field | Meaning | -| ----------- | ------------------------------------------------------------------------------------- | -| `message` | The turn's reply text, from the parts `turn.settled.reply` lists. | -| `status` | `"waiting"` while the session stays open, `"completed"` once it ended, or `"failed"`. | -| `events` | All stream events observed during the turn. | -| `sessionId` | Session ID for streaming and inspection. | -| `data` | Structured output when the turn requested an [output schema](./output-schema). | +| Field | Meaning | +| -------------- | ------------------------------------------------------------------------------------ | +| `reply` | The request's reply text, from the parts `turn.settled.reply` lists. | +| `status` | `working`, `awaiting-input`, `completed`, `failed`, or `cancelled` for this request. | +| `sessionEnded` | Whether the session ended in the delivery's final commit. | +| `deliveryId` | The accepted command's ID. | +| `events` | All stream events observed during the turn. | +| `sessionId` | Session ID for streaming and inspection. | +| `data` | Structured output when the turn requested an [output schema](./output-schema). | -When the turn or the session fails, the result has `status: "failed"` rather than throwing. Transport and route errors throw `ClientError`. +When the turn or the session fails, the result has `status: "failed"` rather than throwing. HTTP failures throw rather than returning a request result. `session.send()` retries `409 session_not_ready` with bounded exponential backoff for up to 20 seconds when a durable run has been accepted but its command inbox is still starting. The caller's abort signal cancels the wait. Other errors, `session.respond()`, control methods, and raw HTTP requests do not use this retry. An unknown -or terminal session throws `ClientError` immediately with `session_not_active`. A +or terminal session throws `SessionNotActiveError` immediately with `retryable: false`. A +`session_not_ready` rejection has `retryable: true` if the bounded send retry expires. A [stranded](../../concepts/execution-model-and-durability#stranded-sessions) session throws `ClientSessionStrandedError`, a `ClientError` with `code: "session_stranded"` and the `eveVersion` that built the session when it recorded one. It is never retried automatically. Call @@ -138,7 +141,7 @@ The reply's delivery records a `response.submitted` for each answer, and each re ## Single-use responses -`MessageResponse` is single-use. Either aggregate it: +`Delivery` is single-use. Either aggregate it: ```ts const result = await response.result(); diff --git a/docs/guides/client/output-schema.mdx b/docs/guides/client/output-schema.mdx index 9b6b72abd3..34eead03f7 100644 --- a/docs/guides/client/output-schema.mdx +++ b/docs/guides/client/output-schema.mdx @@ -1,6 +1,6 @@ --- title: "Output Schema" -description: "Request structured results from eve client turns and read typed data from MessageResult." +description: "Request structured results from eve client turns and read typed data from DeliveryResult." --- Pass `outputSchema` on a client turn when the caller needs structured data instead of only assistant text. The runtime makes the model satisfy the schema before the turn settles, then emits the final payload as a `content.completed` part with `kind: "result"`. @@ -61,7 +61,7 @@ const response = await session.send("Summarize this turn.", { const { data } = await response.result(); ``` -The server is authoritative for validation. The client types `MessageResult.data` from your generic and schema, but it doesn't revalidate the streamed payload client-side. +The server is authoritative for validation. The client types `DeliveryResult.data` from your generic and schema, but it doesn't revalidate the streamed payload client-side. ## Stream the result event diff --git a/docs/guides/client/streaming.mdx b/docs/guides/client/streaming.mdx index f4bada80cd..070e3a4e6f 100644 --- a/docs/guides/client/streaming.mdx +++ b/docs/guides/client/streaming.mdx @@ -3,41 +3,39 @@ title: "Streaming" description: "Consume eve client stream events live, reconnect by line position, and aggregate turn results." --- -Every `ClientSession.send()` call posts the turn, then reads the session's NDJSON (newline-delimited JSON) event stream. `MessageResponse` gives you two ways to consume that stream, aggregating it with `result()` or iterating it live. +`ClientSession.send()` posts a message and returns a `Delivery` when the server accepts it. The receipt has its own `sessionId` and `deliveryId`; it does not wait for a reply. Call `result()` to aggregate its stream, or iterate it live. -Once `send()` is accepted, `response.cancel()` requests cooperative cancellation of that exact turn. Start consuming the response first; cancellation waits for the stream to identify the turn, guards the request with its ID, and never targets a later turn. The result status is `accepted` when the live session durably queues the command. A response that settles before a turn starts returns `no_active_turn`: +Session controls return deliveries too. `session.cancel({ turnId })` targets an observed turn, so a late cancellation cannot stop unrelated later work. Omitting `turnId` cancels the current work. For example, cancel as soon as this message reaches a turn: ```ts -const { response } = await client.sessions.create({ message: "Run the long operation." }); +const { session, response } = await client.sessions.create({ message: "Run the long operation." }); -const resultPromise = response.result(); -const cancellation = await response.cancel(); -if (cancellation.status === "accepted") { - console.log(cancellation.sessionId); +for await (const event of response) { + if ( + event.type === "delivery.consumed" && + event.data.deliveryId === response.deliveryId && + event.data.turnId !== undefined + ) { + const cancellation = await session.cancel({ turnId: event.data.turnId }); + await cancellation.result(); + } } - -const result = await resultPromise; ``` -The cancellation result is discriminated by `status`: only `accepted` includes -`sessionId`; `no_active_turn` has no session identity field. - -Cancellation does not replace stream consumption. Continue reading the response to observe its `turn.settled` with `outcome: "cancelled"` and its delivery's settlement and to advance the client session cursor normally. Use `session.cancel({ turnId })` instead when you have only a fixed session handle and an observed turn ID. +The cancellation delivery settles when the command is applied, including a no-op for a turn that already ended. Continue reading the original message's delivery to observe its outcome and advance its cursor. A live idle session also accepts cancellation; it does not return a separate `no_active_turn` result. -Between turns, `session.compact()` queues context compaction without sending model input. An accepted request reports the session id; consume the durable stream through its `context.settled` before sending the next turn. A `context.settled` with `outcome: "completed"` confirms that summarization succeeded; otherwise eve preserves the previous history, including when the model returns an empty summary. A never-started session returns `no_active_session` as a successful no-op. +Between turns, `session.compact()` queues context compaction without sending model input. `session.clear()` removes model-message history while retaining the session and its durable resources. Await their deliveries before sending another turn: ```ts const compaction = await session.compact(); -console.log(compaction.status); -``` - -Use `session.clear()` to remove model-message history while retaining the session and its durable resources. Consume its `context.settled` (`kind: "clear"`) before sending the next turn. +const compacted = await compaction.result(); -```ts const cleared = await session.clear(); -console.log(cleared.status); +await cleared.result(); ``` +Inspect `context.settled` in `compacted.events` to check whether summarization succeeded. A failed or empty summary preserves the previous history. `session.reset()` instead retires the session; its delivery settles after retirement and address release. + ## Aggregate a turn Use `result()` when you only need the final turn summary: @@ -47,26 +45,27 @@ const response = await session.send("Summarize the latest forecast."); const result = await response.result(); console.log(result.status); -console.log(result.message); +console.log(result.reply); console.log(result.events.length); ``` This consumes the stream until the message's delivery settles (`delivery.settled` for its `deliveryId`), finishing that line, or until the session ends first. A delivery settles when the turn that consumed it settles (`handled`), or when the turn pauses on a person -(`awaiting-input`), such as for an approval or question. A sign-in pauses the turn too, but the -response stays attached through it; see [authorization pauses](#authorization-pauses). +(`awaiting-input`), such as for an approval, question, or sign-in. See +[authorization pauses](#authorization-pauses). A turn that waits on its [tasks](/docs/tools/workflows#run-calls-as-tasks-task) stays open: `result()` keeps reading to its final reply. The text the model wrote before the wait is narration, not the reply. When a call the turn runs asks a question, such as a workflow tool's -`ctx.ask()`, `result()` returns `status: "waiting"` with the question in `interactions`. Answer it +`ctx.ask()`, `result()` returns `status: "awaiting-input"` with the question in `interactions`. Answer it with `respond()`, which keeps reading the same turn. A question stays pending until its `interaction.settled`, which the session you answer on also emits for a subagent's or workflow tool's question. -`result().status` is `"waiting"` while the session stays open for the next message or an answer, -`"completed"` once the session ended, and `"failed"` when the turn or session failed. +`result().status` describes this request: `working`, `awaiting-input`, `completed`, `failed`, or +`cancelled`. A completed request can leave its session open. Read `result().sessionEnded` to +check whether the session itself ended. `result()` closes that HTTP stream, including when fetch instrumentation clones the response for tracing. The durable session remains available for follow-up @@ -153,11 +152,11 @@ File parts in `delivery.consumed.data.parts` never include raw bytes or internal A sign-in is an `interaction.opened` whose `request.kind` is `"sign-in"`. It means a connection needs OAuth or another authorization challenge before the paused turn can continue. Chat UIs should render the sign-in prompt from `request.signIn`, disable ordinary text input for that session, and persist the event with the rest of the chat history. -The sign-in holds the turn open, and the stream emits `turn.paused` awaiting it while the callback is pending. An active `send()` or `respond()` response stays attached across it until the sign-in settles and the resumed turn reaches its next boundary. If you support refresh while an authorization prompt is pending, keep the session cursor from the started session and rehydrate the saved events on load; the callback or a structured decline resumes the same eve session. +The sign-in holds the turn open, and the stream emits `turn.paused` awaiting it while the callback is pending. The current delivery settles with `status: "awaiting-input"`. Follow the session with `session.stream()` to observe the callback and the resumed work. If you support refresh while authorization is pending, save the session cursor and events, then rehydrate them on load. The callback or a structured decline resumes the same eve session. ## Reconnection -HTTP connections can end before a run does. The client reconnects from the next line it hasn't read, so long turns continue without replaying events. By default, a turn response keeps reconnecting until its delivery settles or it is aborted, including while the turn is paused for a sign-in. A stream ends for good on `{"$eve":"stream.ended"}`, which the server sends once the session has ended. +HTTP connections can end before a run does. The client reconnects from the next line it hasn't read, so long turns continue without replaying events. By default, a delivery keeps reconnecting until it settles or is aborted. Use a session stream to follow work beyond that delivery's settlement. A stream ends for good on `{"$eve":"stream.ended"}`, which the server sends once the session has ended. Browser network failures use the same reconnect policy, both when opening a stream (`TypeError: Failed to fetch`, `TypeError: Load failed`, or `TypeError: NetworkError when attempting to fetch resource.`) and while reading a response body (any `TypeError`, such as `TypeError: network error`). Invalid stream events still fail the read. @@ -295,7 +294,7 @@ for await (const event of response) { clearTimeout(timeout); ``` -Once a response is aborted, create a new send for the next turn. Don't reuse the same `MessageResponse`. +Once a response is aborted, create a new send for the next turn. Don't reuse the same `Delivery`. ## What to read next diff --git a/docs/guides/frontend/overview.mdx b/docs/guides/frontend/overview.mdx index eda0ec9426..d506ded694 100644 --- a/docs/guides/frontend/overview.mdx +++ b/docs/guides/frontend/overview.mdx @@ -70,25 +70,27 @@ export function Chat() { `useEveAgent()` returns the current UI state plus commands: -| Field | What it is | -| ----------------- | ------------------------------------------------------------------------------------- | -| `view` | Committed lifecycle tables: turns, calls, tasks, interactions, deliveries, and usage. | -| `previews` | In-progress text and tool-input previews. | -| `messages` | Rendered messages, including local optimistic submissions and answers. | -| `agents` | Followed child sessions and their independent observations. | -| `status` | `"idle"`, `"busy"`, or `"resuming"`. Errors are separate. | -| `error` | The latest session creation, stream, resume, or turn failure. | -| `events` | Raw eve stream events for this session. | -| `session` | Serializable fixed session cursor (`sessionId`, `streamIndex`). | -| `prewarm` | Create the durable session without starting a turn. | -| `send`, `respond` | Submit a message or answers; resolve with `{ sessionId, deliveryId }` when accepted. | -| `resume` | Replay through the current durable tail, then keep following independently. | -| `cancel` | Request durable cancellation of the active turn. | -| `reset` | Clear local state and the local session cursor. | +| Field | What it is | +| ------------------ | ------------------------------------------------------------------------------------- | +| `view` | Committed lifecycle tables: turns, calls, tasks, interactions, deliveries, and usage. | +| `previews` | In-progress text and tool-input previews. | +| `messages` | Rendered messages, including local optimistic submissions and answers. | +| `agents` | Followed child sessions and their independent observations. | +| `status` | `"idle"`, `"busy"`, or `"resuming"`. Errors are separate. | +| `error` | The latest session creation, stream, resume, or turn failure. | +| `events` | Raw eve stream events for this session. | +| `session` | Serializable fixed session cursor (`sessionId`, `streamIndex`). | +| `prewarm` | Create the durable session without starting a turn. | +| `send`, `respond` | Submit a message or answers; resolve with a `Delivery` receipt when accepted. | +| `resume` | Replay through the current durable tail, then keep following independently. | +| `cancel` | Request durable cancellation of the active turn. | +| `clear`, `compact` | Queue a context change and return its delivery. | +| `reset` | Retire the server-side session and return its delivery. | +| `newSession` | Clear local state and the local session cursor. | Most chat UIs only need `messages` and `status`. Use `view` with selectors from `eve/events` for lifecycle state, or `events` when you need the wire log for persistence or audit. Snapshots publish after every fact in a committed line has folded. -Pass `prewarm: true` to start a workflow when the chat mounts and after each reset: +Pass `prewarm: true` to start a workflow when the chat mounts and after each `newSession()` call: ```tsx import { useEveAgent } from "eve/react"; @@ -96,7 +98,7 @@ import { useEveAgent } from "eve/react"; export function Chat() { const agent = useEveAgent({ prewarm: true }); - return ; + return ; } ``` @@ -117,7 +119,7 @@ export function Chat() {