diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index e08eae7b2..e165587b4 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -10,6 +10,8 @@ ### Fixed +- Anthropic prompt caching now retains the previous checkpoint while tool loops append a new result, avoiding repeated prefix reprocessing for API-key and OAuth requests. + ### Removed ## [2026.8.29] - 2026-08-29 diff --git a/packages/ai/changes.md b/packages/ai/changes.md index 1f1563cc2..caac2e4c2 100644 --- a/packages/ai/changes.md +++ b/packages/ai/changes.md @@ -1,5 +1,23 @@ # changes.md — ai +## Anthropic cache checkpoints across tool loops (2026-08-29) + +### What changed + +- Anthropic message serialization now retains the preceding prompt-cache checkpoint only for a genuine tool-loop continuation, including interrupted turns whose tool result and following user text are coalesced into one user message. + +### Why + +- Ordinary multi-turn histories must not create additional premium cache writes, while tool loops need a stable rolling checkpoint across appended results. + +### Why an extension could not handle it + +- Cache markers and Anthropic role coalescing are applied inside the provider wire serializer below extension-visible message handling. + +### Expected merge conflict zones + +- MEDIUM: `src/api/anthropic-messages.ts` message coalescing and final cache-marker pass. + ## Credential pool export wildcard (2026-08-27) ### What changed diff --git a/packages/ai/src/api/anthropic-messages.ts b/packages/ai/src/api/anthropic-messages.ts index c44dbd35e..a5b8679ff 100644 --- a/packages/ai/src/api/anthropic-messages.ts +++ b/packages/ai/src/api/anthropic-messages.ts @@ -1034,6 +1034,48 @@ function isCacheableUserContentBlock( return block?.type === "text" || block?.type === "image" || block?.type === "tool_result"; } +function appendUserBlocks(params: MessageParam[], newBlocks: ContentBlockParam[]): void { + if (newBlocks.length === 0) return; + const lastParam = params[params.length - 1]; + if (lastParam?.role === "user") { + if (typeof lastParam.content === "string") { + lastParam.content = [{ type: "text", text: lastParam.content }, ...newBlocks]; + } else if (Array.isArray(lastParam.content)) { + (lastParam.content as ContentBlockParam[]).push(...newBlocks); + } + } else { + params.push({ role: "user", content: newBlocks }); + } +} + +function isToolLoopContinuation(messages: MessageParam[]): boolean { + const tail = messages[messages.length - 1]; + const preceding = messages[messages.length - 2]; + return ( + tail?.role === "user" && + Array.isArray(tail.content) && + tail.content.some((block) => block.type === "tool_result") && + preceding?.role === "assistant" && + Array.isArray(preceding.content) && + preceding.content.some((block) => block.type === "tool_use") + ); +} + +function markUserMessageCacheCheckpoint(message: MessageParam, cacheControl: CacheControlEphemeral): boolean { + if (message.role !== "user") return false; + if (Array.isArray(message.content)) { + const lastBlock = message.content[message.content.length - 1]; + if (!isCacheableUserContentBlock(lastBlock)) return false; + lastBlock.cache_control = cacheControl; + return true; + } + if (typeof message.content === "string") { + message.content = [{ type: "text", text: message.content, cache_control: cacheControl }]; + return true; + } + return false; +} + const ANTHROPIC_MESSAGE_EVENTS: ReadonlySet = new Set([ "message_start", "message_delta", @@ -2012,7 +2054,7 @@ function buildParams( { type: "text", text: "You are Claude Code, Anthropic's official CLI for Claude.", - ...(cacheControl ? { cache_control: cacheControl } : {}), + ...(!context.systemPrompt && cacheControl ? { cache_control: cacheControl } : {}), }, ]; if (context.systemPrompt) { @@ -2224,10 +2266,12 @@ function convertMessages( if (msg.role === "user") { if (typeof msg.content === "string") { if (msg.content.trim().length > 0) { - params.push({ - role: "user", - content: sanitizeSurrogates(msg.content), - }); + const content = sanitizeSurrogates(msg.content); + if (params[params.length - 1]?.role === "user") { + appendUserBlocks(params, [{ type: "text", text: content }]); + } else { + params.push({ role: "user", content }); + } } } else { const blocks: ContentBlockParam[] = msg.content.map((item) => { @@ -2263,10 +2307,7 @@ function convertMessages( return true; }); if (filteredBlocks.length === 0) continue; - params.push({ - role: "user", - content: filteredBlocks, - }); + appendUserBlocks(params, filteredBlocks); } } else if (msg.role === "assistant") { const blocks: ContentBlockParam[] = []; @@ -2388,30 +2429,15 @@ function convertMessages( if (toolResults.length === 0) continue; // Displaced reference-bearing results must follow every tool_result block. - params.push({ - role: "user", - content: [...toolResults, ...siblingContent], - }); + appendUserBlocks(params, [...toolResults, ...siblingContent]); } } - // Add cache_control to the last user message to cache conversation history - if (cacheControl && params.length > 0) { - const lastMessage = params[params.length - 1]; - if (lastMessage.role === "user") { - if (Array.isArray(lastMessage.content)) { - const lastBlock = lastMessage.content[lastMessage.content.length - 1]; - if (isCacheableUserContentBlock(lastBlock)) { - lastBlock.cache_control = cacheControl; - } - } else if (typeof lastMessage.content === "string") { - lastMessage.content = [ - { - type: "text", - text: lastMessage.content, - cache_control: cacheControl, - }, - ]; + const retainPrecedingCheckpoint = isToolLoopContinuation(params); + if (cacheControl && params.length > 0 && markUserMessageCacheCheckpoint(params[params.length - 1], cacheControl)) { + if (retainPrecedingCheckpoint) { + for (let index = params.length - 2; index >= 0; index--) { + if (markUserMessageCacheCheckpoint(params[index], cacheControl)) break; } } } diff --git a/packages/ai/src/changes.md b/packages/ai/src/changes.md index ad9235917..afa9296eb 100644 --- a/packages/ai/src/changes.md +++ b/packages/ai/src/changes.md @@ -3061,3 +3061,17 @@ Detection has to happen inside the Anthropic SSE loop while the stream is still ### Expected merge conflict zones - OpenAI Completions reasoning conversion and Cloudflare provider generic declarations. + +## 2026-08-22 - Stable Anthropic cache checkpoints across tool loops + +### What changed +- `api/anthropic-messages.ts` now marks the newest and immediately preceding cacheable user-message boundaries, retaining a stable Anthropic prompt-cache checkpoint while tool loops append new results. OAuth requests with a context system prompt keep the checkpoint budget available for message history. + +### Why +- Replacing the sole tail marker on every tool turn invalidated the previous cache boundary and caused repeated prefix reprocessing instead of preserving a reusable checkpoint across adjacent loops. + +### Why an extension could not handle it +- Cache markers are attached while the Anthropic wire payload is built inside `pi-ai`; extensions cannot safely rewrite Anthropic-native message blocks after conversion. + +### Expected merge conflict zones +- MEDIUM: `api/anthropic-messages.ts` cache-control placement in `buildParams()` and the final checkpoint pass in `convertMessages()`. diff --git a/packages/ai/test/anthropic-cache-checkpoint-stability.test.ts b/packages/ai/test/anthropic-cache-checkpoint-stability.test.ts new file mode 100644 index 000000000..137ce0268 --- /dev/null +++ b/packages/ai/test/anthropic-cache-checkpoint-stability.test.ts @@ -0,0 +1,291 @@ +import { Type } from "typebox"; +import { describe, expect, it } from "vitest"; +import { buildAnthropicWarmPromptCacheParams, stream as streamAnthropic } from "../src/api/anthropic-messages.ts"; +import { getBuiltinModel as getModel } from "../src/providers/all.ts"; +import { fauxAssistantMessage, fauxToolCall } from "../src/providers/faux.ts"; +import type { Context, Model, ToolResultMessage } from "../src/types.ts"; + +const FIRST_TOOL_USE_ID = "toolu_first"; +const SECOND_TOOL_USE_ID = "toolu_second"; +const THIRD_TOOL_USE_ID = "toolu_third"; + +function toolResultMessage(toolCallId: string): ToolResultMessage { + return { + role: "toolResult", + toolCallId, + toolName: "read", + content: [{ type: "text", text: `${toolCallId} result` }], + isError: false, + timestamp: 1, + }; +} + +function markedToolResultIds(params: ReturnType): string[] { + const markedIds: string[] = []; + for (const message of params.messages) { + if (!Array.isArray(message.content)) continue; + for (const block of message.content) { + if (block.type === "tool_result" && block.cache_control !== undefined) { + markedIds.push(block.tool_use_id); + } + } + } + return markedIds; +} + +function cacheBreakpointCount(params: ReturnType): number { + return JSON.stringify(params).match(/"cache_control"/g)?.length ?? 0; +} + +interface CacheMarkerSnapshot { + readonly toolResultIds: readonly string[]; + readonly breakpointCount: number; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function cacheMarkerSnapshot(payload: unknown): CacheMarkerSnapshot { + const toolResultIds: string[] = []; + const messages = isRecord(payload) && Array.isArray(payload.messages) ? payload.messages : []; + for (const message of messages) { + if (!isRecord(message) || !Array.isArray(message.content)) continue; + for (const block of message.content) { + if ( + isRecord(block) && + block.type === "tool_result" && + "cache_control" in block && + typeof block.tool_use_id === "string" + ) { + toolResultIds.push(block.tool_use_id); + } + } + } + return { + toolResultIds, + breakpointCount: (JSON.stringify(payload) ?? "").match(/"cache_control"/g)?.length ?? 0, + }; +} + +function oauthSseResponse(): Response { + const events = [ + 'event: message_start\ndata: {"type":"message_start","message":{"id":"msg_test","usage":{"input_tokens":1,"output_tokens":0}}}\n', + 'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}\n', + 'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"ok"}}\n', + 'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}\n', + 'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":1}}\n', + 'event: message_stop\ndata: {"type":"message_stop"}\n', + ]; + return new Response(events.join("\n"), { status: 200, headers: { "content-type": "text/event-stream" } }); +} + +async function captureOAuthCacheMarkers( + model: Model<"anthropic-messages">, + context: Context, + cacheRetention?: "none", +): Promise { + let payload: unknown; + const fetch: typeof globalThis.fetch = async (_input, init) => { + payload = JSON.parse(String(init?.body)); + return oauthSseResponse(); + }; + const response = streamAnthropic(model, context, { + apiKey: "sk-ant-oat-test", + fetch, + ...(cacheRetention ? { cacheRetention } : {}), + }); + await response.result(); + if (payload === undefined) throw new Error("Expected the OAuth request payload"); + return cacheMarkerSnapshot(payload); +} + +function oauthToolLoopContexts(systemPrompt?: string): readonly [Context, Context, Context] { + const firstToolLoop: Context = { + ...(systemPrompt ? { systemPrompt } : {}), + messages: [ + { role: "user", content: "Inspect the repository", timestamp: 1 }, + fauxAssistantMessage(fauxToolCall("read", {}, { id: FIRST_TOOL_USE_ID }), { stopReason: "toolUse" }), + toolResultMessage(FIRST_TOOL_USE_ID), + ], + tools: [{ name: "read", description: "Read a file", parameters: Type.Object({}) }], + }; + const secondToolLoop: Context = { + ...firstToolLoop, + messages: [ + ...firstToolLoop.messages, + fauxAssistantMessage(fauxToolCall("read", {}, { id: SECOND_TOOL_USE_ID }), { stopReason: "toolUse" }), + toolResultMessage(SECOND_TOOL_USE_ID), + ], + }; + const thirdToolLoop: Context = { + ...secondToolLoop, + messages: [ + ...secondToolLoop.messages, + fauxAssistantMessage(fauxToolCall("read", {}, { id: THIRD_TOOL_USE_ID }), { stopReason: "toolUse" }), + toolResultMessage(THIRD_TOOL_USE_ID), + ], + }; + return [firstToolLoop, secondToolLoop, thirdToolLoop]; +} + +describe("Anthropic cache checkpoints", () => { + it("retains the preceding tool-result checkpoint while marking a new tool-result tail", () => { + const model = getModel("anthropic", "claude-sonnet-4-5"); + const firstToolLoop: Context = { + systemPrompt: "You are concise.", + messages: [ + { role: "user", content: "Inspect the repository", timestamp: 1 }, + fauxAssistantMessage(fauxToolCall("read", {}, { id: FIRST_TOOL_USE_ID }), { stopReason: "toolUse" }), + toolResultMessage(FIRST_TOOL_USE_ID), + ], + tools: [{ name: "read", description: "Read a file", parameters: Type.Object({}) }], + }; + const secondToolLoop: Context = { + ...firstToolLoop, + messages: [ + ...firstToolLoop.messages, + fauxAssistantMessage(fauxToolCall("read", {}, { id: SECOND_TOOL_USE_ID }), { stopReason: "toolUse" }), + toolResultMessage(SECOND_TOOL_USE_ID), + ], + }; + const thirdToolLoop: Context = { + ...secondToolLoop, + messages: [ + ...secondToolLoop.messages, + fauxAssistantMessage(fauxToolCall("read", {}, { id: THIRD_TOOL_USE_ID }), { stopReason: "toolUse" }), + toolResultMessage(THIRD_TOOL_USE_ID), + ], + }; + + expect(markedToolResultIds(buildAnthropicWarmPromptCacheParams(model, firstToolLoop))).toEqual([ + FIRST_TOOL_USE_ID, + ]); + expect(markedToolResultIds(buildAnthropicWarmPromptCacheParams(model, secondToolLoop))).toEqual([ + FIRST_TOOL_USE_ID, + SECOND_TOOL_USE_ID, + ]); + expect(markedToolResultIds(buildAnthropicWarmPromptCacheParams(model, thirdToolLoop))).toEqual([ + SECOND_TOOL_USE_ID, + THIRD_TOOL_USE_ID, + ]); + expect(cacheBreakpointCount(buildAnthropicWarmPromptCacheParams(model, thirdToolLoop))).toBe(4); + }); + + it.each([ + ["without", undefined], + ["with", "Keep the response concise."], + ])("retains two rolling OAuth checkpoints %s a context system prompt", async (_label, systemPrompt) => { + const model = getModel("anthropic", "claude-sonnet-4-5"); + const [firstToolLoop, secondToolLoop, thirdToolLoop] = oauthToolLoopContexts(systemPrompt); + + expect((await captureOAuthCacheMarkers(model, firstToolLoop)).toolResultIds).toEqual([FIRST_TOOL_USE_ID]); + expect((await captureOAuthCacheMarkers(model, secondToolLoop)).toolResultIds).toEqual([ + FIRST_TOOL_USE_ID, + SECOND_TOOL_USE_ID, + ]); + const thirdSnapshot = await captureOAuthCacheMarkers(model, thirdToolLoop); + expect(thirdSnapshot.toolResultIds).toEqual([SECOND_TOOL_USE_ID, THIRD_TOOL_USE_ID]); + expect(thirdSnapshot.breakpointCount).toBe(4); + }); + + it("marks only the OAuth tail when there is no preceding tool-result checkpoint", async () => { + const model = getModel("anthropic", "claude-sonnet-4-5"); + const [firstToolLoop] = oauthToolLoopContexts(); + + expect((await captureOAuthCacheMarkers(model, firstToolLoop)).toolResultIds).toEqual([FIRST_TOOL_USE_ID]); + }); + + it("omits every OAuth cache marker when cache retention is none", async () => { + const model = getModel("anthropic", "claude-sonnet-4-5"); + const [, , thirdToolLoop] = oauthToolLoopContexts("Keep the response concise."); + + expect((await captureOAuthCacheMarkers(model, thirdToolLoop, "none")).breakpointCount).toBe(0); + }); + + it("does not mark an unpaired OAuth tool result", async () => { + const model = getModel("anthropic", "claude-sonnet-4-5"); + const [firstToolLoop] = oauthToolLoopContexts(); + const unpairedResultContext: Context = { + ...firstToolLoop, + messages: [...firstToolLoop.messages, toolResultMessage("toolu_unpaired")], + }; + + expect((await captureOAuthCacheMarkers(model, unpairedResultContext)).toolResultIds).toEqual([]); + }); + + it("does not add a rolling checkpoint to an ordinary multi-turn history", () => { + const model = getModel("anthropic", "claude-sonnet-4-5"); + const params = buildAnthropicWarmPromptCacheParams(model, { + messages: [ + { role: "user", content: "first", timestamp: 1 }, + fauxAssistantMessage([{ type: "text", text: "answer" }], { stopReason: "stop" }), + { role: "user", content: "second", timestamp: 2 }, + ], + }); + + expect(cacheBreakpointCount(params)).toBe(1); + }); + + it("preserves standalone string user content when cache retention is none", () => { + const model = getModel("anthropic", "claude-sonnet-4-5"); + const params = buildAnthropicWarmPromptCacheParams( + model, + { messages: [{ role: "user", content: "standalone", timestamp: 1 }] }, + { cacheRetention: "none" }, + ); + + expect(params.messages).toEqual([{ role: "user", content: "standalone" }]); + }); + + it("handles empty history without creating checkpoints", () => { + const model = getModel("anthropic", "claude-sonnet-4-5"); + expect(cacheBreakpointCount(buildAnthropicWarmPromptCacheParams(model, { messages: [] }))).toBe(0); + }); + + it("does not create rolling checkpoints across a long non-tool history", () => { + const model = getModel("anthropic", "claude-sonnet-4-5"); + const messages = Array.from({ length: 13 }, (_, index) => + index % 2 === 0 + ? { role: "user" as const, content: `user ${index}`, timestamp: index } + : fauxAssistantMessage([{ type: "text", text: `assistant ${index}` }], { stopReason: "stop" }), + ); + + expect(cacheBreakpointCount(buildAnthropicWarmPromptCacheParams(model, { messages }))).toBe(1); + }); + + it("retains checkpoints across the compaction boundary before the latest tool loop", () => { + const model = getModel("anthropic", "claude-sonnet-4-5"); + const params = buildAnthropicWarmPromptCacheParams(model, { + messages: [ + { role: "user", content: "compacted summary", timestamp: 1 }, + fauxAssistantMessage(fauxToolCall("read", {}, { id: FIRST_TOOL_USE_ID }), { stopReason: "toolUse" }), + toolResultMessage(FIRST_TOOL_USE_ID), + ], + }); + + expect(markedToolResultIds(params)).toEqual([FIRST_TOOL_USE_ID]); + expect(cacheBreakpointCount(params)).toBe(2); + }); + + it("retains checkpoints after coalescing an interrupted tool turn with user text", () => { + const model = getModel("anthropic", "claude-sonnet-4-5"); + const params = buildAnthropicWarmPromptCacheParams(model, { + messages: [ + { role: "user", content: "first", timestamp: 1 }, + fauxAssistantMessage(fauxToolCall("read", {}, { id: FIRST_TOOL_USE_ID }), { stopReason: "toolUse" }), + toolResultMessage(FIRST_TOOL_USE_ID), + { role: "user", content: "continue", timestamp: 2 }, + ], + }); + const messages = params.messages; + + expect(messages.map((message) => message.role)).toEqual(["user", "assistant", "user"]); + expect(messages[2].content).toEqual([ + { type: "tool_result", tool_use_id: FIRST_TOOL_USE_ID, content: "toolu_first result", is_error: false }, + { type: "text", text: "continue", cache_control: { type: "ephemeral" } }, + ]); + expect(cacheBreakpointCount(params)).toBeLessThanOrEqual(4); + expect(cacheBreakpointCount(params)).toBe(2); + }); +}); diff --git a/packages/ai/test/anthropic-provider-native-replay.test.ts b/packages/ai/test/anthropic-provider-native-replay.test.ts index 52ec6939e..75f27ecc3 100644 --- a/packages/ai/test/anthropic-provider-native-replay.test.ts +++ b/packages/ai/test/anthropic-provider-native-replay.test.ts @@ -660,9 +660,9 @@ describe("Anthropic provider-native replay", () => { ); const closedAssistant = closed.messages?.find((message) => message.role === "assistant"); expect(closedAssistant?.content).not.toContainEqual(pendingUse); - const closedUser = (closed.messages ?? []).find( - (message) => message.role === "user" && Array.isArray(message.content), - ); + const closedUser = [...(closed.messages ?? [])] + .reverse() + .find((message) => message.role === "user" && Array.isArray(message.content)); expect(JSON.stringify(closedUser?.content)).toContain("tool_reference"); }); diff --git a/packages/ai/test/deferred-tools.test.ts b/packages/ai/test/deferred-tools.test.ts index 748059c99..b3d911050 100644 --- a/packages/ai/test/deferred-tools.test.ts +++ b/packages/ai/test/deferred-tools.test.ts @@ -225,6 +225,7 @@ describe("deferred tools", () => { type: "image", source: { type: "base64", media_type: "image/png", data: "aW1hZ2U=" }, }, + { type: "text", text: "Hello", cache_control: { type: "ephemeral" } }, ]); });