Skip to content

Commit 248e768

Browse files
feat: fine-grained tool streaming via TFY beta header (#1088)
* feat: fine-grained tool streaming via TFY beta header TrueFoundry uses anthropic_beta instead of mutating tool defs; native Anthropic keeps call-level toolStreaming. Co-authored-by: Cursor <cursoragent@cursor.com> * refactor: inline TFY fine-grained tool streaming beta string Co-authored-by: Cursor <cursoragent@cursor.com> * fix: use anthropic-beta header name for TFY tool streaming Match Anthropic's wire header spelling instead of anthropic_beta. Co-authored-by: Cursor <cursoragent@cursor.com> * chore: drop TFY beta header unit test; note deprecation Co-authored-by: Cursor <cursoragent@cursor.com> * feat: report invalid tool-arg JSON as a distinct parse error Fine-grained streaming can leave incomplete arguments; surface parse_error without echoing the raw payload. Co-authored-by: Cursor <cursoragent@cursor.com> --------- Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent 0a42995 commit 248e768

5 files changed

Lines changed: 111 additions & 6 deletions

File tree

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@truefoundry/trueforge-core': patch
3+
---
4+
5+
Enable fine-grained tool streaming via TrueFoundry `anthropic-beta` header and Anthropic `toolStreaming`, and report invalid tool-arg JSON as a distinct parse error

‎packages/trueforge-core/src/core/llm/VercelAILLM.ts‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,12 @@ function compatibleModel(config: VercelAIProviderConfig): LanguageModel {
132132
if (baseUrl === undefined) {
133133
throw new Error(`Provider "${provider.type}" requires a baseUrl`);
134134
}
135+
// More straightforward than mutating tool definitions for fine-grained tool streaming.
136+
// TODO: fine-grained-tool-streaming-2025-05-14 is considered deprecated.
137+
const outboundHeaders =
138+
provider.type === 'truefoundry'
139+
? { ...headers, 'anthropic-beta': 'fine-grained-tool-streaming-2025-05-14' }
140+
: headers;
135141
const client = createOpenAICompatible({
136142
name: provider.type,
137143
baseURL: baseUrl,
@@ -141,7 +147,7 @@ function compatibleModel(config: VercelAIProviderConfig): LanguageModel {
141147
supportsStructuredOutputs: true,
142148
// These endpoints omit token counts from streamed responses unless asked.
143149
includeUsage: true,
144-
...(Object.keys(headers).length > 0 ? { headers } : {}),
150+
...(Object.keys(outboundHeaders).length > 0 ? { headers: outboundHeaders } : {}),
145151
});
146152
return client(model.id);
147153
}
@@ -390,6 +396,9 @@ function openaiProviderOptions({
390396
*
391397
* Prompt caching defaults on: Anthropic bills a write once, then cheaper reads on stable prefixes
392398
* (system / tools / early messages). Callers can override via `cache_control` in the request body.
399+
*
400+
* `toolStreaming` is pinned on so every function tool gets `eager_input_streaming` on the Anthropic
401+
* wire (call-level default).
393402
*/
394403
function anthropicProviderOptions(rawBody: unknown): JSONObject | undefined {
395404
const cacheControl = readBodyField({ rawBody, key: 'cache_control' }) ?? { type: 'ephemeral' };
@@ -398,6 +407,7 @@ function anthropicProviderOptions(rawBody: unknown): JSONObject | undefined {
398407
const effort = readBodyField({ rawBody, key: 'effort' });
399408
const opts: JSONObject = {
400409
cacheControl,
410+
toolStreaming: true,
401411
...(disableParallelToolUse !== undefined ? { disableParallelToolUse } : {}),
402412
...(thinking !== undefined ? { thinking } : {}),
403413
...(effort !== undefined ? { effort } : {}),

‎packages/trueforge-core/src/core/mcp/executeToolCalls.ts‎

Lines changed: 48 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,36 @@ export interface ExecuteToolCallsResult {
3737
events: RegisteredPassthroughEvent[];
3838
}
3939

40+
function isJsonObject(value: unknown): value is Record<string, unknown> {
41+
return typeof value === 'object' && value !== null && !Array.isArray(value);
42+
}
43+
44+
/**
45+
* Parse tool-call `function.arguments`. Fine-grained streaming can leave incomplete JSON;
46+
* failures are reported as tool errors so the model can retry rather than treating them as
47+
* generic execution failures.
48+
*/
49+
function parseToolCallArguments(
50+
rawArguments: string,
51+
): { ok: true; args: Record<string, unknown> } | { ok: false; parseError: string } {
52+
let parsed: unknown;
53+
try {
54+
parsed = JSON.parse(rawArguments || '{}');
55+
} catch (error) {
56+
return {
57+
ok: false,
58+
parseError: error instanceof Error ? error.message : 'JSON parse failed',
59+
};
60+
}
61+
if (!isJsonObject(parsed)) {
62+
return {
63+
ok: false,
64+
parseError: 'Tool call arguments must be a JSON object',
65+
};
66+
}
67+
return { ok: true, args: parsed };
68+
}
69+
4070
export async function executeToolCalls({
4171
assistantMessage,
4272
toolMapping,
@@ -82,12 +112,28 @@ export async function executeToolCalls({
82112
};
83113
}
84114

115+
const parsedArgs = parseToolCallArguments(toolCall.function.arguments);
116+
if (!parsedArgs.ok) {
117+
return {
118+
toolCall,
119+
toolInfo,
120+
response: toolResultResponse({
121+
text: JSON.stringify({
122+
error: 'Failed to parse tool call arguments as JSON',
123+
parse_error: parsedArgs.parseError,
124+
}),
125+
isError: true,
126+
}),
127+
failure: true,
128+
completedAt: new Date().toISOString(),
129+
};
130+
}
131+
85132
try {
86-
const args: Record<string, unknown> = JSON.parse(toolCall.function.arguments || '{}') as Record<string, unknown>;
87133
const response = await toolInfo.toolSet.callTool(
88134
{
89135
name: toolInfo.originalToolName,
90-
arguments: args,
136+
arguments: parsedArgs.args,
91137
},
92138
approvalDecisions.get(toolCall.id),
93139
);

‎packages/trueforge-core/tests/core/llm/VercelAILLM.output.test.ts‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -249,7 +249,9 @@ describe('buildProviderOptions', () => {
249249
structuredOutputSpec: textSpec,
250250
rawBody: {},
251251
});
252-
expect(opts).toEqual({ anthropic: { cacheControl: { type: 'ephemeral' } } });
252+
expect(opts).toEqual({
253+
anthropic: { cacheControl: { type: 'ephemeral' }, toolStreaming: true },
254+
});
253255
}
254256
});
255257

@@ -263,6 +265,7 @@ describe('buildProviderOptions', () => {
263265
});
264266
expect(opts['anthropic']).toEqual({
265267
cacheControl: { type: 'ephemeral' },
268+
toolStreaming: true,
266269
thinking: { type: 'adaptive', display: 'summarized' },
267270
effort: 'max',
268271
});
@@ -275,7 +278,9 @@ describe('buildProviderOptions', () => {
275278
structuredOutputSpec: schemaSpecStrict,
276279
rawBody: {},
277280
});
278-
expect(opts).toEqual({ anthropic: { cacheControl: { type: 'ephemeral' } } });
281+
expect(opts).toEqual({
282+
anthropic: { cacheControl: { type: 'ephemeral' }, toolStreaming: true },
283+
});
279284
});
280285

281286
it('defaults cacheControl to ephemeral when cache_control is omitted', () => {
@@ -285,7 +290,7 @@ describe('buildProviderOptions', () => {
285290
structuredOutputSpec: textSpec,
286291
rawBody: {},
287292
});
288-
expect(opts['anthropic']).toEqual({ cacheControl: { type: 'ephemeral' } });
293+
expect(opts['anthropic']).toEqual({ cacheControl: { type: 'ephemeral' }, toolStreaming: true });
289294
});
290295

291296
it('forwards cache_control and disable_parallel_tool_use from rawBody', () => {
@@ -298,6 +303,7 @@ describe('buildProviderOptions', () => {
298303
expect(opts['anthropic']).toMatchObject({
299304
cacheControl: { type: 'ephemeral', ttl: '1h' },
300305
disableParallelToolUse: true,
306+
toolStreaming: true,
301307
});
302308
});
303309
});
Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
import type { MappedMCPTool } from '../../../src/core/mcp/convertMCPServers';
2+
import { executeToolCalls } from '../../../src/core/mcp/executeToolCalls';
3+
import { makeMockIMCPServer } from '../harnessMocks';
4+
5+
describe('executeToolCalls', () => {
6+
it('reports invalid tool-arg JSON as a parse failure without calling the tool', async () => {
7+
const toolSet = makeMockIMCPServer({ name: 'srv', preload: true });
8+
const toolMapping = new Map<string, MappedMCPTool>([['search', { toolSet, originalToolName: 'search' }]]);
9+
10+
const { toolCallResults } = await executeToolCalls({
11+
assistantMessage: {
12+
role: 'assistant',
13+
content: null,
14+
tool_calls: [
15+
{
16+
id: 'call_1',
17+
type: 'function',
18+
function: { name: 'search', arguments: '{"q":' },
19+
tool_info: {
20+
type: 'mcp',
21+
mcp_server_id: 'srv',
22+
mcp_server_name: 'srv',
23+
original_tool_name: 'search',
24+
},
25+
},
26+
],
27+
},
28+
toolMapping,
29+
threadId: 'thread-1',
30+
approvalDecisions: new Map(),
31+
});
32+
33+
expect(toolSet.callTool).not.toHaveBeenCalled();
34+
expect(toolCallResults).toHaveLength(1);
35+
expect(toolCallResults[0]?.failure).toBe(true);
36+
expect(toolCallResults[0]?.message.content).toContain('Failed to parse tool call arguments as JSON');
37+
});
38+
});

0 commit comments

Comments
 (0)