diff --git a/docs/core/model.mdx b/docs/core/model.mdx index 4a264d1..8a595d3 100644 --- a/docs/core/model.mdx +++ b/docs/core/model.mdx @@ -17,6 +17,9 @@ AgentScope provides a unified interface for multiple language model providers: Qwen models from Alibaba Cloud + + Native Gemini Developer API with function calling + Local models via Ollama @@ -116,6 +119,63 @@ const model = new DashScopeChatModel({ }); ``` +### Gemini + +```typescript +import { GeminiChatModel } from '@agentscope-ai/agentscope/model'; +import { createMsg, TextBlock } from '@agentscope-ai/agentscope/message'; + +const model = new GeminiChatModel({ + modelName: 'gemini-3.8-flash', + apiKey: process.env.GEMINI_API_KEY!, + stream: false, + presetGenParams: { + temperature: 0.7, + maxOutputTokens: 1024, + }, +}); + +const response = await model.call({ + messages: [ + createMsg({ + name: 'user', + role: 'user', + content: [TextBlock({ text: 'Explain how a rainbow forms.' })], + }), + ], +}); +``` + +This adapter uses the Gemini Developer API's native `generateContent` and +`streamGenerateContent` endpoints. `baseURL` can override the API root (including +its version); Vertex AI authentication is not supported. No Google SDK dependency +is required. A `GeminiChatFormatter` is provided automatically and can also be +imported from `@agentscope-ai/agentscope/formatter`. + +Pass native camelCase generation settings in `presetGenParams`, or override them +per call using `generationConfig`. Per-call `safetySettings` and `signal` +(`AbortSignal`) are also supported. The adapter uses the first response candidate. + +Function calling supports `auto`, `none`, `required`, and a specific function name. +Tool schemas use Gemini's `parametersJsonSchema` field. Parallel function calls are +returned as separate pending tool-call blocks; their results are grouped into one +user turn on the following request. `callStructured` uses the base class's named +function-calling mechanism. Function arguments are received as complete objects; +experimental partial function-argument streaming is not supported. + +With `stream: true` (the default), `call()` returns an async generator. Each yield +contains incremental content; the generator's return value contains the complete +response and the final token usage, including thought tokens. The `Agent` consumes +both automatically. When using the generator directly, read it with `.next()` if +you need the return value; a `for await` loop only exposes the deltas. + +The formatter accepts inline base64 data and Gemini-compatible file URIs. +HTTP image URLs are not downloaded automatically: use the Gemini Files API or +supply base64 data. Signed response parts retain an optional `thought_signature` +field through message schema parsing. Keep these blocks intact in saved history; +dropping their signatures can cause Gemini to reject subsequent tool turns. +Unsigned thought summaries are omitted from replay. + ### Ollama ```typescript diff --git a/packages/agentscope/src/formatter/gemini-chat-formatter.test.ts b/packages/agentscope/src/formatter/gemini-chat-formatter.test.ts new file mode 100644 index 0000000..ed1b337 --- /dev/null +++ b/packages/agentscope/src/formatter/gemini-chat-formatter.test.ts @@ -0,0 +1,125 @@ +import { GeminiChatFormatter } from './gemini-chat-formatter'; +import { + createMsg, + TextBlock, + DataBlock, + ThinkingBlock, + HintBlock, + ToolCallBlock, + ToolResultBlock, + parseContentBlock, +} from '../message'; + +const formatter = new GeminiChatFormatter(); + +describe('GeminiChatFormatter', () => { + test('formats multimodal input and hints without changing the conversation', async () => { + const msgs = [ + createMsg({ + name: 'user', + role: 'user', + content: [ + TextBlock({ text: 'Describe' }), + DataBlock({ + source: { type: 'base64', data: 'aGVsbG8=', media_type: 'image/png' }, + }), + DataBlock({ + source: { + type: 'url', + url: 'https://generativelanguage.googleapis.com/v1beta/files/sample', + media_type: 'video/mp4', + }, + }), + ], + }), + createMsg({ + name: 'agent', + role: 'assistant', + content: [HintBlock({ hint: 'A hint' })], + }), + ]; + const before = JSON.stringify(msgs); + expect(await formatter.format({ msgs })).toEqual([ + { + role: 'user', + parts: [ + { text: 'Describe' }, + { inlineData: { mimeType: 'image/png', data: 'aGVsbG8=' } }, + { + fileData: { + mimeType: 'video/mp4', + fileUri: + 'https://generativelanguage.googleapis.com/v1beta/files/sample', + }, + }, + { text: 'A hint' }, + ], + }, + ]); + expect(JSON.stringify(msgs)).toBe(before); + }); + + test('retains signed thought parts and omits unsigned summaries', async () => { + const msg = createMsg({ + name: 'agent', + role: 'assistant', + content: [ + ThinkingBlock({ thinking: 'Unsigned summary' }), + ThinkingBlock({ thinking: 'Signed', thought_signature: 'sig' }), + ], + }); + expect(await formatter.format({ msgs: [msg] })).toEqual([ + { role: 'model', parts: [{ text: 'Signed', thought: true, thoughtSignature: 'sig' }] }, + ]); + }); + + test('rejects non-object function arguments instead of sending an invalid request', async () => { + const msg = createMsg({ + name: 'agent', + role: 'assistant', + content: [ToolCallBlock({ id: 'call', name: 'weather', input: '[]' })], + }); + await expect(formatter.format({ msgs: [msg] })).rejects.toThrow('JSON object'); + }); + + test('converts tool failures into native error responses', async () => { + const msg = createMsg({ + name: 'tools', + role: 'assistant', + content: [ + ToolResultBlock({ id: 'call', name: 'weather', output: 'timeout', state: 'error' }), + ], + }); + expect(await formatter.format({ msgs: [msg] })).toEqual([ + { + role: 'user', + parts: [ + { + functionResponse: { + name: 'weather', + id: 'call', + response: { error: 'timeout' }, + }, + }, + ], + }, + ]); + }); + + test('preserves optional signatures on text, tool-call and data blocks during schema parsing', () => { + for (const block of [ + TextBlock({ text: 'hello', thought_signature: 'text-sig' }), + ToolCallBlock({ + id: 'call', + name: 'weather', + input: '{}', + thought_signature: 'call-sig', + }), + DataBlock({ + source: { type: 'base64', data: 'aGVsbG8=', media_type: 'image/png' }, + thought_signature: 'data-sig', + }), + ]) + expect(parseContentBlock(JSON.parse(JSON.stringify(block)))).toEqual(block); + }); +}); diff --git a/packages/agentscope/src/formatter/gemini-chat-formatter.ts b/packages/agentscope/src/formatter/gemini-chat-formatter.ts new file mode 100644 index 0000000..5354b1c --- /dev/null +++ b/packages/agentscope/src/formatter/gemini-chat-formatter.ts @@ -0,0 +1,119 @@ +import { FormatterBase } from './base'; +import type { ContentBlock, DataBlock } from '../message/block'; +import { getContentBlocks } from '../message/message'; +import type { Msg } from '../message/message'; + +/** Gemini REST content part, including the signature needed for history replay. */ +export interface GeminiPart { + text?: string; + thought?: boolean; + thoughtSignature?: string; + functionCall?: { name: string; args?: Record; id?: string }; + functionResponse?: { name: string; response: Record; id?: string }; + inlineData?: { mimeType: string; data: string }; + fileData?: { mimeType: string; fileUri: string }; +} + +/** Converts AgentScope messages to Gemini's native contents format. */ +export class GeminiChatFormatter extends FormatterBase { + /** + * Format messages without mutating the caller's conversation. + * @param root0 + * @param root0.msgs + * @returns The formatted or accumulated result. + */ + async format({ msgs }: { msgs: Msg[] }): Promise[]> { + const result: { role: string; parts: GeminiPart[] }[] = []; + const append = (role: string, parts: GeminiPart[]) => { + if (!parts.length) return; + const last = result.at(-1); + // Parallel function responses must stay in one user turn. + if (last?.role === role) last.parts.push(...parts); + else result.push({ role, parts }); + }; + for (const msg of msgs) { + const role = msg.role === 'assistant' ? 'model' : msg.role; + for (const block of getContentBlocks(msg)) { + if (block.type === 'tool_result') { + const { text } = this.convertToolOutputToString(block.output, false); + append('user', [ + { + functionResponse: { + name: block.name, + id: block.id, + response: + block.state === 'error' ? { error: text } : { output: text }, + }, + }, + ]); + } else if (block.type === 'hint') { + append( + 'user', + typeof block.hint === 'string' + ? [{ text: block.hint }] + : block.hint.map(b => this.formatPart(b)).filter(p => p !== null) + ); + } else { + const part = this.formatPart(block); + if (part) append(role, [part]); + } + } + } + return result; + } + + /** + * Format one content block and preserve its opaque provider signature. + * @param block + * @returns The formatted or accumulated result. + */ + private formatPart(block: ContentBlock): GeminiPart | null { + let part: GeminiPart; + switch (block.type) { + case 'text': + part = { text: block.text }; + break; + case 'thinking': + // Unsigned thought summaries are not user-facing conversation text. + if (!block.thought_signature) return null; + part = { text: block.thinking, thought: true }; + break; + case 'tool_call': { + const args: unknown = JSON.parse(block.input || '{}'); + if (!args || typeof args !== 'object' || Array.isArray(args)) { + throw new Error( + `Gemini tool arguments for ${block.name} must be a JSON object` + ); + } + part = { + functionCall: { + name: block.name, + args: args as Record, + id: block.id, + }, + }; + break; + } + case 'data': + part = this.formatData(block); + break; + default: + return null; + } + if ('thought_signature' in block && typeof block.thought_signature === 'string') { + part.thoughtSignature = block.thought_signature; + } + return part; + } + + /** + * Convert inline data or a Gemini Files API URI to a native data part. + * @param block + * @returns The formatted or accumulated result. + */ + private formatData(block: DataBlock): GeminiPart { + return block.source.type === 'base64' + ? { inlineData: { mimeType: block.source.media_type, data: block.source.data } } + : { fileData: { mimeType: block.source.media_type, fileUri: block.source.url } }; + } +} diff --git a/packages/agentscope/src/formatter/index.ts b/packages/agentscope/src/formatter/index.ts index b25de07..d9ed01c 100644 --- a/packages/agentscope/src/formatter/index.ts +++ b/packages/agentscope/src/formatter/index.ts @@ -3,3 +3,4 @@ export { DashScopeChatFormatter } from './dashscope-chat-formatter'; export { DeepSeekChatFormatter } from './deepseek-chat-formatter'; export { OllamaChatFormatter } from './ollama-chat-formatter'; export { OpenAIChatFormatter } from './openai-chat-formatter'; +export { GeminiChatFormatter } from './gemini-chat-formatter'; diff --git a/packages/agentscope/src/message/block.ts b/packages/agentscope/src/message/block.ts index 902d7e2..3d5c355 100644 --- a/packages/agentscope/src/message/block.ts +++ b/packages/agentscope/src/message/block.ts @@ -2,6 +2,8 @@ import { _generateId, _generateTimestamp } from '../_utils/common'; import type { PermissionRule } from '../permission'; export interface TextBlock { + /** Opaque Gemini part signature, retained for conversation replay. */ + thought_signature?: string; type: 'text'; text: string; id: string; @@ -51,6 +53,8 @@ export interface HintBlock { export type ToolCallState = 'pending' | 'asking' | 'allowed' | 'submitted' | 'finished'; export interface ToolCallBlock { + /** Opaque Gemini part signature, retained for conversation replay. */ + thought_signature?: string; type: 'tool_call'; name: string; id: string; @@ -91,6 +95,8 @@ export interface URLSource { } export interface DataBlock { + /** Opaque Gemini part signature, retained for conversation replay. */ + thought_signature?: string; type: 'data'; source: Base64Source | URLSource; id: string; @@ -113,6 +119,9 @@ export function TextBlock( Partial> ): TextBlock { return { + ...(input.thought_signature !== undefined + ? { thought_signature: input.thought_signature } + : {}), type: 'text', text: input.text, id: input.id ?? _generateId(), @@ -176,6 +185,9 @@ export function DataBlock( Partial> ): DataBlock { return { + ...(input.thought_signature !== undefined + ? { thought_signature: input.thought_signature } + : {}), type: 'data', id: input.id ?? _generateId(), source: input.source, @@ -218,6 +230,9 @@ export function ToolCallBlock( Partial> ): ToolCallBlock { return { + ...(input.thought_signature !== undefined + ? { thought_signature: input.thought_signature } + : {}), type: 'tool_call', id: input.id, name: input.name, diff --git a/packages/agentscope/src/message/schema.ts b/packages/agentscope/src/message/schema.ts index 32a4d90..4acac24 100644 --- a/packages/agentscope/src/message/schema.ts +++ b/packages/agentscope/src/message/schema.ts @@ -27,6 +27,7 @@ const jsonRecordSchema = z.record(z.string(), z.json()); export const TextBlockSchema = z .object({ type: z.literal('text'), + thought_signature: z.string().optional(), text: z.string(), ...entityFields, }) @@ -66,6 +67,7 @@ export const DataSourceSchema = z.union([Base64SourceSchema, URLSourceSchema]); export const DataBlockSchema = z .object({ type: z.literal('data'), + thought_signature: z.string().optional(), source: DataSourceSchema, name: z.string().nullable().optional(), ...entityFields, @@ -91,6 +93,7 @@ export const HintBlockSchema = z export const ToolCallBlockSchema = z .object({ type: z.literal('tool_call'), + thought_signature: z.string().optional(), id: z.string(), name: z.string(), input: z.string(), diff --git a/packages/agentscope/src/model/gemini-model.test.ts b/packages/agentscope/src/model/gemini-model.test.ts new file mode 100644 index 0000000..4d65be8 --- /dev/null +++ b/packages/agentscope/src/model/gemini-model.test.ts @@ -0,0 +1,498 @@ +import { z } from 'zod'; + +import { GeminiChatModel } from './gemini-model'; +import { ChatResponse } from './response'; +import { Agent } from '../agent/agent'; +import { GeminiChatFormatter } from '../formatter/gemini-chat-formatter'; +import { createMsg, parseMsg, TextBlock, ToolResultBlock } from '../message'; +import { Toolkit } from '../tool'; +import { createToolResponse } from '../tool/response'; +import { ToolSchema } from '../type'; + +const tool: ToolSchema = { + type: 'function', + function: { + name: 'weather', + description: 'Look up weather', + parameters: { + type: 'object', + properties: { city: { type: 'string' } }, + required: ['city'], + additionalProperties: false, + }, + }, +}; +const messages = [ + createMsg({ name: 'user', role: 'user', content: [TextBlock({ text: '天气?' })] }), +]; +const originalFetch = global.fetch; +const fetchMock = jest.fn(); + +/** + * Read all deltas and the generator's final accumulated response. + * @param generator + * @returns The formatted or accumulated result. + */ +async function collect(generator: AsyncGenerator) { + const deltas: ChatResponse[] = []; + while (true) { + const next = await generator.next(); + if (next.done) return { deltas, final: next.value }; + deltas.push(next.value); + } +} + +/** + * Create a byte-fragmented SSE response, including CRLF and an unterminated final frame. + * @param chunks + * @returns The formatted or accumulated result. + */ +function streamResponse(chunks: unknown[]) { + const bytes = new TextEncoder().encode( + chunks + .map((c, i) => `data: ${JSON.stringify(c)}${i === chunks.length - 1 ? '' : '\r\n\r\n'}`) + .join('') + ); + return new Response( + new ReadableStream({ + start(controller) { + for (let i = 0; i < bytes.length; i += 7) controller.enqueue(bytes.slice(i, i + 7)); + controller.close(); + }, + }) + ); +} + +/** + * Native response fixture. + * @param parts + * @param extra + * @returns The formatted or accumulated result. + */ +function response(parts: unknown[], extra: Record = {}) { + return { candidates: [{ content: { role: 'model', parts } }], ...extra }; +} + +describe('GeminiChatModel', () => { + beforeEach(() => { + fetchMock.mockReset(); + global.fetch = fetchMock; + }); + afterAll(() => { + global.fetch = originalFetch; + }); + + test('uses native endpoint, headers, system instruction, generation config and tool schemas', async () => { + fetchMock.mockResolvedValue(Response.json(response([{ text: 'OK' }]))); + const model = new GeminiChatModel({ + apiKey: 'test-key', + modelName: 'models/gemini-3.8-flash', + stream: false, + baseURL: 'https://gemini.example/v1beta/', + presetGenParams: { temperature: 0.5, maxOutputTokens: 100 }, + presetHeaders: { 'X-Client': 'test' }, + }); + const result = (await model.call({ + messages: [ + createMsg({ + name: 'system', + role: 'system', + content: [TextBlock({ text: 'Be concise' })], + }), + ...messages, + ], + tools: [tool], + toolChoice: 'weather', + generationConfig: { temperature: 0.2 }, + })) as ChatResponse; + expect(result.content).toEqual([expect.objectContaining({ type: 'text', text: 'OK' })]); + const [url, init] = fetchMock.mock.calls[0]; + expect(url).toBe('https://gemini.example/v1beta/models/gemini-3.8-flash:generateContent'); + expect(url).not.toContain('test-key'); + expect(init.headers).toMatchObject({ 'x-goog-api-key': 'test-key', 'X-Client': 'test' }); + expect(JSON.parse(init.body)).toEqual({ + contents: [{ role: 'user', parts: [{ text: '天气?' }] }], + systemInstruction: { parts: [{ text: 'Be concise' }] }, + tools: [ + { + functionDeclarations: [ + { + name: 'weather', + description: 'Look up weather', + parametersJsonSchema: tool.function.parameters, + }, + ], + }, + ], + toolConfig: { + functionCallingConfig: { mode: 'ANY', allowedFunctionNames: ['weather'] }, + }, + generationConfig: { temperature: 0.2, maxOutputTokens: 100 }, + }); + }); + + test.each([ + ['auto', 'AUTO'], + ['none', 'NONE'], + ['required', 'ANY'], + ])('maps tool choice %s', (choice, mode) => { + const model = new GeminiChatModel({ apiKey: 'key', modelName: 'gemini-test' }); + expect(model._formatToolChoice(choice)).toEqual({ functionCallingConfig: { mode } }); + expect(model._formatToolSchemas()).toEqual([]); + }); + + test('rejects missing tools and unknown tool names before making a request', async () => { + const model = new GeminiChatModel({ apiKey: 'key', modelName: 'gemini-test' }); + await expect(model.call({ messages, toolChoice: 'required' })).rejects.toThrow( + 'requires tools' + ); + await expect( + model.call({ messages, tools: [tool], toolChoice: 'missing' }) + ).rejects.toThrow('Unknown'); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + test('returns parallel function calls, preserves signatures through serialization, and groups responses in one user turn', async () => { + fetchMock.mockResolvedValue( + Response.json( + response( + [ + { + functionCall: { name: 'weather', args: { city: '北京' }, id: 'call-a' }, + thoughtSignature: 'opaque-signature', + }, + { + functionCall: { + name: 'weather', + args: { city: 'Paris' }, + id: 'call-b', + }, + }, + ], + { + usageMetadata: { + promptTokenCount: 20, + candidatesTokenCount: 5, + thoughtsTokenCount: 7, + }, + } + ) + ) + ); + const model = new GeminiChatModel({ + apiKey: 'key', + modelName: 'gemini-test', + stream: false, + }); + const res = (await model.call({ messages, tools: [tool] })) as ChatResponse; + expect(res.content.map(b => b.id)).toEqual(['call-a', 'call-b']); + expect(res.usage).toMatchObject({ inputTokens: 20, outputTokens: 12 }); + const assistant = parseMsg( + JSON.parse( + JSON.stringify( + createMsg({ name: 'agent', role: 'assistant', content: res.content }) + ) + ) + ); + const history = await new GeminiChatFormatter().format({ + msgs: [ + assistant, + createMsg({ + name: 'tools', + role: 'assistant', + content: [ + ToolResultBlock({ + id: 'call-a', + name: 'weather', + output: 'Sunny', + state: 'success', + }), + ], + }), + createMsg({ + name: 'tools', + role: 'assistant', + content: [ + ToolResultBlock({ + id: 'call-b', + name: 'weather', + output: 'Rainy', + state: 'success', + }), + ], + }), + ], + }); + expect(history).toEqual([ + { + role: 'model', + parts: [ + { + functionCall: { name: 'weather', args: { city: '北京' }, id: 'call-a' }, + thoughtSignature: 'opaque-signature', + }, + { functionCall: { name: 'weather', args: { city: 'Paris' }, id: 'call-b' } }, + ], + }, + { + role: 'user', + parts: [ + { + functionResponse: { + name: 'weather', + id: 'call-a', + response: { output: 'Sunny' }, + }, + }, + { + functionResponse: { + name: 'weather', + id: 'call-b', + response: { output: 'Rainy' }, + }, + }, + ], + }, + ]); + }); + + test('streams deltas, final complete function arguments and terminal usage across byte boundaries', async () => { + fetchMock.mockResolvedValue( + streamResponse([ + response([{ text: '分析', thought: true }]), + response([{ text: '天气', thought: true }, { text: '结果：' }]), + response([{ text: '晴天' }]), + response([ + { + functionCall: { name: 'weather', args: { city: '北京' } }, + thoughtSignature: 'sig', + }, + ]), + { + usageMetadata: { + promptTokenCount: 8, + candidatesTokenCount: 4, + thoughtsTokenCount: 3, + }, + }, + { candidates: [{ finishReason: 'STOP' }] }, + ]) + ); + const model = new GeminiChatModel({ apiKey: 'key', modelName: 'gemini-test' }); + const { deltas, final } = await collect( + (await model.call({ messages, tools: [tool] })) as AsyncGenerator< + ChatResponse, + ChatResponse + > + ); + expect(fetchMock.mock.calls[0][0]).toContain(':streamGenerateContent?alt=sse'); + expect(final.content).toEqual([ + expect.objectContaining({ type: 'thinking', thinking: '分析天气' }), + expect.objectContaining({ type: 'text', text: '结果：晴天' }), + expect.objectContaining({ + type: 'tool_call', + name: 'weather', + input: '{"city":"北京"}', + thought_signature: 'sig', + }), + ]); + expect(deltas.every(d => d.id === final.id)).toBe(true); + expect(deltas[2].content[0]).toMatchObject({ type: 'text', text: '晴天' }); + expect(final.usage).toMatchObject({ inputTokens: 8, outputTokens: 7 }); + expect(deltas[0].content[0]).toMatchObject({ thinking: '分析' }); // no mutation after yield + }); + + test('attaches a late signature to the accumulated text part and retains signed part boundaries', async () => { + fetchMock.mockResolvedValue( + streamResponse([ + response([{ text: 'First ' }]), + response([{ text: 'part', thoughtSignature: 'sig-a' }]), + response([{ text: 'Second' }]), + response([{ thoughtSignature: 'sig-b' }]), + ]) + ); + const model = new GeminiChatModel({ apiKey: 'key', modelName: 'gemini-test' }); + const { final } = await collect( + (await model.call({ messages })) as AsyncGenerator + ); + const msg = parseMsg( + JSON.parse( + JSON.stringify( + createMsg({ name: 'agent', role: 'assistant', content: final.content }) + ) + ) + ); + expect(await new GeminiChatFormatter().format({ msgs: [msg] })).toEqual([ + { + role: 'model', + parts: [ + { text: 'First part', thoughtSignature: 'sig-a' }, + { text: 'Second', thoughtSignature: 'sig-b' }, + ], + }, + ]); + }); + + test('callStructured uses named function calling in streaming mode', async () => { + fetchMock.mockResolvedValue( + streamResponse([ + response([ + { + functionCall: { + name: 'GenerateStructuredResponse', + args: { city: 'Paris' }, + }, + }, + ]), + ]) + ); + const model = new GeminiChatModel({ apiKey: 'key', modelName: 'gemini-test' }); + const res = await model.callStructured({ + messages, + schema: z.object({ city: z.string() }), + }); + expect(res.content).toEqual({ city: 'Paris' }); + expect( + JSON.parse(fetchMock.mock.calls[0][1].body).toolConfig.functionCallingConfig + ).toEqual({ mode: 'ANY', allowedFunctionNames: ['GenerateStructuredResponse'] }); + }); + + test('Agent executes a Gemini tool call and replays its signature in the next model request', async () => { + fetchMock + .mockResolvedValueOnce( + streamResponse([ + response([ + { + functionCall: { + name: 'weather', + args: { city: 'Paris' }, + id: 'provider-call', + }, + thoughtSignature: 'round-trip-sig', + }, + ]), + ]) + ) + .mockResolvedValueOnce(streamResponse([response([{ text: 'Sunny in Paris.' }])])); + const call = jest + .fn() + .mockResolvedValue( + createToolResponse({ content: [TextBlock({ text: 'Sunny' })], state: 'success' }) + ); + const toolkit = new Toolkit({ + builtInSkillTool: false, + tools: [ + { + name: 'weather', + description: 'Look up weather', + inputSchema: z.object({ city: z.string() }), + call, + requireUserConfirm: false, + }, + ], + }); + const agent = new Agent({ + name: 'agent', + sysPrompt: 'Help the user', + toolkit, + model: new GeminiChatModel({ apiKey: 'key', modelName: 'gemini-test' }), + }); + for await (const _event of agent.replyStream({ msgs: messages })) { + /* consume the real agent/tool lifecycle */ + } + expect(call).toHaveBeenCalledWith({ city: 'Paris' }); + expect(fetchMock).toHaveBeenCalledTimes(2); + const next = JSON.parse(fetchMock.mock.calls[1][1].body); + expect(next.contents).toEqual([ + { role: 'user', parts: [{ text: '天气?' }] }, + { + role: 'model', + parts: [ + { + functionCall: { + name: 'weather', + args: { city: 'Paris' }, + id: 'provider-call', + }, + thoughtSignature: 'round-trip-sig', + }, + ], + }, + { + role: 'user', + parts: [ + { + functionResponse: { + name: 'weather', + id: 'provider-call', + response: { output: 'Sunny' }, + }, + }, + ], + }, + ]); + expect(agent.context.at(-1)?.content.at(-1)).toMatchObject({ + type: 'text', + text: 'Sunny in Paris.', + }); + }); + + test('cancels the response body if the consumer stops streaming', async () => { + const cancel = jest.fn(); + fetchMock.mockResolvedValue( + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue( + new TextEncoder().encode( + `data: ${JSON.stringify(response([{ text: 'hello' }]))}\n\n` + ) + ); + }, + cancel, + }) + ) + ); + const model = new GeminiChatModel({ apiKey: 'key', modelName: 'gemini-test' }); + const generator = (await model.call({ messages })) as AsyncGenerator< + ChatResponse, + ChatResponse + >; + await generator.next(); + await generator.return({ type: 'chat', id: '', createdAt: '', content: [] }); + expect(cancel).toHaveBeenCalledTimes(1); + }); + + test('surfaces HTTP errors, blocked prompts, empty and malformed stream responses', async () => { + const model = new GeminiChatModel({ + apiKey: 'key', + modelName: 'gemini-test', + stream: false, + }); + fetchMock.mockResolvedValueOnce(new Response('rate limited', { status: 429 })); + await expect(model.call({ messages })).rejects.toThrow('429'); + fetchMock.mockResolvedValueOnce( + Response.json({ promptFeedback: { blockReason: 'SAFETY' } }) + ); + await expect(model.call({ messages })).rejects.toThrow('SAFETY'); + fetchMock.mockResolvedValueOnce( + Response.json({ candidates: [{ finishReason: 'SAFETY' }] }) + ); + await expect(model.call({ messages })).rejects.toThrow('no content'); + const streaming = new GeminiChatModel({ apiKey: 'key', modelName: 'gemini-test' }); + fetchMock.mockResolvedValueOnce(new Response('data: {broken}\n\n')); + const generator = (await streaming.call({ messages })) as AsyncGenerator< + ChatResponse, + ChatResponse + >; + await expect(generator.next()).rejects.toThrow(); + fetchMock.mockResolvedValueOnce( + streamResponse([{ error: { message: 'invalid request' } }]) + ); + const errorStream = (await streaming.call({ messages })) as AsyncGenerator< + ChatResponse, + ChatResponse + >; + await expect(errorStream.next()).rejects.toThrow('invalid request'); + }); +}); diff --git a/packages/agentscope/src/model/gemini-model.ts b/packages/agentscope/src/model/gemini-model.ts new file mode 100644 index 0000000..bf33395 --- /dev/null +++ b/packages/agentscope/src/model/gemini-model.ts @@ -0,0 +1,310 @@ +import { ChatModelBase, ChatModelOptions, ChatModelRequestOptions } from './base'; +import { ChatResponse } from './response'; +import { GeminiChatFormatter, GeminiPart } from '../formatter/gemini-chat-formatter'; +import { DataBlock, TextBlock, ThinkingBlock, ToolCallBlock } from '../message'; +import { ToolChoice, ToolSchema } from '../type'; + +/** Options for the Gemini Developer API (not Vertex AI). */ +export interface GeminiChatModelOptions extends ChatModelOptions { + apiKey: string; + /** REST API root, including the version, e.g. https://generativelanguage.googleapis.com/v1beta. */ + baseURL?: string; + /** Native generationConfig defaults; per-call generationConfig takes precedence. */ + presetGenParams?: Record; + presetHeaders?: Record; +} + +interface GeminiResponse { + responseId?: string; + candidates?: { content?: { parts?: GeminiPart[] }; finishReason?: string }[]; + usageMetadata?: { + promptTokenCount?: number; + candidatesTokenCount?: number; + thoughtsTokenCount?: number; + }; + promptFeedback?: { blockReason?: string }; + error?: { message?: string }; +} + +type GeminiBlock = TextBlock | ThinkingBlock | ToolCallBlock | DataBlock; + +/** Gemini native REST adapter with text, thinking, data and function-call responses. */ +export class GeminiChatModel extends ChatModelBase { + private apiKey: string; + private baseURL: string; + private presetGenParams: Record; + private presetHeaders: Record; + + /** + * Initialize the model with a native Gemini formatter by default. + * @param options + */ + constructor(options: GeminiChatModelOptions) { + super({ ...options, formatter: options.formatter ?? new GeminiChatFormatter() }); + this.apiKey = options.apiKey; + this.baseURL = ( + options.baseURL ?? 'https://generativelanguage.googleapis.com/v1beta' + ).replace(/\/$/, ''); + this.presetGenParams = options.presetGenParams ?? {}; + this.presetHeaders = options.presetHeaders ?? {}; + } + + /** + * Send a generateContent or streamGenerateContent request. + * @param modelName + * @param options + * @returns The formatted or accumulated result. + */ + async _callAPI( + modelName: string, + options: ChatModelRequestOptions> + ): Promise> { + const system = options.messages.filter(m => m.role === 'system'); + const contents = options.messages.filter(m => m.role !== 'system'); + if (!contents.length) throw new Error('Gemini requires at least one non-system message'); + const names = (options.tools ?? []).map(t => t.function.name); + if (options.toolChoice && !['auto', 'none'].includes(options.toolChoice)) { + if (!names.length) throw new Error('Gemini tool choice requires tools'); + if (options.toolChoice !== 'required' && !names.includes(options.toolChoice)) { + throw new Error(`Unknown Gemini tool choice: ${options.toolChoice}`); + } + } + const body = { + contents, + ...(system.length + ? { systemInstruction: { parts: system.flatMap(m => m.parts as GeminiPart[]) } } + : {}), + ...(names.length + ? { + tools: this._formatToolSchemas(options.tools), + toolConfig: this._formatToolChoice(options.toolChoice), + } + : {}), + generationConfig: { + ...this.presetGenParams, + ...((options.generationConfig as Record) ?? {}), + }, + ...(options.safetySettings ? { safetySettings: options.safetySettings } : {}), + }; + const method = this.stream ? 'streamGenerateContent?alt=sse' : 'generateContent'; + const model = encodeURIComponent(modelName.replace(/^models\//, '')); + const startTime = Date.now(); + const response = await fetch(`${this.baseURL}/models/${model}:${method}`, { + method: 'POST', + headers: { + ...this.presetHeaders, + 'Content-Type': 'application/json', + 'x-goog-api-key': this.apiKey, + }, + body: JSON.stringify(body), + signal: options.signal as AbortSignal | undefined, + }); + if (!response.ok) + throw new Error( + `Gemini API request failed with status ${response.status}: ${await response.text()}` + ); + if (this.stream) return this.parseStream(response, startTime); + const raw: GeminiResponse = await response.json(); + const result = this.emptyResponse(); + this.appendResponse(result, raw, startTime); + if (!result.content.length) + throw new Error( + `Gemini returned no content (${raw.promptFeedback?.blockReason ?? raw.candidates?.[0]?.finishReason ?? 'empty response'})` + ); + return result; + } + + /** + * Map unified tool choice modes to Gemini functionCallingConfig. + * @param toolChoice + * @returns The formatted or accumulated result. + */ + _formatToolChoice(toolChoice: ToolChoice = 'auto') { + const mode = toolChoice === 'auto' ? 'AUTO' : toolChoice === 'none' ? 'NONE' : 'ANY'; + return { + functionCallingConfig: { + mode, + ...(!['auto', 'none', 'required'].includes(toolChoice) + ? { allowedFunctionNames: [toolChoice] } + : {}), + }, + }; + } + + /** + * Use the REST API's JSON Schema field without dropping schema constraints. + * @param tools + * @returns The formatted or accumulated result. + */ + _formatToolSchemas(tools: ToolSchema[] = []): Record[] { + if (!tools.length) return []; + return [ + { + functionDeclarations: tools.map(t => ({ + name: t.function.name, + description: t.function.description, + parametersJsonSchema: t.function.parameters, + })), + }, + ]; + } + + /** Create a response whose identity remains stable across streaming deltas. + * @returns The formatted or accumulated result. + */ + private emptyResponse(): ChatResponse { + return { + type: 'chat', + id: crypto.randomUUID(), + createdAt: new Date().toISOString(), + content: [], + }; + } + + /** + * Append incremental native parts and retain usage-only terminal chunks. + * @param result + * @param raw + * @param startTime + * @param streaming + * @returns The formatted or accumulated result. + */ + private appendResponse( + result: ChatResponse, + raw: GeminiResponse, + startTime: number, + streaming = false + ): GeminiBlock[] { + if (raw.error) throw new Error(`Gemini API error: ${raw.error.message ?? 'unknown error'}`); + if (raw.promptFeedback?.blockReason) + throw new Error(`Gemini prompt blocked: ${raw.promptFeedback.blockReason}`); + const deltas: GeminiBlock[] = []; + for (const [index, part] of (raw.candidates?.[0]?.content?.parts ?? []).entries()) { + const common = { + id: crypto.randomUUID(), + created_at: result.createdAt, + ...(part.thoughtSignature ? { thought_signature: part.thoughtSignature } : {}), + }; + let block: GeminiBlock; + if (part.functionCall) { + block = { + ...common, + type: 'tool_call', + id: part.functionCall.id ?? common.id, + name: part.functionCall.name, + input: JSON.stringify(part.functionCall.args ?? {}), + state: 'pending', + }; + } else if (part.inlineData) { + block = { + ...common, + type: 'data', + source: { + type: 'base64', + data: part.inlineData.data, + media_type: part.inlineData.mimeType, + }, + }; + } else if (part.text !== undefined) { + block = part.thought + ? { ...common, type: 'thinking', thinking: part.text } + : { ...common, type: 'text', text: part.text }; + const last = result.content.at(-1); + // Merge deltas of an unfinished text part; a signature closes that part. + if ( + streaming && + index === 0 && + last?.type === block.type && + !('thought_signature' in last) + ) { + block.id = last.id; + if (part.thoughtSignature) + Object.assign(last, { thought_signature: part.thoughtSignature }); + if (last.type === 'text' && block.type === 'text') last.text += block.text; + if (last.type === 'thinking' && block.type === 'thinking') + last.thinking += block.thinking; + deltas.push(block); + continue; + } + } else if (part.thoughtSignature) { + // Providers can finish a text part with a signature-only chunk. + const last = result.content.at(-1); + if (last) { + Object.assign(last, { thought_signature: part.thoughtSignature }); + continue; + } + block = { ...common, type: 'text', text: '' }; + } else continue; + result.content.push({ ...block }); + deltas.push(block); + } + if (raw.usageMetadata) { + result.usage = { + type: 'chat_usage', + inputTokens: raw.usageMetadata.promptTokenCount ?? 0, + outputTokens: + (raw.usageMetadata.candidatesTokenCount ?? 0) + + (raw.usageMetadata.thoughtsTokenCount ?? 0), + time: (Date.now() - startTime) / 1000, + }; + } + return deltas; + } + + /** + * Parse SSE frames across arbitrary network boundaries and cancel on early return. + * @param response + * @param startTime + * @returns The formatted or accumulated result. + */ + private async *parseStream( + response: Response, + startTime: number + ): AsyncGenerator { + const reader = response.body?.getReader(); + if (!reader) throw new Error('Gemini streaming response has no body'); + const decoder = new TextDecoder(); + const result = this.emptyResponse(); + let buffer = ''; + let ended = false; + const parseFrame = (frame: string) => { + const data = frame + .split(/\r?\n/) + .filter(line => line.startsWith('data:')) + .map(line => line.slice(5).trimStart()) + .join('\n'); + if (!data || data === '[DONE]') return null; + const raw: GeminiResponse = JSON.parse(data); + const content = this.appendResponse(result, raw, startTime, true); + return { ...result, content, ...(result.usage ? { usage: { ...result.usage } } : {}) }; + }; + try { + while (true) { + const { value, done } = await reader.read(); + buffer += done ? decoder.decode() : decoder.decode(value, { stream: true }); + let boundary: RegExpExecArray | null; + while ((boundary = /\r?\n\r?\n/.exec(buffer))) { + const chunk = parseFrame(buffer.slice(0, boundary.index)); + buffer = buffer.slice(boundary.index + boundary[0].length); + if (chunk) yield chunk; + } + if (done) { + ended = true; + break; + } + } + if (buffer.trim()) { + const chunk = parseFrame(buffer); + if (chunk) yield chunk; + } + if (!result.content.length) throw new Error('Gemini returned no content in stream'); + return result; + } finally { + try { + if (!ended) await reader.cancel(); + } finally { + reader.releaseLock(); + } + } + } +} diff --git a/packages/agentscope/src/model/index.ts b/packages/agentscope/src/model/index.ts index b49cb7c..6e9a245 100644 --- a/packages/agentscope/src/model/index.ts +++ b/packages/agentscope/src/model/index.ts @@ -5,3 +5,5 @@ export { DashScopeChatModel } from './dashscope-model'; export { DeepSeekChatModel } from './deepseek-model'; export { OllamaChatModel } from './ollama-model'; export { OpenAIChatModel } from './openai-model'; +export { GeminiChatModel } from './gemini-model'; +export type { GeminiChatModelOptions } from './gemini-model';