diff --git a/src/handlers/handlerUtils.ts b/src/handlers/handlerUtils.ts index 9df6c9866..fafcf06f0 100644 --- a/src/handlers/handlerUtils.ts +++ b/src/handlers/handlerUtils.ts @@ -851,6 +851,7 @@ export function constructConfigFromRequestHeaders( requestHeaders[`x-${POWERED_BY}-azure-entra-client-secret`], azureEntraTenantId: requestHeaders[`x-${POWERED_BY}-azure-entra-tenant-id`], azureModelName: requestHeaders[`x-${POWERED_BY}-azure-model-name`], + chatCompletionsApi: requestHeaders[`x-${POWERED_BY}-chat-completions-api`], openaiBeta: requestHeaders[`x-${POWERED_BY}-openai-beta`] || requestHeaders[`openai-beta`], diff --git a/src/providers/azure-openai/api.ts b/src/providers/azure-openai/api.ts index 7ef6f48da..333162977 100644 --- a/src/providers/azure-openai/api.ts +++ b/src/providers/azure-openai/api.ts @@ -6,6 +6,10 @@ import { getAzureWorkloadIdentityToken, } from './utils'; import { getRuntimeKey } from 'hono/adapter'; +import { + getAzureResponsesChatEndpoint, + shouldUseAzureResponsesForChat, +} from './chatCompletionsResponses'; const runtime = getRuntimeKey(); @@ -100,7 +104,12 @@ const AzureOpenAIAPIConfig: ProviderAPIConfig = { } return headersObj; }, - getEndpoint: ({ providerOptions, fn, gatewayRequestURL }) => { + getEndpoint: ({ + providerOptions, + fn, + gatewayRequestBodyJSON, + gatewayRequestURL, + }) => { const { apiVersion, urlToFetch, deploymentId } = providerOptions; let mappedFn = fn; @@ -140,6 +149,13 @@ const AzureOpenAIAPIConfig: ProviderAPIConfig = { searchParams.delete('api-version'); } + if ( + mappedFn === 'chatComplete' && + shouldUseAzureResponsesForChat(gatewayRequestBodyJSON, providerOptions) + ) { + return getAzureResponsesChatEndpoint(); + } + switch (mappedFn) { case 'complete': { return `${prefix}/completions?${searchParams.toString()}`; diff --git a/src/providers/azure-openai/chatComplete.ts b/src/providers/azure-openai/chatComplete.ts index 4a89911d1..b1a31ba36 100644 --- a/src/providers/azure-openai/chatComplete.ts +++ b/src/providers/azure-openai/chatComplete.ts @@ -5,6 +5,10 @@ import { ErrorResponse, ProviderConfig, } from '../types'; +import { + hasFunctionTools, + isAzureResponsesChatCompatibilityEnabled, +} from './chatCompletionsResponses'; import { getAzureModelValue } from './utils'; // TODOS: this configuration does not enforce the maximum token limit for the input parameter. If you want to enforce this, you might need to add a custom validation function or a max property to the ParameterConfig interface, and then use it in the input configuration. However, this might be complex because the token count is not a simple length check, but depends on the specific tokenization method used by the model. @@ -102,6 +106,12 @@ export const AzureOpenAIChatCompleteConfig: ProviderConfig = { }, reasoning_effort: { param: 'reasoning_effort', + transform: (params, providerOptions) => + params.reasoning_effort === 'none' && + isAzureResponsesChatCompatibilityEnabled(providerOptions) && + hasFunctionTools(params) + ? undefined + : params.reasoning_effort, }, stream_options: { param: 'stream_options', diff --git a/src/providers/azure-openai/chatCompletionsResponses.test.ts b/src/providers/azure-openai/chatCompletionsResponses.test.ts new file mode 100644 index 000000000..00a5bb843 --- /dev/null +++ b/src/providers/azure-openai/chatCompletionsResponses.test.ts @@ -0,0 +1,473 @@ +import { Options, Params } from '../../types/requestBody'; +import { ProviderConfig } from '../types'; +import { AzureOpenAIChatCompleteConfig } from './chatComplete'; +import { + AzureOpenAIResponsesChatCompleteConfig, + AzureOpenAIResponsesChatCompleteResponseTransform, + AzureOpenAIResponsesChatCompleteStreamChunkTransform, + getAzureResponsesChatEndpoint, + shouldUseAzureResponsesForChat, +} from './chatCompletionsResponses'; + +const functionTool = { + type: 'function', + function: { + name: 'get_weather', + description: 'Get the weather', + parameters: { + type: 'object', + properties: { city: { type: 'string' } }, + required: ['city'], + }, + strict: true, + }, +}; + +const providerOptions = { + provider: 'azure-openai', + deploymentId: 'gpt-5.6-luna', + apiVersion: '2025-04-01-preview', + chatCompletionsApi: 'responses', +} as any; + +function transformedValue( + config: ProviderConfig, + key: string, + params: Params, + options: Options = providerOptions +) { + const entry = config[key]; + const valueConfig = Array.isArray(entry) ? entry[0] : entry; + return valueConfig.transform + ? valueConfig.transform(params, options) + : (params as any)[key]; +} + +describe('Azure Chat Completions through Responses', () => { + it('bridges opted-in function-tool requests unless reasoning is explicitly disabled', () => { + expect( + shouldUseAzureResponsesForChat( + { tools: [functionTool], reasoning_effort: 'medium' }, + providerOptions + ) + ).toBe(true); + expect( + shouldUseAzureResponsesForChat({ tools: [functionTool] }, providerOptions) + ).toBe(true); + expect( + shouldUseAzureResponsesForChat( + { tools: [functionTool], reasoning_effort: 'none' }, + providerOptions + ) + ).toBe(false); + expect( + shouldUseAzureResponsesForChat( + { tools: [functionTool], reasoning_effort: 'medium' }, + { ...providerOptions, chatCompletionsApi: undefined } + ) + ).toBe(false); + }); + + it('rejects Chat features the bridge cannot preserve', () => { + expect(() => + shouldUseAzureResponsesForChat( + { + tools: [functionTool], + reasoning_effort: 'medium', + logprobs: false as any, + }, + providerOptions + ) + ).not.toThrow(); + expect(() => + shouldUseAzureResponsesForChat( + { + tools: [functionTool], + reasoning_effort: 'medium', + n: 2, + stop: 'END', + }, + providerOptions + ) + ).toThrow('does not support: stop, n'); + expect(() => + shouldUseAzureResponsesForChat( + { + tools: [functionTool, { type: 'web_search' }], + reasoning_effort: 'medium', + }, + providerOptions + ) + ).toThrow('only supports function tools'); + expect(() => + shouldUseAzureResponsesForChat( + { + tools: [functionTool], + reasoning_effort: 'medium', + best_of: 2, + }, + providerOptions + ) + ).toThrow('does not support: best_of'); + expect(() => + shouldUseAzureResponsesForChat( + { + tools: [functionTool], + reasoning_effort: 'medium', + logprobs: true as any, + top_logprobs: 21 as any, + }, + providerOptions + ) + ).toThrow('does not support: top_logprobs, logprobs'); + expect(() => + shouldUseAzureResponsesForChat( + { + functions: [functionTool.function], + reasoning_effort: 'medium', + } as any, + providerOptions + ) + ).toThrow('does not support: functions'); + }); + + it('maps messages, tool history, multimodal input, and direct parameters', () => { + const params = { + model: 'public-alias', + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: 'Read this image.' }, + { + type: 'image_url', + image_url: { url: 'data:image/png;base64,x' }, + }, + { + type: 'file', + file: { + file_data: 'data:application/pdf;base64,eA==', + file_name: 'input.pdf', + }, + }, + ], + }, + { + role: 'assistant', + tool_calls: [ + { + id: 'call_1', + type: 'function', + function: { + name: 'get_weather', + arguments: '{"city":"Paris"}', + }, + }, + ], + }, + { role: 'tool', tool_call_id: 'call_1', content: '18 C' }, + ], + tools: [functionTool], + tool_choice: { + type: 'function', + function: { name: 'get_weather' }, + }, + reasoning_effort: 'medium', + max_completion_tokens: 500, + response_format: { + type: 'json_schema', + json_schema: { + name: 'answer', + schema: { type: 'object' }, + strict: true, + }, + }, + stream: true, + modalities: ['text'], + } as Params; + + expect( + transformedValue(AzureOpenAIResponsesChatCompleteConfig, 'model', params) + ).toBe('gpt-5.6-luna'); + expect( + transformedValue( + AzureOpenAIResponsesChatCompleteConfig, + 'messages', + params + ) + ).toEqual([ + { + type: 'message', + role: 'user', + content: [ + { type: 'input_text', text: 'Read this image.' }, + { type: 'input_image', image_url: 'data:image/png;base64,x' }, + { + type: 'input_file', + file_data: 'data:application/pdf;base64,eA==', + filename: 'input.pdf', + }, + ], + }, + { + type: 'function_call', + call_id: 'call_1', + name: 'get_weather', + arguments: '{"city":"Paris"}', + }, + { + type: 'function_call_output', + call_id: 'call_1', + output: '18 C', + }, + ]); + expect( + transformedValue(AzureOpenAIResponsesChatCompleteConfig, 'tools', params) + ).toEqual([ + { + type: 'function', + name: 'get_weather', + description: 'Get the weather', + parameters: functionTool.function.parameters, + strict: true, + }, + ]); + expect( + transformedValue( + AzureOpenAIResponsesChatCompleteConfig, + 'tool_choice', + params + ) + ).toEqual({ type: 'function', name: 'get_weather' }); + expect( + transformedValue( + AzureOpenAIResponsesChatCompleteConfig, + 'response_format', + params + ) + ).toEqual({ + type: 'json_schema', + name: 'answer', + schema: { type: 'object' }, + strict: true, + }); + expect( + transformedValue( + AzureOpenAIResponsesChatCompleteConfig, + 'reasoning_effort', + params + ) + ).toBe('medium'); + expect(AzureOpenAIResponsesChatCompleteConfig.modalities).toBeUndefined(); + }); + + it('uses Azure v1 Responses independently of the Chat API version', () => { + expect(getAzureResponsesChatEndpoint()).toBe('/v1/responses'); + }); + + it('strips explicit none only for opted-in function-tool requests', () => { + const params = { + tools: [functionTool], + reasoning_effort: 'none', + } as Params; + expect( + transformedValue( + AzureOpenAIChatCompleteConfig, + 'reasoning_effort', + params + ) + ).toBeUndefined(); + expect( + transformedValue( + AzureOpenAIChatCompleteConfig, + 'reasoning_effort', + params, + { ...providerOptions, chatCompletionsApi: undefined } + ) + ).toBe('none'); + }); + + it('maps Responses tool calls and reasoning usage to Chat Completions', () => { + expect( + AzureOpenAIResponsesChatCompleteResponseTransform( + { + id: 'resp_123', + object: 'response', + created_at: 1234, + status: 'completed', + model: 'gpt-5.6-luna', + output: [ + { type: 'reasoning', id: 'rs_1' }, + { + type: 'function_call', + call_id: 'call_1', + name: 'get_weather', + arguments: '{"city":"Paris"}', + }, + ], + usage: { + input_tokens: 30, + output_tokens: 20, + output_tokens_details: { reasoning_tokens: 12 }, + total_tokens: 50, + }, + }, + 200 + ) + ).toMatchObject({ + id: 'resp_123', + object: 'chat.completion', + choices: [ + { + message: { + role: 'assistant', + content: null, + tool_calls: [ + { + id: 'call_1', + type: 'function', + function: { + name: 'get_weather', + arguments: '{"city":"Paris"}', + }, + }, + ], + }, + finish_reason: 'tool_calls', + }, + ], + usage: { + prompt_tokens: 30, + completion_tokens: 20, + total_tokens: 50, + completion_tokens_details: { reasoning_tokens: 12 }, + }, + }); + }); + + it('maps Chat-visible text, errors, and terminal states', () => { + const transformed = AzureOpenAIResponsesChatCompleteResponseTransform( + { + id: 'resp_text', + created_at: 1234, + status: 'completed', + model: 'gpt-5.6-luna', + output: [ + { + type: 'message', + content: [ + { + type: 'output_text', + text: 'Paris weather', + }, + ], + }, + ], + }, + 200 + ) as any; + + expect(transformed.choices[0]).toMatchObject({ + message: { + content: 'Paris weather', + }, + logprobs: null, + finish_reason: 'stop', + }); + + expect( + AzureOpenAIResponsesChatCompleteResponseTransform( + { + id: 'resp_filtered', + created_at: 1234, + status: 'incomplete', + incomplete_details: { reason: 'content_filter' }, + model: 'gpt-5.6-luna', + output: [], + }, + 200 + ) + ).toMatchObject({ choices: [{ finish_reason: 'content_filter' }] }); + + expect( + AzureOpenAIResponsesChatCompleteResponseTransform( + { + status: 'failed', + error: { message: 'upstream failed', code: 'server_error' }, + }, + 200 + ) + ).toMatchObject({ + error: { message: 'azure-openai error: upstream failed' }, + }); + }); + + it('maps Responses SSE tool calls, argument deltas, and usage', () => { + const state = {}; + const request = { + stream_options: { include_usage: true }, + } as any; + const chunks = [ + [ + 'response.created', + { response: { id: 'resp_1', created_at: 1, model: 'gpt-5.6-luna' } }, + ], + [ + 'response.output_item.added', + { + item: { + id: 'fc_1', + call_id: 'call_1', + type: 'function_call', + name: 'get_weather', + }, + }, + ], + [ + 'response.function_call_arguments.delta', + { item_id: 'fc_1', delta: '{"city":"Paris"}' }, + ], + [ + 'response.output_text.delta', + { + item_id: 'msg_1', + delta: 'Checking', + }, + ], + [ + 'response.completed', + { + response: { + id: 'resp_1', + created_at: 1, + status: 'completed', + model: 'gpt-5.6-luna', + usage: { + input_tokens: 30, + output_tokens: 20, + total_tokens: 50, + output_tokens_details: { reasoning_tokens: 12 }, + }, + }, + }, + ], + ] + .map(([event, data]) => + AzureOpenAIResponsesChatCompleteStreamChunkTransform( + `event: ${event}\ndata: ${JSON.stringify(data)}`, + 'fallback', + state, + true, + request + ) + ) + .join(''); + + expect(chunks).toContain('"id":"call_1"'); + expect(chunks).toContain('"arguments":"{\\"city\\":\\"Paris\\"}"'); + expect(chunks).toContain('"finish_reason":"tool_calls"'); + expect(chunks).toContain('"reasoning_tokens":12'); + expect(chunks).toContain('"content":"Checking"'); + expect(chunks).toContain('data: [DONE]'); + }); +}); diff --git a/src/providers/azure-openai/chatCompletionsResponses.ts b/src/providers/azure-openai/chatCompletionsResponses.ts new file mode 100644 index 000000000..fc07d1a07 --- /dev/null +++ b/src/providers/azure-openai/chatCompletionsResponses.ts @@ -0,0 +1,684 @@ +import { GatewayError } from '../../errors/GatewayError'; +import { AZURE_OPEN_AI } from '../../globals'; +import { Message, Options, Params, Tool } from '../../types/requestBody'; +import { OpenAIErrorResponseTransform } from '../openai/utils'; +import { + ChatCompletionResponse, + ErrorResponse, + ProviderConfig, +} from '../types'; + +type BridgeOptions = Options & { chatCompletionsApi?: string }; +type StreamState = { + id?: string; + model?: string; + created?: number; + toolIndexes?: Record; + hasToolCalls?: boolean; + done?: boolean; +}; + +const isFunctionTool = (tool: Tool) => + tool?.type === 'function' && Boolean(tool.function?.name); + +const allowedChatParameters = new Set([ + 'model', + 'messages', + 'max_tokens', + 'max_completion_tokens', + 'temperature', + 'top_p', + 'n', + 'stream', + 'user', + 'tools', + 'tool_choice', + 'response_format', + 'logprobs', + 'top_logprobs', + 'stream_options', + 'service_tier', + 'parallel_tool_calls', + 'store', + 'metadata', + 'modalities', + 'reasoning_effort', + 'prompt_cache_key', + 'safety_identifier', + 'verbosity', +]); + +export function hasFunctionTools(params: Params): boolean { + const request = params as Record; + return Boolean( + params.tools?.some(isFunctionTool) || request.functions?.length + ); +} + +function unsupportedChatParameters(params: Params): string[] { + const request = params as Record; + const unsupported = [ + 'functions', + 'function_call', + 'stop', + 'presence_penalty', + 'frequency_penalty', + 'logit_bias', + 'seed', + 'top_logprobs', + 'audio', + 'prediction', + 'web_search_options', + ].filter((key) => request[key] !== undefined); + + if (params.n !== undefined && params.n !== 1) unsupported.push('n'); + if (request.logprobs !== undefined && request.logprobs !== false) { + unsupported.push('logprobs'); + } + if (params.modalities?.some((modality) => modality !== 'text')) { + unsupported.push('modalities'); + } + + for (const key of Object.keys(request)) { + if ( + request[key] !== undefined && + !allowedChatParameters.has(key) && + !unsupported.includes(key) + ) { + unsupported.push(key); + } + } + + return unsupported; +} + +function assertCompatibleChatRequest(params: Params) { + if (params.tools?.some((tool) => !isFunctionTool(tool))) { + throw new GatewayError( + 'Azure Chat-to-Responses compatibility mode only supports function tools', + 400 + ); + } + if ( + typeof params.tool_choice === 'object' && + (params.tool_choice.type !== 'function' || + !params.tool_choice.function?.name) + ) { + throw new GatewayError( + 'Azure Chat-to-Responses compatibility mode only supports function tool_choice objects', + 400 + ); + } + const request = params as Record; + if ( + request.reasoning_effort !== undefined && + typeof request.reasoning_effort !== 'string' + ) { + throw new GatewayError( + 'Azure Chat-to-Responses compatibility mode requires reasoning_effort to be a string', + 400 + ); + } + const streamOptions = request.stream_options; + if ( + streamOptions && + Object.keys(streamOptions).some((key) => key !== 'include_usage') + ) { + throw new GatewayError( + 'Azure Chat-to-Responses compatibility mode only supports stream_options.include_usage', + 400 + ); + } + + for (const message of params.messages || []) { + const refusal = (message as Record).refusal; + if ( + message.name || + message.role === 'function' || + message.function_call || + message.reasoning_details || + message.content_blocks || + (message as Record).audio || + (refusal !== undefined && + (message.role !== 'assistant' || typeof refusal !== 'string')) || + (['tool', 'function'].includes(message.role) && + message.content === undefined) || + (message.role === 'tool' && !message.tool_call_id) + ) { + throw new GatewayError( + 'Azure Chat-to-Responses compatibility mode cannot preserve named messages, deprecated function messages, audio history, or provider-specific content', + 400 + ); + } + + if ( + Array.isArray(message.tool_calls) && + message.tool_calls.some( + (call: any) => + call?.type !== 'function' || + !call.id || + !call.function?.name || + typeof call.function?.arguments !== 'string' + ) + ) { + throw new GatewayError( + 'Azure Chat-to-Responses compatibility mode only supports valid function tool call history', + 400 + ); + } + + if ( + Array.isArray(message.content) && + message.content.some((part) => { + if (part.type === 'text') return typeof part.text === 'string'; + if (part.type === 'refusal') { + return ( + message.role === 'assistant' && + typeof (part as Record).refusal === 'string' + ); + } + if (part.type === 'image_url') { + return message.role === 'user' && Boolean(part.image_url?.url); + } + if (part.type === 'file') { + const file = part.file as Record | undefined; + return Boolean( + message.role === 'user' && file && (file.file_id || file.file_data) + ); + } + return false; + }) + ) { + throw new GatewayError( + 'Azure Chat-to-Responses compatibility mode only supports Chat text, refusal, image_url, and file message content', + 400 + ); + } + } + + const unsupported = unsupportedChatParameters(params); + if (unsupported.length) { + throw new GatewayError( + `Azure Chat-to-Responses compatibility mode does not support: ${unsupported.join(', ')}`, + 400 + ); + } +} + +export function isAzureResponsesChatCompatibilityEnabled( + providerOptions?: Options +): boolean { + return (providerOptions as BridgeOptions)?.chatCompletionsApi === 'responses'; +} + +/** Route only opted-in function-tool requests that Azure Chat cannot serve. */ +export function shouldUseAzureResponsesForChat( + params: Params, + providerOptions?: Options +): boolean { + const shouldUse = Boolean( + isAzureResponsesChatCompatibilityEnabled(providerOptions) && + hasFunctionTools(params) && + params.reasoning_effort !== 'none' + ); + if (shouldUse) assertCompatibleChatRequest(params); + return shouldUse; +} + +export function getAzureResponsesChatEndpoint(): string { + return '/v1/responses'; +} + +function mapMessageContent(content: Message['content'], role: Message['role']) { + if (typeof content === 'string') { + return [ + { + type: role === 'assistant' ? 'output_text' : 'input_text', + text: content, + }, + ]; + } + if (!Array.isArray(content)) return []; + return content.map((part) => { + if (part.type === 'text') { + return { + type: role === 'assistant' ? 'output_text' : 'input_text', + text: part.text || '', + }; + } + if (part.type === 'refusal') { + return { + type: 'refusal', + refusal: (part as Record).refusal, + }; + } + if (part.type === 'file') { + const file = part.file as Record; + return { + type: 'input_file', + ...(file.file_id && { file_id: file.file_id }), + ...(file.file_data && { file_data: file.file_data }), + ...((file.filename || file.file_name) && { + filename: file.filename || file.file_name, + }), + }; + } + return { + type: 'input_image', + image_url: part.image_url!.url, + ...(part.image_url!.detail && { detail: part.image_url!.detail }), + }; + }); +} + +function toolOutput( + content: Message['content'] +): string | Record[] { + return typeof content === 'string' + ? content + : (mapMessageContent(content, 'user') as any); +} + +/** Convert Chat history, including completed tool rounds, to Responses input. */ +export function chatMessagesToResponsesInput(messages: Message[] = []) { + const input: Record[] = []; + + for (const message of messages) { + if (message.role === 'tool') { + input.push({ + type: 'function_call_output', + call_id: message.tool_call_id, + output: toolOutput(message.content), + }); + continue; + } + const calls = + message.role === 'assistant' && Array.isArray(message.tool_calls) + ? message.tool_calls + : []; + if ( + (message.content !== undefined && + message.content !== null && + (message.content !== '' || calls.length === 0)) || + (message as Record).refusal + ) { + const content = mapMessageContent(message.content, message.role); + const refusal = (message as Record).refusal; + if (refusal && Array.isArray(content)) { + content.push({ type: 'refusal', refusal }); + } + input.push({ + type: 'message', + role: message.role, + content, + }); + } + for (const call of calls) { + input.push({ + type: 'function_call', + call_id: call.id, + name: call.function.name, + arguments: call.function.arguments, + }); + } + } + + return input; +} + +export function chatToolsToResponsesTools(tools: Tool[] = []) { + return tools.map(({ function: fn }) => ({ + type: 'function', + name: fn!.name, + ...(fn!.description !== undefined && { description: fn!.description }), + ...(fn!.parameters !== undefined && { parameters: fn!.parameters }), + ...(fn!.strict !== undefined && { strict: fn!.strict }), + })); +} + +function toolChoice(choice: Params['tool_choice']) { + if ( + typeof choice === 'object' && + choice?.type === 'function' && + choice.function?.name + ) { + return { type: 'function', name: choice.function.name }; + } + return choice; +} + +function textFormat(responseFormat: Params['response_format']) { + if (!responseFormat || responseFormat.type !== 'json_schema') { + return responseFormat; + } + const schema = responseFormat.json_schema || {}; + return { + type: 'json_schema', + name: schema.name, + ...(schema.description !== undefined && { + description: schema.description, + }), + schema: schema.schema, + ...(schema.strict !== undefined && { strict: schema.strict }), + }; +} + +export const AzureOpenAIResponsesChatCompleteConfig: ProviderConfig = { + model: { + param: 'model', + transform: (params: Params, options: Options) => + options.deploymentId || params.model, + }, + messages: { + param: 'input', + transform: (params: Params) => + chatMessagesToResponsesInput(params.messages), + }, + tools: { + param: 'tools', + transform: (params: Params) => chatToolsToResponsesTools(params.tools), + }, + tool_choice: { + param: 'tool_choice', + transform: (params: Params) => toolChoice(params.tool_choice), + }, + reasoning_effort: { param: 'reasoning.effort' }, + max_tokens: { param: 'max_output_tokens' }, + max_completion_tokens: { param: 'max_output_tokens' }, + response_format: { + param: 'text.format', + transform: (params: Params) => textFormat(params.response_format), + }, + verbosity: { param: 'text.verbosity' }, + stream: { param: 'stream' }, + temperature: { param: 'temperature' }, + top_p: { param: 'top_p' }, + parallel_tool_calls: { param: 'parallel_tool_calls' }, + store: { param: 'store' }, + metadata: { param: 'metadata' }, + user: { param: 'user' }, + prompt_cache_key: { param: 'prompt_cache_key' }, + safety_identifier: { param: 'safety_identifier' }, + service_tier: { param: 'service_tier' }, +}; + +function usage(responsesUsage: any) { + if (!responsesUsage) return undefined; + const input = responsesUsage.input_tokens || 0; + const output = responsesUsage.output_tokens || 0; + return { + prompt_tokens: input, + completion_tokens: output, + total_tokens: responsesUsage.total_tokens || input + output, + prompt_tokens_details: { + cached_tokens: responsesUsage.input_tokens_details?.cached_tokens || 0, + }, + completion_tokens_details: { + reasoning_tokens: + responsesUsage.output_tokens_details?.reasoning_tokens || 0, + }, + }; +} + +function finishReason(response: any, hasToolCalls: boolean) { + if (hasToolCalls) return 'tool_calls'; + if (response.status !== 'incomplete') return 'stop'; + return response.incomplete_details?.reason === 'content_filter' + ? 'content_filter' + : 'length'; +} + +function chatError(error: any, fallbackMessage: string): ErrorResponse { + return OpenAIErrorResponseTransform( + { + error: { + message: error?.message || fallbackMessage, + type: error?.type || 'server_error', + param: error?.param || null, + code: error?.code || null, + }, + provider: AZURE_OPEN_AI, + }, + AZURE_OPEN_AI + ); +} + +export const AzureOpenAIResponsesChatCompleteResponseTransform = ( + response: any, + responseStatus: number +): ChatCompletionResponse | ErrorResponse => { + if ( + responseStatus !== 200 || + response.error || + response.status === 'failed' + ) { + return chatError( + response.error, + `Responses request failed with status ${response.status || responseStatus}` + ); + } + + if (!['completed', 'incomplete'].includes(response.status)) { + return chatError( + undefined, + `Responses request ended with unsupported status ${response.status}` + ); + } + + const calls = (response.output || []).filter( + (item: any) => item.type === 'function_call' + ); + const parts = (response.output || []) + .filter((item: any) => ['message', 'output_message'].includes(item.type)) + .flatMap((item: any) => item.content || []); + const content = parts + .filter((part: any) => part.type === 'output_text') + .map((part: any) => part.text || '') + .join(''); + const refusal = parts + .filter((part: any) => part.type === 'refusal') + .map((part: any) => part.refusal || '') + .join(''); + + return { + id: response.id, + object: 'chat.completion', + created: response.created_at, + model: response.model, + choices: [ + { + index: 0, + message: { + role: 'assistant', + content: content || null, + ...(refusal && { refusal }), + ...(calls.length && { + tool_calls: calls.map((call: any) => ({ + id: call.call_id || call.id, + type: 'function', + function: { + name: call.name, + arguments: call.arguments || '', + }, + })), + }), + }, + finish_reason: finishReason(response, calls.length > 0), + logprobs: null, + }, + ], + usage: usage(response.usage), + ...(response.service_tier && { service_tier: response.service_tier }), + } as ChatCompletionResponse; +}; + +function parseEvent(chunk: string) { + const lines = chunk.trim().split('\n'); + return { + event: lines + .find((line) => line.startsWith('event:')) + ?.slice(6) + .trim(), + data: lines + .filter((line) => line.startsWith('data:')) + .map((line) => line.slice(5).trimStart()) + .join('\n'), + }; +} + +function updateMetadata(state: StreamState, response: any) { + if (!response) return; + state.id = response.id || state.id; + state.model = response.model || state.model; + state.created = response.created_at || state.created; +} + +function streamChunk( + state: StreamState, + choices: Record[], + streamUsage?: any +) { + return `data: ${JSON.stringify({ + id: state.id || `chatcmpl-${Date.now()}`, + object: 'chat.completion.chunk', + created: state.created || Math.floor(Date.now() / 1000), + model: state.model || '', + choices, + ...(streamUsage && { usage: streamUsage }), + })}\n\n`; +} + +function toolIndex(state: StreamState, itemId: string) { + state.toolIndexes ||= {}; + if (state.toolIndexes[itemId] === undefined) { + state.toolIndexes[itemId] = Object.keys(state.toolIndexes).length; + } + return state.toolIndexes[itemId]; +} + +function streamError(error: any, fallbackMessage: string) { + return `data: ${JSON.stringify({ + error: { + message: error?.message || fallbackMessage, + type: error?.type || 'server_error', + param: error?.param || null, + code: error?.code || null, + }, + })}\n\ndata: [DONE]\n\n`; +} + +export const AzureOpenAIResponsesChatCompleteStreamChunkTransform = ( + responseChunk: string, + _fallbackId: string, + state: StreamState, + _strictOpenAiCompliance: boolean, + request: Params +): string | undefined => { + const { event: eventName, data } = parseEvent(responseChunk); + if (!data || state.done) return undefined; + if (data === '[DONE]') { + state.done = true; + return 'data: [DONE]\n\n'; + } + const parsed = JSON.parse(data); + const event = eventName || parsed.type; + + if (event === 'response.created') { + updateMetadata(state, parsed.response); + return streamChunk(state, [ + { + index: 0, + delta: { role: 'assistant', content: '' }, + finish_reason: null, + }, + ]); + } + if ( + event === 'response.output_item.added' && + parsed.item?.type === 'function_call' + ) { + state.hasToolCalls = true; + return streamChunk(state, [ + { + index: 0, + delta: { + tool_calls: [ + { + index: toolIndex(state, parsed.item.id || parsed.item.call_id), + id: parsed.item.call_id || parsed.item.id, + type: 'function', + function: { + name: parsed.item.name, + arguments: parsed.item.arguments || '', + }, + }, + ], + }, + finish_reason: null, + }, + ]); + } + if (event === 'response.function_call_arguments.delta') { + state.hasToolCalls = true; + return streamChunk(state, [ + { + index: 0, + delta: { + tool_calls: [ + { + index: toolIndex(state, parsed.item_id || parsed.call_id), + function: { arguments: parsed.delta || '' }, + }, + ], + }, + finish_reason: null, + }, + ]); + } + if (event === 'response.output_text.delta') { + return streamChunk(state, [ + { + index: 0, + delta: { content: parsed.delta || '' }, + finish_reason: null, + }, + ]); + } + if (event === 'response.refusal.delta') { + return streamChunk(state, [ + { + index: 0, + delta: { refusal: parsed.delta || '' }, + finish_reason: null, + }, + ]); + } + if (event === 'response.completed' || event === 'response.incomplete') { + updateMetadata(state, parsed.response); + const finalChunk = streamChunk(state, [ + { + index: 0, + delta: {}, + finish_reason: finishReason( + parsed.response, + Boolean(state.hasToolCalls) + ), + }, + ]); + const usageChunk = (request as any).stream_options?.include_usage + ? streamChunk(state, [], usage(parsed.response.usage)) + : ''; + state.done = true; + return `${finalChunk}${usageChunk}data: [DONE]\n\n`; + } + if ( + event === 'response.failed' || + event === 'response.cancelled' || + event === 'error' + ) { + state.done = true; + const error = parsed.response?.error || parsed.error || parsed; + return streamError(error, `Responses stream failed with event ${event}`); + } + return undefined; +}; diff --git a/src/providers/azure-openai/index.ts b/src/providers/azure-openai/index.ts index 77f4e715b..b32aacf84 100644 --- a/src/providers/azure-openai/index.ts +++ b/src/providers/azure-openai/index.ts @@ -40,8 +40,14 @@ import { OpenAIResponseTransform, } from '../open-ai-base'; import { AZURE_OPEN_AI } from '../../globals'; +import { + AzureOpenAIResponsesChatCompleteConfig, + AzureOpenAIResponsesChatCompleteResponseTransform, + AzureOpenAIResponsesChatCompleteStreamChunkTransform, + shouldUseAzureResponsesForChat, +} from './chatCompletionsResponses'; -const AzureOpenAIConfig: ProviderConfigs = { +const AzureOpenAIBaseConfig: ProviderConfigs = { complete: AzureOpenAICompleteConfig, embed: AzureOpenAIEmbedConfig, api: AzureOpenAIAPIConfig, @@ -103,4 +109,23 @@ const AzureOpenAIConfig: ProviderConfigs = { }, }; +const AzureOpenAIConfig: ProviderConfigs = { + ...AzureOpenAIBaseConfig, + getConfig: ({ params, providerOptions }) => { + if (!shouldUseAzureResponsesForChat(params, providerOptions)) { + return AzureOpenAIBaseConfig; + } + return { + ...AzureOpenAIBaseConfig, + chatComplete: AzureOpenAIResponsesChatCompleteConfig, + responseTransforms: { + ...AzureOpenAIBaseConfig.responseTransforms, + chatComplete: AzureOpenAIResponsesChatCompleteResponseTransform, + 'stream-chatComplete': + AzureOpenAIResponsesChatCompleteStreamChunkTransform, + }, + }; + }, +}; + export default AzureOpenAIConfig;