From a1970ec6c3925630c4cb27901337a633e8213591 Mon Sep 17 00:00:00 2001 From: Slava Imeshev Date: Wed, 22 Jul 2026 07:07:27 -0700 Subject: [PATCH 1/4] #1813 - Fix createObject with Google GenAI thinking fails: answer JSON never selected from multi-part response Select non-thought answer generations for structured output instead of using the first/thought generation or concatenating alternative JSON candidates. Preserve provider metadata, including thoughtSignatures, on no-tool assistant responses by using metadata-capable assistant messages. Add deterministic regression coverage for multi-generation Google GenAI thinking responses, tool-call continuation, metadata preservation, and negative structured-output cases. Add live Google GenAI IT coverage and options converter tests for includeThoughts behavior. Signed-off-by: Slava Imeshev --- .../springai/SpringAiLlmMessageSender.kt | 147 +++- .../spi/support/springai/messageConverters.kt | 10 +- .../support/springai/MessageConversionTest.kt | 89 +++ .../springai/SpringAiLlmMessageSenderTest.kt | 656 ++++++++++++++++++ .../GoogleGenAiChatIntegrationIT.kt | 155 ++++- .../GoogleGenAiOptionsConverterTest.kt | 27 + 6 files changed, 1039 insertions(+), 45 deletions(-) diff --git a/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSender.kt b/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSender.kt index a3e0f89dc..6645a138c 100644 --- a/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSender.kt +++ b/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSender.kt @@ -26,6 +26,7 @@ import com.embabel.common.ai.autoconfig.NativeSupport import com.embabel.common.ai.model.LlmMetadata import com.embabel.common.util.loggerFor import com.embabel.agent.spi.support.nativeoutput.shouldUseNativeStructuredOutput +import org.springframework.ai.chat.messages.AssistantMessage import org.springframework.ai.chat.model.ChatModel import org.springframework.ai.chat.model.ChatResponse import org.springframework.ai.chat.prompt.ChatOptions @@ -96,12 +97,15 @@ internal class SpringAiLlmMessageSender( logger.debug("Prompt: {}\nResponse: {}", prompt, response) - // Convert response to Embabel message - // Note: Some providers (e.g., Bedrock) may return multiple generations where - // the first is empty and the second contains tool calls. We need to find the - // generation with tool calls, or fall back to the first one if none have them. - // See: https://github.com/embabel/embabel-agent/issues/1350 - val assistantMessage = findGenerationWithToolCalls(response) ?: response.result!!.output + // Convert response to Embabel message. + + // Providers may return multiple generations in one ChatResponse: + // - Bedrock: empty first generation, tool calls on a later one (#1350) + // - Google GenAI includeThoughts: thought parts first (isThought=true), answer later. + + // Using only ChatResponse.result (first generation) discards later answer text + // and breaks structured output / createObject after thought-signature support. + val assistantMessage = resolveAssistantMessage(response) val embabelMessage = assistantMessage.toEmbabelMessage() // Extract usage information @@ -115,60 +119,129 @@ internal class SpringAiLlmMessageSender( } /** - * Find the best generation to use from the response. + * Resolve a Spring AI response into the assistant message Embabel should store and inspect. * - * Some providers (e.g., Bedrock) may return multiple generations where - * the first is empty and a subsequent one contains tool calls. + * Spring AI exposes provider response parts as generations. For providers that split a + * single answer across generations, this method preserves the pieces Embabel needs while + * avoiding unsafe concatenation of alternative structured answers. * * Strategy: - * 1. Collect all tool calls from all generations - * 2. Collect all text content from all generations - * 3. If there are tool calls, create a merged AssistantMessage with all tool calls and combined text - * 4. If no tool calls, return null to fall back to first generation - * - * This ensures we don't lose valuable content (text or tool calls) from any generation. * - * @return A merged AssistantMessage with all tool calls and text, or null if no tool calls found + * 1. Collect tool calls and metadata from every generation. + * 2. Prefer non-thought text (Google GenAI sets metadata `isThought=true` on thought parts). + * Fall back to the first non-blank generation text when `isThought` is absent. + * 3. If tool calls exist, return a merged message with all tool calls and selected text. + * 4. If only text exists, return selected answer text with merged metadata. */ - private fun findGenerationWithToolCalls(response: ChatResponse): org.springframework.ai.chat.messages.AssistantMessage? { + private fun resolveAssistantMessage( + response: ChatResponse, + ): AssistantMessage { val allOutputs = response.results.map { it.output } + require(allOutputs.isNotEmpty()) { "ChatResponse contained no generations" } - // Collect all tool calls from all generations - val allToolCalls = allOutputs - .flatMap { it.toolCalls ?: emptyList() } - - if (allToolCalls.isEmpty()) { - return null // No tool calls found, let caller use first generation - } - - // Collect all metadata from all generations + val allToolCalls = allOutputs.flatMap { it.toolCalls ?: emptyList() } val allMetaData: Map = allOutputs .mapNotNull { it.metadata } .fold(emptyMap()) { acc, metadata -> acc + metadata } - // Collect all non-empty text from all generations - val allText = allOutputs - .mapNotNull { it.text?.takeIf { text -> text.isNotBlank() } } - .joinToString("\n") - - // Log if we're merging content from multiple generations + val answerText = if (allToolCalls.isNotEmpty()) { + selectToolCallText(allOutputs) + } else { + selectAnswerText(allOutputs) + } val generationsWithToolCalls = allOutputs.count { !it.toolCalls.isNullOrEmpty() } val generationsWithText = allOutputs.count { !it.text.isNullOrBlank() } if (generationsWithToolCalls > 1 || generationsWithText > 1) { logger.debug( - "Merging content from multiple generations: {} with tool calls, {} with text", + "Resolving multi-generation ChatResponse: {} with tool calls, {} with text, selected answer length={}", generationsWithToolCalls, - generationsWithText + generationsWithText, + answerText.length, ) } - return org.springframework.ai.chat.messages.AssistantMessage.builder() - .content(allText) - .toolCalls(allToolCalls) + if (allToolCalls.isNotEmpty()) { + return AssistantMessage.builder() + .content(answerText) + .toolCalls(allToolCalls) + .properties(allMetaData) + .build() + } + + return AssistantMessage.builder() + .content(answerText) .properties(allMetaData) .build() } + /** + * Select text that should be treated as the model answer for structured conversion. + * + * Google GenAI with includeThoughts emits one generation per part; thought parts are + * marked with metadata isThought=true and must not be used alone as the JSON payload. + * When multiple non-thought texts are present, the first one is selected because those + * generations may be alternative candidates rather than chunks of one JSON document. + */ + private fun selectAnswerText( + allOutputs: List, + ): String { + val nonThoughtTexts = allOutputs + .filterNot { isThoughtGeneration(it) } + .mapNotNull { it.text?.takeIf { text -> text.isNotBlank() } } + if (nonThoughtTexts.isNotEmpty()) { + return nonThoughtTexts.first() + } + // No non-thought text (or provider does not mark thoughts): use first non-blank content + return allOutputs + .mapNotNull { it.text?.takeIf { text -> text.isNotBlank() } } + .firstOrNull() + ?: "" + } + + /** + * Select text for responses that include tool calls. + * + * Tool-call responses need to keep tool calls from all generations. Text handling is more + * conservative: if no generation is marked as thought, all non-blank text is joined to + * preserve Bedrock-style split responses. If thought markers are present, thought text is + * removed so structured answer content and tool continuation metadata stay coherent. + */ + private fun selectToolCallText( + allOutputs: List, + ): String { + val textOutputs = allOutputs.mapNotNull { it.text?.takeIf { text -> text.isNotBlank() } } + if (allOutputs.none { isThoughtGeneration(it) }) { + return textOutputs.joinToString("\n") + } + return allOutputs + .filterNot { isThoughtGeneration(it) } + .mapNotNull { it.text?.takeIf { text -> text.isNotBlank() } } + .joinToString("\n") + } + + /** + * Return true when a generation is provider-marked as model thinking rather than final + * assistant answer text. + * + * Spring AI's Google GenAI adapter uses Boolean `true`; trimmed string values are accepted + * so metadata copied through less strongly typed paths is still filtered correctly. + */ + private fun isThoughtGeneration( + message: AssistantMessage, + ): Boolean = when (val isThought = message.metadata?.get(IS_THOUGHT_METADATA_KEY)) { + true -> true + is String -> isThought.trim().equals("true", ignoreCase = true) + else -> false + } + + companion object { + /** + * Metadata key set by Spring AI Google GenAI on thought parts + * (`GoogleGenAiChatModel.responseCandidateToGeneration`). + */ + const val IS_THOUGHT_METADATA_KEY: String = "isThought" + } + /** * Build ChatOptions with tool definitions. * Tools are passed to the LLM so it knows what's available, diff --git a/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/support/springai/messageConverters.kt b/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/support/springai/messageConverters.kt index c815223b3..ce7e275ed 100644 --- a/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/support/springai/messageConverters.kt +++ b/embabel-agent-api/src/main/kotlin/com/embabel/agent/spi/support/springai/messageConverters.kt @@ -147,11 +147,13 @@ fun SpringAiAssistantMessage.toEmbabelMessage(): Message { val toolCalls = this.toolCalls val content = this.text ?: "" val metadata = this.metadata ?: emptyMap() + val hasProviderMetadata = metadata.keys.any { it != "messageType" } return if (toolCalls.isNullOrEmpty()) { - // AssistantMessage requires non-empty content (TextPart validation). - // For empty content, use AssistantMessageWithToolCalls which handles empty content gracefully. - if (content.isEmpty()) { - AssistantMessageWithToolCalls(content = "", toolCalls = emptyList(), metadata = metadata) + + // AssistantMessage requires non-empty content and does not carry provider metadata. Use + // AssistantMessageWithToolCalls with an empty tool-call list when metadata must survive. + if (content.isEmpty() || hasProviderMetadata) { + AssistantMessageWithToolCalls(content = content, toolCalls = emptyList(), metadata = metadata) } else { AssistantMessage(content = content) } diff --git a/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/MessageConversionTest.kt b/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/MessageConversionTest.kt index 223fd930d..03b88a8b9 100644 --- a/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/MessageConversionTest.kt +++ b/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/MessageConversionTest.kt @@ -349,6 +349,95 @@ class MessageConversionTest { assertThat(signatures[1] as ByteArray).containsExactly(30, 40) } + @Test + fun `preserves thoughtSignatures metadata when tool calls are present with non-empty content`() { + // Prepare + val thoughtSignatures = listOf(byteArrayOf(1, 2, 3)) + val toolCalls = listOf( + SpringAiAssistantMessage.ToolCall("call-1", "function", "lookup", "{}"), + ) + val springMessage = SpringAiAssistantMessage.builder() + .content("calling tool") + .toolCalls(toolCalls) + .properties(mapOf("thoughtSignatures" to thoughtSignatures, "isThought" to false)) + .build() + + // Execute + val embabelMessage = springMessage.toEmbabelMessage() + + // Verify + assertThat(embabelMessage).isInstanceOf(AssistantMessageWithToolCalls::class.java) + val messageWithCalls = embabelMessage as AssistantMessageWithToolCalls + assertThat(messageWithCalls.toolCalls).hasSize(1) + + val signatures = messageWithCalls.metadata["thoughtSignatures"] as? List<*> + assertThat(signatures).isNotNull + assertThat(signatures!![0] as ByteArray).containsExactly(1, 2, 3) + assertThat(messageWithCalls.metadata["isThought"]).isEqualTo(false) + } + + @Test + fun `preserves custom metadata for non-empty assistant content without tool calls`() { + // Prepare + val thoughtSignatures = listOf(byteArrayOf(7, 8)) + val springMessage = SpringAiAssistantMessage.builder() + .content("""{"name":"July"}""") + .properties(mapOf("thoughtSignatures" to thoughtSignatures, "isThought" to false)) + .build() + + // Execute + val embabelMessage = springMessage.toEmbabelMessage() + + // Verify + assertThat(embabelMessage).isInstanceOf(AssistantMessageWithToolCalls::class.java) + assertThat(embabelMessage.content).isEqualTo("""{"name":"July"}""") + + val messageWithCalls = embabelMessage as AssistantMessageWithToolCalls + assertThat(messageWithCalls.toolCalls).isEmpty() + + val signatures = messageWithCalls.metadata["thoughtSignatures"] as? List<*> + assertThat(signatures).isNotNull + assertThat(signatures!![0] as ByteArray).containsExactly(7, 8) + assertThat(messageWithCalls.metadata["isThought"]).isEqualTo(false) + } + + @Test + fun `default messageType metadata alone keeps non-empty content as plain assistant message`() { + // Prepare + val springMessage = SpringAiAssistantMessage.builder() + .content("plain answer") + .properties(mapOf("messageType" to "ASSISTANT")) + .build() + + // Execute + val embabelMessage = springMessage.toEmbabelMessage() + + // Verify + assertThat(embabelMessage).isInstanceOf(AssistantMessage::class.java) + assertThat(embabelMessage).isNotInstanceOf(AssistantMessageWithToolCalls::class.java) + assertThat(embabelMessage.content).isEqualTo("plain answer") + } + + @Test + fun `preserves empty thoughtSignatures metadata for non-empty assistant content without tool calls`() { + // Prepare + val springMessage = SpringAiAssistantMessage.builder() + .content("""{"name":"July"}""") + .properties(mapOf("thoughtSignatures" to emptyList(), "isThought" to false)) + .build() + + // Execute + val embabelMessage = springMessage.toEmbabelMessage() + + // Verify + assertThat(embabelMessage).isInstanceOf(AssistantMessageWithToolCalls::class.java) + + val messageWithCalls = embabelMessage as AssistantMessageWithToolCalls + assertThat(messageWithCalls.toolCalls).isEmpty() + assertThat(messageWithCalls.metadata["thoughtSignatures"] as? List<*>).isEmpty() + assertThat(messageWithCalls.metadata["isThought"]).isEqualTo(false) + } + @Test fun `converts Spring AI AssistantMessage with tool calls`() { val toolCalls = listOf( diff --git a/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSenderTest.kt b/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSenderTest.kt index 8c0a005cc..17994faa3 100644 --- a/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSenderTest.kt +++ b/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSenderTest.kt @@ -27,6 +27,7 @@ import io.mockk.every import io.mockk.mockk import io.mockk.slot import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatThrownBy import org.junit.jupiter.api.Nested import org.junit.jupiter.api.Test import org.springframework.ai.chat.messages.AssistantMessage as SpringAiAssistantMessage @@ -620,6 +621,661 @@ class SpringAiLlmMessageSenderTest { } } + /** + * Google GenAI (and similar providers) emit one Spring AI generation per response part. + * With includeThoughts enabled, thought parts arrive first (metadata isThought=true) + * and the final answer is a later non-thought generation. Using only ChatResponse.result + * would discard the structured answer and break createObject/thinking structured output. + * + * See commit 04a45394 (thought signatures / includeThoughts) and Spring AI + * GoogleGenAiChatModel.responseCandidateToGeneration. + */ + @Nested + inner class GoogleGenAiThoughtAndAnswerGenerationsTests { + + @Test + fun `uses non-thought generation for structured answer when first generation is thought`() { + // Prepare + val thoughtGeneration = Generation( + SpringAiAssistantMessage.builder() + .content("Let me reason carefully about Florida climate...") + .properties(mapOf("isThought" to true, "candidateIndex" to 0)) + .build() + ) + val answerJson = """{"name":"July","temperature":91}""" + val answerGeneration = Generation( + SpringAiAssistantMessage.builder() + .content(answerJson) + .properties(mapOf("isThought" to false, "candidateIndex" to 0)) + .build() + ) + + val mockMetadata = mockk { + every { usage } returns mockk(relaxed = true) + } + val chatResponse = mockk { + every { result } returns thoughtGeneration + every { results } returns listOf(thoughtGeneration, answerGeneration) + every { metadata } returns mockMetadata + } + val chatModel = mockk { + every { call(any()) } returns chatResponse + } + val sender = SpringAiLlmMessageSender(chatModel, testChatOptions()) + + // Execute + val response = sender.call( + messages = listOf(UserMessage("Hottest month in Florida?")), + tools = emptyList(), + ) + + // Verify + assertThat(response.textContent).isEqualTo(answerJson) + assertThat(response.textContent).doesNotContain("Let me reason carefully") + } + + @Test + fun `merges thought and answer so thinking extraction can see both`() { + // Prepare + val thoughtGeneration = Generation( + SpringAiAssistantMessage.builder() + .content("Internal analysis of seasonal temperatures.") + .properties(mapOf("isThought" to true)) + .build() + ) + val answerJson = """{"name":"August","temperature":90}""" + val answerGeneration = Generation( + SpringAiAssistantMessage.builder() + .content(answerJson) + .properties(mapOf("isThought" to false)) + .build() + ) + + val mockMetadata = mockk { + every { usage } returns mockk(relaxed = true) + } + val chatResponse = mockk { + every { result } returns thoughtGeneration + every { results } returns listOf(thoughtGeneration, answerGeneration) + every { metadata } returns mockMetadata + } + val chatModel = mockk { + every { call(any()) } returns chatResponse + } + // Prefer answer-only content for structured parse; do not require thought prefix. + val sender = SpringAiLlmMessageSender(chatModel, testChatOptions()) + + // Execute + val response = sender.call( + messages = listOf(UserMessage("Hottest month?")), + tools = emptyList(), + ) + + // Verify + assertThat(response.textContent).contains(answerJson) + assertThat(response.textContent.trim()).isEqualTo(answerJson) + } + + @Test + fun `falls back to first non-empty generation when isThought metadata is absent`() { + // Prepare + val emptyGeneration = Generation(SpringAiAssistantMessage("")) + val answerGeneration = Generation( + SpringAiAssistantMessage.builder() + .content("""{"ok":true}""") + .build() + ) + + val mockMetadata = mockk { + every { usage } returns mockk(relaxed = true) + } + val chatResponse = mockk { + every { result } returns emptyGeneration + every { results } returns listOf(emptyGeneration, answerGeneration) + every { metadata } returns mockMetadata + } + val chatModel = mockk { + every { call(any()) } returns chatResponse + } + val sender = SpringAiLlmMessageSender(chatModel, testChatOptions()) + + // Execute + val response = sender.call( + messages = listOf(UserMessage("status")), + tools = emptyList(), + ) + + // Verify + assertThat(response.textContent).isEqualTo("""{"ok":true}""") + } + + @Test + fun `returns thought text when only thought generations are present`() { + // Prepare + val thoughtGeneration = Generation( + SpringAiAssistantMessage.builder() + .content("Still thinking, no final answer part yet.") + .properties(mapOf("isThought" to true)) + .build() + ) + val emptyAnswer = Generation( + SpringAiAssistantMessage.builder() + .content("") + .properties(mapOf("isThought" to false)) + .build() + ) + + val mockMetadata = mockk { + every { usage } returns mockk(relaxed = true) + } + val chatResponse = mockk { + every { result } returns thoughtGeneration + every { results } returns listOf(thoughtGeneration, emptyAnswer) + every { metadata } returns mockMetadata + } + val chatModel = mockk { + every { call(any()) } returns chatResponse + } + val sender = SpringAiLlmMessageSender(chatModel, testChatOptions()) + + // Execute + val response = sender.call( + messages = listOf(UserMessage("question")), + tools = emptyList(), + ) + + // Verify + // Prefer non-thought when present; if none, fall back to any non-blank text (thought). + assertThat(response.textContent).isEqualTo("Still thinking, no final answer part yet.") + } + + @Test + fun `uses first non-thought answer generation for structured safety`() { + // Prepare + val thoughtGeneration = Generation( + SpringAiAssistantMessage.builder() + .content("reasoning") + .properties(mapOf("isThought" to true)) + .build() + ) + val answerPart1 = Generation( + SpringAiAssistantMessage.builder() + .content("""{"name":"July"}""") + .properties(mapOf("isThought" to false)) + .build() + ) + val answerPart2 = Generation( + SpringAiAssistantMessage.builder() + .content("""{"temperature":91}""") + .properties(mapOf("isThought" to false)) + .build() + ) + + val mockMetadata = mockk { + every { usage } returns mockk(relaxed = true) + } + val chatResponse = mockk { + every { result } returns thoughtGeneration + every { results } returns listOf(thoughtGeneration, answerPart1, answerPart2) + every { metadata } returns mockMetadata + } + val chatModel = mockk { + every { call(any()) } returns chatResponse + } + val sender = SpringAiLlmMessageSender(chatModel, testChatOptions()) + + // Execute + val response = sender.call( + messages = listOf(UserMessage("question")), + tools = emptyList(), + ) + + // Verify + assertThat(response.textContent).isEqualTo("""{"name":"July"}""") + assertThat(response.textContent).doesNotContain("reasoning") + assertThat(response.textContent).doesNotContain("temperature") + } + + @Test + fun `prefers non-thought text when multi-gen includes tools and thought parts`() { + // Prepare + val thoughtGeneration = Generation( + SpringAiAssistantMessage.builder() + .content("I should call a tool") + .properties(mapOf("isThought" to true, "thoughtSignatures" to listOf(byteArrayOf(1)))) + .build() + ) + val toolGeneration = Generation( + SpringAiAssistantMessage.builder() + .content("""{"status":"calling"}""") + .toolCalls( + listOf( + SpringAiAssistantMessage.ToolCall( + "call-1", + "function", + "lookup_climate", + """{"region":"Florida"}""", + ) + ) + ) + .properties( + mapOf( + "isThought" to false, + "thoughtSignatures" to listOf(byteArrayOf(9, 9)), + ) + ) + .build() + ) + + val mockMetadata = mockk { + every { usage } returns mockk(relaxed = true) + } + val chatResponse = mockk { + every { result } returns thoughtGeneration + every { results } returns listOf(thoughtGeneration, toolGeneration) + every { metadata } returns mockMetadata + } + val chatModel = mockk { + every { call(any()) } returns chatResponse + } + val sender = SpringAiLlmMessageSender(chatModel, testChatOptions()) + + // Execute + val response = sender.call( + messages = listOf(UserMessage("climate?")), + tools = emptyList(), + ) + + // Verify + assertThat(response.message).isInstanceOf(AssistantMessageWithToolCalls::class.java) + val withTools = response.message as AssistantMessageWithToolCalls + assertThat(withTools.toolCalls).hasSize(1) + assertThat(withTools.toolCalls[0].name).isEqualTo("lookup_climate") + // Non-thought answer text is preferred over thought prose for structured conversion. + assertThat(response.textContent).isEqualTo("""{"status":"calling"}""") + // Metadata map merge is last-wins for thoughtSignatures (documented current contract). + val signatures = withTools.metadata["thoughtSignatures"] as? List<*> + assertThat(signatures).isNotNull + assertThat(signatures!![0] as ByteArray).containsExactly(9, 9) + } + } + + /** + * Regression tests for multi-generation ChatResponse handling after Google GenAI + * thought-signature support (#1691) and non-thought answer selection (#1813). + */ + @Nested + inner class MultiGenerationRegressionTests { + + @Test + fun `n-best or multi non-thought texts use first non-thought only for structured safety`() { + // Prepare + // Avoid joining two JSON payloads with newline (breaks Jackson). + // Prefer first non-blank non-thought generation when no isThought=true parts exist. + val candidateA = Generation( + SpringAiAssistantMessage.builder() + .content("""{"name":"July","temperature":91}""") + .build() + ) + val candidateB = Generation( + SpringAiAssistantMessage.builder() + .content("""{"name":"August","temperature":90}""") + .build() + ) + val sender = senderFor(listOf(candidateA, candidateB), resultGeneration = candidateA) + + // Execute + val response = sender.call( + messages = listOf(UserMessage("hottest month?")), + tools = emptyList(), + ) + + // Verify + assertThat(response.textContent) + .describedAs("multi non-thought generations must not be newline-joined for structured bind safety") + .isEqualTo("""{"name":"July","temperature":91}""") + assertThat(response.textContent).doesNotContain("August") + } + + @Test + fun `isThought string true is treated as thought not answer`() { + // Prepare + // Robust against adapters that store isThought as String "true". + val thoughtGeneration = Generation( + SpringAiAssistantMessage.builder() + .content("string-typed thought prose only") + .properties(mapOf("isThought" to "true")) + .build() + ) + val answerJson = """{"ok":true}""" + val answerGeneration = Generation( + SpringAiAssistantMessage.builder() + .content(answerJson) + .properties(mapOf("isThought" to "false")) + .build() + ) + val sender = senderFor(listOf(thoughtGeneration, answerGeneration), resultGeneration = thoughtGeneration) + + // Execute + val response = sender.call( + messages = listOf(UserMessage("q")), + tools = emptyList(), + ) + + // Verify + assertThat(response.textContent) + .describedAs("isThought=\"true\" (String) must not be treated as answer text") + .isEqualTo(answerJson) + assertThat(response.textContent).doesNotContain("string-typed thought") + } + + @Test + fun `isThought string true with surrounding whitespace is treated as thought not answer`() { + // Prepare + val thoughtGeneration = Generation( + SpringAiAssistantMessage.builder() + .content("whitespace-padded thought prose") + .properties(mapOf("isThought" to " TRUE ")) + .build() + ) + val answerJson = """{"ok":true}""" + val answerGeneration = Generation( + SpringAiAssistantMessage.builder() + .content(answerJson) + .properties(mapOf("isThought" to "false")) + .build() + ) + val sender = senderFor(listOf(thoughtGeneration, answerGeneration), resultGeneration = thoughtGeneration) + + // Execute + val response = sender.call( + messages = listOf(UserMessage("q")), + tools = emptyList(), + ) + + // Verify + assertThat(response.textContent).isEqualTo(answerJson) + assertThat(response.textContent).doesNotContain("whitespace-padded thought") + } + + @Test + fun `thought then empty non-thought then JSON selects JSON only`() { + // Prepare + val thoughtGeneration = Generation( + SpringAiAssistantMessage.builder() + .content("reasoning about climate") + .properties(mapOf("isThought" to true)) + .build() + ) + val emptyNonThought = Generation( + SpringAiAssistantMessage.builder() + .content(" ") + .properties(mapOf("isThought" to false)) + .build() + ) + val answerJson = """{"name":"July","temperature":91}""" + val answerGeneration = Generation( + SpringAiAssistantMessage.builder() + .content(answerJson) + .properties(mapOf("isThought" to false)) + .build() + ) + val sender = senderFor( + listOf(thoughtGeneration, emptyNonThought, answerGeneration), + resultGeneration = thoughtGeneration, + ) + + // Execute + val response = sender.call( + messages = listOf(UserMessage("q")), + tools = emptyList(), + ) + + // Verify + assertThat(response.textContent).isEqualTo(answerJson) + assertThat(response.textContent).doesNotContain("reasoning") + } + + @Test + fun `thought prose containing JSON-like snippets does not leak into structured answer text`() { + // Prepare + val thoughtGeneration = Generation( + SpringAiAssistantMessage.builder() + .content("""Maybe the answer shape is {"name":"Fake","temperature":0}.""") + .properties(mapOf("isThought" to true)) + .build() + ) + val answerJson = """{"name":"July","temperature":91}""" + val answerGeneration = Generation( + SpringAiAssistantMessage.builder() + .content(answerJson) + .properties(mapOf("isThought" to false)) + .build() + ) + val sender = senderFor(listOf(thoughtGeneration, answerGeneration), resultGeneration = thoughtGeneration) + + // Execute + val response = sender.call( + messages = listOf(UserMessage("q")), + tools = emptyList(), + ) + + // Verify + assertThat(response.textContent).isEqualTo(answerJson) + assertThat(response.textContent).doesNotContain("Fake") + } + + @Test + fun `empty ChatResponse results fail fast with clear error`() { + // Prepare + val mockMetadata = mockk(relaxed = true) + val chatResponse = mockk { + every { result } returns null + every { results } returns emptyList() + every { metadata } returns mockMetadata + } + val chatModel = mockk { + every { call(any()) } returns chatResponse + } + val sender = SpringAiLlmMessageSender(chatModel, testChatOptions()) + + // Execute + val thrown = assertThatThrownBy { + sender.call(messages = listOf(UserMessage("q")), tools = emptyList()) + } + + // Verify + thrown + .isInstanceOf(IllegalArgumentException::class.java) + .hasMessageContaining("no generations") + } + + @Test + fun `thoughtSignatures from thought gen survive when answer gen is selected without tools`() { + // Prepare + // Selecting non-thought answer text must not drop Google thoughtSignatures + // needed for later tool turns (metadata merge across multi-part response). + val signatures = listOf(byteArrayOf(1, 2, 3, 4)) + val thoughtGeneration = Generation( + SpringAiAssistantMessage.builder() + .content("internal thought") + .properties( + mapOf( + "isThought" to true, + "thoughtSignatures" to signatures, + ) + ) + .build() + ) + val answerJson = """{"name":"July"}""" + val answerGeneration = Generation( + SpringAiAssistantMessage.builder() + .content(answerJson) + .properties(mapOf("isThought" to false)) + .build() + ) + val sender = senderFor(listOf(thoughtGeneration, answerGeneration), resultGeneration = thoughtGeneration) + + // Execute + val response = sender.call(messages = listOf(UserMessage("q")),tools = emptyList(),) + + // Verify + assertThat(response.textContent).isEqualTo(answerJson) + + // After toEmbabelMessage, signatures must still be reachable for Google continuation. + val embabelMeta = when (val msg = response.message) { + is AssistantMessageWithToolCalls -> msg.metadata + else -> emptyMap() + } + val preserved = embabelMeta["thoughtSignatures"] as? List<*> + assertThat(preserved) + .describedAs("thoughtSignatures from thought parts must survive answer selection without tools") + .isNotNull + assertThat(preserved!![0] as ByteArray).containsExactly(1, 2, 3, 4) + } + + @Test + fun `duplicate thoughtSignatures use last generation metadata`() { + // Prepare + val thoughtGeneration = Generation( + SpringAiAssistantMessage.builder() + .content("internal thought") + .properties( + mapOf( + "isThought" to true, + "thoughtSignatures" to listOf(byteArrayOf(1, 1)), + ) + ) + .build() + ) + val answerGeneration = Generation( + SpringAiAssistantMessage.builder() + .content("""{"name":"July"}""") + .properties( + mapOf( + "isThought" to false, + "thoughtSignatures" to listOf(byteArrayOf(2, 2)), + ) + ) + .build() + ) + val sender = senderFor(listOf(thoughtGeneration, answerGeneration), resultGeneration = thoughtGeneration) + + // Execute + val response = sender.call( + messages = listOf(UserMessage("q")), + tools = emptyList(), + ) + + // Verify + val embabelMeta = when (val msg = response.message) { + is AssistantMessageWithToolCalls -> msg.metadata + else -> emptyMap() + } + val preserved = embabelMeta["thoughtSignatures"] as? List<*> + assertThat(preserved).isNotNull + assertThat(preserved!![0] as ByteArray).containsExactly(2, 2) + } + + @Test + fun `tools on thought-marked gen still surface tool calls with non-thought answer text preferred`() { + // Prepare + val thoughtWithTools = Generation( + SpringAiAssistantMessage.builder() + .content("I will call a tool") + .toolCalls( + listOf( + SpringAiAssistantMessage.ToolCall( + "call-thought", + "function", + "from_thought_part", + "{}", + ) + ) + ) + .properties(mapOf("isThought" to true)) + .build() + ) + val answerGeneration = Generation( + SpringAiAssistantMessage.builder() + .content("""{"phase":"answer"}""") + .properties(mapOf("isThought" to false)) + .build() + ) + val sender = senderFor(listOf(thoughtWithTools, answerGeneration), resultGeneration = thoughtWithTools) + + // Execute + val response = sender.call(messages = listOf(UserMessage("q")), tools = emptyList(),) + + // Verify + assertThat(response.message).isInstanceOf(AssistantMessageWithToolCalls::class.java) + + val withTools = response.message as AssistantMessageWithToolCalls + assertThat(withTools.toolCalls.map { it.name }).contains("from_thought_part") + assertThat(response.textContent) + .describedAs("answer text should prefer non-thought generation even when tools sit on thought gen") + .isEqualTo("""{"phase":"answer"}""") + } + + @Test + fun `tool calls survive when only available text is thought prose`() { + // Prepare + val thoughtWithTools = Generation( + SpringAiAssistantMessage.builder() + .content("I should call a tool and not expose this thought") + .toolCalls( + listOf( + SpringAiAssistantMessage.ToolCall( + "call-thought-only", + "function", + "from_thought_only_part", + "{}", + ) + ) + ) + .properties(mapOf("isThought" to true)) + .build() + ) + val sender = senderFor(listOf(thoughtWithTools), resultGeneration = thoughtWithTools) + + // Execute + val response = sender.call( + messages = listOf(UserMessage("q")), + tools = emptyList(), + ) + + // Verify + assertThat(response.message).isInstanceOf(AssistantMessageWithToolCalls::class.java) + + val withTools = response.message as AssistantMessageWithToolCalls + assertThat(withTools.toolCalls.map { it.name }).contains("from_thought_only_part") + assertThat(response.textContent).isBlank() + } + + /** + * Build a sender backed by a mocked [ChatModel] that returns exactly the supplied + * generations. This keeps each regression test focused on response resolution policy + * instead of repeating the same Spring AI mock setup. + */ + private fun senderFor( + generations: List, + resultGeneration: Generation, + ): SpringAiLlmMessageSender { + val mockMetadata = mockk { + every { usage } returns mockk(relaxed = true) + } + val chatResponse = mockk { + every { result } returns resultGeneration + every { results } returns generations + every { metadata } returns mockMetadata + } + val chatModel = mockk { + every { call(any()) } returns chatResponse + } + return SpringAiLlmMessageSender(chatModel, testChatOptions()) + } + } + private fun testChatOptions(): ChatOptions = mockk { every { model } returns "test-model" every { temperature } returns null diff --git a/embabel-agent-autoconfigure/models/embabel-agent-google-genai-autoconfigure/src/test/kotlin/com/embabel/agent/config/models/googlegenai/GoogleGenAiChatIntegrationIT.kt b/embabel-agent-autoconfigure/models/embabel-agent-google-genai-autoconfigure/src/test/kotlin/com/embabel/agent/config/models/googlegenai/GoogleGenAiChatIntegrationIT.kt index 2a5d4498b..3e3ee7043 100644 --- a/embabel-agent-autoconfigure/models/embabel-agent-google-genai-autoconfigure/src/test/kotlin/com/embabel/agent/config/models/googlegenai/GoogleGenAiChatIntegrationIT.kt +++ b/embabel-agent-autoconfigure/models/embabel-agent-google-genai-autoconfigure/src/test/kotlin/com/embabel/agent/config/models/googlegenai/GoogleGenAiChatIntegrationIT.kt @@ -15,14 +15,19 @@ */ package com.embabel.agent.config.models.googlegenai +import com.embabel.agent.api.annotation.LlmTool import com.embabel.agent.api.common.Ai +import com.embabel.agent.api.common.createObjectIfPossible import com.embabel.agent.api.models.GoogleGenAiModels import com.embabel.agent.autoconfigure.models.googlegenai.AgentGoogleGenAiAutoConfiguration import com.embabel.agent.spi.LlmService import com.embabel.common.ai.model.LlmOptions import com.embabel.common.ai.model.Thinking +import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertNotNull import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.DisplayName +import org.junit.jupiter.api.Nested import org.junit.jupiter.api.Test import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable import org.junit.jupiter.params.ParameterizedTest @@ -118,12 +123,14 @@ class GoogleGenAiChatIntegrationIT( @Test fun `creates typed object through google genai thinking mode`() { + // Prepare val modelId = GoogleGenAiModels.GEMINI_2_5_FLASH val runner = ai.withLlm( LlmOptions(modelId) .withThinking(Thinking.withExtraction()) ) + // Execute val response = runGoogleCall(modelId) { runner.thinking().createObject( """ @@ -137,17 +144,153 @@ class GoogleGenAiChatIntegrationIT( ) } + // Verify assertNotNull(response, "Expected Google thinking mode to return a ThinkingResponse") assertTrue(response.hasResult(), "Expected Google thinking mode to return a typed result") assertNotNull(response.thinkingBlocks, "Expected Google thinking mode to always expose thinking blocks collection") + val result = requireNotNull(response.result) { "Expected non-null MonthItem result from Google thinking mode" } - assertTrue( - result.name.lowercase() in setOf("july", "august"), - "Expected a plausible hottest-month result from the model, got: ${result.name}" - ) + assertTrue(result.name.lowercase() in setOf("july", "august"), "Expected a plausible hottest-month result from the model, got: ${result.name}") assertNotNull(result.temperature, "Expected Google thinking mode to populate the temperature field") + + val temperature = requireNotNull(result.temperature) + assertTrue(temperature in 70..110, "Expected a plausible Fahrenheit high for Florida summer, got: $temperature") + } + + /** + * Regression integration tests for Google GenAI multi-part thinking and structured output. + * + * These live tests lock regression-safety contracts for Google GenAI multi-part thinking + * and structured output after #1691 / non-thought generation selection. + * + * Requires GEMINI_API_KEY (class-level EnabledIfEnvironmentVariable). + */ + @Nested + @DisplayName("Google GenAI multi-gen / thinking regression ITs") + inner class GoogleGenAiRegressionITs { + + @Test + fun `createObject without thinking returns typed MonthItem`() { + // Prepare + val modelId = GoogleGenAiModels.GEMINI_2_5_FLASH + val runner = ai.withLlm(modelId) + + // Execute + val result = runGoogleCall(modelId) { + runner.createObject( + """ + Return a JSON object only (no markdown). + What is typically the hottest month in Florida and an approximate average high temperature in Fahrenheit? + Fields: name (month), temperature (integer Fahrenheit). + """.trimIndent(), + MonthItem::class.java, + ) + } + + // Verify + assertNotNull(result) + assertTrue(result.name.lowercase() in setOf("july", "august"), "Expected plausible month, got: ${result.name}",) + assertNotNull(result.temperature) + assertTrue(result.temperature!! in 70..110, "Expected plausible temp, got: ${result.temperature}") + } + + @Test + fun `thinking generateText returns non-blank answer text`() { + // Prepare + val modelId = GoogleGenAiModels.GEMINI_2_5_FLASH + val runner = ai.withLlm( + LlmOptions(modelId).withThinking(Thinking.withExtraction()) + ) + + // Execute + val response = runGoogleCall(modelId) { + runner.thinking().generateText( + "Think briefly, then reply with exactly the single word READY and nothing else." + ) + } + + // Verify + assertNotNull(response) + assertTrue(response.hasResult(), "thinking generateText must expose a result") + val text = requireNotNull(response.result).trim() + assertTrue(text.isNotBlank(), "thinking generateText must not yield blank after multi-gen selection") + assertTrue(text.contains("READY", ignoreCase = true), "Expected READY in answer text, got: $text",) + } + + @Test + fun `thinking createObjectIfPossible returns ThinkingResponse`() { + // Prepare + val modelId = GoogleGenAiModels.GEMINI_2_5_FLASH + val runner = ai.withLlm( + LlmOptions(modelId).withThinking(Thinking.withExtraction()) + ) + + // Execute + val response = runGoogleCall(modelId) { + runner.thinking().createObjectIfPossible( + """ + Think about the coldest month in Alaska and its approximate average low in Fahrenheit. + If possible return JSON with name and temperature; otherwise indicate impossibility. + """.trimIndent(), + MonthItem::class.java, + ) + } + + // Verify + assertNotNull(response, "ThinkingResponse must not be null") + assertNotNull(response.thinkingBlocks, "thinkingBlocks collection must be non-null") + response.result?.let { item -> + assertNotNull(item.name) + assertFalse(item.name.isBlank()) + } + } + + @Test + fun `thinking createObject with tool object completes with typed result`() { + // Prepare + val modelId = GoogleGenAiModels.GEMINI_2_5_FLASH + val runner = ai.withLlm( + LlmOptions(modelId).withThinking(Thinking.withExtraction()) + ).withToolObject(SimpleConversionTooling()) + + // Execute + val response = runGoogleCall(modelId) { + runner.thinking().createObject( + """ + Think carefully, use tools if useful, then return JSON only. + What is typically the hottest month in Florida and an approximate average high temperature in Fahrenheit? + Fields: name (month name), temperature (integer Fahrenheit). + """.trimIndent(), + MonthItem::class.java, + ) + } + + // Verify + assertNotNull(response) + assertTrue(response.hasResult(), "Expected typed result with tools + thinking") + val result = requireNotNull(response.result) + assertTrue(result.name.lowercase() in setOf("july", "august"), "Expected plausible month, got: ${result.name}",) + assertNotNull(result.temperature) + } + } + + /** Minimal tool surface for thinking + tools regression IT. */ + class SimpleConversionTooling { + /** + * Provide a deterministic arithmetic tool that Gemini can call while thinking is + * enabled. The test only needs a simple tool-call continuation path, not domain logic. + */ + @LlmTool(description = "Convert Celsius to Fahrenheit integer approximation") + fun celsiusToFahrenheit(celsius: Int): Int = (celsius * 9 / 5) + 32 } + /** + * Execute a live Google call while converting provider access errors into skipped tests. + * + * Some configured API keys can authenticate but lack access to newer preview model ids. + * Treating that case as an aborted test keeps the integration suite focused on code + * regressions when the environment is otherwise valid. + */ private fun runGoogleCall(modelId: String, block: () -> T): T = try { block() @@ -158,6 +301,10 @@ class GoogleGenAiChatIntegrationIT( throw ex } + /** + * Detect the provider error emitted when credentials are valid but the selected Gemini + * model is not available to the configured Google project. + */ private fun isModelAccessError(ex: Exception): Boolean { val message = generateSequence(ex) { it.cause }.mapNotNull { it.message }.joinToString(" | ") return message.contains("does not have access to model", ignoreCase = true) diff --git a/embabel-agent-autoconfigure/models/embabel-agent-google-genai-autoconfigure/src/test/kotlin/com/embabel/agent/config/models/googlegenai/GoogleGenAiOptionsConverterTest.kt b/embabel-agent-autoconfigure/models/embabel-agent-google-genai-autoconfigure/src/test/kotlin/com/embabel/agent/config/models/googlegenai/GoogleGenAiOptionsConverterTest.kt index 3339a4ff7..503af17ed 100644 --- a/embabel-agent-autoconfigure/models/embabel-agent-google-genai-autoconfigure/src/test/kotlin/com/embabel/agent/config/models/googlegenai/GoogleGenAiOptionsConverterTest.kt +++ b/embabel-agent-autoconfigure/models/embabel-agent-google-genai-autoconfigure/src/test/kotlin/com/embabel/agent/config/models/googlegenai/GoogleGenAiOptionsConverterTest.kt @@ -68,6 +68,33 @@ class GoogleGenAiOptionsConverterTest : OptionsConverterTestSupport Date: Wed, 22 Jul 2026 07:20:39 -0700 Subject: [PATCH 2/4] #1813 - Fix test name says it "merges thought and answer", but the assertions verify answer-only selection (and explicitly avoid requiring a thought prefix). Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Signed-off-by: Slava Imeshev --- .../agent/spi/support/springai/SpringAiLlmMessageSenderTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSenderTest.kt b/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSenderTest.kt index 17994faa3..14190d121 100644 --- a/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSenderTest.kt +++ b/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSenderTest.kt @@ -675,7 +675,7 @@ class SpringAiLlmMessageSenderTest { } @Test - fun `merges thought and answer so thinking extraction can see both`() { + fun `prefers answer-only content for structured parse`() { // Prepare val thoughtGeneration = Generation( SpringAiAssistantMessage.builder() From 2e1d456dcab1fbef1bccdfce2370b32261b544de Mon Sep 17 00:00:00 2001 From: Slava Imeshev Date: Wed, 22 Jul 2026 07:21:52 -0700 Subject: [PATCH 3/4] #1813 - Fix formatting style issue Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Signed-off-by: Slava Imeshev --- .../spi/support/springai/SpringAiLlmMessageSenderTest.kt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSenderTest.kt b/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSenderTest.kt index 14190d121..a87c3a9f2 100644 --- a/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSenderTest.kt +++ b/embabel-agent-api/src/test/kotlin/com/embabel/agent/spi/support/springai/SpringAiLlmMessageSenderTest.kt @@ -1116,9 +1116,7 @@ class SpringAiLlmMessageSenderTest { ) val sender = senderFor(listOf(thoughtGeneration, answerGeneration), resultGeneration = thoughtGeneration) - // Execute - val response = sender.call(messages = listOf(UserMessage("q")),tools = emptyList(),) - + val response = sender.call(messages = listOf(UserMessage("q")), tools = emptyList(),) // Verify assertThat(response.textContent).isEqualTo(answerJson) From 432990c1d7aeebcf7f3643ed5788d88c54beae41 Mon Sep 17 00:00:00 2001 From: Slava Imeshev Date: Wed, 22 Jul 2026 07:22:15 -0700 Subject: [PATCH 4/4] #1813 - Fix formatting style issue Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Signed-off-by: Slava Imeshev --- .../config/models/googlegenai/GoogleGenAiChatIntegrationIT.kt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/embabel-agent-autoconfigure/models/embabel-agent-google-genai-autoconfigure/src/test/kotlin/com/embabel/agent/config/models/googlegenai/GoogleGenAiChatIntegrationIT.kt b/embabel-agent-autoconfigure/models/embabel-agent-google-genai-autoconfigure/src/test/kotlin/com/embabel/agent/config/models/googlegenai/GoogleGenAiChatIntegrationIT.kt index 3e3ee7043..02c9ae6d8 100644 --- a/embabel-agent-autoconfigure/models/embabel-agent-google-genai-autoconfigure/src/test/kotlin/com/embabel/agent/config/models/googlegenai/GoogleGenAiChatIntegrationIT.kt +++ b/embabel-agent-autoconfigure/models/embabel-agent-google-genai-autoconfigure/src/test/kotlin/com/embabel/agent/config/models/googlegenai/GoogleGenAiChatIntegrationIT.kt @@ -188,9 +188,7 @@ class GoogleGenAiChatIntegrationIT( } // Verify - assertNotNull(result) - assertTrue(result.name.lowercase() in setOf("july", "august"), "Expected plausible month, got: ${result.name}",) - assertNotNull(result.temperature) + assertTrue(result.name.lowercase() in setOf("july", "august"), "Expected plausible month, got: ${result.name}") assertTrue(result.temperature!! in 70..110, "Expected plausible temp, got: ${result.temperature}") }