diff --git a/lib/chains/data_extraction_chain.ex b/lib/chains/data_extraction_chain.ex index 2160d9d8..2f5de70d 100644 --- a/lib/chains/data_extraction_chain.ex +++ b/lib/chains/data_extraction_chain.ex @@ -108,15 +108,57 @@ defmodule LangChain.Chains.DataExtractionChain do ] |> FunctionParam.to_parameters_schema() + ## Provider Strategy vs. Tool Strategy + + This chain can ask the LLM to return JSON constrained to a schema natively + ("structured outputs"), without any tool/function calling. The chain checks + whether `llm`'s struct type natively supports this (see + `LangChain.ChatModels.ChatModel.supports_json_output?/1`) — i.e. whether its + bare struct defines both `:json_schema` and `:json_response` fields. If it + does, the chain runs against a patched copy of `llm` with `json_response: + true` and `json_schema` set to `schema_parameters` as given, and parses the + plain JSON response directly with `extract_result/1` instead of requiring a + tool call. Some models require extra parameters alongside the schema itself + (e.g. `LangChain.ChatModels.ChatOpenAIResponses` requires a separate + `:json_schema_name`); the chain fills these in with sensible defaults when + the struct defines them. + + When `opts[:strategy]` is **not given**, the chain defaults to + `:provider_strategy`, but fails gracefully: if `llm`'s struct type doesn't + support it, it falls back to `:tool_strategy` — asking the LLM to call an + `information_extraction` tool built from `schema_parameters`, as shown + above. + + When `opts[:strategy]` **is given** explicitly, it is used strictly — there is + no fallback. Passing `strategy: :provider_strategy` for an `llm` whose struct + type doesn't support it raises from `run_chain/4` (and `run/4` will return an + `{:error, %LangChainError{}}`): + + # Always uses tool calling, regardless of what `llm` supports + {:ok, result} = + DataExtractionChain.run(chat, schema_parameters, data_prompt, strategy: :tool_strategy) + + # Strictly requires provider_strategy support; raises from run_chain/4 otherwise + {:ok, chain} = + DataExtractionChain.run_chain(chat, schema_parameters, data_prompt, strategy: :provider_strategy) + + Note: the exact wire format for structured output still differs by provider + (compare `:json_schema` on `LangChain.ChatModels.ChatAnthropic` vs + `LangChain.ChatModels.ChatMistralAI`, which expects the entire response + format nested under `:json_schema`); the defaults applied here favor the + common bare-schema shape and may not be fully correct for every provider. + """ use Ecto.Schema require Logger alias LangChain.PromptTemplate alias LangChain.Message + alias LangChain.Message.ContentPart alias LangChain.Message.ToolCall alias LangChain.LangChainError alias LangChain.Chains.LLMChain - alias LangChain.ChatModels.ChatOpenAI + alias LangChain.ChatModels.ChatModel + alias LangChain.MessageProcessors.JsonProcessor @function_name "information_extraction" @extraction_template ~s"Extract and save the relevant entities mentioned in the following passage together with their properties. Use the value `null` when missing in the passage. @@ -124,6 +166,12 @@ defmodule LangChain.Chains.DataExtractionChain do Passage: <%= @input %>" + # Sensible defaults applied to extra fields some models require alongside + # `:json_schema`/`:json_response`, when the struct defines them and they're + # not already set. Only fields explicitly known to exist across the chat + # model modules are listed here. + @extra_field_defaults %{json_schema_name: @function_name} + @doc """ Coerces the extraction tool's `info` argument to a list of rows. @@ -159,15 +207,53 @@ Passage: ## Options + - `:strategy` - either `:provider_strategy` or `:tool_strategy`. When + omitted, defaults to `:provider_strategy` but falls back to + `:tool_strategy` if `llm` doesn't support it. When given explicitly, it is + used strictly, raising if `llm` doesn't support it. See the "Provider + Strategy vs. Tool Strategy" section in the module docs. - `:verbose` - when `true`, enables verbose logging on the internally run `LLMChain`. Defaults to `false`. - `:callbacks` - a list of callback handler maps to register on the internally run `LLMChain`. See `LangChain.Chains.ChainCallbacks` for the available events. Defaults to `[]`. """ - @spec run_chain(ChatOpenAI.t(), json_schema :: map(), prompt :: [any()], opts :: Keyword.t()) :: + @spec run_chain(ChatModel.t(), json_schema :: map(), prompt :: [any()], opts :: Keyword.t()) :: {:ok, LLMChain.t()} | {:error, LLMChain.t(), LangChainError.t()} def run_chain(llm, json_schema, prompt, opts \\ []) do + case Keyword.fetch(opts, :strategy) do + :error -> + # No explicit strategy: default to provider_strategy, but fail + # gracefully by falling back to tool_strategy when unsupported. + if ChatModel.supports_json_output?(llm.__struct__) do + run_chain_provider_strategy(llm, json_schema, prompt, opts) + else + Logger.warning( + "#{inspect(llm.__struct__)} does not support :provider_strategy (it does not define both :json_schema and :json_response fields). Falling back to :tool_strategy since no :strategy was explicitly specified." + ) + + run_chain_tool_strategy(llm, json_schema, prompt, opts) + end + + {:ok, :tool_strategy} -> + run_chain_tool_strategy(llm, json_schema, prompt, opts) + + {:ok, :provider_strategy} -> + # Explicit strategy: used strictly, no fallback. + if ChatModel.supports_json_output?(llm.__struct__) do + run_chain_provider_strategy(llm, json_schema, prompt, opts) + else + raise LangChainError, + "`llm`'s struct type does not support :provider_strategy" + end + + {:ok, other} -> + raise LangChainError, + "Invalid :strategy #{inspect(other)}. Expected :tool_strategy or :provider_strategy." + end + end + + defp run_chain_tool_strategy(llm, json_schema, prompt, opts) do verbose = Keyword.get(opts, :verbose, false) callbacks = Keyword.get(opts, :callbacks, []) @@ -187,12 +273,66 @@ Passage: |> LLMChain.run() end + defp run_chain_provider_strategy(llm, json_schema, prompt, opts) do + verbose = Keyword.get(opts, :verbose, false) + callbacks = Keyword.get(opts, :callbacks, []) + + messages = + [ + Message.new_system!( + "You are a helpful assistant that extracts structured data from text passages. Respond only with JSON matching the required schema. Use the value `null` when missing in the passage." + ), + PromptTemplate.new!(%{role: :user, text: @extraction_template}) + ] + |> PromptTemplate.to_messages!(%{input: prompt}) + + %{ + llm: patch_llm_for_provider_strategy(llm, json_schema), + verbose: verbose, + callbacks: callbacks + } + |> LLMChain.new!() + |> LLMChain.message_processors([JsonProcessor.new!()]) + |> LLMChain.add_messages(messages) + |> LLMChain.run() + end + + # Patches a copy of `llm` with the fields needed to request structured JSON + # output for `json_schema`, unwrapped and as given. The schema is always + # taken from the argument passed to this call, never from how `llm` happened + # to be constructed. + defp patch_llm_for_provider_strategy(llm, json_schema) do + llm + |> Map.put(:json_response, true) + |> Map.put(:json_schema, json_schema) + |> apply_extra_field_defaults() + end + + defp apply_extra_field_defaults(llm) do + Enum.reduce(@extra_field_defaults, llm, fn {field, default}, acc -> + if Map.has_key?(acc, field) and is_nil(Map.get(acc, field)) do + Map.put(acc, field, default) + else + acc + end + end) + end + @doc """ Return the extracted data from an executed `LangChain.Chains.LLMChain` that was run by `run_chain/4`. + Under `:tool_strategy`, this reads the `info` array from the extraction + tool call. Under `:provider_strategy`, the response isn't wrapped in an + `info` envelope, so this instead takes whatever JSON the LLM returned + (list or map, matching `json_schema` as given) and normalizes it via + `normalize_extraction_info/1` — this works regardless of the shape of + `json_schema` passed to `run_chain/4`. + Returns an error when the LLM did not respond with the expected extraction - tool call. + tool call, or (under `:provider_strategy`) valid JSON. When `JsonProcessor` + halted on invalid JSON, that corrective error message is surfaced directly + instead of a generic "unexpected response" message. """ @spec extract_result(LLMChain.t()) :: {:ok, result :: [any()]} | {:error, LangChainError.t()} def extract_result(%LLMChain{ @@ -209,8 +349,26 @@ Passage: normalize_extraction_info(info) end + def extract_result(%LLMChain{ + last_message: %Message{role: :assistant, processed_content: processed_content} + }) + when is_list(processed_content) or is_map(processed_content) do + normalize_extraction_info(processed_content) + end + + # Assuming there was no last message. the extraction did not work due to invalid json schema + # we propagate the error forward. + def extract_result(%LLMChain{ + last_message: %Message{role: :user, content: content} + }) do + case ContentPart.content_to_string(content) do + "ERROR: " <> _ = error_text -> {:error, LangChainError.exception(error_text)} + _ -> {:error, LangChainError.exception("Unexpected response.")} + end + end + def extract_result(%LLMChain{} = chain) do - {:error, LangChainError.exception("Unexpected response. #{inspect({:ok, chain})}")} + {:error, LangChainError.exception("Unexpected response. #{inspect(chain.last_message)}")} end @doc """ @@ -221,7 +379,7 @@ Passage: Accepts the same options as `run_chain/4`. """ - @spec run(ChatOpenAI.t(), json_schema :: map(), prompt :: [any()], opts :: Keyword.t()) :: + @spec run(ChatModel.t(), json_schema :: map(), prompt :: [any()], opts :: Keyword.t()) :: {:ok, result :: [any()]} | {:error, LangChainError.t()} def run(llm, json_schema, prompt, opts \\ []) do try do diff --git a/lib/chat_models/chat_model.ex b/lib/chat_models/chat_model.ex index 49db4ec6..a906f5b8 100644 --- a/lib/chat_models/chat_model.ex +++ b/lib/chat_models/chat_model.ex @@ -230,6 +230,22 @@ defmodule LangChain.ChatModels.ChatModel do defp json_response_format?(_), do: false + @doc """ + Returns whether a chat model module natively supports requesting + structured JSON output, i.e. whether its bare struct defines both + `:json_schema` and `:json_response` fields. This reflects what the *type* + supports, not whether a given instance currently has those fields + configured — use `output_type/1` for that. + + Takes the chat model module, not a struct instance — pass `llm.__struct__` + when starting from a configured struct. + """ + @spec supports_json_output?(module()) :: boolean() + def supports_json_output?(module) when is_atom(module) do + pure = struct(module) + Map.has_key?(pure, :json_schema) and Map.has_key?(pure, :json_response) + end + # Request endpoint URL for `server.address`/`server.port`, when the model exposes # one under the conventional `:endpoint` field. @spec endpoint(struct() | nil) :: String.t() | nil diff --git a/test/chains/data_extraction_chain_test.exs b/test/chains/data_extraction_chain_test.exs index ae2dba80..5b4d05d1 100644 --- a/test/chains/data_extraction_chain_test.exs +++ b/test/chains/data_extraction_chain_test.exs @@ -9,6 +9,8 @@ defmodule LangChain.Chains.DataExtractionChainTest do alias LangChain.FunctionParam alias LangChain.Chains.DataExtractionChain alias LangChain.ChatModels.ChatOpenAI + alias LangChain.ChatModels.ChatOpenAIResponses + alias LangChain.ChatModels.ChatGrok alias LangChain.LangChainError alias LangChain.Message alias LangChain.Message.ToolCall @@ -98,7 +100,7 @@ defmodule LangChain.Chains.DataExtractionChainTest do end end - describe "run_chain/4" do + describe "run_chain/4 with strategy: :tool_strategy" do setup do schema_parameters = [FunctionParam.new!(%{name: "person_name", type: :string})] @@ -115,10 +117,15 @@ defmodule LangChain.Chains.DataExtractionChainTest do } do message = extraction_message(%{usage: %TokenUsage{input: 42, output: 7}}) - expect(ChatOpenAI, :call, fn _model, _messages, _tools -> {:ok, [message]} end) + expect(ChatOpenAI, :call, fn _model, _messages, tools -> + assert tools != [] + {:ok, [message]} + end) assert {:ok, %LLMChain{last_message: %Message{role: :assistant} = last_message} = chain} = - DataExtractionChain.run_chain(chat, schema_parameters, "Alex is here.") + DataExtractionChain.run_chain(chat, schema_parameters, "Alex is here.", + strategy: :tool_strategy + ) assert TokenUsage.get(last_message) == %TokenUsage{input: 42, output: 7} assert {:ok, [%{"person_name" => "Alex"}]} = DataExtractionChain.extract_result(chain) @@ -137,7 +144,9 @@ defmodule LangChain.Chains.DataExtractionChainTest do expect(ChatOpenAI, :call, fn _model, _messages, _tools -> {:ok, [message]} end) assert {:ok, %LLMChain{last_message: last_message} = chain} = - DataExtractionChain.run_chain(chat, schema_parameters, "Alex is here.") + DataExtractionChain.run_chain(chat, schema_parameters, "Alex is here.", + strategy: :tool_strategy + ) # the usage is still reportable even though the extraction failed assert TokenUsage.get(last_message) == %TokenUsage{input: 12, output: 3} @@ -145,7 +154,7 @@ defmodule LangChain.Chains.DataExtractionChainTest do end end - describe "run/4" do + describe "run/4 with strategy: :tool_strategy" do setup do schema_parameters = [FunctionParam.new!(%{name: "person_name", type: :string})] @@ -157,12 +166,15 @@ defmodule LangChain.Chains.DataExtractionChainTest do end test "returns the extracted result", %{schema_parameters: schema_parameters, chat: chat} do - expect(ChatOpenAI, :call, fn _model, _messages, _tools -> + expect(ChatOpenAI, :call, fn _model, _messages, tools -> + assert tools != [] {:ok, [extraction_message()]} end) assert {:ok, [%{"person_name" => "Alex"}]} = - DataExtractionChain.run(chat, schema_parameters, "Alex is here.") + DataExtractionChain.run(chat, schema_parameters, "Alex is here.", + strategy: :tool_strategy + ) end test "returns an error when the LLM did not make the extraction tool call", %{ @@ -174,12 +186,158 @@ defmodule LangChain.Chains.DataExtractionChainTest do end) assert {:error, %LangChainError{message: message}} = - DataExtractionChain.run(chat, schema_parameters, "Alex is here.") + DataExtractionChain.run(chat, schema_parameters, "Alex is here.", + strategy: :tool_strategy + ) assert message =~ "Unexpected response." end end + describe "run_chain/4 default strategy (:provider_strategy)" do + setup do + schema_parameters = + [FunctionParam.new!(%{name: "person_name", type: :string})] + |> FunctionParam.to_parameters_schema() + + # Nothing but the model itself is configured up front; the chain patches + # in json_response/json_schema from schema_parameters at call time. + {:ok, chat} = ChatOpenAI.new(%{model: "gpt-4o-mini-2024-07-18", stream: false}) + + %{schema_parameters: schema_parameters, chat: chat} + end + + test "patches json_response/json_schema (unwrapped, as given) onto the llm and skips adding a tool, with no :strategy option given", + %{ + schema_parameters: schema_parameters, + chat: chat + } do + message = + Message.new_assistant!(%{ + content: Jason.encode!(%{"person_name" => "Alex"}) + }) + + expect(ChatOpenAI, :call, fn model, _messages, tools -> + assert tools == [] + assert model.json_response == true + assert model.json_schema == schema_parameters + {:ok, [message]} + end) + + assert {:ok, [%{"person_name" => "Alex"}]} = + DataExtractionChain.run(chat, schema_parameters, "Alex is here.") + end + + test "extracts a list directly when json_schema is array-typed", %{chat: chat} do + item_schema = + [FunctionParam.new!(%{name: "person_name", type: :string})] + |> FunctionParam.to_parameters_schema() + + array_schema = %{type: "array", items: item_schema} + + message = + Message.new_assistant!(%{ + content: Jason.encode!([%{"person_name" => "Alex"}, %{"person_name" => "Claudia"}]) + }) + + expect(ChatOpenAI, :call, fn model, _messages, tools -> + assert tools == [] + assert model.json_schema == array_schema + {:ok, [message]} + end) + + assert {:ok, [%{"person_name" => "Alex"}, %{"person_name" => "Claudia"}]} = + DataExtractionChain.run(chat, array_schema, "Alex and Claudia are here.") + end + + test "surfaces the JsonProcessor's error message when the JSON response is not valid JSON", + %{ + schema_parameters: schema_parameters, + chat: chat + } do + message = Message.new_assistant!(%{content: "not json"}) + + expect(ChatOpenAI, :call, fn _model, _messages, _tools -> {:ok, [message]} end) + + assert {:error, %LangChainError{message: error_message}} = + DataExtractionChain.run(chat, schema_parameters, "Alex is here.") + + assert error_message =~ "ERROR: Invalid JSON data:" + end + + test "fills in extra fields (e.g. json_schema_name) with sensible defaults when the struct defines them", + %{schema_parameters: schema_parameters} do + {:ok, chat} = ChatOpenAIResponses.new(%{stream: false}) + + message = + Message.new_assistant!(%{ + content: Jason.encode!(%{"person_name" => "Alex"}) + }) + + expect(ChatOpenAIResponses, :call, fn model, _messages, tools -> + assert tools == [] + assert model.json_schema == schema_parameters + assert model.json_schema_name == "information_extraction" + {:ok, [message]} + end) + + assert {:ok, [%{"person_name" => "Alex"}]} = + DataExtractionChain.run(chat, schema_parameters, "Alex is here.") + end + end + + describe "run_chain/4 default strategy (:provider_strategy) with an unsupported llm" do + test "fails gracefully by falling back to :tool_strategy, logging a warning" do + schema_parameters = + [FunctionParam.new!(%{name: "person_name", type: :string})] + |> FunctionParam.to_parameters_schema() + + chat = ChatGrok.new!(%{}) + + expect(ChatGrok, :call, fn _model, _messages, tools -> + assert tools != [] + {:ok, [extraction_message()]} + end) + + {result, log} = + ExUnit.CaptureLog.with_log(fn -> + DataExtractionChain.run(chat, schema_parameters, "Alex is here.") + end) + + assert {:ok, [%{"person_name" => "Alex"}]} = result + assert log =~ "does not support :provider_strategy" + assert log =~ "Falling back to :tool_strategy" + end + end + + describe "run_chain/4 :strategy option" do + test "raises for an invalid :strategy value" do + schema_parameters = + [FunctionParam.new!(%{name: "person_name", type: :string})] + |> FunctionParam.to_parameters_schema() + + {:ok, chat} = ChatOpenAI.new(%{model: "gpt-4o-mini-2024-07-18", stream: false}) + + assert_raise LangChainError, ~r/Invalid :strategy/, fn -> + DataExtractionChain.run_chain(chat, schema_parameters, "Alex is here.", strategy: :bogus) + end + end + + test "raises (no fallback) when strategy: :provider_strategy is given explicitly for an unsupported llm" do + schema_parameters = + [FunctionParam.new!(%{name: "person_name", type: :string})] + |> FunctionParam.to_parameters_schema() + + chat = ChatGrok.new!(%{}) + + assert_raise LangChainError, ~r/does not support :provider_strategy/, fn -> + DataExtractionChain.run_chain(chat, schema_parameters, "Alex is here.", + strategy: :provider_strategy + ) + end + end + end + describe "run_chain/4 :callbacks option" do setup do schema_parameters = @@ -208,7 +366,8 @@ defmodule LangChain.Chains.DataExtractionChainTest do assert {:ok, %LLMChain{}} = DataExtractionChain.run_chain(chat, schema_parameters, "Alex is here.", - callbacks: [handler] + callbacks: [handler], + strategy: :tool_strategy ) assert_received {:processed, %Message{}} @@ -228,7 +387,8 @@ defmodule LangChain.Chains.DataExtractionChainTest do assert {:ok, [%{"person_name" => "Alex"}]} = DataExtractionChain.run(chat, schema_parameters, "Alex is here.", - callbacks: [handler] + callbacks: [handler], + strategy: :tool_strategy ) assert_received {:usage, %TokenUsage{input: 42, output: 7}} @@ -254,7 +414,8 @@ defmodule LangChain.Chains.DataExtractionChainTest do assert {:error, %LangChainError{}} = DataExtractionChain.run(chat, schema_parameters, "Alex is here.", - callbacks: [handler] + callbacks: [handler], + strategy: :tool_strategy ) assert_received {:usage, %TokenUsage{input: 12, output: 3}} @@ -305,7 +466,11 @@ defmodule LangChain.Chains.DataExtractionChainTest do Alex's dog Frosty is a labrador and likes to play hide and seek. Identify each person and their relevant information. """ - {:ok, result} = DataExtractionChain.run(chat, schema_parameters, data_prompt, verbose: true) + {:ok, result} = + DataExtractionChain.run(chat, schema_parameters, data_prompt, + verbose: true, + strategy: :tool_strategy + ) assert result == [ %{ diff --git a/test/chat_models/chat_model_test.exs b/test/chat_models/chat_model_test.exs index 212e536b..1ec140f7 100644 --- a/test/chat_models/chat_model_test.exs +++ b/test/chat_models/chat_model_test.exs @@ -328,6 +328,16 @@ defmodule LangChain.ChatModels.ChatModelTest do end end + describe "supports_json_output?/1" do + test "returns true for a module defining both :json_schema and :json_response" do + assert ChatModel.supports_json_output?(ChatOpenAI) + end + + test "returns false for a module missing either field" do + refute ChatModel.supports_json_output?(LangChain.ChatModels.ChatGrok) + end + end + describe "request_options/1" do test "extracts the standard request parameters a model sets, dropping nils" do model = diff --git a/test/test_helper.exs b/test/test_helper.exs index f29aac97..c593733e 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -32,6 +32,7 @@ Mimic.copy(LangChain.Utils.AwsEventstreamDecoder) Mimic.copy(Req) Mimic.copy(LangChain.ChatModels.ChatOpenAI) +Mimic.copy(LangChain.ChatModels.ChatOpenAIResponses) Mimic.copy(LangChain.ChatModels.ChatAnthropic) Mimic.copy(LangChain.ChatModels.ChatMistralAI) Mimic.copy(LangChain.ChatModels.ChatBumblebee)