diff --git a/cmake/deps.txt b/cmake/deps.txt index daec5309b3..3617c92470 100644 --- a/cmake/deps.txt +++ b/cmake/deps.txt @@ -14,7 +14,7 @@ pybind11;https://github.com/pybind/pybind11/archive/refs/tags/v2.13.6.zip;f78029 googletest;https://github.com/google/googletest/archive/530d5c8c84abd2a46f38583ee817743c9b3a42b4.zip;5e3a61db2aa975cfd0f97ba92c818744e7fa7034 microsoft_wil;https://github.com/microsoft/wil/archive/refs/tags/v1.0.230629.1.zip;e4a542a323c070376f7c2d1973d0f7ddbc1d2fa5 directx_headers;https://github.com/microsoft/DirectX-Headers/archive/refs/tags/v1.613.1.zip;47653509a3371eabb156360f42faf582f314bf2e -onnxruntime_extensions;https://github.com/microsoft/onnxruntime-extensions.git;60687b12b9de56df9a702a0e548eacc26836fab9 +onnxruntime_extensions;https://github.com/microsoft/onnxruntime-extensions.git;bd0e21c11187e0b8a2385d1c61122a4d259a53a0 # These two dependencies are for the optional constrained decoding feature (USE_GUIDANCE) llguidance;https://github.com/microsoft/llguidance.git;94fa39128ef184ffeda33845f6d333f332a34b4d diff --git a/src/models/model.cpp b/src/models/model.cpp index fa4e7abed2..48df601e3e 100644 --- a/src/models/model.cpp +++ b/src/models/model.cpp @@ -431,6 +431,22 @@ std::string Tokenizer::ApplyChatTemplate(const char* template_str, const char* m return text_ptr; } +std::string Tokenizer::ApplyChatTemplateWithOptions(const char* template_str, const char* messages, const char* tools, + const char* template_kwargs, bool add_generation_prompt) const { + ort_extensions::OrtxObjectPtr templated_text; + CheckResult(OrtxApplyChatTemplateWithOptions(tokenizer_, template_str, messages, tools, template_kwargs, + templated_text.ToBeAssigned(), add_generation_prompt, + false /*tokenize*/)); + + ort_extensions::OrtxObjectPtr tensor; + CheckResult(OrtxTensorResultGetAt(templated_text.get(), 0, tensor.ToBeAssigned())); + + const char* text_ptr{}; + CheckResult(OrtxGetTensorData(tensor.get(), reinterpret_cast(&text_ptr), nullptr, nullptr)); + + return text_ptr; +} + std::vector Tokenizer::EncodeBatch(std::span strings) const { std::vector> sequences; std::vector> span_sequences; diff --git a/src/models/model.h b/src/models/model.h index 7c082866fa..d0fe1c08ed 100644 --- a/src/models/model.h +++ b/src/models/model.h @@ -132,6 +132,8 @@ struct Tokenizer : std::enable_shared_from_this, LeakChecked Encode(const char* text) const; std::string Decode(std::span tokens) const; std::string ApplyChatTemplate(const char* template_str, const char* messages, const char* tools, bool add_generation_prompt) const; + std::string ApplyChatTemplateWithOptions(const char* template_str, const char* messages, const char* tools, + const char* template_kwargs, bool add_generation_prompt) const; std::vector EncodeBatch(std::span strings) const; std::shared_ptr EncodeBatch(std::span strings) const; diff --git a/src/ort_genai.h b/src/ort_genai.h index ddf1ddd801..31ec14c85c 100644 --- a/src/ort_genai.h +++ b/src/ort_genai.h @@ -420,6 +420,14 @@ struct OgaTokenizer : OgaAbstract { return p; } + OgaString ApplyChatTemplateWithOptions(const char* template_str, const char* messages, const char* tools, + const char* template_kwargs, bool add_generation_prompt) const { + const char* p{}; + OgaCheckResult(OgaTokenizerApplyChatTemplateWithOptions(this, template_str, messages, tools, template_kwargs, + add_generation_prompt, &p)); + return p; + } + #if OGA_USE_SPAN OgaString Decode(std::span tokens) const { const char* p; diff --git a/src/ort_genai_c.cpp b/src/ort_genai_c.cpp index 789896bc44..df81b48ca9 100644 --- a/src/ort_genai_c.cpp +++ b/src/ort_genai_c.cpp @@ -1041,6 +1041,17 @@ OgaResult* OGA_API_CALL OgaTokenizerApplyChatTemplate(const OgaTokenizer* tokeni OGA_CATCH } +OgaResult* OGA_API_CALL OgaTokenizerApplyChatTemplateWithOptions(const OgaTokenizer* tokenizer, const char* template_str, + const char* messages, const char* tools, + const char* template_kwargs, bool add_generation_prompt, + const char** out_string) { + OGA_TRY + *out_string = AllocOgaString(tokenizer->ApplyChatTemplateWithOptions(template_str, messages, tools, + template_kwargs, add_generation_prompt)); + return nullptr; + OGA_CATCH +} + OgaResult* OGA_API_CALL OgaTokenizerDecodeBatch(const OgaTokenizer* tokenizer, const OgaTensor* tensor, OgaStringArray** out) { OGA_TRY auto shape = tensor->GetShape(); diff --git a/src/ort_genai_c.h b/src/ort_genai_c.h index 8fd2b74267..9533b24c55 100644 --- a/src/ort_genai_c.h +++ b/src/ort_genai_c.h @@ -989,6 +989,20 @@ OGA_EXPORT OgaResult* OGA_API_CALL OgaProcessorDecode(const OgaMultiModalProcess */ OGA_EXPORT OgaResult* OGA_API_CALL OgaTokenizerApplyChatTemplate(const OgaTokenizer*, const char* template_str, const char* messages, const char* tools, bool add_generation_prompt, const char** out_string); +/** + * @brief Applies a chat template to input messages with additional template context values. + * + * \param[in] tokenizer OgaTokenizer used for template processing. + * \param[in] template_str Null-terminated string representing the chat template. Use nullptr to fall back to the default chat template from the tokenizer config. + * \param[in] messages Null-terminated string containing the input messages to be processed. + * \param[in] tools Null-terminated string containing the chat function calls if any. Use nullptr if none. + * \param[in] template_kwargs Null-terminated JSON object containing additional template context values. Use nullptr if none. + * \param[in] add_generation_prompt Indicates whether to add a generation prompt to the output. + * \param[out] out_string Pointer to where the output will be stored. The returned pointer must be freed with OgaDestroyString. + * \return OgaResult* containing the error message if the function fails. + */ +OGA_EXPORT OgaResult* OGA_API_CALL OgaTokenizerApplyChatTemplateWithOptions(const OgaTokenizer*, const char* template_str, const char* messages, const char* tools, const char* template_kwargs, bool add_generation_prompt, const char** out_string); + /** OgaTokenizerStream is to decoded token strings incrementally, one token at a time. */ OGA_EXPORT OgaResult* OGA_API_CALL OgaCreateTokenizerStream(const OgaTokenizer*, OgaTokenizerStream** out); diff --git a/test/c_api_tests.cpp b/test/c_api_tests.cpp index a4fded055e..c86c05a884 100644 --- a/test/c_api_tests.cpp +++ b/test/c_api_tests.cpp @@ -243,6 +243,21 @@ TEST(CAPITests, ChatTemplate) { auto out_string = tokenizer->ApplyChatTemplate(chat_template, messages_json, nullptr, true); ASSERT_STREQ(expected_output, out_string); + const char* kwargs_template = + "{% if enable_thinking is defined and not enable_thinking %}NO_THINK{% else %}THINK{% endif %}" + "|{{ reasoning_effort }}|{{ level }}"; + const char* template_kwargs = + R"({"enable_thinking":false,"reasoning_effort":"low","level":2})"; + auto kwargs_output = tokenizer->ApplyChatTemplateWithOptions( + kwargs_template, messages_json, nullptr, template_kwargs, true); + ASSERT_STREQ("NO_THINK|low|2", kwargs_output); + + auto legacy_output = tokenizer->ApplyChatTemplate( + "{{ messages[0].content }}", messages_json, nullptr, true); + auto null_options_output = tokenizer->ApplyChatTemplateWithOptions( + "{{ messages[0].content }}", messages_json, nullptr, nullptr, true); + ASSERT_STREQ(legacy_output, null_options_output); + #endif }