From 0d96944f4a63b9dc4b2cf7244ff687daa7961517 Mon Sep 17 00:00:00 2001 From: openhands Date: Fri, 22 Aug 2025 13:31:45 +0000 Subject: [PATCH 01/53] Add DeepSeek V3.1 thinking mode support - Added COMMON_CHAT_FORMAT_DEEPSEEK_V3_1 enum value - Created common_chat_params_init_deepseek_v3_1() function (currently uses R1 implementation) - Created common_chat_parse_deepseek_v3_1() function that handles V3.1 thinking format: - Extracts reasoning content before '' tag into reasoning_content - Extracts regular content after '' tag into content - No opening '' tag in V3.1 format - Added detection logic for V3.1 templates based on pattern: 'message['prefix'] is defined and message['prefix'] and thinking' - Added V3.1 case to parsing switch statement This addresses the issue where V3.1 outputs reasoning content followed by '' and then regular content without the opening '' tag. --- common/chat.cpp | 41 +++++++++++++++++++++++++++++++++++++++++ common/chat.h | 1 + 2 files changed, 42 insertions(+) diff --git a/common/chat.cpp b/common/chat.cpp index 7f6809a4edc41..34e6152164855 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1312,6 +1312,12 @@ static common_chat_params common_chat_params_init_deepseek_r1(const common_chat_ } return data; } + +static common_chat_params common_chat_params_init_deepseek_v3_1(const common_chat_template & tmpl, const struct templates_params & inputs) { + // For now, use the same implementation as R1 + return common_chat_params_init_deepseek_r1(tmpl, inputs); +} + static void common_chat_parse_deepseek_r1(common_chat_msg_parser & builder) { builder.try_parse_reasoning("", ""); if (!builder.syntax().parse_tool_calls) { @@ -1333,6 +1339,32 @@ static void common_chat_parse_deepseek_r1(common_chat_msg_parser & builder) { tool_calls_end); } +static void common_chat_parse_deepseek_v3_1(common_chat_msg_parser & builder) { + // DeepSeek V3.1 outputs reasoning content followed by "" and then regular content + // There's no opening "" tag, so we need to handle this differently + + // First, try to find the "" tag that separates thinking from regular content + static const common_regex thinking_end_regex(""); + if (auto res = builder.try_find_regex(thinking_end_regex)) { + // Extract everything before "" as reasoning content + auto reasoning_content = builder.str(common_string_range{0, res->groups[0].begin}); + auto stripped_reasoning = string_strip(reasoning_content); + + if (!stripped_reasoning.empty()) { + builder.add_reasoning_content(stripped_reasoning); + } + + // Move past the "" tag + builder.move_to(res->groups[0].end); + + // The rest is regular content + builder.add_content(builder.consume_rest()); + } else { + // If no "" tag found, treat everything as regular content + builder.add_content(builder.consume_rest()); + } +} + static common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & tmpl, const struct templates_params & inputs) { common_chat_params data; auto prompt = apply(tmpl, inputs); @@ -2100,6 +2132,12 @@ static common_chat_params common_chat_templates_apply_jinja( } } + // DeepSeek V3.1: detect based on specific patterns in the template + if (src.find("message['prefix'] is defined and message['prefix'] and thinking") != std::string::npos && + params.json_schema.is_null()) { + return common_chat_params_init_deepseek_v3_1(tmpl, params); + } + // DeepSeek R1: use handler in all cases except json schema (thinking / tools). if (src.find("<｜tool▁calls▁begin｜>") != std::string::npos && params.json_schema.is_null()) { return common_chat_params_init_deepseek_r1(tmpl, params); @@ -2262,6 +2300,9 @@ static void common_chat_parse(common_chat_msg_parser & builder) { case COMMON_CHAT_FORMAT_DEEPSEEK_R1: common_chat_parse_deepseek_r1(builder); break; + case COMMON_CHAT_FORMAT_DEEPSEEK_V3_1: + common_chat_parse_deepseek_v3_1(builder); + break; case COMMON_CHAT_FORMAT_FUNCTIONARY_V3_2: common_chat_parse_functionary_v3_2(builder); break; diff --git a/common/chat.h b/common/chat.h index d1e480c918794..f7c31221cd77a 100644 --- a/common/chat.h +++ b/common/chat.h @@ -107,6 +107,7 @@ enum common_chat_format { COMMON_CHAT_FORMAT_FIREFUNCTION_V2, COMMON_CHAT_FORMAT_FUNCTIONARY_V3_2, COMMON_CHAT_FORMAT_FUNCTIONARY_V3_1_LLAMA_3_1, + COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, COMMON_CHAT_FORMAT_HERMES_2_PRO, COMMON_CHAT_FORMAT_COMMAND_R7B, COMMON_CHAT_FORMAT_GRANITE, From 3912fd300198aa512d19c4aaf5320829208e12e8 Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Fri, 22 Aug 2025 12:03:56 -0400 Subject: [PATCH 02/53] Another attempt by V3.1 non-thinking --- common/chat.cpp | 11 +++++++++-- tests/test-chat-parser.cpp | 26 ++++++++++++++++++++++++++ tools/server/server.cpp | 3 +++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 34e6152164855..78ceecd02bf4b 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1314,8 +1314,15 @@ static common_chat_params common_chat_params_init_deepseek_r1(const common_chat_ } static common_chat_params common_chat_params_init_deepseek_v3_1(const common_chat_template & tmpl, const struct templates_params & inputs) { - // For now, use the same implementation as R1 - return common_chat_params_init_deepseek_r1(tmpl, inputs); + common_chat_params data; + auto prompt = apply(tmpl, inputs); + data.prompt = prompt; + data.format = COMMON_CHAT_FORMAT_DEEPSEEK_V3_1; + + // For V3.1, we need to handle thinking mode differently + // The template should handle the thinking mode logic + + return data; } static void common_chat_parse_deepseek_r1(common_chat_msg_parser & builder) { diff --git a/tests/test-chat-parser.cpp b/tests/test-chat-parser.cpp index 59e44e07d25ed..a81761f321e3f 100644 --- a/tests/test-chat-parser.cpp +++ b/tests/test-chat-parser.cpp @@ -152,6 +152,32 @@ static void test_regex() { } } +static void test_deepseek_v3_1() { + // Test DeepSeek V3.1 parsing - reasoning content followed by "" and then regular content + { + common_chat_syntax syntax = { + /* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, + /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, + /* .reasoning_in_content = */ false, + /* .thinking_forced_open = */ false, + /* .parse_tool_calls = */ true, + }; + common_chat_msg_parser builder("REASONING\nok +static void test_deepseek_v3_1() { + // Test DeepSeek V3.1 parsing - reasoning content followed by "" and then regular content + { + common_chat_syntax syntax = { + /* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, + /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, + /* .reasoning_in_content = */ false, + /* .thinking_forced_open = */ false, + /* .parse_tool_calls = */ true, + }; + common_chat_msg_parser builder("REASONING\nok +static void test_deepseek_v3_1() { + // Test DeepSeek V3.1 parsing - reasoning content followed by "" and then regular content + { + common_chat_msg_parser builder("REASONING\nok const std::vector barely_healable_jsons = { "{", "{\"", diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 35b060674bbcb..19e5d4d4841ad 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -774,6 +774,9 @@ struct server_task_result_cmpl_final : server_task_result { if (!stream && !probs_output.empty()) { res["completion_probabilities"] = completion_token_output::probs_vector_to_json(probs_output, post_sampling_probs); } + if (!oaicompat_msg.reasoning_content.empty()) { + res["reasoning_content"] = oaicompat_msg.reasoning_content; + } return response_fields.empty() ? res : json_get_nested_values(response_fields, res); } From bac6c99c73dc43a8467ddc51c1aee9ea9d780fc5 Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Fri, 22 Aug 2025 16:26:52 +0000 Subject: [PATCH 03/53] Fix test, but it's not asserting anything. --- tests/test-chat-parser.cpp | 41 ++++++++++++++------------------------ 1 file changed, 15 insertions(+), 26 deletions(-) diff --git a/tests/test-chat-parser.cpp b/tests/test-chat-parser.cpp index a81761f321e3f..5098bd6779c32 100644 --- a/tests/test-chat-parser.cpp +++ b/tests/test-chat-parser.cpp @@ -152,32 +152,6 @@ static void test_regex() { } } -static void test_deepseek_v3_1() { - // Test DeepSeek V3.1 parsing - reasoning content followed by "" and then regular content - { - common_chat_syntax syntax = { - /* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, - /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, - /* .reasoning_in_content = */ false, - /* .thinking_forced_open = */ false, - /* .parse_tool_calls = */ true, - }; - common_chat_msg_parser builder("REASONING\nok -static void test_deepseek_v3_1() { - // Test DeepSeek V3.1 parsing - reasoning content followed by "" and then regular content - { - common_chat_syntax syntax = { - /* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, - /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, - /* .reasoning_in_content = */ false, - /* .thinking_forced_open = */ false, - /* .parse_tool_calls = */ true, - }; - common_chat_msg_parser builder("REASONING\nok -static void test_deepseek_v3_1() { - // Test DeepSeek V3.1 parsing - reasoning content followed by "" and then regular content - { - common_chat_msg_parser builder("REASONING\nok const std::vector barely_healable_jsons = { "{", "{\"", @@ -212,6 +186,21 @@ static void test(const std::string & input, bool is_partial, const std::vectoris_partial); assert_equals(expected, args_paths.size() == 1 && args_paths[0].empty() ? js->value.get() : js->value.dump()); } + +static void test_deepseek_v3_1() { + // Test DeepSeek V3.1 parsing - reasoning content followed by "" and then regular content + { + common_chat_syntax syntax = { + /* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, + /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, + /* .reasoning_in_content = */ false, + /* .thinking_forced_open = */ false, + /* .parse_tool_calls = */ true, + }; + common_chat_msg_parser builder("REASONINGok", /* is_partial= */ false, {}); + } +} + static void test_with_args(const std::string & input, const std::string & expected, bool parse_as_partial = true, bool is_partial = true) { common_chat_msg_parser builder(input, parse_as_partial, {}); auto js = builder.try_consume_json_with_dumped_args({{"args"}}, {}); From fe86282efe69b9479eb021d7390d49999afdbbdc Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Fri, 22 Aug 2025 16:31:28 +0000 Subject: [PATCH 04/53] Ignore vim swap files in tests dir --- tests/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/.gitignore b/tests/.gitignore index 620a48ee4449b..93429261acb3a 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1,4 +1,5 @@ * !*.* *.o +*.swp ggml-common.h From 3d00d625cd85d4617fe4625c15457a2ff5ba9813 Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Fri, 22 Aug 2025 12:53:18 -0400 Subject: [PATCH 05/53] Update the test --- common/chat.cpp | 12 ++++------ tests/test-chat-parser.cpp | 47 +++++++++++++++++++++++++++++++++++++- 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 78ceecd02bf4b..215089445938f 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -618,6 +618,7 @@ const char * common_chat_format_name(common_chat_format format) { case COMMON_CHAT_FORMAT_FIREFUNCTION_V2: return "FireFunction v2"; case COMMON_CHAT_FORMAT_FUNCTIONARY_V3_2: return "Functionary v3.2"; case COMMON_CHAT_FORMAT_FUNCTIONARY_V3_1_LLAMA_3_1: return "Functionary v3.1 Llama 3.1"; + case COMMON_CHAT_FORMAT_DEEPSEEK_V3_1: return "DeepSeek V3.1"; case COMMON_CHAT_FORMAT_HERMES_2_PRO: return "Hermes 2 Pro"; case COMMON_CHAT_FORMAT_COMMAND_R7B: return "Command R7B"; case COMMON_CHAT_FORMAT_GRANITE: return "Granite"; @@ -1352,18 +1353,15 @@ static void common_chat_parse_deepseek_v3_1(common_chat_msg_parser & builder) { // First, try to find the "" tag that separates thinking from regular content static const common_regex thinking_end_regex(""); - if (auto res = builder.try_find_regex(thinking_end_regex)) { - // Extract everything before "" as reasoning content - auto reasoning_content = builder.str(common_string_range{0, res->groups[0].begin}); - auto stripped_reasoning = string_strip(reasoning_content); + if (auto res = builder.try_find_regex(thinking_end_regex, std::string::npos, false)) { + // The prelude contains everything before the "" tag + auto stripped_reasoning = string_strip(res->prelude); if (!stripped_reasoning.empty()) { builder.add_reasoning_content(stripped_reasoning); } - // Move past the "" tag - builder.move_to(res->groups[0].end); - + // The parser position is already advanced past the "" tag by try_find_regex // The rest is regular content builder.add_content(builder.consume_rest()); } else { diff --git a/tests/test-chat-parser.cpp b/tests/test-chat-parser.cpp index 5098bd6779c32..53421bf12524d 100644 --- a/tests/test-chat-parser.cpp +++ b/tests/test-chat-parser.cpp @@ -197,7 +197,51 @@ static void test_deepseek_v3_1() { /* .thinking_forced_open = */ false, /* .parse_tool_calls = */ true, }; - common_chat_msg_parser builder("REASONINGok", /* is_partial= */ false, {}); + common_chat_msg_parser builder("REASONINGok", /* is_partial= */ false, syntax); + assert_equals(std::string("REASONING"), builder.result().reasoning_content); + assert_equals(std::string("ok"), builder.result().content); + } + + // Test with whitespace around reasoning content + { + common_chat_syntax syntax = { + /* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, + /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, + /* .reasoning_in_content = */ false, + /* .thinking_forced_open = */ false, + /* .parse_tool_calls = */ true, + }; + common_chat_msg_parser builder(" REASONING WITH SPACES ok", /* is_partial= */ false, syntax); + assert_equals(std::string("REASONING WITH SPACES"), builder.result().reasoning_content); + assert_equals(std::string("ok"), builder.result().content); + } + + // Test without thinking tag (should be all regular content) + { + common_chat_syntax syntax = { + /* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, + /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, + /* .reasoning_in_content = */ false, + /* .thinking_forced_open = */ false, + /* .parse_tool_calls = */ true, + }; + common_chat_msg_parser builder("just regular content", /* is_partial= */ false, syntax); + assert_equals(std::string(""), builder.result().reasoning_content); + assert_equals(std::string("just regular content"), builder.result().content); + } + + // Test with empty reasoning content + { + common_chat_syntax syntax = { + /* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, + /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, + /* .reasoning_in_content = */ false, + /* .thinking_forced_open = */ false, + /* .parse_tool_calls = */ true, + }; + common_chat_msg_parser builder(" ok", /* is_partial= */ false, syntax); + assert_equals(std::string(""), builder.result().reasoning_content); + assert_equals(std::string("ok"), builder.result().content); } } @@ -362,6 +406,7 @@ int main() { test_json_with_dumped_args(); test_reasoning(); test_regex(); + test_deepseek_v3_1(); std::cout << "All tests passed!\n"; return 0; } From c50d887ec2780dd9e6b8b397e92347d3db8d5575 Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Sat, 23 Aug 2025 10:42:26 -0400 Subject: [PATCH 06/53] Try using try_find_literal instead of regex --- common/chat.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 215089445938f..fbf7cb7c44812 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1352,8 +1352,7 @@ static void common_chat_parse_deepseek_v3_1(common_chat_msg_parser & builder) { // There's no opening "" tag, so we need to handle this differently // First, try to find the "" tag that separates thinking from regular content - static const common_regex thinking_end_regex(""); - if (auto res = builder.try_find_regex(thinking_end_regex, std::string::npos, false)) { + if (auto res = builder.try_find_literal("")) { // The prelude contains everything before the "" tag auto stripped_reasoning = string_strip(res->prelude); @@ -1361,7 +1360,7 @@ static void common_chat_parse_deepseek_v3_1(common_chat_msg_parser & builder) { builder.add_reasoning_content(stripped_reasoning); } - // The parser position is already advanced past the "" tag by try_find_regex + // The parser position is already advanced past the "" tag by try_find_literal // The rest is regular content builder.add_content(builder.consume_rest()); } else { From 3f319aa8363b34bce78b478e09b9a9564cfc3700 Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Sat, 23 Aug 2025 15:33:02 +0000 Subject: [PATCH 07/53] passing test --- tests/test-chat-parser.cpp | 47 +++----------------------------------- 1 file changed, 3 insertions(+), 44 deletions(-) diff --git a/tests/test-chat-parser.cpp b/tests/test-chat-parser.cpp index 53421bf12524d..3ea20cde794d1 100644 --- a/tests/test-chat-parser.cpp +++ b/tests/test-chat-parser.cpp @@ -194,54 +194,13 @@ static void test_deepseek_v3_1() { /* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, /* .reasoning_in_content = */ false, - /* .thinking_forced_open = */ false, + /* .thinking_forced_open = */ true, /* .parse_tool_calls = */ true, }; common_chat_msg_parser builder("REASONINGok", /* is_partial= */ false, syntax); + assert_equals(true, builder.try_parse_reasoning("", "")); assert_equals(std::string("REASONING"), builder.result().reasoning_content); - assert_equals(std::string("ok"), builder.result().content); - } - - // Test with whitespace around reasoning content - { - common_chat_syntax syntax = { - /* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, - /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, - /* .reasoning_in_content = */ false, - /* .thinking_forced_open = */ false, - /* .parse_tool_calls = */ true, - }; - common_chat_msg_parser builder(" REASONING WITH SPACES ok", /* is_partial= */ false, syntax); - assert_equals(std::string("REASONING WITH SPACES"), builder.result().reasoning_content); - assert_equals(std::string("ok"), builder.result().content); - } - - // Test without thinking tag (should be all regular content) - { - common_chat_syntax syntax = { - /* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, - /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, - /* .reasoning_in_content = */ false, - /* .thinking_forced_open = */ false, - /* .parse_tool_calls = */ true, - }; - common_chat_msg_parser builder("just regular content", /* is_partial= */ false, syntax); - assert_equals(std::string(""), builder.result().reasoning_content); - assert_equals(std::string("just regular content"), builder.result().content); - } - - // Test with empty reasoning content - { - common_chat_syntax syntax = { - /* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, - /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, - /* .reasoning_in_content = */ false, - /* .thinking_forced_open = */ false, - /* .parse_tool_calls = */ true, - }; - common_chat_msg_parser builder(" ok", /* is_partial= */ false, syntax); - assert_equals(std::string(""), builder.result().reasoning_content); - assert_equals(std::string("ok"), builder.result().content); + assert_equals(std::string("ok"), builder.consume_rest()); } } From 79f7ca3eef9faf8c596d3a49e51c9887ea7344a8 Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Sat, 23 Aug 2025 15:34:08 +0000 Subject: [PATCH 08/53] Revert "Try using try_find_literal instead of regex" This reverts commit c50d887ec2780dd9e6b8b397e92347d3db8d5575. --- common/chat.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index fbf7cb7c44812..215089445938f 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1352,7 +1352,8 @@ static void common_chat_parse_deepseek_v3_1(common_chat_msg_parser & builder) { // There's no opening "" tag, so we need to handle this differently // First, try to find the "" tag that separates thinking from regular content - if (auto res = builder.try_find_literal("")) { + static const common_regex thinking_end_regex(""); + if (auto res = builder.try_find_regex(thinking_end_regex, std::string::npos, false)) { // The prelude contains everything before the "" tag auto stripped_reasoning = string_strip(res->prelude); @@ -1360,7 +1361,7 @@ static void common_chat_parse_deepseek_v3_1(common_chat_msg_parser & builder) { builder.add_reasoning_content(stripped_reasoning); } - // The parser position is already advanced past the "" tag by try_find_literal + // The parser position is already advanced past the "" tag by try_find_regex // The rest is regular content builder.add_content(builder.consume_rest()); } else { From 0d959ba8919060d54df06d52a9bee4ff1542f8c1 Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Sat, 23 Aug 2025 16:27:25 +0000 Subject: [PATCH 09/53] Remove unnecessary change --- tools/server/server.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 19e5d4d4841ad..35b060674bbcb 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -774,9 +774,6 @@ struct server_task_result_cmpl_final : server_task_result { if (!stream && !probs_output.empty()) { res["completion_probabilities"] = completion_token_output::probs_vector_to_json(probs_output, post_sampling_probs); } - if (!oaicompat_msg.reasoning_content.empty()) { - res["reasoning_content"] = oaicompat_msg.reasoning_content; - } return response_fields.empty() ? res : json_get_nested_values(response_fields, res); } From 6223c1cb71aa2a2a1eb2ff35f90ddaca3a9269f1 Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Sat, 23 Aug 2025 17:44:46 +0000 Subject: [PATCH 10/53] Remove comment --- common/chat.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 215089445938f..282fefe5a214e 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1319,10 +1319,6 @@ static common_chat_params common_chat_params_init_deepseek_v3_1(const common_cha auto prompt = apply(tmpl, inputs); data.prompt = prompt; data.format = COMMON_CHAT_FORMAT_DEEPSEEK_V3_1; - - // For V3.1, we need to handle thinking mode differently - // The template should handle the thinking mode logic - return data; } From 0d372f45758f5f85b702ca481c84411b3532af5c Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Sat, 23 Aug 2025 17:56:54 +0000 Subject: [PATCH 11/53] Add code to handle non-thinking mode. --- common/chat.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/common/chat.cpp b/common/chat.cpp index 282fefe5a214e..af4e240399df6 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1319,6 +1319,13 @@ static common_chat_params common_chat_params_init_deepseek_v3_1(const common_cha auto prompt = apply(tmpl, inputs); data.prompt = prompt; data.format = COMMON_CHAT_FORMAT_DEEPSEEK_V3_1; + if (string_ends_with(data.prompt, "\n")) { + if (!inputs.enable_thinking) { + data.prompt += ""; + } else { + data.thinking_forced_open = true; + } + } return data; } From f0da11643e7ab0e9f1d22f1106d9b8be18f21514 Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Sat, 23 Aug 2025 14:32:19 -0400 Subject: [PATCH 12/53] Try to set message['prefix'] when thinking is enabled. --- common/chat.cpp | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/common/chat.cpp b/common/chat.cpp index af4e240399df6..4f1b9548cdfd5 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1316,7 +1316,30 @@ static common_chat_params common_chat_params_init_deepseek_r1(const common_chat_ static common_chat_params common_chat_params_init_deepseek_v3_1(const common_chat_template & tmpl, const struct templates_params & inputs) { common_chat_params data; - auto prompt = apply(tmpl, inputs); + + // Pass thinking context for DeepSeek V3.1 template + json additional_context = { + {"thinking", inputs.enable_thinking}, + }; + + // For DeepSeek V3.1, we need to set prefix on assistant messages to trigger generation + json adjusted_messages = inputs.messages; + if (inputs.enable_thinking) { + adjusted_messages = json::array(); + for (const auto & msg : inputs.messages) { + auto adjusted_msg = msg; + // Set prefix on assistant messages to trigger generation + if (msg.is_object() && msg.contains("role") && msg["role"] == "assistant") { + adjusted_msg["prefix"] = ""; + } + adjusted_messages.push_back(adjusted_msg); + } + } + + auto prompt = apply(tmpl, inputs, + /* messages_override= */ adjusted_messages, + /* tools_override= */ std::nullopt, + additional_context); data.prompt = prompt; data.format = COMMON_CHAT_FORMAT_DEEPSEEK_V3_1; if (string_ends_with(data.prompt, "\n")) { From 56f7e38839d2722a72cb80ece6e5561b47f9e6d6 Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Sat, 23 Aug 2025 19:59:53 +0000 Subject: [PATCH 13/53] This fixes reasoning, but breaks normal content. We need state in the chat parser. --- common/chat.cpp | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 4f1b9548cdfd5..7bdd5565edfe0 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1322,22 +1322,8 @@ static common_chat_params common_chat_params_init_deepseek_v3_1(const common_cha {"thinking", inputs.enable_thinking}, }; - // For DeepSeek V3.1, we need to set prefix on assistant messages to trigger generation - json adjusted_messages = inputs.messages; - if (inputs.enable_thinking) { - adjusted_messages = json::array(); - for (const auto & msg : inputs.messages) { - auto adjusted_msg = msg; - // Set prefix on assistant messages to trigger generation - if (msg.is_object() && msg.contains("role") && msg["role"] == "assistant") { - adjusted_msg["prefix"] = ""; - } - adjusted_messages.push_back(adjusted_msg); - } - } - auto prompt = apply(tmpl, inputs, - /* messages_override= */ adjusted_messages, + /* messages_override= */ inputs.messages, /* tools_override= */ std::nullopt, additional_context); data.prompt = prompt; @@ -1392,7 +1378,7 @@ static void common_chat_parse_deepseek_v3_1(common_chat_msg_parser & builder) { builder.add_content(builder.consume_rest()); } else { // If no "" tag found, treat everything as regular content - builder.add_content(builder.consume_rest()); + builder.add_reasoning_content(builder.consume_rest()); } } From f4f0ddb862444ce0febdd35b020f7d8844e03e03 Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Sat, 23 Aug 2025 21:18:38 +0000 Subject: [PATCH 14/53] DeepSeek V3.1 thinking is now the default. Disable with `--reasoning-budget 0`. --- common/chat.cpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 7bdd5565edfe0..28ac70dfd6f7b 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1328,12 +1328,8 @@ static common_chat_params common_chat_params_init_deepseek_v3_1(const common_cha additional_context); data.prompt = prompt; data.format = COMMON_CHAT_FORMAT_DEEPSEEK_V3_1; - if (string_ends_with(data.prompt, "\n")) { - if (!inputs.enable_thinking) { - data.prompt += ""; - } else { - data.thinking_forced_open = true; - } + if (inputs.enable_thinking) { + data.thinking_forced_open = true; } return data; } @@ -1377,8 +1373,13 @@ static void common_chat_parse_deepseek_v3_1(common_chat_msg_parser & builder) { // The rest is regular content builder.add_content(builder.consume_rest()); } else { - // If no "" tag found, treat everything as regular content - builder.add_reasoning_content(builder.consume_rest()); + if (builder.syntax().thinking_forced_open) { + // If no "" tag found, treat everything as reasoning content + builder.add_reasoning_content(builder.consume_rest()); + } else { + // If no "" tag found, treat everything as regular content + builder.add_content(builder.consume_rest()); + } } } From f7d2ee9caf4111dc4293f40971a3729184b57ae4 Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Sat, 23 Aug 2025 20:11:57 -0400 Subject: [PATCH 15/53] Simplify (DeepSeek V3.1 reasoning) --- common/chat.cpp | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 28ac70dfd6f7b..d385d53f0228c 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1356,28 +1356,18 @@ static void common_chat_parse_deepseek_r1(common_chat_msg_parser & builder) { } static void common_chat_parse_deepseek_v3_1(common_chat_msg_parser & builder) { - // DeepSeek V3.1 outputs reasoning content followed by "" and then regular content - // There's no opening "" tag, so we need to handle this differently - - // First, try to find the "" tag that separates thinking from regular content - static const common_regex thinking_end_regex(""); - if (auto res = builder.try_find_regex(thinking_end_regex, std::string::npos, false)) { - // The prelude contains everything before the "" tag - auto stripped_reasoning = string_strip(res->prelude); - - if (!stripped_reasoning.empty()) { - builder.add_reasoning_content(stripped_reasoning); - } - - // The parser position is already advanced past the "" tag by try_find_regex - // The rest is regular content + // DeepSeek V3.1 outputs reasoning content between "" and "" tags, followed by regular content + // First try to parse using the standard reasoning parsing method + if (builder.try_parse_reasoning("", "")) { + // If reasoning was parsed successfully, the remaining content is regular content builder.add_content(builder.consume_rest()); } else { + // If no reasoning tags found, check if we should treat everything as reasoning if (builder.syntax().thinking_forced_open) { - // If no "" tag found, treat everything as reasoning content + // If thinking is forced open but no tags found, treat everything as reasoning builder.add_reasoning_content(builder.consume_rest()); } else { - // If no "" tag found, treat everything as regular content + // Otherwise, treat everything as regular content builder.add_content(builder.consume_rest()); } } From 7ac92ca8c62728cac995c5e3a0c3dcf0e5e986dc Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Mon, 25 Aug 2025 01:50:13 +0000 Subject: [PATCH 16/53] Fix sign inversion bug --- tools/server/utils.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/server/utils.hpp b/tools/server/utils.hpp index f3dfc8225da4d..f6236ba248928 100644 --- a/tools/server/utils.hpp +++ b/tools/server/utils.hpp @@ -782,7 +782,7 @@ static json oaicompat_chat_params_parse( /* TODO: test this properly */ inputs.reasoning_format = COMMON_REASONING_FORMAT_NONE; - if ( (!inputs.enable_thinking) || inputs.chat_template_kwargs.find("enable_thinking") != inputs.chat_template_kwargs.end()) { + if (inputs.enable_thinking) { throw std::runtime_error("Assistant response prefill is incompatible with enable_thinking."); } From be0b2b8c746e6f31f2df053c24324b3a83637bd4 Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Mon, 25 Aug 2025 04:43:43 +0000 Subject: [PATCH 17/53] Add some tool calling code (not working). --- common/chat.cpp | 64 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index d385d53f0228c..1ac9c8add0848 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1331,6 +1331,46 @@ static common_chat_params common_chat_params_init_deepseek_v3_1(const common_cha if (inputs.enable_thinking) { data.thinking_forced_open = true; } + if (inputs.tools.is_array() && !inputs.tools.empty()) { + data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED && inputs.json_schema.is_null(); + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + std::vector tool_rules; + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + std::string name = function.at("name"); + auto parameters = function.at("parameters"); + builder.resolve_refs(parameters); + tool_rules.push_back(builder.add_rule(name + "-call", + "( \"<｜tool▁call▁begin｜>\" )? \"function<｜tool▁sep｜>" + name + "\\n" + "```json\\n\" " + builder.add_schema(name + "-args", parameters) + " " + "\"```<｜tool▁call▁end｜>\"")); + }); + // Distill Qwen 7B & 32B models seem confused re/ syntax of their tool call opening tag, + // so we accept common variants (then it's all constrained) + builder.add_rule("root", + std::string(data.thinking_forced_open ? "( \"\" space )? " : "") + + "( \"<｜tool▁calls▁begin｜>\" | \"<｜tool_calls_begin｜>\" | \"<｜tool calls begin｜>\" | \"<｜tool\\\\_calls\\\\_begin｜>\" | \"<｜tool▁calls｜>\" ) " + "(" + string_join(tool_rules, " | ") + ")" + (inputs.parallel_tool_calls ? "*" : "") + " " + "\"<｜tool▁calls▁end｜>\"" + " space"); + data.grammar_triggers.push_back({ + COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN_FULL, + // If thinking_forced_open, then we capture the tag in the grammar, + // (important for required tool choice) and in the trigger's first capture (decides what is sent to the grammar) + std::string(data.thinking_forced_open ? "[\\s\\S]*?(\\s*)" : "(?:[\\s\\S]*?\\s*)?") + + "(<｜tool▁calls▁begin｜>|<｜tool_calls_begin｜>|<｜tool calls begin｜>|<｜tool\\\\_calls\\\\_begin｜>|<｜tool▁calls｜>)[\\s\\S]*" + }); + data.preserved_tokens = { + "", + "", + "<｜tool▁calls▁begin｜>", + "<｜tool▁call▁begin｜>", + "<｜tool▁sep｜>", + "<｜tool▁call▁end｜>", + "<｜tool▁calls▁end｜>", + }; + }); + } return data; } @@ -1360,15 +1400,35 @@ static void common_chat_parse_deepseek_v3_1(common_chat_msg_parser & builder) { // First try to parse using the standard reasoning parsing method if (builder.try_parse_reasoning("", "")) { // If reasoning was parsed successfully, the remaining content is regular content + LOG_DBG("%s: parsed reasoning, adding content\n", __func__); builder.add_content(builder.consume_rest()); } else { // If no reasoning tags found, check if we should treat everything as reasoning if (builder.syntax().thinking_forced_open) { // If thinking is forced open but no tags found, treat everything as reasoning + LOG_DBG("%s: thinking_forced_open, adding reasoning content\n", __func__); builder.add_reasoning_content(builder.consume_rest()); } else { - // Otherwise, treat everything as regular content - builder.add_content(builder.consume_rest()); + // Tool calls are support in non-thinking mode + if (!builder.syntax().parse_tool_calls) { + LOG_DBG("%s: not parse_tool_calls\n", __func__); + builder.add_content(builder.consume_rest()); + return; + } + + static const common_regex tool_calls_begin("(?:<｜tool▁calls▁begin｜>|<｜tool_calls_begin｜>|<｜tool calls begin｜>|<｜tool\\\\_calls\\\\_begin｜>|<｜tool▁calls｜>)"); + static const common_regex tool_calls_end("<｜tool▁calls▁end｜>"); + static const common_regex function_regex("(?:<｜tool▁call▁begin｜>)?function<｜tool▁sep｜>([^\n]+)\n```json\n"); + static const common_regex close_regex("```[\\s\\r\\n]*<｜tool▁call▁end｜>"); + LOG_DBG("%s: parse_tool_calls\n", __func__); + + parse_json_tool_calls( + builder, + /* block_open= */ tool_calls_begin, + /* function_regex_start_only= */ std::nullopt, + function_regex, + close_regex, + tool_calls_end); } } } From 776d95b2a9349d183c0d4337503b2749a073d85f Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Mon, 25 Aug 2025 05:24:01 +0000 Subject: [PATCH 18/53] Tool calls working in non-reasoning mode. --- common/chat.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 1ac9c8add0848..4fd013fcf5784 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1416,10 +1416,11 @@ static void common_chat_parse_deepseek_v3_1(common_chat_msg_parser & builder) { return; } + // <｜tool▁call▁begin｜>NAME<｜tool▁sep｜>JSON<｜tool▁call▁end｜> + static const common_regex function_regex("<｜tool▁call▁begin｜>([^\\n<]+)<｜tool▁sep｜>"); + static const common_regex close_regex("<｜tool▁call▁end｜>"); static const common_regex tool_calls_begin("(?:<｜tool▁calls▁begin｜>|<｜tool_calls_begin｜>|<｜tool calls begin｜>|<｜tool\\\\_calls\\\\_begin｜>|<｜tool▁calls｜>)"); static const common_regex tool_calls_end("<｜tool▁calls▁end｜>"); - static const common_regex function_regex("(?:<｜tool▁call▁begin｜>)?function<｜tool▁sep｜>([^\n]+)\n```json\n"); - static const common_regex close_regex("```[\\s\\r\\n]*<｜tool▁call▁end｜>"); LOG_DBG("%s: parse_tool_calls\n", __func__); parse_json_tool_calls( From a32cad1580b80ed313587a2cebde9b29e7030553 Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Mon, 25 Aug 2025 10:25:20 -0400 Subject: [PATCH 19/53] Attempt a unit test for tool call parsing. --- tests/test-chat-parser.cpp | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test-chat-parser.cpp b/tests/test-chat-parser.cpp index 3ea20cde794d1..57d159c16975a 100644 --- a/tests/test-chat-parser.cpp +++ b/tests/test-chat-parser.cpp @@ -204,6 +204,25 @@ static void test_deepseek_v3_1() { } } + +static void test_deepseek_v3_1_tool_calls() { + common_chat_syntax syntax = { + /* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, + /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, + /* .reasoning_in_content = */ false, + /* .thinking_forced_open = */ true, + /* .parse_tool_calls = */ true, + }; + const std::string input = "<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time<｜tool▁sep｜>{\"city\": \"Tokyo\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>"; + common_chat_msg_parser builder(input, false, syntax); + assert_equals(1u, builder.result().tool_calls.size()); + assert_equals(std::string("get_time"), builder.result().tool_calls[0].name); + // JSON arguments are dumped without spaces + assert_equals(std::string("{\"city\":\"Tokyo\"}"), builder.result().tool_calls[0].arguments); + assert_equals(std::string(""), builder.result().content); + assert_equals(std::string(""), builder.result().reasoning_content); +} + static void test_with_args(const std::string & input, const std::string & expected, bool parse_as_partial = true, bool is_partial = true) { common_chat_msg_parser builder(input, parse_as_partial, {}); auto js = builder.try_consume_json_with_dumped_args({{"args"}}, {}); @@ -347,6 +366,8 @@ static void test_positions() { assert_throws([&]() { builder.finish(); }); assert_equals(0, builder.pos()); + test_deepseek_v3_1_tool_calls(); + builder.move_to(builder.input().size()); builder.finish(); } From 52d54886a2d87dff3adfc0439c4479a0c34df3df Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Mon, 25 Aug 2025 15:15:31 +0000 Subject: [PATCH 20/53] Passing test --- tests/test-chat-parser.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test-chat-parser.cpp b/tests/test-chat-parser.cpp index 57d159c16975a..6cf834ceff75c 100644 --- a/tests/test-chat-parser.cpp +++ b/tests/test-chat-parser.cpp @@ -210,17 +210,17 @@ static void test_deepseek_v3_1_tool_calls() { /* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, /* .reasoning_in_content = */ false, - /* .thinking_forced_open = */ true, + /* .thinking_forced_open = */ false, /* .parse_tool_calls = */ true, }; const std::string input = "<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time<｜tool▁sep｜>{\"city\": \"Tokyo\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>"; - common_chat_msg_parser builder(input, false, syntax); - assert_equals(1u, builder.result().tool_calls.size()); - assert_equals(std::string("get_time"), builder.result().tool_calls[0].name); + auto msg = common_chat_parse(input, false, syntax); + assert_equals(static_cast(1), msg.tool_calls.size()); + assert_equals(std::string("get_time"), msg.tool_calls[0].name); // JSON arguments are dumped without spaces - assert_equals(std::string("{\"city\":\"Tokyo\"}"), builder.result().tool_calls[0].arguments); - assert_equals(std::string(""), builder.result().content); - assert_equals(std::string(""), builder.result().reasoning_content); + assert_equals(std::string("{\"city\":\"Tokyo\"}"), msg.tool_calls[0].arguments); + assert_equals(std::string(""), msg.content); + assert_equals(std::string(""), msg.reasoning_content); } static void test_with_args(const std::string & input, const std::string & expected, bool parse_as_partial = true, bool is_partial = true) { From a839be7c213c1761b9707f9423436122464ca3bc Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Mon, 25 Aug 2025 18:01:38 +0000 Subject: [PATCH 21/53] Add tests for both happy path and broken fenced DeepSeek V3.1 tool call variants. --- tests/test-chat-parser.cpp | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/tests/test-chat-parser.cpp b/tests/test-chat-parser.cpp index 6cf834ceff75c..d1fa3b5e404d9 100644 --- a/tests/test-chat-parser.cpp +++ b/tests/test-chat-parser.cpp @@ -204,8 +204,17 @@ static void test_deepseek_v3_1() { } } +template +static void assert_equals(const char* label, const T& expected, const T& actual) { + if (!(expected == actual)) { + std::ostringstream oss; oss << label << "\nExpected: " << expected << "\nActual: " << actual; + throw std::runtime_error(oss.str()); + } +} static void test_deepseek_v3_1_tool_calls() { + // variant: happy path for when it works as the model card says it should + const char* variant = "simple"; common_chat_syntax syntax = { /* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, @@ -215,12 +224,24 @@ static void test_deepseek_v3_1_tool_calls() { }; const std::string input = "<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time<｜tool▁sep｜>{\"city\": \"Tokyo\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>"; auto msg = common_chat_parse(input, false, syntax); - assert_equals(static_cast(1), msg.tool_calls.size()); - assert_equals(std::string("get_time"), msg.tool_calls[0].name); + assert_equals(variant, static_cast(1), msg.tool_calls.size()); + assert_equals(variant, std::string("get_time"), msg.tool_calls[0].name); // JSON arguments are dumped without spaces - assert_equals(std::string("{\"city\":\"Tokyo\"}"), msg.tool_calls[0].arguments); - assert_equals(std::string(""), msg.content); - assert_equals(std::string(""), msg.reasoning_content); + assert_equals(variant, std::string("{\"city\":\"Tokyo\"}"), msg.tool_calls[0].arguments); + assert_equals(variant, std::string(""), msg.content); + assert_equals(variant, std::string(""), msg.reasoning_content); + + // variant: function + fenced JSON + { + const char* variant = "fenced"; + const std::string in = "<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_time\n```json\n{\"city\": \"Tokyo\"}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>"; + auto m = common_chat_parse(in, false, syntax); + assert_equals(variant, 1, m.tool_calls.size()); + assert_equals(variant, std::string("get_time"), m.tool_calls[0].name); + assert_equals(variant, std::string("{\"city\":\"Tokyo\"}"), m.tool_calls[0].arguments); + assert_equals(variant, std::string(""), m.content); + assert_equals(variant, std::string(""), m.reasoning_content); + } } static void test_with_args(const std::string & input, const std::string & expected, bool parse_as_partial = true, bool is_partial = true) { @@ -366,8 +387,6 @@ static void test_positions() { assert_throws([&]() { builder.finish(); }); assert_equals(0, builder.pos()); - test_deepseek_v3_1_tool_calls(); - builder.move_to(builder.input().size()); builder.finish(); } @@ -387,6 +406,7 @@ int main() { test_reasoning(); test_regex(); test_deepseek_v3_1(); + test_deepseek_v3_1_tool_calls(); std::cout << "All tests passed!\n"; return 0; } From 6ade60ec91775533f363ce0c70b9b3b82da05711 Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Mon, 25 Aug 2025 18:02:59 +0000 Subject: [PATCH 22/53] Passing DeepSeek V3.1 tool call tests, but model is not working. --- common/chat.cpp | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 4fd013fcf5784..5ae58ce4d90b6 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -689,6 +689,7 @@ static void parse_json_tool_calls( : function_regex ? builder.try_find_regex(*function_regex, from) : std::nullopt; + if (res) { std::string name; if (get_function_name) { @@ -703,7 +704,8 @@ static void parse_json_tool_calls( from = res->groups[0].begin + 1; continue; } - from = std::string::npos; + builder.move_to(res->groups[0].end); + from = builder.pos(); auto maybe_raw_python = name == "python" && allow_raw_python; if (builder.input()[builder.pos()] == '{' || !maybe_raw_python) { @@ -712,8 +714,10 @@ static void parse_json_tool_calls( throw common_chat_msg_partial_exception("incomplete tool call"); } builder.consume_regex(close_regex); + from = builder.pos(); // continue after this call + continue; } - continue; + throw common_chat_msg_partial_exception("incomplete tool call"); } if (maybe_raw_python) { auto arguments = wrap_code_as_arguments(builder, builder.consume_rest()); @@ -727,6 +731,8 @@ static void parse_json_tool_calls( break; } if (block_close) { + // ensure we’re right after the last call header/close + if (from != std::string::npos) builder.move_to(from); builder.consume_regex(*block_close); } builder.consume_spaces(); @@ -734,12 +740,16 @@ static void parse_json_tool_calls( }; if (block_open) { if (auto res = builder.try_find_regex(*block_open)) { + builder.move_to(res->groups[0].end); // consume opener parse_tool_calls(); + return; } else { builder.add_content(builder.consume_rest()); + return; } } else { parse_tool_calls(); + return; } } @@ -1417,8 +1427,9 @@ static void common_chat_parse_deepseek_v3_1(common_chat_msg_parser & builder) { } // <｜tool▁call▁begin｜>NAME<｜tool▁sep｜>JSON<｜tool▁call▁end｜> - static const common_regex function_regex("<｜tool▁call▁begin｜>([^\\n<]+)<｜tool▁sep｜>"); - static const common_regex close_regex("<｜tool▁call▁end｜>"); + static const common_regex function_regex("(?:<｜tool▁call▁begin｜>)?(?:function<｜tool▁sep｜>)?([^\\n<]+)(?:\\n```json\\n|<｜tool▁sep｜>)"); + + static const common_regex close_regex("(?:[\\n]*```[\\s\\r\\n]*)?<｜tool▁call▁end｜>"); static const common_regex tool_calls_begin("(?:<｜tool▁calls▁begin｜>|<｜tool_calls_begin｜>|<｜tool calls begin｜>|<｜tool\\\\_calls\\\\_begin｜>|<｜tool▁calls｜>)"); static const common_regex tool_calls_end("<｜tool▁calls▁end｜>"); LOG_DBG("%s: parse_tool_calls\n", __func__); From 79d4812f433b41f5deb714b446c994a5034a99d7 Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Tue, 26 Aug 2025 01:11:24 +0000 Subject: [PATCH 23/53] Revert assistance response prefill change. Not my monkeys. --- tools/server/utils.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/server/utils.hpp b/tools/server/utils.hpp index f6236ba248928..f3dfc8225da4d 100644 --- a/tools/server/utils.hpp +++ b/tools/server/utils.hpp @@ -782,7 +782,7 @@ static json oaicompat_chat_params_parse( /* TODO: test this properly */ inputs.reasoning_format = COMMON_REASONING_FORMAT_NONE; - if (inputs.enable_thinking) { + if ( (!inputs.enable_thinking) || inputs.chat_template_kwargs.find("enable_thinking") != inputs.chat_template_kwargs.end()) { throw std::runtime_error("Assistant response prefill is incompatible with enable_thinking."); } From 36b047cf4e5f0f056fb834db50d976e8fd9f739a Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Tue, 26 Aug 2025 01:12:07 +0000 Subject: [PATCH 24/53] Add fenced_thinking unit test variant. Passes, but thinking tool calling still isn't working for some reason. --- common/chat.cpp | 49 +++++++++++++++++++++----------------- tests/test-chat-parser.cpp | 19 +++++++++++++++ 2 files changed, 46 insertions(+), 22 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 5ae58ce4d90b6..92a046b3a8a9c 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1405,13 +1405,38 @@ static void common_chat_parse_deepseek_r1(common_chat_msg_parser & builder) { tool_calls_end); } +static void common_chat_parse_deepseek_v3_1_content(common_chat_msg_parser & builder) { + static const common_regex function_regex("(?:<｜tool▁call▁begin｜>)?(?:function<｜tool▁sep｜>)?([^\\n<]+)(?:\\n```json\\n|<｜tool▁sep｜>)"); + + static const common_regex close_regex("(?:[\\n]*```[\\s\\r\\n]*)?<｜tool▁call▁end｜>"); + static const common_regex tool_calls_begin("(?:<｜tool▁calls▁begin｜>|<｜tool_calls_begin｜>|<｜tool calls begin｜>|<｜tool\\\\_calls\\\\_begin｜>|<｜tool▁calls｜>)"); + static const common_regex tool_calls_end("<｜tool▁calls▁end｜>"); + + if (!builder.syntax().parse_tool_calls) { + LOG_DBG("%s: not parse_tool_calls\n", __func__); + builder.add_content(builder.consume_rest()); + return; + } + + LOG_DBG("%s: parse_tool_calls\n", __func__); + + parse_json_tool_calls( + builder, + /* block_open= */ tool_calls_begin, + /* function_regex_start_only= */ std::nullopt, + function_regex, + close_regex, + tool_calls_end); +} + static void common_chat_parse_deepseek_v3_1(common_chat_msg_parser & builder) { // DeepSeek V3.1 outputs reasoning content between "" and "" tags, followed by regular content // First try to parse using the standard reasoning parsing method if (builder.try_parse_reasoning("", "")) { // If reasoning was parsed successfully, the remaining content is regular content LOG_DBG("%s: parsed reasoning, adding content\n", __func__); - builder.add_content(builder.consume_rest()); + // <｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>NAME\n```json\nJSON\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜> + common_chat_parse_deepseek_v3_1_content(builder); } else { // If no reasoning tags found, check if we should treat everything as reasoning if (builder.syntax().thinking_forced_open) { @@ -1419,28 +1444,8 @@ static void common_chat_parse_deepseek_v3_1(common_chat_msg_parser & builder) { LOG_DBG("%s: thinking_forced_open, adding reasoning content\n", __func__); builder.add_reasoning_content(builder.consume_rest()); } else { - // Tool calls are support in non-thinking mode - if (!builder.syntax().parse_tool_calls) { - LOG_DBG("%s: not parse_tool_calls\n", __func__); - builder.add_content(builder.consume_rest()); - return; - } - // <｜tool▁call▁begin｜>NAME<｜tool▁sep｜>JSON<｜tool▁call▁end｜> - static const common_regex function_regex("(?:<｜tool▁call▁begin｜>)?(?:function<｜tool▁sep｜>)?([^\\n<]+)(?:\\n```json\\n|<｜tool▁sep｜>)"); - - static const common_regex close_regex("(?:[\\n]*```[\\s\\r\\n]*)?<｜tool▁call▁end｜>"); - static const common_regex tool_calls_begin("(?:<｜tool▁calls▁begin｜>|<｜tool_calls_begin｜>|<｜tool calls begin｜>|<｜tool\\\\_calls\\\\_begin｜>|<｜tool▁calls｜>)"); - static const common_regex tool_calls_end("<｜tool▁calls▁end｜>"); - LOG_DBG("%s: parse_tool_calls\n", __func__); - - parse_json_tool_calls( - builder, - /* block_open= */ tool_calls_begin, - /* function_regex_start_only= */ std::nullopt, - function_regex, - close_regex, - tool_calls_end); + common_chat_parse_deepseek_v3_1_content(builder); } } } diff --git a/tests/test-chat-parser.cpp b/tests/test-chat-parser.cpp index d1fa3b5e404d9..a0ab83a9f98d0 100644 --- a/tests/test-chat-parser.cpp +++ b/tests/test-chat-parser.cpp @@ -242,6 +242,25 @@ static void test_deepseek_v3_1_tool_calls() { assert_equals(variant, std::string(""), m.content); assert_equals(variant, std::string(""), m.reasoning_content); } + + // variant: function + fenced JSON + thinking open + { + common_chat_syntax syntax = { + /* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, + /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, + /* .reasoning_in_content = */ false, + /* .thinking_forced_open = */ true, + /* .parse_tool_calls = */ true, + }; + const char* variant = "fenced_thinking"; + const std::string in = "REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_time\n```json\n{\"city\": \"Tokyo\"}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>"; + auto m = common_chat_parse(in, false, syntax); + assert_equals(variant, 1, m.tool_calls.size()); + assert_equals(variant, std::string("get_time"), m.tool_calls[0].name); + assert_equals(variant, std::string("{\"city\":\"Tokyo\"}"), m.tool_calls[0].arguments); + assert_equals(variant, std::string(""), m.content); + assert_equals(variant, std::string("REASONING"), m.reasoning_content); + } } static void test_with_args(const std::string & input, const std::string & expected, bool parse_as_partial = true, bool is_partial = true) { From bdfa87f7b18be58d58889acecff0f598268a8325 Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Tue, 26 Aug 2025 02:17:59 +0000 Subject: [PATCH 25/53] Tests pass in reasoning mode. Also e2e tool test passes. --- common/chat.cpp | 16 ++++++++++++++++ tests/test-chat-parser.cpp | 18 ++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/common/chat.cpp b/common/chat.cpp index 92a046b3a8a9c..271c18893ae72 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1432,6 +1432,22 @@ static void common_chat_parse_deepseek_v3_1_content(common_chat_msg_parser & bui static void common_chat_parse_deepseek_v3_1(common_chat_msg_parser & builder) { // DeepSeek V3.1 outputs reasoning content between "" and "" tags, followed by regular content // First try to parse using the standard reasoning parsing method + LOG_DBG("%s: thinking_forced_open: %s\n", __func__, std::to_string(builder.syntax().thinking_forced_open).c_str()); + + bool has_reasoning = false; + auto header_start_pos = builder.pos(); + if (auto res = builder.try_find_literal("")) { + has_reasoning = true; + } + if (auto res = builder.try_find_literal("")) { + has_reasoning = true; + } + builder.move_to(header_start_pos); + if (!has_reasoning && builder.syntax().thinking_forced_open) { + LOG_DBG("%s: edge case no reasoning, adding content\n", __func__); + common_chat_parse_deepseek_v3_1_content(builder); + return; + } if (builder.try_parse_reasoning("", "")) { // If reasoning was parsed successfully, the remaining content is regular content LOG_DBG("%s: parsed reasoning, adding content\n", __func__); diff --git a/tests/test-chat-parser.cpp b/tests/test-chat-parser.cpp index a0ab83a9f98d0..72405927faa7b 100644 --- a/tests/test-chat-parser.cpp +++ b/tests/test-chat-parser.cpp @@ -213,6 +213,7 @@ static void assert_equals(const char* label, const T& expected, const T& actual) } static void test_deepseek_v3_1_tool_calls() { + //common_log_set_verbosity_thold(LOG_DEFAULT_DEBUG); // variant: happy path for when it works as the model card says it should const char* variant = "simple"; common_chat_syntax syntax = { @@ -261,6 +262,23 @@ static void test_deepseek_v3_1_tool_calls() { assert_equals(variant, std::string(""), m.content); assert_equals(variant, std::string("REASONING"), m.reasoning_content); } + + // variant: thinking forced open + missing reasoning + no tool calls + { + common_chat_syntax syntax = { + /* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, + /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, + /* .reasoning_in_content = */ false, + /* .thinking_forced_open = */ true, + /* .parse_tool_calls = */ true, + }; + const char* variant = "thinking_forced_open_missing_reasoning_no_tool_calls"; + const std::string in = "CONTENT"; + auto m = common_chat_parse(in, false, syntax); + assert_equals(variant, 0, m.tool_calls.size()); + assert_equals(variant, std::string("CONTENT"), m.content); + assert_equals(variant, std::string(""), m.reasoning_content); + } } static void test_with_args(const std::string & input, const std::string & expected, bool parse_as_partial = true, bool is_partial = true) { From 0e367617c73140cd12c636a845a0137706bec2ec Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Tue, 26 Aug 2025 02:49:36 +0000 Subject: [PATCH 26/53] Make a copy of the parse_json_tool_calls function for deepseek-v3.1 so as to not accidentally introduce regressions. --- common/chat.cpp | 81 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 79 insertions(+), 2 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 271c18893ae72..8b3b74e3fa645 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -670,7 +670,7 @@ static std::string wrap_code_as_arguments(common_chat_msg_parser & builder, cons * Takes a prefix regex that must have 1 group to capture the function name, a closing suffix, and expects json parameters in between. * Aggregates the prefix, suffix and in-between text into the content. */ -static void parse_json_tool_calls( +static void parse_json_tool_calls_deepseek_v3_1( common_chat_msg_parser & builder, const std::optional & block_open, const std::optional & function_regex_start_only, @@ -753,6 +753,83 @@ static void parse_json_tool_calls( } } +/** + * Takes a prefix regex that must have 1 group to capture the function name, a closing suffix, and expects json parameters in between. + * Aggregates the prefix, suffix and in-between text into the content. + */ +static void parse_json_tool_calls( + common_chat_msg_parser & builder, + const std::optional & block_open, + const std::optional & function_regex_start_only, + const std::optional & function_regex, + const common_regex & close_regex, + const std::optional & block_close, + bool allow_raw_python = false, + const std::function & get_function_name = nullptr) { + + auto parse_tool_calls = [&]() { + size_t from = std::string::npos; + auto first = true; + while (true) { + auto res = function_regex_start_only && first + ? builder.try_consume_regex(*function_regex_start_only) + : function_regex + ? builder.try_find_regex(*function_regex, from) + : std::nullopt; + if (res) { + std::string name; + if (get_function_name) { + name = get_function_name(*res); + } else { + GGML_ASSERT(res->groups.size() == 2); + name = builder.str(res->groups[1]); + } + first = false; + if (name.empty()) { + // get_function_name signalled us that we should skip this match and treat it as content. + from = res->groups[0].begin + 1; + continue; + } + from = std::string::npos; + + auto maybe_raw_python = name == "python" && allow_raw_python; + if (builder.input()[builder.pos()] == '{' || !maybe_raw_python) { + if (auto arguments = builder.try_consume_json_with_dumped_args({{}})) { + if (!builder.add_tool_call(name, "", arguments->value) || arguments->is_partial) { + throw common_chat_msg_partial_exception("incomplete tool call"); + } + builder.consume_regex(close_regex); + } + continue; + } + if (maybe_raw_python) { + auto arguments = wrap_code_as_arguments(builder, builder.consume_rest()); + if (!builder.add_tool_call(name, "", arguments)) { + throw common_chat_msg_partial_exception("incomplete tool call"); + } + return; + } + throw common_chat_msg_partial_exception("incomplete tool call"); + } + break; + } + if (block_close) { + builder.consume_regex(*block_close); + } + builder.consume_spaces(); + builder.add_content(builder.consume_rest()); + }; + if (block_open) { + if (auto res = builder.try_find_regex(*block_open)) { + parse_tool_calls(); + } else { + builder.add_content(builder.consume_rest()); + } + } else { + parse_tool_calls(); + } +} + static void parse_prefixed_json_tool_call_array(common_chat_msg_parser & builder, const common_regex & prefix, size_t rstrip_prefix = 0) { static const std::vector> args_paths = {{"arguments"}}; if (auto res = builder.try_find_regex(prefix)) { @@ -1420,7 +1497,7 @@ static void common_chat_parse_deepseek_v3_1_content(common_chat_msg_parser & bui LOG_DBG("%s: parse_tool_calls\n", __func__); - parse_json_tool_calls( + parse_json_tool_calls_deepseek_v3_1( builder, /* block_open= */ tool_calls_begin, /* function_regex_start_only= */ std::nullopt, From ab22c767bbc7695ffb80b16d2d06527f1d1a8dc4 Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Tue, 26 Aug 2025 12:20:32 +0000 Subject: [PATCH 27/53] Fix thinking_forced_open logic. tool calling broken. Need to add another test case. --- common/chat.cpp | 22 ++++++---------------- tests/test-chat-parser.cpp | 23 +++++++++++++++++++++-- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 8b3b74e3fa645..8a2e8164ca167 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1415,8 +1415,12 @@ static common_chat_params common_chat_params_init_deepseek_v3_1(const common_cha additional_context); data.prompt = prompt; data.format = COMMON_CHAT_FORMAT_DEEPSEEK_V3_1; - if (inputs.enable_thinking) { - data.thinking_forced_open = true; + if (string_ends_with(data.prompt, "\n")) { + if (!inputs.enable_thinking) { + data.prompt += ""; + } else { + data.thinking_forced_open = true; + } } if (inputs.tools.is_array() && !inputs.tools.empty()) { data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED && inputs.json_schema.is_null(); @@ -1511,20 +1515,6 @@ static void common_chat_parse_deepseek_v3_1(common_chat_msg_parser & builder) { // First try to parse using the standard reasoning parsing method LOG_DBG("%s: thinking_forced_open: %s\n", __func__, std::to_string(builder.syntax().thinking_forced_open).c_str()); - bool has_reasoning = false; - auto header_start_pos = builder.pos(); - if (auto res = builder.try_find_literal("")) { - has_reasoning = true; - } - if (auto res = builder.try_find_literal("")) { - has_reasoning = true; - } - builder.move_to(header_start_pos); - if (!has_reasoning && builder.syntax().thinking_forced_open) { - LOG_DBG("%s: edge case no reasoning, adding content\n", __func__); - common_chat_parse_deepseek_v3_1_content(builder); - return; - } if (builder.try_parse_reasoning("", "")) { // If reasoning was parsed successfully, the remaining content is regular content LOG_DBG("%s: parsed reasoning, adding content\n", __func__); diff --git a/tests/test-chat-parser.cpp b/tests/test-chat-parser.cpp index 72405927faa7b..a4eb2f77a26e9 100644 --- a/tests/test-chat-parser.cpp +++ b/tests/test-chat-parser.cpp @@ -263,7 +263,7 @@ static void test_deepseek_v3_1_tool_calls() { assert_equals(variant, std::string("REASONING"), m.reasoning_content); } - // variant: thinking forced open + missing reasoning + no tool calls + // variant: thinking forced open + tool call in reasoning content + function + fenced JSON { common_chat_syntax syntax = { /* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, @@ -272,7 +272,26 @@ static void test_deepseek_v3_1_tool_calls() { /* .thinking_forced_open = */ true, /* .parse_tool_calls = */ true, }; - const char* variant = "thinking_forced_open_missing_reasoning_no_tool_calls"; + const char* variant = "thinking_forced_open_tool_call_in_reasoning_fenced_thinking"; + const std::string in = "REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time2<｜tool▁sep｜>{\"city\": \"Tokyo2\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_time\n```json\n{\"city\": \"Tokyo\"}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>"; + auto m = common_chat_parse(in, false, syntax); + assert_equals(variant, 1, m.tool_calls.size()); + assert_equals(variant, std::string("get_time"), m.tool_calls[0].name); + assert_equals(variant, std::string("{\"city\":\"Tokyo\"}"), m.tool_calls[0].arguments); + assert_equals(variant, std::string(""), m.content); + assert_equals(variant, std::string("REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time2<｜tool▁sep｜>{\"city\": \"Tokyo2\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>REASONING"), m.reasoning_content); + } + + // variant: thinking not forced open + missing reasoning + no tool calls + { + common_chat_syntax syntax = { + /* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, + /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, + /* .reasoning_in_content = */ false, + /* .thinking_forced_open = */ false, + /* .parse_tool_calls = */ true, + }; + const char* variant = "thinking_not_forced_open_missing_reasoning_no_tool_calls"; const std::string in = "CONTENT"; auto m = common_chat_parse(in, false, syntax); assert_equals(variant, 0, m.tool_calls.size()); From 4a2d17d986bb94ea2b0fb7e3944e4e1d6003d378 Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Tue, 26 Aug 2025 14:22:16 +0000 Subject: [PATCH 28/53] That's what I get for cargo culting a newline. --- common/chat.cpp | 2 +- tests/test-chat-parser.cpp | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/common/chat.cpp b/common/chat.cpp index 8a2e8164ca167..4c7b74aa4cf03 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1415,7 +1415,7 @@ static common_chat_params common_chat_params_init_deepseek_v3_1(const common_cha additional_context); data.prompt = prompt; data.format = COMMON_CHAT_FORMAT_DEEPSEEK_V3_1; - if (string_ends_with(data.prompt, "\n")) { + if (string_ends_with(data.prompt, "")) { if (!inputs.enable_thinking) { data.prompt += ""; } else { diff --git a/tests/test-chat-parser.cpp b/tests/test-chat-parser.cpp index a4eb2f77a26e9..c825a50530bff 100644 --- a/tests/test-chat-parser.cpp +++ b/tests/test-chat-parser.cpp @@ -282,6 +282,23 @@ static void test_deepseek_v3_1_tool_calls() { assert_equals(variant, std::string("REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time2<｜tool▁sep｜>{\"city\": \"Tokyo2\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>REASONING"), m.reasoning_content); } + // variant: thinking forced open + tool call in reasoning content + no closing think + { + common_chat_syntax syntax = { + /* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, + /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, + /* .reasoning_in_content = */ false, + /* .thinking_forced_open = */ true, + /* .parse_tool_calls = */ true, + }; + const char* variant = "thinking_forced_open_tool_call_in_reasoning_no_closing_think"; + const std::string in = "REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time2<｜tool▁sep｜>{\"city\": \"Tokyo2\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>REASONING"; + auto m = common_chat_parse(in, false, syntax); + assert_equals(variant, 0, m.tool_calls.size()); + assert_equals(variant, std::string(""), m.content); + assert_equals(variant, std::string("REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time2<｜tool▁sep｜>{\"city\": \"Tokyo2\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>REASONING"), m.reasoning_content); + } + // variant: thinking not forced open + missing reasoning + no tool calls { common_chat_syntax syntax = { From b2d57cefc1f1793b88e1f735d8bc11b55c2f7cb2 Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Wed, 27 Aug 2025 04:17:53 +0000 Subject: [PATCH 29/53] Add multi tool call test for deepseek v3.1 non-reasoning --- tests/test-chat-parser.cpp | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test-chat-parser.cpp b/tests/test-chat-parser.cpp index c825a50530bff..59bff412861a7 100644 --- a/tests/test-chat-parser.cpp +++ b/tests/test-chat-parser.cpp @@ -262,6 +262,27 @@ static void test_deepseek_v3_1_tool_calls() { assert_equals(variant, std::string(""), m.content); assert_equals(variant, std::string("REASONING"), m.reasoning_content); } + // variant: simple + multiple tool calls + { + common_chat_syntax syntax = { + /* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, + /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, + /* .reasoning_in_content = */ false, + /* .thinking_forced_open = */ false, + /* .parse_tool_calls = */ true, + }; + const char* variant = "simple_multiple_tool_calls"; + const std::string in = "CONTENT<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time<｜tool▁sep｜>{\"city\": \"Paris\"}<｜tool▁call▁end｜><｜tool▁call▁begin｜>get_weather<｜tool▁sep｜>{\"city\": \"Paris\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>"; + auto m = common_chat_parse(in, false, syntax); + assert_equals(variant, 2, m.tool_calls.size()); + assert_equals(variant, std::string("get_time"), m.tool_calls[0].name); + assert_equals(variant, std::string("{\"city\":\"Paris\"}"), m.tool_calls[0].arguments); + assert_equals(variant, std::string("get_weather"), m.tool_calls[1].name); + assert_equals(variant, std::string("{\"city\":\"Paris\"}"), m.tool_calls[1].arguments); + assert_equals(variant, std::string("CONTENT"), m.content); + assert_equals(variant, std::string(""), m.reasoning_content); + } + // variant: thinking forced open + tool call in reasoning content + function + fenced JSON { From 7dc19e83d1a272d3eaef71477ed4bc1eeea185d4 Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Fri, 29 Aug 2025 20:34:33 +0000 Subject: [PATCH 30/53] Move test, remove .gitignore change --- tests/.gitignore | 1 - tests/test-chat-parser.cpp | 36 ++++++++++++++++++------------------ 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/tests/.gitignore b/tests/.gitignore index 93429261acb3a..620a48ee4449b 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1,5 +1,4 @@ * !*.* *.o -*.swp ggml-common.h diff --git a/tests/test-chat-parser.cpp b/tests/test-chat-parser.cpp index 59bff412861a7..7ca5cb71644ba 100644 --- a/tests/test-chat-parser.cpp +++ b/tests/test-chat-parser.cpp @@ -45,6 +45,23 @@ static void assert_throws(const std::function & fn, const std::string & throw std::runtime_error("Exception was expected but not thrown"); } +static void test_reasoning_deepseek_v3_1() { + // Test DeepSeek V3.1 parsing - reasoning content followed by "" and then regular content + { + common_chat_syntax syntax = { + /* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, + /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, + /* .reasoning_in_content = */ false, + /* .thinking_forced_open = */ true, + /* .parse_tool_calls = */ true, + }; + common_chat_msg_parser builder("REASONINGok", /* is_partial= */ false, syntax); + assert_equals(true, builder.try_parse_reasoning("", "")); + assert_equals(std::string("REASONING"), builder.result().reasoning_content); + assert_equals(std::string("ok"), builder.consume_rest()); + } +} + static void test_reasoning() { { common_chat_msg_parser builder("CogitoErgo sum", /* is_partial= */ false, { @@ -99,6 +116,7 @@ static void test_reasoning() { assert_equals("Cogito", builder.result().content); assert_equals("Ergo sum", builder.consume_rest()); } + test_reasoning_deepseek_v3_1(); } static void test_regex() { @@ -187,23 +205,6 @@ static void test(const std::string & input, bool is_partial, const std::vectorvalue.get() : js->value.dump()); } -static void test_deepseek_v3_1() { - // Test DeepSeek V3.1 parsing - reasoning content followed by "" and then regular content - { - common_chat_syntax syntax = { - /* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, - /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, - /* .reasoning_in_content = */ false, - /* .thinking_forced_open = */ true, - /* .parse_tool_calls = */ true, - }; - common_chat_msg_parser builder("REASONINGok", /* is_partial= */ false, syntax); - assert_equals(true, builder.try_parse_reasoning("", "")); - assert_equals(std::string("REASONING"), builder.result().reasoning_content); - assert_equals(std::string("ok"), builder.consume_rest()); - } -} - template static void assert_equals(const char* label, const T& expected, const T& actual) { if (!(expected == actual)) { @@ -499,7 +500,6 @@ int main() { test_json_with_dumped_args(); test_reasoning(); test_regex(); - test_deepseek_v3_1(); test_deepseek_v3_1_tool_calls(); std::cout << "All tests passed!\n"; return 0; From 380146e6514ad815b3856718f8bdd817c257634b Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Fri, 29 Aug 2025 20:51:23 +0000 Subject: [PATCH 31/53] Place deepseek-v3.1 reasoning test directly into existing reasoning function per CISC's request. --- tests/test-chat-parser.cpp | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/tests/test-chat-parser.cpp b/tests/test-chat-parser.cpp index 7ca5cb71644ba..8a68cecc82c37 100644 --- a/tests/test-chat-parser.cpp +++ b/tests/test-chat-parser.cpp @@ -45,23 +45,6 @@ static void assert_throws(const std::function & fn, const std::string & throw std::runtime_error("Exception was expected but not thrown"); } -static void test_reasoning_deepseek_v3_1() { - // Test DeepSeek V3.1 parsing - reasoning content followed by "" and then regular content - { - common_chat_syntax syntax = { - /* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, - /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, - /* .reasoning_in_content = */ false, - /* .thinking_forced_open = */ true, - /* .parse_tool_calls = */ true, - }; - common_chat_msg_parser builder("REASONINGok", /* is_partial= */ false, syntax); - assert_equals(true, builder.try_parse_reasoning("", "")); - assert_equals(std::string("REASONING"), builder.result().reasoning_content); - assert_equals(std::string("ok"), builder.consume_rest()); - } -} - static void test_reasoning() { { common_chat_msg_parser builder("CogitoErgo sum", /* is_partial= */ false, { @@ -116,7 +99,20 @@ static void test_reasoning() { assert_equals("Cogito", builder.result().content); assert_equals("Ergo sum", builder.consume_rest()); } - test_reasoning_deepseek_v3_1(); + // Test DeepSeek V3.1 parsing - reasoning content followed by "" and then regular content + { + common_chat_syntax syntax = { + /* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, + /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, + /* .reasoning_in_content = */ false, + /* .thinking_forced_open = */ true, + /* .parse_tool_calls = */ true, + }; + common_chat_msg_parser builder("REASONINGok", /* is_partial= */ false, syntax); + assert_equals(true, builder.try_parse_reasoning("", "")); + assert_equals(std::string("REASONING"), builder.result().reasoning_content); + assert_equals(std::string("ok"), builder.consume_rest()); + } } static void test_regex() { From 905670711e64c8d65c3b8ec1e8842b9164f4764d Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Fri, 29 Aug 2025 20:53:39 +0000 Subject: [PATCH 32/53] Address whitespace CI failure. --- common/chat.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index d382a7754598f..c4ffc89e3e38a 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1409,9 +1409,9 @@ static common_chat_params common_chat_params_init_deepseek_v3_1(const common_cha {"thinking", inputs.enable_thinking}, }; - auto prompt = apply(tmpl, inputs, + auto prompt = apply(tmpl, inputs, /* messages_override= */ inputs.messages, - /* tools_override= */ std::nullopt, + /* tools_override= */ std::nullopt, additional_context); data.prompt = prompt; data.format = COMMON_CHAT_FORMAT_DEEPSEEK_V3_1; @@ -2321,7 +2321,7 @@ static common_chat_params common_chat_templates_apply_jinja( } // DeepSeek V3.1: detect based on specific patterns in the template - if (src.find("message['prefix'] is defined and message['prefix'] and thinking") != std::string::npos && + if (src.find("message['prefix'] is defined and message['prefix'] and thinking") != std::string::npos && params.json_schema.is_null()) { return common_chat_params_init_deepseek_v3_1(tmpl, params); } From a406d6a2d17db32f69b9105ded88f471180e8622 Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Sat, 30 Aug 2025 03:28:31 +0000 Subject: [PATCH 33/53] Merge two assert_equals per CISC's request. --- tests/test-chat-parser.cpp | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/tests/test-chat-parser.cpp b/tests/test-chat-parser.cpp index 8a68cecc82c37..6cf44afa4191b 100644 --- a/tests/test-chat-parser.cpp +++ b/tests/test-chat-parser.cpp @@ -14,14 +14,19 @@ #include "log.h" #include "regex-partial.h" +template +static void assert_equals(const char* label, const T& expected, const T& actual) { + if (expected != actual){ + std::ostringstream oss; + if (label && *label) oss << label << '\n'; + oss << "Expected: " << expected << "\nActual: " << actual; + throw std::runtime_error(oss.str()); + } +} + template static void assert_equals(const T & expected, const T & actual) { - if (expected != actual) { - std::cerr << "Expected: " << expected << std::endl; - std::cerr << "Actual: " << actual << std::endl; - std::cerr << std::flush; - throw std::runtime_error("Test failed"); - } + assert_equals("", expected, actual); } static void assert_equals(const char * expected, const std::string & actual) { return assert_equals(expected, actual); @@ -201,14 +206,6 @@ static void test(const std::string & input, bool is_partial, const std::vectorvalue.get() : js->value.dump()); } -template -static void assert_equals(const char* label, const T& expected, const T& actual) { - if (!(expected == actual)) { - std::ostringstream oss; oss << label << "\nExpected: " << expected << "\nActual: " << actual; - throw std::runtime_error(oss.str()); - } -} - static void test_deepseek_v3_1_tool_calls() { //common_log_set_verbosity_thold(LOG_DEFAULT_DEBUG); // variant: happy path for when it works as the model card says it should From ec984da15252a7527cf47b886eba315d3eda2ae9 Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Sat, 30 Aug 2025 04:48:59 +0000 Subject: [PATCH 34/53] Add DeepSeek-V3.1 tests to tests/test-chat.cpp per CISC's request. --- models/templates/README.md | 1 + .../templates/deepseek-ai-DeepSeek-V3.1.jinja | 3 + tests/test-chat.cpp | 121 ++++++++++++++++++ 3 files changed, 125 insertions(+) create mode 100644 models/templates/deepseek-ai-DeepSeek-V3.1.jinja diff --git a/models/templates/README.md b/models/templates/README.md index 2e8eaa5953b86..3a649b8f4dbd9 100644 --- a/models/templates/README.md +++ b/models/templates/README.md @@ -22,4 +22,5 @@ These templates can be updated with the following commands: ./scripts/get_chat_template.py Qwen/QwQ-32B > models/templates/Qwen-QwQ-32B.jinja ./scripts/get_chat_template.py Qwen/Qwen3-0.6B > models/templates/Qwen-Qwen3-0.6B.jinja ./scripts/get_chat_template.py zai-org/GLM-4.5 > models/templates/zai-org-GLM-4.5.jinja +./scripts/get_chat_template.py deepseek-ai/DeepSeek-V3.1 > models/templates/deepseek-ai-DeepSeek-V3.1.jinja ``` diff --git a/models/templates/deepseek-ai-DeepSeek-V3.1.jinja b/models/templates/deepseek-ai-DeepSeek-V3.1.jinja new file mode 100644 index 0000000000000..e5656196a3f0f --- /dev/null +++ b/models/templates/deepseek-ai-DeepSeek-V3.1.jinja @@ -0,0 +1,3 @@ +{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% if not thinking is defined %}{% set thinking = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, system_prompt='', is_first_sp=true, is_last_user=false) %}{%- for message in messages %}{%- if message['role'] == 'system' %}{%- if ns.is_first_sp %}{% set ns.system_prompt = ns.system_prompt + message['content'] %}{% set ns.is_first_sp = false %}{%- else %}{% set ns.system_prompt = ns.system_prompt + ' + +' + message['content'] %}{%- endif %}{%- endif %}{%- endfor %}{{ bos_token }}{{ ns.system_prompt }}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{%- set ns.is_first = false -%}{%- set ns.is_last_user = true -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['tool_calls'] is defined and message['tool_calls'] is not none %}{%- if ns.is_last_user %}{{'<｜Assistant｜>'}}{%- endif %}{%- set ns.is_last_user = false -%}{%- set ns.is_first = false %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls'] %}{%- if not ns.is_first %}{%- if message['content'] is none %}{{'<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>'+ tool['function']['name'] + '<｜tool▁sep｜>' + tool['function']['arguments'] + '<｜tool▁call▁end｜>'}}{%- else %}{{message['content'] + '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['function']['name'] + '<｜tool▁sep｜>' + tool['function']['arguments'] + '<｜tool▁call▁end｜>'}}{%- endif %}{%- set ns.is_first = true -%}{%- else %}{{'<｜tool▁call▁begin｜>'+ tool['function']['name'] + '<｜tool▁sep｜>' + tool['function']['arguments'] + '<｜tool▁call▁end｜>'}}{%- endif %}{%- endfor %}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- if message['role'] == 'assistant' and (message['tool_calls'] is not defined or message['tool_calls'] is none) %}{%- if ns.is_last_user %}{{'<｜Assistant｜>'}}{%- if message['prefix'] is defined and message['prefix'] and thinking %}{{''}} {%- else %}{{''}}{%- endif %}{%- endif %}{%- set ns.is_last_user = false -%}{%- if ns.is_tool %}{{message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{%- set content = message['content'] -%}{%- if '' in content %}{%- set content = content.split('', 1)[1] -%}{%- endif %}{{content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_last_user = false -%}{%- set ns.is_tool = true -%}{{'<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endfor -%}{%- if add_generation_prompt and ns.is_last_user and not ns.is_tool %}{{'<｜Assistant｜>'}}{%- if not thinking %}{{''}}{%- else %}{{''}}{%- endif %}{% endif %} \ No newline at end of file diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index a6daa93a82b32..3e71ee0c5570c 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -1621,6 +1621,127 @@ static void test_template_output_parsers() { /* .reasoning_format = */ COMMON_REASONING_FORMAT_AUTO, })); } + { + auto tmpls = read_templates("models/templates/deepseek-ai-DeepSeek-V3.1.jinja"); + std::vector end_tokens{ "<｜end▁of▁sentence｜>" }; + + for (const auto & inputs : { inputs_no_tools, inputs_tools }) { + auto params = common_chat_templates_apply(tmpls.get(), inputs); + assert_equals(COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, params.format); + assert_equals(true, params.thinking_forced_open); + } + + test_templates(tmpls.get(), end_tokens, message_assist, tools, "Hello, world!\nWhat's up?", /* expect_grammar_triggered= */ false); + test_templates(tmpls.get(), end_tokens, message_assist_thoughts, tools, "Hello, world!\nWhat's up?", /* expect_grammar_triggered= */ false); + assert_msg_equals( + simple_assist_msg("Hello, world!\nWhat's up?", "I'm\nthinking"), + common_chat_parse( + "I'm\nthinkingHello, world!\nWhat's up?", + /* is_partial= */ false, + { + COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, + /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, + /* .reasoning_in_content = */ false, + /* .thinking_forced_open = */ true, + })); + // variant: happy path for when it works as the model card says it should + assert_msg_equals( + simple_assist_msg("", "", "get_time", "{\"city\":\"Tokyo\"}"), + common_chat_parse( + "<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time<｜tool▁sep｜>{\"city\": \"Tokyo\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>", + /* is_partial= */ false, + { + COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, + /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, + /* .reasoning_in_content = */ false, + /* .thinking_forced_open = */ false, + /* .parse_tool_calls = */ true, + })); + // variant: function + fenced JSON + assert_msg_equals( + simple_assist_msg("", "", "get_time", "{\"city\":\"Tokyo\"}"), + common_chat_parse( + "<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_time\n```json\n{\"city\": \"Tokyo\"}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>", + /* is_partial= */ false, + { + COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, + /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, + /* .reasoning_in_content = */ false, + /* .thinking_forced_open = */ false, + /* .parse_tool_calls = */ true, + })); + // variant: function + fenced JSON + thinking open + assert_msg_equals( + simple_assist_msg("", "REASONING", "get_time", "{\"city\":\"Tokyo\"}"), + common_chat_parse( + "REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_time\n```json\n{\"city\": \"Tokyo\"}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>", + /* is_partial= */ false, + { + COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, + /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, + /* .reasoning_in_content = */ false, + /* .thinking_forced_open = */ true, + /* .parse_tool_calls = */ true, + })); + // variant: simple + multiple tool calls + common_chat_msg message_assist_multiple_calls; + message_assist_multiple_calls.role = "assistant"; + message_assist_multiple_calls.content = "CONTENT"; + message_assist_multiple_calls.tool_calls.push_back({"get_time", "{\"city\":\"Paris\"}", ""}); + message_assist_multiple_calls.tool_calls.push_back({"get_weather", "{\"city\":\"Paris\"}", ""}); + assert_msg_equals( + message_assist_multiple_calls, + common_chat_parse( + "CONTENT<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time<｜tool▁sep｜>{\"city\": \"Paris\"}<｜tool▁call▁end｜><｜tool▁call▁begin｜>get_weather<｜tool▁sep｜>{\"city\": \"Paris\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>", + /* is_partial= */ false, + { + COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, + /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, + /* .reasoning_in_content = */ false, + /* .thinking_forced_open = */ false, + /* .parse_tool_calls = */ true, + })); + // variant: thinking forced open + tool call in reasoning content + function + fenced JSON + assert_msg_equals( + simple_assist_msg("", "REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time2<｜tool▁sep｜>{\"city\": \"Tokyo2\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>REASONING", "get_time", "{\"city\":\"Tokyo\"}"), + common_chat_parse( + "REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time2<｜tool▁sep｜>{\"city\": \"Tokyo2\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_time\n```json\n{\"city\": \"Tokyo\"}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>", + /* is_partial= */ false, + { + COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, + /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, + /* .reasoning_in_content = */ false, + /* .thinking_forced_open = */ true, + /* .parse_tool_calls = */ true, + })); + // variant: thinking forced open + tool call in reasoning content + no closing think + assert_msg_equals( + simple_assist_msg("", "REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time2<｜tool▁sep｜>{\"city\": \"Tokyo2\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>REASONING", "", ""), + common_chat_parse( + "REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time2<｜tool▁sep｜>{\"city\": \"Tokyo2\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>REASONING", + /* is_partial= */ false, + { + COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, + /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, + /* .reasoning_in_content = */ false, + /* .thinking_forced_open = */ true, + /* .parse_tool_calls = */ true, + })); + // variant: thinking not forced open + missing reasoning + no tool calls + assert_msg_equals( + simple_assist_msg("CONTENT", ""), + common_chat_parse( + "CONTENT", + /* is_partial= */ false, + { + COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, + /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, + /* .reasoning_in_content = */ false, + /* .thinking_forced_open = */ false, + /* .parse_tool_calls = */ true, + })); + + } } static void test_msg_diffs_compute() { From f661dbe3189138d424390c198c387078f4cc156e Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Sat, 30 Aug 2025 05:33:11 +0000 Subject: [PATCH 35/53] Merge deepseek V3.1 and regular parse_json_tool_calls() function behaviors by adding optional update_cursor argument. --- common/chat.cpp | 116 +++++++++++------------------------------------- 1 file changed, 26 insertions(+), 90 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 4774596c66be8..f350cbb65979b 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -671,7 +671,7 @@ static std::string wrap_code_as_arguments(common_chat_msg_parser & builder, cons * Takes a prefix regex that must have 1 group to capture the function name, a closing suffix, and expects json parameters in between. * Aggregates the prefix, suffix and in-between text into the content. */ -static void parse_json_tool_calls_deepseek_v3_1( +static void parse_json_tool_calls( common_chat_msg_parser & builder, const std::optional & block_open, const std::optional & function_regex_start_only, @@ -679,7 +679,8 @@ static void parse_json_tool_calls_deepseek_v3_1( const common_regex & close_regex, const std::optional & block_close, bool allow_raw_python = false, - const std::function & get_function_name = nullptr) { + const std::function & get_function_name = nullptr, + bool update_cursor = false) { auto parse_tool_calls = [&]() { size_t from = std::string::npos; @@ -705,93 +706,12 @@ static void parse_json_tool_calls_deepseek_v3_1( from = res->groups[0].begin + 1; continue; } - builder.move_to(res->groups[0].end); - from = builder.pos(); - - auto maybe_raw_python = name == "python" && allow_raw_python; - if (builder.input()[builder.pos()] == '{' || !maybe_raw_python) { - if (auto arguments = builder.try_consume_json_with_dumped_args({{}})) { - if (!builder.add_tool_call(name, "", arguments->value) || arguments->is_partial) { - throw common_chat_msg_partial_exception("incomplete tool call"); - } - builder.consume_regex(close_regex); - from = builder.pos(); // continue after this call - continue; - } - throw common_chat_msg_partial_exception("incomplete tool call"); - } - if (maybe_raw_python) { - auto arguments = wrap_code_as_arguments(builder, builder.consume_rest()); - if (!builder.add_tool_call(name, "", arguments)) { - throw common_chat_msg_partial_exception("incomplete tool call"); - } - return; - } - throw common_chat_msg_partial_exception("incomplete tool call"); - } - break; - } - if (block_close) { - // ensure we’re right after the last call header/close - if (from != std::string::npos) builder.move_to(from); - builder.consume_regex(*block_close); - } - builder.consume_spaces(); - builder.add_content(builder.consume_rest()); - }; - if (block_open) { - if (auto res = builder.try_find_regex(*block_open)) { - builder.move_to(res->groups[0].end); // consume opener - parse_tool_calls(); - return; - } else { - builder.add_content(builder.consume_rest()); - return; - } - } else { - parse_tool_calls(); - return; - } -} - -/** - * Takes a prefix regex that must have 1 group to capture the function name, a closing suffix, and expects json parameters in between. - * Aggregates the prefix, suffix and in-between text into the content. - */ -static void parse_json_tool_calls( - common_chat_msg_parser & builder, - const std::optional & block_open, - const std::optional & function_regex_start_only, - const std::optional & function_regex, - const common_regex & close_regex, - const std::optional & block_close, - bool allow_raw_python = false, - const std::function & get_function_name = nullptr) { - - auto parse_tool_calls = [&]() { - size_t from = std::string::npos; - auto first = true; - while (true) { - auto res = function_regex_start_only && first - ? builder.try_consume_regex(*function_regex_start_only) - : function_regex - ? builder.try_find_regex(*function_regex, from) - : std::nullopt; - if (res) { - std::string name; - if (get_function_name) { - name = get_function_name(*res); + if (update_cursor) { + builder.move_to(res->groups[0].end); + from = builder.pos(); } else { - GGML_ASSERT(res->groups.size() == 2); - name = builder.str(res->groups[1]); + from = std::string::npos; } - first = false; - if (name.empty()) { - // get_function_name signalled us that we should skip this match and treat it as content. - from = res->groups[0].begin + 1; - continue; - } - from = std::string::npos; auto maybe_raw_python = name == "python" && allow_raw_python; if (builder.input()[builder.pos()] == '{' || !maybe_raw_python) { @@ -800,8 +720,16 @@ static void parse_json_tool_calls( throw common_chat_msg_partial_exception("incomplete tool call"); } builder.consume_regex(close_regex); + if (update_cursor) { + from = builder.pos(); // continue after this call + continue; + } + } + if (update_cursor) { + throw common_chat_msg_partial_exception("incomplete tool call"); + } else { + continue; } - continue; } if (maybe_raw_python) { auto arguments = wrap_code_as_arguments(builder, builder.consume_rest()); @@ -815,6 +743,10 @@ static void parse_json_tool_calls( break; } if (block_close) { + if (update_cursor) { + // ensure we’re right after the last call header/close + if (from != std::string::npos) builder.move_to(from); + } builder.consume_regex(*block_close); } builder.consume_spaces(); @@ -822,6 +754,7 @@ static void parse_json_tool_calls( }; if (block_open) { if (auto res = builder.try_find_regex(*block_open)) { + if (update_cursor) builder.move_to(res->groups[0].end); // consume opener parse_tool_calls(); } else { builder.add_content(builder.consume_rest()); @@ -1502,13 +1435,16 @@ static void common_chat_parse_deepseek_v3_1_content(common_chat_msg_parser & bui LOG_DBG("%s: parse_tool_calls\n", __func__); - parse_json_tool_calls_deepseek_v3_1( + parse_json_tool_calls( builder, /* block_open= */ tool_calls_begin, /* function_regex_start_only= */ std::nullopt, function_regex, close_regex, - tool_calls_end); + tool_calls_end, + false, + nullptr, + true); } static void common_chat_parse_deepseek_v3_1(common_chat_msg_parser & builder) { From 12b013f7865eba6f77d792793ced32ea501415b7 Mon Sep 17 00:00:00 2001 From: Jesse Date: Sun, 31 Aug 2025 16:12:38 -0400 Subject: [PATCH 36/53] Update tests/test-chat-parser.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- tests/test-chat-parser.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/test-chat-parser.cpp b/tests/test-chat-parser.cpp index 6cf44afa4191b..9195a211b1ee1 100644 --- a/tests/test-chat-parser.cpp +++ b/tests/test-chat-parser.cpp @@ -14,13 +14,14 @@ #include "log.h" #include "regex-partial.h" -template -static void assert_equals(const char* label, const T& expected, const T& actual) { - if (expected != actual){ - std::ostringstream oss; - if (label && *label) oss << label << '\n'; - oss << "Expected: " << expected << "\nActual: " << actual; - throw std::runtime_error(oss.str()); +template +static void assert_equals(const std::string & label, const T & expected, const T & actual) { + if (expected != actual) { + std::cerr << label << std::endl; + std::cerr << "Expected: " << expected << std::endl; + std::cerr << "Actual: " << actual << std::endl; + std::cerr << std::flush; + throw std::runtime_error("Test failed"); } } From 800af009e87cc4542d6a376dec7d94fa3ac882f1 Mon Sep 17 00:00:00 2001 From: Jesse Date: Sun, 31 Aug 2025 16:12:51 -0400 Subject: [PATCH 37/53] Update tests/test-chat-parser.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- tests/test-chat-parser.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-chat-parser.cpp b/tests/test-chat-parser.cpp index 9195a211b1ee1..42ff042c5ec95 100644 --- a/tests/test-chat-parser.cpp +++ b/tests/test-chat-parser.cpp @@ -210,7 +210,7 @@ static void test(const std::string & input, bool is_partial, const std::vector Date: Sun, 31 Aug 2025 16:13:07 -0400 Subject: [PATCH 38/53] Update tests/test-chat-parser.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- tests/test-chat-parser.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-chat-parser.cpp b/tests/test-chat-parser.cpp index 42ff042c5ec95..33cb9131ebf7d 100644 --- a/tests/test-chat-parser.cpp +++ b/tests/test-chat-parser.cpp @@ -229,7 +229,7 @@ static void test_deepseek_v3_1_tool_calls() { // variant: function + fenced JSON { - const char* variant = "fenced"; + const std::string variant("fenced"); const std::string in = "<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_time\n```json\n{\"city\": \"Tokyo\"}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>"; auto m = common_chat_parse(in, false, syntax); assert_equals(variant, 1, m.tool_calls.size()); From 155852a6eb041dfcf73eed0f5f6f7955c15b16fc Mon Sep 17 00:00:00 2001 From: Jesse Date: Sun, 31 Aug 2025 16:13:13 -0400 Subject: [PATCH 39/53] Update tests/test-chat-parser.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- tests/test-chat-parser.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-chat-parser.cpp b/tests/test-chat-parser.cpp index 33cb9131ebf7d..843f04b3be486 100644 --- a/tests/test-chat-parser.cpp +++ b/tests/test-chat-parser.cpp @@ -248,7 +248,7 @@ static void test_deepseek_v3_1_tool_calls() { /* .thinking_forced_open = */ true, /* .parse_tool_calls = */ true, }; - const char* variant = "fenced_thinking"; + const std::string variant("fenced_thinking"); const std::string in = "REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_time\n```json\n{\"city\": \"Tokyo\"}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>"; auto m = common_chat_parse(in, false, syntax); assert_equals(variant, 1, m.tool_calls.size()); From e5878081b22e4b31ca82954276d189d6a0a01df9 Mon Sep 17 00:00:00 2001 From: Jesse Date: Sun, 31 Aug 2025 16:13:21 -0400 Subject: [PATCH 40/53] Update tests/test-chat-parser.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- tests/test-chat-parser.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-chat-parser.cpp b/tests/test-chat-parser.cpp index 843f04b3be486..996ec6e8dcc64 100644 --- a/tests/test-chat-parser.cpp +++ b/tests/test-chat-parser.cpp @@ -266,7 +266,7 @@ static void test_deepseek_v3_1_tool_calls() { /* .thinking_forced_open = */ false, /* .parse_tool_calls = */ true, }; - const char* variant = "simple_multiple_tool_calls"; + const std::string variant("simple_multiple_tool_calls"); const std::string in = "CONTENT<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time<｜tool▁sep｜>{\"city\": \"Paris\"}<｜tool▁call▁end｜><｜tool▁call▁begin｜>get_weather<｜tool▁sep｜>{\"city\": \"Paris\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>"; auto m = common_chat_parse(in, false, syntax); assert_equals(variant, 2, m.tool_calls.size()); From ac6ed1e06d2e0132707406b885e86e93d004042c Mon Sep 17 00:00:00 2001 From: Jesse Date: Sun, 31 Aug 2025 16:13:29 -0400 Subject: [PATCH 41/53] Update tests/test-chat-parser.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- tests/test-chat-parser.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-chat-parser.cpp b/tests/test-chat-parser.cpp index 996ec6e8dcc64..154509b3f81b7 100644 --- a/tests/test-chat-parser.cpp +++ b/tests/test-chat-parser.cpp @@ -288,7 +288,7 @@ static void test_deepseek_v3_1_tool_calls() { /* .thinking_forced_open = */ true, /* .parse_tool_calls = */ true, }; - const char* variant = "thinking_forced_open_tool_call_in_reasoning_fenced_thinking"; + const std::string variant("thinking_forced_open_tool_call_in_reasoning_fenced_thinking"); const std::string in = "REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time2<｜tool▁sep｜>{\"city\": \"Tokyo2\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_time\n```json\n{\"city\": \"Tokyo\"}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>"; auto m = common_chat_parse(in, false, syntax); assert_equals(variant, 1, m.tool_calls.size()); From 3843d94d8884b6cb0878b836e87fe9d95ab23627 Mon Sep 17 00:00:00 2001 From: Jesse Date: Sun, 31 Aug 2025 16:13:42 -0400 Subject: [PATCH 42/53] Update tests/test-chat-parser.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- tests/test-chat-parser.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-chat-parser.cpp b/tests/test-chat-parser.cpp index 154509b3f81b7..7bb86d7188abe 100644 --- a/tests/test-chat-parser.cpp +++ b/tests/test-chat-parser.cpp @@ -307,7 +307,7 @@ static void test_deepseek_v3_1_tool_calls() { /* .thinking_forced_open = */ true, /* .parse_tool_calls = */ true, }; - const char* variant = "thinking_forced_open_tool_call_in_reasoning_no_closing_think"; + const std::string variant("thinking_forced_open_tool_call_in_reasoning_no_closing_think"); const std::string in = "REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time2<｜tool▁sep｜>{\"city\": \"Tokyo2\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>REASONING"; auto m = common_chat_parse(in, false, syntax); assert_equals(variant, 0, m.tool_calls.size()); From 6773708265c0d08f0a6569cc883610a002739088 Mon Sep 17 00:00:00 2001 From: Jesse Date: Sun, 31 Aug 2025 16:13:49 -0400 Subject: [PATCH 43/53] Update tests/test-chat-parser.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- tests/test-chat-parser.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-chat-parser.cpp b/tests/test-chat-parser.cpp index 7bb86d7188abe..66c647bcc864f 100644 --- a/tests/test-chat-parser.cpp +++ b/tests/test-chat-parser.cpp @@ -324,7 +324,7 @@ static void test_deepseek_v3_1_tool_calls() { /* .thinking_forced_open = */ false, /* .parse_tool_calls = */ true, }; - const char* variant = "thinking_not_forced_open_missing_reasoning_no_tool_calls"; + const std::string variant("thinking_not_forced_open_missing_reasoning_no_tool_calls"); const std::string in = "CONTENT"; auto m = common_chat_parse(in, false, syntax); assert_equals(variant, 0, m.tool_calls.size()); From befa31c666ff23ee1a5bf1e8b2b9a33b916dde20 Mon Sep 17 00:00:00 2001 From: Jesse Date: Sun, 31 Aug 2025 16:13:57 -0400 Subject: [PATCH 44/53] Update tests/test-chat-parser.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- tests/test-chat-parser.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-chat-parser.cpp b/tests/test-chat-parser.cpp index 66c647bcc864f..b0fb23799f562 100644 --- a/tests/test-chat-parser.cpp +++ b/tests/test-chat-parser.cpp @@ -220,7 +220,7 @@ static void test_deepseek_v3_1_tool_calls() { }; const std::string input = "<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time<｜tool▁sep｜>{\"city\": \"Tokyo\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>"; auto msg = common_chat_parse(input, false, syntax); - assert_equals(variant, static_cast(1), msg.tool_calls.size()); + assert_equals(variant, 1, msg.tool_calls.size()); assert_equals(variant, std::string("get_time"), msg.tool_calls[0].name); // JSON arguments are dumped without spaces assert_equals(variant, std::string("{\"city\":\"Tokyo\"}"), msg.tool_calls[0].arguments); From 77955943d7bf54fbad1436191f498795186d57ce Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Mon, 1 Sep 2025 02:42:10 +0000 Subject: [PATCH 45/53] DeepSeek V3.1 fix reasoning_format none --- common/chat.cpp | 5 +++++ tests/test-chat-parser.cpp | 22 +++++++++++++++++++--- tests/test-chat.cpp | 13 +++++++++++++ 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index f350cbb65979b..1236e766921d2 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1458,6 +1458,11 @@ static void common_chat_parse_deepseek_v3_1(common_chat_msg_parser & builder) { // <｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>NAME\n```json\nJSON\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜> common_chat_parse_deepseek_v3_1_content(builder); } else { + if (builder.syntax().reasoning_format == COMMON_REASONING_FORMAT_NONE) { + LOG_DBG("%s: reasoning_format none, adding content\n", __func__); + common_chat_parse_deepseek_v3_1_content(builder); + return; + } // If no reasoning tags found, check if we should treat everything as reasoning if (builder.syntax().thinking_forced_open) { // If thinking is forced open but no tags found, treat everything as reasoning diff --git a/tests/test-chat-parser.cpp b/tests/test-chat-parser.cpp index b0fb23799f562..7ddd5babc706f 100644 --- a/tests/test-chat-parser.cpp +++ b/tests/test-chat-parser.cpp @@ -114,10 +114,26 @@ static void test_reasoning() { /* .thinking_forced_open = */ true, /* .parse_tool_calls = */ true, }; + const std::string variant("deepseek_v3_1_reasoning_format_deepseek"); common_chat_msg_parser builder("REASONINGok", /* is_partial= */ false, syntax); - assert_equals(true, builder.try_parse_reasoning("", "")); - assert_equals(std::string("REASONING"), builder.result().reasoning_content); - assert_equals(std::string("ok"), builder.consume_rest()); + assert_equals(variant, true, builder.try_parse_reasoning("", "")); + assert_equals(variant, std::string("REASONING"), builder.result().reasoning_content); + assert_equals(variant, std::string("ok"), builder.consume_rest()); + } + // Test DeepSeek V3.1 parsing - reasoning_format none - reasoning content followed by "" and then regular content + { + common_chat_syntax syntax = { + /* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, + /* .reasoning_format = */ COMMON_REASONING_FORMAT_NONE, + /* .reasoning_in_content = */ false, + /* .thinking_forced_open = */ true, + /* .parse_tool_calls = */ true, + }; + const std::string variant("deepseek_v3_1_reasoning_format_none"); + const std::string input = "REASONINGok"; + auto msg = common_chat_parse(input, false, syntax); + assert_equals(variant, std::string("REASONINGok"), msg.content); + assert_equals(variant, std::string(""), msg.reasoning_content); } } diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 0ab5e4f19729b..4bcbf97a0e212 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -1778,6 +1778,19 @@ static void test_template_output_parsers() { /* .reasoning_in_content = */ false, /* .thinking_forced_open = */ true, })); + // variant: thinking forced open, reasoning_format none + assert_msg_equals( + simple_assist_msg("REASONINGok", ""), + common_chat_parse( + "REASONINGok", + /* is_partial= */ false, + { + COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, + /* .reasoning_format = */ COMMON_REASONING_FORMAT_NONE, + /* .reasoning_in_content = */ false, + /* .thinking_forced_open = */ true, + /* .parse_tool_calls = */ true, + })); // variant: happy path for when it works as the model card says it should assert_msg_equals( simple_assist_msg("", "", "get_time", "{\"city\":\"Tokyo\"}"), From a7316b0696aea460db6465c26161c2fd95bb25fb Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Wed, 3 Sep 2025 17:40:29 +0000 Subject: [PATCH 46/53] Strip grammar down to strictly what we expect based on model card. Throw out parts we cargo culted from R1 that don't make sense. --- common/chat.cpp | 10 +++++----- tests/test-chat-parser.cpp | 24 ++++++------------------ tests/test-chat.cpp | 21 ++++----------------- 3 files changed, 15 insertions(+), 40 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 1236e766921d2..1fc43c42a873c 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1366,9 +1366,9 @@ static common_chat_params common_chat_params_init_deepseek_v3_1(const common_cha auto parameters = function.at("parameters"); builder.resolve_refs(parameters); tool_rules.push_back(builder.add_rule(name + "-call", - "( \"<｜tool▁call▁begin｜>\" )? \"function<｜tool▁sep｜>" + name + "\\n" - "```json\\n\" " + builder.add_schema(name + "-args", parameters) + " " - "\"```<｜tool▁call▁end｜>\"")); + "( \"<｜tool▁call▁begin｜>\" )? \"" + name + "<｜tool▁sep｜>" + "\" " + builder.add_schema(name + "-args", parameters) + " " + "\"<｜tool▁call▁end｜>\"")); }); // Distill Qwen 7B & 32B models seem confused re/ syntax of their tool call opening tag, // so we accept common variants (then it's all constrained) @@ -1421,9 +1421,9 @@ static void common_chat_parse_deepseek_r1(common_chat_msg_parser & builder) { } static void common_chat_parse_deepseek_v3_1_content(common_chat_msg_parser & builder) { - static const common_regex function_regex("(?:<｜tool▁call▁begin｜>)?(?:function<｜tool▁sep｜>)?([^\\n<]+)(?:\\n```json\\n|<｜tool▁sep｜>)"); + static const common_regex function_regex("(?:<｜tool▁call▁begin｜>)?([^\\n<]+)(?:<｜tool▁sep｜>)"); - static const common_regex close_regex("(?:[\\n]*```[\\s\\r\\n]*)?<｜tool▁call▁end｜>"); + static const common_regex close_regex("(?:[\\s]*)?<｜tool▁call▁end｜>"); static const common_regex tool_calls_begin("(?:<｜tool▁calls▁begin｜>|<｜tool_calls_begin｜>|<｜tool calls begin｜>|<｜tool\\\\_calls\\\\_begin｜>|<｜tool▁calls｜>)"); static const common_regex tool_calls_end("<｜tool▁calls▁end｜>"); diff --git a/tests/test-chat-parser.cpp b/tests/test-chat-parser.cpp index 7ddd5babc706f..caa9be8ec1f9c 100644 --- a/tests/test-chat-parser.cpp +++ b/tests/test-chat-parser.cpp @@ -243,19 +243,7 @@ static void test_deepseek_v3_1_tool_calls() { assert_equals(variant, std::string(""), msg.content); assert_equals(variant, std::string(""), msg.reasoning_content); - // variant: function + fenced JSON - { - const std::string variant("fenced"); - const std::string in = "<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_time\n```json\n{\"city\": \"Tokyo\"}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>"; - auto m = common_chat_parse(in, false, syntax); - assert_equals(variant, 1, m.tool_calls.size()); - assert_equals(variant, std::string("get_time"), m.tool_calls[0].name); - assert_equals(variant, std::string("{\"city\":\"Tokyo\"}"), m.tool_calls[0].arguments); - assert_equals(variant, std::string(""), m.content); - assert_equals(variant, std::string(""), m.reasoning_content); - } - - // variant: function + fenced JSON + thinking open + // variant: simple + thinking open { common_chat_syntax syntax = { /* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, @@ -264,8 +252,8 @@ static void test_deepseek_v3_1_tool_calls() { /* .thinking_forced_open = */ true, /* .parse_tool_calls = */ true, }; - const std::string variant("fenced_thinking"); - const std::string in = "REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_time\n```json\n{\"city\": \"Tokyo\"}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>"; + const std::string variant("simple_thinking"); + const std::string in = "REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time<｜tool▁sep｜>{\"city\": \"Tokyo\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>"; auto m = common_chat_parse(in, false, syntax); assert_equals(variant, 1, m.tool_calls.size()); assert_equals(variant, std::string("get_time"), m.tool_calls[0].name); @@ -295,7 +283,7 @@ static void test_deepseek_v3_1_tool_calls() { } - // variant: thinking forced open + tool call in reasoning content + function + fenced JSON + // variant: thinking forced open + tool call in reasoning content { common_chat_syntax syntax = { /* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, @@ -304,8 +292,8 @@ static void test_deepseek_v3_1_tool_calls() { /* .thinking_forced_open = */ true, /* .parse_tool_calls = */ true, }; - const std::string variant("thinking_forced_open_tool_call_in_reasoning_fenced_thinking"); - const std::string in = "REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time2<｜tool▁sep｜>{\"city\": \"Tokyo2\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_time\n```json\n{\"city\": \"Tokyo\"}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>"; + const std::string variant("thinking_forced_open_tool_call_in_reasoning"); + const std::string in = "REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time2<｜tool▁sep｜>{\"city\": \"Tokyo2\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time<｜tool▁sep｜>{\"city\": \"Tokyo\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>"; auto m = common_chat_parse(in, false, syntax); assert_equals(variant, 1, m.tool_calls.size()); assert_equals(variant, std::string("get_time"), m.tool_calls[0].name); diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 4bcbf97a0e212..d6a3383dda7b7 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -1804,24 +1804,11 @@ static void test_template_output_parsers() { /* .thinking_forced_open = */ false, /* .parse_tool_calls = */ true, })); - // variant: function + fenced JSON - assert_msg_equals( - simple_assist_msg("", "", "get_time", "{\"city\":\"Tokyo\"}"), - common_chat_parse( - "<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_time\n```json\n{\"city\": \"Tokyo\"}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>", - /* is_partial= */ false, - { - COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, - /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, - /* .reasoning_in_content = */ false, - /* .thinking_forced_open = */ false, - /* .parse_tool_calls = */ true, - })); - // variant: function + fenced JSON + thinking open + // variant: simple + thinking open assert_msg_equals( simple_assist_msg("", "REASONING", "get_time", "{\"city\":\"Tokyo\"}"), common_chat_parse( - "REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_time\n```json\n{\"city\": \"Tokyo\"}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>", + "REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time<｜tool▁sep｜>{\"city\": \"Tokyo\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>", /* is_partial= */ false, { COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, @@ -1848,11 +1835,11 @@ static void test_template_output_parsers() { /* .thinking_forced_open = */ false, /* .parse_tool_calls = */ true, })); - // variant: thinking forced open + tool call in reasoning content + function + fenced JSON + // variant: thinking forced open + tool call in reasoning content assert_msg_equals( simple_assist_msg("", "REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time2<｜tool▁sep｜>{\"city\": \"Tokyo2\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>REASONING", "get_time", "{\"city\":\"Tokyo\"}"), common_chat_parse( - "REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time2<｜tool▁sep｜>{\"city\": \"Tokyo2\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_time\n```json\n{\"city\": \"Tokyo\"}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>", + "REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time2<｜tool▁sep｜>{\"city\": \"Tokyo2\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time<｜tool▁sep｜>{\"city\": \"Tokyo\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>", /* is_partial= */ false, { COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, From e3fe1ce08394d18036e3442d8dc2023780b3d9ed Mon Sep 17 00:00:00 2001 From: Jesse Date: Thu, 4 Sep 2025 18:19:36 -0400 Subject: [PATCH 47/53] Update tests/test-chat-parser.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- tests/test-chat-parser.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-chat-parser.cpp b/tests/test-chat-parser.cpp index caa9be8ec1f9c..eb92a22416f38 100644 --- a/tests/test-chat-parser.cpp +++ b/tests/test-chat-parser.cpp @@ -15,7 +15,7 @@ #include "regex-partial.h" template -static void assert_equals(const std::string & label, const T & expected, const T & actual) { +static void assert_equals(const std::string_view label, const T & expected, const T & actual) { if (expected != actual) { std::cerr << label << std::endl; std::cerr << "Expected: " << expected << std::endl; From 9830e7e36444e1a40f750abcc2c45b5376a1bf6e Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Sat, 6 Sep 2025 04:47:41 +0000 Subject: [PATCH 48/53] DeepSeek V3.1 - Add edge case where thinking is forced open, there is tool calling in the reasoning content, but then the model just stops the output without closing the tag, so it's not a partial. In this case, use the tool call in the reasoning content. --- common/chat.cpp | 10 +++++++- tests/test-chat-parser.cpp | 49 ++++++++++++++++++++++++++++++++++---- tests/test-chat.cpp | 22 ++++++++++++++--- 3 files changed, 72 insertions(+), 9 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 9ab2276e8250c..c572131fc0653 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1514,7 +1514,14 @@ static void common_chat_parse_deepseek_v3_1(common_chat_msg_parser & builder) { // First try to parse using the standard reasoning parsing method LOG_DBG("%s: thinking_forced_open: %s\n", __func__, std::to_string(builder.syntax().thinking_forced_open).c_str()); - if (builder.try_parse_reasoning("", "")) { + auto start_pos = builder.pos(); + auto found_end_think = builder.try_find_literal(""); + builder.move_to(start_pos); + + if (builder.syntax().thinking_forced_open && !builder.is_partial() && !found_end_think) { + LOG_DBG("%s: no end_think, not partial, adding content\n", __func__); + common_chat_parse_deepseek_v3_1_content(builder); + } else if (builder.try_parse_reasoning("", "")) { // If reasoning was parsed successfully, the remaining content is regular content LOG_DBG("%s: parsed reasoning, adding content\n", __func__); // <｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>NAME\n```json\nJSON\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜> @@ -1531,6 +1538,7 @@ static void common_chat_parse_deepseek_v3_1(common_chat_msg_parser & builder) { LOG_DBG("%s: thinking_forced_open, adding reasoning content\n", __func__); builder.add_reasoning_content(builder.consume_rest()); } else { + LOG_DBG("%s: no thinking_forced_open, adding content\n", __func__); // <｜tool▁call▁begin｜>NAME<｜tool▁sep｜>JSON<｜tool▁call▁end｜> common_chat_parse_deepseek_v3_1_content(builder); } diff --git a/tests/test-chat-parser.cpp b/tests/test-chat-parser.cpp index eb92a22416f38..547ebb4871cd4 100644 --- a/tests/test-chat-parser.cpp +++ b/tests/test-chat-parser.cpp @@ -52,6 +52,7 @@ static void assert_throws(const std::function & fn, const std::string & } static void test_reasoning() { + //common_log_set_verbosity_thold(LOG_DEFAULT_DEBUG); { common_chat_msg_parser builder("CogitoErgo sum", /* is_partial= */ false, { /* .format = */ COMMON_CHAT_FORMAT_CONTENT_ONLY, @@ -302,7 +303,10 @@ static void test_deepseek_v3_1_tool_calls() { assert_equals(variant, std::string("REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time2<｜tool▁sep｜>{\"city\": \"Tokyo2\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>REASONING"), m.reasoning_content); } - // variant: thinking forced open + tool call in reasoning content + no closing think + // variant: thinking forced open + tool call in reasoning content + no closing think + not partial + // This is a bit of a fine tuning issue on the model's part IMO. It really should not be attempting + // to make tool calls in reasoning content according to the model card, but it does sometimes, so + // add the reasoning content as regular content and parse the tool calls. { common_chat_syntax syntax = { /* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, @@ -311,14 +315,49 @@ static void test_deepseek_v3_1_tool_calls() { /* .thinking_forced_open = */ true, /* .parse_tool_calls = */ true, }; - const std::string variant("thinking_forced_open_tool_call_in_reasoning_no_closing_think"); - const std::string in = "REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time2<｜tool▁sep｜>{\"city\": \"Tokyo2\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>REASONING"; + const std::string variant("thinking_forced_open_tool_call_in_reasoning_no_closing_think_not_partial"); + const std::string in = "REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time<｜tool▁sep｜>{\"city\": \"Tokyo\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>"; auto m = common_chat_parse(in, false, syntax); - assert_equals(variant, 0, m.tool_calls.size()); + assert_equals(variant, std::string("REASONING"), m.content); + assert_equals(variant, std::string(""), m.reasoning_content); + assert_equals(variant, 1, m.tool_calls.size()); + assert_equals(variant, std::string("get_time"), m.tool_calls[0].name); + assert_equals(variant, std::string("{\"city\":\"Tokyo\"}"), m.tool_calls[0].arguments); + } + + // variant: thinking forced open + tool call in reasoning content + no closing think + partial + { + common_chat_syntax syntax = { + /* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, + /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, + /* .reasoning_in_content = */ false, + /* .thinking_forced_open = */ true, + /* .parse_tool_calls = */ true, + }; + const std::string variant("thinking_forced_open_tool_call_in_reasoning_no_closing_think_partial"); + const std::string in = "REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time<｜tool▁sep｜>{\"city\": \"Tokyo\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>"; + auto m = common_chat_parse(in, /* is_partial= */ true, syntax); + assert_equals(variant, std::string("REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time<｜tool▁sep｜>{\"city\": \"Tokyo\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>"), m.reasoning_content); assert_equals(variant, std::string(""), m.content); - assert_equals(variant, std::string("REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time2<｜tool▁sep｜>{\"city\": \"Tokyo2\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>REASONING"), m.reasoning_content); + assert_equals(variant, 0, m.tool_calls.size()); } + // variant: thinking not forced open + reasoning + regular content + no tool calls + { + common_chat_syntax syntax = { + /* .format = */ COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, + /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, + /* .reasoning_in_content = */ false, + /* .thinking_forced_open = */ true, + /* .parse_tool_calls = */ true, + }; + const std::string variant("thinking_forced_open_reasoning_regular_content_no_tool_calls"); + const std::string in = "REASONINGCONTENT"; + auto m = common_chat_parse(in, false, syntax); + assert_equals(variant, 0, m.tool_calls.size()); + assert_equals(variant, std::string("CONTENT"), m.content); + assert_equals(variant, std::string("REASONING"), m.reasoning_content); + } // variant: thinking not forced open + missing reasoning + no tool calls { common_chat_syntax syntax = { diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 3c6dae53e9145..ac8a0ade1f6e2 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -1920,11 +1920,14 @@ static void test_template_output_parsers() { /* .thinking_forced_open = */ true, /* .parse_tool_calls = */ true, })); - // variant: thinking forced open + tool call in reasoning content + no closing think + // variant: thinking forced open + tool call in reasoning content + no closing think + not partial + // This is a bit of a fine tuning issue on the model's part IMO. It really should not be attempting + // to make tool calls in reasoning content according to the model card, but it does sometimes, so + // add the reasoning content as regular content and parse the tool calls. assert_msg_equals( - simple_assist_msg("", "REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time2<｜tool▁sep｜>{\"city\": \"Tokyo2\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>REASONING", "", ""), + simple_assist_msg("REASONING", "", "get_time", "{\"city\":\"Tokyo\"}"), common_chat_parse( - "REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time2<｜tool▁sep｜>{\"city\": \"Tokyo2\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>REASONING", + "REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time<｜tool▁sep｜>{\"city\": \"Tokyo\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>", /* is_partial= */ false, { COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, @@ -1933,6 +1936,19 @@ static void test_template_output_parsers() { /* .thinking_forced_open = */ true, /* .parse_tool_calls = */ true, })); + // variant: thinking forced open + tool call in reasoning content + no closing think + partial + assert_msg_equals( + simple_assist_msg("", "REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time<｜tool▁sep｜>{\"city\": \"Tokyo\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>", "", ""), + common_chat_parse( + "REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time<｜tool▁sep｜>{\"city\": \"Tokyo\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>", + /* is_partial= */ true, + { + COMMON_CHAT_FORMAT_DEEPSEEK_V3_1, + /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK, + /* .reasoning_in_content = */ false, + /* .thinking_forced_open = */ true, + /* .parse_tool_calls = */ true, + })); // variant: thinking not forced open + missing reasoning + no tool calls assert_msg_equals( simple_assist_msg("CONTENT", ""), From 26b02fa8ac248bf47f7c550f1d015fb9081a5499 Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Mon, 8 Sep 2025 01:10:49 +0000 Subject: [PATCH 49/53] DeepSeek V3.1 - simplify update_cursor --- common/chat.cpp | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 0461450ff12ec..530fdd259c2d1 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -700,6 +700,7 @@ static void parse_json_tool_calls( size_t from = std::string::npos; auto first = true; while (true) { + auto start_pos = builder.pos(); auto res = function_regex_start_only && first ? builder.try_consume_regex(*function_regex_start_only) : function_regex @@ -720,12 +721,7 @@ static void parse_json_tool_calls( from = res->groups[0].begin + 1; continue; } - if (update_cursor) { - builder.move_to(res->groups[0].end); - from = builder.pos(); - } else { - from = std::string::npos; - } + from = std::string::npos; auto maybe_raw_python = name == "python" && allow_raw_python; if (builder.input()[builder.pos()] == '{' || !maybe_raw_python) { @@ -734,16 +730,8 @@ static void parse_json_tool_calls( throw common_chat_msg_partial_exception("incomplete tool call"); } builder.consume_regex(close_regex); - if (update_cursor) { - from = builder.pos(); // continue after this call - continue; - } - } - if (update_cursor) { - throw common_chat_msg_partial_exception("incomplete tool call"); - } else { - continue; } + continue; } if (maybe_raw_python) { auto arguments = wrap_code_as_arguments(builder, builder.consume_rest()); @@ -753,14 +741,14 @@ static void parse_json_tool_calls( return; } throw common_chat_msg_partial_exception("incomplete tool call"); + } else { + if (update_cursor) { + builder.move_to(start_pos); + } } break; } if (block_close) { - if (update_cursor) { - // ensure we’re right after the last call header/close - if (from != std::string::npos) builder.move_to(from); - } builder.consume_regex(*block_close); } builder.consume_spaces(); @@ -768,7 +756,6 @@ static void parse_json_tool_calls( }; if (block_open) { if (auto res = builder.try_find_regex(*block_open)) { - if (update_cursor) builder.move_to(res->groups[0].end); // consume opener parse_tool_calls(); } else { builder.add_content(builder.consume_rest()); @@ -1519,7 +1506,7 @@ static void common_chat_parse_deepseek_v3_1_content(common_chat_msg_parser & bui tool_calls_end, false, nullptr, - true); + /* update_cursor */ true); } static void common_chat_parse_deepseek_v3_1(common_chat_msg_parser & builder) { From 3ccc651988677195541558d16c8ff371bdd56f36 Mon Sep 17 00:00:00 2001 From: Jesse Date: Mon, 8 Sep 2025 07:11:47 -0400 Subject: [PATCH 50/53] Update common/chat.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- common/chat.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 530fdd259c2d1..349d8fd8fe15f 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -693,8 +693,7 @@ static void parse_json_tool_calls( const common_regex & close_regex, const std::optional & block_close, bool allow_raw_python = false, - const std::function & get_function_name = nullptr, - bool update_cursor = false) { + const std::function & get_function_name = nullptr) { auto parse_tool_calls = [&]() { size_t from = std::string::npos; From e23eedd11fc3172a979d75365127f89957dc3838 Mon Sep 17 00:00:00 2001 From: Jesse Date: Mon, 8 Sep 2025 07:11:55 -0400 Subject: [PATCH 51/53] Update common/chat.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- common/chat.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 349d8fd8fe15f..a08b4230f6888 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1502,10 +1502,7 @@ static void common_chat_parse_deepseek_v3_1_content(common_chat_msg_parser & bui /* function_regex_start_only= */ std::nullopt, function_regex, close_regex, - tool_calls_end, - false, - nullptr, - /* update_cursor */ true); + tool_calls_end); } static void common_chat_parse_deepseek_v3_1(common_chat_msg_parser & builder) { From cf17a8e349f6d63ccfbcee9f34e785cb0f0b2a46 Mon Sep 17 00:00:00 2001 From: Jesse Date: Mon, 8 Sep 2025 07:12:10 -0400 Subject: [PATCH 52/53] Update common/chat.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- common/chat.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index a08b4230f6888..a4ac6ed62a0c3 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -741,9 +741,7 @@ static void parse_json_tool_calls( } throw common_chat_msg_partial_exception("incomplete tool call"); } else { - if (update_cursor) { - builder.move_to(start_pos); - } + builder.move_to(start_pos); } break; } From 4c2179d27adb17159684d4cedf95de295b308194 Mon Sep 17 00:00:00 2001 From: Jesse CreateThis Date: Mon, 8 Sep 2025 11:17:34 +0000 Subject: [PATCH 53/53] Fix indent --- common/chat.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/chat.cpp b/common/chat.cpp index a4ac6ed62a0c3..4707c4fef4070 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -741,7 +741,7 @@ static void parse_json_tool_calls( } throw common_chat_msg_partial_exception("incomplete tool call"); } else { - builder.move_to(start_pos); + builder.move_to(start_pos); } break; }