From af928d4890f64fd54f3540abee642ee1a92fe4a4 Mon Sep 17 00:00:00 2001 From: hksdpc255 <43977088+hksdpc255@users.noreply.github.com> Date: Wed, 19 Nov 2025 11:09:21 +0800 Subject: [PATCH 1/4] Fix kimi-k2 parsing --- common/chat.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 6fa05a60416d4..960c7ee23e2dc 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1970,8 +1970,8 @@ static void common_chat_parse_kimi_k2(common_chat_msg_parser & builder) { form.tool_start = "<|tool_call_begin|>"; form.tool_sep = "<|tool_call_argument_begin|>{"; form.key_start = "\""; - form.key_val_sep = "\": "; - form.val_end = ", "; + form.key_val_sep = "\":"; + form.val_end = ","; form.tool_end = "}<|tool_call_end|>"; form.scope_end = "<|tool_calls_section_end|>"; form.raw_argval = false; From f519483d521b6e5fceba303bebac36e90bda0d6d Mon Sep 17 00:00:00 2001 From: hksdpc255 <43977088+hksdpc255@users.noreply.github.com> Date: Wed, 19 Nov 2025 14:15:51 +1100 Subject: [PATCH 2/4] fix template & add more tests for kimi-k2 --- models/templates/Kimi-K2-Instruct.jinja | 2 +- models/templates/Kimi-K2-Thinking.jinja | 2 +- tests/test-chat.cpp | 46 +++++++++++++++++-------- 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/models/templates/Kimi-K2-Instruct.jinja b/models/templates/Kimi-K2-Instruct.jinja index a9439135baa52..5e898c62531c5 100644 --- a/models/templates/Kimi-K2-Instruct.jinja +++ b/models/templates/Kimi-K2-Instruct.jinja @@ -14,7 +14,7 @@ {%- endmacro %} {%- set tool_response_queue = namespace(ids=[]) -%} -{%- set tool_call_counter = namespace(value=1) -%} +{%- set tool_call_counter = namespace(value=0) -%} {%- if tools -%} <|im_system|>tool_declare<|im_middle|>{{ tools | tojson }}<|im_end|> diff --git a/models/templates/Kimi-K2-Thinking.jinja b/models/templates/Kimi-K2-Thinking.jinja index 4c2af6a783236..1df30da752a29 100644 --- a/models/templates/Kimi-K2-Thinking.jinja +++ b/models/templates/Kimi-K2-Thinking.jinja @@ -25,7 +25,7 @@ {%- endmacro -%} {%- set tool_response_queue = namespace(ids=[]) -%} -{%- set tool_call_counter = namespace(value=1) -%} +{%- set tool_call_counter = namespace(value=0) -%} {%- macro render_toolcalls(message) -%} <|tool_calls_section_begin|> diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 62dd1583fa1a5..d121f4b70811c 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -2659,14 +2659,14 @@ Hey there!<|im_end|> // Test parsing tool calls assert_msg_equals(message_assist_call, common_chat_parse( - "<|tool_calls_section_begin|><|tool_call_begin|>functions.special_function:1<|tool_call_argument_begin|>{\"arg1\": 1}<|tool_call_end|><|tool_calls_section_end|>", + "<|tool_calls_section_begin|><|tool_call_begin|>functions.special_function:0<|tool_call_argument_begin|>{\"arg1\": 1}<|tool_call_end|><|tool_calls_section_end|>", /* is_partial= */ false, {COMMON_CHAT_FORMAT_KIMI_K2})); // Test parsing tool calls with thinking assert_msg_equals(message_assist_call_thoughts, common_chat_parse( - "I'm\nthinking<|tool_calls_section_begin|><|tool_call_begin|>functions.special_function:1<|tool_call_argument_begin|>{\"arg1\": 1}<|tool_call_end|><|tool_calls_section_end|>", + "I'm\nthinking<|tool_calls_section_begin|><|tool_call_begin|>functions.special_function:0<|tool_call_argument_begin|>{\"arg1\": 1}<|tool_call_end|><|tool_calls_section_end|>", /* is_partial= */ false, { /* .format = */ COMMON_CHAT_FORMAT_KIMI_K2, @@ -2676,7 +2676,7 @@ Hey there!<|im_end|> // Test tool calls with extra content assert_msg_equals(message_assist_call_content, common_chat_parse( - "<|tool_calls_section_begin|><|tool_call_begin|>functions.special_function:1<|tool_call_argument_begin|>{\"arg1\": 1}<|tool_call_end|><|tool_calls_section_end|>Hello, world!\nWhat's up?", + "<|tool_calls_section_begin|><|tool_call_begin|>functions.special_function:0<|tool_call_argument_begin|>{\"arg1\": 1}<|tool_call_end|><|tool_calls_section_end|>Hello, world!\nWhat's up?", /* is_partial= */ false, {COMMON_CHAT_FORMAT_KIMI_K2} )); @@ -2684,7 +2684,7 @@ Hey there!<|im_end|> // Test tool calls with extra content AND thinking assert_msg_equals(message_assist_call_thoughts_content, common_chat_parse( - "I'm\nthinking<|tool_calls_section_begin|><|tool_call_begin|>functions.special_function:1<|tool_call_argument_begin|>{\"arg1\": 1}<|tool_call_end|><|tool_calls_section_end|>Hello, world!\nWhat's up?", + "I'm\nthinking<|tool_calls_section_begin|><|tool_call_begin|>functions.special_function:0<|tool_call_argument_begin|>{\"arg1\": 1}<|tool_call_end|><|tool_calls_section_end|>Hello, world!\nWhat's up?", /* is_partial= */ false, { /* .format = */ COMMON_CHAT_FORMAT_KIMI_K2, @@ -2693,47 +2693,65 @@ Hey there!<|im_end|> // Test streaming test_parser_with_streaming(message_assist_call_thoughts_content, - "I'm\nthinking\nHello, world!\nWhat's up?\n<|tool_calls_section_begin|><|tool_call_begin|>functions.special_function:1<|tool_call_argument_begin|>{\"arg1\": 1}<|tool_call_end|><|tool_calls_section_end|>", + "I'm\nthinking\nHello, world!\nWhat's up?\n<|tool_calls_section_begin|><|tool_call_begin|>functions.special_function:0<|tool_call_argument_begin|>{\"arg1\": 1}<|tool_call_end|><|tool_calls_section_end|>", [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, { /* .format = */ COMMON_CHAT_FORMAT_KIMI_K2, /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK }); }); test_parser_with_streaming(message_assist_call_thoughts_unparsed, - "I'm\nthinking\n\n<|tool_calls_section_begin|><|tool_call_begin|>functions.special_function:1<|tool_call_argument_begin|>{\"arg1\": 1}<|tool_call_end|><|tool_calls_section_end|>", + "I'm\nthinking\n\n<|tool_calls_section_begin|><|tool_call_begin|>functions.special_function:0<|tool_call_argument_begin|>{\"arg1\": 1}<|tool_call_end|><|tool_calls_section_end|>", [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, { /* .format = */ COMMON_CHAT_FORMAT_KIMI_K2, /* .reasoning_format = */ COMMON_REASONING_FORMAT_NONE }); }); test_parser_with_streaming(message_assist_call_thoughts_content, - "I'm\nthinking\n\n\nHello, world!\nWhat's up?\n\n<|tool_calls_section_begin|><|tool_call_begin|>functions.special_function:1<|tool_call_argument_begin|>{\"arg1\": 1}<|tool_call_end|><|tool_calls_section_end|>\n", + "I'm\nthinking\n\n\nHello, world!\nWhat's up?\n\n<|tool_calls_section_begin|><|tool_call_begin|>functions.special_function:0<|tool_call_argument_begin|>{\"arg1\": 1}<|tool_call_end|><|tool_calls_section_end|>\n", [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, { /* .format = */ COMMON_CHAT_FORMAT_KIMI_K2, /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK }); }); test_parser_with_streaming(message_assist_call_withopt, - "<|tool_calls_section_begin|><|tool_call_begin|>functions.special_function_with_opt:1<|tool_call_argument_begin|>{\"arg1\": 1, \"arg2\": 2}<|tool_call_end|><|tool_calls_section_end|>", + "<|tool_calls_section_begin|><|tool_call_begin|>functions.special_function_with_opt:0<|tool_call_argument_begin|>{\"arg1\": 1, \"arg2\": 2}<|tool_call_end|><|tool_calls_section_end|>", [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, { /* .format = */ COMMON_CHAT_FORMAT_KIMI_K2, /* .reasoning_format = */ COMMON_REASONING_FORMAT_NONE }); }); test_parser_with_streaming(simple_assist_msg("Hello, world!\nWhat's up?", "I'm\nthinking", "special_function", "{\"arg1\": \"123456\"}"), - "I'm\nthinkingHello, world!\nWhat's up?\n<|tool_calls_section_begin|><|tool_call_begin|>functions.special_function:1<|tool_call_argument_begin|>{\"arg1\": \"123456\"}<|tool_call_end|><|tool_calls_section_end|>", + "I'm\nthinkingHello, world!\nWhat's up?\n<|tool_calls_section_begin|><|tool_call_begin|>functions.special_function:0<|tool_call_argument_begin|>{\"arg1\": \"123456\"}<|tool_call_end|><|tool_calls_section_end|>", [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, { /* .format = */ COMMON_CHAT_FORMAT_KIMI_K2, /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK }); }); test_parser_with_streaming(simple_assist_msg("Hello, world!\nWhat's up?", "I'm\nthinking", "special_function", "{\"arg1\": [1, 2, \"345\", 6]}"), - "I'm\nthinkingHello, world!\nWhat's up?\n<|tool_calls_section_begin|><|tool_call_begin|>functions.special_function:1<|tool_call_argument_begin|>{\"arg1\": [1, 2, \"345\", 6]}<|tool_call_end|><|tool_calls_section_end|>", + "I'm\nthinkingHello, world!\nWhat's up?\n<|tool_calls_section_begin|><|tool_call_begin|>functions.special_function:0<|tool_call_argument_begin|>{\"arg1\": [1, 2, \"345\", 6]}<|tool_call_end|><|tool_calls_section_end|>", [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, { /* .format = */ COMMON_CHAT_FORMAT_KIMI_K2, /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK }); }); test_parser_with_streaming(simple_assist_msg("Hello, world!\nWhat's up?", "I'm\nthinking", "special_function", "{\"arg1\": {\"12\": 34, \"5\": [67, 8], \"9\": \"10\"}}"), - "I'm\nthinkingHello, world!\nWhat's up?\n<|tool_calls_section_begin|><|tool_call_begin|>functions.special_function:1<|tool_call_argument_begin|>{\"arg1\": {\"12\": 34, \"5\": [67, 8], \"9\": \"10\"}}<|tool_call_end|><|tool_calls_section_end|>", + "I'm\nthinkingHello, world!\nWhat's up?\n<|tool_calls_section_begin|><|tool_call_begin|>functions.special_function:0<|tool_call_argument_begin|>{\"arg1\": {\"12\": 34, \"5\": [67, 8], \"9\": \"10\"}}<|tool_call_end|><|tool_calls_section_end|>", [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, { /* .format = */ COMMON_CHAT_FORMAT_KIMI_K2, /* .reasoning_format = */ COMMON_REASONING_FORMAT_DEEPSEEK }); }); + test_parser_with_streaming( + simple_assist_msg("", "", "complex_function", "{\"name\":\"John Doe\",\"age\":30,\"active\":true,\"score\":95.5}"), + "<|tool_calls_section_begin|><|tool_call_begin|>functions.complex_function:0<|tool_call_argument_begin|>" + "{\"name\": \"John Doe\", \"age\": 30, \"active\": true, \"score\": 95.5}" + "<|tool_call_end|><|tool_calls_section_end|>", + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_KIMI_K2}); }); + test_parser_with_streaming( + simple_assist_msg("", "", "web_search", "{\"query\":\"\\\"From Zero\\\" Linkin Park album tracklist complete songs\",\"limit\":3,\"type\":\"text\"}"), + "<|tool_calls_section_begin|><|tool_call_begin|>functions.web_search:0<|tool_call_argument_begin|>" + "{\"query\":\"\\\"From Zero\\\" Linkin Park album tracklist complete songs\",\"limit\":3,\"type\":\"text\"}" + "<|tool_call_end|><|tool_calls_section_end|>", + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_KIMI_K2}); }); + test_parser_with_streaming( + simple_assist_msg("", "", "read_file", "{\"args\": [{\"path\": \"src/providers/ThemeProvider.tsx\"}, {\"path\": \"src/components/Header.tsx\"}, {\"path\": \"src/components/ThemeToggle.tsx\"}, {\"path\": \"src/app/globals.css\"}, {\"path\": \"src/app/layout.tsx\"}]}"), + "<|tool_calls_section_begin|><|tool_call_begin|>functions.read_file:0<|tool_call_argument_begin|>" + "{\"args\": [{\"path\": \"src/providers/ThemeProvider.tsx\"}, {\"path\": \"src/components/Header.tsx\"}, {\"path\": \"src/components/ThemeToggle.tsx\"}, {\"path\": \"src/app/globals.css\"}, {\"path\": \"src/app/layout.tsx\"}]}" + "<|tool_call_end|><|tool_calls_section_end|>", + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_KIMI_K2}); }); // Test template generation for regular content test_templates(tmpls.get(), end_tokens, message_assist, tools, @@ -2742,7 +2760,7 @@ Hey there!<|im_end|> // Test template generation for tool calls test_templates(tmpls.get(), end_tokens, message_assist_call, tools, - "<|tool_calls_section_begin|><|tool_call_begin|>functions.special_function:1<|tool_call_argument_begin|>{\"arg1\": 1}<|tool_call_end|><|tool_calls_section_end|>", + "<|tool_calls_section_begin|><|tool_call_begin|>functions.special_function:0<|tool_call_argument_begin|>{\"arg1\": 1}<|tool_call_end|><|tool_calls_section_end|>", /* expect_grammar_triggered= */ true, /* test_grammar_if_triggered= */ true, /* common_reasoning_format= */ COMMON_REASONING_FORMAT_DEEPSEEK, @@ -2751,14 +2769,14 @@ Hey there!<|im_end|> // Test template generation for tools with optional parameters test_templates(tmpls.get(), end_tokens, message_assist_call_noopt, tools, - "<|tool_calls_section_begin|><|tool_call_begin|>functions.special_function_with_opt:1<|tool_call_argument_begin|>{\"arg1\": 1}<|tool_call_end|><|tool_calls_section_end|>", + "<|tool_calls_section_begin|><|tool_call_begin|>functions.special_function_with_opt:0<|tool_call_argument_begin|>{\"arg1\": 1}<|tool_call_end|><|tool_calls_section_end|>", /* expect_grammar_triggered= */ true, /* test_grammar_if_triggered= */ true, /* common_reasoning_format= */ COMMON_REASONING_FORMAT_DEEPSEEK, /* ignore_whitespace_differences= */ true ); test_templates(tmpls.get(), end_tokens, message_assist_call_withopt, tools, - "<|tool_calls_section_begin|><|tool_call_begin|>functions.special_function_with_opt:1<|tool_call_argument_begin|>{\"arg1\": 1, \"arg2\": 2}<|tool_call_end|><|tool_calls_section_end|>", + "<|tool_calls_section_begin|><|tool_call_begin|>functions.special_function_with_opt:0<|tool_call_argument_begin|>{\"arg1\": 1, \"arg2\": 2}<|tool_call_end|><|tool_calls_section_end|>", /* expect_grammar_triggered= */ true, /* test_grammar_if_triggered= */ true, /* common_reasoning_format= */ COMMON_REASONING_FORMAT_DEEPSEEK, From e0eda17d42a8213ecf84504e562926176abb0ba4 Mon Sep 17 00:00:00 2001 From: hksdpc255 <43977088+hksdpc255@users.noreply.github.com> Date: Fri, 21 Nov 2025 14:29:51 +1100 Subject: [PATCH 3/4] Another fix for Kimi-K2 chat template. --- models/templates/Kimi-K2-Instruct.jinja | 8 ++------ models/templates/Kimi-K2-Thinking.jinja | 8 ++------ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/models/templates/Kimi-K2-Instruct.jinja b/models/templates/Kimi-K2-Instruct.jinja index 5e898c62531c5..6204fb3960286 100644 --- a/models/templates/Kimi-K2-Instruct.jinja +++ b/models/templates/Kimi-K2-Instruct.jinja @@ -36,12 +36,8 @@ {%- if message['role'] == 'assistant' and message.get('tool_calls') -%} {{render_content(message)}}<|tool_calls_section_begin|> {%- for tool_call in message['tool_calls'] -%} - {%- if tool_call['id'] is defined -%} - {%- set formatted_id = tool_call['id'] -%} - {%- else -%} - {%- set formatted_id = 'functions.' + tool_call['function']['name'] + ':' + (tool_call_counter.value | string) -%} - {%- set tool_call_counter.value = tool_call_counter.value + 1 -%} - {%- endif -%} + {%- set formatted_id = 'functions.' + tool_call['function']['name'] + ':' + (tool_call_counter.value | string) -%} + {%- set tool_call_counter.value = tool_call_counter.value + 1 -%} {%- set _ = tool_response_queue.ids.append(formatted_id) -%} <|tool_call_begin|>{{ formatted_id }}<|tool_call_argument_begin|>{% if tool_call['function']['arguments'] is string %}{{ tool_call['function']['arguments'] }}{% else %}{{ tool_call['function']['arguments'] | tojson }}{% endif %}<|tool_call_end|> {%- endfor -%} diff --git a/models/templates/Kimi-K2-Thinking.jinja b/models/templates/Kimi-K2-Thinking.jinja index 1df30da752a29..5641429f5356f 100644 --- a/models/templates/Kimi-K2-Thinking.jinja +++ b/models/templates/Kimi-K2-Thinking.jinja @@ -30,12 +30,8 @@ {%- macro render_toolcalls(message) -%} <|tool_calls_section_begin|> {%- for tool_call in message['tool_calls'] -%} - {%- if tool_call['id'] is defined -%} - {%- set formatted_id = tool_call['id'] -%} - {%- else -%} - {%- set formatted_id = 'functions.' + tool_call['function']['name'] + ':' + (tool_call_counter.value | string) -%} - {%- set tool_call_counter.value = tool_call_counter.value + 1 -%} - {%- endif -%} + {%- set formatted_id = 'functions.' + tool_call['function']['name'] + ':' + (tool_call_counter.value | string) -%} + {%- set tool_call_counter.value = tool_call_counter.value + 1 -%} {%- set _ = tool_response_queue.ids.append(formatted_id) -%} <|tool_call_begin|>{{ formatted_id }}<|tool_call_argument_begin|>{% if tool_call['function']['arguments'] is string %}{{ tool_call['function']['arguments'] }}{% else %}{{ tool_call['function']['arguments'] | tojson }}{% endif %}<|tool_call_end|> {%- endfor -%} From 043a6a7f65c8b7b0a439f005db578975bf8596d4 Mon Sep 17 00:00:00 2001 From: hksdpc255 <43977088+hksdpc255@users.noreply.github.com> Date: Fri, 21 Nov 2025 21:23:50 +1100 Subject: [PATCH 4/4] enable allow_toolcall_in_think for Kimi-K2 --- common/chat-parser-xml-toolcall.cpp | 53 +++++++++---- common/chat.cpp | 1 + tests/test-chat.cpp | 117 +++++++++++++++++++++++++++- 3 files changed, 153 insertions(+), 18 deletions(-) diff --git a/common/chat-parser-xml-toolcall.cpp b/common/chat-parser-xml-toolcall.cpp index 7349895550f85..142cd623b35a1 100644 --- a/common/chat-parser-xml-toolcall.cpp +++ b/common/chat-parser-xml-toolcall.cpp @@ -724,16 +724,10 @@ inline void parse_msg_with_xml_tool_calls(common_chat_msg_parser & builder, cons if (reasoning_unclosed) { if (auto pos = content.find(end_think); pos == std::string::npos && builder.pos() != builder.input().size()) { unclosed_reasoning_content += content; - if (form.allow_toolcall_in_think) { - builder.move_to(tc->groups[0].begin); - if (!builder.try_consume_xml_tool_calls(form)) { - unclosed_reasoning_content += tool_call_start; - builder.move_to(tc->groups[0].end); - } - } else { + if (!(form.allow_toolcall_in_think && tc)) { unclosed_reasoning_content += tool_call_start; + continue; } - continue; } else { reasoning_unclosed = false; std::string reasoning_content; @@ -781,8 +775,12 @@ inline void parse_msg_with_xml_tool_calls(common_chat_msg_parser & builder, cons } } else { // This start is in thinking block, skip this tool call - auto pos = think_start + start_think.size(); - unclosed_reasoning_content = content.substr(pos) + tool_call_start; + // This start is in thinking block + if (form.allow_toolcall_in_think) { + unclosed_reasoning_content = content.substr(think_start + start_think.size()); + } else { + unclosed_reasoning_content = content.substr(think_start + start_think.size()) + tool_call_start; + } reasoning_unclosed = true; content.resize(think_start); toolcall_in_think = true; @@ -805,14 +803,35 @@ inline void parse_msg_with_xml_tool_calls(common_chat_msg_parser & builder, cons } // remove potential partial suffix - if (content.size() > 0 && builder.pos() == builder.input().size() && unclosed_reasoning_content.empty()) { - rstrip(content); - trim_potential_partial_word(content); - rstrip(content); + if (builder.pos() == builder.input().size()) { + if (unclosed_reasoning_content.empty()) { + rstrip(content); + trim_potential_partial_word(content); + rstrip(content); + } else { + rstrip(unclosed_reasoning_content); + trim_potential_partial_word(unclosed_reasoning_content); + rstrip(unclosed_reasoning_content); + } + } + + // consume unclosed_reasoning_content if allow_toolcall_in_think is set + if (form.allow_toolcall_in_think && !unclosed_reasoning_content.empty()) { + if (builder.syntax().reasoning_format != COMMON_REASONING_FORMAT_NONE && !builder.syntax().reasoning_in_content) { + builder.add_reasoning_content(unclosed_reasoning_content); + } else { + if (content.empty()) { + content = start_think + unclosed_reasoning_content; + } else { + content += "\n\n" + start_think; + content += unclosed_reasoning_content; + } + } + unclosed_reasoning_content.clear(); } // Add content - if (content.size() != 0) { + if (!content.empty()) { // If there are multiple content blocks if (builder.syntax().reasoning_format != COMMON_REASONING_FORMAT_NONE && !builder.syntax().reasoning_in_content && builder.result().content.size() != 0) { builder.add_content("\n\n"); @@ -820,7 +839,7 @@ inline void parse_msg_with_xml_tool_calls(common_chat_msg_parser & builder, cons builder.add_content(content); } - // This start is in thinking block, skip this tool call + // This start is in thinking block and toolcall_in_think not set, skip this tool call if (toolcall_in_think && !form.allow_toolcall_in_think) { continue; } @@ -829,7 +848,7 @@ inline void parse_msg_with_xml_tool_calls(common_chat_msg_parser & builder, cons if (!tc) { GGML_ASSERT(builder.pos() == builder.input().size()); GGML_ASSERT(unclosed_reasoning_content.empty()); - GGML_ASSERT(!reasoning_unclosed); + if (!form.allow_toolcall_in_think) GGML_ASSERT(!reasoning_unclosed); break; } diff --git a/common/chat.cpp b/common/chat.cpp index 960c7ee23e2dc..a492d803feeb0 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1976,6 +1976,7 @@ static void common_chat_parse_kimi_k2(common_chat_msg_parser & builder) { form.scope_end = "<|tool_calls_section_end|>"; form.raw_argval = false; form.last_val_end = ""; + form.allow_toolcall_in_think = true; return form; })(); builder.consume_reasoning_with_xml_tool_calls(form, "", ""); diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index d121f4b70811c..b87bc3c34e1cc 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -428,10 +428,38 @@ static void test_templates(const struct common_chat_templates * tmpls, const std */ template static void test_parser_with_streaming(const common_chat_msg & expected, const std::string & raw_message, T parse_msg) { + constexpr auto utf8_truncate_safe = [](const std::string_view s) -> size_t { + auto len = s.size(); + if (len == 0) return 0; + auto i = len; + for (size_t back = 0; back < 4 && i > 0; ++back) { + --i; + unsigned char c = s[i]; + if ((c & 0x80) == 0) { + return len; + } else if ((c & 0xC0) == 0xC0) { + size_t expected_len = 0; + if ((c & 0xE0) == 0xC0) expected_len = 2; + else if ((c & 0xF0) == 0xE0) expected_len = 3; + else if ((c & 0xF8) == 0xF0) expected_len = 4; + else return i; + if (len - i >= expected_len) { + return len; + } else { + return i; + } + } + } + return len - std::min(len, size_t(3)); + }; + constexpr auto utf8_truncate_safe_view = [utf8_truncate_safe](const std::string_view s) { + return s.substr(0, utf8_truncate_safe(s)); + }; + auto merged = simple_assist_msg(""); auto last_msg = parse_msg(""); for (size_t i = 1; i <= raw_message.size(); ++i) { - auto curr_msg = parse_msg(raw_message.substr(0, i)); + auto curr_msg = parse_msg(std::string(utf8_truncate_safe_view(std::string_view(raw_message).substr(0, i)))); if (curr_msg == simple_assist_msg("")) continue; LOG_INF("Streaming msg: %s\n", common_chat_msgs_to_json_oaicompat({curr_msg}).dump().c_str()); for (auto diff: common_chat_msg_diff::compute_diffs(last_msg, curr_msg)) { @@ -2752,6 +2780,93 @@ Hey there!<|im_end|> "{\"args\": [{\"path\": \"src/providers/ThemeProvider.tsx\"}, {\"path\": \"src/components/Header.tsx\"}, {\"path\": \"src/components/ThemeToggle.tsx\"}, {\"path\": \"src/app/globals.css\"}, {\"path\": \"src/app/layout.tsx\"}]}" "<|tool_call_end|><|tool_calls_section_end|>", [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_KIMI_K2}); }); + test_parser_with_streaming( + simple_assist_msg( + "Let me start by examining the relevant files to understand the current implementation.", "", + "read_file", + "{\"files\": [{\"path\": \"src/app/Partners.tsx\", \"line_ranges\": [\"1-100\"]}]}"), + "Let me start by examining the relevant files to understand the current implementation." + "<|tool_calls_section_begin|><|tool_call_begin|>functions.read_file:0<|tool_call_argument_begin|>" + "{\"files\":[{\"path\":\"src/app/Partners.tsx\",\"line_ranges\":[\"1-100\"]}]}" + "<|tool_call_end|><|tool_calls_section_end|>", + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_KIMI_K2}); }); + auto multi_tool_msg = simple_assist_msg("Let me call multiple tools.", "I'm thinking."); + multi_tool_msg.tool_calls.push_back({ "read_file", "{\"files\": [{\"path\": \"src/app/Partners.tsx\", \"line_ranges\": [\"1-100\"]}]}", "" }); + multi_tool_msg.tool_calls.push_back({ "web_search", "{\"query\":\"\\\"From Zero\\\" Linkin Park album tracklist complete songs\",\"limit\":3,\"type\":\"text\"}", "" }); + multi_tool_msg.tool_calls.push_back({ "complex_function", "{\"name\": \"John Doe\", \"age\": 30, \"active\": true, \"score\": 95.5}", "" }); + multi_tool_msg.tool_calls.push_back({ "emoji_function", "{\"message\":\"Hello! 👋 🌟 🚀 Testing emojis: 😀😃😄😁 and symbols: ∑∏∆∇\"}", "" }); + test_parser_with_streaming(multi_tool_msg, + "I'm thinking.Let me call multiple tools." + "<|tool_calls_section_begin|>" + "<|tool_call_begin|>functions.read_file:0<|tool_call_argument_begin|>" + "{\"files\":[{\"path\":\"src/app/Partners.tsx\",\"line_ranges\":[\"1-100\"]}]}" + "<|tool_call_end|>" + "<|tool_call_begin|>functions.web_search:1<|tool_call_argument_begin|>" + "{\"query\":\"\\\"From Zero\\\" Linkin Park album tracklist complete songs\",\"limit\":3,\"type\":\"text\"}" + "<|tool_call_end|>" + "<|tool_call_begin|>functions.complex_function:2<|tool_call_argument_begin|>" + "{\"name\": \"John Doe\", \"age\": 30, \"active\": true, \"score\": 95.5}" + "<|tool_call_end|>" + "<|tool_call_begin|>functions.emoji_function:3<|tool_call_argument_begin|>" + "{\"message\":\"Hello! 👋 🌟 🚀 Testing emojis: 😀😃😄😁 and symbols: ∑∏∆∇\"}" + "<|tool_call_end|>" + "<|tool_calls_section_end|>", + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, { + COMMON_CHAT_FORMAT_KIMI_K2, + COMMON_REASONING_FORMAT_DEEPSEEK + }); }); + test_parser_with_streaming( + simple_assist_msg("", "I'm thinking", "complex_function_in_think", "{\"name\":\"John Doe\",\"age\":30,\"active\":true,\"score\":95.5}"), + "I'm thinking<|tool_calls_section_begin|><|tool_call_begin|>functions.complex_function_in_think:0<|tool_call_argument_begin|>" + "{\"name\": \"John Doe\", \"age\": 30, \"active\": true, \"score\": 95.5}" + "<|tool_call_end|><|tool_calls_section_end|>", + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, { + COMMON_CHAT_FORMAT_KIMI_K2, + COMMON_REASONING_FORMAT_DEEPSEEK + }); }); + test_parser_with_streaming( + simple_assist_msg("Hello", "I'm thinkingI'm still thinking", "complex_function_in_think", "{\"name\":\"John Doe\",\"age\":30,\"active\":true,\"score\":95.5}"), + "I'm thinking<|tool_calls_section_begin|><|tool_call_begin|>functions.complex_function_in_think:0<|tool_call_argument_begin|>" + "{\"name\": \"John Doe\", \"age\": 30, \"active\": true, \"score\": 95.5}" + "<|tool_call_end|><|tool_calls_section_end|>I'm still thinkingHello", + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, { + COMMON_CHAT_FORMAT_KIMI_K2, + COMMON_REASONING_FORMAT_DEEPSEEK + }); }); + + // Test template rendering + common_chat_templates_inputs conversation_with_tools = inputs_tools; + conversation_with_tools.messages.push_back(simple_assist_msg("Let's do it", "Think first", "complex_function", "{\"name\":\"John Doe\",\"age\":30,\"active\":true,\"score\":95.5}")); + conversation_with_tools.messages.push_back({ + "tool", + "Tool response 1", + /* .content_parts = */ {}, + /* .tool_calls = */ {}, + /* .reasoning_content = */ "", + /* .tool_name = */ "complex_function", + /* .tool_call_id = */ "", + }); + conversation_with_tools.messages.push_back(simple_assist_msg("Continue", "Think next", "web_search", "{\"query\":\"\\\"From Zero\\\" Linkin Park album tracklist complete songs\",\"limit\":3,\"type\":\"text\"}")); + conversation_with_tools.messages.push_back({ + "tool", + "Tool response 2", + /* .content_parts = */ {}, + /* .tool_calls = */ {}, + /* .reasoning_content = */ "", + /* .tool_name = */ "web_search", + /* .tool_call_id = */ "", + }); + conversation_with_tools.messages.push_back(simple_assist_msg("CC", "Think last", "read_file", "{\"args\": [{\"path\": \"src/providers/ThemeProvider.tsx\"}, {\"path\": \"src/components/Header.tsx\"}, {\"path\": \"src/components/ThemeToggle.tsx\"}, {\"path\": \"src/app/globals.css\"}, {\"path\": \"src/app/layout.tsx\"}]}")); + conversation_with_tools.messages.push_back({ + "tool", + "Tool response 3", + /* .content_parts = */ {}, + /* .tool_calls = */ {}, + /* .reasoning_content = */ "", + /* .tool_name = */ "read_file", + /* .tool_call_id = */ "", + }); + assert_equals(common_chat_templates_apply(tmpls.get(), conversation_with_tools).prompt, std::string("<|im_system|>tool_declare<|im_middle|>[{\"type\": \"function\", \"function\": {\"name\": \"special_function\", \"description\": \"I'm special\", \"parameters\": {\"type\": \"object\", \"properties\": {\"arg1\": {\"type\": \"integer\", \"description\": \"The arg.\"}}, \"required\": [\"arg1\"]}}}]<|im_end|><|im_system|>system<|im_middle|>You are Kimi, an AI assistant created by Moonshot AI.<|im_end|><|im_user|>user<|im_middle|>Hey there!<|im_end|><|im_assistant|>assistant<|im_middle|>Think firstLet's do it<|tool_calls_section_begin|><|tool_call_begin|>functions.complex_function:0<|tool_call_argument_begin|>{\"name\":\"John Doe\",\"age\":30,\"active\":true,\"score\":95.5}<|tool_call_end|><|tool_calls_section_end|><|im_end|><|im_system|>complex_function<|im_middle|>## Return of functions.complex_function:0\nTool response 1<|im_end|><|im_assistant|>assistant<|im_middle|>Think nextContinue<|tool_calls_section_begin|><|tool_call_begin|>functions.web_search:1<|tool_call_argument_begin|>{\"query\":\"\\\"From Zero\\\" Linkin Park album tracklist complete songs\",\"limit\":3,\"type\":\"text\"}<|tool_call_end|><|tool_calls_section_end|><|im_end|><|im_system|>web_search<|im_middle|>## Return of functions.web_search:1\nTool response 2<|im_end|><|im_assistant|>assistant<|im_middle|>Think lastCC<|tool_calls_section_begin|><|tool_call_begin|>functions.read_file:2<|tool_call_argument_begin|>{\"args\": [{\"path\": \"src/providers/ThemeProvider.tsx\"}, {\"path\": \"src/components/Header.tsx\"}, {\"path\": \"src/components/ThemeToggle.tsx\"}, {\"path\": \"src/app/globals.css\"}, {\"path\": \"src/app/layout.tsx\"}]}<|tool_call_end|><|tool_calls_section_end|><|im_end|><|im_system|>read_file<|im_middle|>## Return of functions.read_file:2\nTool response 3<|im_end|><|im_assistant|>assistant<|im_middle|>")); // Test template generation for regular content test_templates(tmpls.get(), end_tokens, message_assist, tools,