From c822e73c4d5484a03ea0c367ded40ab3f832a56d Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 9 Nov 2025 20:07:12 -0600 Subject: [PATCH 001/183] common : implement parser combinators to simplify chat parsing --- common/CMakeLists.txt | 2 + common/chat-parser-combinator.cpp | 819 ++++++++++++++++++++++++++ common/chat-parser-combinator.h | 158 +++++ tests/CMakeLists.txt | 1 + tests/test-chat-parser-combinator.cpp | 472 +++++++++++++++ 5 files changed, 1452 insertions(+) create mode 100644 common/chat-parser-combinator.cpp create mode 100644 common/chat-parser-combinator.h create mode 100644 tests/test-chat-parser-combinator.cpp diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 7086d08e5e5e9..7bdc9aab5995f 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -48,6 +48,8 @@ add_library(${TARGET} STATIC arg.cpp arg.h base64.hpp + chat-parser-combinator.cpp + chat-parser-combinator.h chat-parser.cpp chat-parser.h chat.cpp diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp new file mode 100644 index 0000000000000..f2182980b3bac --- /dev/null +++ b/common/chat-parser-combinator.cpp @@ -0,0 +1,819 @@ +#include "chat-parser-combinator.h" +#include "common.h" +#include "log.h" + +#include +#include + +class parser_base { + protected: + int id_; + + void set_id(int id) { id_ = id; } + + public: + parser_base(int id) : id_(id) {} + + virtual parser_type type() const = 0; + virtual parser_result parse(parser_context & ctx, size_t start = 0) = 0; + virtual std::string dump() const = 0; + virtual void assign_ids_internal(int& next_id) { + if (id_ == -1) { + id_ = next_id++; + } + } +}; + +class literal_parser : public parser_base { + std::string literal_; + + public: + literal_parser(const std::string & literal, int id) : parser_base(id), literal_(literal) {} + + parser_type type() const override { return PARSER_LITERAL; } + + parser_result parse(parser_context & ctx, size_t start = 0) override { + auto cached = ctx.memo.get(id_, start); + if (cached != std::nullopt) { + return *cached; + } + + auto pos = start; + for (auto i = 0u; i < literal_.size(); ++i) { + if (pos >= ctx.input.size()) { + if (ctx.input_is_complete) { + return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_FAIL, start)); + } + if (i > 0) { + return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, pos); + } + return parser_result(PARSER_RESULT_FAIL, start); + } + if (ctx.input[pos] != literal_[i]) { + return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_FAIL, start)); + } + ++pos; + } + + return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_SUCCESS, start, pos)); + } + + std::string dump() const override { + return "Literal(" + literal_ + ")"; + } +}; + +class sequence_parser : public parser_base { + std::vector parsers_; + + public: + sequence_parser(std::initializer_list parsers, int id) : parser_base(id) { + for (const auto & p : parsers) { + if (p.is_sequence()) { + // Flatten sequences + for (const auto & embedded : p.to_sequence()->parsers()) { + parsers_.push_back(embedded); + } + } else { + parsers_.push_back(p); + } + } + } + + parser_type type() const override { return PARSER_SEQUENCE; } + + parser_result parse(parser_context & ctx, size_t start = 0) override { + auto cached = ctx.memo.get(id_, start); + if (cached != std::nullopt) { + return *cached; + } + + std::unordered_map groups; + + auto pos = start; + for (const auto & p : parsers_) { + auto result = p->parse(ctx, pos); + + // Copy groups + groups.insert(result.groups.begin(), result.groups.end()); + + if (result.is_fail()) { + if (result.end >= ctx.input.size() && !ctx.input_is_complete) { + // If we fail because we don't have enough input, then return success + return parser_result(PARSER_RESULT_SUCCESS, start, result.end, groups); + } + return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_FAIL, start, result.end, groups)); + } + + if (result.is_need_more_input()) { + return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, result.end, groups); + } + + pos = result.end; + } + + return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_SUCCESS, start, pos, groups)); + } + + std::string dump() const override { + std::vector parts; + parts.reserve(parsers_.size()); + for (const auto & p : parsers_) { + parts.push_back(p->dump()); + } + return "Sequence(" + string_join(parts, ", ") + ")"; + } + + const std::vector & parsers() const { return parsers_; } + + void assign_ids_internal(int& next_id) override { + if (id_ == -1) { + id_ = next_id++; + } + for (auto & p : parsers_) { + p->assign_ids_internal(next_id); + } + } +}; + +class choice_parser : public parser_base { + std::vector parsers_; + + public: + choice_parser(std::initializer_list parsers, int id) : parser_base(id) { + for (const auto & p : parsers) { + if (p.is_choice()) { + // Flatten choices + for (const auto & embedded : p.to_choice()->parsers()) { + parsers_.push_back(embedded); + } + } else { + parsers_.push_back(p); + } + } + } + + parser_type type() const override { return PARSER_CHOICE; } + + parser_result parse(parser_context & ctx, size_t start = 0) override { + auto cached = ctx.memo.get(id_, start); + if (cached != std::nullopt) { + return *cached; + } + + auto pos = start; + for (const auto & p : parsers_) { + auto result = p->parse(ctx, pos); + + if (result.is_success()) { + return ctx.memo.set(id_, start, result); + } + + if (result.is_need_more_input()) { + return result; + } + } + + return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_FAIL, start)); + } + + std::string dump() const override { + std::vector parts; + parts.reserve(parsers_.size()); + for (const auto & p : parsers_) { + parts.push_back(p->dump()); + } + return "Choice(" + string_join(parts, ", ") + ")"; + } + + const std::vector & parsers() const { return parsers_; } + + void assign_ids_internal(int& next_id) override { + if (id_ == -1) { + id_ = next_id++; + } + for (auto & p : parsers_) { + p->assign_ids_internal(next_id); + } + } +}; + +class one_or_more_parser : public parser_base { + parser parser_; + + public: + one_or_more_parser(const parser & parser, int id) : parser_base(id), parser_(parser) {} + + parser_type type() const override { return PARSER_ONE_OR_MORE; } + + parser_result parse(parser_context & ctx, size_t start = 0) override { + auto cached = ctx.memo.get(id_, start); + std::unordered_map groups; + + // We can't return back the cached result, since there may be more + // repetitions since the last parsing attempt. Instead, resume parsing from + // the last successful repetition found. + auto pos = start; + if (cached != std::nullopt) { + pos = cached->end; + groups.insert(cached->groups.begin(), cached->groups.end()); + } + + if (pos == start) { + auto first_result = parser_->parse(ctx, pos); + if (!first_result.is_success()) { + return first_result; + } + + pos = first_result.end; + groups.insert(first_result.groups.begin(), first_result.groups.end()); + } + + for (;;) { + auto result = parser_->parse(ctx, pos); + groups.insert(result.groups.begin(), result.groups.end()); + + if (result.is_need_more_input()) { + return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, pos, groups); + } + + if (result.is_fail()) { + // Done with repetitions + break; + } + + if (result.end == pos) { + break; // Prevent an infinite loop + } + + pos = result.end; + } + + return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_SUCCESS, start, pos, groups)); + } + + std::string dump() const override { + return "OneOrMore(" + parser_->dump() + ")"; + } + + const parser & child() const { return parser_; } + + void assign_ids_internal(int& next_id) override { + if (id_ == -1) { + id_ = next_id++; + } + parser_->assign_ids_internal(next_id); + } +}; + +class zero_or_more_parser : public parser_base { + parser parser_; + + public: + zero_or_more_parser(const parser & parser, int id) : parser_base(id), parser_(parser) {} + + parser_type type() const override { return PARSER_ZERO_OR_MORE; } + + parser_result parse(parser_context & ctx, size_t start = 0) override { + auto cached = ctx.memo.get(id_, start); + std::unordered_map groups; + + // We can't return back the cached result, since there may be more + // repetitions since the last parsing attempt. Instead, resume parsing from + // the last successful repetition found. + auto pos = start; + if (cached != std::nullopt) { + pos = cached->end; + groups.insert(cached->groups.begin(), cached->groups.end()); + } + + for (;;) { + auto result = parser_->parse(ctx, pos); + groups.insert(result.groups.begin(), result.groups.end()); + + if (result.is_need_more_input()) { + return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, pos, groups); + } + + if (result.is_fail()) { + // Done with repetitions (zero or more is always valid) + break; + } + + if (result.end == pos) { + break; // Prevent an infinite loop + } + + pos = result.end; + } + + return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_SUCCESS, start, pos, groups)); + } + + std::string dump() const override { + return "ZeroOrMore(" + parser_->dump() + ")"; + } + + const parser & child() const { return parser_; } + + void assign_ids_internal(int& next_id) override { + if (id_ == -1) { + id_ = next_id++; + } + parser_->assign_ids_internal(next_id); + } +}; + +class optional_parser : public parser_base { + parser parser_; + + public: + optional_parser(const parser & parser, int id) : parser_base(id), parser_(parser) {} + + parser_type type() const override { return PARSER_OPTIONAL; } + + parser_result parse(parser_context & ctx, size_t start = 0) override { + auto cached = ctx.memo.get(id_, start); + if (cached != std::nullopt) { + return *cached; + } + + auto result = parser_->parse(ctx, start); + + if (result.is_success()) { + // Matched successfully + return ctx.memo.set(id_, start, result); + } + + if (result.is_need_more_input()) { + // Propagate - need more input to determine if optional matches + return result; + } + + // No match, but optional always succeeds with zero matches + return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_SUCCESS, start, start)); + } + + std::string dump() const override { + return "Optional(" + parser_->dump() + ")"; + } + + const parser & child() const { return parser_; } + + void assign_ids_internal(int& next_id) override { + if (id_ == -1) { + id_ = next_id++; + } + parser_->assign_ids_internal(next_id); + } +}; + +class not_parser : public parser_base { + parser parser_; + + public: + not_parser(const parser & parser, int id) : parser_base(id), parser_(parser) {} + + parser_type type() const override { return PARSER_NOT; } + + parser_result parse(parser_context & ctx, size_t start = 0) override { + auto cached = ctx.memo.get(id_, start); + if (cached != std::nullopt) { + return *cached; + } + + auto result = parser_->parse(ctx, start); + + if (result.is_success()) { + // Fail if the underlying parser matches + return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_FAIL, start)); + } + + if (result.is_need_more_input()) { + // Propagate - need to know what child would match before negating + return result; + } + + // Child failed, so negation succeeds + return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_SUCCESS, start)); + } + + std::string dump() const override { + return "Not(" + parser_->dump() + ")"; + } + + const parser & child() const { return parser_; } + + void assign_ids_internal(int& next_id) override { + if (id_ == -1) { + id_ = next_id++; + } + parser_->assign_ids_internal(next_id); + } +}; + +class any_parser : public parser_base { + public: + any_parser(int id) : parser_base(id) {} + + parser_type type() const override { return PARSER_ANY; } + + parser_result parse(parser_context & ctx, size_t start = 0) override { + auto cached = ctx.memo.get(id_, start); + if (cached != std::nullopt) { + return *cached; + } + + if (start >= ctx.input.size()) { + if (ctx.input_is_complete) { + return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_FAIL, start)); + } + return parser_result(PARSER_RESULT_FAIL, start); + } + + return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_SUCCESS, start, start + 1)); + } + + std::string dump() const override { + return "Any"; + } +}; + +class char_class_parser : public parser_base { + struct char_range { + int start; + int end; + + bool contains(char c) const { return (int)c >= start && int(c) <= end; } + }; + + std::string pattern_; + std::vector ranges_; + + public: + char_class_parser(const std::string & classes, int id) : parser_base(id), pattern_(classes) { + std::string content = classes; + if (content.front() == '[') { + content = content.substr(1); + } + + if (content.back() == ']') { + content.pop_back(); + } + + auto parse_char = [&](size_t pos) -> std::pair { + if (content[pos] == '\\' && pos + 1 < content.length()) { + char next = content[pos + 1]; + switch (next) { + case 'n': return {'\n', 2}; + case 't': return {'\t', 2}; + case 'r': return {'\r', 2}; + case '\\': return {'\\', 2}; + case ']': return {']', 2}; + case '-': return {'-', 2}; + case '[': return {'[', 2}; + default: return {next, 2}; // Treat as literal escaped character + } + } + return {content[pos], 1}; + }; + + size_t i = 0; + while (i < content.length()) { + auto [start, start_len] = parse_char(i); + i += start_len; + + if (i + 1 < content.length() && content[i] == '-') { + // Range detected + auto [end, end_len] = parse_char(i + 1); + ranges_.push_back(char_range{start, end}); + i += 1 + end_len; + } else { + ranges_.push_back(char_range{start, start}); + } + } + } + + parser_type type() const override { return PARSER_CHAR_CLASS; } + + parser_result parse(parser_context & ctx, size_t start = 0) override { + auto cached = ctx.memo.get(id_, start); + if (cached != std::nullopt) { + return *cached; + } + + if (start >= ctx.input.size()) { + if (ctx.input_is_complete) { + return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_FAIL, start)); + } + return parser_result(PARSER_RESULT_FAIL, start); + } + + for (const auto & range : ranges_) { + if (range.contains(ctx.input[start])) { + return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_SUCCESS, start, start + 1)); + } + } + + return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_FAIL, start)); + } + + std::string dump() const override { + return "Char(" + pattern_ + ")"; + } +}; + +class group_parser : public parser_base { + std::string name_; + parser parser_; + + public: + group_parser(const std::string & name, const parser & parser, int id) : parser_base(id), name_(name), parser_(parser) {} + + parser_type type() const override { return PARSER_GROUP; } + + parser_result parse(parser_context & ctx, size_t start = 0) override { + auto result = parser_->parse(ctx, start); + + // Store result + result.groups[name_] = parser_match_location{result.start, result.end}; + return ctx.memo.set(id_, start, result); + } + + std::string dump() const override { + return "Group(" + name_ + ", " + parser_->dump() + ")"; + } + + void assign_ids_internal(int& next_id) override { + if (id_ == -1) { + id_ = next_id++; + } + parser_->assign_ids_internal(next_id); + } +}; + +class rule_parser : public parser_base { + std::string rule_name_; + std::shared_ptr> rules_; + + public: + rule_parser(const std::string & name, std::shared_ptr> rules, int id) + : parser_base(id), rule_name_(name), rules_(std::move(rules)) {} + + parser_type type() const override { return PARSER_RULE; } + + parser_result parse(parser_context & ctx, size_t start = 0) override { + auto cached = ctx.memo.get(id_, start); + if (cached != std::nullopt) { + return *cached; + } + + if (!rules_) { + LOG_ERR("rule_parser::parse called without rule registry\n"); + return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_FAIL, start)); + } + + auto it = rules_->find(rule_name_); + if (it == rules_->end()) { + LOG_ERR("rule_parser::parse rule '%s' not found in registry\n", rule_name_.c_str()); + return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_FAIL, start)); + } + + auto result = it->second->parse(ctx, start); + return ctx.memo.set(id_, start, result); + } + + std::string dump() const override { + return "Rule(" + rule_name_ + ")"; + } +}; + +std::optional parser_result::group(const std::string & name, std::string_view input) const { + auto it = groups.find(name); + if (it == groups.end()) { + return std::nullopt; + } + + return std::string(it->second.view(input)); +} + +parser_result parse_cache::set(int id, size_t start, parser_result result) { + if (id == -1) { + // Don't cache parsers with ID -1 (from operators and global factory functions) + return result; + } + results[parse_cache_key{id, start}] = result; + return result; +} + +std::optional parse_cache::get(int id, size_t start) { + if (id == -1) { + // Don't cache parsers with ID -1 (from operators and global factory functions) + return std::nullopt; + } + auto it = results.find(parse_cache_key{id, start}); + if (it != results.end()) { + return it->second; + } + return std::nullopt; +} + +void parse_cache::clear() { + results.clear(); +} + +parser::parser() {} + +parser::parser(std::shared_ptr parser) : ptr(std::move(parser)) {} + +parser parser::operator~() const { + return parser(std::make_shared(*this, -1)); +} + +parser parser::operator+(const parser & other) const { + return parser(std::shared_ptr(new sequence_parser({*this, other}, -1))); +} + +parser parser::operator|(const parser & other) const { + return parser(std::shared_ptr(new choice_parser({*this, other}, -1))); +} + +parser_base & parser::operator*() const { + return *ptr; +} + +parser_base * parser::operator->() const { + return ptr.get(); +} + +bool parser::is_sequence() const { + return ptr->type() == PARSER_SEQUENCE; +} + +std::shared_ptr parser::to_sequence() const { + return std::dynamic_pointer_cast(ptr); +} + +bool parser::is_choice() const { + return ptr->type() == PARSER_CHOICE; +} + +std::shared_ptr parser::to_choice() const { + return std::dynamic_pointer_cast(ptr); +} + +parser_type parser::type() const { + return ptr->type(); +} + +parser_result parser::parse(parser_context & ctx, size_t start) const { + return ptr->parse(ctx, start); +} + +std::string parser::dump() const { + return ptr->dump(); +} + +parser_builder::parser_builder() + : rules_(std::make_shared>()) + , next_id_(0) {} + +parser parser_builder::literal(const std::string & literal) { + return parser(std::make_shared(literal, next_id_++)); +} + +parser parser_builder::sequence(std::initializer_list parsers) { + return parser(std::shared_ptr(new sequence_parser(parsers, next_id_++))); +} + +parser parser_builder::choice(std::initializer_list parsers) { + return parser(std::shared_ptr(new choice_parser(parsers, next_id_++))); +} + +parser parser_builder::one_or_more(const parser & p) { + return parser(std::make_shared(p, next_id_++)); +} + +parser parser_builder::zero_or_more(const parser & p) { + return parser(std::make_shared(p, next_id_++)); +} + +parser parser_builder::optional(const parser & p) { + return parser(std::make_shared(p, next_id_++)); +} + +parser parser_builder::negate(const parser & p) { + return parser(std::make_shared(p, next_id_++)); +} + +parser parser_builder::any() { + return parser(std::make_shared(next_id_++)); +} + +parser parser_builder::char_class(const std::string & classes) { + return parser(std::make_shared(classes, next_id_++)); +} + +parser parser_builder::group(const std::string & name, const parser & p) { + return parser(std::make_shared(name, p, next_id_++)); +} + +parser parser_builder::rule(const std::string & name) { + return parser(std::make_shared(name, rules_, next_id_++)); +} + +parser parser_builder::space() { + return zero_or_more(char_class("[ \\t\\n\\r]")); +} + +parser parser_builder::add_rule(const std::string & name, const parser & p) { + (*rules_)[name] = p; + return rule(name); +} + +void parser_builder::assign_ids(parser & p) { + if (p.ptr) { + p.ptr->assign_ids_internal(next_id_); + } +} + +parser parser_builder::add_json_rule(const std::string & name) { + // Whitespace: space, tab, newline, carriage return + auto ws = zero_or_more(char_class("[ \\t\\n\\r]")); + + // Number components + auto digit = char_class("[0-9]"); + auto digit1_9 = char_class("[1-9]"); + auto digits = one_or_more(digit); + + // Integer part: 0 or non-zero digit followed by more digits + auto int_part = literal("0") | (digit1_9 + zero_or_more(digit)); + + // Optional fractional part + auto frac = literal(".") + digits; + + // Optional exponent part + auto exp = (literal("e") | literal("E")) + optional(char_class("[+\\-]")) + digits; + + // Complete number + auto number = optional(literal("-")) + int_part + optional(frac) + optional(exp); + + add_rule("json_number", number); + + // String components + auto hex = char_class("[0-9a-fA-F]"); + auto unicode_escape = literal("\\u") + hex + hex + hex + hex; + auto simple_escape = literal("\\") + char_class("[\"\\\\bfnrt/]"); + auto escape = simple_escape | unicode_escape; + + // String character: escape sequence or any char except quote and backslash + auto string_char = escape | (~char_class("[\"\\\\]") + any()); + auto string = literal("\"") + zero_or_more(string_char) + literal("\""); + + add_rule("json_string", string); + + // Literals + auto true_lit = literal("true"); + auto false_lit = literal("false"); + auto null_lit = literal("null"); + + // Value - uses forward references for recursive structures + add_rule("json_value", + rule("json_object") | + rule("json_array") | + rule("json_string") | + rule("json_number") | + true_lit | + false_lit | + null_lit + ); + + // Object: { "key": value, ... } + auto member = rule("json_string") + ws + literal(":") + ws + rule("json_value"); + auto members = member + zero_or_more(ws + literal(",") + ws + member); + + // Empty object or object with members + auto object = (literal("{") + ws + literal("}")) | + (literal("{") + ws + members + ws + literal("}")); + + add_rule("json_object", object); + + // Array: [ value, ... ] + auto elements = rule("json_value") + zero_or_more(ws + literal(",") + ws + rule("json_value")); + + // Empty array or array with elements + auto array = (literal("[") + ws + literal("]")) | + (literal("[") + ws + elements + ws + literal("]")); + + add_rule("json_array", array); + + // Register the main rule with the provided name + return add_rule(name, rule("json_value")); +} + +parser build_parser(const std::function & fn) { + parser_builder builder; + auto root = fn(builder); + builder.assign_ids(root); // Assign IDs to rules that were created with operators + return root; +} diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h new file mode 100644 index 0000000000000..72adf523c489a --- /dev/null +++ b/common/chat-parser-combinator.h @@ -0,0 +1,158 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +enum parser_type { + PARSER_LITERAL = 0, + PARSER_SEQUENCE = 1, + PARSER_CHOICE = 2, + PARSER_ZERO_OR_MORE = 3, + PARSER_ONE_OR_MORE = 4, + PARSER_NOT = 5, + PARSER_ANY = 6, + PARSER_CHAR_CLASS = 7, + PARSER_GROUP = 8, + PARSER_RULE = 9, + PARSER_OPTIONAL = 10, +}; + +enum parser_result_type { + PARSER_RESULT_FAIL = 0, + PARSER_RESULT_NEED_MORE_INPUT = 1, + PARSER_RESULT_SUCCESS = 2, +}; + +struct parse_cache_key { + int id; + size_t start; + + bool operator==(const parse_cache_key & other) const { + return id == other.id && start == other.start; + } +}; + +template <> +struct std::hash { + std::size_t operator()(const parse_cache_key & k) const { + return std::hash{}(((size_t)k.id << 32) | k.start); + } +}; + +struct parser_match_location { + size_t start; + size_t end; + + size_t length() const { return end - start; } + + std::string_view view(std::string_view sv) const { + return sv.substr(start, length()); + } +}; + +struct parser_result { + parser_result_type type = PARSER_RESULT_FAIL; + size_t start = 0; + size_t end = 0; + + std::unordered_map groups; + + parser_result() : type(PARSER_RESULT_FAIL) {} + parser_result(parser_result_type type, size_t start) : type(type), start(start), end(start) {} + parser_result(parser_result_type type, size_t start, size_t end) : type(type), start(start), end(end) {} + parser_result(parser_result_type type, size_t start, size_t end, const std::unordered_map & groups) : type(type), start(start), end(end), groups(groups) {} + + bool is_fail() const { return type == PARSER_RESULT_FAIL; } + bool is_need_more_input() const { return type == PARSER_RESULT_NEED_MORE_INPUT; } + bool is_success() const { return type == PARSER_RESULT_SUCCESS; } + + std::optional group(const std::string & name, std::string_view input) const; +}; + +class parse_cache { + std::unordered_map results; + + public: + parser_result set(int id, size_t start, parser_result result); + std::optional get(int id, size_t start); + void clear(); +}; + +class parser; + +struct parser_context { + std::string_view input; + parse_cache memo; + bool input_is_complete = true; +}; + +class parser_base; +class sequence_parser; +class choice_parser; +class parser_builder; + +class parser { + std::shared_ptr ptr; + + friend class parser_builder; + + public: + parser(); + parser(std::shared_ptr parser); + parser(const parser & other) = default; + parser & operator=(const parser & other) { + if (this != &other) { + ptr = other.ptr; + } + return *this; + } + + parser operator~() const; + parser operator+(const parser & other) const; + parser operator|(const parser & other) const; + + parser_base & operator*() const; + parser_base * operator->() const; + + bool is_sequence() const; + std::shared_ptr to_sequence() const; + + bool is_choice() const; + std::shared_ptr to_choice() const; + + parser_type type() const; + parser_result parse(parser_context & ctx, size_t start = 0) const; + std::string dump() const; +}; + +class parser_builder { + std::shared_ptr> rules_; + int next_id_; + + public: + parser_builder(); + + parser literal(const std::string & literal); + parser sequence(std::initializer_list parsers); + parser choice(std::initializer_list parsers); + parser one_or_more(const parser & p); + parser zero_or_more(const parser & p); + parser optional(const parser & p); + parser negate(const parser & p); + parser any(); + parser char_class(const std::string & classes); + parser group(const std::string & name, const parser & p); + parser rule(const std::string & name); + parser space(); + + parser add_rule(const std::string & name, const parser & p); + parser add_json_rule(const std::string & name); + + void assign_ids(parser & p); +}; + +parser build_parser(const std::function & fn); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d9cc5e933f4ce..90badf62af667 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -180,6 +180,7 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS) endif() llama_build_and_test(test-chat-parser.cpp) +llama_build_and_test(test-chat-parser-combinator.cpp) llama_build_and_test(test-chat-template.cpp) llama_build_and_test(test-json-partial.cpp) llama_build_and_test(test-log.cpp) diff --git a/tests/test-chat-parser-combinator.cpp b/tests/test-chat-parser-combinator.cpp new file mode 100644 index 0000000000000..55a443aed3e38 --- /dev/null +++ b/tests/test-chat-parser-combinator.cpp @@ -0,0 +1,472 @@ +#include +#include + +#include "chat-parser-combinator.h" + +template +static void assert_equals(const std::string_view label, const T & expected, const T & actual) { + if (expected != actual) { + std::cerr << label << "\n"; + std::cerr << "Expected: " << expected << "\n"; + std::cerr << "Actual: " << actual << "\n"; + std::cerr << std::flush; + throw std::runtime_error("Test failed"); + } +} + +template +static void assert_equals(const T & expected, const T & actual) { + assert_equals("", expected, actual); +} + +static void assert_equals(const char * expected, const std::string & actual) { + assert_equals(expected, actual); +} + +static void test_partial_parsing() { + { + // Test literal + auto parser = build_parser([](parser_builder& p) { + return p.literal("hello"); + }); + + parser_context ctx; + parser_result result; + + ctx = parser_context{"hello", parse_cache()}; + result = parser.parse(ctx); + assert_equals(true, result.is_success()); + } + { + // Test char class + auto parser = build_parser([](parser_builder& p) { + return p.char_class("a-z"); + }); + + parser_context ctx; + parser_result result; + + ctx = parser_context{"a", parse_cache()}; + result = parser.parse(ctx); + assert_equals(true, result.is_success()); + + ctx = parser_context{"A", parse_cache()}; + result = parser.parse(ctx); + assert_equals(true, result.is_fail()); + + parser = build_parser([](parser_builder& p) { + return p.char_class("a-z-"); + }); + + ctx = parser_context{"f", parse_cache()}; + result = parser.parse(ctx); + assert_equals(true, result.is_success()); + + ctx = parser_context{"-", parse_cache()}; + result = parser.parse(ctx); + assert_equals(true, result.is_success()); + + ctx = parser_context{"A", parse_cache()}; + result = parser.parse(ctx); + assert_equals(true, result.is_fail()); + } + { + // Test sequences and literals + auto parser = build_parser([](parser_builder& p) { + return p.literal("") + p.literal(""); + }); + + // Partial matches + auto ctx = parser_context{"", parse_cache(), false}; + result = parser.parse(ctx); + assert_equals(true, result.is_success()); + + ctx = parser_context{"", parse_cache(), true}; + result = parser.parse(ctx); + assert_equals(true, result.is_success()); + + // No match, since it does not adhere to the grammar + ctx = parser_context{"I am parser", parse_cache(), false}; + result = parser.parse(ctx); + assert_equals(true, result.is_fail()); + } + { + // Test choices + auto parser = build_parser([](parser_builder& p) { + return p.literal("") | p.literal(""); + }); + + // Partial matches + auto ctx = parser_context{"", parse_cache(), true}; + result = parser.parse(ctx); + assert_equals(true, result.is_success()); + + ctx = parser_context{"", parse_cache(), true}; + result = parser.parse(ctx); + assert_equals(true, result.is_success()); + + // No match + ctx = parser_context{"", parse_cache(), true}; + result = parser.parse(ctx); + assert_equals(true, result.is_fail()); + } + { + // Test zero_or_more + auto parser = build_parser([](parser_builder& p) { + return p.zero_or_more(p.literal("ab")); + }); + + // Partial matches + auto ctx = parser_context{"a", parse_cache(), false}; + auto result = parser.parse(ctx); + assert_equals(true, result.is_need_more_input()); + + ctx = parser_context{"aba", parse_cache(), false}; + result = parser.parse(ctx); + assert_equals(true, result.is_need_more_input()); + + // Full match + ctx = parser_context{"ab", parse_cache(), true}; + result = parser.parse(ctx); + assert_equals(true, result.is_success()); + } + { + // Test one_or_more + auto parser = build_parser([](parser_builder& p) { + return p.one_or_more(p.literal("ab")); + }); + + // Partial matches + auto ctx = parser_context{"a", parse_cache(), false}; + auto result = parser.parse(ctx); + assert_equals(true, result.is_need_more_input()); + + ctx = parser_context{"aba", parse_cache(), false}; + result = parser.parse(ctx); + assert_equals(true, result.is_need_more_input()); + + // Full match + ctx = parser_context{"ab", parse_cache(), true}; + result = parser.parse(ctx); + assert_equals(true, result.is_success()); + + // No match + ctx = parser_context{"cd", parse_cache(), true}; + result = parser.parse(ctx); + assert_equals(true, result.is_fail()); + } +} + +static void test_capture_groups() { + { + auto parser = build_parser([](parser_builder& p) { + return p.literal("") + + p.group("reasoning_content", + p.zero_or_more(~p.literal("") + p.any()) + ) + + p.literal(""); + }); + + std::string input = "I have a thought"; + auto ctx = parser_context{input, parse_cache()}; + auto result = parser.parse(ctx); + + assert_equals(true, result.is_success()); + + auto it = result.groups.find("reasoning_content"); + assert_equals(true, it != result.groups.end()); + assert_equals("I have a thought", std::string(it->second.view(input))); + } + { + auto parser = build_parser([](parser_builder& p) { + return p.literal("") + + p.group("reasoning_content", + p.zero_or_more(~p.literal("") + p.any()) + ) + + p.literal(""); + }); + + std::string input = "I have a "; + auto ctx = parser_context{input, parse_cache(), false}; + auto result = parser.parse(ctx); + + assert_equals(true, result.is_success()); + + auto it = result.groups.find("reasoning_content"); + assert_equals(true, it != result.groups.end()); + assert_equals("I have a ", std::string(it->second.view(input))); + } + { + auto parser = build_parser([](parser_builder& p) { + return p.literal("") + + p.group("reasoning_content", + p.zero_or_more(~p.literal("") + p.any()) + ) + + p.literal("") + + p.group("content", p.zero_or_more(p.any())); + }); + + std::string input = "The user said hello.Hello!"; + auto ctx = parser_context{input, parse_cache(), true}; + auto result = parser.parse(ctx); + + assert_equals(true, result.is_success()); + + auto it = result.groups.find("reasoning_content"); + assert_equals(true, it != result.groups.end()); + assert_equals("The user said hello.", std::string(it->second.view(input))); + + it = result.groups.find("content"); + assert_equals(true, it != result.groups.end()); + assert_equals("Hello!", std::string(it->second.view(input))); + } +} + +static void test_char_class() { + { + // Test common escape sequences + auto parser = build_parser([](parser_builder& p) { + return p.char_class("[\\n\\t\\\\]"); + }); + + parser_context ctx; + parser_result result; + + ctx = parser_context{"\n", parse_cache()}; + result = parser.parse(ctx); + assert_equals(true, result.is_success()); + + ctx = parser_context{"\t", parse_cache()}; + result = parser.parse(ctx); + assert_equals(true, result.is_success()); + + ctx = parser_context{"\\", parse_cache()}; + result = parser.parse(ctx); + assert_equals(true, result.is_success()); + + ctx = parser_context{" ", parse_cache()}; + result = parser.parse(ctx); + assert_equals(true, result.is_fail()); + } + { + // Test escaped dash (literal dash, not a range) + auto parser = build_parser([](parser_builder& p) { + return p.char_class("[a\\-z]"); + }); + + parser_context ctx; + parser_result result; + + ctx = parser_context{"a", parse_cache()}; + result = parser.parse(ctx); + assert_equals(true, result.is_success()); + + ctx = parser_context{"-", parse_cache()}; + result = parser.parse(ctx); + assert_equals(true, result.is_success()); + + ctx = parser_context{"z", parse_cache()}; + result = parser.parse(ctx); + assert_equals(true, result.is_success()); + + // Should NOT match 'b' since \- is a literal dash, not a range + ctx = parser_context{"b", parse_cache()}; + result = parser.parse(ctx); + assert_equals(true, result.is_fail()); + } +} + +static void test_recursive_references() { + auto value_parser = build_parser([](parser_builder& p) { + p.add_rule("number", p.one_or_more(p.char_class("0-9"))); + p.add_rule("list", p.sequence({ + p.literal("["), + p.rule("value"), + p.literal("]") + })); + return p.add_rule("value", p.rule("number") | p.rule("list")); + }); + + parser_context ctx; + parser_result result; + + // Test simple number + ctx = parser_context{"1", parse_cache(), true}; + result = value_parser.parse(ctx); + assert_equals(true, result.is_success()); + + // Test simple list + ctx = parser_context{"[1]", parse_cache(), true}; + result = value_parser.parse(ctx); + assert_equals(true, result.is_success()); + + // Test nested list + ctx = parser_context{"[[2]]", parse_cache(), true}; + result = value_parser.parse(ctx); + assert_equals(true, result.is_success()); + + // Test deeply nested list + ctx = parser_context{"[[[3]]]", parse_cache(), true}; + result = value_parser.parse(ctx); + assert_equals(true, result.is_success()); + + // Test partial match + ctx = parser_context{"[[", parse_cache(), false}; + result = value_parser.parse(ctx); + assert_equals(true, result.is_success()); + + // Test no match + ctx = parser_context{"[a]", parse_cache(), true}; + result = value_parser.parse(ctx); + assert_equals(true, result.is_fail()); +} + +static void test_optional() { + // Test optional with a match + auto parser = build_parser([](parser_builder& p) { + return p.literal("hello") + p.optional(p.literal(" world")); + }); + + // Full match with optional part present + auto ctx = parser_context{"hello world", parse_cache()}; + auto result = parser.parse(ctx); + assert_equals(true, result.is_success()); + assert_equals((size_t)11, result.end); + + // Full match with optional part absent + ctx = parser_context{"hello", parse_cache(), true}; + result = parser.parse(ctx); + assert_equals(true, result.is_success()); + assert_equals((size_t)5, result.end); + + // Partial match - waiting for more input to determine if optional matches + ctx = parser_context{"hello ", parse_cache(), false}; + result = parser.parse(ctx); + assert_equals(true, result.is_need_more_input()); +} + +static void test_json_parser() { + auto json = build_parser([](parser_builder & p) { + return p.add_json_rule("json"); + }); + + // Test parsing a simple JSON object + std::string input = R"({"name": "test", "value": 42, "flag": true})"; + parser_context ctx{input, parse_cache()}; + + auto result = json.parse(ctx); + + assert_equals(true, result.is_success()); + assert_equals(input.size(), result.end); +} + +static void test_complete_example() { + auto parser = build_parser([](parser_builder & p) { + auto space = p.add_rule("space", p.space()); + + auto reasoning = p.add_rule("reasoning", + p.literal("") + space + + p.group("reasoning-content", + p.zero_or_more(~(space + p.literal("")) + p.any())) + + space + p.literal("")); + + auto content = p.add_rule("content", + p.group("content", + p.zero_or_more(~(space + p.literal("")) + p.any()))); + + auto ident_chars = p.add_rule("ident-chars", p.char_class("[a-zA-Z\\-_]")); + auto json = p.add_json_rule("json"); + + auto tool_call_name = p.add_rule("tool-call-name", + p.literal("") + space + + p.group("tool-name", p.one_or_more(~p.literal("") + ident_chars)) + + space + p.literal("")); + + auto tool_call_args = p.add_rule("tool-call-args", + p.literal("") + space + + p.group("tool-args", json) + + space + p.literal("")); + + auto tool_call = p.add_rule("tool-call", + p.literal("") + space + + tool_call_name + space + + tool_call_args + space + + p.literal("")); + + return p.add_rule("root", reasoning + p.optional(content) + p.optional(tool_call)); + }); + + // Test complete input + std::string input = R"(I need to call get_weather with city = New Yorkget_weather{"city": "New York"})"; + parser_context ctx{input, parse_cache()}; + + auto result = parser.parse(ctx); + + assert_equals(true, result.is_success()); + assert_equals(input.size(), result.end); + assert_equals(std::string("I need to call get_weather with city = New York"), *result.group("reasoning-content", ctx.input)); + assert_equals(std::string("get_weather"), *result.group("tool-name", ctx.input)); + assert_equals(std::string(R"({"city": "New York"})"), *result.group("tool-args", ctx.input)); + + // Test partial input + input = R"(I need to call get_weather )"; + ctx = parser_context{input, parse_cache(), /* .is_input_complete = */ false}; + result = parser.parse(ctx); + + assert_equals(true, result.is_success()); + assert_equals(std::string("I need to call get_weather"), *result.group("reasoning-content", ctx.input)); + + input = R"(I need to call get_weatherget_weather)"; + ctx = parser_context{input, parse_cache(), /* .is_input_complete = */ false}; + result = parser.parse(ctx); + + assert_equals(true, result.is_success()); + assert_equals(std::string("I need to call get_weather"), *result.group("reasoning-content", ctx.input)); + + input = R"(I need to call get_weatherget_weatherI need to call get_weatherget_weather{"cit)"; + ctx = parser_context{input, parse_cache(), /* .is_input_complete = */ false}; + result = parser.parse(ctx); + + assert_equals(true, result.is_success()); + assert_equals(std::string("I need to call get_weather"), *result.group("reasoning-content", ctx.input)); + assert_equals(std::string("get_weather"), *result.group("tool-name", ctx.input)); + assert_equals(std::string(R"({"cit)"), *result.group("tool-args", ctx.input)); +} + +int main() { + test_partial_parsing(); + test_char_class(); + test_capture_groups(); + test_recursive_references(); + test_optional(); + test_json_parser(); + test_complete_example(); + std::cout << "All tests passed!\n"; + return 0; +} From e6153bb14a0728ed7fcce7dadf54c3c7f670dca0 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 9 Nov 2025 22:34:24 -0600 Subject: [PATCH 002/183] add virtual destructor to parser_base --- common/chat-parser-combinator.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index f2182980b3bac..a8d8b10fe4fcd 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -13,6 +13,7 @@ class parser_base { public: parser_base(int id) : id_(id) {} + virtual ~parser_base() = default; virtual parser_type type() const = 0; virtual parser_result parse(parser_context & ctx, size_t start = 0) = 0; From 4ced9996e65817c0da27acca05d64b3acb6a6a08 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 9 Nov 2025 22:52:20 -0600 Subject: [PATCH 003/183] fix memory leak from circular references of rules --- common/chat-parser-combinator.cpp | 53 +++++++++++++++++++++++++------ common/chat-parser-combinator.h | 2 ++ 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index a8d8b10fe4fcd..cd915afcd7b88 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -555,11 +555,11 @@ class group_parser : public parser_base { class rule_parser : public parser_base { std::string rule_name_; - std::shared_ptr> rules_; + std::weak_ptr> rules_; public: rule_parser(const std::string & name, std::shared_ptr> rules, int id) - : parser_base(id), rule_name_(name), rules_(std::move(rules)) {} + : parser_base(id), rule_name_(name), rules_(rules) {} parser_type type() const override { return PARSER_RULE; } @@ -569,13 +569,14 @@ class rule_parser : public parser_base { return *cached; } - if (!rules_) { - LOG_ERR("rule_parser::parse called without rule registry\n"); + auto rules = rules_.lock(); + if (!rules) { + LOG_ERR("rule_parser::parse called with expired rule registry\n"); return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_FAIL, start)); } - auto it = rules_->find(rule_name_); - if (it == rules_->end()) { + auto it = rules->find(rule_name_); + if (it == rules->end()) { LOG_ERR("rule_parser::parse rule '%s' not found in registry\n", rule_name_.c_str()); return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_FAIL, start)); } @@ -589,6 +590,32 @@ class rule_parser : public parser_base { } }; +class root_parser : public parser_base { + parser root_; + std::shared_ptr> rules_; + + public: + root_parser(const parser & root, std::shared_ptr> rules, int id) + : parser_base(id), root_(root), rules_(std::move(rules)) {} + + parser_type type() const override { return root_->type(); } + + parser_result parse(parser_context & ctx, size_t start = 0) override { + return root_->parse(ctx, start); + } + + std::string dump() const override { + return root_->dump(); + } + + void assign_ids_internal(int& next_id) override { + if (id_ == -1) { + id_ = next_id++; + } + root_->assign_ids_internal(next_id); + } +}; + std::optional parser_result::group(const std::string & name, std::string_view input) const { auto it = groups.find(name); if (it == groups.end()) { @@ -632,11 +659,11 @@ parser parser::operator~() const { } parser parser::operator+(const parser & other) const { - return parser(std::shared_ptr(new sequence_parser({*this, other}, -1))); + return parser(std::make_shared(std::initializer_list{*this, other}, -1)); } parser parser::operator|(const parser & other) const { - return parser(std::shared_ptr(new choice_parser({*this, other}, -1))); + return parser(std::make_shared(std::initializer_list{*this, other}, -1)); } parser_base & parser::operator*() const { @@ -684,11 +711,11 @@ parser parser_builder::literal(const std::string & literal) { } parser parser_builder::sequence(std::initializer_list parsers) { - return parser(std::shared_ptr(new sequence_parser(parsers, next_id_++))); + return parser(std::make_shared(parsers, next_id_++)); } parser parser_builder::choice(std::initializer_list parsers) { - return parser(std::shared_ptr(new choice_parser(parsers, next_id_++))); + return parser(std::make_shared(parsers, next_id_++)); } parser parser_builder::one_or_more(const parser & p) { @@ -816,5 +843,11 @@ parser build_parser(const std::function & fn) { parser_builder builder; auto root = fn(builder); builder.assign_ids(root); // Assign IDs to rules that were created with operators + + // Wrap the root parser in a root_parser to own the rules and break circular references + auto rules = builder.rules(); + if (rules && !rules->empty()) { + return parser(std::make_shared(root, rules, -1)); + } return root; } diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index 72adf523c489a..edebd0bef75db 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -153,6 +153,8 @@ class parser_builder { parser add_json_rule(const std::string & name); void assign_ids(parser & p); + + std::shared_ptr> rules() const { return rules_; } }; parser build_parser(const std::function & fn); From 2a9a13de753dafa7e3caa64b9cd5dafcdf8bda49 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Mon, 10 Nov 2025 03:44:21 -0600 Subject: [PATCH 004/183] implement gbnf grammar building --- common/chat-parser-combinator.cpp | 570 +++++++++++++++++++++++--- common/chat-parser-combinator.h | 14 +- tests/test-chat-parser-combinator.cpp | 276 +++++++++++-- 3 files changed, 782 insertions(+), 78 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index cd915afcd7b88..897c4f6f75b4b 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -1,10 +1,17 @@ #include "chat-parser-combinator.h" +#include "json-schema-to-grammar.h" #include "common.h" #include "log.h" +#include + #include #include +class gbnf_visitor; + +static parser json_parser(); + class parser_base { protected: int id_; @@ -18,6 +25,7 @@ class parser_base { virtual parser_type type() const = 0; virtual parser_result parse(parser_context & ctx, size_t start = 0) = 0; virtual std::string dump() const = 0; + virtual std::string accept(gbnf_visitor & visitor) const = 0; virtual void assign_ids_internal(int& next_id) { if (id_ == -1) { id_ = next_id++; @@ -28,6 +36,8 @@ class parser_base { class literal_parser : public parser_base { std::string literal_; + friend class gbnf_visitor; + public: literal_parser(const std::string & literal, int id) : parser_base(id), literal_(literal) {} @@ -62,11 +72,15 @@ class literal_parser : public parser_base { std::string dump() const override { return "Literal(" + literal_ + ")"; } + + std::string accept(gbnf_visitor & visitor) const override; }; class sequence_parser : public parser_base { std::vector parsers_; + friend class gbnf_visitor; + public: sequence_parser(std::initializer_list parsers, int id) : parser_base(id) { for (const auto & p : parsers) { @@ -125,6 +139,8 @@ class sequence_parser : public parser_base { return "Sequence(" + string_join(parts, ", ") + ")"; } + std::string accept(gbnf_visitor & visitor) const override; + const std::vector & parsers() const { return parsers_; } void assign_ids_internal(int& next_id) override { @@ -140,6 +156,8 @@ class sequence_parser : public parser_base { class choice_parser : public parser_base { std::vector parsers_; + friend class gbnf_visitor; + public: choice_parser(std::initializer_list parsers, int id) : parser_base(id) { for (const auto & p : parsers) { @@ -187,6 +205,8 @@ class choice_parser : public parser_base { return "Choice(" + string_join(parts, ", ") + ")"; } + std::string accept(gbnf_visitor & visitor) const override; + const std::vector & parsers() const { return parsers_; } void assign_ids_internal(int& next_id) override { @@ -202,6 +222,8 @@ class choice_parser : public parser_base { class one_or_more_parser : public parser_base { parser parser_; + friend class gbnf_visitor; + public: one_or_more_parser(const parser & parser, int id) : parser_base(id), parser_(parser) {} @@ -257,6 +279,8 @@ class one_or_more_parser : public parser_base { return "OneOrMore(" + parser_->dump() + ")"; } + std::string accept(gbnf_visitor & visitor) const override; + const parser & child() const { return parser_; } void assign_ids_internal(int& next_id) override { @@ -270,6 +294,8 @@ class one_or_more_parser : public parser_base { class zero_or_more_parser : public parser_base { parser parser_; + friend class gbnf_visitor; + public: zero_or_more_parser(const parser & parser, int id) : parser_base(id), parser_(parser) {} @@ -315,6 +341,8 @@ class zero_or_more_parser : public parser_base { return "ZeroOrMore(" + parser_->dump() + ")"; } + std::string accept(gbnf_visitor & visitor) const override; + const parser & child() const { return parser_; } void assign_ids_internal(int& next_id) override { @@ -328,6 +356,8 @@ class zero_or_more_parser : public parser_base { class optional_parser : public parser_base { parser parser_; + friend class gbnf_visitor; + public: optional_parser(const parser & parser, int id) : parser_base(id), parser_(parser) {} @@ -359,6 +389,8 @@ class optional_parser : public parser_base { return "Optional(" + parser_->dump() + ")"; } + std::string accept(gbnf_visitor & visitor) const override; + const parser & child() const { return parser_; } void assign_ids_internal(int& next_id) override { @@ -369,9 +401,55 @@ class optional_parser : public parser_base { } }; +class until_parser : public parser_base { + std::string delimiter_; + bool include_spaces_; + parser parser_; + + friend class gbnf_visitor; + + public: + until_parser(const std::string & delimiter, bool include_spaces, int id, parser_builder & builder) + : parser_base(id), delimiter_(delimiter), include_spaces_(include_spaces) { + if (include_spaces) { + auto ws = builder.zero_or_more(builder.char_class("[ \\t\\n\\r]")); + parser_ = builder.zero_or_more(builder.negate(ws + builder.literal(delimiter)) + builder.any()); + } else { + parser_ = builder.zero_or_more(builder.negate(builder.literal(delimiter)) + builder.any()); + } + } + + parser_type type() const override { return PARSER_UNTIL; } + + parser_result parse(parser_context & ctx, size_t start = 0) override { + auto cached = ctx.memo.get(id_, start); + if (cached != std::nullopt) { + return *cached; + } + + auto result = parser_->parse(ctx, start); + return ctx.memo.set(id_, start, result); + } + + std::string dump() const override { + return "Until(" + delimiter_ + ")"; + } + + std::string accept(gbnf_visitor & visitor) const override; + + void assign_ids_internal(int& next_id) override { + if (id_ == -1) { + id_ = next_id++; + } + parser_->assign_ids_internal(next_id); + } +}; + class not_parser : public parser_base { parser parser_; + friend class gbnf_visitor; + public: not_parser(const parser & parser, int id) : parser_base(id), parser_(parser) {} @@ -403,6 +481,8 @@ class not_parser : public parser_base { return "Not(" + parser_->dump() + ")"; } + std::string accept(gbnf_visitor & visitor) const override; + const parser & child() const { return parser_; } void assign_ids_internal(int& next_id) override { @@ -414,6 +494,8 @@ class not_parser : public parser_base { }; class any_parser : public parser_base { + friend class gbnf_visitor; + public: any_parser(int id) : parser_base(id) {} @@ -438,6 +520,42 @@ class any_parser : public parser_base { std::string dump() const override { return "Any"; } + + std::string accept(gbnf_visitor & visitor) const override; +}; + +class space_parser : public parser_base { + friend class gbnf_visitor; + + public: + space_parser(int id) : parser_base(id) {} + + parser_type type() const override { return PARSER_SPACE; } + + parser_result parse(parser_context & ctx, size_t start = 0) override { + auto cached = ctx.memo.get(id_, start); + if (cached != std::nullopt) { + return *cached; + } + + auto pos = start; + while (pos < ctx.input.size()) { + char c = ctx.input[pos]; + if (c == ' ' || c == '\t' || c == '\n') { + ++pos; + } else { + break; + } + } + + return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_SUCCESS, start, pos)); + } + + std::string dump() const override { + return "Space"; + } + + std::string accept(gbnf_visitor & visitor) const override; }; class char_class_parser : public parser_base { @@ -450,9 +568,12 @@ class char_class_parser : public parser_base { std::string pattern_; std::vector ranges_; + bool negated_; + + friend class gbnf_visitor; public: - char_class_parser(const std::string & classes, int id) : parser_base(id), pattern_(classes) { + char_class_parser(const std::string & classes, int id) : parser_base(id), pattern_(classes), negated_(false) { std::string content = classes; if (content.front() == '[') { content = content.substr(1); @@ -462,6 +583,12 @@ class char_class_parser : public parser_base { content.pop_back(); } + // Check for negation + if (!content.empty() && content.front() == '^') { + negated_ = true; + content = content.substr(1); + } + auto parse_char = [&](size_t pos) -> std::pair { if (content[pos] == '\\' && pos + 1 < content.length()) { char next = content[pos + 1]; @@ -510,24 +637,39 @@ class char_class_parser : public parser_base { return parser_result(PARSER_RESULT_FAIL, start); } + bool matches = false; for (const auto & range : ranges_) { if (range.contains(ctx.input[start])) { - return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_SUCCESS, start, start + 1)); + matches = true; + break; } } + // If negated, invert the match result + if (negated_) { + matches = !matches; + } + + if (matches) { + return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_SUCCESS, start, start + 1)); + } + return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_FAIL, start)); } std::string dump() const override { return "Char(" + pattern_ + ")"; } + + std::string accept(gbnf_visitor & visitor) const override; }; class group_parser : public parser_base { std::string name_; parser parser_; + friend class gbnf_visitor; + public: group_parser(const std::string & name, const parser & parser, int id) : parser_base(id), name_(name), parser_(parser) {} @@ -545,6 +687,8 @@ class group_parser : public parser_base { return "Group(" + name_ + ", " + parser_->dump() + ")"; } + std::string accept(gbnf_visitor & visitor) const override; + void assign_ids_internal(int& next_id) override { if (id_ == -1) { id_ = next_id++; @@ -553,10 +697,36 @@ class group_parser : public parser_base { } }; +class schema_parser : public parser_base { + parser parser_; + std::string name_; + nlohmann::ordered_json schema_; + + friend class gbnf_visitor; + + public: + schema_parser(const parser & parser, const std::string & name, const nlohmann::ordered_json & schema, int id) + : parser_base(id), parser_(parser), name_(name), schema_(schema) {} + + parser_type type() const override { return PARSER_SCHEMA; } + + parser_result parse(parser_context & ctx, size_t start = 0) override { + return parser_->parse(ctx, start); + } + + std::string dump() const override { + return "Schema(" + parser_->dump() + ", " + schema_.dump() + ")"; + } + + std::string accept(gbnf_visitor & visitor) const override; +}; + class rule_parser : public parser_base { std::string rule_name_; std::weak_ptr> rules_; + friend class gbnf_visitor; + public: rule_parser(const std::string & name, std::shared_ptr> rules, int id) : parser_base(id), rule_name_(name), rules_(rules) {} @@ -588,12 +758,16 @@ class rule_parser : public parser_base { std::string dump() const override { return "Rule(" + rule_name_ + ")"; } + + std::string accept(gbnf_visitor & visitor) const override; }; class root_parser : public parser_base { parser root_; std::shared_ptr> rules_; + friend class gbnf_visitor; + public: root_parser(const parser & root, std::shared_ptr> rules, int id) : parser_base(id), root_(root), rules_(std::move(rules)) {} @@ -608,6 +782,8 @@ class root_parser : public parser_base { return root_->dump(); } + std::string accept(gbnf_visitor & visitor) const override; + void assign_ids_internal(int& next_id) override { if (id_ == -1) { id_ = next_id++; @@ -616,6 +792,269 @@ class root_parser : public parser_base { } }; +class gbnf_visitor { + common_grammar_builder& builder_; + std::unordered_map rule_name_mapping_; + + public: + gbnf_visitor(common_grammar_builder& builder) : builder_(builder) {} + + private: + // Escape special characters for GBNF literals + static std::string escape_literal(const std::string & s) { + std::string escaped; + for (char c : s) { + switch (c) { + case '\n': escaped += "\\n"; break; + case '\t': escaped += "\\t"; break; + case '\r': escaped += "\\r"; break; + case '\\': escaped += "\\\\"; break; + case '"': escaped += "\\\""; break; + default: escaped += c; break; + } + } + return escaped; + } + + // Escape a single character for use in character classes + static std::string escape_char_class(char c) { + switch (c) { + case '\n': return "\\n"; + case '\t': return "\\t"; + case '\r': return "\\r"; + case '\\': return "\\\\"; + case ']': return "\\]"; + case '-': return "\\-"; + case '^': return "\\^"; + default: return std::string(1, c); + } + } + + // Generate pattern for until() that matches prefixes but prevents full delimiter match + // For "" generates: ( [^<] | "<" [^/] | " alternatives; + + // First alternative: match any character that's not the start of the delimiter + alternatives.push_back("[^" + escape_char_class(delimiter[0]) + "]"); + + // For each prefix, match the prefix followed by a char that's not the next delimiter char + for (size_t i = 1; i < delimiter.length(); ++i) { + std::string prefix = "\"" + escape_literal(delimiter.substr(0, i)) + "\""; + std::string next_char_negated = "[^" + escape_char_class(delimiter[i]) + "]"; + alternatives.push_back(prefix + " " + next_char_negated); + } + + // Combine alternatives with | + std::string result = "("; + for (size_t i = 0; i < alternatives.size(); ++i) { + if (i > 0) { + result += " | "; + } + result += alternatives[i]; + } + result += ")"; + + return result; + } + + // Check if expression needs parentheses + static bool needs_parens(parser_type type) { + return type == PARSER_CHOICE || type == PARSER_SEQUENCE; + } + + public: + std::string visit(const literal_parser & p) { + return "\"" + escape_literal(p.literal_) + "\""; + } + + std::string visit(const sequence_parser & p) { + std::string s; + for (size_t i = 0; i < p.parsers_.size(); ++i) { + if (i > 0) s += " "; + auto child_result = p.parsers_[i]->accept(*this); + s += child_result; + } + return s; + } + + std::string visit(const choice_parser & p) { + std::string s; + for (size_t i = 0; i < p.parsers_.size(); ++i) { + if (i > 0) { + s += " | "; + } + + auto child_type = p.parsers_[i]->type(); + auto child_result = p.parsers_[i]->accept(*this); + + // Parenthesize sequences in choices + if (child_type == PARSER_SEQUENCE) { + s += "(" + child_result + ")"; + } else { + s += child_result; + } + } + return s; + } + + std::string visit(const one_or_more_parser & p) { + auto child_type = p.parser_->type(); + auto child_result = p.parser_->accept(*this); + if (needs_parens(child_type)) { + return "(" + child_result + ")+"; + } + return child_result + "+"; + } + + std::string visit(const zero_or_more_parser & p) { + auto child_type = p.parser_->type(); + auto child_result = p.parser_->accept(*this); + if (needs_parens(child_type)) { + return "(" + child_result + ")*"; + } + return child_result + "*"; + } + + std::string visit(const optional_parser & p) { + auto child_type = p.parser_->type(); + auto child_result = p.parser_->accept(*this); + if (needs_parens(child_type)) { + return "(" + child_result + ")?"; + } + return child_result + "?"; + } + + std::string visit(const until_parser & p) { + // Generate pattern that matches prefixes but prevents full delimiter match + return generate_until_pattern(p.delimiter_) + "*"; + } + + std::string visit(const not_parser &) { + // NOT is tricky in GBNF - for now, emit error + LOG_ERR("NOT operator not directly supported in GBNF generation\n"); + return ""; // This will cause compilation errors, which is intended + } + + std::string visit(const any_parser &) { + // Match any single character + return "[\\x00-\\x{10FFFF}]"; + } + + std::string visit(const space_parser &) { + // Reference the built-in space rule + return "space"; + } + + std::string visit(const char_class_parser & p) { + // Return pattern as-is (already in GBNF format) + return p.pattern_; + } + + std::string visit(const group_parser & p) { + // Groups are transparent - just visit child + return p.parser_->accept(*this); + } + + std::string visit(const schema_parser & p) { + return builder_.add_schema(p.name_, p.schema_); + } + + std::string visit(const rule_parser & p) { + // Return canonical rule reference + auto it = rule_name_mapping_.find(p.rule_name_); + if (it != rule_name_mapping_.end()) { + return it->second; + } + // Fallback to original name if not in mapping (shouldn't happen in valid usage) + return p.rule_name_; + } + + std::string visit(const root_parser & p) { + // Generate named rules first + if (p.rules_) { + for (const auto & [name, rule] : *p.rules_) { + auto rule_body = rule->accept(*this); + auto canonical_name = builder_.add_rule(name, rule_body); + rule_name_mapping_[name] = canonical_name; + } + } + + // Return root body for composition + return p.root_->accept(*this); + } +}; + +// Implement accept() methods for all parser classes +std::string literal_parser::accept(gbnf_visitor & visitor) const { + return visitor.visit(*this); +} + +std::string sequence_parser::accept(gbnf_visitor & visitor) const { + return visitor.visit(*this); +} + +std::string choice_parser::accept(gbnf_visitor & visitor) const { + return visitor.visit(*this); +} + +std::string one_or_more_parser::accept(gbnf_visitor & visitor) const { + return visitor.visit(*this); +} + +std::string zero_or_more_parser::accept(gbnf_visitor & visitor) const { + return visitor.visit(*this); +} + +std::string optional_parser::accept(gbnf_visitor & visitor) const { + return visitor.visit(*this); +} + +std::string until_parser::accept(gbnf_visitor & visitor) const { + return visitor.visit(*this); +} + +std::string not_parser::accept(gbnf_visitor & visitor) const { + return visitor.visit(*this); +} + +std::string any_parser::accept(gbnf_visitor & visitor) const { + return visitor.visit(*this); +} + +std::string space_parser::accept(gbnf_visitor & visitor) const { + return visitor.visit(*this); +} + +std::string char_class_parser::accept(gbnf_visitor & visitor) const { + return visitor.visit(*this); +} + +std::string group_parser::accept(gbnf_visitor & visitor) const { + return visitor.visit(*this); +} + +std::string schema_parser::accept(gbnf_visitor & visitor) const { + return visitor.visit(*this); +} + +std::string rule_parser::accept(gbnf_visitor & visitor) const { + return visitor.visit(*this); +} + +std::string root_parser::accept(gbnf_visitor & visitor) const { + return visitor.visit(*this); +} + std::optional parser_result::group(const std::string & name, std::string_view input) const { auto it = groups.find(name); if (it == groups.end()) { @@ -666,6 +1105,11 @@ parser parser::operator|(const parser & other) const { return parser(std::make_shared(std::initializer_list{*this, other}, -1)); } +parser parser::operator<<(const parser & other) const { + auto ws = parser(std::make_shared(-1)); + return parser(std::make_shared(std::initializer_list{*this, ws, other}, -1)); +} + parser_base & parser::operator*() const { return *ptr; } @@ -702,6 +1146,16 @@ std::string parser::dump() const { return ptr->dump(); } +void parser::build_grammar(common_grammar_builder& builder) const { + gbnf_visitor visitor(builder); + auto result = ptr->accept(visitor); + // The visitor returns the GBNF string for this parser + // root_parser registers its named rules and returns its root body + if (!result.empty()) { + builder.add_rule("root", result); + } +} + parser_builder::parser_builder() : rules_(std::make_shared>()) , next_id_(0) {} @@ -751,7 +1205,15 @@ parser parser_builder::rule(const std::string & name) { } parser parser_builder::space() { - return zero_or_more(char_class("[ \\t\\n\\r]")); + return parser(std::make_shared(next_id_++)); +} + +parser parser_builder::until(const std::string & delimiter, bool include_spaces) { + return parser(std::make_shared(delimiter, include_spaces, next_id_++, *this)); +} + +parser parser_builder::schema(const parser & p, const std::string & name, const nlohmann::ordered_json & schema) { + return parser(std::make_shared(p, name, schema, next_id_++)); } parser parser_builder::add_rule(const std::string & name, const parser & p) { @@ -765,89 +1227,99 @@ void parser_builder::assign_ids(parser & p) { } } -parser parser_builder::add_json_rule(const std::string & name) { +parser build_parser(const std::function & fn) { + parser_builder builder; + auto root = fn(builder); + builder.assign_ids(root); // Assign IDs to rules that were created with operators + + // Wrap the root parser in a root_parser to own the rules and break circular references + auto rules = builder.rules(); + if (rules && !rules->empty()) { + return parser(std::make_shared(root, rules, -1)); + } + return root; +} + +static parser json_parser() { + parser_builder builder; + // Whitespace: space, tab, newline, carriage return - auto ws = zero_or_more(char_class("[ \\t\\n\\r]")); + auto ws = builder.zero_or_more(builder.char_class("[ \\t\\n\\r]")); // Number components - auto digit = char_class("[0-9]"); - auto digit1_9 = char_class("[1-9]"); - auto digits = one_or_more(digit); + auto digit = builder.char_class("[0-9]"); + auto digit1_9 = builder.char_class("[1-9]"); + auto digits = builder.one_or_more(digit); // Integer part: 0 or non-zero digit followed by more digits - auto int_part = literal("0") | (digit1_9 + zero_or_more(digit)); + auto int_part = builder.literal("0") | (digit1_9 + builder.zero_or_more(digit)); // Optional fractional part - auto frac = literal(".") + digits; + auto frac = builder.literal(".") + digits; // Optional exponent part - auto exp = (literal("e") | literal("E")) + optional(char_class("[+\\-]")) + digits; + auto exp = (builder.literal("e") | builder.literal("E")) + builder.optional(builder.char_class("[+\\-]")) + digits; // Complete number - auto number = optional(literal("-")) + int_part + optional(frac) + optional(exp); + auto number = builder.optional(builder.literal("-")) + int_part + builder.optional(frac) + builder.optional(exp); - add_rule("json_number", number); + builder.add_rule("json_number", number); // String components - auto hex = char_class("[0-9a-fA-F]"); - auto unicode_escape = literal("\\u") + hex + hex + hex + hex; - auto simple_escape = literal("\\") + char_class("[\"\\\\bfnrt/]"); + auto hex = builder.char_class("[0-9a-fA-F]"); + auto unicode_escape = builder.literal("\\u") + hex + hex + hex + hex; + auto simple_escape = builder.literal("\\") + builder.char_class("[\"\\\\bfnrt/]"); auto escape = simple_escape | unicode_escape; // String character: escape sequence or any char except quote and backslash - auto string_char = escape | (~char_class("[\"\\\\]") + any()); - auto string = literal("\"") + zero_or_more(string_char) + literal("\""); + auto string_char = escape | builder.char_class("[^\"\\\\]"); + auto string = builder.literal("\"") + builder.zero_or_more(string_char) + builder.literal("\""); - add_rule("json_string", string); + builder.add_rule("json_string", string); // Literals - auto true_lit = literal("true"); - auto false_lit = literal("false"); - auto null_lit = literal("null"); + auto true_lit = builder.literal("true"); + auto false_lit = builder.literal("false"); + auto null_lit = builder.literal("null"); // Value - uses forward references for recursive structures - add_rule("json_value", - rule("json_object") | - rule("json_array") | - rule("json_string") | - rule("json_number") | + builder.add_rule("json_value", + builder.rule("json_object") | + builder.rule("json_array") | + builder.rule("json_string") | + builder.rule("json_number") | true_lit | false_lit | null_lit ); // Object: { "key": value, ... } - auto member = rule("json_string") + ws + literal(":") + ws + rule("json_value"); - auto members = member + zero_or_more(ws + literal(",") + ws + member); + auto member = builder.rule("json_string") + ws + builder.literal(":") + ws + builder.rule("json_value"); + auto members = member + builder.zero_or_more(ws + builder.literal(",") + ws + member); // Empty object or object with members - auto object = (literal("{") + ws + literal("}")) | - (literal("{") + ws + members + ws + literal("}")); + auto object = (builder.literal("{") + ws + builder.literal("}")) | + (builder.literal("{") + ws + members + ws + builder.literal("}")); - add_rule("json_object", object); + builder.add_rule("json_object", object); // Array: [ value, ... ] - auto elements = rule("json_value") + zero_or_more(ws + literal(",") + ws + rule("json_value")); + auto elements = builder.rule("json_value") + builder.zero_or_more(ws + builder.literal(",") + ws + builder.rule("json_value")); // Empty array or array with elements - auto array = (literal("[") + ws + literal("]")) | - (literal("[") + ws + elements + ws + literal("]")); + auto array = (builder.literal("[") + ws + builder.literal("]")) | + (builder.literal("[") + ws + elements + ws + builder.literal("]")); - add_rule("json_array", array); + builder.add_rule("json_array", array); - // Register the main rule with the provided name - return add_rule(name, rule("json_value")); -} + // Get the json_value rule as the root + auto root = builder.rule("json_value"); + builder.assign_ids(root); -parser build_parser(const std::function & fn) { - parser_builder builder; - auto root = fn(builder); - builder.assign_ids(root); // Assign IDs to rules that were created with operators + // Wrap in root_parser to own the rules + return parser(std::make_shared(root, builder.rules(), -1)); +} - // Wrap the root parser in a root_parser to own the rules and break circular references - auto rules = builder.rules(); - if (rules && !rules->empty()) { - return parser(std::make_shared(root, rules, -1)); - } - return root; +parser parser_builder::json() { + return json_parser(); } diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index edebd0bef75db..1ef3996dc862a 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -1,5 +1,7 @@ #pragma once +#include + #include #include #include @@ -7,6 +9,8 @@ #include #include +struct common_grammar_builder; + enum parser_type { PARSER_LITERAL = 0, PARSER_SEQUENCE = 1, @@ -19,6 +23,9 @@ enum parser_type { PARSER_GROUP = 8, PARSER_RULE = 9, PARSER_OPTIONAL = 10, + PARSER_UNTIL = 11, + PARSER_SPACE = 12, + PARSER_SCHEMA = 13, }; enum parser_result_type { @@ -94,6 +101,7 @@ class parser_base; class sequence_parser; class choice_parser; class parser_builder; +class gbnf_visitor; class parser { std::shared_ptr ptr; @@ -114,6 +122,7 @@ class parser { parser operator~() const; parser operator+(const parser & other) const; parser operator|(const parser & other) const; + parser operator<<(const parser & other) const; parser_base & operator*() const; parser_base * operator->() const; @@ -127,6 +136,7 @@ class parser { parser_type type() const; parser_result parse(parser_context & ctx, size_t start = 0) const; std::string dump() const; + void build_grammar(common_grammar_builder& builder) const; }; class parser_builder { @@ -148,9 +158,11 @@ class parser_builder { parser group(const std::string & name, const parser & p); parser rule(const std::string & name); parser space(); + parser until(const std::string & delimiter, bool include_spaces = true); + parser json(); + parser schema(const parser & p, const std::string & name, const nlohmann::ordered_json & schema); parser add_rule(const std::string & name, const parser & p); - parser add_json_rule(const std::string & name); void assign_ids(parser & p); diff --git a/tests/test-chat-parser-combinator.cpp b/tests/test-chat-parser-combinator.cpp index 55a443aed3e38..1fffbfbf040a0 100644 --- a/tests/test-chat-parser-combinator.cpp +++ b/tests/test-chat-parser-combinator.cpp @@ -2,6 +2,9 @@ #include #include "chat-parser-combinator.h" +#include "json-schema-to-grammar.h" +#include "nlohmann/json.hpp" +#include "nlohmann/json_fwd.hpp" template static void assert_equals(const std::string_view label, const T & expected, const T & actual) { @@ -365,53 +368,110 @@ static void test_optional() { static void test_json_parser() { auto json = build_parser([](parser_builder & p) { - return p.add_json_rule("json"); + return p.json(); }); - // Test parsing a simple JSON object - std::string input = R"({"name": "test", "value": 42, "flag": true})"; - parser_context ctx{input, parse_cache()}; + { + // Test parsing a simple JSON object + std::string input = R"({"name": "test", "value": 42, "flag": true})"; + parser_context ctx{input, parse_cache()}; - auto result = json.parse(ctx); + auto result = json.parse(ctx); - assert_equals(true, result.is_success()); - assert_equals(input.size(), result.end); + assert_equals(true, result.is_success()); + assert_equals(input.size(), result.end); + } + { + // Test parsing a JSON array with mixed types + std::string input = R"([1, "hello", true, null, 3.14])"; + parser_context ctx{input, parse_cache()}; + + auto result = json.parse(ctx); + + assert_equals(true, result.is_success()); + assert_equals(input.size(), result.end); + } + { + // Test parsing nested JSON with objects and arrays + std::string input = R"({"users": [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}], "count": 2, "metadata": {"version": "1.0", "tags": ["admin", "user"]}})"; + parser_context ctx{input, parse_cache()}; + + auto result = json.parse(ctx); + + assert_equals(true, result.is_success()); + assert_equals(input.size(), result.end); + } + { + // Test partial parsing - incomplete object + std::string input = R"({"name": "test", "value": )"; + parser_context ctx{input, parse_cache(), false}; + + auto result = json.parse(ctx); + + assert_equals(true, result.is_success()); + } + { + // Test partial parsing - incomplete array + std::string input = R"([1, 2, 3, )"; + parser_context ctx{input, parse_cache(), false}; + + auto result = json.parse(ctx); + + assert_equals(true, result.is_success()); + } + { + // Test partial parsing - incomplete nested structure + std::string input = R"({"data": {"nested": )"; + parser_context ctx{input, parse_cache(), false}; + + auto result = json.parse(ctx); + + assert_equals(true, result.is_success()); + } } static void test_complete_example() { + // Parser for a fictitious model that outputs: + // + // + // ... reasoning content ... + // + // ... content ... + // + // tool_name + // { ... json args ... } + // + // auto parser = build_parser([](parser_builder & p) { - auto space = p.add_rule("space", p.space()); - auto reasoning = p.add_rule("reasoning", - p.literal("") + space + - p.group("reasoning-content", - p.zero_or_more(~(space + p.literal("")) + p.any())) + - space + p.literal("")); + p.literal("") + << p.group("reasoning-content", p.until("")) + << p.literal("")); auto content = p.add_rule("content", - p.group("content", - p.zero_or_more(~(space + p.literal("")) + p.any()))); + p.group("content", p.until(""))); - auto ident_chars = p.add_rule("ident-chars", p.char_class("[a-zA-Z\\-_]")); - auto json = p.add_json_rule("json"); + auto json = p.json(); auto tool_call_name = p.add_rule("tool-call-name", - p.literal("") + space + - p.group("tool-name", p.one_or_more(~p.literal("") + ident_chars)) + - space + p.literal("")); + p.literal("") + << p.group("tool-name", p.one_or_more(p.char_class("[a-zA-Z\\-_]"))) + << p.literal("")); + + auto schema = nlohmann::ordered_json::parse(R"({"type": "object"})"); auto tool_call_args = p.add_rule("tool-call-args", - p.literal("") + space + - p.group("tool-args", json) + - space + p.literal("")); + p.literal("") + << p.group("tool-args", p.schema(json, "get_weather", schema)) + << p.literal("")); auto tool_call = p.add_rule("tool-call", - p.literal("") + space + - tool_call_name + space + - tool_call_args + space + - p.literal("")); + p.literal("") + << tool_call_name + << tool_call_args + << p.literal("")); - return p.add_rule("root", reasoning + p.optional(content) + p.optional(tool_call)); + return reasoning << p.optional(content) << p.optional(tool_call); }); // Test complete input @@ -457,6 +517,165 @@ static void test_complete_example() { assert_equals(std::string("I need to call get_weather"), *result.group("reasoning-content", ctx.input)); assert_equals(std::string("get_weather"), *result.group("tool-name", ctx.input)); assert_equals(std::string(R"({"cit)"), *result.group("tool-args", ctx.input)); + + auto gbnf = ::build_grammar([&](const common_grammar_builder& builder) { + parser.build_grammar(const_cast(builder)); + }); + + std::cout << "Grammar:\n" << gbnf << "\n"; +} + +static void test_gbnf_generation() { + { + // Test literal + auto parser = build_parser([](parser_builder& p) { + return p.literal("hello"); + }); + + auto gbnf = ::build_grammar([&](const common_grammar_builder& builder) { + parser.build_grammar(const_cast(builder)); + }); + assert_equals(true, gbnf.find("root ::= \"hello\"") != std::string::npos); + assert_equals(true, gbnf.find("space ::=") != std::string::npos); + } + { + // Test char class + auto parser = build_parser([](parser_builder& p) { + return p.char_class("[a-z]"); + }); + + auto gbnf = ::build_grammar([&](const common_grammar_builder& builder) { + parser.build_grammar(const_cast(builder)); + }); + assert_equals(true, gbnf.find("root ::= [a-z]") != std::string::npos); + } + { + // Test sequence + auto parser = build_parser([](parser_builder& p) { + return p.literal("hello") + p.literal(" ") + p.literal("world"); + }); + + auto gbnf = ::build_grammar([&](const common_grammar_builder& builder) { + parser.build_grammar(const_cast(builder)); + }); + assert_equals(true, gbnf.find("root ::= \"hello\" \" \" \"world\"") != std::string::npos); + } + { + // Test choice + auto parser = build_parser([](parser_builder& p) { + return p.literal("cat") | p.literal("dog"); + }); + + auto gbnf = ::build_grammar([&](const common_grammar_builder& builder) { + parser.build_grammar(const_cast(builder)); + }); + assert_equals(true, gbnf.find("root ::= \"cat\" | \"dog\"") != std::string::npos); + } + { + // Test one_or_more + auto parser = build_parser([](parser_builder& p) { + return p.one_or_more(p.char_class("[0-9]")); + }); + + auto gbnf = ::build_grammar([&](const common_grammar_builder& builder) { + parser.build_grammar(const_cast(builder)); + }); + assert_equals(true, gbnf.find("root ::= [0-9]+") != std::string::npos); + } + { + // Test zero_or_more + auto parser = build_parser([](parser_builder& p) { + return p.zero_or_more(p.char_class("[a-z]")); + }); + + auto gbnf = ::build_grammar([&](const common_grammar_builder& builder) { + parser.build_grammar(const_cast(builder)); + }); + assert_equals(true, gbnf.find("root ::= [a-z]*") != std::string::npos); + } + { + // Test optional + auto parser = build_parser([](parser_builder& p) { + return p.literal("hello") + p.optional(p.literal(" world")); + }); + + auto gbnf = ::build_grammar([&](const common_grammar_builder& builder) { + parser.build_grammar(const_cast(builder)); + }); + assert_equals(true, gbnf.find("root ::= \"hello\" \" world\"?") != std::string::npos); + } + { + // Test until + auto parser = build_parser([](parser_builder& p) { + return p.until(""); + }); + + auto gbnf = ::build_grammar([&](const common_grammar_builder& builder) { + parser.build_grammar(const_cast(builder)); + }); + // Should generate pattern that prevents matching the full delimiter + assert_equals(true, gbnf.find("root ::= ([^<] | \"<\" [^/] | \"])*") != std::string::npos); + } + { + // Test groups are transparent + auto parser = build_parser([](parser_builder& p) { + return p.group("test", p.literal("hello")); + }); + + auto gbnf = ::build_grammar([&](const common_grammar_builder& builder) { + parser.build_grammar(const_cast(builder)); + }); + assert_equals(true, gbnf.find("root ::= \"hello\"") != std::string::npos); + } + { + // Test complex expression with parentheses + auto parser = build_parser([](parser_builder& p) { + return p.one_or_more(p.literal("a") | p.literal("b")); + }); + + auto gbnf = ::build_grammar([&](const common_grammar_builder& builder) { + parser.build_grammar(const_cast(builder)); + }); + assert_equals(true, gbnf.find("root ::= (\"a\" | \"b\")+") != std::string::npos); + } + { + // Test rule references + auto parser = build_parser([](parser_builder& p) { + auto digit = p.add_rule("digit", p.char_class("[0-9]")); + return p.one_or_more(digit); + }); + + auto gbnf = ::build_grammar([&](const common_grammar_builder& builder) { + parser.build_grammar(const_cast(builder)); + }); + // Should have digit rule defined and referenced + assert_equals(true, gbnf.find("digit ::= [0-9]") != std::string::npos); + assert_equals(true, gbnf.find("root ::= digit+") != std::string::npos); + } + { + // Test escaping in literals + auto parser = build_parser([](parser_builder& p) { + return p.literal("hello\nworld\t!"); + }); + + auto gbnf = ::build_grammar([&](const common_grammar_builder& builder) { + parser.build_grammar(const_cast(builder)); + }); + assert_equals(true, gbnf.find("root ::= \"hello\\nworld\\t!\"") != std::string::npos); + } + { + // Test operator<< (whitespace insertion) + auto parser = build_parser([](parser_builder& p) { + return p.literal("hello") << p.literal("world"); + }); + + auto gbnf = ::build_grammar([&](const common_grammar_builder& builder) { + parser.build_grammar(const_cast(builder)); + }); + // Should inline the whitespace pattern + assert_equals(true, gbnf.find("\"hello\"") != std::string::npos); + assert_equals(true, gbnf.find("\"world\"") != std::string::npos); + } } int main() { @@ -467,6 +686,7 @@ int main() { test_optional(); test_json_parser(); test_complete_example(); + test_gbnf_generation(); std::cout << "All tests passed!\n"; return 0; } From 228653248e1994bbe82f99c90f7b8a607a34d461 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Mon, 10 Nov 2025 04:06:59 -0600 Subject: [PATCH 005/183] remove unused private variable --- common/chat-parser-combinator.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index 897c4f6f75b4b..1a340f6162656 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -403,14 +403,13 @@ class optional_parser : public parser_base { class until_parser : public parser_base { std::string delimiter_; - bool include_spaces_; parser parser_; friend class gbnf_visitor; public: until_parser(const std::string & delimiter, bool include_spaces, int id, parser_builder & builder) - : parser_base(id), delimiter_(delimiter), include_spaces_(include_spaces) { + : parser_base(id), delimiter_(delimiter) { if (include_spaces) { auto ws = builder.zero_or_more(builder.char_class("[ \\t\\n\\r]")); parser_ = builder.zero_or_more(builder.negate(ws + builder.literal(delimiter)) + builder.any()); From 3e6662f66c030d2804736e515f554b4e6ed4ac11 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Mon, 10 Nov 2025 20:17:09 -0600 Subject: [PATCH 006/183] create a base visitor and implement id assignment as a visitor --- common/chat-parser-combinator.cpp | 461 +++++++++++++++--------------- common/chat-parser-combinator.h | 4 +- 2 files changed, 230 insertions(+), 235 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index 1a340f6162656..598fd74a93657 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -8,7 +8,7 @@ #include #include -class gbnf_visitor; +class id_assignment_visitor; static parser json_parser(); @@ -16,7 +16,7 @@ class parser_base { protected: int id_; - void set_id(int id) { id_ = id; } + friend class id_assignment_visitor; public: parser_base(int id) : id_(id) {} @@ -25,19 +25,12 @@ class parser_base { virtual parser_type type() const = 0; virtual parser_result parse(parser_context & ctx, size_t start = 0) = 0; virtual std::string dump() const = 0; - virtual std::string accept(gbnf_visitor & visitor) const = 0; - virtual void assign_ids_internal(int& next_id) { - if (id_ == -1) { - id_ = next_id++; - } - } + virtual void accept(parser_visitor & visitor) = 0; }; class literal_parser : public parser_base { std::string literal_; - friend class gbnf_visitor; - public: literal_parser(const std::string & literal, int id) : parser_base(id), literal_(literal) {} @@ -73,14 +66,14 @@ class literal_parser : public parser_base { return "Literal(" + literal_ + ")"; } - std::string accept(gbnf_visitor & visitor) const override; + void accept(parser_visitor & visitor) override; + + const std::string & literal() const { return literal_; } }; class sequence_parser : public parser_base { std::vector parsers_; - friend class gbnf_visitor; - public: sequence_parser(std::initializer_list parsers, int id) : parser_base(id) { for (const auto & p : parsers) { @@ -139,25 +132,14 @@ class sequence_parser : public parser_base { return "Sequence(" + string_join(parts, ", ") + ")"; } - std::string accept(gbnf_visitor & visitor) const override; + void accept(parser_visitor & visitor) override; const std::vector & parsers() const { return parsers_; } - - void assign_ids_internal(int& next_id) override { - if (id_ == -1) { - id_ = next_id++; - } - for (auto & p : parsers_) { - p->assign_ids_internal(next_id); - } - } }; class choice_parser : public parser_base { std::vector parsers_; - friend class gbnf_visitor; - public: choice_parser(std::initializer_list parsers, int id) : parser_base(id) { for (const auto & p : parsers) { @@ -205,25 +187,14 @@ class choice_parser : public parser_base { return "Choice(" + string_join(parts, ", ") + ")"; } - std::string accept(gbnf_visitor & visitor) const override; + void accept(parser_visitor & visitor) override; const std::vector & parsers() const { return parsers_; } - - void assign_ids_internal(int& next_id) override { - if (id_ == -1) { - id_ = next_id++; - } - for (auto & p : parsers_) { - p->assign_ids_internal(next_id); - } - } }; class one_or_more_parser : public parser_base { parser parser_; - friend class gbnf_visitor; - public: one_or_more_parser(const parser & parser, int id) : parser_base(id), parser_(parser) {} @@ -279,23 +250,14 @@ class one_or_more_parser : public parser_base { return "OneOrMore(" + parser_->dump() + ")"; } - std::string accept(gbnf_visitor & visitor) const override; + void accept(parser_visitor & visitor) override; const parser & child() const { return parser_; } - - void assign_ids_internal(int& next_id) override { - if (id_ == -1) { - id_ = next_id++; - } - parser_->assign_ids_internal(next_id); - } }; class zero_or_more_parser : public parser_base { parser parser_; - friend class gbnf_visitor; - public: zero_or_more_parser(const parser & parser, int id) : parser_base(id), parser_(parser) {} @@ -341,23 +303,14 @@ class zero_or_more_parser : public parser_base { return "ZeroOrMore(" + parser_->dump() + ")"; } - std::string accept(gbnf_visitor & visitor) const override; + void accept(parser_visitor & visitor) override; const parser & child() const { return parser_; } - - void assign_ids_internal(int& next_id) override { - if (id_ == -1) { - id_ = next_id++; - } - parser_->assign_ids_internal(next_id); - } }; class optional_parser : public parser_base { parser parser_; - friend class gbnf_visitor; - public: optional_parser(const parser & parser, int id) : parser_base(id), parser_(parser) {} @@ -389,24 +342,15 @@ class optional_parser : public parser_base { return "Optional(" + parser_->dump() + ")"; } - std::string accept(gbnf_visitor & visitor) const override; + void accept(parser_visitor & visitor) override; const parser & child() const { return parser_; } - - void assign_ids_internal(int& next_id) override { - if (id_ == -1) { - id_ = next_id++; - } - parser_->assign_ids_internal(next_id); - } }; class until_parser : public parser_base { std::string delimiter_; parser parser_; - friend class gbnf_visitor; - public: until_parser(const std::string & delimiter, bool include_spaces, int id, parser_builder & builder) : parser_base(id), delimiter_(delimiter) { @@ -434,21 +378,16 @@ class until_parser : public parser_base { return "Until(" + delimiter_ + ")"; } - std::string accept(gbnf_visitor & visitor) const override; + void accept(parser_visitor & visitor) override; - void assign_ids_internal(int& next_id) override { - if (id_ == -1) { - id_ = next_id++; - } - parser_->assign_ids_internal(next_id); - } + const std::string & delimiter() const { return delimiter_; } + + const parser & child() const { return parser_; } }; class not_parser : public parser_base { parser parser_; - friend class gbnf_visitor; - public: not_parser(const parser & parser, int id) : parser_base(id), parser_(parser) {} @@ -480,21 +419,12 @@ class not_parser : public parser_base { return "Not(" + parser_->dump() + ")"; } - std::string accept(gbnf_visitor & visitor) const override; + void accept(parser_visitor & visitor) override; const parser & child() const { return parser_; } - - void assign_ids_internal(int& next_id) override { - if (id_ == -1) { - id_ = next_id++; - } - parser_->assign_ids_internal(next_id); - } }; class any_parser : public parser_base { - friend class gbnf_visitor; - public: any_parser(int id) : parser_base(id) {} @@ -520,12 +450,10 @@ class any_parser : public parser_base { return "Any"; } - std::string accept(gbnf_visitor & visitor) const override; + void accept(parser_visitor & visitor) override; }; class space_parser : public parser_base { - friend class gbnf_visitor; - public: space_parser(int id) : parser_base(id) {} @@ -554,7 +482,7 @@ class space_parser : public parser_base { return "Space"; } - std::string accept(gbnf_visitor & visitor) const override; + void accept(parser_visitor & visitor) override; }; class char_class_parser : public parser_base { @@ -569,8 +497,6 @@ class char_class_parser : public parser_base { std::vector ranges_; bool negated_; - friend class gbnf_visitor; - public: char_class_parser(const std::string & classes, int id) : parser_base(id), pattern_(classes), negated_(false) { std::string content = classes; @@ -660,15 +586,15 @@ class char_class_parser : public parser_base { return "Char(" + pattern_ + ")"; } - std::string accept(gbnf_visitor & visitor) const override; + void accept(parser_visitor & visitor) override; + + const std::string & pattern() const { return pattern_; } }; class group_parser : public parser_base { std::string name_; parser parser_; - friend class gbnf_visitor; - public: group_parser(const std::string & name, const parser & parser, int id) : parser_base(id), name_(name), parser_(parser) {} @@ -686,14 +612,9 @@ class group_parser : public parser_base { return "Group(" + name_ + ", " + parser_->dump() + ")"; } - std::string accept(gbnf_visitor & visitor) const override; + void accept(parser_visitor & visitor) override; - void assign_ids_internal(int& next_id) override { - if (id_ == -1) { - id_ = next_id++; - } - parser_->assign_ids_internal(next_id); - } + const parser & child() const { return parser_; } }; class schema_parser : public parser_base { @@ -701,8 +622,6 @@ class schema_parser : public parser_base { std::string name_; nlohmann::ordered_json schema_; - friend class gbnf_visitor; - public: schema_parser(const parser & parser, const std::string & name, const nlohmann::ordered_json & schema, int id) : parser_base(id), parser_(parser), name_(name), schema_(schema) {} @@ -717,18 +636,22 @@ class schema_parser : public parser_base { return "Schema(" + parser_->dump() + ", " + schema_.dump() + ")"; } - std::string accept(gbnf_visitor & visitor) const override; + void accept(parser_visitor & visitor) override; + + const parser & child() const { return parser_; } + + const std::string & name() const { return name_; } + + const nlohmann::ordered_json & schema() const { return schema_; } }; class rule_parser : public parser_base { - std::string rule_name_; + std::string name_; std::weak_ptr> rules_; - friend class gbnf_visitor; - public: - rule_parser(const std::string & name, std::shared_ptr> rules, int id) - : parser_base(id), rule_name_(name), rules_(rules) {} + rule_parser(const std::string & name, const std::shared_ptr> & rules, int id) + : parser_base(id), name_(name), rules_(rules) {} parser_type type() const override { return PARSER_RULE; } @@ -744,9 +667,9 @@ class rule_parser : public parser_base { return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_FAIL, start)); } - auto it = rules->find(rule_name_); + auto it = rules->find(name_); if (it == rules->end()) { - LOG_ERR("rule_parser::parse rule '%s' not found in registry\n", rule_name_.c_str()); + LOG_ERR("rule_parser::parse rule '%s' not found in registry\n", name_.c_str()); return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_FAIL, start)); } @@ -755,17 +678,19 @@ class rule_parser : public parser_base { } std::string dump() const override { - return "Rule(" + rule_name_ + ")"; + return "Rule(" + name_ + ")"; } - std::string accept(gbnf_visitor & visitor) const override; + void accept(parser_visitor & visitor) override; + + const std::string & name() const { return name_; } }; class root_parser : public parser_base { parser root_; std::shared_ptr> rules_; - friend class gbnf_visitor; + friend class parser_visitor; public: root_parser(const parser & root, std::shared_ptr> rules, int id) @@ -781,23 +706,45 @@ class root_parser : public parser_base { return root_->dump(); } - std::string accept(gbnf_visitor & visitor) const override; + void accept(parser_visitor & visitor) override; - void assign_ids_internal(int& next_id) override { - if (id_ == -1) { - id_ = next_id++; - } - root_->assign_ids_internal(next_id); - } + const parser & root() const { return root_; } + + std::shared_ptr> rules() const { return rules_; } +}; + +// Base visitor class for parser tree traversal +class parser_visitor { + public: + virtual ~parser_visitor() = default; + + virtual void visit(literal_parser & p) = 0; + virtual void visit(sequence_parser & p) = 0; + virtual void visit(choice_parser & p) = 0; + virtual void visit(one_or_more_parser & p) = 0; + virtual void visit(zero_or_more_parser & p) = 0; + virtual void visit(optional_parser & p) = 0; + virtual void visit(until_parser & p) = 0; + virtual void visit(not_parser & p) = 0; + virtual void visit(any_parser & p) = 0; + virtual void visit(space_parser & p) = 0; + virtual void visit(char_class_parser & p) = 0; + virtual void visit(group_parser & p) = 0; + virtual void visit(schema_parser & p) = 0; + virtual void visit(rule_parser & p) = 0; + virtual void visit(root_parser & p) = 0; }; -class gbnf_visitor { +class gbnf_visitor : public parser_visitor { common_grammar_builder& builder_; std::unordered_map rule_name_mapping_; + std::string current_result_; public: gbnf_visitor(common_grammar_builder& builder) : builder_(builder) {} + const std::string& result() const { return current_result_; } + private: // Escape special characters for GBNF literals static std::string escape_literal(const std::string & s) { @@ -872,187 +819,235 @@ class gbnf_visitor { } public: - std::string visit(const literal_parser & p) { - return "\"" + escape_literal(p.literal_) + "\""; + void visit(literal_parser & p) override { + current_result_ = "\"" + escape_literal(p.literal()) + "\""; } - std::string visit(const sequence_parser & p) { + void visit(sequence_parser & p) override { std::string s; - for (size_t i = 0; i < p.parsers_.size(); ++i) { - if (i > 0) s += " "; - auto child_result = p.parsers_[i]->accept(*this); - s += child_result; + for (const auto & child : p.parsers()) { + if (!s.empty()) { + s += " "; + } + child->accept(*this); + s += current_result_; } - return s; + current_result_ = s; } - std::string visit(const choice_parser & p) { + void visit(choice_parser & p) override { std::string s; - for (size_t i = 0; i < p.parsers_.size(); ++i) { - if (i > 0) { + for (const auto & child : p.parsers()) { + if (!s.empty()) { s += " | "; } - auto child_type = p.parsers_[i]->type(); - auto child_result = p.parsers_[i]->accept(*this); + child->accept(*this); // Parenthesize sequences in choices - if (child_type == PARSER_SEQUENCE) { - s += "(" + child_result + ")"; + if (child->type() == PARSER_SEQUENCE) { + s += "(" + current_result_ + ")"; } else { - s += child_result; + s += current_result_; } } - return s; + current_result_ = s; } - std::string visit(const one_or_more_parser & p) { - auto child_type = p.parser_->type(); - auto child_result = p.parser_->accept(*this); - if (needs_parens(child_type)) { - return "(" + child_result + ")+"; + void visit(one_or_more_parser & p) override { + p.child()->accept(*this); + if (needs_parens(p.child()->type())) { + current_result_ = "(" + current_result_ + ")+"; + } else { + current_result_ = current_result_ + "+"; } - return child_result + "+"; } - std::string visit(const zero_or_more_parser & p) { - auto child_type = p.parser_->type(); - auto child_result = p.parser_->accept(*this); - if (needs_parens(child_type)) { - return "(" + child_result + ")*"; + void visit(zero_or_more_parser & p) override { + p.child()->accept(*this); + if (needs_parens(p.child()->type())) { + current_result_ = "(" + current_result_ + ")*"; + } else { + current_result_ = current_result_ + "*"; } - return child_result + "*"; } - std::string visit(const optional_parser & p) { - auto child_type = p.parser_->type(); - auto child_result = p.parser_->accept(*this); - if (needs_parens(child_type)) { - return "(" + child_result + ")?"; + void visit(optional_parser & p) override { + p.child()->accept(*this); + if (needs_parens(p.child()->type())) { + current_result_ = "(" + current_result_ + ")?"; + } else { + current_result_ = current_result_ + "?"; } - return child_result + "?"; } - std::string visit(const until_parser & p) { + void visit(until_parser & p) override { // Generate pattern that matches prefixes but prevents full delimiter match - return generate_until_pattern(p.delimiter_) + "*"; + current_result_ = generate_until_pattern(p.delimiter()) + "*"; } - std::string visit(const not_parser &) { + void visit(not_parser &) override { // NOT is tricky in GBNF - for now, emit error LOG_ERR("NOT operator not directly supported in GBNF generation\n"); - return ""; // This will cause compilation errors, which is intended + current_result_ = ""; } - std::string visit(const any_parser &) { + void visit(any_parser &) override { // Match any single character - return "[\\x00-\\x{10FFFF}]"; + current_result_ = "[\\x00-\\x{10FFFF}]"; } - std::string visit(const space_parser &) { + void visit(space_parser &) override { // Reference the built-in space rule - return "space"; + current_result_ = "space"; } - std::string visit(const char_class_parser & p) { + void visit(char_class_parser & p) override { // Return pattern as-is (already in GBNF format) - return p.pattern_; + current_result_ = p.pattern(); } - std::string visit(const group_parser & p) { + void visit(group_parser & p) override { // Groups are transparent - just visit child - return p.parser_->accept(*this); + p.child()->accept(*this); } - std::string visit(const schema_parser & p) { - return builder_.add_schema(p.name_, p.schema_); + void visit(schema_parser & p) override { + current_result_ = builder_.add_schema(p.name(), p.schema()); } - std::string visit(const rule_parser & p) { + void visit(rule_parser & p) override { // Return canonical rule reference - auto it = rule_name_mapping_.find(p.rule_name_); + auto it = rule_name_mapping_.find(p.name()); if (it != rule_name_mapping_.end()) { - return it->second; + current_result_ = it->second; + } else { + // Fallback to original name if not in mapping (shouldn't happen in valid usage) + current_result_ = p.name(); } - // Fallback to original name if not in mapping (shouldn't happen in valid usage) - return p.rule_name_; } - std::string visit(const root_parser & p) { + void visit(root_parser & p) override { // Generate named rules first - if (p.rules_) { - for (const auto & [name, rule] : *p.rules_) { - auto rule_body = rule->accept(*this); + auto rules = p.rules(); + if (rules) { + for (const auto & [name, rule] : *rules) { + rule->accept(*this); + auto rule_body = current_result_; auto canonical_name = builder_.add_rule(name, rule_body); rule_name_mapping_[name] = canonical_name; } } // Return root body for composition - return p.root_->accept(*this); + p.root()->accept(*this); } }; -// Implement accept() methods for all parser classes -std::string literal_parser::accept(gbnf_visitor & visitor) const { - return visitor.visit(*this); -} +// ID assignment visitor for assigning unique IDs to parsers +class id_assignment_visitor : public parser_visitor { + int & next_id_; -std::string sequence_parser::accept(gbnf_visitor & visitor) const { - return visitor.visit(*this); -} + public: + id_assignment_visitor(int & next_id) : next_id_(next_id) {} -std::string choice_parser::accept(gbnf_visitor & visitor) const { - return visitor.visit(*this); -} + void assign_id(parser_base & p) { + if (p.id_ == -1) { + p.id_ = next_id_++; + } + } -std::string one_or_more_parser::accept(gbnf_visitor & visitor) const { - return visitor.visit(*this); -} + void visit(literal_parser & p) override { + assign_id(p); + } -std::string zero_or_more_parser::accept(gbnf_visitor & visitor) const { - return visitor.visit(*this); -} + void visit(any_parser & p) override { + assign_id(p); + } -std::string optional_parser::accept(gbnf_visitor & visitor) const { - return visitor.visit(*this); -} + void visit(space_parser & p) override { + assign_id(p); + } -std::string until_parser::accept(gbnf_visitor & visitor) const { - return visitor.visit(*this); -} + void visit(char_class_parser & p) override { + assign_id(p); + } -std::string not_parser::accept(gbnf_visitor & visitor) const { - return visitor.visit(*this); -} + void visit(schema_parser & p) override { + assign_id(p); + } -std::string any_parser::accept(gbnf_visitor & visitor) const { - return visitor.visit(*this); -} + void visit(rule_parser & p) override { + assign_id(p); + } -std::string space_parser::accept(gbnf_visitor & visitor) const { - return visitor.visit(*this); -} + // Composite parsers - assign ID and traverse children + void visit(sequence_parser & p) override { + assign_id(p); + for (const auto & child : p.parsers()) { + child->accept(*this); + } + } -std::string char_class_parser::accept(gbnf_visitor & visitor) const { - return visitor.visit(*this); -} + void visit(choice_parser & p) override { + assign_id(p); + for (const auto & child : p.parsers()) { + child->accept(*this); + } + } -std::string group_parser::accept(gbnf_visitor & visitor) const { - return visitor.visit(*this); -} + void visit(one_or_more_parser & p) override { + assign_id(p); + p.child()->accept(*this); + } -std::string schema_parser::accept(gbnf_visitor & visitor) const { - return visitor.visit(*this); -} + void visit(zero_or_more_parser & p) override { + assign_id(p); + p.child()->accept(*this); + } -std::string rule_parser::accept(gbnf_visitor & visitor) const { - return visitor.visit(*this); -} + void visit(optional_parser & p) override { + assign_id(p); + p.child()->accept(*this); + } -std::string root_parser::accept(gbnf_visitor & visitor) const { - return visitor.visit(*this); -} + void visit(until_parser & p) override { + assign_id(p); + p.child()->accept(*this); + } + + void visit(not_parser & p) override { + assign_id(p); + p.child()->accept(*this); + } + + void visit(group_parser & p) override { + assign_id(p); + p.child()->accept(*this); + } + + void visit(root_parser & p) override { + assign_id(p); + p.root()->accept(*this); + } +}; + +// Implement accept() methods for all parser classes +void literal_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } +void sequence_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } +void choice_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } +void one_or_more_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } +void zero_or_more_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } +void optional_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } +void until_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } +void not_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } +void any_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } +void space_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } +void char_class_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } +void group_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } +void schema_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } +void rule_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } +void root_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } std::optional parser_result::group(const std::string & name, std::string_view input) const { auto it = groups.find(name); @@ -1145,11 +1140,10 @@ std::string parser::dump() const { return ptr->dump(); } -void parser::build_grammar(common_grammar_builder& builder) const { +void parser::build_grammar(common_grammar_builder& builder) { gbnf_visitor visitor(builder); - auto result = ptr->accept(visitor); - // The visitor returns the GBNF string for this parser - // root_parser registers its named rules and returns its root body + ptr->accept(visitor); + auto result = visitor.result(); if (!result.empty()) { builder.add_rule("root", result); } @@ -1222,7 +1216,8 @@ parser parser_builder::add_rule(const std::string & name, const parser & p) { void parser_builder::assign_ids(parser & p) { if (p.ptr) { - p.ptr->assign_ids_internal(next_id_); + id_assignment_visitor visitor(next_id_); + p.ptr->accept(visitor); } } diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index 1ef3996dc862a..f0cb1d24ff7bb 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -101,7 +101,7 @@ class parser_base; class sequence_parser; class choice_parser; class parser_builder; -class gbnf_visitor; +class parser_visitor; class parser { std::shared_ptr ptr; @@ -136,7 +136,7 @@ class parser { parser_type type() const; parser_result parse(parser_context & ctx, size_t start = 0) const; std::string dump() const; - void build_grammar(common_grammar_builder& builder) const; + void build_grammar(common_grammar_builder& builder); }; class parser_builder { From 76cf0b5b6197d427e3c48aa4d24f549a3d3a4167 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Mon, 10 Nov 2025 20:21:04 -0600 Subject: [PATCH 007/183] fix const ref for grammar builder --- common/chat-parser-combinator.cpp | 6 +-- common/chat-parser-combinator.h | 3 +- tests/test-chat-parser-combinator.cpp | 69 ++++++++++++++++----------- 3 files changed, 46 insertions(+), 32 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index 598fd74a93657..aff72b67fd68d 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -736,12 +736,12 @@ class parser_visitor { }; class gbnf_visitor : public parser_visitor { - common_grammar_builder& builder_; + const common_grammar_builder & builder_; std::unordered_map rule_name_mapping_; std::string current_result_; public: - gbnf_visitor(common_grammar_builder& builder) : builder_(builder) {} + gbnf_visitor(const common_grammar_builder & builder) : builder_(builder) {} const std::string& result() const { return current_result_; } @@ -1140,7 +1140,7 @@ std::string parser::dump() const { return ptr->dump(); } -void parser::build_grammar(common_grammar_builder& builder) { +void parser::build_grammar(const common_grammar_builder & builder) { gbnf_visitor visitor(builder); ptr->accept(visitor); auto result = visitor.result(); diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index f0cb1d24ff7bb..e56a6adf24c17 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -136,7 +136,8 @@ class parser { parser_type type() const; parser_result parse(parser_context & ctx, size_t start = 0) const; std::string dump() const; - void build_grammar(common_grammar_builder& builder); + + void build_grammar(const common_grammar_builder & builder); }; class parser_builder { diff --git a/tests/test-chat-parser-combinator.cpp b/tests/test-chat-parser-combinator.cpp index 1fffbfbf040a0..e4f637af9f797 100644 --- a/tests/test-chat-parser-combinator.cpp +++ b/tests/test-chat-parser-combinator.cpp @@ -518,8 +518,8 @@ static void test_complete_example() { assert_equals(std::string("get_weather"), *result.group("tool-name", ctx.input)); assert_equals(std::string(R"({"cit)"), *result.group("tool-args", ctx.input)); - auto gbnf = ::build_grammar([&](const common_grammar_builder& builder) { - parser.build_grammar(const_cast(builder)); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); }); std::cout << "Grammar:\n" << gbnf << "\n"; @@ -532,9 +532,10 @@ static void test_gbnf_generation() { return p.literal("hello"); }); - auto gbnf = ::build_grammar([&](const common_grammar_builder& builder) { - parser.build_grammar(const_cast(builder)); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); }); + assert_equals(true, gbnf.find("root ::= \"hello\"") != std::string::npos); assert_equals(true, gbnf.find("space ::=") != std::string::npos); } @@ -544,9 +545,10 @@ static void test_gbnf_generation() { return p.char_class("[a-z]"); }); - auto gbnf = ::build_grammar([&](const common_grammar_builder& builder) { - parser.build_grammar(const_cast(builder)); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); }); + assert_equals(true, gbnf.find("root ::= [a-z]") != std::string::npos); } { @@ -555,9 +557,10 @@ static void test_gbnf_generation() { return p.literal("hello") + p.literal(" ") + p.literal("world"); }); - auto gbnf = ::build_grammar([&](const common_grammar_builder& builder) { - parser.build_grammar(const_cast(builder)); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); }); + assert_equals(true, gbnf.find("root ::= \"hello\" \" \" \"world\"") != std::string::npos); } { @@ -566,9 +569,10 @@ static void test_gbnf_generation() { return p.literal("cat") | p.literal("dog"); }); - auto gbnf = ::build_grammar([&](const common_grammar_builder& builder) { - parser.build_grammar(const_cast(builder)); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); }); + assert_equals(true, gbnf.find("root ::= \"cat\" | \"dog\"") != std::string::npos); } { @@ -577,9 +581,10 @@ static void test_gbnf_generation() { return p.one_or_more(p.char_class("[0-9]")); }); - auto gbnf = ::build_grammar([&](const common_grammar_builder& builder) { - parser.build_grammar(const_cast(builder)); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); }); + assert_equals(true, gbnf.find("root ::= [0-9]+") != std::string::npos); } { @@ -588,9 +593,10 @@ static void test_gbnf_generation() { return p.zero_or_more(p.char_class("[a-z]")); }); - auto gbnf = ::build_grammar([&](const common_grammar_builder& builder) { - parser.build_grammar(const_cast(builder)); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); }); + assert_equals(true, gbnf.find("root ::= [a-z]*") != std::string::npos); } { @@ -599,9 +605,10 @@ static void test_gbnf_generation() { return p.literal("hello") + p.optional(p.literal(" world")); }); - auto gbnf = ::build_grammar([&](const common_grammar_builder& builder) { - parser.build_grammar(const_cast(builder)); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); }); + assert_equals(true, gbnf.find("root ::= \"hello\" \" world\"?") != std::string::npos); } { @@ -610,9 +617,10 @@ static void test_gbnf_generation() { return p.until(""); }); - auto gbnf = ::build_grammar([&](const common_grammar_builder& builder) { - parser.build_grammar(const_cast(builder)); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); }); + // Should generate pattern that prevents matching the full delimiter assert_equals(true, gbnf.find("root ::= ([^<] | \"<\" [^/] | \"])*") != std::string::npos); } @@ -622,9 +630,10 @@ static void test_gbnf_generation() { return p.group("test", p.literal("hello")); }); - auto gbnf = ::build_grammar([&](const common_grammar_builder& builder) { - parser.build_grammar(const_cast(builder)); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); }); + assert_equals(true, gbnf.find("root ::= \"hello\"") != std::string::npos); } { @@ -633,9 +642,10 @@ static void test_gbnf_generation() { return p.one_or_more(p.literal("a") | p.literal("b")); }); - auto gbnf = ::build_grammar([&](const common_grammar_builder& builder) { - parser.build_grammar(const_cast(builder)); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); }); + assert_equals(true, gbnf.find("root ::= (\"a\" | \"b\")+") != std::string::npos); } { @@ -645,9 +655,10 @@ static void test_gbnf_generation() { return p.one_or_more(digit); }); - auto gbnf = ::build_grammar([&](const common_grammar_builder& builder) { - parser.build_grammar(const_cast(builder)); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); }); + // Should have digit rule defined and referenced assert_equals(true, gbnf.find("digit ::= [0-9]") != std::string::npos); assert_equals(true, gbnf.find("root ::= digit+") != std::string::npos); @@ -658,9 +669,10 @@ static void test_gbnf_generation() { return p.literal("hello\nworld\t!"); }); - auto gbnf = ::build_grammar([&](const common_grammar_builder& builder) { - parser.build_grammar(const_cast(builder)); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); }); + assert_equals(true, gbnf.find("root ::= \"hello\\nworld\\t!\"") != std::string::npos); } { @@ -669,9 +681,10 @@ static void test_gbnf_generation() { return p.literal("hello") << p.literal("world"); }); - auto gbnf = ::build_grammar([&](const common_grammar_builder& builder) { - parser.build_grammar(const_cast(builder)); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); }); + // Should inline the whitespace pattern assert_equals(true, gbnf.find("\"hello\"") != std::string::npos); assert_equals(true, gbnf.find("\"world\"") != std::string::npos); From 9c7b3e8bcf57ea416d21a90d219c39d06e16b426 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Mon, 10 Nov 2025 20:33:26 -0600 Subject: [PATCH 008/183] clean up types, friend classes, and class declarations --- common/chat-parser-combinator.cpp | 76 +++++++++++++++---------------- common/chat-parser-combinator.h | 39 ++-------------- 2 files changed, 43 insertions(+), 72 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index aff72b67fd68d..56215302df99e 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -8,7 +8,24 @@ #include #include -class id_assignment_visitor; +enum parser_type { + PARSER_LITERAL = 0, + PARSER_SEQUENCE = 1, + PARSER_CHOICE = 2, + PARSER_ZERO_OR_MORE = 3, + PARSER_ONE_OR_MORE = 4, + PARSER_NOT = 5, + PARSER_ANY = 6, + PARSER_CHAR_CLASS = 7, + PARSER_GROUP = 8, + PARSER_RULE = 9, + PARSER_OPTIONAL = 10, + PARSER_UNTIL = 11, + PARSER_SPACE = 12, + PARSER_SCHEMA = 13, +}; + +class parser_visitor; static parser json_parser(); @@ -16,12 +33,13 @@ class parser_base { protected: int id_; - friend class id_assignment_visitor; - public: parser_base(int id) : id_(id) {} virtual ~parser_base() = default; + int id() const { return id_; } + void set_id(int id) { id_ = id; } + virtual parser_type type() const = 0; virtual parser_result parse(parser_context & ctx, size_t start = 0) = 0; virtual std::string dump() const = 0; @@ -77,9 +95,10 @@ class sequence_parser : public parser_base { public: sequence_parser(std::initializer_list parsers, int id) : parser_base(id) { for (const auto & p : parsers) { - if (p.is_sequence()) { + if (p->type() == PARSER_SEQUENCE) { // Flatten sequences - for (const auto & embedded : p.to_sequence()->parsers()) { + auto seq = std::static_pointer_cast(p.ptr()); + for (const auto & embedded : seq->parsers()) { parsers_.push_back(embedded); } } else { @@ -143,9 +162,10 @@ class choice_parser : public parser_base { public: choice_parser(std::initializer_list parsers, int id) : parser_base(id) { for (const auto & p : parsers) { - if (p.is_choice()) { + if (p->type() == PARSER_CHOICE) { // Flatten choices - for (const auto & embedded : p.to_choice()->parsers()) { + auto choice = std::static_pointer_cast(p.ptr()); + for (const auto & embedded : choice->parsers()) { parsers_.push_back(embedded); } } else { @@ -952,8 +972,8 @@ class id_assignment_visitor : public parser_visitor { id_assignment_visitor(int & next_id) : next_id_(next_id) {} void assign_id(parser_base & p) { - if (p.id_ == -1) { - p.id_ = next_id_++; + if (p.id() == -1) { + p.set_id(next_id_++); } } @@ -1085,7 +1105,7 @@ void parse_cache::clear() { parser::parser() {} -parser::parser(std::shared_ptr parser) : ptr(std::move(parser)) {} +parser::parser(std::shared_ptr parser) : ptr_(std::move(parser)) {} parser parser::operator~() const { return parser(std::make_shared(*this, -1)); @@ -1105,44 +1125,24 @@ parser parser::operator<<(const parser & other) const { } parser_base & parser::operator*() const { - return *ptr; + return *ptr_; } parser_base * parser::operator->() const { - return ptr.get(); -} - -bool parser::is_sequence() const { - return ptr->type() == PARSER_SEQUENCE; -} - -std::shared_ptr parser::to_sequence() const { - return std::dynamic_pointer_cast(ptr); -} - -bool parser::is_choice() const { - return ptr->type() == PARSER_CHOICE; -} - -std::shared_ptr parser::to_choice() const { - return std::dynamic_pointer_cast(ptr); -} - -parser_type parser::type() const { - return ptr->type(); + return ptr_.get(); } parser_result parser::parse(parser_context & ctx, size_t start) const { - return ptr->parse(ctx, start); + return ptr_->parse(ctx, start); } std::string parser::dump() const { - return ptr->dump(); + return ptr_->dump(); } -void parser::build_grammar(const common_grammar_builder & builder) { +void parser::build_grammar(const common_grammar_builder & builder) const { gbnf_visitor visitor(builder); - ptr->accept(visitor); + ptr_->accept(visitor); auto result = visitor.result(); if (!result.empty()) { builder.add_rule("root", result); @@ -1215,9 +1215,9 @@ parser parser_builder::add_rule(const std::string & name, const parser & p) { } void parser_builder::assign_ids(parser & p) { - if (p.ptr) { + if (p.ptr()) { id_assignment_visitor visitor(next_id_); - p.ptr->accept(visitor); + p.ptr()->accept(visitor); } } diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index e56a6adf24c17..6c7b86d4e04e9 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -11,23 +11,6 @@ struct common_grammar_builder; -enum parser_type { - PARSER_LITERAL = 0, - PARSER_SEQUENCE = 1, - PARSER_CHOICE = 2, - PARSER_ZERO_OR_MORE = 3, - PARSER_ONE_OR_MORE = 4, - PARSER_NOT = 5, - PARSER_ANY = 6, - PARSER_CHAR_CLASS = 7, - PARSER_GROUP = 8, - PARSER_RULE = 9, - PARSER_OPTIONAL = 10, - PARSER_UNTIL = 11, - PARSER_SPACE = 12, - PARSER_SCHEMA = 13, -}; - enum parser_result_type { PARSER_RESULT_FAIL = 0, PARSER_RESULT_NEED_MORE_INPUT = 1, @@ -89,8 +72,6 @@ class parse_cache { void clear(); }; -class parser; - struct parser_context { std::string_view input; parse_cache memo; @@ -98,15 +79,9 @@ struct parser_context { }; class parser_base; -class sequence_parser; -class choice_parser; -class parser_builder; -class parser_visitor; class parser { - std::shared_ptr ptr; - - friend class parser_builder; + std::shared_ptr ptr_; public: parser(); @@ -114,7 +89,7 @@ class parser { parser(const parser & other) = default; parser & operator=(const parser & other) { if (this != &other) { - ptr = other.ptr; + ptr_ = other.ptr_; } return *this; } @@ -127,17 +102,13 @@ class parser { parser_base & operator*() const; parser_base * operator->() const; - bool is_sequence() const; - std::shared_ptr to_sequence() const; - - bool is_choice() const; - std::shared_ptr to_choice() const; + std::shared_ptr ptr() const { return ptr_; } - parser_type type() const; parser_result parse(parser_context & ctx, size_t start = 0) const; + std::string dump() const; - void build_grammar(const common_grammar_builder & builder); + void build_grammar(const common_grammar_builder & builder) const; }; class parser_builder { From f02e2b06fa0ef29aa647060ec74f7f0c6224606b Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Mon, 10 Nov 2025 20:47:43 -0600 Subject: [PATCH 009/183] remove builder usage from until_parser --- common/chat-parser-combinator.cpp | 84 ++++++++++++++++--------------- common/chat-parser-combinator.h | 2 +- 2 files changed, 45 insertions(+), 41 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index 56215302df99e..0081606516d40 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -367,44 +367,6 @@ class optional_parser : public parser_base { const parser & child() const { return parser_; } }; -class until_parser : public parser_base { - std::string delimiter_; - parser parser_; - - public: - until_parser(const std::string & delimiter, bool include_spaces, int id, parser_builder & builder) - : parser_base(id), delimiter_(delimiter) { - if (include_spaces) { - auto ws = builder.zero_or_more(builder.char_class("[ \\t\\n\\r]")); - parser_ = builder.zero_or_more(builder.negate(ws + builder.literal(delimiter)) + builder.any()); - } else { - parser_ = builder.zero_or_more(builder.negate(builder.literal(delimiter)) + builder.any()); - } - } - - parser_type type() const override { return PARSER_UNTIL; } - - parser_result parse(parser_context & ctx, size_t start = 0) override { - auto cached = ctx.memo.get(id_, start); - if (cached != std::nullopt) { - return *cached; - } - - auto result = parser_->parse(ctx, start); - return ctx.memo.set(id_, start, result); - } - - std::string dump() const override { - return "Until(" + delimiter_ + ")"; - } - - void accept(parser_visitor & visitor) override; - - const std::string & delimiter() const { return delimiter_; } - - const parser & child() const { return parser_; } -}; - class not_parser : public parser_base { parser parser_; @@ -637,6 +599,48 @@ class group_parser : public parser_base { const parser & child() const { return parser_; } }; +class until_parser : public parser_base { + std::string delimiter_; + parser parser_; + + public: + until_parser(const std::string & delimiter, bool consume_spaces, int id) + : parser_base(id), delimiter_(delimiter) { + + auto delim = parser(std::make_shared(delimiter, -1)); + auto any = parser(std::make_shared(-1)); + + if (consume_spaces) { + auto ws = parser(std::make_shared(-1)); + parser_ = parser(std::make_shared(~(ws + delim) + any, -1)); + } else { + parser_ = parser(std::make_shared(~delim + any, -1)); + } + } + + parser_type type() const override { return PARSER_UNTIL; } + + parser_result parse(parser_context & ctx, size_t start = 0) override { + auto cached = ctx.memo.get(id_, start); + if (cached != std::nullopt) { + return *cached; + } + + auto result = parser_->parse(ctx, start); + return ctx.memo.set(id_, start, result); + } + + std::string dump() const override { + return "Until(" + delimiter_ + ")"; + } + + void accept(parser_visitor & visitor) override; + + const std::string & delimiter() const { return delimiter_; } + + const parser & child() const { return parser_; } +}; + class schema_parser : public parser_base { parser parser_; std::string name_; @@ -1201,8 +1205,8 @@ parser parser_builder::space() { return parser(std::make_shared(next_id_++)); } -parser parser_builder::until(const std::string & delimiter, bool include_spaces) { - return parser(std::make_shared(delimiter, include_spaces, next_id_++, *this)); +parser parser_builder::until(const std::string & delimiter, bool consume_spaces) { + return parser(std::make_shared(delimiter, consume_spaces, next_id_++)); } parser parser_builder::schema(const parser & p, const std::string & name, const nlohmann::ordered_json & schema) { diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index 6c7b86d4e04e9..e5f508d963011 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -130,7 +130,7 @@ class parser_builder { parser group(const std::string & name, const parser & p); parser rule(const std::string & name); parser space(); - parser until(const std::string & delimiter, bool include_spaces = true); + parser until(const std::string & delimiter, bool consume_spaces = true); parser json(); parser schema(const parser & p, const std::string & name, const nlohmann::ordered_json & schema); From 66cf038a37596bee9771142d9900c7aea35c0b32 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Mon, 10 Nov 2025 21:08:17 -0600 Subject: [PATCH 010/183] Use a counter class to help assign rule ids --- common/chat-parser-combinator.cpp | 74 +++++++++++++++---------------- common/chat-parser-combinator.h | 10 ++++- 2 files changed, 45 insertions(+), 39 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index 0081606516d40..998e5511d857a 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -27,8 +27,6 @@ enum parser_type { class parser_visitor; -static parser json_parser(); - class parser_base { protected: int id_; @@ -970,14 +968,14 @@ class gbnf_visitor : public parser_visitor { // ID assignment visitor for assigning unique IDs to parsers class id_assignment_visitor : public parser_visitor { - int & next_id_; + std::shared_ptr counter_; public: - id_assignment_visitor(int & next_id) : next_id_(next_id) {} + id_assignment_visitor(const std::shared_ptr & counter) : counter_(counter) {} void assign_id(parser_base & p) { if (p.id() == -1) { - p.set_id(next_id_++); + p.set_id(counter_->next()); } } @@ -1155,62 +1153,66 @@ void parser::build_grammar(const common_grammar_builder & builder) const { parser_builder::parser_builder() : rules_(std::make_shared>()) - , next_id_(0) {} + , counter_(std::make_shared(0)) {} + +parser_builder::parser_builder(std::shared_ptr counter) + : rules_(std::make_shared>()) + , counter_(std::move(counter)) {} parser parser_builder::literal(const std::string & literal) { - return parser(std::make_shared(literal, next_id_++)); + return parser(std::make_shared(literal, counter_->next())); } parser parser_builder::sequence(std::initializer_list parsers) { - return parser(std::make_shared(parsers, next_id_++)); + return parser(std::make_shared(parsers, counter_->next())); } parser parser_builder::choice(std::initializer_list parsers) { - return parser(std::make_shared(parsers, next_id_++)); + return parser(std::make_shared(parsers, counter_->next())); } parser parser_builder::one_or_more(const parser & p) { - return parser(std::make_shared(p, next_id_++)); + return parser(std::make_shared(p, counter_->next())); } parser parser_builder::zero_or_more(const parser & p) { - return parser(std::make_shared(p, next_id_++)); + return parser(std::make_shared(p, counter_->next())); } parser parser_builder::optional(const parser & p) { - return parser(std::make_shared(p, next_id_++)); + return parser(std::make_shared(p, counter_->next())); } parser parser_builder::negate(const parser & p) { - return parser(std::make_shared(p, next_id_++)); + return parser(std::make_shared(p, counter_->next())); } parser parser_builder::any() { - return parser(std::make_shared(next_id_++)); + return parser(std::make_shared(counter_->next())); } parser parser_builder::char_class(const std::string & classes) { - return parser(std::make_shared(classes, next_id_++)); + return parser(std::make_shared(classes, counter_->next())); } parser parser_builder::group(const std::string & name, const parser & p) { - return parser(std::make_shared(name, p, next_id_++)); + return parser(std::make_shared(name, p, counter_->next())); } parser parser_builder::rule(const std::string & name) { - return parser(std::make_shared(name, rules_, next_id_++)); + return parser(std::make_shared(name, rules_, counter_->next())); } parser parser_builder::space() { - return parser(std::make_shared(next_id_++)); + return parser(std::make_shared(counter_->next())); } parser parser_builder::until(const std::string & delimiter, bool consume_spaces) { - return parser(std::make_shared(delimiter, consume_spaces, next_id_++)); + return parser(std::make_shared(delimiter, consume_spaces, counter_->next())); } parser parser_builder::schema(const parser & p, const std::string & name, const nlohmann::ordered_json & schema) { - return parser(std::make_shared(p, name, schema, next_id_++)); + return parser(std::make_shared(p, name, schema, counter_->next())); } parser parser_builder::add_rule(const std::string & name, const parser & p) { @@ -1220,7 +1222,7 @@ parser parser_builder::add_rule(const std::string & name, const parser & p) { void parser_builder::assign_ids(parser & p) { if (p.ptr()) { - id_assignment_visitor visitor(next_id_); + id_assignment_visitor visitor(counter_); p.ptr()->accept(visitor); } } @@ -1238,8 +1240,8 @@ parser build_parser(const std::function & fn) { return root; } -static parser json_parser() { - parser_builder builder; +static parser json_parser(std::shared_ptr counter) { + parser_builder builder(std::move(counter)); // Whitespace: space, tab, newline, carriage return auto ws = builder.zero_or_more(builder.char_class("[ \\t\\n\\r]")); @@ -1280,17 +1282,6 @@ static parser json_parser() { auto false_lit = builder.literal("false"); auto null_lit = builder.literal("null"); - // Value - uses forward references for recursive structures - builder.add_rule("json_value", - builder.rule("json_object") | - builder.rule("json_array") | - builder.rule("json_string") | - builder.rule("json_number") | - true_lit | - false_lit | - null_lit - ); - // Object: { "key": value, ... } auto member = builder.rule("json_string") + ws + builder.literal(":") + ws + builder.rule("json_value"); auto members = member + builder.zero_or_more(ws + builder.literal(",") + ws + member); @@ -1310,14 +1301,21 @@ static parser json_parser() { builder.add_rule("json_array", array); - // Get the json_value rule as the root - auto root = builder.rule("json_value"); - builder.assign_ids(root); + // Value - uses forward references for recursive structures + auto root = builder.add_rule("json_value", + builder.rule("json_object") | + builder.rule("json_array") | + builder.rule("json_string") | + builder.rule("json_number") | + true_lit | + false_lit | + null_lit + ); // Wrap in root_parser to own the rules return parser(std::make_shared(root, builder.rules(), -1)); } parser parser_builder::json() { - return json_parser(); + return json_parser(counter_); } diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index e5f508d963011..56a522394bf75 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -111,12 +111,20 @@ class parser { void build_grammar(const common_grammar_builder & builder) const; }; +class parser_id_counter { + int next_id_; + public: + parser_id_counter(int start) : next_id_(start) {} + int next() { return next_id_++; } +}; + class parser_builder { std::shared_ptr> rules_; - int next_id_; + std::shared_ptr counter_; public: parser_builder(); + parser_builder(std::shared_ptr counter); parser literal(const std::string & literal); parser sequence(std::initializer_list parsers); From 2b3caefde82282c5933c9ad53014c336184cedfb Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Mon, 10 Nov 2025 21:24:05 -0600 Subject: [PATCH 011/183] cache everything --- common/chat-parser-combinator.cpp | 419 ++++++++++++++---------------- common/chat-parser-combinator.h | 2 + 2 files changed, 194 insertions(+), 227 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index 998e5511d857a..fb3b0fb083a0a 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -53,29 +53,26 @@ class literal_parser : public parser_base { parser_type type() const override { return PARSER_LITERAL; } parser_result parse(parser_context & ctx, size_t start = 0) override { - auto cached = ctx.memo.get(id_, start); - if (cached != std::nullopt) { - return *cached; - } - - auto pos = start; - for (auto i = 0u; i < literal_.size(); ++i) { - if (pos >= ctx.input.size()) { - if (ctx.input_is_complete) { - return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_FAIL, start)); + return ctx.memo.cached(id_, start, [&]() { + auto pos = start; + for (auto i = 0u; i < literal_.size(); ++i) { + if (pos >= ctx.input.size()) { + if (ctx.input_is_complete) { + return parser_result(PARSER_RESULT_FAIL, start); + } + if (i > 0) { + return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, pos); + } + return parser_result(PARSER_RESULT_FAIL, start); } - if (i > 0) { - return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, pos); + if (ctx.input[pos] != literal_[i]) { + return parser_result(PARSER_RESULT_FAIL, start); } - return parser_result(PARSER_RESULT_FAIL, start); - } - if (ctx.input[pos] != literal_[i]) { - return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_FAIL, start)); + ++pos; } - ++pos; - } - return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_SUCCESS, start, pos)); + return parser_result(PARSER_RESULT_SUCCESS, start, pos); + }); } std::string dump() const override { @@ -108,36 +105,33 @@ class sequence_parser : public parser_base { parser_type type() const override { return PARSER_SEQUENCE; } parser_result parse(parser_context & ctx, size_t start = 0) override { - auto cached = ctx.memo.get(id_, start); - if (cached != std::nullopt) { - return *cached; - } - - std::unordered_map groups; - - auto pos = start; - for (const auto & p : parsers_) { - auto result = p->parse(ctx, pos); - - // Copy groups - groups.insert(result.groups.begin(), result.groups.end()); + return ctx.memo.cached(id_, start, [&]() { + std::unordered_map groups; + + auto pos = start; + for (const auto & p : parsers_) { + auto result = p->parse(ctx, pos); + + // Copy groups + groups.insert(result.groups.begin(), result.groups.end()); + + if (result.is_fail()) { + if (result.end >= ctx.input.size() && !ctx.input_is_complete) { + // If we fail because we don't have enough input, then return success + return parser_result(PARSER_RESULT_SUCCESS, start, result.end, groups); + } + return parser_result(PARSER_RESULT_FAIL, start, result.end, groups); + } - if (result.is_fail()) { - if (result.end >= ctx.input.size() && !ctx.input_is_complete) { - // If we fail because we don't have enough input, then return success - return parser_result(PARSER_RESULT_SUCCESS, start, result.end, groups); + if (result.is_need_more_input()) { + return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, result.end, groups); } - return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_FAIL, start, result.end, groups)); - } - if (result.is_need_more_input()) { - return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, result.end, groups); + pos = result.end; } - pos = result.end; - } - - return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_SUCCESS, start, pos, groups)); + return parser_result(PARSER_RESULT_SUCCESS, start, pos, groups); + }); } std::string dump() const override { @@ -175,25 +169,22 @@ class choice_parser : public parser_base { parser_type type() const override { return PARSER_CHOICE; } parser_result parse(parser_context & ctx, size_t start = 0) override { - auto cached = ctx.memo.get(id_, start); - if (cached != std::nullopt) { - return *cached; - } + return ctx.memo.cached(id_, start, [&]() { + auto pos = start; + for (const auto & p : parsers_) { + auto result = p->parse(ctx, pos); - auto pos = start; - for (const auto & p : parsers_) { - auto result = p->parse(ctx, pos); - - if (result.is_success()) { - return ctx.memo.set(id_, start, result); - } + if (result.is_success()) { + return result; + } - if (result.is_need_more_input()) { - return result; + if (result.is_need_more_input()) { + return result; + } } - } - return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_FAIL, start)); + return parser_result(PARSER_RESULT_FAIL, start); + }); } std::string dump() const override { @@ -219,49 +210,41 @@ class one_or_more_parser : public parser_base { parser_type type() const override { return PARSER_ONE_OR_MORE; } parser_result parse(parser_context & ctx, size_t start = 0) override { - auto cached = ctx.memo.get(id_, start); - std::unordered_map groups; - - // We can't return back the cached result, since there may be more - // repetitions since the last parsing attempt. Instead, resume parsing from - // the last successful repetition found. - auto pos = start; - if (cached != std::nullopt) { - pos = cached->end; - groups.insert(cached->groups.begin(), cached->groups.end()); - } + return ctx.memo.cached(id_, start, [&]() { + std::unordered_map groups; - if (pos == start) { - auto first_result = parser_->parse(ctx, pos); + // Parse at least once + auto first_result = parser_->parse(ctx, start); if (!first_result.is_success()) { return first_result; } - pos = first_result.end; + auto pos = first_result.end; groups.insert(first_result.groups.begin(), first_result.groups.end()); - } - for (;;) { - auto result = parser_->parse(ctx, pos); - groups.insert(result.groups.begin(), result.groups.end()); + // Parse zero or more additional times + for (;;) { + auto result = parser_->parse(ctx, pos); + groups.insert(result.groups.begin(), result.groups.end()); - if (result.is_need_more_input()) { - return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, pos, groups); - } + if (result.is_need_more_input()) { + return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, pos, groups); + } - if (result.is_fail()) { - // Done with repetitions - break; - } + if (result.is_fail()) { + // Done with repetitions + break; + } - if (result.end == pos) { - break; // Prevent an infinite loop - } + if (result.end == pos) { + break; // Prevent an infinite loop + } - pos = result.end; - } + pos = result.end; + } - return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_SUCCESS, start, pos, groups)); + return parser_result(PARSER_RESULT_SUCCESS, start, pos, groups); + }); } std::string dump() const override { @@ -282,39 +265,32 @@ class zero_or_more_parser : public parser_base { parser_type type() const override { return PARSER_ZERO_OR_MORE; } parser_result parse(parser_context & ctx, size_t start = 0) override { - auto cached = ctx.memo.get(id_, start); - std::unordered_map groups; - - // We can't return back the cached result, since there may be more - // repetitions since the last parsing attempt. Instead, resume parsing from - // the last successful repetition found. - auto pos = start; - if (cached != std::nullopt) { - pos = cached->end; - groups.insert(cached->groups.begin(), cached->groups.end()); - } + return ctx.memo.cached(id_, start, [&]() { + std::unordered_map groups; + auto pos = start; - for (;;) { - auto result = parser_->parse(ctx, pos); - groups.insert(result.groups.begin(), result.groups.end()); + for (;;) { + auto result = parser_->parse(ctx, pos); + groups.insert(result.groups.begin(), result.groups.end()); - if (result.is_need_more_input()) { - return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, pos, groups); - } + if (result.is_need_more_input()) { + return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, pos, groups); + } - if (result.is_fail()) { - // Done with repetitions (zero or more is always valid) - break; - } + if (result.is_fail()) { + // Done with repetitions (zero or more is always valid) + break; + } - if (result.end == pos) { - break; // Prevent an infinite loop - } + if (result.end == pos) { + break; // Prevent an infinite loop + } - pos = result.end; - } + pos = result.end; + } - return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_SUCCESS, start, pos, groups)); + return parser_result(PARSER_RESULT_SUCCESS, start, pos, groups); + }); } std::string dump() const override { @@ -335,25 +311,22 @@ class optional_parser : public parser_base { parser_type type() const override { return PARSER_OPTIONAL; } parser_result parse(parser_context & ctx, size_t start = 0) override { - auto cached = ctx.memo.get(id_, start); - if (cached != std::nullopt) { - return *cached; - } - - auto result = parser_->parse(ctx, start); + return ctx.memo.cached(id_, start, [&]() { + auto result = parser_->parse(ctx, start); - if (result.is_success()) { - // Matched successfully - return ctx.memo.set(id_, start, result); - } + if (result.is_success()) { + // Matched successfully + return result; + } - if (result.is_need_more_input()) { - // Propagate - need more input to determine if optional matches - return result; - } + if (result.is_need_more_input()) { + // Propagate - need more input to determine if optional matches + return result; + } - // No match, but optional always succeeds with zero matches - return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_SUCCESS, start, start)); + // No match, but optional always succeeds with zero matches + return parser_result(PARSER_RESULT_SUCCESS, start, start); + }); } std::string dump() const override { @@ -374,25 +347,22 @@ class not_parser : public parser_base { parser_type type() const override { return PARSER_NOT; } parser_result parse(parser_context & ctx, size_t start = 0) override { - auto cached = ctx.memo.get(id_, start); - if (cached != std::nullopt) { - return *cached; - } - - auto result = parser_->parse(ctx, start); + return ctx.memo.cached(id_, start, [&]() { + auto result = parser_->parse(ctx, start); - if (result.is_success()) { - // Fail if the underlying parser matches - return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_FAIL, start)); - } + if (result.is_success()) { + // Fail if the underlying parser matches + return parser_result(PARSER_RESULT_FAIL, start); + } - if (result.is_need_more_input()) { - // Propagate - need to know what child would match before negating - return result; - } + if (result.is_need_more_input()) { + // Propagate - need to know what child would match before negating + return result; + } - // Child failed, so negation succeeds - return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_SUCCESS, start)); + // Child failed, so negation succeeds + return parser_result(PARSER_RESULT_SUCCESS, start); + }); } std::string dump() const override { @@ -411,19 +381,16 @@ class any_parser : public parser_base { parser_type type() const override { return PARSER_ANY; } parser_result parse(parser_context & ctx, size_t start = 0) override { - auto cached = ctx.memo.get(id_, start); - if (cached != std::nullopt) { - return *cached; - } - - if (start >= ctx.input.size()) { - if (ctx.input_is_complete) { - return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_FAIL, start)); + return ctx.memo.cached(id_, start, [&]() { + if (start >= ctx.input.size()) { + if (ctx.input_is_complete) { + return parser_result(PARSER_RESULT_FAIL, start); + } + return parser_result(PARSER_RESULT_FAIL, start); } - return parser_result(PARSER_RESULT_FAIL, start); - } - return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_SUCCESS, start, start + 1)); + return parser_result(PARSER_RESULT_SUCCESS, start, start + 1); + }); } std::string dump() const override { @@ -440,22 +407,19 @@ class space_parser : public parser_base { parser_type type() const override { return PARSER_SPACE; } parser_result parse(parser_context & ctx, size_t start = 0) override { - auto cached = ctx.memo.get(id_, start); - if (cached != std::nullopt) { - return *cached; - } - - auto pos = start; - while (pos < ctx.input.size()) { - char c = ctx.input[pos]; - if (c == ' ' || c == '\t' || c == '\n') { - ++pos; - } else { - break; + return ctx.memo.cached(id_, start, [&]() { + auto pos = start; + while (pos < ctx.input.size()) { + char c = ctx.input[pos]; + if (c == ' ' || c == '\t' || c == '\n') { + ++pos; + } else { + break; + } } - } - return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_SUCCESS, start, pos)); + return parser_result(PARSER_RESULT_SUCCESS, start, pos); + }); } std::string dump() const override { @@ -530,36 +494,33 @@ class char_class_parser : public parser_base { parser_type type() const override { return PARSER_CHAR_CLASS; } parser_result parse(parser_context & ctx, size_t start = 0) override { - auto cached = ctx.memo.get(id_, start); - if (cached != std::nullopt) { - return *cached; - } - - if (start >= ctx.input.size()) { - if (ctx.input_is_complete) { - return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_FAIL, start)); + return ctx.memo.cached(id_, start, [&]() { + if (start >= ctx.input.size()) { + if (ctx.input_is_complete) { + return parser_result(PARSER_RESULT_FAIL, start); + } + return parser_result(PARSER_RESULT_FAIL, start); } - return parser_result(PARSER_RESULT_FAIL, start); - } - bool matches = false; - for (const auto & range : ranges_) { - if (range.contains(ctx.input[start])) { - matches = true; - break; + bool matches = false; + for (const auto & range : ranges_) { + if (range.contains(ctx.input[start])) { + matches = true; + break; + } } - } - // If negated, invert the match result - if (negated_) { - matches = !matches; - } + // If negated, invert the match result + if (negated_) { + matches = !matches; + } - if (matches) { - return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_SUCCESS, start, start + 1)); - } + if (matches) { + return parser_result(PARSER_RESULT_SUCCESS, start, start + 1); + } - return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_FAIL, start)); + return parser_result(PARSER_RESULT_FAIL, start); + }); } std::string dump() const override { @@ -581,11 +542,13 @@ class group_parser : public parser_base { parser_type type() const override { return PARSER_GROUP; } parser_result parse(parser_context & ctx, size_t start = 0) override { - auto result = parser_->parse(ctx, start); + return ctx.memo.cached(id_, start, [&]() { + auto result = parser_->parse(ctx, start); - // Store result - result.groups[name_] = parser_match_location{result.start, result.end}; - return ctx.memo.set(id_, start, result); + // Store result + result.groups[name_] = parser_match_location{result.start, result.end}; + return result; + }); } std::string dump() const override { @@ -619,13 +582,9 @@ class until_parser : public parser_base { parser_type type() const override { return PARSER_UNTIL; } parser_result parse(parser_context & ctx, size_t start = 0) override { - auto cached = ctx.memo.get(id_, start); - if (cached != std::nullopt) { - return *cached; - } - - auto result = parser_->parse(ctx, start); - return ctx.memo.set(id_, start, result); + return ctx.memo.cached(id_, start, [&]() { + return parser_->parse(ctx, start); + }); } std::string dump() const override { @@ -651,7 +610,9 @@ class schema_parser : public parser_base { parser_type type() const override { return PARSER_SCHEMA; } parser_result parse(parser_context & ctx, size_t start = 0) override { - return parser_->parse(ctx, start); + return ctx.memo.cached(id_, start, [&]() { + return parser_->parse(ctx, start); + }); } std::string dump() const override { @@ -678,25 +639,21 @@ class rule_parser : public parser_base { parser_type type() const override { return PARSER_RULE; } parser_result parse(parser_context & ctx, size_t start = 0) override { - auto cached = ctx.memo.get(id_, start); - if (cached != std::nullopt) { - return *cached; - } - - auto rules = rules_.lock(); - if (!rules) { - LOG_ERR("rule_parser::parse called with expired rule registry\n"); - return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_FAIL, start)); - } + return ctx.memo.cached(id_, start, [&]() { + auto rules = rules_.lock(); + if (!rules) { + LOG_ERR("rule_parser::parse called with expired rule registry\n"); + return parser_result(PARSER_RESULT_FAIL, start); + } - auto it = rules->find(name_); - if (it == rules->end()) { - LOG_ERR("rule_parser::parse rule '%s' not found in registry\n", name_.c_str()); - return ctx.memo.set(id_, start, parser_result(PARSER_RESULT_FAIL, start)); - } + auto it = rules->find(name_); + if (it == rules->end()) { + LOG_ERR("rule_parser::parse rule '%s' not found in registry\n", name_.c_str()); + return parser_result(PARSER_RESULT_FAIL, start); + } - auto result = it->second->parse(ctx, start); - return ctx.memo.set(id_, start, result); + return it->second->parse(ctx, start); + }); } std::string dump() const override { @@ -1105,6 +1062,14 @@ void parse_cache::clear() { results.clear(); } +parser_result parse_cache::cached(int id, size_t start, const std::function & fn) { + auto result = get(id, start); + if (result) { + return *result; + } + return set(id, start, fn()); +} + parser::parser() {} parser::parser(std::shared_ptr parser) : ptr_(std::move(parser)) {} diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index 56a522394bf75..4dbbca3f6d1c5 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -70,6 +70,8 @@ class parse_cache { parser_result set(int id, size_t start, parser_result result); std::optional get(int id, size_t start); void clear(); + + parser_result cached(int id, size_t start, const std::function & fn); }; struct parser_context { From adac6bae7f8a53493a192f72cdb7302ef1cf7f62 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Mon, 10 Nov 2025 21:32:29 -0600 Subject: [PATCH 012/183] add short description for each parser --- common/chat-parser-combinator.cpp | 30 +++++++++++++++++++++ common/chat-parser-combinator.h | 44 +++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index fb3b0fb083a0a..2b46a097d7a23 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -44,6 +44,8 @@ class parser_base { virtual void accept(parser_visitor & visitor) = 0; }; +// Matches an exact literal string. +// S -> "hello" class literal_parser : public parser_base { std::string literal_; @@ -84,6 +86,8 @@ class literal_parser : public parser_base { const std::string & literal() const { return literal_; } }; +// Matches a sequence of parsers in order, all must succeed. +// S -> A B C class sequence_parser : public parser_base { std::vector parsers_; @@ -148,6 +152,8 @@ class sequence_parser : public parser_base { const std::vector & parsers() const { return parsers_; } }; +// Matches the first parser that succeeds from a list of alternatives. +// S -> A | B | C class choice_parser : public parser_base { std::vector parsers_; @@ -201,6 +207,8 @@ class choice_parser : public parser_base { const std::vector & parsers() const { return parsers_; } }; +// Matches one or more repetitions of a parser. +// S -> A+ class one_or_more_parser : public parser_base { parser parser_; @@ -256,6 +264,8 @@ class one_or_more_parser : public parser_base { const parser & child() const { return parser_; } }; +// Matches zero or more repetitions of a parser, always succeeds. +// S -> A* class zero_or_more_parser : public parser_base { parser parser_; @@ -302,6 +312,8 @@ class zero_or_more_parser : public parser_base { const parser & child() const { return parser_; } }; +// Matches zero or one occurrence of a parser, always succeeds. +// S -> A? class optional_parser : public parser_base { parser parser_; @@ -338,6 +350,8 @@ class optional_parser : public parser_base { const parser & child() const { return parser_; } }; +// Negative lookahead: succeeds if child parser fails, consumes no input. +// S -> !A class not_parser : public parser_base { parser parser_; @@ -374,6 +388,8 @@ class not_parser : public parser_base { const parser & child() const { return parser_; } }; +// Matches any single character. +// S -> . class any_parser : public parser_base { public: any_parser(int id) : parser_base(id) {} @@ -400,6 +416,8 @@ class any_parser : public parser_base { void accept(parser_visitor & visitor) override; }; +// Matches zero or more whitespace characters (space, tab, newline). +// S -> [ \t\n]* class space_parser : public parser_base { public: space_parser(int id) : parser_base(id) {} @@ -429,6 +447,8 @@ class space_parser : public parser_base { void accept(parser_visitor & visitor) override; }; +// Matches a single character from a character class or range. +// S -> [a-z] or S -> [^0-9] class char_class_parser : public parser_base { struct char_range { int start; @@ -532,6 +552,8 @@ class char_class_parser : public parser_base { const std::string & pattern() const { return pattern_; } }; +// Captures the matched text from a parser and stores it with a name. +// S -> class group_parser : public parser_base { std::string name_; parser parser_; @@ -560,6 +582,8 @@ class group_parser : public parser_base { const parser & child() const { return parser_; } }; +// Matches all characters until a delimiter is found (delimiter not consumed). +// S -> (!delim .)* class until_parser : public parser_base { std::string delimiter_; parser parser_; @@ -598,6 +622,8 @@ class until_parser : public parser_base { const parser & child() const { return parser_; } }; +// Wraps a parser with JSON schema metadata for grammar generation. +// Used internally to convert JSON schemas to GBNF grammar rules. class schema_parser : public parser_base { parser parser_; std::string name_; @@ -628,6 +654,8 @@ class schema_parser : public parser_base { const nlohmann::ordered_json & schema() const { return schema_; } }; +// References a named rule for recursive or reusable grammar definitions. +// expr -> term | expr "+" term class rule_parser : public parser_base { std::string name_; std::weak_ptr> rules_; @@ -665,6 +693,8 @@ class rule_parser : public parser_base { const std::string & name() const { return name_; } }; +// Container for the root parser and all named rules in the grammar. +// Manages ownership of rule registry to enable recursive grammar definitions. class root_parser : public parser_base { parser root_; std::shared_ptr> rules_; diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index 4dbbca3f6d1c5..25ce7f7c11cb0 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -128,20 +128,64 @@ class parser_builder { parser_builder(); parser_builder(std::shared_ptr counter); + // Matches an exact literal string. + // S -> "hello" parser literal(const std::string & literal); + + // Matches a sequence of parsers in order, all must succeed. + // S -> A B C parser sequence(std::initializer_list parsers); + + // Matches the first parser that succeeds from a list of alternatives. + // S -> A | B | C parser choice(std::initializer_list parsers); + + // Matches one or more repetitions of a parser. + // S -> A+ parser one_or_more(const parser & p); + + // Matches zero or more repetitions of a parser, always succeeds. + // S -> A* parser zero_or_more(const parser & p); + + // Matches zero or one occurrence of a parser, always succeeds. + // S -> A? parser optional(const parser & p); + + // Negative lookahead: succeeds if child parser fails, consumes no input. + // S -> !A parser negate(const parser & p); + + // Matches any single character. + // S -> . parser any(); + + // Matches a single character from a character class or range. + // S -> [a-z] or S -> [^0-9] parser char_class(const std::string & classes); + + // Captures the matched text from a parser and stores it with a name. + // S -> parser group(const std::string & name, const parser & p); + + // References a named rule for recursive or reusable grammar definitions. + // expr -> term | expr "+" term parser rule(const std::string & name); + + // Matches zero or more whitespace characters (space, tab, newline). + // S -> [ \t\n]* parser space(); + + // Matches all characters until a delimiter is found (delimiter not consumed). + // S -> (!delim .)* parser until(const std::string & delimiter, bool consume_spaces = true); + + // Creates a complete JSON parser supporting objects, arrays, strings, numbers, booleans, and null. + // value -> object | array | string | number | true | false | null parser json(); + + // Wraps a parser with JSON schema metadata for grammar generation. + // Used internally to convert JSON schemas to GBNF grammar rules. parser schema(const parser & p, const std::string & name, const nlohmann::ordered_json & schema); parser add_rule(const std::string & name, const parser & p); From 0be2a93eb7ea86d3836d986007e8ba0e451dd834 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Mon, 10 Nov 2025 21:49:24 -0600 Subject: [PATCH 013/183] create a type for the root parser --- common/chat-parser-combinator.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index 2b46a097d7a23..d6aee652d2ea0 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -23,6 +23,7 @@ enum parser_type { PARSER_UNTIL = 11, PARSER_SPACE = 12, PARSER_SCHEMA = 13, + PARSER_ROOT = 14, }; class parser_visitor; @@ -705,7 +706,7 @@ class root_parser : public parser_base { root_parser(const parser & root, std::shared_ptr> rules, int id) : parser_base(id), root_(root), rules_(std::move(rules)) {} - parser_type type() const override { return root_->type(); } + parser_type type() const override { return PARSER_ROOT; } parser_result parse(parser_context & ctx, size_t start = 0) override { return root_->parse(ctx, start); From 31b386f6ef431220840e869da5b54c34804f058b Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Mon, 10 Nov 2025 22:16:30 -0600 Subject: [PATCH 014/183] implement repetition parser --- common/chat-parser-combinator.cpp | 194 ++++++++++++++++++------------ common/chat-parser-combinator.h | 9 ++ 2 files changed, 129 insertions(+), 74 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index d6aee652d2ea0..4121bfcde1c2e 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -24,6 +24,7 @@ enum parser_type { PARSER_SPACE = 12, PARSER_SCHEMA = 13, PARSER_ROOT = 14, + PARSER_REPETITION = 15, }; class parser_visitor; @@ -208,48 +209,52 @@ class choice_parser : public parser_base { const std::vector & parsers() const { return parsers_; } }; -// Matches one or more repetitions of a parser. -// S -> A+ -class one_or_more_parser : public parser_base { +// Matches between min and max repetitions of a parser (inclusive). +// S -> A{m,n} +// Use -1 for max_count to represent unbounded repetition (equivalent to {m,}) +class repetition_parser : public parser_base { parser parser_; + int min_count_; + int max_count_; public: - one_or_more_parser(const parser & parser, int id) : parser_base(id), parser_(parser) {} + repetition_parser(const parser & parser, int min_count, int max_count, int id) + : parser_base(id), parser_(parser), min_count_(min_count), max_count_(max_count) {} - parser_type type() const override { return PARSER_ONE_OR_MORE; } + parser_type type() const override { return PARSER_REPETITION; } parser_result parse(parser_context & ctx, size_t start = 0) override { return ctx.memo.cached(id_, start, [&]() { std::unordered_map groups; + auto pos = start; + int match_count = 0; - // Parse at least once - auto first_result = parser_->parse(ctx, start); - if (!first_result.is_success()) { - return first_result; - } - - auto pos = first_result.end; - groups.insert(first_result.groups.begin(), first_result.groups.end()); - - // Parse zero or more additional times - for (;;) { + // Try to match up to max_count times (or unlimited if max_count is -1) + while (max_count_ == -1 || match_count < max_count_) { auto result = parser_->parse(ctx, pos); groups.insert(result.groups.begin(), result.groups.end()); - if (result.is_need_more_input()) { - return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, pos, groups); + if (result.is_success()) { + // Prevent infinite loop on empty matches + if (result.end == pos) { + break; + } + pos = result.end; + match_count++; + continue; } - if (result.is_fail()) { - // Done with repetitions - break; + if (result.is_need_more_input()) { + return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, pos, groups); } - if (result.end == pos) { - break; // Prevent an infinite loop - } + // Child failed - stop trying + break; + } - pos = result.end; + // Check if we got enough matches + if (match_count < min_count_) { + return parser_result(PARSER_RESULT_FAIL, start, pos, groups); } return parser_result(PARSER_RESULT_SUCCESS, start, pos, groups); @@ -257,98 +262,106 @@ class one_or_more_parser : public parser_base { } std::string dump() const override { - return "OneOrMore(" + parser_->dump() + ")"; + if (max_count_ == -1) { + return "Repetition(" + parser_->dump() + ", " + std::to_string(min_count_) + ", unbounded)"; + } + return "Repetition(" + parser_->dump() + ", " + std::to_string(min_count_) + ", " + std::to_string(max_count_) + ")"; } void accept(parser_visitor & visitor) override; const parser & child() const { return parser_; } + + int min_count() const { return min_count_; } + + int max_count() const { return max_count_; } }; -// Matches zero or more repetitions of a parser, always succeeds. -// S -> A* -class zero_or_more_parser : public parser_base { - parser parser_; +// Matches one or more repetitions of a parser. +// S -> A+ +class one_or_more_parser : public parser_base { + parser delegate_; public: - zero_or_more_parser(const parser & parser, int id) : parser_base(id), parser_(parser) {} + one_or_more_parser(const parser & p, int id) : parser_base(id) { + delegate_ = parser(std::make_shared(p, 1, -1, id)); + } - parser_type type() const override { return PARSER_ZERO_OR_MORE; } + parser_type type() const override { return PARSER_ONE_OR_MORE; } parser_result parse(parser_context & ctx, size_t start = 0) override { - return ctx.memo.cached(id_, start, [&]() { - std::unordered_map groups; - auto pos = start; + return delegate_->parse(ctx, start); + } - for (;;) { - auto result = parser_->parse(ctx, pos); - groups.insert(result.groups.begin(), result.groups.end()); + std::string dump() const override { + auto rep = std::static_pointer_cast(delegate_.ptr()); + return "OneOrMore(" + rep->child()->dump() + ")"; + } - if (result.is_need_more_input()) { - return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, pos, groups); - } + void accept(parser_visitor & visitor) override; - if (result.is_fail()) { - // Done with repetitions (zero or more is always valid) - break; - } + const parser & child() const { + auto rep = std::static_pointer_cast(delegate_.ptr()); + return rep->child(); + } +}; - if (result.end == pos) { - break; // Prevent an infinite loop - } +// Matches zero or more repetitions of a parser, always succeeds. +// S -> A* +class zero_or_more_parser : public parser_base { + parser delegate_; - pos = result.end; - } + public: + zero_or_more_parser(const parser & p, int id) : parser_base(id) { + delegate_ = parser(std::make_shared(p, 0, -1, id)); + } - return parser_result(PARSER_RESULT_SUCCESS, start, pos, groups); - }); + parser_type type() const override { return PARSER_ZERO_OR_MORE; } + + parser_result parse(parser_context & ctx, size_t start = 0) override { + return delegate_->parse(ctx, start); } std::string dump() const override { - return "ZeroOrMore(" + parser_->dump() + ")"; + auto rep = std::static_pointer_cast(delegate_.ptr()); + return "ZeroOrMore(" + rep->child()->dump() + ")"; } void accept(parser_visitor & visitor) override; - const parser & child() const { return parser_; } + const parser & child() const { + auto rep = std::static_pointer_cast(delegate_.ptr()); + return rep->child(); + } }; // Matches zero or one occurrence of a parser, always succeeds. // S -> A? class optional_parser : public parser_base { - parser parser_; + parser delegate_; public: - optional_parser(const parser & parser, int id) : parser_base(id), parser_(parser) {} + optional_parser(const parser & p, int id) : parser_base(id) { + delegate_ = parser(std::make_shared(p, 0, 1, id)); + } parser_type type() const override { return PARSER_OPTIONAL; } parser_result parse(parser_context & ctx, size_t start = 0) override { - return ctx.memo.cached(id_, start, [&]() { - auto result = parser_->parse(ctx, start); - - if (result.is_success()) { - // Matched successfully - return result; - } - - if (result.is_need_more_input()) { - // Propagate - need more input to determine if optional matches - return result; - } - - // No match, but optional always succeeds with zero matches - return parser_result(PARSER_RESULT_SUCCESS, start, start); - }); + return delegate_->parse(ctx, start); } std::string dump() const override { - return "Optional(" + parser_->dump() + ")"; + auto rep = std::static_pointer_cast(delegate_.ptr()); + return "Optional(" + rep->child()->dump() + ")"; } void accept(parser_visitor & visitor) override; - const parser & child() const { return parser_; } + const parser & child() const { + auto rep = std::static_pointer_cast(delegate_.ptr()); + return rep->child(); + } }; // Negative lookahead: succeeds if child parser fails, consumes no input. @@ -734,6 +747,7 @@ class parser_visitor { virtual void visit(one_or_more_parser & p) = 0; virtual void visit(zero_or_more_parser & p) = 0; virtual void visit(optional_parser & p) = 0; + virtual void visit(repetition_parser & p) = 0; virtual void visit(until_parser & p) = 0; virtual void visit(not_parser & p) = 0; virtual void visit(any_parser & p) = 0; @@ -891,6 +905,24 @@ class gbnf_visitor : public parser_visitor { } } + void visit(repetition_parser & p) override { + p.child()->accept(*this); + std::string child_result = current_result_; + + if (needs_parens(p.child()->type())) { + child_result = "(" + child_result + ")"; + } + + if (p.max_count() == -1) { + // Unbounded: {n,} + current_result_ = child_result + "{" + std::to_string(p.min_count()) + ",}"; + } else { + // Bounded: {n,m} + current_result_ = child_result + "{" + std::to_string(p.min_count()) + "," + + std::to_string(p.max_count()) + "}"; + } + } + void visit(until_parser & p) override { // Generate pattern that matches prefixes but prevents full delimiter match current_result_ = generate_until_pattern(p.delimiter()) + "*"; @@ -1021,6 +1053,11 @@ class id_assignment_visitor : public parser_visitor { p.child()->accept(*this); } + void visit(repetition_parser & p) override { + assign_id(p); + p.child()->accept(*this); + } + void visit(until_parser & p) override { assign_id(p); p.child()->accept(*this); @@ -1049,6 +1086,7 @@ void choice_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void one_or_more_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void zero_or_more_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void optional_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } +void repetition_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void until_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void not_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void any_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } @@ -1207,6 +1245,14 @@ parser parser_builder::until(const std::string & delimiter, bool consume_spaces) return parser(std::make_shared(delimiter, consume_spaces, counter_->next())); } +parser parser_builder::repeat(const parser & p, int min, int max) { + return parser(std::make_shared(p, min, max, counter_->next())); +} + +parser parser_builder::repeat(const parser & p, int n) { + return repeat(p, n, n); +} + parser parser_builder::schema(const parser & p, const std::string & name, const nlohmann::ordered_json & schema) { return parser(std::make_shared(p, name, schema, counter_->next())); } diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index 25ce7f7c11cb0..b295b4b520498 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -180,6 +180,15 @@ class parser_builder { // S -> (!delim .)* parser until(const std::string & delimiter, bool consume_spaces = true); + // Matches between min and max repetitions of a parser (inclusive). + // S -> A{m,n} + // Use -1 for max to represent unbounded repetition (equivalent to {m,}) + parser repeat(const parser & p, int min, int max); + + // Matches exactly n repetitions of a parser. + // S -> A{n} + parser repeat(const parser & p, int n); + // Creates a complete JSON parser supporting objects, arrays, strings, numbers, booleans, and null. // value -> object | array | string | number | true | false | null parser json(); From ffb7a6f77db113c16ecf3cd2de4db9fc2f5344fb Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Mon, 10 Nov 2025 22:27:54 -0600 Subject: [PATCH 015/183] Make optional, one_or_more, and zero_or_more subclasses of repetition --- common/chat-parser-combinator.cpp | 86 ++++++++----------------------- 1 file changed, 22 insertions(+), 64 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index 4121bfcde1c2e..23bce3b3a50d6 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -12,19 +12,19 @@ enum parser_type { PARSER_LITERAL = 0, PARSER_SEQUENCE = 1, PARSER_CHOICE = 2, - PARSER_ZERO_OR_MORE = 3, - PARSER_ONE_OR_MORE = 4, - PARSER_NOT = 5, - PARSER_ANY = 6, - PARSER_CHAR_CLASS = 7, - PARSER_GROUP = 8, - PARSER_RULE = 9, - PARSER_OPTIONAL = 10, - PARSER_UNTIL = 11, - PARSER_SPACE = 12, - PARSER_SCHEMA = 13, - PARSER_ROOT = 14, - PARSER_REPETITION = 15, + PARSER_REPETITION = 3, + PARSER_OPTIONAL = 4, + PARSER_ZERO_OR_MORE = 5, + PARSER_ONE_OR_MORE = 6, + PARSER_NOT = 7, + PARSER_ANY = 8, + PARSER_CHAR_CLASS = 9, + PARSER_GROUP = 10, + PARSER_RULE = 11, + PARSER_UNTIL = 12, + PARSER_SPACE = 13, + PARSER_SCHEMA = 14, + PARSER_ROOT = 15, }; class parser_visitor; @@ -279,89 +279,47 @@ class repetition_parser : public parser_base { // Matches one or more repetitions of a parser. // S -> A+ -class one_or_more_parser : public parser_base { - parser delegate_; - +class one_or_more_parser : public repetition_parser { public: - one_or_more_parser(const parser & p, int id) : parser_base(id) { - delegate_ = parser(std::make_shared(p, 1, -1, id)); - } + one_or_more_parser(const parser & p, int id) : repetition_parser(p, 1, -1, id) {} parser_type type() const override { return PARSER_ONE_OR_MORE; } - parser_result parse(parser_context & ctx, size_t start = 0) override { - return delegate_->parse(ctx, start); - } - std::string dump() const override { - auto rep = std::static_pointer_cast(delegate_.ptr()); - return "OneOrMore(" + rep->child()->dump() + ")"; + return "OneOrMore(" + child()->dump() + ")"; } void accept(parser_visitor & visitor) override; - - const parser & child() const { - auto rep = std::static_pointer_cast(delegate_.ptr()); - return rep->child(); - } }; // Matches zero or more repetitions of a parser, always succeeds. // S -> A* -class zero_or_more_parser : public parser_base { - parser delegate_; - +class zero_or_more_parser : public repetition_parser { public: - zero_or_more_parser(const parser & p, int id) : parser_base(id) { - delegate_ = parser(std::make_shared(p, 0, -1, id)); - } + zero_or_more_parser(const parser & p, int id) : repetition_parser(p, 0, -1, id) {} parser_type type() const override { return PARSER_ZERO_OR_MORE; } - parser_result parse(parser_context & ctx, size_t start = 0) override { - return delegate_->parse(ctx, start); - } - std::string dump() const override { - auto rep = std::static_pointer_cast(delegate_.ptr()); - return "ZeroOrMore(" + rep->child()->dump() + ")"; + return "ZeroOrMore(" + child()->dump() + ")"; } void accept(parser_visitor & visitor) override; - - const parser & child() const { - auto rep = std::static_pointer_cast(delegate_.ptr()); - return rep->child(); - } }; // Matches zero or one occurrence of a parser, always succeeds. // S -> A? -class optional_parser : public parser_base { - parser delegate_; - +class optional_parser : public repetition_parser { public: - optional_parser(const parser & p, int id) : parser_base(id) { - delegate_ = parser(std::make_shared(p, 0, 1, id)); - } + optional_parser(const parser & p, int id) : repetition_parser(p, 0, 1, id) {} parser_type type() const override { return PARSER_OPTIONAL; } - parser_result parse(parser_context & ctx, size_t start = 0) override { - return delegate_->parse(ctx, start); - } - std::string dump() const override { - auto rep = std::static_pointer_cast(delegate_.ptr()); - return "Optional(" + rep->child()->dump() + ")"; + return "Optional(" + child()->dump() + ")"; } void accept(parser_visitor & visitor) override; - - const parser & child() const { - auto rep = std::static_pointer_cast(delegate_.ptr()); - return rep->child(); - } }; // Negative lookahead: succeeds if child parser fails, consumes no input. From 085404a326000a195efb2ca550cec2dccf684273 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Mon, 10 Nov 2025 22:44:55 -0600 Subject: [PATCH 016/183] improve context constructor --- common/chat-parser-combinator.h | 26 ++++++- tests/test-chat-parser-combinator.cpp | 108 +++++++++++++------------- 2 files changed, 76 insertions(+), 58 deletions(-) diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index b295b4b520498..ab839971b725c 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -52,9 +52,15 @@ struct parser_result { std::unordered_map groups; parser_result() : type(PARSER_RESULT_FAIL) {} - parser_result(parser_result_type type, size_t start) : type(type), start(start), end(start) {} - parser_result(parser_result_type type, size_t start, size_t end) : type(type), start(start), end(end) {} - parser_result(parser_result_type type, size_t start, size_t end, const std::unordered_map & groups) : type(type), start(start), end(end), groups(groups) {} + + parser_result(parser_result_type type, size_t start) + : type(type), start(start), end(start) {} + + parser_result(parser_result_type type, size_t start, size_t end) + : type(type), start(start), end(end) {} + + parser_result(parser_result_type type, size_t start, size_t end, const std::unordered_map & groups) + : type(type), start(start), end(end), groups(groups) {} bool is_fail() const { return type == PARSER_RESULT_FAIL; } bool is_need_more_input() const { return type == PARSER_RESULT_NEED_MORE_INPUT; } @@ -77,7 +83,19 @@ class parse_cache { struct parser_context { std::string_view input; parse_cache memo; - bool input_is_complete = true; + bool input_is_complete; + + parser_context() + : memo(), input_is_complete(true) {} + + parser_context(std::string_view input) + : input(input), memo(), input_is_complete(true) {} + + parser_context(std::string_view input, bool complete) + : input(input), memo(), input_is_complete(complete) {} + + parser_context(std::string_view input, parse_cache memo, bool complete = true) + : input(input), memo(std::move(memo)), input_is_complete(complete) {} }; class parser_base; diff --git a/tests/test-chat-parser-combinator.cpp b/tests/test-chat-parser-combinator.cpp index e4f637af9f797..83ff2ba4a67fd 100644 --- a/tests/test-chat-parser-combinator.cpp +++ b/tests/test-chat-parser-combinator.cpp @@ -36,7 +36,7 @@ static void test_partial_parsing() { parser_context ctx; parser_result result; - ctx = parser_context{"hello", parse_cache()}; + ctx = parser_context("hello"); result = parser.parse(ctx); assert_equals(true, result.is_success()); } @@ -49,11 +49,11 @@ static void test_partial_parsing() { parser_context ctx; parser_result result; - ctx = parser_context{"a", parse_cache()}; + ctx = parser_context("a"); result = parser.parse(ctx); assert_equals(true, result.is_success()); - ctx = parser_context{"A", parse_cache()}; + ctx = parser_context("A"); result = parser.parse(ctx); assert_equals(true, result.is_fail()); @@ -61,15 +61,15 @@ static void test_partial_parsing() { return p.char_class("a-z-"); }); - ctx = parser_context{"f", parse_cache()}; + ctx = parser_context("f"); result = parser.parse(ctx); assert_equals(true, result.is_success()); - ctx = parser_context{"-", parse_cache()}; + ctx = parser_context("-"); result = parser.parse(ctx); assert_equals(true, result.is_success()); - ctx = parser_context{"A", parse_cache()}; + ctx = parser_context("A"); result = parser.parse(ctx); assert_equals(true, result.is_fail()); } @@ -80,25 +80,25 @@ static void test_partial_parsing() { }); // Partial matches - auto ctx = parser_context{"", parse_cache(), false}; + ctx = parser_context("", false); result = parser.parse(ctx); assert_equals(true, result.is_success()); - ctx = parser_context{"", parse_cache(), true}; + ctx = parser_context("", true); result = parser.parse(ctx); assert_equals(true, result.is_success()); // No match, since it does not adhere to the grammar - ctx = parser_context{"I am parser", parse_cache(), false}; + ctx = parser_context("I am parser", false); result = parser.parse(ctx); assert_equals(true, result.is_fail()); } @@ -109,25 +109,25 @@ static void test_partial_parsing() { }); // Partial matches - auto ctx = parser_context{"", parse_cache(), true}; + ctx = parser_context("", true); result = parser.parse(ctx); assert_equals(true, result.is_success()); - ctx = parser_context{"", parse_cache(), true}; + ctx = parser_context("", true); result = parser.parse(ctx); assert_equals(true, result.is_success()); // No match - ctx = parser_context{"", parse_cache(), true}; + ctx = parser_context("", true); result = parser.parse(ctx); assert_equals(true, result.is_fail()); } @@ -138,16 +138,16 @@ static void test_partial_parsing() { }); // Partial matches - auto ctx = parser_context{"a", parse_cache(), false}; + auto ctx = parser_context("a", false); auto result = parser.parse(ctx); assert_equals(true, result.is_need_more_input()); - ctx = parser_context{"aba", parse_cache(), false}; + ctx = parser_context("aba", false); result = parser.parse(ctx); assert_equals(true, result.is_need_more_input()); // Full match - ctx = parser_context{"ab", parse_cache(), true}; + ctx = parser_context("ab", true); result = parser.parse(ctx); assert_equals(true, result.is_success()); } @@ -158,21 +158,21 @@ static void test_partial_parsing() { }); // Partial matches - auto ctx = parser_context{"a", parse_cache(), false}; + auto ctx = parser_context("a", false); auto result = parser.parse(ctx); assert_equals(true, result.is_need_more_input()); - ctx = parser_context{"aba", parse_cache(), false}; + ctx = parser_context("aba", false); result = parser.parse(ctx); assert_equals(true, result.is_need_more_input()); // Full match - ctx = parser_context{"ab", parse_cache(), true}; + ctx = parser_context("ab", true); result = parser.parse(ctx); assert_equals(true, result.is_success()); // No match - ctx = parser_context{"cd", parse_cache(), true}; + ctx = parser_context("cd", true); result = parser.parse(ctx); assert_equals(true, result.is_fail()); } @@ -189,7 +189,7 @@ static void test_capture_groups() { }); std::string input = "I have a thought"; - auto ctx = parser_context{input, parse_cache()}; + auto ctx = parser_context(input); auto result = parser.parse(ctx); assert_equals(true, result.is_success()); @@ -208,7 +208,7 @@ static void test_capture_groups() { }); std::string input = "I have a "; - auto ctx = parser_context{input, parse_cache(), false}; + auto ctx = parser_context(input, false); auto result = parser.parse(ctx); assert_equals(true, result.is_success()); @@ -228,7 +228,7 @@ static void test_capture_groups() { }); std::string input = "The user said hello.Hello!"; - auto ctx = parser_context{input, parse_cache(), true}; + auto ctx = parser_context(input, true); auto result = parser.parse(ctx); assert_equals(true, result.is_success()); @@ -253,19 +253,19 @@ static void test_char_class() { parser_context ctx; parser_result result; - ctx = parser_context{"\n", parse_cache()}; + ctx = parser_context("\n"); result = parser.parse(ctx); assert_equals(true, result.is_success()); - ctx = parser_context{"\t", parse_cache()}; + ctx = parser_context("\t"); result = parser.parse(ctx); assert_equals(true, result.is_success()); - ctx = parser_context{"\\", parse_cache()}; + ctx = parser_context("\\"); result = parser.parse(ctx); assert_equals(true, result.is_success()); - ctx = parser_context{" ", parse_cache()}; + ctx = parser_context(" "); result = parser.parse(ctx); assert_equals(true, result.is_fail()); } @@ -278,20 +278,20 @@ static void test_char_class() { parser_context ctx; parser_result result; - ctx = parser_context{"a", parse_cache()}; + ctx = parser_context("a"); result = parser.parse(ctx); assert_equals(true, result.is_success()); - ctx = parser_context{"-", parse_cache()}; + ctx = parser_context("-"); result = parser.parse(ctx); assert_equals(true, result.is_success()); - ctx = parser_context{"z", parse_cache()}; + ctx = parser_context("z"); result = parser.parse(ctx); assert_equals(true, result.is_success()); // Should NOT match 'b' since \- is a literal dash, not a range - ctx = parser_context{"b", parse_cache()}; + ctx = parser_context("b"); result = parser.parse(ctx); assert_equals(true, result.is_fail()); } @@ -312,32 +312,32 @@ static void test_recursive_references() { parser_result result; // Test simple number - ctx = parser_context{"1", parse_cache(), true}; + ctx = parser_context("1", true); result = value_parser.parse(ctx); assert_equals(true, result.is_success()); // Test simple list - ctx = parser_context{"[1]", parse_cache(), true}; + ctx = parser_context("[1]", true); result = value_parser.parse(ctx); assert_equals(true, result.is_success()); // Test nested list - ctx = parser_context{"[[2]]", parse_cache(), true}; + ctx = parser_context("[[2]]", true); result = value_parser.parse(ctx); assert_equals(true, result.is_success()); // Test deeply nested list - ctx = parser_context{"[[[3]]]", parse_cache(), true}; + ctx = parser_context("[[[3]]]", true); result = value_parser.parse(ctx); assert_equals(true, result.is_success()); // Test partial match - ctx = parser_context{"[[", parse_cache(), false}; + ctx = parser_context("[[", false); result = value_parser.parse(ctx); assert_equals(true, result.is_success()); // Test no match - ctx = parser_context{"[a]", parse_cache(), true}; + ctx = parser_context("[a]", true); result = value_parser.parse(ctx); assert_equals(true, result.is_fail()); } @@ -349,19 +349,19 @@ static void test_optional() { }); // Full match with optional part present - auto ctx = parser_context{"hello world", parse_cache()}; + auto ctx = parser_context("hello world"); auto result = parser.parse(ctx); assert_equals(true, result.is_success()); assert_equals((size_t)11, result.end); // Full match with optional part absent - ctx = parser_context{"hello", parse_cache(), true}; + ctx = parser_context("hello", true); result = parser.parse(ctx); assert_equals(true, result.is_success()); assert_equals((size_t)5, result.end); // Partial match - waiting for more input to determine if optional matches - ctx = parser_context{"hello ", parse_cache(), false}; + ctx = parser_context("hello ", false); result = parser.parse(ctx); assert_equals(true, result.is_need_more_input()); } @@ -374,7 +374,7 @@ static void test_json_parser() { { // Test parsing a simple JSON object std::string input = R"({"name": "test", "value": 42, "flag": true})"; - parser_context ctx{input, parse_cache()}; + parser_context ctx(input); auto result = json.parse(ctx); @@ -384,7 +384,7 @@ static void test_json_parser() { { // Test parsing a JSON array with mixed types std::string input = R"([1, "hello", true, null, 3.14])"; - parser_context ctx{input, parse_cache()}; + parser_context ctx(input); auto result = json.parse(ctx); @@ -394,7 +394,7 @@ static void test_json_parser() { { // Test parsing nested JSON with objects and arrays std::string input = R"({"users": [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}], "count": 2, "metadata": {"version": "1.0", "tags": ["admin", "user"]}})"; - parser_context ctx{input, parse_cache()}; + parser_context ctx(input); auto result = json.parse(ctx); @@ -404,7 +404,7 @@ static void test_json_parser() { { // Test partial parsing - incomplete object std::string input = R"({"name": "test", "value": )"; - parser_context ctx{input, parse_cache(), false}; + parser_context ctx(input, false); auto result = json.parse(ctx); @@ -413,7 +413,7 @@ static void test_json_parser() { { // Test partial parsing - incomplete array std::string input = R"([1, 2, 3, )"; - parser_context ctx{input, parse_cache(), false}; + parser_context ctx(input, false); auto result = json.parse(ctx); @@ -422,7 +422,7 @@ static void test_json_parser() { { // Test partial parsing - incomplete nested structure std::string input = R"({"data": {"nested": )"; - parser_context ctx{input, parse_cache(), false}; + parser_context ctx(input, false); auto result = json.parse(ctx); @@ -476,7 +476,7 @@ static void test_complete_example() { // Test complete input std::string input = R"(I need to call get_weather with city = New Yorkget_weather{"city": "New York"})"; - parser_context ctx{input, parse_cache()}; + parser_context ctx(input); auto result = parser.parse(ctx); @@ -488,21 +488,21 @@ static void test_complete_example() { // Test partial input input = R"(I need to call get_weather )"; - ctx = parser_context{input, parse_cache(), /* .is_input_complete = */ false}; + ctx = parser_context(input, /* .is_input_complete = */ false); result = parser.parse(ctx); assert_equals(true, result.is_success()); assert_equals(std::string("I need to call get_weather"), *result.group("reasoning-content", ctx.input)); input = R"(I need to call get_weatherget_weather)"; - ctx = parser_context{input, parse_cache(), /* .is_input_complete = */ false}; + ctx = parser_context(input, /* .is_input_complete = */ false); result = parser.parse(ctx); assert_equals(true, result.is_success()); assert_equals(std::string("I need to call get_weather"), *result.group("reasoning-content", ctx.input)); input = R"(I need to call get_weatherget_weatherI need to call get_weatherget_weather{"cit)"; - ctx = parser_context{input, parse_cache(), /* .is_input_complete = */ false}; + ctx = parser_context(input, /* .is_input_complete = */ false); result = parser.parse(ctx); assert_equals(true, result.is_success()); From 6bd9a9502679080641e31db2a1486d3ebd081d02 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Tue, 11 Nov 2025 22:22:53 -0600 Subject: [PATCH 017/183] improve until parsing and add benchmarks --- common/chat-parser-combinator.cpp | 71 ++++++-- common/chat-parser-combinator.h | 5 + tests/test-chat-parser-combinator.cpp | 237 +++++++++++++++++++++++++- 3 files changed, 293 insertions(+), 20 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index 23bce3b3a50d6..0f5adfb662e27 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -46,6 +46,12 @@ class parser_base { virtual void accept(parser_visitor & visitor) = 0; }; +// We define our own space function because MSVC's std::isspace() +// crashes for non-printable characters in Debug builds. +static bool is_space(const char c) { + return (c == ' ' || c == '\t' || c == '\n'); +} + // Matches an exact literal string. // S -> "hello" class literal_parser : public parser_base { @@ -401,7 +407,7 @@ class space_parser : public parser_base { auto pos = start; while (pos < ctx.input.size()) { char c = ctx.input[pos]; - if (c == ' ' || c == '\t' || c == '\n') { + if (is_space(c)) { ++pos; } else { break; @@ -558,28 +564,46 @@ class group_parser : public parser_base { // S -> (!delim .)* class until_parser : public parser_base { std::string delimiter_; - parser parser_; + bool consume_spaces_; + + std::boyer_moore_searcher searcher_; public: until_parser(const std::string & delimiter, bool consume_spaces, int id) - : parser_base(id), delimiter_(delimiter) { - - auto delim = parser(std::make_shared(delimiter, -1)); - auto any = parser(std::make_shared(-1)); - - if (consume_spaces) { - auto ws = parser(std::make_shared(-1)); - parser_ = parser(std::make_shared(~(ws + delim) + any, -1)); - } else { - parser_ = parser(std::make_shared(~delim + any, -1)); - } + : parser_base(id), delimiter_(delimiter), consume_spaces_(consume_spaces), searcher_(delimiter_.begin(), delimiter_.end()) { } parser_type type() const override { return PARSER_UNTIL; } parser_result parse(parser_context & ctx, size_t start = 0) override { return ctx.memo.cached(id_, start, [&]() { - return parser_->parse(ctx, start); + parser_result result(PARSER_RESULT_SUCCESS, start, ctx.input.size()); + + // Search for the delimiter + const auto * it = std::search(ctx.input.begin(), ctx.input.end(), searcher_); + + if (it != ctx.input.end()) { + result.type = PARSER_RESULT_SUCCESS; + result.end = std::distance(ctx.input.begin(), it); + } else { + // If not found, check if the input ends with a prefix of the delimiter + size_t max_overlap = std::min(ctx.input.size(), delimiter_.size() - 1); + for (size_t overlap = max_overlap; overlap > 0; --overlap) { + if (std::equal(ctx.input.end() - overlap, ctx.input.end(), delimiter_.begin())) { + result.type = PARSER_RESULT_NEED_MORE_INPUT; + result.end = ctx.input.size() - overlap; + } + } + } + + if (consume_spaces_) { + // Remove trailing spaces + while (result.end > start && is_space(ctx.input[result.end - 1])) { + result.end--; + } + } + + return result; }); } @@ -590,8 +614,6 @@ class until_parser : public parser_base { void accept(parser_visitor & visitor) override; const std::string & delimiter() const { return delimiter_; } - - const parser & child() const { return parser_; } }; // Wraps a parser with JSON schema metadata for grammar generation. @@ -1018,7 +1040,6 @@ class id_assignment_visitor : public parser_visitor { void visit(until_parser & p) override { assign_id(p); - p.child()->accept(*this); } void visit(not_parser & p) override { @@ -1215,6 +1236,22 @@ parser parser_builder::schema(const parser & p, const std::string & name, const return parser(std::make_shared(p, name, schema, counter_->next())); } +parser parser_builder::json_key(const std::string & name, const parser & p) { + return literal("\"" + name + "\"") << literal(":") << p; +} + +parser parser_builder::json_string(const parser & p) { + auto quote = literal("\""); + return quote + p + quote; +} + +parser parser_builder::between(const std::string & left, const parser & p, const std::string & right, bool allow_spaces) { + if (allow_spaces) { + return literal(left) << p << literal(right); + } + return literal(left) + p + literal(right); +} + parser parser_builder::add_rule(const std::string & name, const parser & p) { (*rules_)[name] = p; return rule(name); diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index ab839971b725c..034aa773d6a37 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -211,6 +211,11 @@ class parser_builder { // value -> object | array | string | number | true | false | null parser json(); + parser json_key(const std::string & name, const parser & p); + parser json_string(const parser & p); + + parser between(const std::string & left, const parser & p, const std::string & right, bool allow_spaces = true); + // Wraps a parser with JSON schema metadata for grammar generation. // Used internally to convert JSON schemas to GBNF grammar rules. parser schema(const parser & p, const std::string & name, const nlohmann::ordered_json & schema); diff --git a/tests/test-chat-parser-combinator.cpp b/tests/test-chat-parser-combinator.cpp index 83ff2ba4a67fd..a0e007c79c879 100644 --- a/tests/test-chat-parser-combinator.cpp +++ b/tests/test-chat-parser-combinator.cpp @@ -1,10 +1,13 @@ #include #include +#include +#include "nlohmann/json.hpp" + +#include "chat-parser.h" #include "chat-parser-combinator.h" +#include "common.h" #include "json-schema-to-grammar.h" -#include "nlohmann/json.hpp" -#include "nlohmann/json_fwd.hpp" template static void assert_equals(const std::string_view label, const T & expected, const T & actual) { @@ -455,7 +458,7 @@ static void test_complete_example() { auto tool_call_name = p.add_rule("tool-call-name", p.literal("") - << p.group("tool-name", p.one_or_more(p.char_class("[a-zA-Z\\-_]"))) + << p.group("tool-name", p.until("")) << p.literal("")); auto schema = nlohmann::ordered_json::parse(R"({"type": "object"})"); @@ -494,6 +497,20 @@ static void test_complete_example() { assert_equals(true, result.is_success()); assert_equals(std::string("I need to call get_weather"), *result.group("reasoning-content", ctx.input)); + input = R"(I need to call I need to call get_weatherI need to call get_weatherget_weather)"; ctx = parser_context(input, /* .is_input_complete = */ false); result = parser.parse(ctx); @@ -691,6 +708,184 @@ static void test_gbnf_generation() { } } +static parser create_command_r7b_parser() { + return build_parser([](parser_builder & p) { + auto thinking = p.literal("<|START_THINKING|>") + << p.until("<|END_THINKING|>") + << p.literal("<|END_THINKING|>"); + + auto response = p.literal("<|START_RESPONSE|>") + << p.until("<|END_RESPONSE|>") + << p.literal("<|END_RESPONSE|>"); + + auto json = p.json(); + auto tool_call_id = p.json_key("tool_call_id", p.json_string(p.until("\""))); + auto tool_call_name = p.json_key("tool_name", p.json_string(p.until("\""))); + auto tool_call_args = p.json_key("parameters", json); + auto tool_call_fields = tool_call_id | tool_call_name | tool_call_args; + + auto tool_call = p.between("{", + tool_call_fields << p.zero_or_more(p.literal(",") << tool_call_fields), + "}"); + + auto tool_calls = p.literal("<|START_ACTION|>") + << p.literal("[") + << tool_call + << p.zero_or_more(p.literal(",") << tool_call) + << p.literal("]") + << p.literal("<|END_ACTION|>"); + + return p.optional(thinking) << (tool_calls | response); + }); +} + +static void test_command_r7b_parser(const parser & p, const std::string & input, bool partial) { + parser_context ctx(input, !partial); + p.parse(ctx); +} + +static void test_command_r7b_legacy_parser(const std::string & input, bool partial) { + // Original parser taken from chat.cpp + common_chat_msg_parser builder(input, + /* is_partial= */ partial, { + /* .format = */ COMMON_CHAT_FORMAT_GENERIC, + /* .reasoning_format = */ COMMON_REASONING_FORMAT_AUTO, + /* .reasoning_in_content = */ false, + /* .thinking_forced_open = */ false, + }); + + builder.try_parse_reasoning("<|START_THINKING|>", "<|END_THINKING|>"); + + static const common_regex start_action_regex("<\\|START_ACTION\\|>"); + static const common_regex end_action_regex("<\\|END_ACTION\\|>"); + static const common_regex start_response_regex("<\\|START_RESPONSE\\|>"); + static const common_regex end_response_regex("<\\|END_RESPONSE\\|>"); + + if (auto res = builder.try_find_regex(start_action_regex)) { + // If we didn't extract thoughts, prelude includes them. + auto tool_calls = builder.consume_json_with_dumped_args({{"parameters"}}); + for (const auto & tool_call : tool_calls.value) { + std::string name = tool_call.contains("tool_name") ? tool_call.at("tool_name") : ""; + std::string id = tool_call.contains("tool_call_id") ? tool_call.at("tool_call_id") : ""; + std::string arguments = tool_call.contains("parameters") ? tool_call.at("parameters") : ""; + if (!builder.add_tool_call(name, id, arguments) || tool_calls.is_partial) { + throw common_chat_msg_partial_exception("incomplete tool call"); + } + } + if (tool_calls.is_partial) { + throw common_chat_msg_partial_exception("incomplete tool call"); + } + builder.consume_regex(end_action_regex); + } else if (auto res = builder.try_find_regex(start_response_regex)) { + if (!builder.try_find_regex(end_response_regex)) { + builder.add_content(builder.consume_rest()); + throw common_chat_msg_partial_exception(end_response_regex.str()); + } + } else { + builder.add_content(builder.consume_rest()); + } +} + +struct bench_tool_call { + std::string id; + std::string name; + nlohmann::ordered_json args; +}; + +// Simple tokenize function that splits by space +static std::vector simple_tokenize(const std::string & input) { + std::vector result; + std::string current; + + for (size_t i = 0; i < input.size(); i++) { + if (input[i] == ' ') { + if (!current.empty()) { + result.push_back(current); + current.clear(); + } + current += ' '; + } else { + current += input[i]; + } + } + + if (!current.empty()) { + result.push_back(current); + } + + return result; +} + +static void benchmark_compare( + const std::string & reasoning, + const std::string & content, + const std::vector & tool_calls, + int iterations) { + + // Build response + std::vector tokens; // Since we don't have a command r7b tokenizer, we're going to "simulate" them. + + if (!reasoning.empty()) { + auto tokenized = simple_tokenize(reasoning); + tokens.emplace_back("<|START_THINKING|>"); + tokens.insert(tokens.end(), tokenized.begin(), tokenized.end()); + tokens.emplace_back("<|END_THINKING|>"); + } + + if (!content.empty()) { + auto tokenized = simple_tokenize(content); + tokens.emplace_back("<|START_RESPONSE|>"); + tokens.insert(tokens.end(), tokenized.begin(), tokenized.end()); + tokens.emplace_back("<|END_RESPONSE|>"); + } + + if (!tool_calls.empty()) { + tokens.emplace_back("<|START_ACTION|>"); + + auto json = nlohmann::json::array(); + for (const auto & tc : tool_calls) { + auto tc_json = nlohmann::json::object(); + tc_json["tool_call_id"] = tc.id; + tc_json["tool_name"] = tc.name; + tc_json["parameters"] = tc.args; + json.push_back(tc_json); + } + + auto tokenized = simple_tokenize(json.dump(-1, ' ', true)); + tokens.insert(tokens.end(), tokenized.begin(), tokenized.end()); + + tokens.emplace_back("<|END_ACTION|>"); + } + + auto run = [&](const std::function & fn) { + std::string input = std::accumulate(tokens.begin(), tokens.end(), std::string()); + + std::chrono::microseconds duration(0); + for (int i = 0; i < iterations; i++) { + auto start = std::chrono::high_resolution_clock::now(); + fn(input, false); + auto end = std::chrono::high_resolution_clock::now(); + duration += std::chrono::duration_cast(end - start); + } + return duration.count() / iterations; + }; + + auto parser = create_command_r7b_parser(); + + auto duration_new = run([&](const std::string & input, bool partial) { + test_command_r7b_parser(parser, input, partial); + }); + + auto duration_legacy = run([&](const std::string & input, bool partial) { + try { + test_command_r7b_legacy_parser(input, partial); + } catch (const common_chat_msg_partial_exception &) { } + }); + + std::cout << " New parser avg: " << duration_new << " us\n"; + std::cout << "Legacy parser avg: " << duration_legacy << " us\n"; +} + int main() { test_partial_parsing(); test_char_class(); @@ -701,5 +896,41 @@ int main() { test_complete_example(); test_gbnf_generation(); std::cout << "All tests passed!\n"; + + std::cout << "\n== Benchmarks ==\n"; + std::string example_reasoning = + "To plan an effective trip to Japan that includes both historical sites and modern attractions within a budget of $4000 for a two-week stay, we need to:\n\n" + "1. Identify key historical sites and modern attractions in Japan.\n" + "2. Find affordable accommodation options that provide a balance between comfort and cost.\n" + "3. Determine the best modes of transportation for getting around Japan.\n" + "4. Create a day-by-day itinerary that ensures the user gets to see a variety of attractions without overspending.\n" + "5. Provide a detailed cost breakdown that includes accommodation, transportation, meals, and entry fees to attractions."; + + std::string example_content = + "For a two-week trip to Japan with a $4,000 budget, I recommend planning an itinerary that balances historical sites with modern attractions. The destination will be Japan, with a duration of 14 days.\n\n" + "Given your interests in both historical sites and modern attractions, you'll want to focus on cities like Kyoto for its temples and traditional culture, Tokyo for its cutting-edge technology and entertainment districts, and possibly Hiroshima or Nara for additional historical significance.\n\n" + "For accommodation, I suggest looking for affordable options such as budget hotels, hostels, or guesthouses that offer good value without sacrificing too much comfort. Japan has excellent mid-range accommodation options that can keep your lodging costs manageable.\n\n" + "Transportation should prioritize efficiencyβ€”consider getting a JR Rail Pass for intercity travel, which allows unlimited rides on most JR trains including the Shinkansen (bullet train). Within cities, use local trains and subways, which are both affordable and highly reliable.\n\n" + "For meals, embrace local cuisine by eating at neighborhood restaurants, ramen shops, and izakayas rather than touristy establishments. This will give you an authentic experience while keeping costs reasonableβ€”you can enjoy excellent meals for $10-20 per person at local spots.\n\n"; + + std::vector example_tool_calls = {{ + "call_0", + "plan_trip", + nlohmann::json::parse(R"({ + "destination": "Japan", + "duration": 14, + "budget": 4000, + "interests": ["historical sites", "modern attractions"], + "accommodation_preferences": "affordable", + "transportation_preferences": "efficient", + "meal_preferences": "local cuisine" + })") + }}; + + std::cout << "\nReasoning + Content:\n"; + benchmark_compare(example_reasoning, example_content, std::vector(), 100); + + std::cout << "\nReasoning + Tool Call:\n"; + benchmark_compare(example_reasoning, "", example_tool_calls, 100); return 0; } From 62656db20a67340f81de99eaf5120e97229fd66e Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Tue, 11 Nov 2025 22:36:07 -0600 Subject: [PATCH 018/183] remove cached() pattern, cache in parser_base with specialized parsing functions for each parser --- common/chat-parser-combinator.cpp | 375 +++++++++++++++--------------- common/chat-parser-combinator.h | 2 - 2 files changed, 182 insertions(+), 195 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index 0f5adfb662e27..871a12cde4be9 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -41,7 +41,28 @@ class parser_base { void set_id(int id) { id_ = id; } virtual parser_type type() const = 0; - virtual parser_result parse(parser_context & ctx, size_t start = 0) = 0; + + // Template Method: handles caching, delegates to parse_uncached() + virtual parser_result parse(parser_context & ctx, size_t start = 0) { + if (id_ == -1) { + // Don't cache parsers with ID -1 (from operators) + return parse_uncached(ctx, start); + } + + // Check cache + auto cached = ctx.memo.get(id_, start); + if (cached) { + return *cached; + } + + // Execute and cache + auto result = parse_uncached(ctx, start); + return ctx.memo.set(id_, start, result); + } + + // Actual parsing implementation (to be overridden by subclasses) + virtual parser_result parse_uncached(parser_context & ctx, size_t start = 0) = 0; + virtual std::string dump() const = 0; virtual void accept(parser_visitor & visitor) = 0; }; @@ -62,27 +83,25 @@ class literal_parser : public parser_base { parser_type type() const override { return PARSER_LITERAL; } - parser_result parse(parser_context & ctx, size_t start = 0) override { - return ctx.memo.cached(id_, start, [&]() { - auto pos = start; - for (auto i = 0u; i < literal_.size(); ++i) { - if (pos >= ctx.input.size()) { - if (ctx.input_is_complete) { - return parser_result(PARSER_RESULT_FAIL, start); - } - if (i > 0) { - return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, pos); - } + parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { + auto pos = start; + for (auto i = 0u; i < literal_.size(); ++i) { + if (pos >= ctx.input.size()) { + if (ctx.input_is_complete) { return parser_result(PARSER_RESULT_FAIL, start); } - if (ctx.input[pos] != literal_[i]) { - return parser_result(PARSER_RESULT_FAIL, start); + if (i > 0) { + return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, pos); } - ++pos; + return parser_result(PARSER_RESULT_FAIL, start); + } + if (ctx.input[pos] != literal_[i]) { + return parser_result(PARSER_RESULT_FAIL, start); } + ++pos; + } - return parser_result(PARSER_RESULT_SUCCESS, start, pos); - }); + return parser_result(PARSER_RESULT_SUCCESS, start, pos); } std::string dump() const override { @@ -116,34 +135,32 @@ class sequence_parser : public parser_base { parser_type type() const override { return PARSER_SEQUENCE; } - parser_result parse(parser_context & ctx, size_t start = 0) override { - return ctx.memo.cached(id_, start, [&]() { - std::unordered_map groups; + parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { + std::unordered_map groups; - auto pos = start; - for (const auto & p : parsers_) { - auto result = p->parse(ctx, pos); + auto pos = start; + for (const auto & p : parsers_) { + auto result = p->parse(ctx, pos); - // Copy groups - groups.insert(result.groups.begin(), result.groups.end()); + // Copy groups + groups.insert(result.groups.begin(), result.groups.end()); - if (result.is_fail()) { - if (result.end >= ctx.input.size() && !ctx.input_is_complete) { - // If we fail because we don't have enough input, then return success - return parser_result(PARSER_RESULT_SUCCESS, start, result.end, groups); - } - return parser_result(PARSER_RESULT_FAIL, start, result.end, groups); - } - - if (result.is_need_more_input()) { - return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, result.end, groups); + if (result.is_fail()) { + if (result.end >= ctx.input.size() && !ctx.input_is_complete) { + // If we fail because we don't have enough input, then return success + return parser_result(PARSER_RESULT_SUCCESS, start, result.end, groups); } + return parser_result(PARSER_RESULT_FAIL, start, result.end, groups); + } - pos = result.end; + if (result.is_need_more_input()) { + return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, result.end, groups); } - return parser_result(PARSER_RESULT_SUCCESS, start, pos, groups); - }); + pos = result.end; + } + + return parser_result(PARSER_RESULT_SUCCESS, start, pos, groups); } std::string dump() const override { @@ -182,23 +199,21 @@ class choice_parser : public parser_base { parser_type type() const override { return PARSER_CHOICE; } - parser_result parse(parser_context & ctx, size_t start = 0) override { - return ctx.memo.cached(id_, start, [&]() { - auto pos = start; - for (const auto & p : parsers_) { - auto result = p->parse(ctx, pos); + parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { + auto pos = start; + for (const auto & p : parsers_) { + auto result = p->parse(ctx, pos); - if (result.is_success()) { - return result; - } + if (result.is_success()) { + return result; + } - if (result.is_need_more_input()) { - return result; - } + if (result.is_need_more_input()) { + return result; } + } - return parser_result(PARSER_RESULT_FAIL, start); - }); + return parser_result(PARSER_RESULT_FAIL, start); } std::string dump() const override { @@ -229,42 +244,40 @@ class repetition_parser : public parser_base { parser_type type() const override { return PARSER_REPETITION; } - parser_result parse(parser_context & ctx, size_t start = 0) override { - return ctx.memo.cached(id_, start, [&]() { - std::unordered_map groups; - auto pos = start; - int match_count = 0; - - // Try to match up to max_count times (or unlimited if max_count is -1) - while (max_count_ == -1 || match_count < max_count_) { - auto result = parser_->parse(ctx, pos); - groups.insert(result.groups.begin(), result.groups.end()); - - if (result.is_success()) { - // Prevent infinite loop on empty matches - if (result.end == pos) { - break; - } - pos = result.end; - match_count++; - continue; - } + parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { + std::unordered_map groups; + auto pos = start; + int match_count = 0; - if (result.is_need_more_input()) { - return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, pos, groups); - } + // Try to match up to max_count times (or unlimited if max_count is -1) + while (max_count_ == -1 || match_count < max_count_) { + auto result = parser_->parse(ctx, pos); + groups.insert(result.groups.begin(), result.groups.end()); - // Child failed - stop trying - break; + if (result.is_success()) { + // Prevent infinite loop on empty matches + if (result.end == pos) { + break; + } + pos = result.end; + match_count++; + continue; } - // Check if we got enough matches - if (match_count < min_count_) { - return parser_result(PARSER_RESULT_FAIL, start, pos, groups); + if (result.is_need_more_input()) { + return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, pos, groups); } - return parser_result(PARSER_RESULT_SUCCESS, start, pos, groups); - }); + // Child failed - stop trying + break; + } + + // Check if we got enough matches + if (match_count < min_count_) { + return parser_result(PARSER_RESULT_FAIL, start, pos, groups); + } + + return parser_result(PARSER_RESULT_SUCCESS, start, pos, groups); } std::string dump() const override { @@ -338,23 +351,21 @@ class not_parser : public parser_base { parser_type type() const override { return PARSER_NOT; } - parser_result parse(parser_context & ctx, size_t start = 0) override { - return ctx.memo.cached(id_, start, [&]() { - auto result = parser_->parse(ctx, start); + parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { + auto result = parser_->parse(ctx, start); - if (result.is_success()) { - // Fail if the underlying parser matches - return parser_result(PARSER_RESULT_FAIL, start); - } + if (result.is_success()) { + // Fail if the underlying parser matches + return parser_result(PARSER_RESULT_FAIL, start); + } - if (result.is_need_more_input()) { - // Propagate - need to know what child would match before negating - return result; - } + if (result.is_need_more_input()) { + // Propagate - need to know what child would match before negating + return result; + } - // Child failed, so negation succeeds - return parser_result(PARSER_RESULT_SUCCESS, start); - }); + // Child failed, so negation succeeds + return parser_result(PARSER_RESULT_SUCCESS, start); } std::string dump() const override { @@ -374,17 +385,15 @@ class any_parser : public parser_base { parser_type type() const override { return PARSER_ANY; } - parser_result parse(parser_context & ctx, size_t start = 0) override { - return ctx.memo.cached(id_, start, [&]() { - if (start >= ctx.input.size()) { - if (ctx.input_is_complete) { - return parser_result(PARSER_RESULT_FAIL, start); - } + parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { + if (start >= ctx.input.size()) { + if (ctx.input_is_complete) { return parser_result(PARSER_RESULT_FAIL, start); } + return parser_result(PARSER_RESULT_FAIL, start); + } - return parser_result(PARSER_RESULT_SUCCESS, start, start + 1); - }); + return parser_result(PARSER_RESULT_SUCCESS, start, start + 1); } std::string dump() const override { @@ -402,20 +411,18 @@ class space_parser : public parser_base { parser_type type() const override { return PARSER_SPACE; } - parser_result parse(parser_context & ctx, size_t start = 0) override { - return ctx.memo.cached(id_, start, [&]() { - auto pos = start; - while (pos < ctx.input.size()) { - char c = ctx.input[pos]; - if (is_space(c)) { - ++pos; - } else { - break; - } + parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { + auto pos = start; + while (pos < ctx.input.size()) { + char c = ctx.input[pos]; + if (is_space(c)) { + ++pos; + } else { + break; } + } - return parser_result(PARSER_RESULT_SUCCESS, start, pos); - }); + return parser_result(PARSER_RESULT_SUCCESS, start, pos); } std::string dump() const override { @@ -491,34 +498,32 @@ class char_class_parser : public parser_base { parser_type type() const override { return PARSER_CHAR_CLASS; } - parser_result parse(parser_context & ctx, size_t start = 0) override { - return ctx.memo.cached(id_, start, [&]() { - if (start >= ctx.input.size()) { - if (ctx.input_is_complete) { - return parser_result(PARSER_RESULT_FAIL, start); - } + parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { + if (start >= ctx.input.size()) { + if (ctx.input_is_complete) { return parser_result(PARSER_RESULT_FAIL, start); } + return parser_result(PARSER_RESULT_FAIL, start); + } - bool matches = false; - for (const auto & range : ranges_) { - if (range.contains(ctx.input[start])) { - matches = true; - break; - } + bool matches = false; + for (const auto & range : ranges_) { + if (range.contains(ctx.input[start])) { + matches = true; + break; } + } - // If negated, invert the match result - if (negated_) { - matches = !matches; - } + // If negated, invert the match result + if (negated_) { + matches = !matches; + } - if (matches) { - return parser_result(PARSER_RESULT_SUCCESS, start, start + 1); - } + if (matches) { + return parser_result(PARSER_RESULT_SUCCESS, start, start + 1); + } - return parser_result(PARSER_RESULT_FAIL, start); - }); + return parser_result(PARSER_RESULT_FAIL, start); } std::string dump() const override { @@ -541,14 +546,12 @@ class group_parser : public parser_base { parser_type type() const override { return PARSER_GROUP; } - parser_result parse(parser_context & ctx, size_t start = 0) override { - return ctx.memo.cached(id_, start, [&]() { - auto result = parser_->parse(ctx, start); + parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { + auto result = parser_->parse(ctx, start); - // Store result - result.groups[name_] = parser_match_location{result.start, result.end}; - return result; - }); + // Store result + result.groups[name_] = parser_match_location{result.start, result.end}; + return result; } std::string dump() const override { @@ -575,36 +578,34 @@ class until_parser : public parser_base { parser_type type() const override { return PARSER_UNTIL; } - parser_result parse(parser_context & ctx, size_t start = 0) override { - return ctx.memo.cached(id_, start, [&]() { - parser_result result(PARSER_RESULT_SUCCESS, start, ctx.input.size()); + parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { + parser_result result(PARSER_RESULT_SUCCESS, start, ctx.input.size()); - // Search for the delimiter - const auto * it = std::search(ctx.input.begin(), ctx.input.end(), searcher_); + // Search for the delimiter + const auto * it = std::search(ctx.input.begin(), ctx.input.end(), searcher_); - if (it != ctx.input.end()) { - result.type = PARSER_RESULT_SUCCESS; - result.end = std::distance(ctx.input.begin(), it); - } else { - // If not found, check if the input ends with a prefix of the delimiter - size_t max_overlap = std::min(ctx.input.size(), delimiter_.size() - 1); - for (size_t overlap = max_overlap; overlap > 0; --overlap) { - if (std::equal(ctx.input.end() - overlap, ctx.input.end(), delimiter_.begin())) { - result.type = PARSER_RESULT_NEED_MORE_INPUT; - result.end = ctx.input.size() - overlap; - } + if (it != ctx.input.end()) { + result.type = PARSER_RESULT_SUCCESS; + result.end = std::distance(ctx.input.begin(), it); + } else { + // If not found, check if the input ends with a prefix of the delimiter + size_t max_overlap = std::min(ctx.input.size(), delimiter_.size() - 1); + for (size_t overlap = max_overlap; overlap > 0; --overlap) { + if (std::equal(ctx.input.end() - overlap, ctx.input.end(), delimiter_.begin())) { + result.type = (ctx.input_is_complete) ? PARSER_RESULT_FAIL : PARSER_RESULT_NEED_MORE_INPUT; + result.end = ctx.input.size() - overlap; } } + } - if (consume_spaces_) { - // Remove trailing spaces - while (result.end > start && is_space(ctx.input[result.end - 1])) { - result.end--; - } + if (consume_spaces_) { + // Remove trailing spaces + while (result.end > start && is_space(ctx.input[result.end - 1])) { + result.end--; } + } - return result; - }); + return result; } std::string dump() const override { @@ -629,10 +630,8 @@ class schema_parser : public parser_base { parser_type type() const override { return PARSER_SCHEMA; } - parser_result parse(parser_context & ctx, size_t start = 0) override { - return ctx.memo.cached(id_, start, [&]() { - return parser_->parse(ctx, start); - }); + parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { + return parser_->parse(ctx, start); } std::string dump() const override { @@ -660,22 +659,20 @@ class rule_parser : public parser_base { parser_type type() const override { return PARSER_RULE; } - parser_result parse(parser_context & ctx, size_t start = 0) override { - return ctx.memo.cached(id_, start, [&]() { - auto rules = rules_.lock(); - if (!rules) { - LOG_ERR("rule_parser::parse called with expired rule registry\n"); - return parser_result(PARSER_RESULT_FAIL, start); - } + parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { + auto rules = rules_.lock(); + if (!rules) { + LOG_ERR("rule_parser::parse called with expired rule registry\n"); + return parser_result(PARSER_RESULT_FAIL, start); + } - auto it = rules->find(name_); - if (it == rules->end()) { - LOG_ERR("rule_parser::parse rule '%s' not found in registry\n", name_.c_str()); - return parser_result(PARSER_RESULT_FAIL, start); - } + auto it = rules->find(name_); + if (it == rules->end()) { + LOG_ERR("rule_parser::parse rule '%s' not found in registry\n", name_.c_str()); + return parser_result(PARSER_RESULT_FAIL, start); + } - return it->second->parse(ctx, start); - }); + return it->second->parse(ctx, start); } std::string dump() const override { @@ -701,7 +698,7 @@ class root_parser : public parser_base { parser_type type() const override { return PARSER_ROOT; } - parser_result parse(parser_context & ctx, size_t start = 0) override { + parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { return root_->parse(ctx, start); } @@ -1110,14 +1107,6 @@ void parse_cache::clear() { results.clear(); } -parser_result parse_cache::cached(int id, size_t start, const std::function & fn) { - auto result = get(id, start); - if (result) { - return *result; - } - return set(id, start, fn()); -} - parser::parser() {} parser::parser(std::shared_ptr parser) : ptr_(std::move(parser)) {} diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index 034aa773d6a37..4a1242a718362 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -76,8 +76,6 @@ class parse_cache { parser_result set(int id, size_t start, parser_result result); std::optional get(int id, size_t start); void clear(); - - parser_result cached(int id, size_t start, const std::function & fn); }; struct parser_context { From 18557f3a5ebe37bbe1502480c84ba38e40501854 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Tue, 11 Nov 2025 23:54:43 -0600 Subject: [PATCH 019/183] improve json parsing performance to better match legacy parsing --- common/chat-parser-combinator.cpp | 257 +++++++++++++++++++++----- common/chat-parser-combinator.h | 14 +- tests/test-chat-parser-combinator.cpp | 30 +-- 3 files changed, 241 insertions(+), 60 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index 871a12cde4be9..de825a8e6a2ee 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -18,13 +18,14 @@ enum parser_type { PARSER_ONE_OR_MORE = 6, PARSER_NOT = 7, PARSER_ANY = 8, - PARSER_CHAR_CLASS = 9, + PARSER_CHARS = 9, PARSER_GROUP = 10, PARSER_RULE = 11, PARSER_UNTIL = 12, PARSER_SPACE = 13, PARSER_SCHEMA = 14, PARSER_ROOT = 15, + PARSER_JSON_STRING = 16, }; class parser_visitor; @@ -93,7 +94,7 @@ class literal_parser : public parser_base { if (i > 0) { return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, pos); } - return parser_result(PARSER_RESULT_FAIL, start); + return parser_result(PARSER_RESULT_FAIL, start, pos); } if (ctx.input[pos] != literal_[i]) { return parser_result(PARSER_RESULT_FAIL, start); @@ -432,9 +433,9 @@ class space_parser : public parser_base { void accept(parser_visitor & visitor) override; }; -// Matches a single character from a character class or range. -// S -> [a-z] or S -> [^0-9] -class char_class_parser : public parser_base { +// Matches between min and max repetitions of characters from a character class. +// S -> [a-z]{m,n} +class chars_parser : public parser_base { struct char_range { int start; int end; @@ -445,9 +446,13 @@ class char_class_parser : public parser_base { std::string pattern_; std::vector ranges_; bool negated_; + int min_count_; + int max_count_; public: - char_class_parser(const std::string & classes, int id) : parser_base(id), pattern_(classes), negated_(false) { + chars_parser(const std::string & classes, int min_count, int max_count, int id) + : parser_base(id), pattern_(classes), negated_(false), min_count_(min_count), max_count_(max_count) { + std::string content = classes; if (content.front() == '[') { content = content.substr(1); @@ -496,43 +501,164 @@ class char_class_parser : public parser_base { } } - parser_type type() const override { return PARSER_CHAR_CLASS; } + parser_type type() const override { return PARSER_CHARS; } parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { - if (start >= ctx.input.size()) { - if (ctx.input_is_complete) { - return parser_result(PARSER_RESULT_FAIL, start); + auto pos = start; + int match_count = 0; + + // Try to match up to max_count times (or unlimited if max_count is -1) + while (max_count_ == -1 || match_count < max_count_) { + if (pos >= ctx.input.size()) { + break; } - return parser_result(PARSER_RESULT_FAIL, start); - } - bool matches = false; - for (const auto & range : ranges_) { - if (range.contains(ctx.input[start])) { - matches = true; + bool matches = false; + for (const auto & range : ranges_) { + if (range.contains(ctx.input[pos])) { + matches = true; + break; + } + } + + // If negated, invert the match result + if (negated_) { + matches = !matches; + } + + if (matches) { + ++pos; + ++match_count; + } else { break; } } - // If negated, invert the match result - if (negated_) { - matches = !matches; + // Check if we got enough matches + if (match_count < min_count_) { + return parser_result(PARSER_RESULT_FAIL, start); } - if (matches) { - return parser_result(PARSER_RESULT_SUCCESS, start, start + 1); + return parser_result(PARSER_RESULT_SUCCESS, start, pos); + } + + std::string dump() const override { + if (max_count_ == -1) { + return "CharRepeat(" + pattern_ + ", " + std::to_string(min_count_) + ", unbounded)"; } + return "CharRepeat(" + pattern_ + ", " + std::to_string(min_count_) + ", " + std::to_string(max_count_) + ")"; + } - return parser_result(PARSER_RESULT_FAIL, start); + void accept(parser_visitor & visitor) override; + + const std::string & pattern() const { return pattern_; } + + int min_count() const { return min_count_; } + + int max_count() const { return max_count_; } +}; + +// Specialized parser for JSON string content (without quotes). +// Parses the content between quotes with single-pass streaming support. +// Stops before the closing quote (doesn't consume it). +// Handles escape sequences and emits NEED_MORE_INPUT for incomplete input. +// S -> (regular chars and escape sequences)* until closing " +class json_string_parser : public parser_base { + std::optional capture_name_; + + static bool is_hex_digit(char c) { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); + } + + public: + json_string_parser(std::optional capture_name, int id) + : parser_base(id), capture_name_(std::move(capture_name)) {} + + parser_type type() const override { return PARSER_JSON_STRING; } + + parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { + std::unordered_map groups; + auto pos = start; + + // Parse string content (without quotes) + while (pos < ctx.input.size()) { + char c = ctx.input[pos]; + + if (c == '"') { + // Found closing quote - success (don't consume it) + if (capture_name_) { + groups[*capture_name_] = parser_match_location{start, pos}; + } + return parser_result(PARSER_RESULT_SUCCESS, start, pos, groups); + } + + if (c == '\\') { + // Handle escape sequence + ++pos; + if (pos >= ctx.input.size()) { + // Mid-escape sequence + if (ctx.input_is_complete) { + return parser_result(PARSER_RESULT_FAIL, start); + } + return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, pos); + } + + char escape = ctx.input[pos]; + switch (escape) { + case '"': + case '\\': + case '/': + case 'b': + case 'f': + case 'n': + case 'r': + case 't': + // Valid escape + ++pos; + break; + + case 'u': + // Unicode escape: must be followed by 4 hex digits + ++pos; + for (int i = 0; i < 4; ++i) { + if (pos >= ctx.input.size()) { + // Incomplete unicode escape + if (ctx.input_is_complete) { + return parser_result(PARSER_RESULT_FAIL, start); + } + return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, pos); + } + if (!is_hex_digit(ctx.input[pos])) { + return parser_result(PARSER_RESULT_FAIL, start); + } + ++pos; + } + break; + + default: + // Invalid escape sequence + return parser_result(PARSER_RESULT_FAIL, start); + } + } else { + // Regular character + ++pos; + } + } + + // Reached end without finding closing quote + return parser_result(PARSER_RESULT_FAIL, start, pos); } std::string dump() const override { - return "Char(" + pattern_ + ")"; + if (capture_name_) { + return "JsonString(" + *capture_name_ + ")"; + } + return "JsonString()"; } void accept(parser_visitor & visitor) override; - const std::string & pattern() const { return pattern_; } + const std::optional & capture_name() const { return capture_name_; } }; // Captures the matched text from a parser and stores it with a name. @@ -729,7 +855,8 @@ class parser_visitor { virtual void visit(not_parser & p) = 0; virtual void visit(any_parser & p) = 0; virtual void visit(space_parser & p) = 0; - virtual void visit(char_class_parser & p) = 0; + virtual void visit(chars_parser & p) = 0; + virtual void visit(json_string_parser & p) = 0; virtual void visit(group_parser & p) = 0; virtual void visit(schema_parser & p) = 0; virtual void visit(rule_parser & p) = 0; @@ -921,9 +1048,36 @@ class gbnf_visitor : public parser_visitor { current_result_ = "space"; } - void visit(char_class_parser & p) override { - // Return pattern as-is (already in GBNF format) - current_result_ = p.pattern(); + void visit(chars_parser & p) override { + const std::string & pattern = p.pattern(); + + if (p.min_count() == 0 && p.max_count() == -1) { + // Zero or more: * + current_result_ = pattern + "*"; + } else if (p.min_count() == 1 && p.max_count() == -1) { + // One or more: + + current_result_ = pattern + "+"; + } else if (p.max_count() == -1) { + // Unbounded: {n,} + current_result_ = pattern + "{" + std::to_string(p.min_count()) + ",}"; + } else if (p.min_count() == p.max_count()) { + // Exact count: {n} or just pattern for n=1 + if (p.min_count() == 1) { + current_result_ = pattern; + } else { + current_result_ = pattern + "{" + std::to_string(p.min_count()) + "}"; + } + } else { + // Bounded: {n,m} + current_result_ = pattern + "{" + std::to_string(p.min_count()) + "," + + std::to_string(p.max_count()) + "}"; + } + } + + void visit(json_string_parser &) override { + // JSON string content (without quotes) + // Pattern: (any non-quote/backslash OR escape sequences)* until closing quote + current_result_ = R"(( [^"\\] | "\\" ( ["\\/ bfnrt] | "u" [0-9a-fA-F]{4} ) )*)"; } void visit(group_parser & p) override { @@ -988,7 +1142,11 @@ class id_assignment_visitor : public parser_visitor { assign_id(p); } - void visit(char_class_parser & p) override { + void visit(chars_parser & p) override { + assign_id(p); + } + + void visit(json_string_parser & p) override { assign_id(p); } @@ -1067,7 +1225,8 @@ void until_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void not_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void any_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void space_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } -void char_class_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } +void chars_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } +void json_string_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void group_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void schema_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void rule_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } @@ -1193,8 +1352,20 @@ parser parser_builder::any() { return parser(std::make_shared(counter_->next())); } -parser parser_builder::char_class(const std::string & classes) { - return parser(std::make_shared(classes, counter_->next())); +parser parser_builder::chars(const std::string & classes, int min, int max) { + return parser(std::make_shared(classes, min, max, counter_->next())); +} + +parser parser_builder::one(const std::string & classes) { + return chars(classes, 1, 1); +} + +parser parser_builder::json_string() { + return parser(std::make_shared(std::nullopt, counter_->next())); +} + +parser parser_builder::json_string(const std::string & name) { + return parser(std::make_shared(name, counter_->next())); } parser parser_builder::group(const std::string & name, const parser & p) { @@ -1270,36 +1441,28 @@ static parser json_parser(std::shared_ptr counter) { parser_builder builder(std::move(counter)); // Whitespace: space, tab, newline, carriage return - auto ws = builder.zero_or_more(builder.char_class("[ \\t\\n\\r]")); + auto ws = builder.chars("[ \\t\\n\\r]", 0, -1); // Number components - auto digit = builder.char_class("[0-9]"); - auto digit1_9 = builder.char_class("[1-9]"); - auto digits = builder.one_or_more(digit); + auto digit1_9 = builder.chars("[1-9]", 1, 1); + auto digits = builder.chars("[0-9]"); // Integer part: 0 or non-zero digit followed by more digits - auto int_part = builder.literal("0") | (digit1_9 + builder.zero_or_more(digit)); + auto int_part = builder.literal("0") | (digit1_9 + builder.chars("[0-9]", 0, -1)); // Optional fractional part auto frac = builder.literal(".") + digits; // Optional exponent part - auto exp = (builder.literal("e") | builder.literal("E")) + builder.optional(builder.char_class("[+\\-]")) + digits; + auto exp = (builder.literal("e") | builder.literal("E")) + builder.optional(builder.chars("[+\\-]", 1, 1)) + digits; // Complete number auto number = builder.optional(builder.literal("-")) + int_part + builder.optional(frac) + builder.optional(exp); builder.add_rule("json_number", number); - // String components - auto hex = builder.char_class("[0-9a-fA-F]"); - auto unicode_escape = builder.literal("\\u") + hex + hex + hex + hex; - auto simple_escape = builder.literal("\\") + builder.char_class("[\"\\\\bfnrt/]"); - auto escape = simple_escape | unicode_escape; - - // String character: escape sequence or any char except quote and backslash - auto string_char = escape | builder.char_class("[^\"\\\\]"); - auto string = builder.literal("\"") + builder.zero_or_more(string_char) + builder.literal("\""); + // String: specialized single-pass parser (content only, wrapped with quotes) + auto string = builder.literal("\"") + builder.json_string() + builder.literal("\""); builder.add_rule("json_string", string); diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index 4a1242a718362..6275de1fcde62 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -176,9 +176,17 @@ class parser_builder { // S -> . parser any(); + // Matches between min and max repetitions of characters from a character class. + // S -> [a-z]{m,n} + // + // Use -1 for max to represent unbounded repetition (equivalent to {m,}) + parser chars(const std::string & classes, int min = 1, int max = -1); + // Matches a single character from a character class or range. // S -> [a-z] or S -> [^0-9] - parser char_class(const std::string & classes); + // + // Equivalent to chars(classes, 1, 1) + parser one(const std::string & classes); // Captures the matched text from a parser and stores it with a name. // S -> @@ -209,6 +217,10 @@ class parser_builder { // value -> object | array | string | number | true | false | null parser json(); + // Specialized single-pass JSON string parser with escape sequence handling + parser json_string(); + parser json_string(const std::string & name); + parser json_key(const std::string & name, const parser & p); parser json_string(const parser & p); diff --git a/tests/test-chat-parser-combinator.cpp b/tests/test-chat-parser-combinator.cpp index a0e007c79c879..6264d72f76fbb 100644 --- a/tests/test-chat-parser-combinator.cpp +++ b/tests/test-chat-parser-combinator.cpp @@ -46,7 +46,7 @@ static void test_partial_parsing() { { // Test char class auto parser = build_parser([](parser_builder& p) { - return p.char_class("a-z"); + return p.one("a-z"); }); parser_context ctx; @@ -61,7 +61,7 @@ static void test_partial_parsing() { assert_equals(true, result.is_fail()); parser = build_parser([](parser_builder& p) { - return p.char_class("a-z-"); + return p.one("a-z-"); }); ctx = parser_context("f"); @@ -246,11 +246,11 @@ static void test_capture_groups() { } } -static void test_char_class() { +static void test_one() { { // Test common escape sequences auto parser = build_parser([](parser_builder& p) { - return p.char_class("[\\n\\t\\\\]"); + return p.one("[\\n\\t\\\\]"); }); parser_context ctx; @@ -275,7 +275,7 @@ static void test_char_class() { { // Test escaped dash (literal dash, not a range) auto parser = build_parser([](parser_builder& p) { - return p.char_class("[a\\-z]"); + return p.one("[a\\-z]"); }); parser_context ctx; @@ -302,7 +302,7 @@ static void test_char_class() { static void test_recursive_references() { auto value_parser = build_parser([](parser_builder& p) { - p.add_rule("number", p.one_or_more(p.char_class("0-9"))); + p.add_rule("number", p.one_or_more(p.one("0-9"))); p.add_rule("list", p.sequence({ p.literal("["), p.rule("value"), @@ -559,7 +559,7 @@ static void test_gbnf_generation() { { // Test char class auto parser = build_parser([](parser_builder& p) { - return p.char_class("[a-z]"); + return p.one("[a-z]"); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { @@ -595,7 +595,7 @@ static void test_gbnf_generation() { { // Test one_or_more auto parser = build_parser([](parser_builder& p) { - return p.one_or_more(p.char_class("[0-9]")); + return p.one_or_more(p.one("[0-9]")); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { @@ -607,7 +607,7 @@ static void test_gbnf_generation() { { // Test zero_or_more auto parser = build_parser([](parser_builder& p) { - return p.zero_or_more(p.char_class("[a-z]")); + return p.zero_or_more(p.one("[a-z]")); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { @@ -668,7 +668,7 @@ static void test_gbnf_generation() { { // Test rule references auto parser = build_parser([](parser_builder& p) { - auto digit = p.add_rule("digit", p.char_class("[0-9]")); + auto digit = p.add_rule("digit", p.one("[0-9]")); return p.one_or_more(digit); }); @@ -709,7 +709,7 @@ static void test_gbnf_generation() { } static parser create_command_r7b_parser() { - return build_parser([](parser_builder & p) { + auto parser = build_parser([](parser_builder & p) { auto thinking = p.literal("<|START_THINKING|>") << p.until("<|END_THINKING|>") << p.literal("<|END_THINKING|>"); @@ -737,6 +737,12 @@ static parser create_command_r7b_parser() { return p.optional(thinking) << (tool_calls | response); }); + + auto grammar = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); + + return parser; } static void test_command_r7b_parser(const parser & p, const std::string & input, bool partial) { @@ -888,7 +894,7 @@ static void benchmark_compare( int main() { test_partial_parsing(); - test_char_class(); + test_one(); test_capture_groups(); test_recursive_references(); test_optional(); From f6aa60857a72120aecf40775d68cb34c94124e03 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Wed, 12 Nov 2025 00:20:21 -0600 Subject: [PATCH 020/183] fix const auto * it for windows --- common/chat-parser-combinator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index de825a8e6a2ee..2481443be9c44 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -708,7 +708,7 @@ class until_parser : public parser_base { parser_result result(PARSER_RESULT_SUCCESS, start, ctx.input.size()); // Search for the delimiter - const auto * it = std::search(ctx.input.begin(), ctx.input.end(), searcher_); + const auto it = std::search(ctx.input.begin(), ctx.input.end(), searcher_); if (it != ctx.input.end()) { result.type = PARSER_RESULT_SUCCESS; From d58dacea18127adfc37272c018d8f50a0c5d9b25 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Wed, 12 Nov 2025 00:35:33 -0600 Subject: [PATCH 021/183] move id assignment to classes instead of using a visitor --- common/chat-parser-combinator.cpp | 139 +++++++++--------------------- 1 file changed, 41 insertions(+), 98 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index 2481443be9c44..064afd80f1686 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -64,6 +64,12 @@ class parser_base { // Actual parsing implementation (to be overridden by subclasses) virtual parser_result parse_uncached(parser_context & ctx, size_t start = 0) = 0; + virtual void assign_id(std::shared_ptr counter) { + if (id_ == -1) { + id_ = counter->next(); + } + } + virtual std::string dump() const = 0; virtual void accept(parser_visitor & visitor) = 0; }; @@ -164,6 +170,13 @@ class sequence_parser : public parser_base { return parser_result(PARSER_RESULT_SUCCESS, start, pos, groups); } + void assign_id(std::shared_ptr counter) override { + parser_base::assign_id(counter); + for (auto & p : parsers_) { + p->assign_id(counter); + } + } + std::string dump() const override { std::vector parts; parts.reserve(parsers_.size()); @@ -217,6 +230,13 @@ class choice_parser : public parser_base { return parser_result(PARSER_RESULT_FAIL, start); } + void assign_id(std::shared_ptr counter) override { + parser_base::assign_id(counter); + for (auto & p : parsers_) { + p->assign_id(counter); + } + } + std::string dump() const override { std::vector parts; parts.reserve(parsers_.size()); @@ -281,6 +301,11 @@ class repetition_parser : public parser_base { return parser_result(PARSER_RESULT_SUCCESS, start, pos, groups); } + void assign_id(std::shared_ptr counter) override { + parser_base::assign_id(counter); + parser_->assign_id(counter); + } + std::string dump() const override { if (max_count_ == -1) { return "Repetition(" + parser_->dump() + ", " + std::to_string(min_count_) + ", unbounded)"; @@ -369,6 +394,11 @@ class not_parser : public parser_base { return parser_result(PARSER_RESULT_SUCCESS, start); } + void assign_id(std::shared_ptr counter) override { + parser_base::assign_id(counter); + parser_->assign_id(counter); + } + std::string dump() const override { return "Not(" + parser_->dump() + ")"; } @@ -680,6 +710,11 @@ class group_parser : public parser_base { return result; } + void assign_id(std::shared_ptr counter) override { + parser_base::assign_id(counter); + parser_->assign_id(counter); + } + std::string dump() const override { return "Group(" + name_ + ", " + parser_->dump() + ")"; } @@ -828,6 +863,11 @@ class root_parser : public parser_base { return root_->parse(ctx, start); } + void assign_id(std::shared_ptr counter) override { + parser_base::assign_id(counter); + root_->assign_id(counter); + } + std::string dump() const override { return root_->dump(); } @@ -1117,102 +1157,6 @@ class gbnf_visitor : public parser_visitor { } }; -// ID assignment visitor for assigning unique IDs to parsers -class id_assignment_visitor : public parser_visitor { - std::shared_ptr counter_; - - public: - id_assignment_visitor(const std::shared_ptr & counter) : counter_(counter) {} - - void assign_id(parser_base & p) { - if (p.id() == -1) { - p.set_id(counter_->next()); - } - } - - void visit(literal_parser & p) override { - assign_id(p); - } - - void visit(any_parser & p) override { - assign_id(p); - } - - void visit(space_parser & p) override { - assign_id(p); - } - - void visit(chars_parser & p) override { - assign_id(p); - } - - void visit(json_string_parser & p) override { - assign_id(p); - } - - void visit(schema_parser & p) override { - assign_id(p); - } - - void visit(rule_parser & p) override { - assign_id(p); - } - - // Composite parsers - assign ID and traverse children - void visit(sequence_parser & p) override { - assign_id(p); - for (const auto & child : p.parsers()) { - child->accept(*this); - } - } - - void visit(choice_parser & p) override { - assign_id(p); - for (const auto & child : p.parsers()) { - child->accept(*this); - } - } - - void visit(one_or_more_parser & p) override { - assign_id(p); - p.child()->accept(*this); - } - - void visit(zero_or_more_parser & p) override { - assign_id(p); - p.child()->accept(*this); - } - - void visit(optional_parser & p) override { - assign_id(p); - p.child()->accept(*this); - } - - void visit(repetition_parser & p) override { - assign_id(p); - p.child()->accept(*this); - } - - void visit(until_parser & p) override { - assign_id(p); - } - - void visit(not_parser & p) override { - assign_id(p); - p.child()->accept(*this); - } - - void visit(group_parser & p) override { - assign_id(p); - p.child()->accept(*this); - } - - void visit(root_parser & p) override { - assign_id(p); - p.root()->accept(*this); - } -}; - // Implement accept() methods for all parser classes void literal_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void sequence_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } @@ -1419,8 +1363,7 @@ parser parser_builder::add_rule(const std::string & name, const parser & p) { void parser_builder::assign_ids(parser & p) { if (p.ptr()) { - id_assignment_visitor visitor(counter_); - p.ptr()->accept(visitor); + p.ptr()->assign_id(counter_); } } From 20f9a1b83b3d7516a44e1ba726930f57395a63c3 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Wed, 12 Nov 2025 00:48:05 -0600 Subject: [PATCH 022/183] create named rules in the command r7b example --- common/chat-parser-combinator.cpp | 2 +- tests/test-chat-parser-combinator.cpp | 30 ++++++++++++++------------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index 064afd80f1686..e1a1b8f082a52 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -1384,7 +1384,7 @@ static parser json_parser(std::shared_ptr counter) { parser_builder builder(std::move(counter)); // Whitespace: space, tab, newline, carriage return - auto ws = builder.chars("[ \\t\\n\\r]", 0, -1); + auto ws = builder.space(); // Number components auto digit1_9 = builder.chars("[1-9]", 1, 1); diff --git a/tests/test-chat-parser-combinator.cpp b/tests/test-chat-parser-combinator.cpp index 6264d72f76fbb..032acba9b8d3b 100644 --- a/tests/test-chat-parser-combinator.cpp +++ b/tests/test-chat-parser-combinator.cpp @@ -710,38 +710,40 @@ static void test_gbnf_generation() { static parser create_command_r7b_parser() { auto parser = build_parser([](parser_builder & p) { - auto thinking = p.literal("<|START_THINKING|>") + auto thinking = p.add_rule("thinking", p.literal("<|START_THINKING|>") << p.until("<|END_THINKING|>") - << p.literal("<|END_THINKING|>"); + << p.literal("<|END_THINKING|>")); - auto response = p.literal("<|START_RESPONSE|>") + auto response = p.add_rule("response", p.literal("<|START_RESPONSE|>") << p.until("<|END_RESPONSE|>") - << p.literal("<|END_RESPONSE|>"); + << p.literal("<|END_RESPONSE|>")); - auto json = p.json(); - auto tool_call_id = p.json_key("tool_call_id", p.json_string(p.until("\""))); - auto tool_call_name = p.json_key("tool_name", p.json_string(p.until("\""))); - auto tool_call_args = p.json_key("parameters", json); - auto tool_call_fields = tool_call_id | tool_call_name | tool_call_args; + auto json = p.add_rule("json", p.json()); + auto tool_call_id = p.add_rule("tool-call-id", p.json_key("tool_call_id", p.json_string(p.until("\"")))); + auto tool_call_name = p.add_rule("tool-name", p.json_key("tool_name", p.json_string(p.until("\"")))); + auto tool_call_args = p.add_rule("tool-args", p.json_key("parameters", json)); + auto tool_call_fields = p.add_rule("tool-call-fields", tool_call_id | tool_call_name | tool_call_args); - auto tool_call = p.between("{", + auto tool_call = p.add_rule("tool-call", p.between("{", tool_call_fields << p.zero_or_more(p.literal(",") << tool_call_fields), - "}"); + "}")); - auto tool_calls = p.literal("<|START_ACTION|>") + auto tool_calls = p.add_rule("tool-calls", p.literal("<|START_ACTION|>") << p.literal("[") << tool_call << p.zero_or_more(p.literal(",") << tool_call) << p.literal("]") - << p.literal("<|END_ACTION|>"); + << p.literal("<|END_ACTION|>")); - return p.optional(thinking) << (tool_calls | response); + return p.optional(thinking) << p.add_rule("content", (tool_calls | response)); }); auto grammar = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); + std::cout << "=== Grammar ===\n\n" << grammar << "\n\n"; + return parser; } From 35b164037e19be2d971514be636a62de69ae92a2 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Wed, 12 Nov 2025 01:09:39 -0600 Subject: [PATCH 023/183] use '.' for any in GBNF --- common/chat-parser-combinator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index e1a1b8f082a52..f4633de06f962 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -1080,7 +1080,7 @@ class gbnf_visitor : public parser_visitor { void visit(any_parser &) override { // Match any single character - current_result_ = "[\\x00-\\x{10FFFF}]"; + current_result_ = "."; } void visit(space_parser &) override { From bcb1c03c02cbcf03ea0c9e59dc2c6423d10c3d79 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Wed, 12 Nov 2025 01:16:24 -0600 Subject: [PATCH 024/183] fix parens around choices in gbnf grammar --- common/chat-parser-combinator.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index f4633de06f962..4ab0835abcc2e 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -998,7 +998,13 @@ class gbnf_visitor : public parser_visitor { s += " "; } child->accept(*this); - s += current_result_; + + // Parenthesize choices + if (needs_parens(child->type())) { + s += "(" + current_result_ + ")"; + } else { + s += current_result_; + } } current_result_ = s; } @@ -1012,8 +1018,8 @@ class gbnf_visitor : public parser_visitor { child->accept(*this); - // Parenthesize sequences in choices - if (child->type() == PARSER_SEQUENCE) { + // Parenthesize choices + if (child->type() == PARSER_CHOICE) { s += "(" + current_result_ + ")"; } else { s += current_result_; From 4bed84dfe0341e46305d3a7f5f575b7853b59267 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Wed, 12 Nov 2025 01:29:44 -0600 Subject: [PATCH 025/183] add convenience operators to turn strings to literals --- common/chat-parser-combinator.cpp | 18 ++++++++++++++++++ common/chat-parser-combinator.h | 7 +++++++ tests/test-chat-parser-combinator.cpp | 25 ++++++++++--------------- 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index 4ab0835abcc2e..a096745130b57 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -1220,6 +1220,8 @@ parser::parser() {} parser::parser(std::shared_ptr parser) : ptr_(std::move(parser)) {} +parser::parser(const std::string & literal) : ptr_(std::make_shared(literal, -1)) {} + parser parser::operator~() const { return parser(std::make_shared(*this, -1)); } @@ -1228,15 +1230,31 @@ parser parser::operator+(const parser & other) const { return parser(std::make_shared(std::initializer_list{*this, other}, -1)); } +parser parser::operator+(const std::string & literal) const { + auto lit = parser(std::make_shared(literal, -1)); + return parser(std::make_shared(std::initializer_list{*this, lit}, -1)); +} + parser parser::operator|(const parser & other) const { return parser(std::make_shared(std::initializer_list{*this, other}, -1)); } +parser parser::operator|(const std::string & literal) const { + auto lit = parser(std::make_shared(literal, -1)); + return parser(std::make_shared(std::initializer_list{*this, lit}, -1)); +} + parser parser::operator<<(const parser & other) const { auto ws = parser(std::make_shared(-1)); return parser(std::make_shared(std::initializer_list{*this, ws, other}, -1)); } +parser parser::operator<<(const std::string & literal) const { + auto ws = parser(std::make_shared(-1)); + auto lit = parser(std::make_shared(literal, -1)); + return parser(std::make_shared(std::initializer_list{*this, ws, lit}, -1)); +} + parser_base & parser::operator*() const { return *ptr_; } diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index 6275de1fcde62..9e5270150bb48 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -105,6 +105,7 @@ class parser { parser(); parser(std::shared_ptr parser); parser(const parser & other) = default; + parser(const std::string & literal); parser & operator=(const parser & other) { if (this != &other) { ptr_ = other.ptr_; @@ -113,9 +114,15 @@ class parser { } parser operator~() const; + parser operator+(const parser & other) const; + parser operator+(const std::string & literal) const; + parser operator|(const parser & other) const; + parser operator|(const std::string & literal) const; + parser operator<<(const parser & other) const; + parser operator<<(const std::string & literal) const; parser_base & operator*() const; parser_base * operator->() const; diff --git a/tests/test-chat-parser-combinator.cpp b/tests/test-chat-parser-combinator.cpp index 032acba9b8d3b..752afee4349d1 100644 --- a/tests/test-chat-parser-combinator.cpp +++ b/tests/test-chat-parser-combinator.cpp @@ -710,13 +710,11 @@ static void test_gbnf_generation() { static parser create_command_r7b_parser() { auto parser = build_parser([](parser_builder & p) { - auto thinking = p.add_rule("thinking", p.literal("<|START_THINKING|>") - << p.until("<|END_THINKING|>") - << p.literal("<|END_THINKING|>")); + auto thinking = p.add_rule("thinking", + p.literal("<|START_THINKING|>") << p.until("<|END_THINKING|>") << "<|END_THINKING|>"); - auto response = p.add_rule("response", p.literal("<|START_RESPONSE|>") - << p.until("<|END_RESPONSE|>") - << p.literal("<|END_RESPONSE|>")); + auto response = p.add_rule("response", + p.literal("<|START_RESPONSE|>") << p.until("<|END_RESPONSE|>") << "<|END_RESPONSE|>"); auto json = p.add_rule("json", p.json()); auto tool_call_id = p.add_rule("tool-call-id", p.json_key("tool_call_id", p.json_string(p.until("\"")))); @@ -724,16 +722,13 @@ static parser create_command_r7b_parser() { auto tool_call_args = p.add_rule("tool-args", p.json_key("parameters", json)); auto tool_call_fields = p.add_rule("tool-call-fields", tool_call_id | tool_call_name | tool_call_args); - auto tool_call = p.add_rule("tool-call", p.between("{", - tool_call_fields << p.zero_or_more(p.literal(",") << tool_call_fields), - "}")); + auto tool_call = p.add_rule("tool-call", + p.between("{", tool_call_fields << p.zero_or_more(p.literal(",") << tool_call_fields), "}")); - auto tool_calls = p.add_rule("tool-calls", p.literal("<|START_ACTION|>") - << p.literal("[") - << tool_call - << p.zero_or_more(p.literal(",") << tool_call) - << p.literal("]") - << p.literal("<|END_ACTION|>")); + auto tool_calls = p.add_rule("tool-calls", + p.literal("<|START_ACTION|>") + << "[" << tool_call << p.zero_or_more(p.literal(",") << tool_call) << "]" + << "<|END_ACTION|>"); return p.optional(thinking) << p.add_rule("content", (tool_calls | response)); }); From c02aaa61b53fc1f99dd7ca236ffabeaf9540a1cd Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Wed, 12 Nov 2025 01:44:20 -0600 Subject: [PATCH 026/183] add free-form operators for const char * to simplify defining literals --- common/chat-parser-combinator.cpp | 27 +++++---------------------- common/chat-parser-combinator.h | 14 ++++++-------- tests/test-chat-parser-combinator.cpp | 12 ++++++------ 3 files changed, 17 insertions(+), 36 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index a096745130b57..81b55240a842f 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -1222,6 +1222,8 @@ parser::parser(std::shared_ptr parser) : ptr_(std::move(parser)) {} parser::parser(const std::string & literal) : ptr_(std::make_shared(literal, -1)) {} +parser::parser(const char * literal) : ptr_(std::make_shared(literal, -1)) {} + parser parser::operator~() const { return parser(std::make_shared(*this, -1)); } @@ -1230,30 +1232,18 @@ parser parser::operator+(const parser & other) const { return parser(std::make_shared(std::initializer_list{*this, other}, -1)); } -parser parser::operator+(const std::string & literal) const { - auto lit = parser(std::make_shared(literal, -1)); - return parser(std::make_shared(std::initializer_list{*this, lit}, -1)); -} - parser parser::operator|(const parser & other) const { return parser(std::make_shared(std::initializer_list{*this, other}, -1)); } -parser parser::operator|(const std::string & literal) const { - auto lit = parser(std::make_shared(literal, -1)); - return parser(std::make_shared(std::initializer_list{*this, lit}, -1)); -} - parser parser::operator<<(const parser & other) const { auto ws = parser(std::make_shared(-1)); return parser(std::make_shared(std::initializer_list{*this, ws, other}, -1)); } -parser parser::operator<<(const std::string & literal) const { - auto ws = parser(std::make_shared(-1)); - auto lit = parser(std::make_shared(literal, -1)); - return parser(std::make_shared(std::initializer_list{*this, ws, lit}, -1)); -} +parser operator+(const char * lhs, const parser & rhs) { return parser(lhs) + rhs; } +parser operator|(const char * lhs, const parser & rhs) { return parser(lhs) | rhs; } +parser operator<<(const char * lhs, const parser & rhs) { return parser(lhs) << rhs; } parser_base & parser::operator*() const { return *ptr_; @@ -1373,13 +1363,6 @@ parser parser_builder::json_string(const parser & p) { return quote + p + quote; } -parser parser_builder::between(const std::string & left, const parser & p, const std::string & right, bool allow_spaces) { - if (allow_spaces) { - return literal(left) << p << literal(right); - } - return literal(left) + p + literal(right); -} - parser parser_builder::add_rule(const std::string & name, const parser & p) { (*rules_)[name] = p; return rule(name); diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index 9e5270150bb48..5b8b08aeb528f 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -106,6 +106,8 @@ class parser { parser(std::shared_ptr parser); parser(const parser & other) = default; parser(const std::string & literal); + parser(const char * literal); + parser & operator=(const parser & other) { if (this != &other) { ptr_ = other.ptr_; @@ -114,15 +116,9 @@ class parser { } parser operator~() const; - parser operator+(const parser & other) const; - parser operator+(const std::string & literal) const; - parser operator|(const parser & other) const; - parser operator|(const std::string & literal) const; - parser operator<<(const parser & other) const; - parser operator<<(const std::string & literal) const; parser_base & operator*() const; parser_base * operator->() const; @@ -136,6 +132,10 @@ class parser { void build_grammar(const common_grammar_builder & builder) const; }; +parser operator+(const char * lhs, const parser & rhs); +parser operator|(const char * lhs, const parser & rhs); +parser operator<<(const char * lhs, const parser & rhs); + class parser_id_counter { int next_id_; public: @@ -231,8 +231,6 @@ class parser_builder { parser json_key(const std::string & name, const parser & p); parser json_string(const parser & p); - parser between(const std::string & left, const parser & p, const std::string & right, bool allow_spaces = true); - // Wraps a parser with JSON schema metadata for grammar generation. // Used internally to convert JSON schemas to GBNF grammar rules. parser schema(const parser & p, const std::string & name, const nlohmann::ordered_json & schema); diff --git a/tests/test-chat-parser-combinator.cpp b/tests/test-chat-parser-combinator.cpp index 752afee4349d1..c20cf5417af2c 100644 --- a/tests/test-chat-parser-combinator.cpp +++ b/tests/test-chat-parser-combinator.cpp @@ -711,10 +711,10 @@ static void test_gbnf_generation() { static parser create_command_r7b_parser() { auto parser = build_parser([](parser_builder & p) { auto thinking = p.add_rule("thinking", - p.literal("<|START_THINKING|>") << p.until("<|END_THINKING|>") << "<|END_THINKING|>"); + "<|START_THINKING|>" << p.until("<|END_THINKING|>") << "<|END_THINKING|>"); auto response = p.add_rule("response", - p.literal("<|START_RESPONSE|>") << p.until("<|END_RESPONSE|>") << "<|END_RESPONSE|>"); + "<|START_RESPONSE|>" << p.until("<|END_RESPONSE|>") << "<|END_RESPONSE|>"); auto json = p.add_rule("json", p.json()); auto tool_call_id = p.add_rule("tool-call-id", p.json_key("tool_call_id", p.json_string(p.until("\"")))); @@ -723,14 +723,14 @@ static parser create_command_r7b_parser() { auto tool_call_fields = p.add_rule("tool-call-fields", tool_call_id | tool_call_name | tool_call_args); auto tool_call = p.add_rule("tool-call", - p.between("{", tool_call_fields << p.zero_or_more(p.literal(",") << tool_call_fields), "}")); + "{" << tool_call_fields << p.zero_or_more(p.literal(",") << tool_call_fields) << "}"); auto tool_calls = p.add_rule("tool-calls", - p.literal("<|START_ACTION|>") - << "[" << tool_call << p.zero_or_more(p.literal(",") << tool_call) << "]" + "<|START_ACTION|>" + << ("[" << tool_call << p.zero_or_more(p.literal(",") << tool_call) << "]") << "<|END_ACTION|>"); - return p.optional(thinking) << p.add_rule("content", (tool_calls | response)); + return p.optional(thinking) << p.add_rule("content", tool_calls | response); }); auto grammar = build_grammar([&](const common_grammar_builder & builder) { From 8e821275f07b7a031ee95e9246030e43a7035222 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Wed, 12 Nov 2025 01:51:43 -0600 Subject: [PATCH 027/183] simplify test case parser --- tests/test-chat-parser-combinator.cpp | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/tests/test-chat-parser-combinator.cpp b/tests/test-chat-parser-combinator.cpp index c20cf5417af2c..93d730ee8ffe2 100644 --- a/tests/test-chat-parser-combinator.cpp +++ b/tests/test-chat-parser-combinator.cpp @@ -447,9 +447,7 @@ static void test_complete_example() { // auto parser = build_parser([](parser_builder & p) { auto reasoning = p.add_rule("reasoning", - p.literal("") - << p.group("reasoning-content", p.until("")) - << p.literal("")); + "" << p.group("reasoning-content", p.until("")) << ""); auto content = p.add_rule("content", p.group("content", p.until(""))); @@ -457,22 +455,15 @@ static void test_complete_example() { auto json = p.json(); auto tool_call_name = p.add_rule("tool-call-name", - p.literal("") - << p.group("tool-name", p.until("")) - << p.literal("")); + "" << p.group("tool-name", p.until("")) << ""); auto schema = nlohmann::ordered_json::parse(R"({"type": "object"})"); auto tool_call_args = p.add_rule("tool-call-args", - p.literal("") - << p.group("tool-args", p.schema(json, "get_weather", schema)) - << p.literal("")); + "" << p.group("tool-args", p.schema(json, "get_weather", schema)) << ""); auto tool_call = p.add_rule("tool-call", - p.literal("") - << tool_call_name - << tool_call_args - << p.literal("")); + "" << tool_call_name << tool_call_args << ""); return reasoning << p.optional(content) << p.optional(tool_call); }); From 9685b696584b960d04225e67a18ce4be02fe0f9a Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Wed, 12 Nov 2025 02:32:01 -0600 Subject: [PATCH 028/183] implement semantic actions --- common/chat-parser-combinator.cpp | 49 +++++++++++ common/chat-parser-combinator.h | 33 +++++++- tests/test-chat-parser-combinator.cpp | 112 ++++++++++++++++++++++++++ 3 files changed, 190 insertions(+), 4 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index 81b55240a842f..cf68c646ac1b8 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -26,6 +26,7 @@ enum parser_type { PARSER_SCHEMA = 14, PARSER_ROOT = 15, PARSER_JSON_STRING = 16, + PARSER_ACTION = 17, }; class parser_visitor; @@ -879,6 +880,43 @@ class root_parser : public parser_base { std::shared_ptr> rules() const { return rules_; } }; +// Wraps a parser with a semantic action callback. +class action_parser : public parser_base { + parser parser_; + std::function action_; + + public: + action_parser(const parser & parser, std::function action, int id) + : parser_base(id), parser_(parser), action_(std::move(action)) {} + + parser_type type() const override { return PARSER_ACTION; } + + parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { + auto result = parser_->parse(ctx, start); + + // Invoke action callback on success if environment is available + if (result.is_success() && ctx.env && action_) { + std::string_view matched = ctx.input.substr(result.start, result.end - result.start); + action_(result, matched, *ctx.env); + } + + return result; + } + + void assign_id(std::shared_ptr counter) override { + parser_base::assign_id(counter); + parser_->assign_id(counter); + } + + std::string dump() const override { + return "Action(" + parser_->dump() + ")"; + } + + void accept(parser_visitor & visitor) override; + + const parser & child() const { return parser_; } +}; + // Base visitor class for parser tree traversal class parser_visitor { public: @@ -901,6 +939,7 @@ class parser_visitor { virtual void visit(schema_parser & p) = 0; virtual void visit(rule_parser & p) = 0; virtual void visit(root_parser & p) = 0; + virtual void visit(action_parser & p) = 0; }; class gbnf_visitor : public parser_visitor { @@ -1161,6 +1200,11 @@ class gbnf_visitor : public parser_visitor { // Return root body for composition p.root()->accept(*this); } + + void visit(action_parser & p) override { + // Actions are transparent for grammar generation - just visit child + p.child()->accept(*this); + } }; // Implement accept() methods for all parser classes @@ -1181,6 +1225,7 @@ void group_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void schema_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void rule_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void root_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } +void action_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } std::optional parser_result::group(const std::string & name, std::string_view input) const { auto it = groups.find(name); @@ -1354,6 +1399,10 @@ parser parser_builder::schema(const parser & p, const std::string & name, const return parser(std::make_shared(p, name, schema, counter_->next())); } +parser parser_builder::action(const parser & p, std::function fn) { + return parser(std::make_shared(p, std::move(fn), counter_->next())); +} + parser parser_builder::json_key(const std::string & name, const parser & p) { return literal("\"" + name + "\"") << literal(":") << p; } diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index 5b8b08aeb528f..585ca44d04b69 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -8,9 +8,19 @@ #include #include #include +#include +#include +struct common_chat_tool_call; struct common_grammar_builder; +struct parser_environment { + std::string content; + std::string reasoning; + std::vector tool_calls; + std::unordered_map> scratchpad; +}; + enum parser_result_type { PARSER_RESULT_FAIL = 0, PARSER_RESULT_NEED_MORE_INPUT = 1, @@ -82,18 +92,28 @@ struct parser_context { std::string_view input; parse_cache memo; bool input_is_complete; + parser_environment * env; parser_context() - : memo(), input_is_complete(true) {} + : memo(), input_is_complete(true), env(nullptr) {} parser_context(std::string_view input) - : input(input), memo(), input_is_complete(true) {} + : input(input), memo(), input_is_complete(true), env(nullptr) {} parser_context(std::string_view input, bool complete) - : input(input), memo(), input_is_complete(complete) {} + : input(input), memo(), input_is_complete(complete), env(nullptr) {} parser_context(std::string_view input, parse_cache memo, bool complete = true) - : input(input), memo(std::move(memo)), input_is_complete(complete) {} + : input(input), memo(std::move(memo)), input_is_complete(complete), env(nullptr) {} + + parser_context(std::string_view input, parser_environment * environment) + : input(input), memo(), input_is_complete(true), env(environment) {} + + parser_context(std::string_view input, parser_environment * environment, bool complete) + : input(input), memo(), input_is_complete(complete), env(environment) {} + + parser_context(std::string_view input, parse_cache memo, parser_environment * environment, bool complete = true) + : input(input), memo(std::move(memo)), input_is_complete(complete), env(environment) {} }; class parser_base; @@ -235,6 +255,11 @@ class parser_builder { // Used internally to convert JSON schemas to GBNF grammar rules. parser schema(const parser & p, const std::string & name, const nlohmann::ordered_json & schema); + // Wraps a parser with a semantic action callback. + // The callback is invoked on successful parse with the result, matched text, and environment. + // S -> A [action] + parser action(const parser & p, std::function fn); + parser add_rule(const std::string & name, const parser & p); void assign_ids(parser & p); diff --git a/tests/test-chat-parser-combinator.cpp b/tests/test-chat-parser-combinator.cpp index 93d730ee8ffe2..57795f5403bf1 100644 --- a/tests/test-chat-parser-combinator.cpp +++ b/tests/test-chat-parser-combinator.cpp @@ -533,6 +533,117 @@ static void test_complete_example() { std::cout << "Grammar:\n" << gbnf << "\n"; } +static void test_actions() { + { + // Test simple action - append matched text to content + auto parser = build_parser([](parser_builder& p) { + auto word = p.chars("[a-z]+"); + return p.action(word, [](const parser_result &, std::string_view matched, parser_environment & env) { + env.content += std::string(matched); + }); + }); + + parser_environment env; + parser_context ctx("hello", &env); + auto result = parser.parse(ctx); + + assert_equals(true, result.is_success()); + assert_equals("hello", env.content); + } + { + // Test multiple sequential actions - build a sentence + auto parser = build_parser([](parser_builder& p) { + auto greeting = p.action(p.literal("hello"), [](const parser_result &, std::string_view matched, parser_environment & env) { + env.content += std::string(matched) + " "; + }); + + auto name = p.action(p.chars("[A-Z][a-z]+"), [](const parser_result &, std::string_view matched, parser_environment & env) { + env.content += std::string(matched); + env.scratchpad["name"] = std::string(matched); + }); + + return greeting + p.literal(" ") + name; + }); + + parser_environment env; + parser_context ctx("hello Alice", &env); + auto result = parser.parse(ctx); + + assert_equals(true, result.is_success()); + assert_equals("hello Alice", env.content); + assert_equals("Alice", std::get(env.scratchpad["name"])); + } + { + // Test using scratchpad for intermediate calculations + auto parser = build_parser([](parser_builder& p) { + auto digit = p.action(p.one("[0-9]"), [](const parser_result &, std::string_view matched, parser_environment & env) { + auto it = env.scratchpad.find("sum"); + int current_sum = it != env.scratchpad.end() ? std::get(it->second) : 0; + current_sum += (matched[0] - '0'); + env.scratchpad["sum"] = current_sum; + }); + + return p.one_or_more(digit + p.optional(p.literal("+"))); + }); + + parser_environment env; + parser_context ctx("1+2+3+4", &env); + auto result = parser.parse(ctx); + + assert_equals(true, result.is_success()); + assert_equals(10, std::get(env.scratchpad["sum"])); // 1+2+3+4 = 10 + } + { + // Test actions don't run when parse fails + auto parser = build_parser([](parser_builder& p) { + return p.action(p.literal("success"), [](const parser_result &, std::string_view, parser_environment & env) { + env.content = "action_ran"; + }); + }); + + parser_environment env; + parser_context ctx("failure", &env); + auto result = parser.parse(ctx); + + assert_equals(true, result.is_fail()); + assert_equals("", env.content); // Action should not have run + } + { + // Test Actions work with partial parsing + auto parser = build_parser([](parser_builder& p) { + auto content = p.action(p.until(""), [](const parser_result &, std::string_view matched, parser_environment & env) { + env.content += std::string(matched); + }); + return "" << content << ""; + }); + + { + parser_environment env; + parser_context ctx("hello ", &env, false); + auto result = parser.parse(ctx); + + assert_equals(true, result.is_success()); + assert_equals("hello", env.content); + } + { + parser_environment env; + parser_context ctx("hello world", &env, false); + auto result = parser.parse(ctx); + + assert_equals(true, result.is_success()); + assert_equals("hello world", env.content); + } + { + parser_environment env; + parser_context ctx("hello world", &env, true); + auto result = parser.parse(ctx); + + assert_equals(true, result.is_success()); + assert_equals("hello world", env.content); + } + } +} + static void test_gbnf_generation() { { // Test literal @@ -888,6 +999,7 @@ int main() { test_optional(); test_json_parser(); test_complete_example(); + test_actions(); test_gbnf_generation(); std::cout << "All tests passed!\n"; From d9a62295b8fa9e39c58fda1e286c80fb352e3714 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Wed, 12 Nov 2025 03:06:47 -0600 Subject: [PATCH 029/183] remove groups in favor of actions and a scratchpad --- common/chat-parser-combinator.cpp | 118 +++----------- common/chat-parser-combinator.h | 25 +-- tests/test-chat-parser-combinator.cpp | 222 ++++++++++++-------------- 3 files changed, 120 insertions(+), 245 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index cf68c646ac1b8..04f354232f548 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -19,14 +19,13 @@ enum parser_type { PARSER_NOT = 7, PARSER_ANY = 8, PARSER_CHARS = 9, - PARSER_GROUP = 10, - PARSER_RULE = 11, - PARSER_UNTIL = 12, - PARSER_SPACE = 13, - PARSER_SCHEMA = 14, - PARSER_ROOT = 15, - PARSER_JSON_STRING = 16, - PARSER_ACTION = 17, + PARSER_RULE = 10, + PARSER_UNTIL = 11, + PARSER_SPACE = 12, + PARSER_SCHEMA = 13, + PARSER_ROOT = 14, + PARSER_JSON_STRING = 15, + PARSER_ACTION = 16, }; class parser_visitor; @@ -81,6 +80,10 @@ static bool is_space(const char c) { return (c == ' ' || c == '\t' || c == '\n'); } +static bool is_hex_digit(const char c) { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); +} + // Matches an exact literal string. // S -> "hello" class literal_parser : public parser_base { @@ -144,31 +147,26 @@ class sequence_parser : public parser_base { parser_type type() const override { return PARSER_SEQUENCE; } parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { - std::unordered_map groups; - auto pos = start; for (const auto & p : parsers_) { auto result = p->parse(ctx, pos); - // Copy groups - groups.insert(result.groups.begin(), result.groups.end()); - if (result.is_fail()) { if (result.end >= ctx.input.size() && !ctx.input_is_complete) { // If we fail because we don't have enough input, then return success - return parser_result(PARSER_RESULT_SUCCESS, start, result.end, groups); + return parser_result(PARSER_RESULT_SUCCESS, start, result.end); } - return parser_result(PARSER_RESULT_FAIL, start, result.end, groups); + return parser_result(PARSER_RESULT_FAIL, start, result.end); } if (result.is_need_more_input()) { - return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, result.end, groups); + return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, result.end); } pos = result.end; } - return parser_result(PARSER_RESULT_SUCCESS, start, pos, groups); + return parser_result(PARSER_RESULT_SUCCESS, start, pos); } void assign_id(std::shared_ptr counter) override { @@ -267,14 +265,12 @@ class repetition_parser : public parser_base { parser_type type() const override { return PARSER_REPETITION; } parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { - std::unordered_map groups; auto pos = start; int match_count = 0; // Try to match up to max_count times (or unlimited if max_count is -1) while (max_count_ == -1 || match_count < max_count_) { auto result = parser_->parse(ctx, pos); - groups.insert(result.groups.begin(), result.groups.end()); if (result.is_success()) { // Prevent infinite loop on empty matches @@ -287,7 +283,7 @@ class repetition_parser : public parser_base { } if (result.is_need_more_input()) { - return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, pos, groups); + return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, pos); } // Child failed - stop trying @@ -296,10 +292,10 @@ class repetition_parser : public parser_base { // Check if we got enough matches if (match_count < min_count_) { - return parser_result(PARSER_RESULT_FAIL, start, pos, groups); + return parser_result(PARSER_RESULT_FAIL, start, pos); } - return parser_result(PARSER_RESULT_SUCCESS, start, pos, groups); + return parser_result(PARSER_RESULT_SUCCESS, start, pos); } void assign_id(std::shared_ptr counter) override { @@ -595,20 +591,13 @@ class chars_parser : public parser_base { // Handles escape sequences and emits NEED_MORE_INPUT for incomplete input. // S -> (regular chars and escape sequences)* until closing " class json_string_parser : public parser_base { - std::optional capture_name_; - - static bool is_hex_digit(char c) { - return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); - } public: - json_string_parser(std::optional capture_name, int id) - : parser_base(id), capture_name_(std::move(capture_name)) {} + json_string_parser(int id) : parser_base(id) {} parser_type type() const override { return PARSER_JSON_STRING; } parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { - std::unordered_map groups; auto pos = start; // Parse string content (without quotes) @@ -617,10 +606,7 @@ class json_string_parser : public parser_base { if (c == '"') { // Found closing quote - success (don't consume it) - if (capture_name_) { - groups[*capture_name_] = parser_match_location{start, pos}; - } - return parser_result(PARSER_RESULT_SUCCESS, start, pos, groups); + return parser_result(PARSER_RESULT_SUCCESS, start, pos); } if (c == '\\') { @@ -681,48 +667,10 @@ class json_string_parser : public parser_base { } std::string dump() const override { - if (capture_name_) { - return "JsonString(" + *capture_name_ + ")"; - } return "JsonString()"; } void accept(parser_visitor & visitor) override; - - const std::optional & capture_name() const { return capture_name_; } -}; - -// Captures the matched text from a parser and stores it with a name. -// S -> -class group_parser : public parser_base { - std::string name_; - parser parser_; - - public: - group_parser(const std::string & name, const parser & parser, int id) : parser_base(id), name_(name), parser_(parser) {} - - parser_type type() const override { return PARSER_GROUP; } - - parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { - auto result = parser_->parse(ctx, start); - - // Store result - result.groups[name_] = parser_match_location{result.start, result.end}; - return result; - } - - void assign_id(std::shared_ptr counter) override { - parser_base::assign_id(counter); - parser_->assign_id(counter); - } - - std::string dump() const override { - return "Group(" + name_ + ", " + parser_->dump() + ")"; - } - - void accept(parser_visitor & visitor) override; - - const parser & child() const { return parser_; } }; // Matches all characters until a delimiter is found (delimiter not consumed). @@ -935,7 +883,6 @@ class parser_visitor { virtual void visit(space_parser & p) = 0; virtual void visit(chars_parser & p) = 0; virtual void visit(json_string_parser & p) = 0; - virtual void visit(group_parser & p) = 0; virtual void visit(schema_parser & p) = 0; virtual void visit(rule_parser & p) = 0; virtual void visit(root_parser & p) = 0; @@ -1165,11 +1112,6 @@ class gbnf_visitor : public parser_visitor { current_result_ = R"(( [^"\\] | "\\" ( ["\\/ bfnrt] | "u" [0-9a-fA-F]{4} ) )*)"; } - void visit(group_parser & p) override { - // Groups are transparent - just visit child - p.child()->accept(*this); - } - void visit(schema_parser & p) override { current_result_ = builder_.add_schema(p.name(), p.schema()); } @@ -1221,21 +1163,11 @@ void any_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void space_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void chars_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void json_string_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } -void group_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void schema_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void rule_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void root_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void action_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } -std::optional parser_result::group(const std::string & name, std::string_view input) const { - auto it = groups.find(name); - if (it == groups.end()) { - return std::nullopt; - } - - return std::string(it->second.view(input)); -} - parser_result parse_cache::set(int id, size_t start, parser_result result) { if (id == -1) { // Don't cache parsers with ID -1 (from operators and global factory functions) @@ -1364,15 +1296,7 @@ parser parser_builder::one(const std::string & classes) { } parser parser_builder::json_string() { - return parser(std::make_shared(std::nullopt, counter_->next())); -} - -parser parser_builder::json_string(const std::string & name) { - return parser(std::make_shared(name, counter_->next())); -} - -parser parser_builder::group(const std::string & name, const parser & p) { - return parser(std::make_shared(name, p, counter_->next())); + return parser(std::make_shared(counter_->next())); } parser parser_builder::rule(const std::string & name) { diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index 585ca44d04b69..cb7da41740a75 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -16,7 +16,7 @@ struct common_grammar_builder; struct parser_environment { std::string content; - std::string reasoning; + std::string reasoning_content; std::vector tool_calls; std::unordered_map> scratchpad; }; @@ -43,24 +43,11 @@ struct std::hash { } }; -struct parser_match_location { - size_t start; - size_t end; - - size_t length() const { return end - start; } - - std::string_view view(std::string_view sv) const { - return sv.substr(start, length()); - } -}; - struct parser_result { parser_result_type type = PARSER_RESULT_FAIL; size_t start = 0; size_t end = 0; - std::unordered_map groups; - parser_result() : type(PARSER_RESULT_FAIL) {} parser_result(parser_result_type type, size_t start) @@ -69,14 +56,9 @@ struct parser_result { parser_result(parser_result_type type, size_t start, size_t end) : type(type), start(start), end(end) {} - parser_result(parser_result_type type, size_t start, size_t end, const std::unordered_map & groups) - : type(type), start(start), end(end), groups(groups) {} - bool is_fail() const { return type == PARSER_RESULT_FAIL; } bool is_need_more_input() const { return type == PARSER_RESULT_NEED_MORE_INPUT; } bool is_success() const { return type == PARSER_RESULT_SUCCESS; } - - std::optional group(const std::string & name, std::string_view input) const; }; class parse_cache { @@ -215,10 +197,6 @@ class parser_builder { // Equivalent to chars(classes, 1, 1) parser one(const std::string & classes); - // Captures the matched text from a parser and stores it with a name. - // S -> - parser group(const std::string & name, const parser & p); - // References a named rule for recursive or reusable grammar definitions. // expr -> term | expr "+" term parser rule(const std::string & name); @@ -246,7 +224,6 @@ class parser_builder { // Specialized single-pass JSON string parser with escape sequence handling parser json_string(); - parser json_string(const std::string & name); parser json_key(const std::string & name, const parser & p); parser json_string(const parser & p); diff --git a/tests/test-chat-parser-combinator.cpp b/tests/test-chat-parser-combinator.cpp index 57795f5403bf1..a309f7fce5a1c 100644 --- a/tests/test-chat-parser-combinator.cpp +++ b/tests/test-chat-parser-combinator.cpp @@ -4,6 +4,7 @@ #include "nlohmann/json.hpp" +#include "chat.h" #include "chat-parser.h" #include "chat-parser-combinator.h" #include "common.h" @@ -181,71 +182,6 @@ static void test_partial_parsing() { } } -static void test_capture_groups() { - { - auto parser = build_parser([](parser_builder& p) { - return p.literal("") + - p.group("reasoning_content", - p.zero_or_more(~p.literal("") + p.any()) - ) + - p.literal(""); - }); - - std::string input = "I have a thought"; - auto ctx = parser_context(input); - auto result = parser.parse(ctx); - - assert_equals(true, result.is_success()); - - auto it = result.groups.find("reasoning_content"); - assert_equals(true, it != result.groups.end()); - assert_equals("I have a thought", std::string(it->second.view(input))); - } - { - auto parser = build_parser([](parser_builder& p) { - return p.literal("") + - p.group("reasoning_content", - p.zero_or_more(~p.literal("") + p.any()) - ) + - p.literal(""); - }); - - std::string input = "I have a "; - auto ctx = parser_context(input, false); - auto result = parser.parse(ctx); - - assert_equals(true, result.is_success()); - - auto it = result.groups.find("reasoning_content"); - assert_equals(true, it != result.groups.end()); - assert_equals("I have a ", std::string(it->second.view(input))); - } - { - auto parser = build_parser([](parser_builder& p) { - return p.literal("") + - p.group("reasoning_content", - p.zero_or_more(~p.literal("") + p.any()) - ) + - p.literal("") + - p.group("content", p.zero_or_more(p.any())); - }); - - std::string input = "The user said hello.Hello!"; - auto ctx = parser_context(input, true); - auto result = parser.parse(ctx); - - assert_equals(true, result.is_success()); - - auto it = result.groups.find("reasoning_content"); - assert_equals(true, it != result.groups.end()); - assert_equals("The user said hello.", std::string(it->second.view(input))); - - it = result.groups.find("content"); - assert_equals(true, it != result.groups.end()); - assert_equals("Hello!", std::string(it->second.view(input))); - } -} - static void test_one() { { // Test common escape sequences @@ -446,85 +382,136 @@ static void test_complete_example() { // // auto parser = build_parser([](parser_builder & p) { + auto handle_reasoning = [](const parser_result &, std::string_view match, parser_environment & env) { + env.reasoning_content += match; + }; + + auto handle_content = [](const parser_result &, std::string_view match, parser_environment & env) { + env.content += match; + }; + + auto handle_tool_call_name = [](const parser_result &, std::string_view match, parser_environment & env) { + env.scratchpad["tool_name"] = std::string(match); + }; + + auto handle_tool_call_args = [](const parser_result &, std::string_view match, parser_environment & env) { + env.scratchpad["tool_args"] = std::string(match); + }; + + auto handle_tool_call = [](const parser_result &, std::string_view, parser_environment & env) { + auto name = env.scratchpad.find("tool_name"); + auto args = env.scratchpad.find("tool_args"); + if (name != env.scratchpad.end() && args != env.scratchpad.end()) { + auto tool_call = common_chat_tool_call{ + std::get(name->second), + std::get(args->second), + std::string() + }; + + env.tool_calls.push_back(tool_call); + } + }; + auto reasoning = p.add_rule("reasoning", - "" << p.group("reasoning-content", p.until("")) << ""); + "" << p.action(p.until(""), handle_reasoning) << ""); auto content = p.add_rule("content", - p.group("content", p.until(""))); + p.action(p.until(""), handle_content)); auto json = p.json(); auto tool_call_name = p.add_rule("tool-call-name", - "" << p.group("tool-name", p.until("")) << ""); + "" << p.action(p.until(""), handle_tool_call_name) << ""); auto schema = nlohmann::ordered_json::parse(R"({"type": "object"})"); auto tool_call_args = p.add_rule("tool-call-args", - "" << p.group("tool-args", p.schema(json, "get_weather", schema)) << ""); + "" << p.action(p.schema(json, "get_weather", schema), handle_tool_call_args) << ""); auto tool_call = p.add_rule("tool-call", - "" << tool_call_name << tool_call_args << ""); + "" << p.action(tool_call_name << tool_call_args, handle_tool_call) << ""); return reasoning << p.optional(content) << p.optional(tool_call); }); // Test complete input - std::string input = R"(I need to call get_weather with city = New Yorkget_weather{"city": "New York"})"; - parser_context ctx(input); + { + std::string input = R"(I need to call get_weather with city = New Yorkget_weather{"city": "New York"})"; + parser_environment env; + parser_context ctx(input, &env); - auto result = parser.parse(ctx); + auto result = parser.parse(ctx); - assert_equals(true, result.is_success()); - assert_equals(input.size(), result.end); - assert_equals(std::string("I need to call get_weather with city = New York"), *result.group("reasoning-content", ctx.input)); - assert_equals(std::string("get_weather"), *result.group("tool-name", ctx.input)); - assert_equals(std::string(R"({"city": "New York"})"), *result.group("tool-args", ctx.input)); + assert_equals(true, result.is_success()); + assert_equals(input.size(), result.end); + assert_equals("I need to call get_weather with city = New York", env.reasoning_content); + assert_equals((size_t)1, env.tool_calls.size()); + assert_equals("", env.tool_calls[0].id); + assert_equals("get_weather", env.tool_calls[0].name); + assert_equals(R"({"city": "New York"})", env.tool_calls[0].arguments); + } // Test partial input - input = R"(I need to call get_weather )"; - ctx = parser_context(input, /* .is_input_complete = */ false); - result = parser.parse(ctx); + { + std::string input = R"(I need to call get_weather )"; + parser_environment env = parser_environment(); + parser_context ctx = parser_context(input, &env, /* .is_input_complete = */ false); - assert_equals(true, result.is_success()); - assert_equals(std::string("I need to call get_weather"), *result.group("reasoning-content", ctx.input)); + auto result = parser.parse(ctx); - input = R"(I need to call I need to call I need to call get_weatherI need to call get_weatherI need to call get_weatherget_weather)"; - ctx = parser_context(input, /* .is_input_complete = */ false); - result = parser.parse(ctx); + assert_equals(true, result.is_need_more_input()); + } + { + std::string input = R"(I need to call get_weatherget_weather)"; + parser_environment env = parser_environment(); + parser_context ctx = parser_context(input, &env, /* .is_input_complete = */ false); - assert_equals(true, result.is_success()); - assert_equals(std::string("I need to call get_weather"), *result.group("reasoning-content", ctx.input)); + auto result = parser.parse(ctx); - input = R"(I need to call get_weatherget_weatherI need to call get_weatherget_weatherI need to call get_weatherget_weather{"cit)"; - ctx = parser_context(input, /* .is_input_complete = */ false); - result = parser.parse(ctx); + assert_equals(true, result.is_need_more_input()); + assert_equals("I need to call get_weather", env.reasoning_content); + } + { + std::string input = R"(I need to call get_weatherget_weather{"cit)"; + parser_environment env = parser_environment(); + parser_context ctx = parser_context(input, &env, /* .is_input_complete = */ false); - assert_equals(true, result.is_success()); - assert_equals(std::string("I need to call get_weather"), *result.group("reasoning-content", ctx.input)); - assert_equals(std::string("get_weather"), *result.group("tool-name", ctx.input)); - assert_equals(std::string(R"({"cit)"), *result.group("tool-args", ctx.input)); + auto result = parser.parse(ctx); + + assert_equals(true, result.is_success()); + assert_equals("I need to call get_weather", env.reasoning_content); + assert_equals("get_weather", std::get(env.scratchpad["tool_name"])); + assert_equals(R"({"cit)", std::get(env.scratchpad["tool_args"])); + } auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); @@ -743,18 +730,6 @@ static void test_gbnf_generation() { // Should generate pattern that prevents matching the full delimiter assert_equals(true, gbnf.find("root ::= ([^<] | \"<\" [^/] | \"])*") != std::string::npos); } - { - // Test groups are transparent - auto parser = build_parser([](parser_builder& p) { - return p.group("test", p.literal("hello")); - }); - - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder); - }); - - assert_equals(true, gbnf.find("root ::= \"hello\"") != std::string::npos); - } { // Test complex expression with parentheses auto parser = build_parser([](parser_builder& p) { @@ -994,7 +969,6 @@ static void benchmark_compare( int main() { test_partial_parsing(); test_one(); - test_capture_groups(); test_recursive_references(); test_optional(); test_json_parser(); From 117d908c6ef4c9038cc9611307b809a55d02a206 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Wed, 12 Nov 2025 03:31:01 -0600 Subject: [PATCH 030/183] add built in actions for common operations --- common/chat-parser-combinator.cpp | 76 +++++++++++++++++++++++++++ common/chat-parser-combinator.h | 36 +++++++++++++ tests/test-chat-parser-combinator.cpp | 44 +++------------- 3 files changed, 119 insertions(+), 37 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index 04f354232f548..60cb54f84cc1f 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -1,6 +1,7 @@ #include "chat-parser-combinator.h" #include "json-schema-to-grammar.h" #include "common.h" +#include "chat.h" #include "log.h" #include @@ -84,6 +85,22 @@ static bool is_hex_digit(const char c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); } +// Unescapes a JSON string (without the surrounding quotes) +// Uses nlohmann::json::parse to handle all JSON escape sequences +static std::string unescape_json_string(std::string_view str) { + try { + // Wrap in quotes and parse as JSON string + std::string quoted = "\"" + std::string(str) + "\""; + auto parsed = nlohmann::json::parse(quoted); + if (parsed.is_string()) { + return parsed.get(); + } + } catch (...) { + // If parsing fails, return original string + } + return std::string(str); +} + // Matches an exact literal string. // S -> "hello" class literal_parser : public parser_base { @@ -1327,6 +1344,65 @@ parser parser_builder::action(const parser & p, std::function(p, std::move(fn), counter_->next())); } +parser parser_builder::append_reasoning(const parser & p) { + return action(p, [](const parser_result &, std::string_view matched, parser_environment & env) { + if (!env.reasoning_content.empty()) { + env.reasoning_content += "\n"; + } + env.reasoning_content += matched; + }); +} + +parser parser_builder::append_content(const parser & p) { + return action(p, [](const parser_result &, std::string_view matched, parser_environment & env) { + if (!env.content.empty()) { + env.content += "\n"; + } + env.content += matched; + }); +} + +parser parser_builder::capture(const parser & p, const std::string & key, bool unescape_json) { + return action(p, [key, unescape_json](const parser_result &, std::string_view matched, parser_environment & env) { + std::string value = unescape_json ? unescape_json_string(matched) : std::string(matched); + env.scratchpad[key] = std::move(value); + }); +} + +parser parser_builder::capture_tool_call_id(const parser & p, bool unescape_json) { + return action(p, [unescape_json](const parser_result &, std::string_view matched, parser_environment & env) { + env.tool_call_id = unescape_json ? unescape_json_string(matched) : std::string(matched); + }); +} + +parser parser_builder::capture_tool_call_name(const parser & p, bool unescape_json) { + return action(p, [unescape_json](const parser_result &, std::string_view matched, parser_environment & env) { + env.tool_call_name = unescape_json ? unescape_json_string(matched) : std::string(matched); + }); +} + +parser parser_builder::capture_tool_call_args(const parser & p, bool unescape_json) { + return action(p, [unescape_json](const parser_result &, std::string_view matched, parser_environment & env) { + env.tool_call_args = unescape_json ? unescape_json_string(matched) : std::string(matched); + }); +} + +parser parser_builder::add_tool_call(const parser & p) { + return action(p, [](const parser_result &, std::string_view, parser_environment & env) { + auto tool_call = common_chat_tool_call{ + env.tool_call_name, + env.tool_call_args, + env.tool_call_id + }; + env.tool_calls.push_back(tool_call); + + // Clear the fields to prevent bleeding to next tool call + env.tool_call_id.clear(); + env.tool_call_name.clear(); + env.tool_call_args.clear(); + }); +} + parser parser_builder::json_key(const std::string & name, const parser & p) { return literal("\"" + name + "\"") << literal(":") << p; } diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index cb7da41740a75..98ca83f7bfb02 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -18,6 +18,13 @@ struct parser_environment { std::string content; std::string reasoning_content; std::vector tool_calls; + + // Tool call fields for building tool calls + std::string tool_call_id; + std::string tool_call_name; + std::string tool_call_args; + + // Scratch pad for any custom logic std::unordered_map> scratchpad; }; @@ -225,6 +232,7 @@ class parser_builder { // Specialized single-pass JSON string parser with escape sequence handling parser json_string(); + // TODO: improve convenience functions to allow users to build specific JSON fields parser json_key(const std::string & name, const parser & p); parser json_string(const parser & p); @@ -237,6 +245,34 @@ class parser_builder { // S -> A [action] parser action(const parser & p, std::function fn); + // Convenience action wrappers for common patterns + + // Appends matched text to env.reasoning_content + parser append_reasoning(const parser & p); + + // Appends matched text to env.content + parser append_content(const parser & p); + + // Captures matched text to env.scratchpad[key] + // If unescape_json is true, the matched text is unescaped as a JSON string + parser capture(const parser & p, const std::string & key, bool unescape_json = false); + + // Captures matched text to env.tool_call_id + // If unescape_json is true, the matched text is unescaped as a JSON string + parser capture_tool_call_id(const parser & p, bool unescape_json = false); + + // Captures matched text to env.tool_call_name + // If unescape_json is true, the matched text is unescaped as a JSON string + parser capture_tool_call_name(const parser & p, bool unescape_json = false); + + // Captures matched text to env.tool_call_args + // If unescape_json is true, the matched text is unescaped as a JSON string + parser capture_tool_call_args(const parser & p, bool unescape_json = false); + + // Adds a tool call to env.tool_calls using env.tool_call_{id,name,args} + // Clears the tool call fields after adding + parser add_tool_call(const parser & p); + parser add_rule(const std::string & name, const parser & p); void assign_ids(parser & p); diff --git a/tests/test-chat-parser-combinator.cpp b/tests/test-chat-parser-combinator.cpp index a309f7fce5a1c..69506ba0cc314 100644 --- a/tests/test-chat-parser-combinator.cpp +++ b/tests/test-chat-parser-combinator.cpp @@ -382,54 +382,24 @@ static void test_complete_example() { // // auto parser = build_parser([](parser_builder & p) { - auto handle_reasoning = [](const parser_result &, std::string_view match, parser_environment & env) { - env.reasoning_content += match; - }; - - auto handle_content = [](const parser_result &, std::string_view match, parser_environment & env) { - env.content += match; - }; - - auto handle_tool_call_name = [](const parser_result &, std::string_view match, parser_environment & env) { - env.scratchpad["tool_name"] = std::string(match); - }; - - auto handle_tool_call_args = [](const parser_result &, std::string_view match, parser_environment & env) { - env.scratchpad["tool_args"] = std::string(match); - }; - - auto handle_tool_call = [](const parser_result &, std::string_view, parser_environment & env) { - auto name = env.scratchpad.find("tool_name"); - auto args = env.scratchpad.find("tool_args"); - if (name != env.scratchpad.end() && args != env.scratchpad.end()) { - auto tool_call = common_chat_tool_call{ - std::get(name->second), - std::get(args->second), - std::string() - }; - - env.tool_calls.push_back(tool_call); - } - }; - auto reasoning = p.add_rule("reasoning", - "" << p.action(p.until(""), handle_reasoning) << ""); + "" << p.append_reasoning(p.until("")) << ""); auto content = p.add_rule("content", - p.action(p.until(""), handle_content)); + p.append_content(p.until(""))); auto json = p.json(); auto tool_call_name = p.add_rule("tool-call-name", - "" << p.action(p.until(""), handle_tool_call_name) << ""); + "" << p.capture_tool_call_name(p.until("")) << ""); auto schema = nlohmann::ordered_json::parse(R"({"type": "object"})"); auto tool_call_args = p.add_rule("tool-call-args", - "" << p.action(p.schema(json, "get_weather", schema), handle_tool_call_args) << ""); + "" << p.capture_tool_call_args(p.schema(json, "get_weather", schema)) << ""); auto tool_call = p.add_rule("tool-call", - "" << p.action(tool_call_name << tool_call_args, handle_tool_call) << ""); + "" << p.add_tool_call(tool_call_name << tool_call_args) << ""); return reasoning << p.optional(content) << p.optional(tool_call); }); @@ -509,8 +479,8 @@ static void test_complete_example() { assert_equals(true, result.is_success()); assert_equals("I need to call get_weather", env.reasoning_content); - assert_equals("get_weather", std::get(env.scratchpad["tool_name"])); - assert_equals(R"({"cit)", std::get(env.scratchpad["tool_args"])); + assert_equals("get_weather", env.tool_calls[0].name); + assert_equals(R"({"cit)", env.tool_calls[0].arguments); } auto gbnf = build_grammar([&](const common_grammar_builder & builder) { From f97abde571c5f978f704fec598ef246acdba9894 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Wed, 12 Nov 2025 03:42:27 -0600 Subject: [PATCH 031/183] add actions to command r7b example --- tests/test-chat-parser-combinator.cpp | 65 ++++++++++++++++++++------- 1 file changed, 50 insertions(+), 15 deletions(-) diff --git a/tests/test-chat-parser-combinator.cpp b/tests/test-chat-parser-combinator.cpp index 69506ba0cc314..0bfa0b45b6364 100644 --- a/tests/test-chat-parser-combinator.cpp +++ b/tests/test-chat-parser-combinator.cpp @@ -758,19 +758,25 @@ static void test_gbnf_generation() { static parser create_command_r7b_parser() { auto parser = build_parser([](parser_builder & p) { auto thinking = p.add_rule("thinking", - "<|START_THINKING|>" << p.until("<|END_THINKING|>") << "<|END_THINKING|>"); + "<|START_THINKING|>" << p.append_reasoning(p.until("<|END_THINKING|>")) << "<|END_THINKING|>"); auto response = p.add_rule("response", - "<|START_RESPONSE|>" << p.until("<|END_RESPONSE|>") << "<|END_RESPONSE|>"); + "<|START_RESPONSE|>" << p.append_content(p.until("<|END_RESPONSE|>")) << "<|END_RESPONSE|>"); auto json = p.add_rule("json", p.json()); - auto tool_call_id = p.add_rule("tool-call-id", p.json_key("tool_call_id", p.json_string(p.until("\"")))); - auto tool_call_name = p.add_rule("tool-name", p.json_key("tool_name", p.json_string(p.until("\"")))); - auto tool_call_args = p.add_rule("tool-args", p.json_key("parameters", json)); + + auto tool_call_id = p.add_rule("tool-call-id", + p.json_key("tool_call_id", "\"" + p.capture_tool_call_id(p.json_string(), /* unescape_json = */ true) + "\"")); + + auto tool_call_name = p.add_rule("tool-name", + p.json_key("tool_name", "\"" + p.capture_tool_call_name(p.json_string(), /* unescape_json = */ true) + "\"")); + + auto tool_call_args = p.add_rule("tool-args", p.json_key("parameters", p.capture_tool_call_args(json))); + auto tool_call_fields = p.add_rule("tool-call-fields", tool_call_id | tool_call_name | tool_call_args); auto tool_call = p.add_rule("tool-call", - "{" << tool_call_fields << p.zero_or_more(p.literal(",") << tool_call_fields) << "}"); + "{" << p.add_tool_call(tool_call_fields << p.zero_or_more(p.literal(",") << tool_call_fields)) << "}"); auto tool_calls = p.add_rule("tool-calls", "<|START_ACTION|>" @@ -789,12 +795,27 @@ static parser create_command_r7b_parser() { return parser; } -static void test_command_r7b_parser(const parser & p, const std::string & input, bool partial) { - parser_context ctx(input, !partial); +static void test_command_r7b_parser(const parser & p, const std::string & input, bool partial, bool print_results = false) { + parser_environment env; + parser_context ctx(input, &env, !partial); p.parse(ctx); + + if (print_results) { + std::cout << "== Parsed (new) ==\n"; + std::cout << "=== Reasoning ===\n"; + std::cout << env.reasoning_content << "\n"; + std::cout << "\n\n=== Content ===\n"; + std::cout << env.content << "\n"; + std::cout << "\n\n=== Tool Calls ===\n"; + for (const auto & tc : env.tool_calls) { + std::cout << "id: " << tc.id << "\n"; + std::cout << "name: " << tc.name << "\n"; + std::cout << "args: " << tc.arguments << "\n"; + } + } } -static void test_command_r7b_legacy_parser(const std::string & input, bool partial) { +static void test_command_r7b_legacy_parser(const std::string & input, bool partial, bool print_results = false) { // Original parser taken from chat.cpp common_chat_msg_parser builder(input, /* is_partial= */ partial, { @@ -834,6 +855,20 @@ static void test_command_r7b_legacy_parser(const std::string & input, bool parti } else { builder.add_content(builder.consume_rest()); } + + if (print_results) { + std::cout << "== Parsed (legacy) ==\n"; + std::cout << "=== Reasoning ===\n"; + std::cout << builder.result().reasoning_content << "\n"; + std::cout << "\n\n=== Content ===\n"; + std::cout << builder.result().content << "\n"; + std::cout << "\n\n=== Tool Calls ===\n"; + for (const auto & tc : builder.result().tool_calls) { + std::cout << "id: " << tc.id << "\n"; + std::cout << "name: " << tc.name << "\n"; + std::cout << "args: " << tc.arguments << "\n"; + } + } } struct bench_tool_call { @@ -907,13 +942,13 @@ static void benchmark_compare( tokens.emplace_back("<|END_ACTION|>"); } - auto run = [&](const std::function & fn) { + auto run = [&](const std::function & fn) { std::string input = std::accumulate(tokens.begin(), tokens.end(), std::string()); std::chrono::microseconds duration(0); for (int i = 0; i < iterations; i++) { auto start = std::chrono::high_resolution_clock::now(); - fn(input, false); + fn(input, false, i == 0); auto end = std::chrono::high_resolution_clock::now(); duration += std::chrono::duration_cast(end - start); } @@ -922,13 +957,13 @@ static void benchmark_compare( auto parser = create_command_r7b_parser(); - auto duration_new = run([&](const std::string & input, bool partial) { - test_command_r7b_parser(parser, input, partial); + auto duration_new = run([&](const std::string & input, bool partial, bool print_content) { + test_command_r7b_parser(parser, input, partial, print_content); }); - auto duration_legacy = run([&](const std::string & input, bool partial) { + auto duration_legacy = run([&](const std::string & input, bool partial, bool print_content) { try { - test_command_r7b_legacy_parser(input, partial); + test_command_r7b_legacy_parser(input, partial, print_content); } catch (const common_chat_msg_partial_exception &) { } }); From 3114a0e679d3447a672fb1a20c81e844afa1301d Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Wed, 12 Nov 2025 03:53:41 -0600 Subject: [PATCH 032/183] use std::default_searcher for platforms that don't have bm --- common/chat-parser-combinator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index 60cb54f84cc1f..331f00198ea56 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -696,7 +696,7 @@ class until_parser : public parser_base { std::string delimiter_; bool consume_spaces_; - std::boyer_moore_searcher searcher_; + std::default_searcher searcher_; public: until_parser(const std::string & delimiter, bool consume_spaces, int id) From cc4d52c05983e7146280d589e4c9540a43923bd3 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Wed, 12 Nov 2025 22:56:16 -0600 Subject: [PATCH 033/183] improve parser_type handling and add cast helper --- common/chat-parser-combinator.cpp | 124 ++++++++++++++++++++---------- 1 file changed, 84 insertions(+), 40 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index 331f00198ea56..f8b5dbaa32109 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -10,23 +10,23 @@ #include enum parser_type { - PARSER_LITERAL = 0, - PARSER_SEQUENCE = 1, - PARSER_CHOICE = 2, - PARSER_REPETITION = 3, - PARSER_OPTIONAL = 4, - PARSER_ZERO_OR_MORE = 5, - PARSER_ONE_OR_MORE = 6, - PARSER_NOT = 7, - PARSER_ANY = 8, - PARSER_CHARS = 9, - PARSER_RULE = 10, - PARSER_UNTIL = 11, - PARSER_SPACE = 12, - PARSER_SCHEMA = 13, - PARSER_ROOT = 14, - PARSER_JSON_STRING = 15, - PARSER_ACTION = 16, + PARSER_LITERAL, + PARSER_SEQUENCE, + PARSER_CHOICE, + PARSER_REPETITION, + PARSER_OPTIONAL, + PARSER_ZERO_OR_MORE, + PARSER_ONE_OR_MORE, + PARSER_NOT, + PARSER_ANY, + PARSER_CHARS, + PARSER_RULE, + PARSER_UNTIL, + PARSER_SPACE, + PARSER_SCHEMA, + PARSER_ROOT, + PARSER_JSON_STRING, + PARSER_ACTION, }; class parser_visitor; @@ -75,6 +75,20 @@ class parser_base { virtual void accept(parser_visitor & visitor) = 0; }; +// Convenience cast functions +template +static std::shared_ptr cast(const std::shared_ptr & p) { + if (p->type() != T::type_value) { + return nullptr; + } + return std::static_pointer_cast(p); +} + +template +static std::shared_ptr cast(const parser & p) { + return cast(p.ptr()); +} + // We define our own space function because MSVC's std::isspace() // crashes for non-printable characters in Debug builds. static bool is_space(const char c) { @@ -107,9 +121,11 @@ class literal_parser : public parser_base { std::string literal_; public: + static constexpr parser_type type_value = PARSER_LITERAL; + literal_parser(const std::string & literal, int id) : parser_base(id), literal_(literal) {} - parser_type type() const override { return PARSER_LITERAL; } + parser_type type() const override { return type_value; } parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { auto pos = start; @@ -147,11 +163,11 @@ class sequence_parser : public parser_base { std::vector parsers_; public: + static constexpr parser_type type_value = PARSER_SEQUENCE; + sequence_parser(std::initializer_list parsers, int id) : parser_base(id) { for (const auto & p : parsers) { - if (p->type() == PARSER_SEQUENCE) { - // Flatten sequences - auto seq = std::static_pointer_cast(p.ptr()); + if (auto seq = cast(p)) { for (const auto & embedded : seq->parsers()) { parsers_.push_back(embedded); } @@ -161,7 +177,7 @@ class sequence_parser : public parser_base { } } - parser_type type() const override { return PARSER_SEQUENCE; } + parser_type type() const override { return type_value; } parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { auto pos = start; @@ -213,11 +229,11 @@ class choice_parser : public parser_base { std::vector parsers_; public: + static constexpr parser_type type_value = PARSER_CHOICE; + choice_parser(std::initializer_list parsers, int id) : parser_base(id) { for (const auto & p : parsers) { - if (p->type() == PARSER_CHOICE) { - // Flatten choices - auto choice = std::static_pointer_cast(p.ptr()); + if (auto choice = cast(p)) { for (const auto & embedded : choice->parsers()) { parsers_.push_back(embedded); } @@ -227,7 +243,7 @@ class choice_parser : public parser_base { } } - parser_type type() const override { return PARSER_CHOICE; } + parser_type type() const override { return type_value; } parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { auto pos = start; @@ -276,10 +292,12 @@ class repetition_parser : public parser_base { int max_count_; public: + static constexpr parser_type type_value = PARSER_REPETITION; + repetition_parser(const parser & parser, int min_count, int max_count, int id) : parser_base(id), parser_(parser), min_count_(min_count), max_count_(max_count) {} - parser_type type() const override { return PARSER_REPETITION; } + parser_type type() const override { return type_value; } parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { auto pos = start; @@ -340,9 +358,11 @@ class repetition_parser : public parser_base { // S -> A+ class one_or_more_parser : public repetition_parser { public: + static constexpr parser_type type_value = PARSER_ONE_OR_MORE; + one_or_more_parser(const parser & p, int id) : repetition_parser(p, 1, -1, id) {} - parser_type type() const override { return PARSER_ONE_OR_MORE; } + parser_type type() const override { return type_value; } std::string dump() const override { return "OneOrMore(" + child()->dump() + ")"; @@ -355,9 +375,11 @@ class one_or_more_parser : public repetition_parser { // S -> A* class zero_or_more_parser : public repetition_parser { public: + static constexpr parser_type type_value = PARSER_ZERO_OR_MORE; + zero_or_more_parser(const parser & p, int id) : repetition_parser(p, 0, -1, id) {} - parser_type type() const override { return PARSER_ZERO_OR_MORE; } + parser_type type() const override { return type_value; } std::string dump() const override { return "ZeroOrMore(" + child()->dump() + ")"; @@ -370,9 +392,11 @@ class zero_or_more_parser : public repetition_parser { // S -> A? class optional_parser : public repetition_parser { public: + static constexpr parser_type type_value = PARSER_OPTIONAL; + optional_parser(const parser & p, int id) : repetition_parser(p, 0, 1, id) {} - parser_type type() const override { return PARSER_OPTIONAL; } + parser_type type() const override { return type_value; } std::string dump() const override { return "Optional(" + child()->dump() + ")"; @@ -387,9 +411,11 @@ class not_parser : public parser_base { parser parser_; public: + static constexpr parser_type type_value = PARSER_NOT; + not_parser(const parser & parser, int id) : parser_base(id), parser_(parser) {} - parser_type type() const override { return PARSER_NOT; } + parser_type type() const override { return type_value; } parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { auto result = parser_->parse(ctx, start); @@ -426,9 +452,11 @@ class not_parser : public parser_base { // S -> . class any_parser : public parser_base { public: + static constexpr parser_type type_value = PARSER_ANY; + any_parser(int id) : parser_base(id) {} - parser_type type() const override { return PARSER_ANY; } + parser_type type() const override { return type_value; } parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { if (start >= ctx.input.size()) { @@ -452,9 +480,11 @@ class any_parser : public parser_base { // S -> [ \t\n]* class space_parser : public parser_base { public: + static constexpr parser_type type_value = PARSER_SPACE; + space_parser(int id) : parser_base(id) {} - parser_type type() const override { return PARSER_SPACE; } + parser_type type() const override { return type_value; } parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { auto pos = start; @@ -545,7 +575,9 @@ class chars_parser : public parser_base { } } - parser_type type() const override { return PARSER_CHARS; } + static constexpr parser_type type_value = PARSER_CHARS; + + parser_type type() const override { return type_value; } parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { auto pos = start; @@ -610,9 +642,11 @@ class chars_parser : public parser_base { class json_string_parser : public parser_base { public: + static constexpr parser_type type_value = PARSER_JSON_STRING; + json_string_parser(int id) : parser_base(id) {} - parser_type type() const override { return PARSER_JSON_STRING; } + parser_type type() const override { return type_value; } parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { auto pos = start; @@ -699,11 +733,13 @@ class until_parser : public parser_base { std::default_searcher searcher_; public: + static constexpr parser_type type_value = PARSER_UNTIL; + until_parser(const std::string & delimiter, bool consume_spaces, int id) : parser_base(id), delimiter_(delimiter), consume_spaces_(consume_spaces), searcher_(delimiter_.begin(), delimiter_.end()) { } - parser_type type() const override { return PARSER_UNTIL; } + parser_type type() const override { return type_value; } parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { parser_result result(PARSER_RESULT_SUCCESS, start, ctx.input.size()); @@ -752,10 +788,12 @@ class schema_parser : public parser_base { nlohmann::ordered_json schema_; public: + static constexpr parser_type type_value = PARSER_SCHEMA; + schema_parser(const parser & parser, const std::string & name, const nlohmann::ordered_json & schema, int id) : parser_base(id), parser_(parser), name_(name), schema_(schema) {} - parser_type type() const override { return PARSER_SCHEMA; } + parser_type type() const override { return type_value; } parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { return parser_->parse(ctx, start); @@ -781,10 +819,12 @@ class rule_parser : public parser_base { std::weak_ptr> rules_; public: + static constexpr parser_type type_value = PARSER_RULE; + rule_parser(const std::string & name, const std::shared_ptr> & rules, int id) : parser_base(id), name_(name), rules_(rules) {} - parser_type type() const override { return PARSER_RULE; } + parser_type type() const override { return type_value; } parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { auto rules = rules_.lock(); @@ -820,10 +860,12 @@ class root_parser : public parser_base { friend class parser_visitor; public: + static constexpr parser_type type_value = PARSER_ROOT; + root_parser(const parser & root, std::shared_ptr> rules, int id) : parser_base(id), root_(root), rules_(std::move(rules)) {} - parser_type type() const override { return PARSER_ROOT; } + parser_type type() const override { return type_value; } parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { return root_->parse(ctx, start); @@ -851,10 +893,12 @@ class action_parser : public parser_base { std::function action_; public: + static constexpr parser_type type_value = PARSER_ACTION; + action_parser(const parser & parser, std::function action, int id) : parser_base(id), parser_(parser), action_(std::move(action)) {} - parser_type type() const override { return PARSER_ACTION; } + parser_type type() const override { return type_value; } parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { auto result = parser_->parse(ctx, start); From c119c1290a087aaa67cd7df8ea3efbda02213869 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Thu, 13 Nov 2025 02:14:21 -0600 Subject: [PATCH 034/183] add partial result type to better control when to run actions --- common/chat-parser-combinator.cpp | 109 ++++++++++++++------------ common/chat-parser-combinator.h | 13 ++- tests/test-chat-parser-combinator.cpp | 24 +++--- 3 files changed, 79 insertions(+), 67 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index f8b5dbaa32109..0a754bc012c58 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -134,10 +134,7 @@ class literal_parser : public parser_base { if (ctx.input_is_complete) { return parser_result(PARSER_RESULT_FAIL, start); } - if (i > 0) { - return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, pos); - } - return parser_result(PARSER_RESULT_FAIL, start, pos); + return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, pos); } if (ctx.input[pos] != literal_[i]) { return parser_result(PARSER_RESULT_FAIL, start); @@ -183,17 +180,8 @@ class sequence_parser : public parser_base { auto pos = start; for (const auto & p : parsers_) { auto result = p->parse(ctx, pos); - - if (result.is_fail()) { - if (result.end >= ctx.input.size() && !ctx.input_is_complete) { - // If we fail because we don't have enough input, then return success - return parser_result(PARSER_RESULT_SUCCESS, start, result.end); - } - return parser_result(PARSER_RESULT_FAIL, start, result.end); - } - - if (result.is_need_more_input()) { - return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, result.end); + if (!result.is_success()) { + return parser_result(result.type, start, result.end); } pos = result.end; @@ -249,12 +237,7 @@ class choice_parser : public parser_base { auto pos = start; for (const auto & p : parsers_) { auto result = p->parse(ctx, pos); - - if (result.is_success()) { - return result; - } - - if (result.is_need_more_input()) { + if (!result.is_fail()) { return result; } } @@ -305,6 +288,10 @@ class repetition_parser : public parser_base { // Try to match up to max_count times (or unlimited if max_count is -1) while (max_count_ == -1 || match_count < max_count_) { + if (pos >= ctx.input.size()) { + break; + } + auto result = parser_->parse(ctx, pos); if (result.is_success()) { @@ -317,8 +304,8 @@ class repetition_parser : public parser_base { continue; } - if (result.is_need_more_input()) { - return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, pos); + if (result.is_need_more_input() || result.is_partial()) { + return parser_result(result.type, start, result.end); } // Child failed - stop trying @@ -420,7 +407,7 @@ class not_parser : public parser_base { parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { auto result = parser_->parse(ctx, start); - if (result.is_success()) { + if (result.is_success() || result.is_partial()) { // Fail if the underlying parser matches return parser_result(PARSER_RESULT_FAIL, start); } @@ -463,9 +450,8 @@ class any_parser : public parser_base { if (ctx.input_is_complete) { return parser_result(PARSER_RESULT_FAIL, start); } - return parser_result(PARSER_RESULT_FAIL, start); + return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start); } - return parser_result(PARSER_RESULT_SUCCESS, start, start + 1); } @@ -612,7 +598,10 @@ class chars_parser : public parser_base { // Check if we got enough matches if (match_count < min_count_) { - return parser_result(PARSER_RESULT_FAIL, start); + if (pos >= ctx.input.size() && !ctx.input_is_complete) { + return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, pos); + } + return parser_result(PARSER_RESULT_FAIL, start, pos); } return parser_result(PARSER_RESULT_SUCCESS, start, pos); @@ -714,7 +703,10 @@ class json_string_parser : public parser_base { } // Reached end without finding closing quote - return parser_result(PARSER_RESULT_FAIL, start, pos); + if (ctx.input_is_complete) { + return parser_result(PARSER_RESULT_FAIL, start, pos); + } + return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, pos); } std::string dump() const override { @@ -748,15 +740,19 @@ class until_parser : public parser_base { const auto it = std::search(ctx.input.begin(), ctx.input.end(), searcher_); if (it != ctx.input.end()) { - result.type = PARSER_RESULT_SUCCESS; result.end = std::distance(ctx.input.begin(), it); + result.type = PARSER_RESULT_SUCCESS; } else { // If not found, check if the input ends with a prefix of the delimiter size_t max_overlap = std::min(ctx.input.size(), delimiter_.size() - 1); for (size_t overlap = max_overlap; overlap > 0; --overlap) { if (std::equal(ctx.input.end() - overlap, ctx.input.end(), delimiter_.begin())) { - result.type = (ctx.input_is_complete) ? PARSER_RESULT_FAIL : PARSER_RESULT_NEED_MORE_INPUT; result.end = ctx.input.size() - overlap; + if (ctx.input_is_complete) { + result.type = PARSER_RESULT_FAIL; + } else { + result.type = PARSER_RESULT_NEED_MORE_INPUT; + } } } } @@ -890,21 +886,25 @@ class root_parser : public parser_base { // Wraps a parser with a semantic action callback. class action_parser : public parser_base { parser parser_; - std::function action_; + std::function action_; + int when_; public: static constexpr parser_type type_value = PARSER_ACTION; - action_parser(const parser & parser, std::function action, int id) - : parser_base(id), parser_(parser), action_(std::move(action)) {} + action_parser( + const parser & parser, + std::function action, + int when, + int id + ) : parser_base(id), parser_(parser), action_(std::move(action)), when_(when) {} parser_type type() const override { return type_value; } parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { auto result = parser_->parse(ctx, start); - // Invoke action callback on success if environment is available - if (result.is_success() && ctx.env && action_) { + if ((result.type & when_) && ctx.env && action_) { std::string_view matched = ctx.input.substr(result.start, result.end - result.start); action_(result, matched, *ctx.env); } @@ -918,7 +918,7 @@ class action_parser : public parser_base { } std::string dump() const override { - return "Action(" + parser_->dump() + ")"; + return "Action(" + parser_->dump() + ", when=" + std::to_string(when_) +")"; } void accept(parser_visitor & visitor) override; @@ -926,6 +926,7 @@ class action_parser : public parser_base { const parser & child() const { return parser_; } }; + // Base visitor class for parser tree traversal class parser_visitor { public: @@ -1384,51 +1385,57 @@ parser parser_builder::schema(const parser & p, const std::string & name, const return parser(std::make_shared(p, name, schema, counter_->next())); } -parser parser_builder::action(const parser & p, std::function fn) { - return parser(std::make_shared(p, std::move(fn), counter_->next())); +parser parser_builder::action(const parser & p, std::function fn, int when) { + return parser(std::make_shared(p, std::move(fn), when, counter_->next())); +} + +parser parser_builder::partial(const parser & p) { + return action(p, [](parser_result &result, std::string_view, parser_environment &) { + result.type = PARSER_RESULT_PARTIAL; + }, PARSER_RESULT_NEED_MORE_INPUT); } parser parser_builder::append_reasoning(const parser & p) { - return action(p, [](const parser_result &, std::string_view matched, parser_environment & env) { + return action(p, [](parser_result &, std::string_view matched, parser_environment & env) { if (!env.reasoning_content.empty()) { env.reasoning_content += "\n"; } env.reasoning_content += matched; - }); + }, PARSER_RESULT_SUCCESS | PARSER_RESULT_PARTIAL); } parser parser_builder::append_content(const parser & p) { - return action(p, [](const parser_result &, std::string_view matched, parser_environment & env) { + return action(p, [](parser_result &, std::string_view matched, parser_environment & env) { if (!env.content.empty()) { env.content += "\n"; } env.content += matched; - }); + }, PARSER_RESULT_SUCCESS | PARSER_RESULT_PARTIAL); } parser parser_builder::capture(const parser & p, const std::string & key, bool unescape_json) { - return action(p, [key, unescape_json](const parser_result &, std::string_view matched, parser_environment & env) { + return action(p, [key, unescape_json](parser_result &, std::string_view matched, parser_environment & env) { std::string value = unescape_json ? unescape_json_string(matched) : std::string(matched); env.scratchpad[key] = std::move(value); - }); + }, PARSER_RESULT_SUCCESS | PARSER_RESULT_PARTIAL); } parser parser_builder::capture_tool_call_id(const parser & p, bool unescape_json) { - return action(p, [unescape_json](const parser_result &, std::string_view matched, parser_environment & env) { + return action(p, [unescape_json](parser_result &, std::string_view matched, parser_environment & env) { env.tool_call_id = unescape_json ? unescape_json_string(matched) : std::string(matched); - }); + }, PARSER_RESULT_SUCCESS | PARSER_RESULT_PARTIAL); } parser parser_builder::capture_tool_call_name(const parser & p, bool unescape_json) { - return action(p, [unescape_json](const parser_result &, std::string_view matched, parser_environment & env) { + return action(p, [unescape_json](parser_result &, std::string_view matched, parser_environment & env) { env.tool_call_name = unescape_json ? unescape_json_string(matched) : std::string(matched); - }); + }, PARSER_RESULT_SUCCESS | PARSER_RESULT_PARTIAL); } parser parser_builder::capture_tool_call_args(const parser & p, bool unescape_json) { - return action(p, [unescape_json](const parser_result &, std::string_view matched, parser_environment & env) { + return action(p, [unescape_json](parser_result &, std::string_view matched, parser_environment & env) { env.tool_call_args = unescape_json ? unescape_json_string(matched) : std::string(matched); - }); + }, PARSER_RESULT_SUCCESS | PARSER_RESULT_PARTIAL); } parser parser_builder::add_tool_call(const parser & p) { @@ -1444,7 +1451,7 @@ parser parser_builder::add_tool_call(const parser & p) { env.tool_call_id.clear(); env.tool_call_name.clear(); env.tool_call_args.clear(); - }); + }, PARSER_RESULT_SUCCESS | PARSER_RESULT_PARTIAL); } parser parser_builder::json_key(const std::string & name, const parser & p) { diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index 98ca83f7bfb02..1a38fde861dcc 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -29,9 +29,10 @@ struct parser_environment { }; enum parser_result_type { - PARSER_RESULT_FAIL = 0, - PARSER_RESULT_NEED_MORE_INPUT = 1, - PARSER_RESULT_SUCCESS = 2, + PARSER_RESULT_FAIL = 1 << 0, + PARSER_RESULT_SUCCESS = 1 << 1, + PARSER_RESULT_NEED_MORE_INPUT = 1 << 2, + PARSER_RESULT_PARTIAL = 1 << 3, }; struct parse_cache_key { @@ -65,6 +66,7 @@ struct parser_result { bool is_fail() const { return type == PARSER_RESULT_FAIL; } bool is_need_more_input() const { return type == PARSER_RESULT_NEED_MORE_INPUT; } + bool is_partial() const { return type == PARSER_RESULT_PARTIAL; } bool is_success() const { return type == PARSER_RESULT_SUCCESS; } }; @@ -243,10 +245,13 @@ class parser_builder { // Wraps a parser with a semantic action callback. // The callback is invoked on successful parse with the result, matched text, and environment. // S -> A [action] - parser action(const parser & p, std::function fn); + parser action(const parser & p, std::function fn, int when = PARSER_RESULT_SUCCESS); // Convenience action wrappers for common patterns + // Converts PARSER_RESULT_NEED_MORE_INPUT to PARSER_RESULT_PARTIAL + parser partial(const parser & p); + // Appends matched text to env.reasoning_content parser append_reasoning(const parser & p); diff --git a/tests/test-chat-parser-combinator.cpp b/tests/test-chat-parser-combinator.cpp index 0bfa0b45b6364..2affdd7718a0c 100644 --- a/tests/test-chat-parser-combinator.cpp +++ b/tests/test-chat-parser-combinator.cpp @@ -90,7 +90,7 @@ static void test_partial_parsing() { ctx = parser_context("", false); result = parser.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.is_need_more_input()); ctx = parser_context("" << p.capture_tool_call_args(p.schema(json, "get_weather", schema)) << ""); + "" << p.capture_tool_call_args(p.schema(p.partial(json), "get_weather", schema)) << ""); auto tool_call = p.add_rule("tool-call", - "" << p.add_tool_call(tool_call_name << tool_call_args) << ""); + "" << p.add_tool_call(tool_call_name << p.partial(tool_call_args)) << ""); return reasoning << p.optional(content) << p.optional(tool_call); }); @@ -429,7 +429,7 @@ static void test_complete_example() { auto result = parser.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.is_need_more_input()); assert_equals("I need to call get_weather", env.reasoning_content); } { @@ -457,7 +457,7 @@ static void test_complete_example() { auto result = parser.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.is_need_more_input()); assert_equals("I need to call get_weather", env.reasoning_content); } { @@ -477,7 +477,7 @@ static void test_complete_example() { auto result = parser.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.is_partial()); assert_equals("I need to call get_weather", env.reasoning_content); assert_equals("get_weather", env.tool_calls[0].name); assert_equals(R"({"cit)", env.tool_calls[0].arguments); @@ -579,7 +579,7 @@ static void test_actions() { parser_context ctx("hello ", &env, false); auto result = parser.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.is_need_more_input()); assert_equals("hello", env.content); } { @@ -587,7 +587,7 @@ static void test_actions() { parser_context ctx("hello world", &env, false); auto result = parser.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.is_need_more_input()); assert_equals("hello world", env.content); } { From 39d1095e4ff9c0c7d3848436cc2b1c68bcf8003f Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Thu, 13 Nov 2025 03:05:51 -0600 Subject: [PATCH 035/183] fix bug in until() --- common/chat-parser-combinator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index 0a754bc012c58..83967ec4b9f20 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -737,7 +737,7 @@ class until_parser : public parser_base { parser_result result(PARSER_RESULT_SUCCESS, start, ctx.input.size()); // Search for the delimiter - const auto it = std::search(ctx.input.begin(), ctx.input.end(), searcher_); + const auto it = std::search(ctx.input.begin() + start, ctx.input.end(), searcher_); if (it != ctx.input.end()) { result.end = std::distance(ctx.input.begin(), it); From eabdb85e5492dec3bdad9104c19ab08fa98b7984 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Thu, 13 Nov 2025 03:06:04 -0600 Subject: [PATCH 036/183] run actions on partial results by default --- common/chat-parser-combinator.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index 1a38fde861dcc..c2d6953586270 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -245,7 +245,7 @@ class parser_builder { // Wraps a parser with a semantic action callback. // The callback is invoked on successful parse with the result, matched text, and environment. // S -> A [action] - parser action(const parser & p, std::function fn, int when = PARSER_RESULT_SUCCESS); + parser action(const parser & p, std::function fn, int when = PARSER_RESULT_SUCCESS | PARSER_RESULT_PARTIAL); // Convenience action wrappers for common patterns From 692ade29a62ad794d178705783a5e85ec7dc59aa Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Thu, 13 Nov 2025 04:28:19 -0600 Subject: [PATCH 037/183] use common_chat_msg for result --- common/chat-parser-combinator.cpp | 14 +++++++------- common/chat-parser-combinator.h | 8 +++----- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index 83967ec4b9f20..0ffc5a0617220 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -1397,19 +1397,19 @@ parser parser_builder::partial(const parser & p) { parser parser_builder::append_reasoning(const parser & p) { return action(p, [](parser_result &, std::string_view matched, parser_environment & env) { - if (!env.reasoning_content.empty()) { - env.reasoning_content += "\n"; + if (!env.result.reasoning_content.empty()) { + env.result.reasoning_content += "\n"; } - env.reasoning_content += matched; + env.result.reasoning_content += matched; }, PARSER_RESULT_SUCCESS | PARSER_RESULT_PARTIAL); } parser parser_builder::append_content(const parser & p) { return action(p, [](parser_result &, std::string_view matched, parser_environment & env) { - if (!env.content.empty()) { - env.content += "\n"; + if (!env.result.content.empty()) { + env.result.content += "\n"; } - env.content += matched; + env.result.content += matched; }, PARSER_RESULT_SUCCESS | PARSER_RESULT_PARTIAL); } @@ -1445,7 +1445,7 @@ parser parser_builder::add_tool_call(const parser & p) { env.tool_call_args, env.tool_call_id }; - env.tool_calls.push_back(tool_call); + env.result.tool_calls.push_back(tool_call); // Clear the fields to prevent bleeding to next tool call env.tool_call_id.clear(); diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index c2d6953586270..1c0dd01d5a7b4 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -1,5 +1,7 @@ #pragma once +#include "chat.h" + #include #include @@ -9,15 +11,11 @@ #include #include #include -#include -struct common_chat_tool_call; struct common_grammar_builder; struct parser_environment { - std::string content; - std::string reasoning_content; - std::vector tool_calls; + common_chat_msg result; // Tool call fields for building tool calls std::string tool_call_id; From 6478050c0a7739af71032e91e19f8a6a567e59ec Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Thu, 13 Nov 2025 04:28:36 -0600 Subject: [PATCH 038/183] add qwen3 example wip --- tests/test-chat-parser-combinator.cpp | 218 +++++++++++++++++++++++--- 1 file changed, 193 insertions(+), 25 deletions(-) diff --git a/tests/test-chat-parser-combinator.cpp b/tests/test-chat-parser-combinator.cpp index 2affdd7718a0c..e227dfb6748c7 100644 --- a/tests/test-chat-parser-combinator.cpp +++ b/tests/test-chat-parser-combinator.cpp @@ -414,11 +414,11 @@ static void test_complete_example() { assert_equals(true, result.is_success()); assert_equals(input.size(), result.end); - assert_equals("I need to call get_weather with city = New York", env.reasoning_content); - assert_equals((size_t)1, env.tool_calls.size()); - assert_equals("", env.tool_calls[0].id); - assert_equals("get_weather", env.tool_calls[0].name); - assert_equals(R"({"city": "New York"})", env.tool_calls[0].arguments); + assert_equals("I need to call get_weather with city = New York", env.result.reasoning_content); + assert_equals((size_t)1, env.result.tool_calls.size()); + assert_equals("", env.result.tool_calls[0].id); + assert_equals("get_weather", env.result.tool_calls[0].name); + assert_equals(R"({"city": "New York"})", env.result.tool_calls[0].arguments); } // Test partial input @@ -430,7 +430,7 @@ static void test_complete_example() { auto result = parser.parse(ctx); assert_equals(true, result.is_need_more_input()); - assert_equals("I need to call get_weather", env.reasoning_content); + assert_equals("I need to call get_weather", env.result.reasoning_content); } { std::string input = R"(I need to call I need to call get_weatherget_weatherI need to call get_weatherget_weather{"cit)"; @@ -478,9 +478,9 @@ static void test_complete_example() { auto result = parser.parse(ctx); assert_equals(true, result.is_partial()); - assert_equals("I need to call get_weather", env.reasoning_content); - assert_equals("get_weather", env.tool_calls[0].name); - assert_equals(R"({"cit)", env.tool_calls[0].arguments); + assert_equals("I need to call get_weather", env.result.reasoning_content); + assert_equals("get_weather", env.result.tool_calls[0].name); + assert_equals(R"({"cit)", env.result.tool_calls[0].arguments); } auto gbnf = build_grammar([&](const common_grammar_builder & builder) { @@ -496,7 +496,7 @@ static void test_actions() { auto parser = build_parser([](parser_builder& p) { auto word = p.chars("[a-z]+"); return p.action(word, [](const parser_result &, std::string_view matched, parser_environment & env) { - env.content += std::string(matched); + env.result.content += std::string(matched); }); }); @@ -505,17 +505,17 @@ static void test_actions() { auto result = parser.parse(ctx); assert_equals(true, result.is_success()); - assert_equals("hello", env.content); + assert_equals("hello", env.result.content); } { // Test multiple sequential actions - build a sentence auto parser = build_parser([](parser_builder& p) { auto greeting = p.action(p.literal("hello"), [](const parser_result &, std::string_view matched, parser_environment & env) { - env.content += std::string(matched) + " "; + env.result.content += std::string(matched) + " "; }); auto name = p.action(p.chars("[A-Z][a-z]+"), [](const parser_result &, std::string_view matched, parser_environment & env) { - env.content += std::string(matched); + env.result.content += std::string(matched); env.scratchpad["name"] = std::string(matched); }); @@ -527,7 +527,7 @@ static void test_actions() { auto result = parser.parse(ctx); assert_equals(true, result.is_success()); - assert_equals("hello Alice", env.content); + assert_equals("hello Alice", env.result.content); assert_equals("Alice", std::get(env.scratchpad["name"])); } { @@ -554,7 +554,7 @@ static void test_actions() { // Test actions don't run when parse fails auto parser = build_parser([](parser_builder& p) { return p.action(p.literal("success"), [](const parser_result &, std::string_view, parser_environment & env) { - env.content = "action_ran"; + env.result.content = "action_ran"; }); }); @@ -563,13 +563,13 @@ static void test_actions() { auto result = parser.parse(ctx); assert_equals(true, result.is_fail()); - assert_equals("", env.content); // Action should not have run + assert_equals("", env.result.content); // Action should not have run } { // Test Actions work with partial parsing auto parser = build_parser([](parser_builder& p) { auto content = p.action(p.until(""), [](const parser_result &, std::string_view matched, parser_environment & env) { - env.content += std::string(matched); + env.result.content += std::string(matched); }); return "" << content << ""; }); @@ -580,7 +580,7 @@ static void test_actions() { auto result = parser.parse(ctx); assert_equals(true, result.is_need_more_input()); - assert_equals("hello", env.content); + assert_equals("hello", env.result.content); } { parser_environment env; @@ -588,7 +588,7 @@ static void test_actions() { auto result = parser.parse(ctx); assert_equals(true, result.is_need_more_input()); - assert_equals("hello world", env.content); + assert_equals("hello world", env.result.content); } { parser_environment env; @@ -596,7 +596,7 @@ static void test_actions() { auto result = parser.parse(ctx); assert_equals(true, result.is_success()); - assert_equals("hello world", env.content); + assert_equals("hello world", env.result.content); } } } @@ -755,6 +755,172 @@ static void test_gbnf_generation() { } } +static void example_qwen3_coder() { + auto parser = build_parser([](parser_builder & p) { + auto thinking = p.add_rule("thinking", + "" << p.append_reasoning(p.until("")) << ""); + + auto content = p.add_rule("content", p.append_content(p.until(""))); + + auto arg_start = p.add_rule("arg-start", + p.action("", [](const parser_result &, std::string_view, parser_environment & env) { + env.tool_call_args += "\":"; + })); + + auto arg_end = p.add_rule("arg-end", ""); + + auto string_arg = p.add_rule("arg-string", + p.action(arg_start, [&](const parser_result &, std::string_view, parser_environment & env) { + env.tool_call_args += "{"; + }) + << p.action(p.until(""), [&](const parser_result &, std::string_view match, parser_environment & env) { + // TODO: add a JSON escape helper + env.tool_call_args += std::string(match); + }) + << p.action(arg_end, [&](const parser_result &, std::string_view, parser_environment & env) { + env.tool_call_args += "\""; + })); + + auto json = p.json(); + + auto json_arg = p.add_rule("arg-json", + arg_start + << p.action(p.partial(json), [&](const parser_result &, std::string_view match, parser_environment & env) { + // JSON should already be properly formatted + env.tool_call_args += std::string(match); + }) + << arg_end); + + auto function = p.add_rule("function", p.add_tool_call( + "", [&](const parser_result &, std::string_view, parser_environment & env) { + env.tool_call_args += "{"; + }) + + p.one_or_more(p.space() + (p.partial(json_arg) | p.partial(string_arg))) + << p.action("", [&](const parser_result &, std::string_view, parser_environment & env) { + env.tool_call_args += "}"; + }))); + + auto tool_call = p.add_rule("tool-call", + "" << p.one_or_more(function) << ""); + + + return p.partial(thinking) + p.optional(p.space() + p.partial(content)) + p.zero_or_more(p.space() + tool_call); + }); + + std::string input = + "The user wants to find large log files that haven't been accessed recently. " + "I should search for files with .log extension, filter by size (over 100MB), " + "and check access time within the last 30 days. I'll need to use the search_files function." + "Based on your requirements, I'll search for log files over 100MB that haven't been " + "accessed in the last month. This will help identify candidates for cleanup or archival.\n\n" + "\n" + "\n" + "/var/log\n" + "*.log\n" + "100\n" + "5\n" + "false\n" + "30\n" + "true\n" + "size\n" + "{\"exclude_patterns\": [\"*temp*\", \"*cache*\"], \"file_types\": [\"regular\"]}\n" + "\n" + ""; + + static const std::regex token_regex(R"(([A-Za-z0-9]+|[^A-Za-z0-9]+))"); + std::vector tokens; + std::sregex_iterator it(input.begin(), input.end(), token_regex); + std::sregex_iterator end; + for (; it != end; ++it) { + tokens.push_back(it->str()); + } + + common_chat_msg prev; + + for (auto it = tokens.begin(); it != tokens.end(); it++) { + std::string in = std::accumulate(tokens.begin(), it, std::string()); + + parser_environment env; + parser_context ctx(in, &env, it + 1 == tokens.end()); + + auto result = parser.parse(ctx); + + if (result.is_need_more_input()) { + continue; + } + + auto diffs = common_chat_msg_diff::compute_diffs(prev, env.result); + prev = env.result; + + std::cout << "=== Diffs ===\n\n"; + if (!diffs.empty()) { + for (size_t i = 0; i < diffs.size(); ++i) { + const auto& diff = diffs[i]; + + std::cout << "Diff #" << (i + 1) << "\n"; + + if (!diff.reasoning_content_delta.empty()) { + std::cout << " [Reasoning Content]: " << diff.reasoning_content_delta << "\n"; + } + + if (!diff.content_delta.empty()) { + std::cout << " [Content]: " << diff.content_delta << "\n"; + } + + if (diff.tool_call_index != std::string::npos) { + std::cout << " [Tool Call #" << diff.tool_call_index << "]" << "\n"; + + if (!diff.tool_call_delta.id.empty()) { + std::cout << " ID: " << diff.tool_call_delta.id << "\n"; + } + + if (!diff.tool_call_delta.name.empty()) { + std::cout << " Name: " << diff.tool_call_delta.name << "\n"; + } + + if (!diff.tool_call_delta.arguments.empty()) { + std::cout << " Arguments: " << diff.tool_call_delta.arguments << "\n"; + } + } + + std::cout << "\n"; + } + } else { + std::cout << "No changes detected.\n"; + } + + /* + if (!env.result.reasoning_content.empty()) { + std::cout << "=== Reasoning ===\n"; + std::cout << env.result.reasoning_content << "\n"; + } + if (!env.result.content.empty()) { + std::cout << "\n=== Content ===\n"; + std::cout << env.result.content << "\n"; + } + if (!env.result.tool_calls.empty()) { + std::cout << "\n=== Tool Calls ===\n"; + for (const auto & tc : env.result.tool_calls) { + std::cout << "id: " << tc.id << "\n"; + std::cout << "name: " << tc.name << "\n"; + std::cout << "args: " << tc.arguments << "\n"; + } + } + */ + } +} + static parser create_command_r7b_parser() { auto parser = build_parser([](parser_builder & p) { auto thinking = p.add_rule("thinking", @@ -803,11 +969,11 @@ static void test_command_r7b_parser(const parser & p, const std::string & input, if (print_results) { std::cout << "== Parsed (new) ==\n"; std::cout << "=== Reasoning ===\n"; - std::cout << env.reasoning_content << "\n"; + std::cout << env.result.reasoning_content << "\n"; std::cout << "\n\n=== Content ===\n"; - std::cout << env.content << "\n"; + std::cout << env.result.content << "\n"; std::cout << "\n\n=== Tool Calls ===\n"; - for (const auto & tc : env.tool_calls) { + for (const auto & tc : env.result.tool_calls) { std::cout << "id: " << tc.id << "\n"; std::cout << "name: " << tc.name << "\n"; std::cout << "args: " << tc.arguments << "\n"; @@ -982,6 +1148,8 @@ int main() { test_gbnf_generation(); std::cout << "All tests passed!\n"; + //example_qwen3_coder(); + std::cout << "\n== Benchmarks ==\n"; std::string example_reasoning = "To plan an effective trip to Japan that includes both historical sites and modern attractions within a budget of $4000 for a two-week stay, we need to:\n\n" From bbdf45fbd5bf16520bb13f2dc96725c1937c05dd Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Thu, 13 Nov 2025 06:46:07 -0600 Subject: [PATCH 039/183] trash partial idea and simplify --- common/chat-parser-combinator.cpp | 60 +++++------ common/chat-parser-combinator.h | 10 +- tests/test-chat-parser-combinator.cpp | 146 +++++++++++++++----------- 3 files changed, 112 insertions(+), 104 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index 0ffc5a0617220..6ae8e906c568b 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -304,7 +304,7 @@ class repetition_parser : public parser_base { continue; } - if (result.is_need_more_input() || result.is_partial()) { + if (result.is_need_more_input()) { return parser_result(result.type, start, result.end); } @@ -407,7 +407,7 @@ class not_parser : public parser_base { parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { auto result = parser_->parse(ctx, start); - if (result.is_success() || result.is_partial()) { + if (result.is_success()) { // Fail if the underlying parser matches return parser_result(PARSER_RESULT_FAIL, start); } @@ -720,15 +720,14 @@ class json_string_parser : public parser_base { // S -> (!delim .)* class until_parser : public parser_base { std::string delimiter_; - bool consume_spaces_; std::default_searcher searcher_; public: static constexpr parser_type type_value = PARSER_UNTIL; - until_parser(const std::string & delimiter, bool consume_spaces, int id) - : parser_base(id), delimiter_(delimiter), consume_spaces_(consume_spaces), searcher_(delimiter_.begin(), delimiter_.end()) { + until_parser(const std::string & delimiter, int id) + : parser_base(id), delimiter_(delimiter), searcher_(delimiter_.begin(), delimiter_.end()) { } parser_type type() const override { return type_value; } @@ -741,29 +740,16 @@ class until_parser : public parser_base { if (it != ctx.input.end()) { result.end = std::distance(ctx.input.begin(), it); - result.type = PARSER_RESULT_SUCCESS; } else { // If not found, check if the input ends with a prefix of the delimiter size_t max_overlap = std::min(ctx.input.size(), delimiter_.size() - 1); for (size_t overlap = max_overlap; overlap > 0; --overlap) { if (std::equal(ctx.input.end() - overlap, ctx.input.end(), delimiter_.begin())) { result.end = ctx.input.size() - overlap; - if (ctx.input_is_complete) { - result.type = PARSER_RESULT_FAIL; - } else { - result.type = PARSER_RESULT_NEED_MORE_INPUT; - } } } } - if (consume_spaces_) { - // Remove trailing spaces - while (result.end > start && is_space(ctx.input[result.end - 1])) { - result.end--; - } - } - return result; } @@ -1369,8 +1355,8 @@ parser parser_builder::space() { return parser(std::make_shared(counter_->next())); } -parser parser_builder::until(const std::string & delimiter, bool consume_spaces) { - return parser(std::make_shared(delimiter, consume_spaces, counter_->next())); +parser parser_builder::until(const std::string & delimiter) { + return parser(std::make_shared(delimiter, counter_->next())); } parser parser_builder::repeat(const parser & p, int min, int max) { @@ -1389,10 +1375,10 @@ parser parser_builder::action(const parser & p, std::function(p, std::move(fn), when, counter_->next())); } -parser parser_builder::partial(const parser & p) { +parser parser_builder::succeed(const parser & p, int when) { return action(p, [](parser_result &result, std::string_view, parser_environment &) { - result.type = PARSER_RESULT_PARTIAL; - }, PARSER_RESULT_NEED_MORE_INPUT); + result.type = PARSER_RESULT_SUCCESS; + }, when); } parser parser_builder::append_reasoning(const parser & p) { @@ -1401,7 +1387,7 @@ parser parser_builder::append_reasoning(const parser & p) { env.result.reasoning_content += "\n"; } env.result.reasoning_content += matched; - }, PARSER_RESULT_SUCCESS | PARSER_RESULT_PARTIAL); + }, PARSER_RESULT_SUCCESS | PARSER_RESULT_NEED_MORE_INPUT); } parser parser_builder::append_content(const parser & p) { @@ -1410,48 +1396,50 @@ parser parser_builder::append_content(const parser & p) { env.result.content += "\n"; } env.result.content += matched; - }, PARSER_RESULT_SUCCESS | PARSER_RESULT_PARTIAL); + }, PARSER_RESULT_SUCCESS | PARSER_RESULT_NEED_MORE_INPUT); } parser parser_builder::capture(const parser & p, const std::string & key, bool unescape_json) { return action(p, [key, unescape_json](parser_result &, std::string_view matched, parser_environment & env) { std::string value = unescape_json ? unescape_json_string(matched) : std::string(matched); env.scratchpad[key] = std::move(value); - }, PARSER_RESULT_SUCCESS | PARSER_RESULT_PARTIAL); + }, PARSER_RESULT_SUCCESS); } parser parser_builder::capture_tool_call_id(const parser & p, bool unescape_json) { return action(p, [unescape_json](parser_result &, std::string_view matched, parser_environment & env) { env.tool_call_id = unescape_json ? unescape_json_string(matched) : std::string(matched); - }, PARSER_RESULT_SUCCESS | PARSER_RESULT_PARTIAL); + }, PARSER_RESULT_SUCCESS); } parser parser_builder::capture_tool_call_name(const parser & p, bool unescape_json) { return action(p, [unescape_json](parser_result &, std::string_view matched, parser_environment & env) { env.tool_call_name = unescape_json ? unescape_json_string(matched) : std::string(matched); - }, PARSER_RESULT_SUCCESS | PARSER_RESULT_PARTIAL); + }, PARSER_RESULT_SUCCESS); } parser parser_builder::capture_tool_call_args(const parser & p, bool unescape_json) { return action(p, [unescape_json](parser_result &, std::string_view matched, parser_environment & env) { env.tool_call_args = unescape_json ? unescape_json_string(matched) : std::string(matched); - }, PARSER_RESULT_SUCCESS | PARSER_RESULT_PARTIAL); + }, PARSER_RESULT_SUCCESS | PARSER_RESULT_NEED_MORE_INPUT); } parser parser_builder::add_tool_call(const parser & p) { return action(p, [](const parser_result &, std::string_view, parser_environment & env) { - auto tool_call = common_chat_tool_call{ - env.tool_call_name, - env.tool_call_args, - env.tool_call_id - }; - env.result.tool_calls.push_back(tool_call); + if (!env.tool_call_name.empty() && !env.tool_call_args.empty()) { + auto tool_call = common_chat_tool_call{ + env.tool_call_name, + env.tool_call_args, + env.tool_call_id + }; + env.result.tool_calls.push_back(tool_call); + } // Clear the fields to prevent bleeding to next tool call env.tool_call_id.clear(); env.tool_call_name.clear(); env.tool_call_args.clear(); - }, PARSER_RESULT_SUCCESS | PARSER_RESULT_PARTIAL); + }, PARSER_RESULT_SUCCESS | PARSER_RESULT_NEED_MORE_INPUT); } parser parser_builder::json_key(const std::string & name, const parser & p) { diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index 1c0dd01d5a7b4..2a29e0fadde89 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -30,7 +30,6 @@ enum parser_result_type { PARSER_RESULT_FAIL = 1 << 0, PARSER_RESULT_SUCCESS = 1 << 1, PARSER_RESULT_NEED_MORE_INPUT = 1 << 2, - PARSER_RESULT_PARTIAL = 1 << 3, }; struct parse_cache_key { @@ -64,7 +63,6 @@ struct parser_result { bool is_fail() const { return type == PARSER_RESULT_FAIL; } bool is_need_more_input() const { return type == PARSER_RESULT_NEED_MORE_INPUT; } - bool is_partial() const { return type == PARSER_RESULT_PARTIAL; } bool is_success() const { return type == PARSER_RESULT_SUCCESS; } }; @@ -214,7 +212,7 @@ class parser_builder { // Matches all characters until a delimiter is found (delimiter not consumed). // S -> (!delim .)* - parser until(const std::string & delimiter, bool consume_spaces = true); + parser until(const std::string & delimiter); // Matches between min and max repetitions of a parser (inclusive). // S -> A{m,n} @@ -243,12 +241,12 @@ class parser_builder { // Wraps a parser with a semantic action callback. // The callback is invoked on successful parse with the result, matched text, and environment. // S -> A [action] - parser action(const parser & p, std::function fn, int when = PARSER_RESULT_SUCCESS | PARSER_RESULT_PARTIAL); + parser action(const parser & p, std::function fn, int when = PARSER_RESULT_SUCCESS); // Convenience action wrappers for common patterns - // Converts PARSER_RESULT_NEED_MORE_INPUT to PARSER_RESULT_PARTIAL - parser partial(const parser & p); + // Causes a rule to succeed + parser succeed(const parser & p, int when = PARSER_RESULT_NEED_MORE_INPUT); // Appends matched text to env.reasoning_content parser append_reasoning(const parser & p); diff --git a/tests/test-chat-parser-combinator.cpp b/tests/test-chat-parser-combinator.cpp index e227dfb6748c7..29f4cc335d91a 100644 --- a/tests/test-chat-parser-combinator.cpp +++ b/tests/test-chat-parser-combinator.cpp @@ -396,10 +396,10 @@ static void test_complete_example() { auto schema = nlohmann::ordered_json::parse(R"({"type": "object"})"); auto tool_call_args = p.add_rule("tool-call-args", - "" << p.capture_tool_call_args(p.schema(p.partial(json), "get_weather", schema)) << ""); + "" << p.capture_tool_call_args(p.schema(p.succeed(json), "get_weather", schema)) << ""); auto tool_call = p.add_rule("tool-call", - "" << p.add_tool_call(tool_call_name << p.partial(tool_call_args)) << ""); + "" << p.add_tool_call(tool_call_name << p.succeed(tool_call_args)) << ""); return reasoning << p.optional(content) << p.optional(tool_call); }); @@ -423,7 +423,7 @@ static void test_complete_example() { // Test partial input { - std::string input = R"(I need to call get_weather )"; + std::string input = R"(I need to call get_weather)"; parser_environment env = parser_environment(); parser_context ctx = parser_context(input, &env, /* .is_input_complete = */ false); @@ -477,7 +477,7 @@ static void test_complete_example() { auto result = parser.parse(ctx); - assert_equals(true, result.is_partial()); + assert_equals(true, result.is_need_more_input()); assert_equals("I need to call get_weather", env.result.reasoning_content); assert_equals("get_weather", env.result.tool_calls[0].name); assert_equals(R"({"cit)", env.result.tool_calls[0].arguments); @@ -580,7 +580,7 @@ static void test_actions() { auto result = parser.parse(ctx); assert_equals(true, result.is_need_more_input()); - assert_equals("hello", env.result.content); + assert_equals("hello ", env.result.content); } { parser_environment env; @@ -755,6 +755,42 @@ static void test_gbnf_generation() { } } +// Simple tokenize function that splits by space and special chars +static std::vector simple_tokenize(const std::string & input) { + std::vector result; + std::string current; + + for (size_t i = 0; i < input.size(); i++) { + switch (input[i]) { + case ' ': + case '\n': + case '\t': + case '{': + case '}': + case ',': + case '[': + case '"': + case ']': + case '.': + case '<': + case '>': + case '=': + case '/': + if (!current.empty()) { + result.push_back(current); + current.clear(); + } + } + current += input[i]; + } + + if (!current.empty()) { + result.push_back(current); + } + + return result; +} + static void example_qwen3_coder() { auto parser = build_parser([](parser_builder & p) { auto thinking = p.add_rule("thinking", @@ -780,7 +816,7 @@ static void example_qwen3_coder() { auto string_arg = p.add_rule("arg-string", p.action(arg_start, [&](const parser_result &, std::string_view, parser_environment & env) { - env.tool_call_args += "{"; + env.tool_call_args += "\""; }) << p.action(p.until(""), [&](const parser_result &, std::string_view match, parser_environment & env) { // TODO: add a JSON escape helper @@ -794,9 +830,13 @@ static void example_qwen3_coder() { auto json_arg = p.add_rule("arg-json", arg_start - << p.action(p.partial(json), [&](const parser_result &, std::string_view match, parser_environment & env) { + << p.action(json, [&](const parser_result &, std::string_view match, parser_environment & env) { // JSON should already be properly formatted env.tool_call_args += std::string(match); + + // This can be streamed by passing p.success(json), but we have + // to be mindful of the potential backtracking--it only works + // if we only keep the last value... }) << arg_end); @@ -806,7 +846,7 @@ static void example_qwen3_coder() { + p.action(">", [&](const parser_result &, std::string_view, parser_environment & env) { env.tool_call_args += "{"; }) - + p.one_or_more(p.space() + (p.partial(json_arg) | p.partial(string_arg))) + + p.one_or_more(p.space() + (json_arg | string_arg)) << p.action("", [&](const parser_result &, std::string_view, parser_environment & env) { env.tool_call_args += "}"; }))); @@ -815,7 +855,7 @@ static void example_qwen3_coder() { "" << p.one_or_more(function) << ""); - return p.partial(thinking) + p.optional(p.space() + p.partial(content)) + p.zero_or_more(p.space() + tool_call); + return thinking + p.optional(p.space() + content) + p.zero_or_more(p.space() + tool_call); }); std::string input = @@ -838,13 +878,7 @@ static void example_qwen3_coder() { "\n" ""; - static const std::regex token_regex(R"(([A-Za-z0-9]+|[^A-Za-z0-9]+))"); - std::vector tokens; - std::sregex_iterator it(input.begin(), input.end(), token_regex); - std::sregex_iterator end; - for (; it != end; ++it) { - tokens.push_back(it->str()); - } + std::vector tokens = simple_tokenize(input); common_chat_msg prev; @@ -852,17 +886,47 @@ static void example_qwen3_coder() { std::string in = std::accumulate(tokens.begin(), it, std::string()); parser_environment env; - parser_context ctx(in, &env, it + 1 == tokens.end()); + parser_context ctx(in, &env, it == tokens.end() - 1); auto result = parser.parse(ctx); + if (result.is_fail()) { + break; + } - if (result.is_need_more_input()) { - continue; + /* + std::cout << "Input:\n" << in << "\n\n"; + std::cout << "Reasoning: " << prev.reasoning_content << "\n"; + std::cout << "Content : " << prev.content << "\n"; + if (!prev.tool_calls.empty()) { + std::cout << "\n=== Tool Calls ===\n"; + for (const auto & tc : prev.tool_calls) { + std::cout << "ID : " << tc.id << "\n"; + std::cout << "Name: " << tc.name << "\n"; + std::cout << "Args: " << tc.arguments << "\n"; + } } + */ + // This shouldn't emit any runtime errors auto diffs = common_chat_msg_diff::compute_diffs(prev, env.result); prev = env.result; + /* + std::cout << "----\n"; + std::cout << "Reasoning: " << prev.reasoning_content << "\n"; + std::cout << "Content : " << prev.content << "\n"; + if (!prev.tool_calls.empty()) { + std::cout << "\n=== Tool Calls ===\n"; + for (const auto & tc : prev.tool_calls) { + std::cout << "ID : " << tc.id << "\n"; + std::cout << "Name: " << tc.name << "\n"; + std::cout << "Args: " << tc.arguments << "\n"; + } + } + std::cout << "======================\n"; + */ + + /* std::cout << "=== Diffs ===\n\n"; if (!diffs.empty()) { for (size_t i = 0; i < diffs.size(); ++i) { @@ -899,24 +963,6 @@ static void example_qwen3_coder() { } else { std::cout << "No changes detected.\n"; } - - /* - if (!env.result.reasoning_content.empty()) { - std::cout << "=== Reasoning ===\n"; - std::cout << env.result.reasoning_content << "\n"; - } - if (!env.result.content.empty()) { - std::cout << "\n=== Content ===\n"; - std::cout << env.result.content << "\n"; - } - if (!env.result.tool_calls.empty()) { - std::cout << "\n=== Tool Calls ===\n"; - for (const auto & tc : env.result.tool_calls) { - std::cout << "id: " << tc.id << "\n"; - std::cout << "name: " << tc.name << "\n"; - std::cout << "args: " << tc.arguments << "\n"; - } - } */ } } @@ -1043,30 +1089,6 @@ struct bench_tool_call { nlohmann::ordered_json args; }; -// Simple tokenize function that splits by space -static std::vector simple_tokenize(const std::string & input) { - std::vector result; - std::string current; - - for (size_t i = 0; i < input.size(); i++) { - if (input[i] == ' ') { - if (!current.empty()) { - result.push_back(current); - current.clear(); - } - current += ' '; - } else { - current += input[i]; - } - } - - if (!current.empty()) { - result.push_back(current); - } - - return result; -} - static void benchmark_compare( const std::string & reasoning, const std::string & content, @@ -1148,7 +1170,7 @@ int main() { test_gbnf_generation(); std::cout << "All tests passed!\n"; - //example_qwen3_coder(); + example_qwen3_coder(); std::cout << "\n== Benchmarks ==\n"; std::string example_reasoning = From dd069728ca9a401ba541b23ab888ef744b509cf7 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Thu, 13 Nov 2025 06:58:51 -0600 Subject: [PATCH 040/183] move action arguments to a struct --- common/chat-parser-combinator.cpp | 68 +++++++++++++------------ common/chat-parser-combinator.h | 8 ++- tests/test-chat-parser-combinator.cpp | 72 +++++++++++++-------------- 3 files changed, 79 insertions(+), 69 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index 6ae8e906c568b..f308c26ea5610 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -872,7 +872,7 @@ class root_parser : public parser_base { // Wraps a parser with a semantic action callback. class action_parser : public parser_base { parser parser_; - std::function action_; + std::function action_; int when_; public: @@ -880,7 +880,7 @@ class action_parser : public parser_base { action_parser( const parser & parser, - std::function action, + std::function action, int when, int id ) : parser_base(id), parser_(parser), action_(std::move(action)), when_(when) {} @@ -892,7 +892,11 @@ class action_parser : public parser_base { if ((result.type & when_) && ctx.env && action_) { std::string_view matched = ctx.input.substr(result.start, result.end - result.start); - action_(result, matched, *ctx.env); + action_({ + result, + *ctx.env, + matched, + }); } return result; @@ -1371,74 +1375,74 @@ parser parser_builder::schema(const parser & p, const std::string & name, const return parser(std::make_shared(p, name, schema, counter_->next())); } -parser parser_builder::action(const parser & p, std::function fn, int when) { +parser parser_builder::action(const parser & p, std::function fn, int when) { return parser(std::make_shared(p, std::move(fn), when, counter_->next())); } parser parser_builder::succeed(const parser & p, int when) { - return action(p, [](parser_result &result, std::string_view, parser_environment &) { - result.type = PARSER_RESULT_SUCCESS; + return action(p, [](const parser_action & act) { + act.result.type = PARSER_RESULT_SUCCESS; }, when); } parser parser_builder::append_reasoning(const parser & p) { - return action(p, [](parser_result &, std::string_view matched, parser_environment & env) { - if (!env.result.reasoning_content.empty()) { - env.result.reasoning_content += "\n"; + return action(p, [](const parser_action & act) { + if (!act.env.result.reasoning_content.empty()) { + act.env.result.reasoning_content += "\n"; } - env.result.reasoning_content += matched; + act.env.result.reasoning_content += act.match; }, PARSER_RESULT_SUCCESS | PARSER_RESULT_NEED_MORE_INPUT); } parser parser_builder::append_content(const parser & p) { - return action(p, [](parser_result &, std::string_view matched, parser_environment & env) { - if (!env.result.content.empty()) { - env.result.content += "\n"; + return action(p, [](const parser_action & act) { + if (!act.env.result.content.empty()) { + act.env.result.content += "\n"; } - env.result.content += matched; + act.env.result.content += act.match; }, PARSER_RESULT_SUCCESS | PARSER_RESULT_NEED_MORE_INPUT); } parser parser_builder::capture(const parser & p, const std::string & key, bool unescape_json) { - return action(p, [key, unescape_json](parser_result &, std::string_view matched, parser_environment & env) { - std::string value = unescape_json ? unescape_json_string(matched) : std::string(matched); - env.scratchpad[key] = std::move(value); + return action(p, [key, unescape_json](const parser_action & act) { + std::string value = unescape_json ? unescape_json_string(act.match) : std::string(act.match); + act.env.scratchpad[key] = std::move(value); }, PARSER_RESULT_SUCCESS); } parser parser_builder::capture_tool_call_id(const parser & p, bool unescape_json) { - return action(p, [unescape_json](parser_result &, std::string_view matched, parser_environment & env) { - env.tool_call_id = unescape_json ? unescape_json_string(matched) : std::string(matched); + return action(p, [unescape_json](const parser_action & act) { + act.env.tool_call_id = unescape_json ? unescape_json_string(act.match) : std::string(act.match); }, PARSER_RESULT_SUCCESS); } parser parser_builder::capture_tool_call_name(const parser & p, bool unescape_json) { - return action(p, [unescape_json](parser_result &, std::string_view matched, parser_environment & env) { - env.tool_call_name = unescape_json ? unescape_json_string(matched) : std::string(matched); + return action(p, [unescape_json](const parser_action & act) { + act.env.tool_call_name = unescape_json ? unescape_json_string(act.match) : std::string(act.match); }, PARSER_RESULT_SUCCESS); } parser parser_builder::capture_tool_call_args(const parser & p, bool unescape_json) { - return action(p, [unescape_json](parser_result &, std::string_view matched, parser_environment & env) { - env.tool_call_args = unescape_json ? unescape_json_string(matched) : std::string(matched); + return action(p, [unescape_json](const parser_action & act) { + act.env.tool_call_args = unescape_json ? unescape_json_string(act.match) : std::string(act.match); }, PARSER_RESULT_SUCCESS | PARSER_RESULT_NEED_MORE_INPUT); } parser parser_builder::add_tool_call(const parser & p) { - return action(p, [](const parser_result &, std::string_view, parser_environment & env) { - if (!env.tool_call_name.empty() && !env.tool_call_args.empty()) { + return action(p, [](const parser_action & act) { + if (!act.env.tool_call_name.empty() && !act.env.tool_call_args.empty()) { auto tool_call = common_chat_tool_call{ - env.tool_call_name, - env.tool_call_args, - env.tool_call_id + act.env.tool_call_name, + act.env.tool_call_args, + act.env.tool_call_id }; - env.result.tool_calls.push_back(tool_call); + act.env.result.tool_calls.push_back(tool_call); } // Clear the fields to prevent bleeding to next tool call - env.tool_call_id.clear(); - env.tool_call_name.clear(); - env.tool_call_args.clear(); + act.env.tool_call_id.clear(); + act.env.tool_call_name.clear(); + act.env.tool_call_args.clear(); }, PARSER_RESULT_SUCCESS | PARSER_RESULT_NEED_MORE_INPUT); } diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index 2a29e0fadde89..fcb99d6d68c22 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -66,6 +66,12 @@ struct parser_result { bool is_success() const { return type == PARSER_RESULT_SUCCESS; } }; +struct parser_action { + parser_result & result; + parser_environment & env; + std::string_view match; +}; + class parse_cache { std::unordered_map results; @@ -241,7 +247,7 @@ class parser_builder { // Wraps a parser with a semantic action callback. // The callback is invoked on successful parse with the result, matched text, and environment. // S -> A [action] - parser action(const parser & p, std::function fn, int when = PARSER_RESULT_SUCCESS); + parser action(const parser & p, std::function fn, int when = PARSER_RESULT_SUCCESS); // Convenience action wrappers for common patterns diff --git a/tests/test-chat-parser-combinator.cpp b/tests/test-chat-parser-combinator.cpp index 29f4cc335d91a..f4ad8a64c3c70 100644 --- a/tests/test-chat-parser-combinator.cpp +++ b/tests/test-chat-parser-combinator.cpp @@ -495,8 +495,8 @@ static void test_actions() { // Test simple action - append matched text to content auto parser = build_parser([](parser_builder& p) { auto word = p.chars("[a-z]+"); - return p.action(word, [](const parser_result &, std::string_view matched, parser_environment & env) { - env.result.content += std::string(matched); + return p.action(word, [](const parser_action & act) { + act.env.result.content += std::string(act.match); }); }); @@ -510,13 +510,13 @@ static void test_actions() { { // Test multiple sequential actions - build a sentence auto parser = build_parser([](parser_builder& p) { - auto greeting = p.action(p.literal("hello"), [](const parser_result &, std::string_view matched, parser_environment & env) { - env.result.content += std::string(matched) + " "; + auto greeting = p.action(p.literal("hello"), [](const parser_action & act) { + act.env.result.content += std::string(act.match) + " "; }); - auto name = p.action(p.chars("[A-Z][a-z]+"), [](const parser_result &, std::string_view matched, parser_environment & env) { - env.result.content += std::string(matched); - env.scratchpad["name"] = std::string(matched); + auto name = p.action(p.chars("[A-Z][a-z]+"), [](const parser_action & act) { + act.env.result.content += std::string(act.match); + act.env.scratchpad["name"] = std::string(act.match); }); return greeting + p.literal(" ") + name; @@ -533,11 +533,11 @@ static void test_actions() { { // Test using scratchpad for intermediate calculations auto parser = build_parser([](parser_builder& p) { - auto digit = p.action(p.one("[0-9]"), [](const parser_result &, std::string_view matched, parser_environment & env) { - auto it = env.scratchpad.find("sum"); - int current_sum = it != env.scratchpad.end() ? std::get(it->second) : 0; - current_sum += (matched[0] - '0'); - env.scratchpad["sum"] = current_sum; + auto digit = p.action(p.one("[0-9]"), [](const parser_action & act) { + auto it = act.env.scratchpad.find("sum"); + int current_sum = it != act.env.scratchpad.end() ? std::get(it->second) : 0; + current_sum += (act.match[0] - '0'); + act.env.scratchpad["sum"] = current_sum; }); return p.one_or_more(digit + p.optional(p.literal("+"))); @@ -553,8 +553,8 @@ static void test_actions() { { // Test actions don't run when parse fails auto parser = build_parser([](parser_builder& p) { - return p.action(p.literal("success"), [](const parser_result &, std::string_view, parser_environment & env) { - env.result.content = "action_ran"; + return p.action(p.literal("success"), [](const parser_action & act) { + act.env.result.content = "action_ran"; }); }); @@ -568,8 +568,8 @@ static void test_actions() { { // Test Actions work with partial parsing auto parser = build_parser([](parser_builder& p) { - auto content = p.action(p.until(""), [](const parser_result &, std::string_view matched, parser_environment & env) { - env.result.content += std::string(matched); + auto content = p.action(p.until(""), [](const parser_action & act) { + act.env.result.content += std::string(act.match); }); return "" << content << ""; }); @@ -799,40 +799,40 @@ static void example_qwen3_coder() { auto content = p.add_rule("content", p.append_content(p.until(""))); auto arg_start = p.add_rule("arg-start", - p.action("", [](const parser_result &, std::string_view, parser_environment & env) { - env.tool_call_args += "\":"; + + p.action(">", [](const parser_action & act) { + act.env.tool_call_args += "\":"; })); auto arg_end = p.add_rule("arg-end", ""); auto string_arg = p.add_rule("arg-string", - p.action(arg_start, [&](const parser_result &, std::string_view, parser_environment & env) { - env.tool_call_args += "\""; + p.action(arg_start, [&](const parser_action & act) { + act.env.tool_call_args += "\""; }) - << p.action(p.until(""), [&](const parser_result &, std::string_view match, parser_environment & env) { + << p.action(p.until(""), [&](const parser_action & act) { // TODO: add a JSON escape helper - env.tool_call_args += std::string(match); + act.env.tool_call_args += std::string(act.match); }) - << p.action(arg_end, [&](const parser_result &, std::string_view, parser_environment & env) { - env.tool_call_args += "\""; + << p.action(arg_end, [&](const parser_action & act) { + act.env.tool_call_args += "\""; })); auto json = p.json(); auto json_arg = p.add_rule("arg-json", arg_start - << p.action(json, [&](const parser_result &, std::string_view match, parser_environment & env) { + << p.action(json, [&](const parser_action & act) { // JSON should already be properly formatted - env.tool_call_args += std::string(match); + act.env.tool_call_args += std::string(act.match); // This can be streamed by passing p.success(json), but we have // to be mindful of the potential backtracking--it only works @@ -843,12 +843,12 @@ static void example_qwen3_coder() { auto function = p.add_rule("function", p.add_tool_call( "", [&](const parser_result &, std::string_view, parser_environment & env) { - env.tool_call_args += "{"; + + p.action(">", [&](const parser_action & act) { + act.env.tool_call_args += "{"; }) + p.one_or_more(p.space() + (json_arg | string_arg)) - << p.action("", [&](const parser_result &, std::string_view, parser_environment & env) { - env.tool_call_args += "}"; + << p.action("", [&](const parser_action & act) { + act.env.tool_call_args += "}"; }))); auto tool_call = p.add_rule("tool-call", From 94bd7007abcc28470268bce258b65cb56326ab70 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Fri, 14 Nov 2025 03:29:18 -0600 Subject: [PATCH 041/183] implement aho-corasick matcher for until_parser and to build exclusion grammars --- common/chat-parser-combinator.cpp | 326 +++++++++++++++++++++--------- 1 file changed, 235 insertions(+), 91 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index f308c26ea5610..2c366176d11f5 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -6,6 +6,7 @@ #include +#include #include #include @@ -115,6 +116,202 @@ static std::string unescape_json_string(std::string_view str) { return std::string(str); } +// Aho-Corasick automation for matching multiple literals. +// This is used in until_parser and to build a GBNF exclusion grammar by +// exploiting its trie structure. +class aho_corasick_matcher { + struct node { + size_t fail = 0; + size_t depth = 0; + std::map children; + std::vector word_lengths; + }; + + std::vector trie; + + public: + aho_corasick_matcher(const std::vector & words) { + create_node(); // root node + for (const auto & w : words) { + insert(w); + } + build_fail_links(); + } + + size_t search(std::string_view sv, size_t start = 0) { + size_t current = 0; + + for (auto i = start; i < sv.size(); ++i) { + // Aho-Corasick transition + while (current != 0 && trie[current].children.find(sv[i]) == trie[current].children.end()) { + current = trie[current].fail; + } + + auto it = trie[current].children.find(sv[i]); + if (it != trie[current].children.end()) { + current = it->second; + } else { + current = 0; + } + + if (!trie[current].word_lengths.empty()) { + // Return back the longest word + size_t pos = sv.size(); + for (const auto & len : trie[current].word_lengths) { + pos = std::min(pos, i - len + 1); + } + return pos; + } + } + + if (trie[current].depth > 0) { + return sv.size() - trie[current].depth; + } + return sv.size(); + } + + struct prefix_and_next { + std::string prefix; + std::string next_chars; + }; + + std::vector collect_prefix_and_next() { + std::string prefix; + std::vector result; + collect_prefix_and_next(0, prefix, result); + return result; + } + + private: + void collect_prefix_and_next(size_t index, std::string & prefix, std::vector & out) { + if (trie[index].word_lengths.empty()) { + if (!trie[index].children.empty()) { + std::string chars; + chars.reserve(trie[index].children.size()); + for (const auto & p : trie[index].children) { + chars.push_back(p.first); + } + out.emplace_back(prefix_and_next{prefix, chars}); + } + } + + for (const auto & p : trie[index].children) { + unsigned char ch = p.first; + int child = p.second; + prefix.push_back(ch); + collect_prefix_and_next(child, prefix, out); + prefix.pop_back(); + } + } + + size_t create_node() { + size_t index = trie.size(); + trie.emplace_back(); + return index; + } + + void insert(const std::string & word) { + size_t current = 0; + for (unsigned char ch : word) { + auto it = trie[current].children.find(ch); + if (it == trie[current].children.end()) { + size_t child = create_node(); + trie[child].depth = trie[current].depth + 1; + trie[current].children[ch] = child; + current = child; + } else { + current = it->second; + } + } + trie[current].word_lengths.push_back(word.length()); + } + + void build_fail_links() { + std::deque queue; + + size_t root = 0; + trie[root].fail = 0; + for (const auto & it : trie[root].children) { + size_t child = it.second; + trie[child].fail = 0; + queue.push_back(child); + } + + while (!queue.empty()) { + size_t current = queue.front(); + queue.pop_front(); + + for (const auto & p : trie[current].children) { + unsigned char ch = p.first; + size_t child = p.second; + queue.push_back(child); + + auto fail = trie[current].fail; + while (fail != 0 && trie[fail].children.find(p.first) == trie[fail].children.end()) { + fail = trie[fail].fail; + } + + auto fail_it = trie[fail].children.find(ch); + trie[child].fail = fail_it != trie[fail].children.end() ? fail_it->second : 0; + } + } + } +}; + +// Generate an excluding pattern, with customized escaping +static std::string generic_excluding_pattern( + const std::vector & strings, + const std::function & literal, + const std::function & escape_char_class, + bool pad = false) { + + // Use the aho_corasick_matcher to grab an exhaustive list of prefixes and + // potential next characters. We can use this to build an exclusion for + // multiple strings. + aho_corasick_matcher matcher(strings); + auto pieces = matcher.collect_prefix_and_next(); + + std::string pattern; + for (size_t i = 0; i < pieces.size(); ++i) { + if (i > 0) { + pattern += pad ? " | " : "|"; + } + + const auto & pre = pieces[i].prefix; + const auto & chars = pieces[i].next_chars; + + std::string cls; + cls.reserve(chars.size()); + for (const auto & ch : chars) { + cls += escape_char_class(ch); + } + + if (!pre.empty()) { + pattern += literal(pre) + (pad ? " [^" : "[^") + cls + "]"; + } else { + pattern += "[^" + cls + "]"; + } + } + + return "(" + pattern + ")*"; +} + +// Escape a single character for use in regex character classes +static std::string regex_escape_char_class(char c) { + switch (c) { + case '\\': return "\\\\"; + case ']': return "\\]"; + case '-': return "\\-"; + case '^': return "\\^"; + default: return std::string(1, c); + } +} + +// Create a regex excluding pattern +static std::string regex_excluding_pattern(const std::vector & strings) { + return generic_excluding_pattern(strings, regex_escape, regex_escape_char_class); +} + // Matches an exact literal string. // S -> "hello" class literal_parser : public parser_base { @@ -721,36 +918,19 @@ class json_string_parser : public parser_base { class until_parser : public parser_base { std::string delimiter_; - std::default_searcher searcher_; + aho_corasick_matcher matcher_; public: static constexpr parser_type type_value = PARSER_UNTIL; until_parser(const std::string & delimiter, int id) - : parser_base(id), delimiter_(delimiter), searcher_(delimiter_.begin(), delimiter_.end()) { + : parser_base(id), delimiter_(delimiter), matcher_({delimiter}) { } parser_type type() const override { return type_value; } parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { - parser_result result(PARSER_RESULT_SUCCESS, start, ctx.input.size()); - - // Search for the delimiter - const auto it = std::search(ctx.input.begin() + start, ctx.input.end(), searcher_); - - if (it != ctx.input.end()) { - result.end = std::distance(ctx.input.begin(), it); - } else { - // If not found, check if the input ends with a prefix of the delimiter - size_t max_overlap = std::min(ctx.input.size(), delimiter_.size() - 1); - for (size_t overlap = max_overlap; overlap > 0; --overlap) { - if (std::equal(ctx.input.end() - overlap, ctx.input.end(), delimiter_.begin())) { - result.end = ctx.input.size() - overlap; - } - } - } - - return result; + return parser_result(PARSER_RESULT_SUCCESS, start, matcher_.search(ctx.input, start)); } std::string dump() const override { @@ -891,7 +1071,8 @@ class action_parser : public parser_base { auto result = parser_->parse(ctx, start); if ((result.type & when_) && ctx.env && action_) { - std::string_view matched = ctx.input.substr(result.start, result.end - result.start); + std::string_view matched = ctx.input; + matched = matched.substr(result.start, result.end - result.start); action_({ result, *ctx.env, @@ -916,7 +1097,6 @@ class action_parser : public parser_base { const parser & child() const { return parser_; } }; - // Base visitor class for parser tree traversal class parser_visitor { public: @@ -941,6 +1121,37 @@ class parser_visitor { virtual void visit(action_parser & p) = 0; }; +// Escape special characters for GBNF literals +static std::string gbnf_literal(const std::string & s) { + std::string escaped; + for (char c : s) { + switch (c) { + case '\n': escaped += "\\n"; break; + case '\t': escaped += "\\t"; break; + case '\r': escaped += "\\r"; break; + case '\\': escaped += "\\\\"; break; + case '"': escaped += "\\\""; break; + default: escaped += c; break; + } + } + return "\"" + escaped + "\""; +} + +// Escape a single character for use in gbnf character classes +static std::string gbnf_escape_char_class(char c) { + switch (c) { + case '\n': return "\\n"; + case '\t': return "\\t"; + case '\r': return "\\r"; + default: return regex_escape_char_class(c); // these too + } +} + +// Create a GBNF excluding pattern +static std::string gbnf_excluding_pattern(const std::vector & strings) { + return generic_excluding_pattern(strings, gbnf_literal, gbnf_escape_char_class, true); +} + class gbnf_visitor : public parser_visitor { const common_grammar_builder & builder_; std::unordered_map rule_name_mapping_; @@ -952,73 +1163,6 @@ class gbnf_visitor : public parser_visitor { const std::string& result() const { return current_result_; } private: - // Escape special characters for GBNF literals - static std::string escape_literal(const std::string & s) { - std::string escaped; - for (char c : s) { - switch (c) { - case '\n': escaped += "\\n"; break; - case '\t': escaped += "\\t"; break; - case '\r': escaped += "\\r"; break; - case '\\': escaped += "\\\\"; break; - case '"': escaped += "\\\""; break; - default: escaped += c; break; - } - } - return escaped; - } - - // Escape a single character for use in character classes - static std::string escape_char_class(char c) { - switch (c) { - case '\n': return "\\n"; - case '\t': return "\\t"; - case '\r': return "\\r"; - case '\\': return "\\\\"; - case ']': return "\\]"; - case '-': return "\\-"; - case '^': return "\\^"; - default: return std::string(1, c); - } - } - - // Generate pattern for until() that matches prefixes but prevents full delimiter match - // For "" generates: ( [^<] | "<" [^/] | " alternatives; - - // First alternative: match any character that's not the start of the delimiter - alternatives.push_back("[^" + escape_char_class(delimiter[0]) + "]"); - - // For each prefix, match the prefix followed by a char that's not the next delimiter char - for (size_t i = 1; i < delimiter.length(); ++i) { - std::string prefix = "\"" + escape_literal(delimiter.substr(0, i)) + "\""; - std::string next_char_negated = "[^" + escape_char_class(delimiter[i]) + "]"; - alternatives.push_back(prefix + " " + next_char_negated); - } - - // Combine alternatives with | - std::string result = "("; - for (size_t i = 0; i < alternatives.size(); ++i) { - if (i > 0) { - result += " | "; - } - result += alternatives[i]; - } - result += ")"; - - return result; - } - // Check if expression needs parentheses static bool needs_parens(parser_type type) { return type == PARSER_CHOICE || type == PARSER_SEQUENCE; @@ -1026,7 +1170,7 @@ class gbnf_visitor : public parser_visitor { public: void visit(literal_parser & p) override { - current_result_ = "\"" + escape_literal(p.literal()) + "\""; + current_result_ = gbnf_literal(p.literal()); } void visit(sequence_parser & p) override { @@ -1113,7 +1257,7 @@ class gbnf_visitor : public parser_visitor { void visit(until_parser & p) override { // Generate pattern that matches prefixes but prevents full delimiter match - current_result_ = generate_until_pattern(p.delimiter()) + "*"; + current_result_ = gbnf_excluding_pattern({p.delimiter()}); } void visit(not_parser &) override { From 599e2fdd2ac981ec0742b785a443b7852b49d9df Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Fri, 14 Nov 2025 03:29:39 -0600 Subject: [PATCH 042/183] use std::string for input, since std::string_view is incompatible with std::regex --- common/chat-parser-combinator.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index fcb99d6d68c22..60eacc8a7b8bf 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -82,7 +82,7 @@ class parse_cache { }; struct parser_context { - std::string_view input; + std::string input; parse_cache memo; bool input_is_complete; parser_environment * env; @@ -90,22 +90,22 @@ struct parser_context { parser_context() : memo(), input_is_complete(true), env(nullptr) {} - parser_context(std::string_view input) + parser_context(const std::string & input) : input(input), memo(), input_is_complete(true), env(nullptr) {} - parser_context(std::string_view input, bool complete) + parser_context(const std::string & input, bool complete) : input(input), memo(), input_is_complete(complete), env(nullptr) {} - parser_context(std::string_view input, parse_cache memo, bool complete = true) + parser_context(const std::string & input, parse_cache memo, bool complete = true) : input(input), memo(std::move(memo)), input_is_complete(complete), env(nullptr) {} - parser_context(std::string_view input, parser_environment * environment) + parser_context(const std::string & input, parser_environment * environment) : input(input), memo(), input_is_complete(true), env(environment) {} - parser_context(std::string_view input, parser_environment * environment, bool complete) + parser_context(const std::string & input, parser_environment * environment, bool complete) : input(input), memo(), input_is_complete(complete), env(environment) {} - parser_context(std::string_view input, parse_cache memo, parser_environment * environment, bool complete = true) + parser_context(const std::string & input, parse_cache memo, parser_environment * environment, bool complete = true) : input(input), memo(std::move(memo)), input_is_complete(complete), env(environment) {} }; From c40b03e361f346dff8dbe6a6ebcab2a13e828cf2 Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Fri, 14 Nov 2025 11:49:05 +0100 Subject: [PATCH 043/183] Refactor tests --- tests/.gitignore | 1 + tests/CMakeLists.txt | 12 + tests/combinator/test-actions.cpp | 127 ++++++++ tests/combinator/test-combinator-all.cpp | 36 +++ tests/combinator/test-complete-example.cpp | 145 +++++++++ tests/combinator/test-gbnf-generation.cpp | 176 +++++++++++ tests/combinator/test-json-parser.cpp | 99 ++++++ tests/combinator/test-one.cpp | 124 ++++++++ tests/combinator/test-optional.cpp | 49 +++ tests/combinator/test-partial-parsing.cpp | 283 ++++++++++++++++++ .../combinator/test-recursive-references.cpp | 120 ++++++++ tests/combinator/tests.h | 15 + tests/testcase.hpp | 147 +++++++++ 13 files changed, 1334 insertions(+) create mode 100644 tests/combinator/test-actions.cpp create mode 100644 tests/combinator/test-combinator-all.cpp create mode 100644 tests/combinator/test-complete-example.cpp create mode 100644 tests/combinator/test-gbnf-generation.cpp create mode 100644 tests/combinator/test-json-parser.cpp create mode 100644 tests/combinator/test-one.cpp create mode 100644 tests/combinator/test-optional.cpp create mode 100644 tests/combinator/test-partial-parsing.cpp create mode 100644 tests/combinator/test-recursive-references.cpp create mode 100644 tests/combinator/tests.h create mode 100644 tests/testcase.hpp diff --git a/tests/.gitignore b/tests/.gitignore index cbc381606cb7f..51b51ee6b29b6 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1,4 +1,5 @@ * +!combinator !*.* *.o ggml-common.h diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 90badf62af667..a8ff1debc6661 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -181,6 +181,18 @@ endif() llama_build_and_test(test-chat-parser.cpp) llama_build_and_test(test-chat-parser-combinator.cpp) + +# Combinator tests (modular) +file(GLOB_RECURSE COMBINATOR_TEST_SOURCES + combinator/*.cpp + combinator/*.hpp +) +add_executable(test-combinator ${COMBINATOR_TEST_SOURCES}) +target_link_libraries(test-combinator PRIVATE common) +install(TARGETS test-combinator RUNTIME) +add_test(NAME test-combinator COMMAND test-combinator) +set_property(TEST test-combinator PROPERTY LABELS main) + llama_build_and_test(test-chat-template.cpp) llama_build_and_test(test-json-partial.cpp) llama_build_and_test(test-log.cpp) diff --git a/tests/combinator/test-actions.cpp b/tests/combinator/test-actions.cpp new file mode 100644 index 0000000000000..3ec274ad19b8d --- /dev/null +++ b/tests/combinator/test-actions.cpp @@ -0,0 +1,127 @@ +#include "tests.h" + +class test_actions : public compound_test { +public: + test_actions() : compound_test("test_actions") { + // Test simple action - append matched text to content + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + auto word = p.chars("[a-z]+"); + return p.action(word, [](const parser_action & act) { + act.env.result.content += std::string(act.match); + }); + }); + + parser_environment env; + parser_context ctx("hello", &env); + auto result = parser.parse(ctx); + + h.assert_equals("result_is_success", true, result.is_success()); + h.assert_equals("result_is_hello", std::string("hello"), env.result.content); + }, "simple action - append matched text to content"); + + // Test multiple sequential actions - build a sentence + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + auto greeting = p.action(p.literal("hello"), [](const parser_action & act) { + act.env.result.content += std::string(act.match) + " "; + }); + + auto name = p.action(p.chars("[A-Z][a-z]+"), [](const parser_action & act) { + act.env.result.content += std::string(act.match); + act.env.scratchpad["name"] = std::string(act.match); + }); + + return greeting + p.literal(" ") + name; + }); + + parser_environment env; + parser_context ctx("hello Alice", &env); + auto result = parser.parse(ctx); + + h.assert_equals("result_is_success", true, result.is_success()); + h.assert_equals("result_content", std::string("hello Alice"), env.result.content); + h.assert_equals("scratchpad_name", std::string("Alice"), std::get(env.scratchpad["name"])); + }, "multiple sequential actions - build a sentence"); + + // Test using scratchpad for intermediate calculations + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + auto digit = p.action(p.one("[0-9]"), [](const parser_action & act) { + auto it = act.env.scratchpad.find("sum"); + int current_sum = it != act.env.scratchpad.end() ? std::get(it->second) : 0; + current_sum += (act.match[0] - '0'); + act.env.scratchpad["sum"] = current_sum; + }); + + return p.one_or_more(digit + p.optional(p.literal("+"))); + }); + + parser_environment env; + parser_context ctx("1+2+3+4", &env); + auto result = parser.parse(ctx); + + h.assert_equals("result_is_success", true, result.is_success()); + h.assert_equals("scratchpad_sum", 10, std::get(env.scratchpad["sum"])); // 1+2+3+4 = 10 + }, "using scratchpad for intermediate calculations"); + + // Test actions don't run when parse fails + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + return p.action(p.literal("success"), [](const parser_action & act) { + act.env.result.content = "action_ran"; + }); + }); + + parser_environment env; + parser_context ctx("failure", &env); + auto result = parser.parse(ctx); + + h.assert_equals("result_is_fail", true, result.is_fail()); + h.assert_equals("result_content_empty", std::string(""), env.result.content); // Action should not have run + }, "actions don't run when parse fails"); + + // Test Actions work with partial parsing + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + auto content = p.action(p.until(""), [](const parser_action & act) { + act.env.result.content += std::string(act.match); + }); + return "" << content << ""; + }); + + { + parser_environment env; + parser_context ctx("hello ", &env, false); + auto result = parser.parse(ctx); + + h.assert_equals("result_is_need_more_input_1", true, result.is_need_more_input()); + h.assert_equals("result_content_1", std::string("hello "), env.result.content); + } + + { + parser_environment env; + parser_context ctx("hello world", &env, false); + auto result = parser.parse(ctx); + + h.assert_equals("result_is_need_more_input_2", true, result.is_need_more_input()); + h.assert_equals("result_content_2", std::string("hello world"), env.result.content); + } + + { + parser_environment env; + parser_context ctx("hello world", &env, true); + auto result = parser.parse(ctx); + + h.assert_equals("result_is_success", true, result.is_success()); + h.assert_equals("result_content_final", std::string("hello world"), env.result.content); + } + }, "actions work with partial parsing"); + } + + // Provide a convenient way to run all tests + void run_all_tests() { + run_all(); + summary(); + } +}; \ No newline at end of file diff --git a/tests/combinator/test-combinator-all.cpp b/tests/combinator/test-combinator-all.cpp new file mode 100644 index 0000000000000..90df6daa9aba3 --- /dev/null +++ b/tests/combinator/test-combinator-all.cpp @@ -0,0 +1,36 @@ +#include "test-partial-parsing.cpp" +#include "test-one.cpp" +#include "test-optional.cpp" +#include "test-recursive-references.cpp" +#include "test-json-parser.cpp" +#include "test-complete-example.cpp" +#include "test-actions.cpp" +#include "test-gbnf-generation.cpp" + +int main() { + test_partial_parsing partial_parsing_test; + partial_parsing_test.run_all_tests(); + + test_one one_test; + one_test.run_all_tests(); + + test_optional optional_test; + optional_test.run_all_tests(); + + test_recursive_references recursive_references_test; + recursive_references_test.run_all_tests(); + + test_json_parser json_parser_test; + json_parser_test.run_all_tests(); + + test_complete_example complete_example_test; + complete_example_test.run_all_tests(); + + test_actions actions_test; + actions_test.run_all_tests(); + + test_gbnf_generation gbnf_generation_test; + gbnf_generation_test.run_all_tests(); + + return 0; +} \ No newline at end of file diff --git a/tests/combinator/test-complete-example.cpp b/tests/combinator/test-complete-example.cpp new file mode 100644 index 0000000000000..8a09bc70cb50b --- /dev/null +++ b/tests/combinator/test-complete-example.cpp @@ -0,0 +1,145 @@ +#include "json-schema-to-grammar.h" +#include "tests.h" +#include + +class test_complete_example : public compound_test { + public: + test_complete_example() : compound_test("test_complete_example") { + /* Parser for a fictitious model that outputs: + * + * + * ... reasoning content ... + * + * ... content ... + * + * tool_name + * { ... json args ... } + * + */ + auto parser = build_parser([](parser_builder & p) { + auto reasoning = + p.add_rule("reasoning", "" << p.append_reasoning(p.until("")) << ""); + + auto content = p.add_rule("content", p.append_content(p.until(""))); + + auto json = p.json(); + + auto tool_call_name = + p.add_rule("tool-call-name", "" << p.capture_tool_call_name(p.until("")) << ""); + + auto schema = nlohmann::json::parse(R"({"type": "object"})"); + + auto tool_call_args = p.add_rule( + "tool-call-args", + "" << p.capture_tool_call_args(p.schema(p.succeed(json), "get_weather", schema)) << ""); + + auto tool_call = + p.add_rule("tool-call", "" << p.add_tool_call(tool_call_name << p.succeed(tool_call_args)) + << ""); + + return reasoning << p.optional(content) << p.optional(tool_call); + }); + + // Test complete input + std::string input = + std::string(R"(I need to call get_weather with city = New Yorkget_weather{"city": "New York"})"); + parser_environment env; + parser_context ctx(input, &env); + + auto result = parser.parse(ctx); + + // Test complete input with reasoning and tool call + add_test( + [env, input, result](test_harness h) { + h.assert_equals("parse_success", true, result.is_success()); + h.assert_equals("parse_end", (size_t) input.size(), result.end); + h.assert_equals("reasoning_content", std::string("I need to call get_weather with city = New York"), + env.result.reasoning_content); + h.assert_equals("tool_calls_size", (size_t) 1, env.result.tool_calls.size()); + h.assert_equals("tool_call_id", std::string(""), env.result.tool_calls[0].id); + h.assert_equals("tool_call_name", std::string("get_weather"), env.result.tool_calls[0].name); + h.assert_equals("tool_call_args", std::string(R"({"city": "New York"})"), + env.result.tool_calls[0].arguments); + }, + "complete_tool_call_parsing"); + + // Test partial input + add_test([parser](test_harness h) { + std::string input = R"(I need to call get_weather)"; + parser_environment env = parser_environment(); + parser_context ctx = parser_context(input, &env, /* .is_input_complete = */ false); + + auto result = parser.parse(ctx); + + h.assert_equals("needs_more_input", true, result.is_need_more_input()); + h.assert_equals("reasoning_content", std::string("I need to call get_weather"), env.result.reasoning_content); + }, "partial_input"); + + + add_test([parser](test_harness h) { + std::string input = R"(I need to call I need to call get_weatherI need to call get_weatherget_weather)"; + parser_environment env = parser_environment(); + parser_context ctx = parser_context(input, &env, /* .is_input_complete = */ false); + + auto result = parser.parse(ctx); + + h.assert_equals("needs_more_input", true, result.is_need_more_input()); + h.assert_equals("reasoning_content", std::string("I need to call get_weather"), env.result.reasoning_content); + }, "tool_call_incomplete"); + add_test([parser](test_harness h) { + std::string input = R"(I need to call get_weatherget_weatherI need to call get_weatherget_weather{"cit)"; + parser_environment env = parser_environment(); + parser_context ctx = parser_context(input, &env, /* .is_input_complete = */ false); + + auto result = parser.parse(ctx); + + h.assert_equals("needs_more_input", true, result.is_need_more_input()); + h.assert_equals("reasoning_content", std::string("I need to call get_weather"), env.result.reasoning_content); + h.assert_equals("tool_name", std::string("get_weather"), env.result.tool_calls[0].name); + h.assert_equals("tool_incomplete_arg", std::string(R"({"cit)"), env.result.tool_calls[0].arguments); + }, "tool_call_arg_incomplete"); + + auto gbnf = build_grammar([parser](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); + + add_test([gbnf](test_harness h) { + h.assert_equals("not_empty", false, gbnf.empty()); + }, "grammar_is_there"); + } + + // Provide a convenient way to run all tests + void run_all_tests() { + run_all(); + summary(); + } +}; diff --git a/tests/combinator/test-gbnf-generation.cpp b/tests/combinator/test-gbnf-generation.cpp new file mode 100644 index 0000000000000..685e4374db689 --- /dev/null +++ b/tests/combinator/test-gbnf-generation.cpp @@ -0,0 +1,176 @@ +#include "tests.h" +#include "json-schema-to-grammar.h" + +class test_gbnf_generation : public compound_test { +public: + test_gbnf_generation() : compound_test("test_gbnf_generation") { + // Test literal + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + return p.literal("hello"); + }); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); + + h.assert_equals("has_root_hello", true, gbnf.find("root ::= \"hello\"") != std::string::npos); + h.assert_equals("has_space", true, gbnf.find("space ::=") != std::string::npos); + }, "literal grammar generation"); + + // Test char class + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + return p.one("[a-z]"); + }); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); + + h.assert_equals("has_char_class", true, gbnf.find("root ::= [a-z]") != std::string::npos); + }, "char class grammar"); + + // Test sequence + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + return p.literal("hello") + p.literal(" ") + p.literal("world"); + }); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); + + h.assert_equals("has_proper_sequence", true, gbnf.find("root ::= \"hello\" \" \" \"world\"") != std::string::npos); + }, "sequence grammar"); + + // Test choice + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + return p.literal("cat") | p.literal("dog"); + }); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); + + h.assert_equals("has_proper_choice", true, gbnf.find("root ::= \"cat\" | \"dog\"") != std::string::npos); + }, "choice grammar"); + + // Test one_or_more + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + return p.one_or_more(p.one("[0-9]")); + }); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); + + h.assert_equals("has_proper_one_or_more", true, gbnf.find("root ::= [0-9]+") != std::string::npos); + }, "one_or_more grammar"); + + // Test zero_or_more + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + return p.zero_or_more(p.one("[a-z]")); + }); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); + + h.assert_equals("has_proper_zero_or_more", true, gbnf.find("root ::= [a-z]*") != std::string::npos); + }, "zero_or_more grammar"); + + // Test optional + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + return p.literal("hello") + p.optional(p.literal(" world")); + }); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); + + h.assert_equals("has_proper_optional", true, gbnf.find("root ::= \"hello\" \" world\"?") != std::string::npos); + }, "optional grammar"); + + // Test until + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + return p.until(""); + }); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); + + // Should generate pattern that prevents matching the full delimiter + h.assert_equals("has_proper_until", true, gbnf.find("root ::= ([^<] | \"<\" [^/] | \"])*") != std::string::npos); + }, "until grammar"); + + // Test complex expression with parentheses + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + return p.one_or_more(p.literal("a") | p.literal("b")); + }); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); + + h.assert_equals("has_proper_complex", true, gbnf.find("root ::= (\"a\" | \"b\")+") != std::string::npos); + }, "complex expressions with parentheses"); + + // Test rule references + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + auto digit = p.add_rule("digit", p.one("[0-9]")); + return p.one_or_more(digit); + }); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); + + // Should have digit rule defined and referenced + h.assert_equals("has_digit_rule", true, gbnf.find("digit ::= [0-9]") != std::string::npos); + h.assert_equals("has_root_digit_ref", true, gbnf.find("root ::= digit+") != std::string::npos); + }, "rule references"); + + // Test escaping in literals + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + return p.literal("hello\nworld\t!"); + }); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); + + h.assert_equals("has_escaping", true, gbnf.find("root ::= \"hello\\nworld\\t!\"") != std::string::npos); + }, "escaping in literals"); + + // Test operator<< (whitespace insertion) + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + return p.literal("hello") << p.literal("world"); + }); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); + + // Should inline the whitespace pattern + h.assert_equals("has_inlined_hello", true, gbnf.find("\"hello\"") != std::string::npos); + h.assert_equals("has_inlined_world", true, gbnf.find("\"world\"") != std::string::npos); + }, "operator<< (whitespace insertion)"); + } + + // Provide a convenient way to run all tests + void run_all_tests() { + run_all(); + summary(); + } +}; \ No newline at end of file diff --git a/tests/combinator/test-json-parser.cpp b/tests/combinator/test-json-parser.cpp new file mode 100644 index 0000000000000..ed238270bf3d2 --- /dev/null +++ b/tests/combinator/test-json-parser.cpp @@ -0,0 +1,99 @@ +#include "tests.h" + +class test_json_parser : public compound_test { +public: + test_json_parser() : compound_test("test_json_parser") { + // Test parsing a simple JSON object + add_test([](test_harness h) { + auto json = build_parser([](parser_builder & p) { + return p.json(); + }); + + std::string input = R"({"name": "test", "value": 42, "flag": true})"; + parser_context ctx(input); + + auto result = json.parse(ctx); + + h.assert_equals("result_is_success", true, result.is_success()); + h.assert_equals("result_end", input.size(), result.end); + }, "simple JSON object parsing"); + + // Test parsing a JSON array with mixed types + add_test([](test_harness h) { + auto json = build_parser([](parser_builder & p) { + return p.json(); + }); + + std::string input = R"([1, "hello", true, null, 3.14])"; + parser_context ctx(input); + + auto result = json.parse(ctx); + + h.assert_equals("result_is_success", true, result.is_success()); + h.assert_equals("result_end", input.size(), result.end); + }, "JSON array with mixed types"); + + // Test parsing nested JSON with objects and arrays + add_test([](test_harness h) { + auto json = build_parser([](parser_builder & p) { + return p.json(); + }); + + std::string input = R"({"users": [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}], "count": 2, "metadata": {"version": "1.0", "tags": ["admin", "user"]}})"; + parser_context ctx(input); + + auto result = json.parse(ctx); + + h.assert_equals("result_is_success", true, result.is_success()); + h.assert_equals("result_end", input.size(), result.end); + }, "nested JSON with objects and arrays"); + + // Test partial parsing - incomplete object + add_test([](test_harness h) { + auto json = build_parser([](parser_builder & p) { + return p.json(); + }); + + std::string input = R"({"name": "test", "value": )"; + parser_context ctx(input, false); + + auto result = json.parse(ctx); + + h.assert_equals("result_is_need_more_input", true, result.is_need_more_input()); + }, "partial parsing - incomplete object"); + + // Test partial parsing - incomplete array + add_test([](test_harness h) { + auto json = build_parser([](parser_builder & p) { + return p.json(); + }); + + std::string input = R"([1, 2, 3, )"; + parser_context ctx(input, false); + + auto result = json.parse(ctx); + + h.assert_equals("result_is_need_more_input", true, result.is_need_more_input()); + }, "partial parsing - incomplete array"); + + // Test partial parsing - incomplete nested structure + add_test([](test_harness h) { + auto json = build_parser([](parser_builder & p) { + return p.json(); + }); + + std::string input = R"({"data": {"nested": )"; + parser_context ctx(input, false); + + auto result = json.parse(ctx); + + h.assert_equals("result_is_need_more_input", true, result.is_need_more_input()); + }, "partial parsing - incomplete nested structure"); + } + + // Provide a convenient way to run all tests + void run_all_tests() { + run_all(); + summary(); + } +}; \ No newline at end of file diff --git a/tests/combinator/test-one.cpp b/tests/combinator/test-one.cpp new file mode 100644 index 0000000000000..a6f74c6cbfa3a --- /dev/null +++ b/tests/combinator/test-one.cpp @@ -0,0 +1,124 @@ +#include "tests.h" + +class test_one : public compound_test { +public: + test_one() : compound_test("test_one") { + // Test common escape sequences - newline + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + return p.one("[\\n\\t\\\\]"); + }); + + parser_context ctx; + parser_result result; + + ctx = parser_context("\n"); + result = parser.parse(ctx); + h.assert_equals("escape_sequence_newline", true, result.is_success()); + }, "escape_sequence_newline"); + + // Test common escape sequences - tab + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + return p.one("[\\n\\t\\\\]"); + }); + + parser_context ctx; + parser_result result; + + ctx = parser_context("\t"); + result = parser.parse(ctx); + h.assert_equals("escape_sequence_tab", true, result.is_success()); + }, "escape_sequence_tab"); + + // Test common escape sequences - backslash + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + return p.one("[\\n\\t\\\\]"); + }); + + parser_context ctx; + parser_result result; + + ctx = parser_context("\\"); + result = parser.parse(ctx); + h.assert_equals("escape_sequence_backslash", true, result.is_success()); + }, "escape_sequence_backslash"); + + // Test common escape sequences - space (should fail) + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + return p.one("[\\n\\t\\\\]"); + }); + + parser_context ctx; + parser_result result; + + ctx = parser_context(" "); + result = parser.parse(ctx); + h.assert_equals("escape_sequence_space_fail", true, result.is_fail()); + }, "escape_sequence_space_fail"); + + // Test escaped dash - 'a' should succeed + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + return p.one("[a\\-z]"); + }); + + parser_context ctx; + parser_result result; + + ctx = parser_context("a"); + result = parser.parse(ctx); + h.assert_equals("escaped_dash_a", true, result.is_success()); + }, "escaped_dash_a"); + + // Test escaped dash - '-' should succeed (literal dash) + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + return p.one("[a\\-z]"); + }); + + parser_context ctx; + parser_result result; + + ctx = parser_context("-"); + result = parser.parse(ctx); + h.assert_equals("escaped_dash_literal", true, result.is_success()); + }, "escaped_dash_literal"); + + // Test escaped dash - 'z' should succeed + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + return p.one("[a\\-z]"); + }); + + parser_context ctx; + parser_result result; + + ctx = parser_context("z"); + result = parser.parse(ctx); + h.assert_equals("escaped_dash_z", true, result.is_success()); + }, "escaped_dash_z"); + + // Test escaped dash - 'b' should NOT match (since \- is literal dash, not range) + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + return p.one("[a\\-z]"); + }); + + parser_context ctx; + parser_result result; + + ctx = parser_context("b"); + result = parser.parse(ctx); + h.assert_equals("escaped_dash_b_fail", true, result.is_fail()); + }, "escaped_dash_b_fail"); + } + + // Provide a convenient way to run all tests + void run_all_tests() { + run_all(); + summary(); + } +}; \ No newline at end of file diff --git a/tests/combinator/test-optional.cpp b/tests/combinator/test-optional.cpp new file mode 100644 index 0000000000000..16a3c66f81e2f --- /dev/null +++ b/tests/combinator/test-optional.cpp @@ -0,0 +1,49 @@ +#include "tests.h" + +class test_optional : public compound_test { +public: + test_optional() : compound_test("test_optional") { + // Full match with optional part present + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + return p.literal("hello") + p.optional(p.literal(" world")); + }); + + auto ctx = parser_context("hello world"); + auto result = parser.parse(ctx); + h.assert_equals("optional_present", true, result.is_success()); + int end_pos = result.end; + h.assert_equals("optional_present_end", 11, end_pos); + }, "optional_present"); + + // Full match with optional part absent + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + return p.literal("hello") + p.optional(p.literal(" world")); + }); + + auto ctx = parser_context("hello", true); + auto result = parser.parse(ctx); + h.assert_equals("optional_absent", true, result.is_success()); + int end_pos = result.end; + h.assert_equals("optional_absent_end", 5, end_pos); + }, "optional_absent"); + + // Partial match - waiting for more input to determine if optional matches + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + return p.literal("hello") + p.optional(p.literal(" world")); + }); + + auto ctx = parser_context("hello ", false); + auto result = parser.parse(ctx); + h.assert_equals("partial_match_need_more", true, result.is_need_more_input()); + }, "partial_match_need_more"); + } + + // Provide a convenient way to run all tests + void run_all_tests() { + run_all(); + summary(); + } +}; \ No newline at end of file diff --git a/tests/combinator/test-partial-parsing.cpp b/tests/combinator/test-partial-parsing.cpp new file mode 100644 index 0000000000000..8045ee9cd0475 --- /dev/null +++ b/tests/combinator/test-partial-parsing.cpp @@ -0,0 +1,283 @@ +#include "tests.h" + +class test_partial_parsing : public compound_test { +public: + test_partial_parsing() : compound_test("test_partial_parsing") { + // Literals - Basic Success + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + return p.literal("hello"); + }); + + parser_context ctx; + parser_result result; + + ctx = parser_context("hello"); + result = parser.parse(ctx); + h.assert_equals("literal_success", true, result.is_success()); + }, "literal_success"); + + // Char Classes - Basic Lowercase Success + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + return p.one("a-z"); + }); + + parser_context ctx; + parser_result result; + + ctx = parser_context("a"); + result = parser.parse(ctx); + h.assert_equals("char_class_lowercase_success", true, result.is_success()); + }, "char_class_lowercase_success"); + + // Char Classes - Uppercase Fail + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + return p.one("a-z"); + }); + + parser_context ctx; + parser_result result; + + ctx = parser_context("A"); + result = parser.parse(ctx); + h.assert_equals("char_class_uppercase_fail", true, result.is_fail()); + }, "char_class_uppercase_fail"); + + // Char Classes with Dash - Lowercase Success + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + return p.one("a-z-"); + }); + + parser_context ctx; + parser_result result; + + ctx = parser_context("f"); + result = parser.parse(ctx); + h.assert_equals("char_class_with_dash_lowercase", true, result.is_success()); + }, "char_class_with_dash_lowercase"); + + // Char Classes with Dash - Literal Dash Success + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + return p.one("a-z-"); + }); + + parser_context ctx; + parser_result result; + + ctx = parser_context("-"); + result = parser.parse(ctx); + h.assert_equals("char_class_with_dash_literal_dash", true, result.is_success()); + }, "char_class_with_dash_literal_dash"); + + // Char Classes with Dash - Uppercase Fail + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + return p.one("a-z-"); + }); + + parser_context ctx; + parser_result result; + + ctx = parser_context("A"); + result = parser.parse(ctx); + h.assert_equals("char_class_with_dash_uppercase_fail", true, result.is_fail()); + }, "char_class_with_dash_uppercase_fail"); + + // Sequences - Partial Match 1 + add_test([](test_harness h) { + auto parser = build_parser([](parser_builder& p) { + return p.literal(" +#include +#include +#include +#include +#include +#include + +class test_harness { // TODO: more prototypes? +private: + int& successes_; + int& failures_; + std::ostream& error_stream_; + +public: + test_harness(int& successes, int& failures, std::ostream& error_stream = std::cerr) + : successes_(successes), failures_(failures), error_stream_(error_stream) {} + + template + bool assert_equals(const std::string &label, T expected, T actual) { + if (expected != actual) { + error_stream_ << "[" << label << "] FAILED\n"; + error_stream_ << "Expected: " << expected << "\n"; + error_stream_ << "Actual: " << actual << "\n"; + error_stream_ << std::flush; + failures_++; + return false; + } + error_stream_ << "[" << label << "] PASSED\n"; + successes_++; + return true; + } +}; + +class base_test_case { +public: + virtual ~base_test_case() = default; + virtual bool run() = 0; + virtual std::string get_name() const = 0; +}; + +class test_case : public base_test_case { +private: + std::function test_func_; + std::string name_; + int successes = 0, failures = 0; + test_harness harness; + +public: + test_case(std::function test_func, const std::string& name) + : test_func_(std::move(test_func)), name_(name), + harness(successes, failures) {} + + bool run() override { + // clean counters on run + successes = 0; + failures = 0; + // execute run with harness + test_func_(harness); + std::cerr << "[" << get_name() << "] "; + if (is_success()) { + std::cerr << "PASSED" << '\n'; + return true; + } + std::cerr << "FAILED (" << successes << "/" << (successes + failures) << ")\n"; + return false; + } + + std::string get_name() const override { return name_; } + bool is_success() const { return successes > 0 && failures == 0; } +}; + +class compound_test { +private: + std::vector> test_cases_; + std::string name_; + int successes_ = 0; + int failures_ = 0; + std::unordered_map test_name_to_index_; + +public: + explicit compound_test(const std::string& name) : name_(name) {} + + // Add a test case + void add_test(const std::function& test_func, const std::string& test_name) { + auto test = std::make_unique(test_func, test_name); + int index = test_cases_.size(); + test_name_to_index_[test_name] = index; + test_cases_.push_back(std::move(test)); + } + + // Access test by name + bool operator[](const std::string& test_name) { + auto it = test_name_to_index_.find(test_name); + if (it == test_name_to_index_.end()) { + std::cerr << "Test case '" << test_name << "' not found in compound test '" << name_ << "'\n"; + return false; + } + + int index = it->second; + bool result = test_cases_[index]->run(); + + if (result) { + successes_++; + } else { + failures_++; + } + + return result; + } + + // Execute all tests + void run_all() { + std::cerr << "Running all tests for: " << name_ << "\n"; + for (auto& test_case : test_cases_) { + bool result = test_case->run(); + if (result) { + successes_++; + } else { + failures_++; + } + } + } + + // Display summary + void summary() { + std::cerr << "\n=== Compound Test Summary: " << name_ << " ===\n"; + std::cerr << "Successes: " << successes_ << "\n"; + std::cerr << "Failures: " << failures_ << "\n"; + std::cerr << "Total: " << (successes_ + failures_) << "\n"; + if (successes_ + failures_ > 0) { + std::cerr << "Pass Rate: " << (successes_ * 100.0 / (successes_ + failures_)) << "%\n"; + } + std::cerr << "========================================\n"; + } + + // Get results + int get_successes() const { return successes_; } + int get_failures() const { return failures_; } + int get_total() const { return successes_ + failures_; } + double get_pass_rate() const { + int total = successes_ + failures_; + return total > 0 ? (successes_ * 100.0 / total) : 0.0; + } +}; \ No newline at end of file From 7745261ad847d6d9b361e15d2c7b743b6c324c04 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Fri, 14 Nov 2025 05:02:05 -0600 Subject: [PATCH 044/183] improve qwen3 example --- common/chat-parser-combinator.cpp | 36 ++++-- common/chat-parser-combinator.h | 1 + tests/test-chat-parser-combinator.cpp | 166 ++++++++++++++++---------- 3 files changed, 129 insertions(+), 74 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index 2c366176d11f5..1776a3eecb187 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -138,7 +138,13 @@ class aho_corasick_matcher { build_fail_links(); } - size_t search(std::string_view sv, size_t start = 0) { + struct search_result { + size_t pos; + bool found; + bool is_partial; + }; + + search_result search(std::string_view sv, size_t start = 0) { size_t current = 0; for (auto i = start; i < sv.size(); ++i) { @@ -160,14 +166,14 @@ class aho_corasick_matcher { for (const auto & len : trie[current].word_lengths) { pos = std::min(pos, i - len + 1); } - return pos; + return search_result{pos, true, false}; } } if (trie[current].depth > 0) { - return sv.size() - trie[current].depth; + return search_result{sv.size() - trie[current].depth, true, true}; } - return sv.size(); + return search_result{sv.size(), false, false}; } struct prefix_and_next { @@ -916,30 +922,32 @@ class json_string_parser : public parser_base { // Matches all characters until a delimiter is found (delimiter not consumed). // S -> (!delim .)* class until_parser : public parser_base { - std::string delimiter_; - + std::vector delimiters_; aho_corasick_matcher matcher_; public: static constexpr parser_type type_value = PARSER_UNTIL; + until_parser(const std::vector & delimiters, int id) + : parser_base(id), delimiters_(delimiters), matcher_(delimiters) {} + until_parser(const std::string & delimiter, int id) - : parser_base(id), delimiter_(delimiter), matcher_({delimiter}) { - } + : until_parser(std::vector{delimiter}, id) {} parser_type type() const override { return type_value; } parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { - return parser_result(PARSER_RESULT_SUCCESS, start, matcher_.search(ctx.input, start)); + auto search_result = matcher_.search(ctx.input, start); + return parser_result(PARSER_RESULT_SUCCESS, start, search_result.pos); } std::string dump() const override { - return "Until(" + delimiter_ + ")"; + return "Until(" + string_join(delimiters_, " | ") + ")"; } void accept(parser_visitor & visitor) override; - const std::string & delimiter() const { return delimiter_; } + std::vector delimiters() const { return delimiters_; } }; // Wraps a parser with JSON schema metadata for grammar generation. @@ -1257,7 +1265,7 @@ class gbnf_visitor : public parser_visitor { void visit(until_parser & p) override { // Generate pattern that matches prefixes but prevents full delimiter match - current_result_ = gbnf_excluding_pattern({p.delimiter()}); + current_result_ = gbnf_excluding_pattern(p.delimiters()); } void visit(not_parser &) override { @@ -1507,6 +1515,10 @@ parser parser_builder::until(const std::string & delimiter) { return parser(std::make_shared(delimiter, counter_->next())); } +parser parser_builder::until_one_of(const std::vector & delimiters) { + return parser(std::make_shared(delimiters, counter_->next())); +} + parser parser_builder::repeat(const parser & p, int min, int max) { return parser(std::make_shared(p, min, max, counter_->next())); } diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index 60eacc8a7b8bf..d695494f1f193 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -219,6 +219,7 @@ class parser_builder { // Matches all characters until a delimiter is found (delimiter not consumed). // S -> (!delim .)* parser until(const std::string & delimiter); + parser until_one_of(const std::vector & delimiters); // Matches between min and max repetitions of a parser (inclusive). // S -> A{m,n} diff --git a/tests/test-chat-parser-combinator.cpp b/tests/test-chat-parser-combinator.cpp index f4ad8a64c3c70..058f0492e7104 100644 --- a/tests/test-chat-parser-combinator.cpp +++ b/tests/test-chat-parser-combinator.cpp @@ -793,66 +793,110 @@ static std::vector simple_tokenize(const std::string & input) { static void example_qwen3_coder() { auto parser = build_parser([](parser_builder & p) { + // ===== Actions ===== + + auto start_arg = [&](const parser_action & act) { + if (act.env.tool_call_args != "{") { + act.env.tool_call_args += ","; + } + act.env.tool_call_args += "\""; + }; + + auto close_string_arg = [&](const parser_action & act) { + if (act.env.scratchpad.find("in-string-arg") != act.env.scratchpad.end()) { + act.env.tool_call_args += "\""; + act.env.scratchpad.erase("in-string-arg"); + } + }; + + auto append_arg_name = [](const parser_action & act) { + act.env.tool_call_args += std::string(act.match); + }; + + auto append_arg_colon = [](const parser_action & act) { + act.env.tool_call_args += "\":"; + }; + + auto open_function_args = [&](const parser_action & act) { + act.env.tool_call_args += "{"; + }; + + auto close_function_args = [&](const parser_action & act) { + close_string_arg(act); + act.env.tool_call_args += "}"; + }; + + auto open_string_arg = [&](const parser_action & act) { + act.env.tool_call_args += "\""; + act.env.scratchpad["in-string-arg"] = true; + }; + + auto append_string_content = [&](const parser_action & act) { + // TODO: add a JSON escape helper + act.env.tool_call_args += std::string(act.match); + }; + + auto append_json_arg = [&](const parser_action & act) { + // JSON should already be properly formatted + act.env.tool_call_args += std::string(act.match); + + // This can be streamed by passing p.success(json), but we have + // to be mindful of the potential backtracking--it only works + // if we only keep the last value... + }; + + // ===== Grammar Rules ===== + auto thinking = p.add_rule("thinking", "" << p.append_reasoning(p.until("")) << ""); auto content = p.add_rule("content", p.append_content(p.until(""))); auto arg_start = p.add_rule("arg-start", - p.action("", [](const parser_action & act) { - act.env.tool_call_args += "\":"; + p.action(""); + auto arg_name = p.add_rule("arg-name", + p.action(p.chars("[a-zA-Z0-9_]"), append_arg_name) + + p.action(">", append_arg_colon)); + + auto arg_end = p.add_rule("arg-end", + "" + p.choice({ + arg_start, + p.action("", close_function_args) + })); + + // Consume string argument until either another parameter or the + // function closing tag follows. + auto string_arg_content = p.add_rule("arg-string-content", + p.until_one_of({ + "" + })); auto string_arg = p.add_rule("arg-string", - p.action(arg_start, [&](const parser_action & act) { - act.env.tool_call_args += "\""; - }) - << p.action(p.until(""), [&](const parser_action & act) { - // TODO: add a JSON escape helper - act.env.tool_call_args += std::string(act.match); - }) - << p.action(arg_end, [&](const parser_action & act) { - act.env.tool_call_args += "\""; - })); + p.action(arg_name, open_string_arg) + << p.action(string_arg_content, append_string_content) + << arg_end); auto json = p.json(); auto json_arg = p.add_rule("arg-json", - arg_start - << p.action(json, [&](const parser_action & act) { - // JSON should already be properly formatted - act.env.tool_call_args += std::string(act.match); - - // This can be streamed by passing p.success(json), but we have - // to be mindful of the potential backtracking--it only works - // if we only keep the last value... - }) + arg_name + << p.action(json, append_json_arg) << arg_end); auto function = p.add_rule("function", p.add_tool_call( "", [&](const parser_action & act) { - act.env.tool_call_args += "{"; - }) - + p.one_or_more(p.space() + (json_arg | string_arg)) - << p.action("", [&](const parser_action & act) { - act.env.tool_call_args += "}"; - }))); + + p.action(">", open_function_args) + + arg_start + + p.one_or_more(json_arg | string_arg))); auto tool_call = p.add_rule("tool-call", - "" << p.one_or_more(function) << ""); + "" + p.one_or_more(function) + ""); return thinking + p.optional(p.space() + content) + p.zero_or_more(p.space() + tool_call); @@ -864,18 +908,19 @@ static void example_qwen3_coder() { "and check access time within the last 30 days. I'll need to use the search_files function." "Based on your requirements, I'll search for log files over 100MB that haven't been " "accessed in the last month. This will help identify candidates for cleanup or archival.\n\n" - "\n" - "\n" - "/var/log\n" - "*.log\n" - "100\n" - "5\n" - "false\n" - "30\n" - "true\n" - "size\n" - "{\"exclude_patterns\": [\"*temp*\", \"*cache*\"], \"file_types\": [\"regular\"]}\n" - "\n" + "" + "" + "/var/log" + "*.log" + "searching for blah" + "100" + "5" + "false" + "30" + "true" + "size" + "{\"exclude_patterns\": [\"*temp*\", \"*cache*\"], \"file_types\": [\"regular\"]}" + "" ""; std::vector tokens = simple_tokenize(input); @@ -889,12 +934,11 @@ static void example_qwen3_coder() { parser_context ctx(in, &env, it == tokens.end() - 1); auto result = parser.parse(ctx); - if (result.is_fail()) { - break; - } + assert_equals(false, result.is_fail()); + std::cout << "=================================\n"; + std::cout << in << "\n\n"; /* - std::cout << "Input:\n" << in << "\n\n"; std::cout << "Reasoning: " << prev.reasoning_content << "\n"; std::cout << "Content : " << prev.content << "\n"; if (!prev.tool_calls.empty()) { @@ -911,20 +955,17 @@ static void example_qwen3_coder() { auto diffs = common_chat_msg_diff::compute_diffs(prev, env.result); prev = env.result; - /* std::cout << "----\n"; std::cout << "Reasoning: " << prev.reasoning_content << "\n"; std::cout << "Content : " << prev.content << "\n"; if (!prev.tool_calls.empty()) { - std::cout << "\n=== Tool Calls ===\n"; + std::cout << "\n-- tool calls --\n"; for (const auto & tc : prev.tool_calls) { - std::cout << "ID : " << tc.id << "\n"; - std::cout << "Name: " << tc.name << "\n"; - std::cout << "Args: " << tc.arguments << "\n"; + std::cout << " ID : " << tc.id << "\n"; + std::cout << " Name: " << tc.name << "\n"; + std::cout << " Args: " << tc.arguments << "\n\n"; } } - std::cout << "======================\n"; - */ /* std::cout << "=== Diffs ===\n\n"; @@ -1171,6 +1212,7 @@ int main() { std::cout << "All tests passed!\n"; example_qwen3_coder(); + return 0; std::cout << "\n== Benchmarks ==\n"; std::string example_reasoning = From 749212346774841e0b86743f0f0c8e9ad3834de1 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Fri, 14 Nov 2025 07:31:46 -0600 Subject: [PATCH 045/183] implement sax-style parsing and refactor --- common/chat-parser-combinator.cpp | 164 +++++----- common/chat-parser-combinator.h | 92 +++--- tests/test-chat-parser-combinator.cpp | 418 ++++++++------------------ 3 files changed, 266 insertions(+), 408 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index 1776a3eecb187..6f9a07b7db268 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -18,6 +18,7 @@ enum parser_type { PARSER_OPTIONAL, PARSER_ZERO_OR_MORE, PARSER_ONE_OR_MORE, + PARSER_AND, PARSER_NOT, PARSER_ANY, PARSER_CHARS, @@ -595,6 +596,43 @@ class optional_parser : public repetition_parser { void accept(parser_visitor & visitor) override; }; +// Positive lookahead: succeeds if child parser succeeds, consumes no input. +// S -> &A +class and_parser : public parser_base { + parser parser_; + + public: + static constexpr parser_type type_value = PARSER_AND; + + and_parser(const parser & parser, int id) : parser_base(id), parser_(parser) {} + + parser_type type() const override { return type_value; } + + parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { + auto result = parser_->parse(ctx, start); + if (result.is_success()) { + return parser_result(PARSER_RESULT_SUCCESS, start); + } + if (result.is_need_more_input()) { + return result; + } + return parser_result(PARSER_RESULT_SUCCESS, start); + } + + void assign_id(std::shared_ptr counter) override { + parser_base::assign_id(counter); + parser_->assign_id(counter); + } + + std::string dump() const override { + return "And(" + parser_->dump() + ")"; + } + + void accept(parser_visitor & visitor) override; + + const parser & child() const { return parser_; } +}; + // Negative lookahead: succeeds if child parser fails, consumes no input. // S -> !A class not_parser : public parser_base { @@ -1009,7 +1047,44 @@ class rule_parser : public parser_base { return parser_result(PARSER_RESULT_FAIL, start); } - return it->second->parse(ctx, start); + // Fire NODE_START event + if (ctx.event_handler && ctx.env) { + ctx.event_handler(parse_event{ + PARSER_EVENT_NODE_START, + name_, + start, + start, + "", + PARSER_RESULT_FAIL, + ctx.current_depth + }, *ctx.env); + ctx.current_depth++; + } + + // Parse the referenced rule + auto result = it->second->parse(ctx, start); + + // Fire NODE_END event + if (ctx.event_handler && ctx.env) { + ctx.current_depth--; + std::string_view text = ctx.input; + if (result.start < ctx.input.size()) { + text = text.substr(result.start, result.end - result.start); + } else { + text = ""; + } + ctx.event_handler(parse_event{ + PARSER_EVENT_NODE_END, + name_, + result.start, + result.end, + text, + result.type, + ctx.current_depth + }, *ctx.env); + } + + return result; } std::string dump() const override { @@ -1118,6 +1193,7 @@ class parser_visitor { virtual void visit(optional_parser & p) = 0; virtual void visit(repetition_parser & p) = 0; virtual void visit(until_parser & p) = 0; + virtual void visit(and_parser & p) = 0; virtual void visit(not_parser & p) = 0; virtual void visit(any_parser & p) = 0; virtual void visit(space_parser & p) = 0; @@ -1268,6 +1344,10 @@ class gbnf_visitor : public parser_visitor { current_result_ = gbnf_excluding_pattern(p.delimiters()); } + void visit(and_parser &) override { + current_result_ = ""; + } + void visit(not_parser &) override { // NOT is tricky in GBNF - for now, emit error LOG_ERR("NOT operator not directly supported in GBNF generation\n"); @@ -1362,6 +1442,7 @@ void zero_or_more_parser::accept(parser_visitor & visitor) { visitor.visit(*this void optional_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void repetition_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void until_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } +void and_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void not_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void any_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void space_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } @@ -1483,6 +1564,10 @@ parser parser_builder::optional(const parser & p) { return parser(std::make_shared(p, counter_->next())); } +parser parser_builder::peek(const parser & p) { + return parser(std::make_shared(p, counter_->next())); +} + parser parser_builder::negate(const parser & p) { return parser(std::make_shared(p, counter_->next())); } @@ -1535,82 +1620,13 @@ parser parser_builder::action(const parser & p, std::function(p, std::move(fn), when, counter_->next())); } -parser parser_builder::succeed(const parser & p, int when) { - return action(p, [](const parser_action & act) { - act.result.type = PARSER_RESULT_SUCCESS; - }, when); -} - -parser parser_builder::append_reasoning(const parser & p) { - return action(p, [](const parser_action & act) { - if (!act.env.result.reasoning_content.empty()) { - act.env.result.reasoning_content += "\n"; - } - act.env.result.reasoning_content += act.match; - }, PARSER_RESULT_SUCCESS | PARSER_RESULT_NEED_MORE_INPUT); -} - -parser parser_builder::append_content(const parser & p) { - return action(p, [](const parser_action & act) { - if (!act.env.result.content.empty()) { - act.env.result.content += "\n"; - } - act.env.result.content += act.match; - }, PARSER_RESULT_SUCCESS | PARSER_RESULT_NEED_MORE_INPUT); -} - -parser parser_builder::capture(const parser & p, const std::string & key, bool unescape_json) { - return action(p, [key, unescape_json](const parser_action & act) { - std::string value = unescape_json ? unescape_json_string(act.match) : std::string(act.match); - act.env.scratchpad[key] = std::move(value); +parser parser_builder::capture(const std::string & key, const parser & p) { + return action(p, [key](const parser_action & act) { + std::string value = std::string(act.match); + act.env.captures[key] = std::move(value); }, PARSER_RESULT_SUCCESS); } -parser parser_builder::capture_tool_call_id(const parser & p, bool unescape_json) { - return action(p, [unescape_json](const parser_action & act) { - act.env.tool_call_id = unescape_json ? unescape_json_string(act.match) : std::string(act.match); - }, PARSER_RESULT_SUCCESS); -} - -parser parser_builder::capture_tool_call_name(const parser & p, bool unescape_json) { - return action(p, [unescape_json](const parser_action & act) { - act.env.tool_call_name = unescape_json ? unescape_json_string(act.match) : std::string(act.match); - }, PARSER_RESULT_SUCCESS); -} - -parser parser_builder::capture_tool_call_args(const parser & p, bool unescape_json) { - return action(p, [unescape_json](const parser_action & act) { - act.env.tool_call_args = unescape_json ? unescape_json_string(act.match) : std::string(act.match); - }, PARSER_RESULT_SUCCESS | PARSER_RESULT_NEED_MORE_INPUT); -} - -parser parser_builder::add_tool_call(const parser & p) { - return action(p, [](const parser_action & act) { - if (!act.env.tool_call_name.empty() && !act.env.tool_call_args.empty()) { - auto tool_call = common_chat_tool_call{ - act.env.tool_call_name, - act.env.tool_call_args, - act.env.tool_call_id - }; - act.env.result.tool_calls.push_back(tool_call); - } - - // Clear the fields to prevent bleeding to next tool call - act.env.tool_call_id.clear(); - act.env.tool_call_name.clear(); - act.env.tool_call_args.clear(); - }, PARSER_RESULT_SUCCESS | PARSER_RESULT_NEED_MORE_INPUT); -} - -parser parser_builder::json_key(const std::string & name, const parser & p) { - return literal("\"" + name + "\"") << literal(":") << p; -} - -parser parser_builder::json_string(const parser & p) { - auto quote = literal("\""); - return quote + p + quote; -} - parser parser_builder::add_rule(const std::string & name, const parser & p) { (*rules_)[name] = p; return rule(name); diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index d695494f1f193..d64b08cc27d8b 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -10,20 +10,13 @@ #include #include #include -#include struct common_grammar_builder; struct parser_environment { common_chat_msg result; - // Tool call fields for building tool calls - std::string tool_call_id; - std::string tool_call_name; - std::string tool_call_args; - - // Scratch pad for any custom logic - std::unordered_map> scratchpad; + std::unordered_map captures; }; enum parser_result_type { @@ -72,6 +65,30 @@ struct parser_action { std::string_view match; }; +enum parse_event_type { + PARSER_EVENT_NODE_START, + PARSER_EVENT_NODE_END, +}; + +struct parse_event { + parse_event_type type; + std::string rule; + size_t start; + size_t end; + std::string_view text; + parser_result_type status; + int depth; + + bool starting() const { return type == PARSER_EVENT_NODE_START; } + bool ending() const { return type == PARSER_EVENT_NODE_END; } + + bool success() const { return status == PARSER_RESULT_SUCCESS; } + bool partial() const { return status == PARSER_RESULT_NEED_MORE_INPUT; } + bool fail() const { return status == PARSER_RESULT_FAIL; } +}; + +using parse_event_handler = std::function; + class parse_cache { std::unordered_map results; @@ -86,27 +103,32 @@ struct parser_context { parse_cache memo; bool input_is_complete; parser_environment * env; + parse_event_handler event_handler; + int current_depth; parser_context() - : memo(), input_is_complete(true), env(nullptr) {} + : memo(), input_is_complete(true), env(nullptr), event_handler(nullptr), current_depth(0) {} parser_context(const std::string & input) - : input(input), memo(), input_is_complete(true), env(nullptr) {} + : input(input), memo(), input_is_complete(true), env(nullptr), event_handler(nullptr), current_depth(0) {} parser_context(const std::string & input, bool complete) - : input(input), memo(), input_is_complete(complete), env(nullptr) {} + : input(input), memo(), input_is_complete(complete), env(nullptr), event_handler(nullptr), current_depth(0) {} parser_context(const std::string & input, parse_cache memo, bool complete = true) - : input(input), memo(std::move(memo)), input_is_complete(complete), env(nullptr) {} + : input(input), memo(std::move(memo)), input_is_complete(complete), env(nullptr), event_handler(nullptr), current_depth(0) {} parser_context(const std::string & input, parser_environment * environment) - : input(input), memo(), input_is_complete(true), env(environment) {} + : input(input), memo(), input_is_complete(true), env(environment), event_handler(nullptr), current_depth(0) {} parser_context(const std::string & input, parser_environment * environment, bool complete) - : input(input), memo(), input_is_complete(complete), env(environment) {} + : input(input), memo(), input_is_complete(complete), env(environment), event_handler(nullptr), current_depth(0) {} parser_context(const std::string & input, parse_cache memo, parser_environment * environment, bool complete = true) - : input(input), memo(std::move(memo)), input_is_complete(complete), env(environment) {} + : input(input), memo(std::move(memo)), input_is_complete(complete), env(environment), event_handler(nullptr), current_depth(0) {} + + parser_context(const std::string & input, parser_environment * environment, parse_event_handler handler, bool complete = true) + : input(input), memo(), input_is_complete(complete), env(environment), event_handler(std::move(handler)), current_depth(0) {} }; class parser_base; @@ -188,6 +210,10 @@ class parser_builder { // S -> A? parser optional(const parser & p); + // Negative lookahead: succeeds if child parser fails, consumes no input. + // S -> !A + parser peek(const parser & p); + // Negative lookahead: succeeds if child parser fails, consumes no input. // S -> !A parser negate(const parser & p); @@ -237,10 +263,6 @@ class parser_builder { // Specialized single-pass JSON string parser with escape sequence handling parser json_string(); - // TODO: improve convenience functions to allow users to build specific JSON fields - parser json_key(const std::string & name, const parser & p); - parser json_string(const parser & p); - // Wraps a parser with JSON schema metadata for grammar generation. // Used internally to convert JSON schemas to GBNF grammar rules. parser schema(const parser & p, const std::string & name, const nlohmann::ordered_json & schema); @@ -250,36 +272,8 @@ class parser_builder { // S -> A [action] parser action(const parser & p, std::function fn, int when = PARSER_RESULT_SUCCESS); - // Convenience action wrappers for common patterns - - // Causes a rule to succeed - parser succeed(const parser & p, int when = PARSER_RESULT_NEED_MORE_INPUT); - - // Appends matched text to env.reasoning_content - parser append_reasoning(const parser & p); - - // Appends matched text to env.content - parser append_content(const parser & p); - - // Captures matched text to env.scratchpad[key] - // If unescape_json is true, the matched text is unescaped as a JSON string - parser capture(const parser & p, const std::string & key, bool unescape_json = false); - - // Captures matched text to env.tool_call_id - // If unescape_json is true, the matched text is unescaped as a JSON string - parser capture_tool_call_id(const parser & p, bool unescape_json = false); - - // Captures matched text to env.tool_call_name - // If unescape_json is true, the matched text is unescaped as a JSON string - parser capture_tool_call_name(const parser & p, bool unescape_json = false); - - // Captures matched text to env.tool_call_args - // If unescape_json is true, the matched text is unescaped as a JSON string - parser capture_tool_call_args(const parser & p, bool unescape_json = false); - - // Adds a tool call to env.tool_calls using env.tool_call_{id,name,args} - // Clears the tool call fields after adding - parser add_tool_call(const parser & p); + // Captures matched text to env.captures[key] + parser capture(const std::string & key, const parser & p); parser add_rule(const std::string & name, const parser & p); diff --git a/tests/test-chat-parser-combinator.cpp b/tests/test-chat-parser-combinator.cpp index 058f0492e7104..fc6e863b8820b 100644 --- a/tests/test-chat-parser-combinator.cpp +++ b/tests/test-chat-parser-combinator.cpp @@ -369,127 +369,6 @@ static void test_json_parser() { } } -static void test_complete_example() { - // Parser for a fictitious model that outputs: - // - // - // ... reasoning content ... - // - // ... content ... - // - // tool_name - // { ... json args ... } - // - // - auto parser = build_parser([](parser_builder & p) { - auto reasoning = p.add_rule("reasoning", - "" << p.append_reasoning(p.until("")) << ""); - - auto content = p.add_rule("content", - p.append_content(p.until(""))); - - auto json = p.json(); - - auto tool_call_name = p.add_rule("tool-call-name", - "" << p.capture_tool_call_name(p.until("")) << ""); - - auto schema = nlohmann::ordered_json::parse(R"({"type": "object"})"); - - auto tool_call_args = p.add_rule("tool-call-args", - "" << p.capture_tool_call_args(p.schema(p.succeed(json), "get_weather", schema)) << ""); - - auto tool_call = p.add_rule("tool-call", - "" << p.add_tool_call(tool_call_name << p.succeed(tool_call_args)) << ""); - - return reasoning << p.optional(content) << p.optional(tool_call); - }); - - // Test complete input - { - std::string input = R"(I need to call get_weather with city = New Yorkget_weather{"city": "New York"})"; - parser_environment env; - parser_context ctx(input, &env); - - auto result = parser.parse(ctx); - - assert_equals(true, result.is_success()); - assert_equals(input.size(), result.end); - assert_equals("I need to call get_weather with city = New York", env.result.reasoning_content); - assert_equals((size_t)1, env.result.tool_calls.size()); - assert_equals("", env.result.tool_calls[0].id); - assert_equals("get_weather", env.result.tool_calls[0].name); - assert_equals(R"({"city": "New York"})", env.result.tool_calls[0].arguments); - } - - // Test partial input - { - std::string input = R"(I need to call get_weather)"; - parser_environment env = parser_environment(); - parser_context ctx = parser_context(input, &env, /* .is_input_complete = */ false); - - auto result = parser.parse(ctx); - - assert_equals(true, result.is_need_more_input()); - assert_equals("I need to call get_weather", env.result.reasoning_content); - } - { - std::string input = R"(I need to call I need to call get_weatherI need to call get_weatherget_weather)"; - parser_environment env = parser_environment(); - parser_context ctx = parser_context(input, &env, /* .is_input_complete = */ false); - - auto result = parser.parse(ctx); - - assert_equals(true, result.is_need_more_input()); - assert_equals("I need to call get_weather", env.result.reasoning_content); - } - { - std::string input = R"(I need to call get_weatherget_weatherI need to call get_weatherget_weather{"cit)"; - parser_environment env = parser_environment(); - parser_context ctx = parser_context(input, &env, /* .is_input_complete = */ false); - - auto result = parser.parse(ctx); - - assert_equals(true, result.is_need_more_input()); - assert_equals("I need to call get_weather", env.result.reasoning_content); - assert_equals("get_weather", env.result.tool_calls[0].name); - assert_equals(R"({"cit)", env.result.tool_calls[0].arguments); - } - - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder); - }); - - std::cout << "Grammar:\n" << gbnf << "\n"; -} - static void test_actions() { { // Test simple action - append matched text to content @@ -516,7 +395,7 @@ static void test_actions() { auto name = p.action(p.chars("[A-Z][a-z]+"), [](const parser_action & act) { act.env.result.content += std::string(act.match); - act.env.scratchpad["name"] = std::string(act.match); + act.env.captures["name"] = std::string(act.match); }); return greeting + p.literal(" ") + name; @@ -528,27 +407,7 @@ static void test_actions() { assert_equals(true, result.is_success()); assert_equals("hello Alice", env.result.content); - assert_equals("Alice", std::get(env.scratchpad["name"])); - } - { - // Test using scratchpad for intermediate calculations - auto parser = build_parser([](parser_builder& p) { - auto digit = p.action(p.one("[0-9]"), [](const parser_action & act) { - auto it = act.env.scratchpad.find("sum"); - int current_sum = it != act.env.scratchpad.end() ? std::get(it->second) : 0; - current_sum += (act.match[0] - '0'); - act.env.scratchpad["sum"] = current_sum; - }); - - return p.one_or_more(digit + p.optional(p.literal("+"))); - }); - - parser_environment env; - parser_context ctx("1+2+3+4", &env); - auto result = parser.parse(ctx); - - assert_equals(true, result.is_success()); - assert_equals(10, std::get(env.scratchpad["sum"])); // 1+2+3+4 = 10 + assert_equals("Alice", env.captures["name"]); } { // Test actions don't run when parse fails @@ -601,6 +460,39 @@ static void test_actions() { } } +static void test_sax_events() { + { + // Test basic event firing + auto parser = build_parser([](parser_builder& p) { + return p.add_rule("greeting", p.literal("hello")); + }); + + parser_environment env; + std::vector events; + + parser_context ctx("hello", &env, [&](const parse_event& evt, parser_environment&) { + events.push_back(evt); + }); + + auto result = parser.parse(ctx); + + assert_equals(true, result.is_success()); + assert_equals((size_t)2, events.size()); + assert_equals(PARSER_EVENT_NODE_START, events[0].type); + assert_equals("greeting", events[0].rule); + assert_equals((size_t)0, events[0].start); + assert_equals(0, events[0].depth); + + assert_equals(PARSER_EVENT_NODE_END, events[1].type); + assert_equals("greeting", events[1].rule); + assert_equals((size_t)0, events[1].start); + assert_equals((size_t)5, events[1].end); + assert_equals("hello", std::string(events[1].text)); + assert_equals(PARSER_RESULT_SUCCESS, events[1].status); + assert_equals(0, events[1].depth); + } +} + static void test_gbnf_generation() { { // Test literal @@ -793,114 +685,75 @@ static std::vector simple_tokenize(const std::string & input) { static void example_qwen3_coder() { auto parser = build_parser([](parser_builder & p) { - // ===== Actions ===== - - auto start_arg = [&](const parser_action & act) { - if (act.env.tool_call_args != "{") { - act.env.tool_call_args += ","; - } - act.env.tool_call_args += "\""; - }; - - auto close_string_arg = [&](const parser_action & act) { - if (act.env.scratchpad.find("in-string-arg") != act.env.scratchpad.end()) { - act.env.tool_call_args += "\""; - act.env.scratchpad.erase("in-string-arg"); - } - }; - - auto append_arg_name = [](const parser_action & act) { - act.env.tool_call_args += std::string(act.match); - }; - - auto append_arg_colon = [](const parser_action & act) { - act.env.tool_call_args += "\":"; - }; + auto thinking = p.add_rule("raw-reasoning", + "" << p.add_rule("reasoning-content", p.until("")) << ""); - auto open_function_args = [&](const parser_action & act) { - act.env.tool_call_args += "{"; - }; + auto content = p.add_rule("content", p.until("")); - auto close_function_args = [&](const parser_action & act) { - close_string_arg(act); - act.env.tool_call_args += "}"; - }; + auto arg_name = p.add_rule("arg-start", ""); + auto arg_end = p.add_rule("arg-end", "" + p.peek(p.literal("")); - auto open_string_arg = [&](const parser_action & act) { - act.env.tool_call_args += "\""; - act.env.scratchpad["in-string-arg"] = true; - }; - - auto append_string_content = [&](const parser_action & act) { - // TODO: add a JSON escape helper - act.env.tool_call_args += std::string(act.match); - }; - - auto append_json_arg = [&](const parser_action & act) { - // JSON should already be properly formatted - act.env.tool_call_args += std::string(act.match); - - // This can be streamed by passing p.success(json), but we have - // to be mindful of the potential backtracking--it only works - // if we only keep the last value... - }; + auto string_arg_content = p.add_rule("arg-string-content", + p.until_one_of({""})); - // ===== Grammar Rules ===== + auto string_arg = p.add_rule("arg-string", arg_name + string_arg_content + arg_end); - auto thinking = p.add_rule("thinking", - "" << p.append_reasoning(p.until("")) << ""); - - auto content = p.add_rule("content", p.append_content(p.until(""))); + auto json = p.json(); - auto arg_start = p.add_rule("arg-start", - p.action("", append_arg_colon)); + auto function = p.add_rule("function", + p.add_rule("function-start", "") + + p.one_or_more(json_arg | string_arg) + + ""); - auto arg_end = p.add_rule("arg-end", - "" + p.choice({ - arg_start, - p.action("", close_function_args) - })); + auto tool_call = p.add_rule("tool-call", + "" + p.one_or_more(function) + ""); - // Consume string argument until either another parameter or the - // function closing tag follows. - auto string_arg_content = p.add_rule("arg-string-content", - p.until_one_of({ - "" - })); + return thinking + p.optional(p.space() + content) + p.zero_or_more(p.space() + tool_call); + }); - auto string_arg = p.add_rule("arg-string", - p.action(arg_name, open_string_arg) - << p.action(string_arg_content, append_string_content) - << arg_end); + auto handler = [&](const parse_event & ev, parser_environment & env) { + if (ev.rule == "reasoning-content" && ev.ending()) { + env.result.reasoning_content = ev.text; + } - auto json = p.json(); + if (ev.rule == "content" && ev.ending()) { + env.result.content = ev.text; + } - auto json_arg = p.add_rule("arg-json", - arg_name - << p.action(json, append_json_arg) - << arg_end); + if (ev.rule == "function-start" && ev.ending() && ev.success()) { + env.result.tool_calls.emplace_back(); + auto & tc = env.result.tool_calls.back(); + tc.name = env.captures["tool-name"]; + } - auto function = p.add_rule("function", p.add_tool_call( - "", open_function_args) - + arg_start - + p.one_or_more(json_arg | string_arg))); + if (ev.rule == "arg-start" && ev.ending() && ev.success()) { + auto & tc = env.result.tool_calls.back(); + auto name = env.captures["arg-name"]; + if (tc.arguments.empty()) { + tc.arguments += "{"; + } else { + tc.arguments += ", "; + } + tc.arguments += "\"" + name + "\": "; + } - auto tool_call = p.add_rule("tool-call", - "" + p.one_or_more(function) + ""); + if (ev.rule == "arg-string-content" && ev.ending() && ev.success()) { + auto & tc = env.result.tool_calls.back(); + tc.arguments += "\"" + std::string(ev.text); + } + if (ev.rule == "arg-string" && ev.ending() && ev.success()) { + auto & tc = env.result.tool_calls.back(); + tc.arguments += "\""; + } - return thinking + p.optional(p.space() + content) + p.zero_or_more(p.space() + tool_call); - }); + if (ev.rule == "arg-json-content" && ev.ending() && (ev.success() || ev.partial())) { + auto & tc = env.result.tool_calls.back(); + tc.arguments += std::string(ev.text); + } + }; std::string input = "The user wants to find large log files that haven't been accessed recently. " @@ -932,9 +785,10 @@ static void example_qwen3_coder() { parser_environment env; parser_context ctx(in, &env, it == tokens.end() - 1); + ctx.event_handler = handler; - auto result = parser.parse(ctx); - assert_equals(false, result.is_fail()); + auto parse_result = parser.parse(ctx); + assert_equals(false, parse_result.is_fail()); std::cout << "=================================\n"; std::cout << in << "\n\n"; @@ -955,6 +809,7 @@ static void example_qwen3_coder() { auto diffs = common_chat_msg_diff::compute_diffs(prev, env.result); prev = env.result; +#if 0 std::cout << "----\n"; std::cout << "Reasoning: " << prev.reasoning_content << "\n"; std::cout << "Content : " << prev.content << "\n"; @@ -966,77 +821,40 @@ static void example_qwen3_coder() { std::cout << " Args: " << tc.arguments << "\n\n"; } } - - /* - std::cout << "=== Diffs ===\n\n"; - if (!diffs.empty()) { - for (size_t i = 0; i < diffs.size(); ++i) { - const auto& diff = diffs[i]; - - std::cout << "Diff #" << (i + 1) << "\n"; - - if (!diff.reasoning_content_delta.empty()) { - std::cout << " [Reasoning Content]: " << diff.reasoning_content_delta << "\n"; - } - - if (!diff.content_delta.empty()) { - std::cout << " [Content]: " << diff.content_delta << "\n"; - } - - if (diff.tool_call_index != std::string::npos) { - std::cout << " [Tool Call #" << diff.tool_call_index << "]" << "\n"; - - if (!diff.tool_call_delta.id.empty()) { - std::cout << " ID: " << diff.tool_call_delta.id << "\n"; - } - - if (!diff.tool_call_delta.name.empty()) { - std::cout << " Name: " << diff.tool_call_delta.name << "\n"; - } - - if (!diff.tool_call_delta.arguments.empty()) { - std::cout << " Arguments: " << diff.tool_call_delta.arguments << "\n"; - } - } - - std::cout << "\n"; - } - } else { - std::cout << "No changes detected.\n"; - } - */ +#endif } } static parser create_command_r7b_parser() { auto parser = build_parser([](parser_builder & p) { auto thinking = p.add_rule("thinking", - "<|START_THINKING|>" << p.append_reasoning(p.until("<|END_THINKING|>")) << "<|END_THINKING|>"); + "<|START_THINKING|>" << p.add_rule("reasoning-content", p.until("<|END_THINKING|>")) << "<|END_THINKING|>"); auto response = p.add_rule("response", - "<|START_RESPONSE|>" << p.append_content(p.until("<|END_RESPONSE|>")) << "<|END_RESPONSE|>"); + "<|START_RESPONSE|>" << p.add_rule("content", p.until("<|END_RESPONSE|>")) << "<|END_RESPONSE|>"); auto json = p.add_rule("json", p.json()); auto tool_call_id = p.add_rule("tool-call-id", - p.json_key("tool_call_id", "\"" + p.capture_tool_call_id(p.json_string(), /* unescape_json = */ true) + "\"")); + "\"tool_call_id\"" << (":" << p.add_rule("tool-call-id-value", "\"" + p.json_string() + "\""))); auto tool_call_name = p.add_rule("tool-name", - p.json_key("tool_name", "\"" + p.capture_tool_call_name(p.json_string(), /* unescape_json = */ true) + "\"")); + "\"tool_name\"" << (":" << p.add_rule("tool-name-value", "\"" + p.json_string() + "\""))); - auto tool_call_args = p.add_rule("tool-args", p.json_key("parameters", p.capture_tool_call_args(json))); + auto tool_call_args = p.add_rule("tool-args", + "\"parameters\"" << (":" << p.add_rule("tool-args-value", json))); auto tool_call_fields = p.add_rule("tool-call-fields", tool_call_id | tool_call_name | tool_call_args); auto tool_call = p.add_rule("tool-call", - "{" << p.add_tool_call(tool_call_fields << p.zero_or_more(p.literal(",") << tool_call_fields)) << "}"); + "{" << tool_call_fields << p.zero_or_more(p.literal(",") << tool_call_fields) << "}"); auto tool_calls = p.add_rule("tool-calls", "<|START_ACTION|>" << ("[" << tool_call << p.zero_or_more(p.literal(",") << tool_call) << "]") << "<|END_ACTION|>"); - return p.optional(thinking) << p.add_rule("content", tool_calls | response); + return p.optional(thinking) << (tool_calls | response); }); auto grammar = build_grammar([&](const common_grammar_builder & builder) { @@ -1051,6 +869,36 @@ static parser create_command_r7b_parser() { static void test_command_r7b_parser(const parser & p, const std::string & input, bool partial, bool print_results = false) { parser_environment env; parser_context ctx(input, &env, !partial); + + ctx.event_handler = [&](const parse_event & ev, parser_environment & env) { + if (ev.rule == "reasoning-content" && ev.ending()) { + env.result.reasoning_content = ev.text; + } + + if (ev.rule == "content" && ev.ending()) { + env.result.content = ev.text; + } + + if (ev.rule == "tool-call" && ev.starting()) { + env.result.tool_calls.emplace_back(); + } + + if (ev.rule == "tool-call-id-value" && ev.ending() && ev.success()) { + auto & tc = env.result.tool_calls.back(); + tc.id = ev.text; + } + + if (ev.rule == "tool-name-value" && ev.ending() && ev.success()) { + auto & tc = env.result.tool_calls.back(); + tc.name = ev.text; + } + + if (ev.rule == "tool-args-value" && ev.ending() && (ev.success() || ev.partial())) { + auto & tc = env.result.tool_calls.back(); + tc.arguments = ev.text; + } + }; + p.parse(ctx); if (print_results) { @@ -1206,13 +1054,13 @@ int main() { test_recursive_references(); test_optional(); test_json_parser(); - test_complete_example(); test_actions(); + test_sax_events(); test_gbnf_generation(); std::cout << "All tests passed!\n"; example_qwen3_coder(); - return 0; + //return 0; std::cout << "\n== Benchmarks ==\n"; std::string example_reasoning = From 843a2793f223cb666cfdcda4cdd9374048e6272b Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Fri, 14 Nov 2025 07:35:00 -0600 Subject: [PATCH 046/183] fix json string in test --- tests/test-chat-parser-combinator.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test-chat-parser-combinator.cpp b/tests/test-chat-parser-combinator.cpp index fc6e863b8820b..8d8653f052ba2 100644 --- a/tests/test-chat-parser-combinator.cpp +++ b/tests/test-chat-parser-combinator.cpp @@ -885,12 +885,12 @@ static void test_command_r7b_parser(const parser & p, const std::string & input, if (ev.rule == "tool-call-id-value" && ev.ending() && ev.success()) { auto & tc = env.result.tool_calls.back(); - tc.id = ev.text; + tc.id = nlohmann::json::parse(ev.text).get(); } if (ev.rule == "tool-name-value" && ev.ending() && ev.success()) { auto & tc = env.result.tool_calls.back(); - tc.name = ev.text; + tc.name = nlohmann::json::parse(ev.text).get(); } if (ev.rule == "tool-args-value" && ev.ending() && (ev.success() || ev.partial())) { From 9f9fd1c436caa7ef4c4652961d8b8fdb12910e0c Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Fri, 14 Nov 2025 07:56:59 -0600 Subject: [PATCH 047/183] rename classes to use common_chat_ prefix --- common/chat-parser-combinator.cpp | 511 +++++++++++++------------- common/chat-parser-combinator.h | 218 +++++------ tests/test-chat-parser-combinator.cpp | 236 ++++++------ 3 files changed, 482 insertions(+), 483 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index 6f9a07b7db268..e3c95bb6893de 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -1,7 +1,6 @@ #include "chat-parser-combinator.h" #include "json-schema-to-grammar.h" #include "common.h" -#include "chat.h" #include "log.h" #include @@ -11,35 +10,35 @@ #include enum parser_type { - PARSER_LITERAL, - PARSER_SEQUENCE, - PARSER_CHOICE, - PARSER_REPETITION, - PARSER_OPTIONAL, - PARSER_ZERO_OR_MORE, - PARSER_ONE_OR_MORE, - PARSER_AND, - PARSER_NOT, - PARSER_ANY, - PARSER_CHARS, - PARSER_RULE, - PARSER_UNTIL, - PARSER_SPACE, - PARSER_SCHEMA, - PARSER_ROOT, - PARSER_JSON_STRING, - PARSER_ACTION, + LITERAL, + SEQUENCE, + CHOICE, + REPETITION, + OPTIONAL, + ZERO_OR_MORE, + ONE_OR_MORE, + AND, + NOT, + ANY, + CHARS, + RULE, + UNTIL, + SPACE, + SCHEMA, + ROOT, + JSON_STRING, + ACTION, }; class parser_visitor; -class parser_base { +class common_chat_combinator_parser_base { protected: int id_; public: - parser_base(int id) : id_(id) {} - virtual ~parser_base() = default; + common_chat_combinator_parser_base(int id) : id_(id) {} + virtual ~common_chat_combinator_parser_base() = default; int id() const { return id_; } void set_id(int id) { id_ = id; } @@ -47,27 +46,27 @@ class parser_base { virtual parser_type type() const = 0; // Template Method: handles caching, delegates to parse_uncached() - virtual parser_result parse(parser_context & ctx, size_t start = 0) { + virtual common_chat_parse_result parse(common_chat_parse_context & ctx, size_t start = 0) { if (id_ == -1) { // Don't cache parsers with ID -1 (from operators) return parse_uncached(ctx, start); } // Check cache - auto cached = ctx.memo.get(id_, start); + auto cached = ctx.cache.get(id_, start); if (cached) { return *cached; } // Execute and cache auto result = parse_uncached(ctx, start); - return ctx.memo.set(id_, start, result); + return ctx.cache.set(id_, start, result); } // Actual parsing implementation (to be overridden by subclasses) - virtual parser_result parse_uncached(parser_context & ctx, size_t start = 0) = 0; + virtual common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) = 0; - virtual void assign_id(std::shared_ptr counter) { + virtual void assign_id(std::shared_ptr counter) { if (id_ == -1) { id_ = counter->next(); } @@ -79,7 +78,7 @@ class parser_base { // Convenience cast functions template -static std::shared_ptr cast(const std::shared_ptr & p) { +static std::shared_ptr cast(const std::shared_ptr & p) { if (p->type() != T::type_value) { return nullptr; } @@ -87,7 +86,7 @@ static std::shared_ptr cast(const std::shared_ptr & p) { } template -static std::shared_ptr cast(const parser & p) { +static std::shared_ptr cast(const common_chat_combinator_parser & p) { return cast(p.ptr()); } @@ -321,32 +320,32 @@ static std::string regex_excluding_pattern(const std::vector & stri // Matches an exact literal string. // S -> "hello" -class literal_parser : public parser_base { +class literal_parser : public common_chat_combinator_parser_base { std::string literal_; public: - static constexpr parser_type type_value = PARSER_LITERAL; + static constexpr parser_type type_value = LITERAL; - literal_parser(const std::string & literal, int id) : parser_base(id), literal_(literal) {} + literal_parser(const std::string & literal, int id) : common_chat_combinator_parser_base(id), literal_(literal) {} parser_type type() const override { return type_value; } - parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { + common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto pos = start; for (auto i = 0u; i < literal_.size(); ++i) { if (pos >= ctx.input.size()) { if (ctx.input_is_complete) { - return parser_result(PARSER_RESULT_FAIL, start); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); } - return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, pos); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start, pos); } if (ctx.input[pos] != literal_[i]) { - return parser_result(PARSER_RESULT_FAIL, start); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); } ++pos; } - return parser_result(PARSER_RESULT_SUCCESS, start, pos); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); } std::string dump() const override { @@ -360,13 +359,13 @@ class literal_parser : public parser_base { // Matches a sequence of parsers in order, all must succeed. // S -> A B C -class sequence_parser : public parser_base { - std::vector parsers_; +class sequence_parser : public common_chat_combinator_parser_base { + std::vector parsers_; public: - static constexpr parser_type type_value = PARSER_SEQUENCE; + static constexpr parser_type type_value = SEQUENCE; - sequence_parser(std::initializer_list parsers, int id) : parser_base(id) { + sequence_parser(std::initializer_list parsers, int id) : common_chat_combinator_parser_base(id) { for (const auto & p : parsers) { if (auto seq = cast(p)) { for (const auto & embedded : seq->parsers()) { @@ -380,22 +379,22 @@ class sequence_parser : public parser_base { parser_type type() const override { return type_value; } - parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { + common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto pos = start; for (const auto & p : parsers_) { auto result = p->parse(ctx, pos); if (!result.is_success()) { - return parser_result(result.type, start, result.end); + return common_chat_parse_result(result.type, start, result.end); } pos = result.end; } - return parser_result(PARSER_RESULT_SUCCESS, start, pos); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); } - void assign_id(std::shared_ptr counter) override { - parser_base::assign_id(counter); + void assign_id(std::shared_ptr counter) override { + common_chat_combinator_parser_base::assign_id(counter); for (auto & p : parsers_) { p->assign_id(counter); } @@ -412,18 +411,18 @@ class sequence_parser : public parser_base { void accept(parser_visitor & visitor) override; - const std::vector & parsers() const { return parsers_; } + const std::vector & parsers() const { return parsers_; } }; // Matches the first parser that succeeds from a list of alternatives. // S -> A | B | C -class choice_parser : public parser_base { - std::vector parsers_; +class choice_parser : public common_chat_combinator_parser_base { + std::vector parsers_; public: - static constexpr parser_type type_value = PARSER_CHOICE; + static constexpr parser_type type_value = CHOICE; - choice_parser(std::initializer_list parsers, int id) : parser_base(id) { + choice_parser(std::initializer_list parsers, int id) : common_chat_combinator_parser_base(id) { for (const auto & p : parsers) { if (auto choice = cast(p)) { for (const auto & embedded : choice->parsers()) { @@ -437,7 +436,7 @@ class choice_parser : public parser_base { parser_type type() const override { return type_value; } - parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { + common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto pos = start; for (const auto & p : parsers_) { auto result = p->parse(ctx, pos); @@ -446,11 +445,11 @@ class choice_parser : public parser_base { } } - return parser_result(PARSER_RESULT_FAIL, start); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); } - void assign_id(std::shared_ptr counter) override { - parser_base::assign_id(counter); + void assign_id(std::shared_ptr counter) override { + common_chat_combinator_parser_base::assign_id(counter); for (auto & p : parsers_) { p->assign_id(counter); } @@ -467,26 +466,26 @@ class choice_parser : public parser_base { void accept(parser_visitor & visitor) override; - const std::vector & parsers() const { return parsers_; } + const std::vector & parsers() const { return parsers_; } }; // Matches between min and max repetitions of a parser (inclusive). // S -> A{m,n} // Use -1 for max_count to represent unbounded repetition (equivalent to {m,}) -class repetition_parser : public parser_base { - parser parser_; +class repetition_parser : public common_chat_combinator_parser_base { + common_chat_combinator_parser parser_; int min_count_; int max_count_; public: - static constexpr parser_type type_value = PARSER_REPETITION; + static constexpr parser_type type_value = REPETITION; - repetition_parser(const parser & parser, int min_count, int max_count, int id) - : parser_base(id), parser_(parser), min_count_(min_count), max_count_(max_count) {} + repetition_parser(const common_chat_combinator_parser & parser, int min_count, int max_count, int id) + : common_chat_combinator_parser_base(id), parser_(parser), min_count_(min_count), max_count_(max_count) {} parser_type type() const override { return type_value; } - parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { + common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto pos = start; int match_count = 0; @@ -509,7 +508,7 @@ class repetition_parser : public parser_base { } if (result.is_need_more_input()) { - return parser_result(result.type, start, result.end); + return common_chat_parse_result(result.type, start, result.end); } // Child failed - stop trying @@ -518,14 +517,14 @@ class repetition_parser : public parser_base { // Check if we got enough matches if (match_count < min_count_) { - return parser_result(PARSER_RESULT_FAIL, start, pos); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start, pos); } - return parser_result(PARSER_RESULT_SUCCESS, start, pos); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); } - void assign_id(std::shared_ptr counter) override { - parser_base::assign_id(counter); + void assign_id(std::shared_ptr counter) override { + common_chat_combinator_parser_base::assign_id(counter); parser_->assign_id(counter); } @@ -538,7 +537,7 @@ class repetition_parser : public parser_base { void accept(parser_visitor & visitor) override; - const parser & child() const { return parser_; } + const common_chat_combinator_parser & child() const { return parser_; } int min_count() const { return min_count_; } @@ -549,9 +548,9 @@ class repetition_parser : public parser_base { // S -> A+ class one_or_more_parser : public repetition_parser { public: - static constexpr parser_type type_value = PARSER_ONE_OR_MORE; + static constexpr parser_type type_value = ONE_OR_MORE; - one_or_more_parser(const parser & p, int id) : repetition_parser(p, 1, -1, id) {} + one_or_more_parser(const common_chat_combinator_parser & p, int id) : repetition_parser(p, 1, -1, id) {} parser_type type() const override { return type_value; } @@ -566,9 +565,9 @@ class one_or_more_parser : public repetition_parser { // S -> A* class zero_or_more_parser : public repetition_parser { public: - static constexpr parser_type type_value = PARSER_ZERO_OR_MORE; + static constexpr parser_type type_value = ZERO_OR_MORE; - zero_or_more_parser(const parser & p, int id) : repetition_parser(p, 0, -1, id) {} + zero_or_more_parser(const common_chat_combinator_parser & p, int id) : repetition_parser(p, 0, -1, id) {} parser_type type() const override { return type_value; } @@ -583,9 +582,9 @@ class zero_or_more_parser : public repetition_parser { // S -> A? class optional_parser : public repetition_parser { public: - static constexpr parser_type type_value = PARSER_OPTIONAL; + static constexpr parser_type type_value = OPTIONAL; - optional_parser(const parser & p, int id) : repetition_parser(p, 0, 1, id) {} + optional_parser(const common_chat_combinator_parser & p, int id) : repetition_parser(p, 0, 1, id) {} parser_type type() const override { return type_value; } @@ -598,29 +597,29 @@ class optional_parser : public repetition_parser { // Positive lookahead: succeeds if child parser succeeds, consumes no input. // S -> &A -class and_parser : public parser_base { - parser parser_; +class and_parser : public common_chat_combinator_parser_base { + common_chat_combinator_parser parser_; public: - static constexpr parser_type type_value = PARSER_AND; + static constexpr parser_type type_value = AND; - and_parser(const parser & parser, int id) : parser_base(id), parser_(parser) {} + and_parser(const common_chat_combinator_parser & parser, int id) : common_chat_combinator_parser_base(id), parser_(parser) {} parser_type type() const override { return type_value; } - parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { + common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto result = parser_->parse(ctx, start); if (result.is_success()) { - return parser_result(PARSER_RESULT_SUCCESS, start); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start); } if (result.is_need_more_input()) { return result; } - return parser_result(PARSER_RESULT_SUCCESS, start); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start); } - void assign_id(std::shared_ptr counter) override { - parser_base::assign_id(counter); + void assign_id(std::shared_ptr counter) override { + common_chat_combinator_parser_base::assign_id(counter); parser_->assign_id(counter); } @@ -630,27 +629,27 @@ class and_parser : public parser_base { void accept(parser_visitor & visitor) override; - const parser & child() const { return parser_; } + const common_chat_combinator_parser & child() const { return parser_; } }; // Negative lookahead: succeeds if child parser fails, consumes no input. // S -> !A -class not_parser : public parser_base { - parser parser_; +class not_parser : public common_chat_combinator_parser_base { + common_chat_combinator_parser parser_; public: - static constexpr parser_type type_value = PARSER_NOT; + static constexpr parser_type type_value = NOT; - not_parser(const parser & parser, int id) : parser_base(id), parser_(parser) {} + not_parser(const common_chat_combinator_parser & parser, int id) : common_chat_combinator_parser_base(id), parser_(parser) {} parser_type type() const override { return type_value; } - parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { + common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto result = parser_->parse(ctx, start); if (result.is_success()) { // Fail if the underlying parser matches - return parser_result(PARSER_RESULT_FAIL, start); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); } if (result.is_need_more_input()) { @@ -659,11 +658,11 @@ class not_parser : public parser_base { } // Child failed, so negation succeeds - return parser_result(PARSER_RESULT_SUCCESS, start); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start); } - void assign_id(std::shared_ptr counter) override { - parser_base::assign_id(counter); + void assign_id(std::shared_ptr counter) override { + common_chat_combinator_parser_base::assign_id(counter); parser_->assign_id(counter); } @@ -673,27 +672,27 @@ class not_parser : public parser_base { void accept(parser_visitor & visitor) override; - const parser & child() const { return parser_; } + const common_chat_combinator_parser & child() const { return parser_; } }; // Matches any single character. // S -> . -class any_parser : public parser_base { +class any_parser : public common_chat_combinator_parser_base { public: - static constexpr parser_type type_value = PARSER_ANY; + static constexpr parser_type type_value = ANY; - any_parser(int id) : parser_base(id) {} + any_parser(int id) : common_chat_combinator_parser_base(id) {} parser_type type() const override { return type_value; } - parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { + common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { if (start >= ctx.input.size()) { if (ctx.input_is_complete) { - return parser_result(PARSER_RESULT_FAIL, start); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); } - return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start); } - return parser_result(PARSER_RESULT_SUCCESS, start, start + 1); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, start + 1); } std::string dump() const override { @@ -705,15 +704,15 @@ class any_parser : public parser_base { // Matches zero or more whitespace characters (space, tab, newline). // S -> [ \t\n]* -class space_parser : public parser_base { +class space_parser : public common_chat_combinator_parser_base { public: - static constexpr parser_type type_value = PARSER_SPACE; + static constexpr parser_type type_value = SPACE; - space_parser(int id) : parser_base(id) {} + space_parser(int id) : common_chat_combinator_parser_base(id) {} parser_type type() const override { return type_value; } - parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { + common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto pos = start; while (pos < ctx.input.size()) { char c = ctx.input[pos]; @@ -724,7 +723,7 @@ class space_parser : public parser_base { } } - return parser_result(PARSER_RESULT_SUCCESS, start, pos); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); } std::string dump() const override { @@ -736,7 +735,7 @@ class space_parser : public parser_base { // Matches between min and max repetitions of characters from a character class. // S -> [a-z]{m,n} -class chars_parser : public parser_base { +class chars_parser : public common_chat_combinator_parser_base { struct char_range { int start; int end; @@ -752,7 +751,7 @@ class chars_parser : public parser_base { public: chars_parser(const std::string & classes, int min_count, int max_count, int id) - : parser_base(id), pattern_(classes), negated_(false), min_count_(min_count), max_count_(max_count) { + : common_chat_combinator_parser_base(id), pattern_(classes), negated_(false), min_count_(min_count), max_count_(max_count) { std::string content = classes; if (content.front() == '[') { @@ -802,11 +801,11 @@ class chars_parser : public parser_base { } } - static constexpr parser_type type_value = PARSER_CHARS; + static constexpr parser_type type_value = CHARS; parser_type type() const override { return type_value; } - parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { + common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto pos = start; int match_count = 0; @@ -840,12 +839,12 @@ class chars_parser : public parser_base { // Check if we got enough matches if (match_count < min_count_) { if (pos >= ctx.input.size() && !ctx.input_is_complete) { - return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, pos); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start, pos); } - return parser_result(PARSER_RESULT_FAIL, start, pos); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start, pos); } - return parser_result(PARSER_RESULT_SUCCESS, start, pos); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); } std::string dump() const override { @@ -869,16 +868,16 @@ class chars_parser : public parser_base { // Stops before the closing quote (doesn't consume it). // Handles escape sequences and emits NEED_MORE_INPUT for incomplete input. // S -> (regular chars and escape sequences)* until closing " -class json_string_parser : public parser_base { +class json_string_parser : public common_chat_combinator_parser_base { public: - static constexpr parser_type type_value = PARSER_JSON_STRING; + static constexpr parser_type type_value = JSON_STRING; - json_string_parser(int id) : parser_base(id) {} + json_string_parser(int id) : common_chat_combinator_parser_base(id) {} parser_type type() const override { return type_value; } - parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { + common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto pos = start; // Parse string content (without quotes) @@ -887,7 +886,7 @@ class json_string_parser : public parser_base { if (c == '"') { // Found closing quote - success (don't consume it) - return parser_result(PARSER_RESULT_SUCCESS, start, pos); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); } if (c == '\\') { @@ -896,9 +895,9 @@ class json_string_parser : public parser_base { if (pos >= ctx.input.size()) { // Mid-escape sequence if (ctx.input_is_complete) { - return parser_result(PARSER_RESULT_FAIL, start); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); } - return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, pos); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start, pos); } char escape = ctx.input[pos]; @@ -922,12 +921,12 @@ class json_string_parser : public parser_base { if (pos >= ctx.input.size()) { // Incomplete unicode escape if (ctx.input_is_complete) { - return parser_result(PARSER_RESULT_FAIL, start); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); } - return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, pos); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start, pos); } if (!is_hex_digit(ctx.input[pos])) { - return parser_result(PARSER_RESULT_FAIL, start); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); } ++pos; } @@ -935,7 +934,7 @@ class json_string_parser : public parser_base { default: // Invalid escape sequence - return parser_result(PARSER_RESULT_FAIL, start); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); } } else { // Regular character @@ -945,9 +944,9 @@ class json_string_parser : public parser_base { // Reached end without finding closing quote if (ctx.input_is_complete) { - return parser_result(PARSER_RESULT_FAIL, start, pos); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start, pos); } - return parser_result(PARSER_RESULT_NEED_MORE_INPUT, start, pos); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start, pos); } std::string dump() const override { @@ -959,24 +958,24 @@ class json_string_parser : public parser_base { // Matches all characters until a delimiter is found (delimiter not consumed). // S -> (!delim .)* -class until_parser : public parser_base { +class until_parser : public common_chat_combinator_parser_base { std::vector delimiters_; aho_corasick_matcher matcher_; public: - static constexpr parser_type type_value = PARSER_UNTIL; + static constexpr parser_type type_value = UNTIL; until_parser(const std::vector & delimiters, int id) - : parser_base(id), delimiters_(delimiters), matcher_(delimiters) {} + : common_chat_combinator_parser_base(id), delimiters_(delimiters), matcher_(delimiters) {} until_parser(const std::string & delimiter, int id) : until_parser(std::vector{delimiter}, id) {} parser_type type() const override { return type_value; } - parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { + common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto search_result = matcher_.search(ctx.input, start); - return parser_result(PARSER_RESULT_SUCCESS, start, search_result.pos); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, search_result.pos); } std::string dump() const override { @@ -990,20 +989,20 @@ class until_parser : public parser_base { // Wraps a parser with JSON schema metadata for grammar generation. // Used internally to convert JSON schemas to GBNF grammar rules. -class schema_parser : public parser_base { - parser parser_; +class schema_parser : public common_chat_combinator_parser_base { + common_chat_combinator_parser parser_; std::string name_; nlohmann::ordered_json schema_; public: - static constexpr parser_type type_value = PARSER_SCHEMA; + static constexpr parser_type type_value = SCHEMA; - schema_parser(const parser & parser, const std::string & name, const nlohmann::ordered_json & schema, int id) - : parser_base(id), parser_(parser), name_(name), schema_(schema) {} + schema_parser(const common_chat_combinator_parser & parser, const std::string & name, const nlohmann::ordered_json & schema, int id) + : common_chat_combinator_parser_base(id), parser_(parser), name_(name), schema_(schema) {} parser_type type() const override { return type_value; } - parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { + common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { return parser_->parse(ctx, start); } @@ -1013,7 +1012,7 @@ class schema_parser : public parser_base { void accept(parser_visitor & visitor) override; - const parser & child() const { return parser_; } + const common_chat_combinator_parser & child() const { return parser_; } const std::string & name() const { return name_; } @@ -1022,40 +1021,40 @@ class schema_parser : public parser_base { // References a named rule for recursive or reusable grammar definitions. // expr -> term | expr "+" term -class rule_parser : public parser_base { +class rule_parser : public common_chat_combinator_parser_base { std::string name_; - std::weak_ptr> rules_; + std::weak_ptr> rules_; public: - static constexpr parser_type type_value = PARSER_RULE; + static constexpr parser_type type_value = RULE; - rule_parser(const std::string & name, const std::shared_ptr> & rules, int id) - : parser_base(id), name_(name), rules_(rules) {} + rule_parser(const std::string & name, const std::shared_ptr> & rules, int id) + : common_chat_combinator_parser_base(id), name_(name), rules_(rules) {} parser_type type() const override { return type_value; } - parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { + common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto rules = rules_.lock(); if (!rules) { LOG_ERR("rule_parser::parse called with expired rule registry\n"); - return parser_result(PARSER_RESULT_FAIL, start); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); } auto it = rules->find(name_); if (it == rules->end()) { LOG_ERR("rule_parser::parse rule '%s' not found in registry\n", name_.c_str()); - return parser_result(PARSER_RESULT_FAIL, start); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); } // Fire NODE_START event if (ctx.event_handler && ctx.env) { - ctx.event_handler(parse_event{ - PARSER_EVENT_NODE_START, + ctx.event_handler(common_chat_parse_event{ + COMMON_CHAT_PARSE_EVENT_NODE_START, name_, start, start, "", - PARSER_RESULT_FAIL, + COMMON_CHAT_PARSE_RESULT_FAIL, ctx.current_depth }, *ctx.env); ctx.current_depth++; @@ -1073,8 +1072,8 @@ class rule_parser : public parser_base { } else { text = ""; } - ctx.event_handler(parse_event{ - PARSER_EVENT_NODE_END, + ctx.event_handler(common_chat_parse_event{ + COMMON_CHAT_PARSE_EVENT_NODE_END, name_, result.start, result.end, @@ -1098,26 +1097,26 @@ class rule_parser : public parser_base { // Container for the root parser and all named rules in the grammar. // Manages ownership of rule registry to enable recursive grammar definitions. -class root_parser : public parser_base { - parser root_; - std::shared_ptr> rules_; +class root_parser : public common_chat_combinator_parser_base { + common_chat_combinator_parser root_; + std::shared_ptr> rules_; friend class parser_visitor; public: - static constexpr parser_type type_value = PARSER_ROOT; + static constexpr parser_type type_value = ROOT; - root_parser(const parser & root, std::shared_ptr> rules, int id) - : parser_base(id), root_(root), rules_(std::move(rules)) {} + root_parser(const common_chat_combinator_parser & root, std::shared_ptr> rules, int id) + : common_chat_combinator_parser_base(id), root_(root), rules_(std::move(rules)) {} parser_type type() const override { return type_value; } - parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { + common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { return root_->parse(ctx, start); } - void assign_id(std::shared_ptr counter) override { - parser_base::assign_id(counter); + void assign_id(std::shared_ptr counter) override { + common_chat_combinator_parser_base::assign_id(counter); root_->assign_id(counter); } @@ -1127,30 +1126,30 @@ class root_parser : public parser_base { void accept(parser_visitor & visitor) override; - const parser & root() const { return root_; } + const common_chat_combinator_parser & root() const { return root_; } - std::shared_ptr> rules() const { return rules_; } + std::shared_ptr> rules() const { return rules_; } }; // Wraps a parser with a semantic action callback. -class action_parser : public parser_base { - parser parser_; - std::function action_; +class action_parser : public common_chat_combinator_parser_base { + common_chat_combinator_parser parser_; + std::function action_; int when_; public: - static constexpr parser_type type_value = PARSER_ACTION; + static constexpr parser_type type_value = ACTION; action_parser( - const parser & parser, - std::function action, + const common_chat_combinator_parser & parser, + std::function action, int when, int id - ) : parser_base(id), parser_(parser), action_(std::move(action)), when_(when) {} + ) : common_chat_combinator_parser_base(id), parser_(parser), action_(std::move(action)), when_(when) {} parser_type type() const override { return type_value; } - parser_result parse_uncached(parser_context & ctx, size_t start = 0) override { + common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto result = parser_->parse(ctx, start); if ((result.type & when_) && ctx.env && action_) { @@ -1166,8 +1165,8 @@ class action_parser : public parser_base { return result; } - void assign_id(std::shared_ptr counter) override { - parser_base::assign_id(counter); + void assign_id(std::shared_ptr counter) override { + common_chat_combinator_parser_base::assign_id(counter); parser_->assign_id(counter); } @@ -1177,7 +1176,7 @@ class action_parser : public parser_base { void accept(parser_visitor & visitor) override; - const parser & child() const { return parser_; } + const common_chat_combinator_parser & child() const { return parser_; } }; // Base visitor class for parser tree traversal @@ -1249,7 +1248,7 @@ class gbnf_visitor : public parser_visitor { private: // Check if expression needs parentheses static bool needs_parens(parser_type type) { - return type == PARSER_CHOICE || type == PARSER_SEQUENCE; + return type == CHOICE || type == SEQUENCE; } public: @@ -1285,7 +1284,7 @@ class gbnf_visitor : public parser_visitor { child->accept(*this); // Parenthesize choices - if (child->type() == PARSER_CHOICE) { + if (child->type() == CHOICE) { s += "(" + current_result_ + ")"; } else { s += current_result_; @@ -1453,77 +1452,77 @@ void rule_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void root_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void action_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } -parser_result parse_cache::set(int id, size_t start, parser_result result) { +common_chat_parse_result common_chat_parse_cache::set(int id, size_t start, common_chat_parse_result result) { if (id == -1) { // Don't cache parsers with ID -1 (from operators and global factory functions) return result; } - results[parse_cache_key{id, start}] = result; + results[common_chat_parse_cache_key{id, start}] = result; return result; } -std::optional parse_cache::get(int id, size_t start) { +std::optional common_chat_parse_cache::get(int id, size_t start) { if (id == -1) { // Don't cache parsers with ID -1 (from operators and global factory functions) return std::nullopt; } - auto it = results.find(parse_cache_key{id, start}); + auto it = results.find(common_chat_parse_cache_key{id, start}); if (it != results.end()) { return it->second; } return std::nullopt; } -void parse_cache::clear() { +void common_chat_parse_cache::clear() { results.clear(); } -parser::parser() {} +common_chat_combinator_parser::common_chat_combinator_parser() {} -parser::parser(std::shared_ptr parser) : ptr_(std::move(parser)) {} +common_chat_combinator_parser::common_chat_combinator_parser(std::shared_ptr parser) : ptr_(std::move(parser)) {} -parser::parser(const std::string & literal) : ptr_(std::make_shared(literal, -1)) {} +common_chat_combinator_parser::common_chat_combinator_parser(const std::string & literal) : ptr_(std::make_shared(literal, -1)) {} -parser::parser(const char * literal) : ptr_(std::make_shared(literal, -1)) {} +common_chat_combinator_parser::common_chat_combinator_parser(const char * literal) : ptr_(std::make_shared(literal, -1)) {} -parser parser::operator~() const { - return parser(std::make_shared(*this, -1)); +common_chat_combinator_parser common_chat_combinator_parser::operator~() const { + return common_chat_combinator_parser(std::make_shared(*this, -1)); } -parser parser::operator+(const parser & other) const { - return parser(std::make_shared(std::initializer_list{*this, other}, -1)); +common_chat_combinator_parser common_chat_combinator_parser::operator+(const common_chat_combinator_parser & other) const { + return common_chat_combinator_parser(std::make_shared(std::initializer_list{*this, other}, -1)); } -parser parser::operator|(const parser & other) const { - return parser(std::make_shared(std::initializer_list{*this, other}, -1)); +common_chat_combinator_parser common_chat_combinator_parser::operator|(const common_chat_combinator_parser & other) const { + return common_chat_combinator_parser(std::make_shared(std::initializer_list{*this, other}, -1)); } -parser parser::operator<<(const parser & other) const { - auto ws = parser(std::make_shared(-1)); - return parser(std::make_shared(std::initializer_list{*this, ws, other}, -1)); +common_chat_combinator_parser common_chat_combinator_parser::operator<<(const common_chat_combinator_parser & other) const { + auto ws = common_chat_combinator_parser(std::make_shared(-1)); + return common_chat_combinator_parser(std::make_shared(std::initializer_list{*this, ws, other}, -1)); } -parser operator+(const char * lhs, const parser & rhs) { return parser(lhs) + rhs; } -parser operator|(const char * lhs, const parser & rhs) { return parser(lhs) | rhs; } -parser operator<<(const char * lhs, const parser & rhs) { return parser(lhs) << rhs; } +common_chat_combinator_parser operator+(const char * lhs, const common_chat_combinator_parser & rhs) { return common_chat_combinator_parser(lhs) + rhs; } +common_chat_combinator_parser operator|(const char * lhs, const common_chat_combinator_parser & rhs) { return common_chat_combinator_parser(lhs) | rhs; } +common_chat_combinator_parser operator<<(const char * lhs, const common_chat_combinator_parser & rhs) { return common_chat_combinator_parser(lhs) << rhs; } -parser_base & parser::operator*() const { +common_chat_combinator_parser_base & common_chat_combinator_parser::operator*() const { return *ptr_; } -parser_base * parser::operator->() const { +common_chat_combinator_parser_base * common_chat_combinator_parser::operator->() const { return ptr_.get(); } -parser_result parser::parse(parser_context & ctx, size_t start) const { +common_chat_parse_result common_chat_combinator_parser::parse(common_chat_parse_context & ctx, size_t start) const { return ptr_->parse(ctx, start); } -std::string parser::dump() const { +std::string common_chat_combinator_parser::dump() const { return ptr_->dump(); } -void parser::build_grammar(const common_grammar_builder & builder) const { +void common_chat_combinator_parser::build_grammar(const common_grammar_builder & builder) const { gbnf_visitor visitor(builder); ptr_->accept(visitor); auto result = visitor.result(); @@ -1532,127 +1531,127 @@ void parser::build_grammar(const common_grammar_builder & builder) const { } } -parser_builder::parser_builder() - : rules_(std::make_shared>()) - , counter_(std::make_shared(0)) {} +common_chat_combinator_parser_builder::common_chat_combinator_parser_builder() + : rules_(std::make_shared>()) + , counter_(std::make_shared(0)) {} -parser_builder::parser_builder(std::shared_ptr counter) - : rules_(std::make_shared>()) +common_chat_combinator_parser_builder::common_chat_combinator_parser_builder(std::shared_ptr counter) + : rules_(std::make_shared>()) , counter_(std::move(counter)) {} -parser parser_builder::literal(const std::string & literal) { - return parser(std::make_shared(literal, counter_->next())); +common_chat_combinator_parser common_chat_combinator_parser_builder::literal(const std::string & literal) { + return common_chat_combinator_parser(std::make_shared(literal, counter_->next())); } -parser parser_builder::sequence(std::initializer_list parsers) { - return parser(std::make_shared(parsers, counter_->next())); +common_chat_combinator_parser common_chat_combinator_parser_builder::sequence(std::initializer_list parsers) { + return common_chat_combinator_parser(std::make_shared(parsers, counter_->next())); } -parser parser_builder::choice(std::initializer_list parsers) { - return parser(std::make_shared(parsers, counter_->next())); +common_chat_combinator_parser common_chat_combinator_parser_builder::choice(std::initializer_list parsers) { + return common_chat_combinator_parser(std::make_shared(parsers, counter_->next())); } -parser parser_builder::one_or_more(const parser & p) { - return parser(std::make_shared(p, counter_->next())); +common_chat_combinator_parser common_chat_combinator_parser_builder::one_or_more(const common_chat_combinator_parser & p) { + return common_chat_combinator_parser(std::make_shared(p, counter_->next())); } -parser parser_builder::zero_or_more(const parser & p) { - return parser(std::make_shared(p, counter_->next())); +common_chat_combinator_parser common_chat_combinator_parser_builder::zero_or_more(const common_chat_combinator_parser & p) { + return common_chat_combinator_parser(std::make_shared(p, counter_->next())); } -parser parser_builder::optional(const parser & p) { - return parser(std::make_shared(p, counter_->next())); +common_chat_combinator_parser common_chat_combinator_parser_builder::optional(const common_chat_combinator_parser & p) { + return common_chat_combinator_parser(std::make_shared(p, counter_->next())); } -parser parser_builder::peek(const parser & p) { - return parser(std::make_shared(p, counter_->next())); +common_chat_combinator_parser common_chat_combinator_parser_builder::peek(const common_chat_combinator_parser & p) { + return common_chat_combinator_parser(std::make_shared(p, counter_->next())); } -parser parser_builder::negate(const parser & p) { - return parser(std::make_shared(p, counter_->next())); +common_chat_combinator_parser common_chat_combinator_parser_builder::negate(const common_chat_combinator_parser & p) { + return common_chat_combinator_parser(std::make_shared(p, counter_->next())); } -parser parser_builder::any() { - return parser(std::make_shared(counter_->next())); +common_chat_combinator_parser common_chat_combinator_parser_builder::any() { + return common_chat_combinator_parser(std::make_shared(counter_->next())); } -parser parser_builder::chars(const std::string & classes, int min, int max) { - return parser(std::make_shared(classes, min, max, counter_->next())); +common_chat_combinator_parser common_chat_combinator_parser_builder::chars(const std::string & classes, int min, int max) { + return common_chat_combinator_parser(std::make_shared(classes, min, max, counter_->next())); } -parser parser_builder::one(const std::string & classes) { +common_chat_combinator_parser common_chat_combinator_parser_builder::one(const std::string & classes) { return chars(classes, 1, 1); } -parser parser_builder::json_string() { - return parser(std::make_shared(counter_->next())); +common_chat_combinator_parser common_chat_combinator_parser_builder::json_string() { + return common_chat_combinator_parser(std::make_shared(counter_->next())); } -parser parser_builder::rule(const std::string & name) { - return parser(std::make_shared(name, rules_, counter_->next())); +common_chat_combinator_parser common_chat_combinator_parser_builder::rule(const std::string & name) { + return common_chat_combinator_parser(std::make_shared(name, rules_, counter_->next())); } -parser parser_builder::space() { - return parser(std::make_shared(counter_->next())); +common_chat_combinator_parser common_chat_combinator_parser_builder::space() { + return common_chat_combinator_parser(std::make_shared(counter_->next())); } -parser parser_builder::until(const std::string & delimiter) { - return parser(std::make_shared(delimiter, counter_->next())); +common_chat_combinator_parser common_chat_combinator_parser_builder::until(const std::string & delimiter) { + return common_chat_combinator_parser(std::make_shared(delimiter, counter_->next())); } -parser parser_builder::until_one_of(const std::vector & delimiters) { - return parser(std::make_shared(delimiters, counter_->next())); +common_chat_combinator_parser common_chat_combinator_parser_builder::until_one_of(const std::vector & delimiters) { + return common_chat_combinator_parser(std::make_shared(delimiters, counter_->next())); } -parser parser_builder::repeat(const parser & p, int min, int max) { - return parser(std::make_shared(p, min, max, counter_->next())); +common_chat_combinator_parser common_chat_combinator_parser_builder::repeat(const common_chat_combinator_parser & p, int min, int max) { + return common_chat_combinator_parser(std::make_shared(p, min, max, counter_->next())); } -parser parser_builder::repeat(const parser & p, int n) { +common_chat_combinator_parser common_chat_combinator_parser_builder::repeat(const common_chat_combinator_parser & p, int n) { return repeat(p, n, n); } -parser parser_builder::schema(const parser & p, const std::string & name, const nlohmann::ordered_json & schema) { - return parser(std::make_shared(p, name, schema, counter_->next())); +common_chat_combinator_parser common_chat_combinator_parser_builder::schema(const common_chat_combinator_parser & p, const std::string & name, const nlohmann::ordered_json & schema) { + return common_chat_combinator_parser(std::make_shared(p, name, schema, counter_->next())); } -parser parser_builder::action(const parser & p, std::function fn, int when) { - return parser(std::make_shared(p, std::move(fn), when, counter_->next())); +common_chat_combinator_parser common_chat_combinator_parser_builder::action(const common_chat_combinator_parser & p, std::function fn, int when) { + return common_chat_combinator_parser(std::make_shared(p, std::move(fn), when, counter_->next())); } -parser parser_builder::capture(const std::string & key, const parser & p) { - return action(p, [key](const parser_action & act) { +common_chat_combinator_parser common_chat_combinator_parser_builder::capture(const std::string & key, const common_chat_combinator_parser & p) { + return action(p, [key](const common_chat_parse_action & act) { std::string value = std::string(act.match); act.env.captures[key] = std::move(value); - }, PARSER_RESULT_SUCCESS); + }, COMMON_CHAT_PARSE_RESULT_SUCCESS); } -parser parser_builder::add_rule(const std::string & name, const parser & p) { +common_chat_combinator_parser common_chat_combinator_parser_builder::add_rule(const std::string & name, const common_chat_combinator_parser & p) { (*rules_)[name] = p; return rule(name); } -void parser_builder::assign_ids(parser & p) { +void common_chat_combinator_parser_builder::assign_ids(common_chat_combinator_parser & p) { if (p.ptr()) { p.ptr()->assign_id(counter_); } } -parser build_parser(const std::function & fn) { - parser_builder builder; +common_chat_combinator_parser build_combinator_parser(const std::function & fn) { + common_chat_combinator_parser_builder builder; auto root = fn(builder); builder.assign_ids(root); // Assign IDs to rules that were created with operators // Wrap the root parser in a root_parser to own the rules and break circular references auto rules = builder.rules(); if (rules && !rules->empty()) { - return parser(std::make_shared(root, rules, -1)); + return common_chat_combinator_parser(std::make_shared(root, rules, -1)); } return root; } -static parser json_parser(std::shared_ptr counter) { - parser_builder builder(std::move(counter)); +static common_chat_combinator_parser json_parser(std::shared_ptr counter) { + common_chat_combinator_parser_builder builder(std::move(counter)); // Whitespace: space, tab, newline, carriage return auto ws = builder.space(); @@ -1716,9 +1715,9 @@ static parser json_parser(std::shared_ptr counter) { ); // Wrap in root_parser to own the rules - return parser(std::make_shared(root, builder.rules(), -1)); + return common_chat_combinator_parser(std::make_shared(root, builder.rules(), -1)); } -parser parser_builder::json() { +common_chat_combinator_parser common_chat_combinator_parser_builder::json() { return json_parser(counter_); } diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index d64b08cc27d8b..6156d62128700 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -13,273 +13,273 @@ struct common_grammar_builder; -struct parser_environment { +struct common_chat_parse_semantics { common_chat_msg result; std::unordered_map captures; }; -enum parser_result_type { - PARSER_RESULT_FAIL = 1 << 0, - PARSER_RESULT_SUCCESS = 1 << 1, - PARSER_RESULT_NEED_MORE_INPUT = 1 << 2, +enum common_chat_parse_result_type { + COMMON_CHAT_PARSE_RESULT_FAIL = 1 << 0, + COMMON_CHAT_PARSE_RESULT_SUCCESS = 1 << 1, + COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT = 1 << 2, }; -struct parse_cache_key { +struct common_chat_parse_cache_key { int id; size_t start; - bool operator==(const parse_cache_key & other) const { + bool operator==(const common_chat_parse_cache_key & other) const { return id == other.id && start == other.start; } }; template <> -struct std::hash { - std::size_t operator()(const parse_cache_key & k) const { +struct std::hash { + std::size_t operator()(const common_chat_parse_cache_key & k) const { return std::hash{}(((size_t)k.id << 32) | k.start); } }; -struct parser_result { - parser_result_type type = PARSER_RESULT_FAIL; +struct common_chat_parse_result { + common_chat_parse_result_type type = COMMON_CHAT_PARSE_RESULT_FAIL; size_t start = 0; size_t end = 0; - parser_result() : type(PARSER_RESULT_FAIL) {} + common_chat_parse_result() : type(COMMON_CHAT_PARSE_RESULT_FAIL) {} - parser_result(parser_result_type type, size_t start) + common_chat_parse_result(common_chat_parse_result_type type, size_t start) : type(type), start(start), end(start) {} - parser_result(parser_result_type type, size_t start, size_t end) + common_chat_parse_result(common_chat_parse_result_type type, size_t start, size_t end) : type(type), start(start), end(end) {} - bool is_fail() const { return type == PARSER_RESULT_FAIL; } - bool is_need_more_input() const { return type == PARSER_RESULT_NEED_MORE_INPUT; } - bool is_success() const { return type == PARSER_RESULT_SUCCESS; } + bool is_fail() const { return type == COMMON_CHAT_PARSE_RESULT_FAIL; } + bool is_need_more_input() const { return type == COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT; } + bool is_success() const { return type == COMMON_CHAT_PARSE_RESULT_SUCCESS; } }; -struct parser_action { - parser_result & result; - parser_environment & env; +struct common_chat_parse_action { + common_chat_parse_result & result; + common_chat_parse_semantics & env; std::string_view match; }; -enum parse_event_type { - PARSER_EVENT_NODE_START, - PARSER_EVENT_NODE_END, +enum common_chat_parse_event_type { + COMMON_CHAT_PARSE_EVENT_NODE_START, + COMMON_CHAT_PARSE_EVENT_NODE_END, }; -struct parse_event { - parse_event_type type; +struct common_chat_parse_event { + common_chat_parse_event_type type; std::string rule; size_t start; size_t end; std::string_view text; - parser_result_type status; + common_chat_parse_result_type status; int depth; - bool starting() const { return type == PARSER_EVENT_NODE_START; } - bool ending() const { return type == PARSER_EVENT_NODE_END; } + bool starting() const { return type == COMMON_CHAT_PARSE_EVENT_NODE_START; } + bool ending() const { return type == COMMON_CHAT_PARSE_EVENT_NODE_END; } - bool success() const { return status == PARSER_RESULT_SUCCESS; } - bool partial() const { return status == PARSER_RESULT_NEED_MORE_INPUT; } - bool fail() const { return status == PARSER_RESULT_FAIL; } + bool success() const { return status == COMMON_CHAT_PARSE_RESULT_SUCCESS; } + bool partial() const { return status == COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT; } + bool fail() const { return status == COMMON_CHAT_PARSE_RESULT_FAIL; } }; -using parse_event_handler = std::function; +using common_chat_parse_event_handler = std::function; -class parse_cache { - std::unordered_map results; +class common_chat_parse_cache { + std::unordered_map results; public: - parser_result set(int id, size_t start, parser_result result); - std::optional get(int id, size_t start); + common_chat_parse_result set(int id, size_t start, common_chat_parse_result result); + std::optional get(int id, size_t start); void clear(); }; -struct parser_context { +struct common_chat_parse_context { std::string input; - parse_cache memo; + common_chat_parse_cache cache; bool input_is_complete; - parser_environment * env; - parse_event_handler event_handler; + common_chat_parse_semantics * env; + common_chat_parse_event_handler event_handler; int current_depth; - parser_context() - : memo(), input_is_complete(true), env(nullptr), event_handler(nullptr), current_depth(0) {} + common_chat_parse_context() + : cache(), input_is_complete(true), env(nullptr), event_handler(nullptr), current_depth(0) {} - parser_context(const std::string & input) - : input(input), memo(), input_is_complete(true), env(nullptr), event_handler(nullptr), current_depth(0) {} + common_chat_parse_context(const std::string & input) + : input(input), cache(), input_is_complete(true), env(nullptr), event_handler(nullptr), current_depth(0) {} - parser_context(const std::string & input, bool complete) - : input(input), memo(), input_is_complete(complete), env(nullptr), event_handler(nullptr), current_depth(0) {} + common_chat_parse_context(const std::string & input, bool complete) + : input(input), cache(), input_is_complete(complete), env(nullptr), event_handler(nullptr), current_depth(0) {} - parser_context(const std::string & input, parse_cache memo, bool complete = true) - : input(input), memo(std::move(memo)), input_is_complete(complete), env(nullptr), event_handler(nullptr), current_depth(0) {} + common_chat_parse_context(const std::string & input, common_chat_parse_cache memo, bool complete = true) + : input(input), cache(std::move(memo)), input_is_complete(complete), env(nullptr), event_handler(nullptr), current_depth(0) {} - parser_context(const std::string & input, parser_environment * environment) - : input(input), memo(), input_is_complete(true), env(environment), event_handler(nullptr), current_depth(0) {} + common_chat_parse_context(const std::string & input, common_chat_parse_semantics * environment) + : input(input), cache(), input_is_complete(true), env(environment), event_handler(nullptr), current_depth(0) {} - parser_context(const std::string & input, parser_environment * environment, bool complete) - : input(input), memo(), input_is_complete(complete), env(environment), event_handler(nullptr), current_depth(0) {} + common_chat_parse_context(const std::string & input, common_chat_parse_semantics * environment, bool complete) + : input(input), cache(), input_is_complete(complete), env(environment), event_handler(nullptr), current_depth(0) {} - parser_context(const std::string & input, parse_cache memo, parser_environment * environment, bool complete = true) - : input(input), memo(std::move(memo)), input_is_complete(complete), env(environment), event_handler(nullptr), current_depth(0) {} + common_chat_parse_context(const std::string & input, common_chat_parse_cache memo, common_chat_parse_semantics * environment, bool complete = true) + : input(input), cache(std::move(memo)), input_is_complete(complete), env(environment), event_handler(nullptr), current_depth(0) {} - parser_context(const std::string & input, parser_environment * environment, parse_event_handler handler, bool complete = true) - : input(input), memo(), input_is_complete(complete), env(environment), event_handler(std::move(handler)), current_depth(0) {} + common_chat_parse_context(const std::string & input, common_chat_parse_semantics * environment, common_chat_parse_event_handler handler, bool complete = true) + : input(input), cache(), input_is_complete(complete), env(environment), event_handler(std::move(handler)), current_depth(0) {} }; -class parser_base; +class common_chat_combinator_parser_base; -class parser { - std::shared_ptr ptr_; +class common_chat_combinator_parser { + std::shared_ptr ptr_; public: - parser(); - parser(std::shared_ptr parser); - parser(const parser & other) = default; - parser(const std::string & literal); - parser(const char * literal); + common_chat_combinator_parser(); + common_chat_combinator_parser(std::shared_ptr parser); + common_chat_combinator_parser(const common_chat_combinator_parser & other) = default; + common_chat_combinator_parser(const std::string & literal); + common_chat_combinator_parser(const char * literal); - parser & operator=(const parser & other) { + common_chat_combinator_parser & operator=(const common_chat_combinator_parser & other) { if (this != &other) { ptr_ = other.ptr_; } return *this; } - parser operator~() const; - parser operator+(const parser & other) const; - parser operator|(const parser & other) const; - parser operator<<(const parser & other) const; + common_chat_combinator_parser operator~() const; + common_chat_combinator_parser operator+(const common_chat_combinator_parser & other) const; + common_chat_combinator_parser operator|(const common_chat_combinator_parser & other) const; + common_chat_combinator_parser operator<<(const common_chat_combinator_parser & other) const; - parser_base & operator*() const; - parser_base * operator->() const; + common_chat_combinator_parser_base & operator*() const; + common_chat_combinator_parser_base * operator->() const; - std::shared_ptr ptr() const { return ptr_; } + std::shared_ptr ptr() const { return ptr_; } - parser_result parse(parser_context & ctx, size_t start = 0) const; + common_chat_parse_result parse(common_chat_parse_context & ctx, size_t start = 0) const; std::string dump() const; void build_grammar(const common_grammar_builder & builder) const; }; -parser operator+(const char * lhs, const parser & rhs); -parser operator|(const char * lhs, const parser & rhs); -parser operator<<(const char * lhs, const parser & rhs); +common_chat_combinator_parser operator+(const char * lhs, const common_chat_combinator_parser & rhs); +common_chat_combinator_parser operator|(const char * lhs, const common_chat_combinator_parser & rhs); +common_chat_combinator_parser operator<<(const char * lhs, const common_chat_combinator_parser & rhs); -class parser_id_counter { +class common_chat_combinator_parser_id_counter { int next_id_; public: - parser_id_counter(int start) : next_id_(start) {} + common_chat_combinator_parser_id_counter(int start) : next_id_(start) {} int next() { return next_id_++; } }; -class parser_builder { - std::shared_ptr> rules_; - std::shared_ptr counter_; +class common_chat_combinator_parser_builder { + std::shared_ptr> rules_; + std::shared_ptr counter_; public: - parser_builder(); - parser_builder(std::shared_ptr counter); + common_chat_combinator_parser_builder(); + common_chat_combinator_parser_builder(std::shared_ptr counter); // Matches an exact literal string. // S -> "hello" - parser literal(const std::string & literal); + common_chat_combinator_parser literal(const std::string & literal); // Matches a sequence of parsers in order, all must succeed. // S -> A B C - parser sequence(std::initializer_list parsers); + common_chat_combinator_parser sequence(std::initializer_list parsers); // Matches the first parser that succeeds from a list of alternatives. // S -> A | B | C - parser choice(std::initializer_list parsers); + common_chat_combinator_parser choice(std::initializer_list parsers); // Matches one or more repetitions of a parser. // S -> A+ - parser one_or_more(const parser & p); + common_chat_combinator_parser one_or_more(const common_chat_combinator_parser & p); // Matches zero or more repetitions of a parser, always succeeds. // S -> A* - parser zero_or_more(const parser & p); + common_chat_combinator_parser zero_or_more(const common_chat_combinator_parser & p); // Matches zero or one occurrence of a parser, always succeeds. // S -> A? - parser optional(const parser & p); + common_chat_combinator_parser optional(const common_chat_combinator_parser & p); // Negative lookahead: succeeds if child parser fails, consumes no input. // S -> !A - parser peek(const parser & p); + common_chat_combinator_parser peek(const common_chat_combinator_parser & p); // Negative lookahead: succeeds if child parser fails, consumes no input. // S -> !A - parser negate(const parser & p); + common_chat_combinator_parser negate(const common_chat_combinator_parser & p); // Matches any single character. // S -> . - parser any(); + common_chat_combinator_parser any(); // Matches between min and max repetitions of characters from a character class. // S -> [a-z]{m,n} // // Use -1 for max to represent unbounded repetition (equivalent to {m,}) - parser chars(const std::string & classes, int min = 1, int max = -1); + common_chat_combinator_parser chars(const std::string & classes, int min = 1, int max = -1); // Matches a single character from a character class or range. // S -> [a-z] or S -> [^0-9] // // Equivalent to chars(classes, 1, 1) - parser one(const std::string & classes); + common_chat_combinator_parser one(const std::string & classes); // References a named rule for recursive or reusable grammar definitions. // expr -> term | expr "+" term - parser rule(const std::string & name); + common_chat_combinator_parser rule(const std::string & name); // Matches zero or more whitespace characters (space, tab, newline). // S -> [ \t\n]* - parser space(); + common_chat_combinator_parser space(); // Matches all characters until a delimiter is found (delimiter not consumed). // S -> (!delim .)* - parser until(const std::string & delimiter); - parser until_one_of(const std::vector & delimiters); + common_chat_combinator_parser until(const std::string & delimiter); + common_chat_combinator_parser until_one_of(const std::vector & delimiters); // Matches between min and max repetitions of a parser (inclusive). // S -> A{m,n} // Use -1 for max to represent unbounded repetition (equivalent to {m,}) - parser repeat(const parser & p, int min, int max); + common_chat_combinator_parser repeat(const common_chat_combinator_parser & p, int min, int max); // Matches exactly n repetitions of a parser. // S -> A{n} - parser repeat(const parser & p, int n); + common_chat_combinator_parser repeat(const common_chat_combinator_parser & p, int n); // Creates a complete JSON parser supporting objects, arrays, strings, numbers, booleans, and null. // value -> object | array | string | number | true | false | null - parser json(); + common_chat_combinator_parser json(); // Specialized single-pass JSON string parser with escape sequence handling - parser json_string(); + common_chat_combinator_parser json_string(); // Wraps a parser with JSON schema metadata for grammar generation. // Used internally to convert JSON schemas to GBNF grammar rules. - parser schema(const parser & p, const std::string & name, const nlohmann::ordered_json & schema); + common_chat_combinator_parser schema(const common_chat_combinator_parser & p, const std::string & name, const nlohmann::ordered_json & schema); // Wraps a parser with a semantic action callback. // The callback is invoked on successful parse with the result, matched text, and environment. // S -> A [action] - parser action(const parser & p, std::function fn, int when = PARSER_RESULT_SUCCESS); + common_chat_combinator_parser action(const common_chat_combinator_parser & p, std::function fn, int when = COMMON_CHAT_PARSE_RESULT_SUCCESS); // Captures matched text to env.captures[key] - parser capture(const std::string & key, const parser & p); + common_chat_combinator_parser capture(const std::string & key, const common_chat_combinator_parser & p); - parser add_rule(const std::string & name, const parser & p); + common_chat_combinator_parser add_rule(const std::string & name, const common_chat_combinator_parser & p); - void assign_ids(parser & p); + void assign_ids(common_chat_combinator_parser & p); - std::shared_ptr> rules() const { return rules_; } + std::shared_ptr> rules() const { return rules_; } }; -parser build_parser(const std::function & fn); +common_chat_combinator_parser build_combinator_parser(const std::function & fn); diff --git a/tests/test-chat-parser-combinator.cpp b/tests/test-chat-parser-combinator.cpp index 8d8653f052ba2..486fe1deb34bc 100644 --- a/tests/test-chat-parser-combinator.cpp +++ b/tests/test-chat-parser-combinator.cpp @@ -33,150 +33,150 @@ static void assert_equals(const char * expected, const std::string & actual) { static void test_partial_parsing() { { // Test literal - auto parser = build_parser([](parser_builder& p) { + auto parser = build_combinator_parser([](common_chat_combinator_parser_builder& p) { return p.literal("hello"); }); - parser_context ctx; - parser_result result; + common_chat_parse_context ctx; + common_chat_parse_result result; - ctx = parser_context("hello"); + ctx = common_chat_parse_context("hello"); result = parser.parse(ctx); assert_equals(true, result.is_success()); } { // Test char class - auto parser = build_parser([](parser_builder& p) { + auto parser = build_combinator_parser([](common_chat_combinator_parser_builder& p) { return p.one("a-z"); }); - parser_context ctx; - parser_result result; + common_chat_parse_context ctx; + common_chat_parse_result result; - ctx = parser_context("a"); + ctx = common_chat_parse_context("a"); result = parser.parse(ctx); assert_equals(true, result.is_success()); - ctx = parser_context("A"); + ctx = common_chat_parse_context("A"); result = parser.parse(ctx); assert_equals(true, result.is_fail()); - parser = build_parser([](parser_builder& p) { + parser = build_combinator_parser([](common_chat_combinator_parser_builder& p) { return p.one("a-z-"); }); - ctx = parser_context("f"); + ctx = common_chat_parse_context("f"); result = parser.parse(ctx); assert_equals(true, result.is_success()); - ctx = parser_context("-"); + ctx = common_chat_parse_context("-"); result = parser.parse(ctx); assert_equals(true, result.is_success()); - ctx = parser_context("A"); + ctx = common_chat_parse_context("A"); result = parser.parse(ctx); assert_equals(true, result.is_fail()); } { // Test sequences and literals - auto parser = build_parser([](parser_builder& p) { + auto parser = build_combinator_parser([](common_chat_combinator_parser_builder& p) { return p.literal("") + p.literal(""); }); // Partial matches - auto ctx = parser_context("", false); + ctx = common_chat_parse_context("", false); result = parser.parse(ctx); assert_equals(true, result.is_need_more_input()); - ctx = parser_context("", true); + ctx = common_chat_parse_context("", true); result = parser.parse(ctx); assert_equals(true, result.is_success()); // No match, since it does not adhere to the grammar - ctx = parser_context("I am parser", false); + ctx = common_chat_parse_context("I am parser", false); result = parser.parse(ctx); assert_equals(true, result.is_fail()); } { // Test choices - auto parser = build_parser([](parser_builder& p) { + auto parser = build_combinator_parser([](common_chat_combinator_parser_builder& p) { return p.literal("") | p.literal(""); }); // Partial matches - auto ctx = parser_context("", true); + ctx = common_chat_parse_context("", true); result = parser.parse(ctx); assert_equals(true, result.is_success()); - ctx = parser_context("", true); + ctx = common_chat_parse_context("", true); result = parser.parse(ctx); assert_equals(true, result.is_success()); // No match - ctx = parser_context("", true); + ctx = common_chat_parse_context("", true); result = parser.parse(ctx); assert_equals(true, result.is_fail()); } { // Test zero_or_more - auto parser = build_parser([](parser_builder& p) { + auto parser = build_combinator_parser([](common_chat_combinator_parser_builder& p) { return p.zero_or_more(p.literal("ab")); }); // Partial matches - auto ctx = parser_context("a", false); + auto ctx = common_chat_parse_context("a", false); auto result = parser.parse(ctx); assert_equals(true, result.is_need_more_input()); - ctx = parser_context("aba", false); + ctx = common_chat_parse_context("aba", false); result = parser.parse(ctx); assert_equals(true, result.is_need_more_input()); // Full match - ctx = parser_context("ab", true); + ctx = common_chat_parse_context("ab", true); result = parser.parse(ctx); assert_equals(true, result.is_success()); } { // Test one_or_more - auto parser = build_parser([](parser_builder& p) { + auto parser = build_combinator_parser([](common_chat_combinator_parser_builder& p) { return p.one_or_more(p.literal("ab")); }); // Partial matches - auto ctx = parser_context("a", false); + auto ctx = common_chat_parse_context("a", false); auto result = parser.parse(ctx); assert_equals(true, result.is_need_more_input()); - ctx = parser_context("aba", false); + ctx = common_chat_parse_context("aba", false); result = parser.parse(ctx); assert_equals(true, result.is_need_more_input()); // Full match - ctx = parser_context("ab", true); + ctx = common_chat_parse_context("ab", true); result = parser.parse(ctx); assert_equals(true, result.is_success()); // No match - ctx = parser_context("cd", true); + ctx = common_chat_parse_context("cd", true); result = parser.parse(ctx); assert_equals(true, result.is_fail()); } @@ -185,59 +185,59 @@ static void test_partial_parsing() { static void test_one() { { // Test common escape sequences - auto parser = build_parser([](parser_builder& p) { + auto parser = build_combinator_parser([](common_chat_combinator_parser_builder& p) { return p.one("[\\n\\t\\\\]"); }); - parser_context ctx; - parser_result result; + common_chat_parse_context ctx; + common_chat_parse_result result; - ctx = parser_context("\n"); + ctx = common_chat_parse_context("\n"); result = parser.parse(ctx); assert_equals(true, result.is_success()); - ctx = parser_context("\t"); + ctx = common_chat_parse_context("\t"); result = parser.parse(ctx); assert_equals(true, result.is_success()); - ctx = parser_context("\\"); + ctx = common_chat_parse_context("\\"); result = parser.parse(ctx); assert_equals(true, result.is_success()); - ctx = parser_context(" "); + ctx = common_chat_parse_context(" "); result = parser.parse(ctx); assert_equals(true, result.is_fail()); } { // Test escaped dash (literal dash, not a range) - auto parser = build_parser([](parser_builder& p) { + auto parser = build_combinator_parser([](common_chat_combinator_parser_builder& p) { return p.one("[a\\-z]"); }); - parser_context ctx; - parser_result result; + common_chat_parse_context ctx; + common_chat_parse_result result; - ctx = parser_context("a"); + ctx = common_chat_parse_context("a"); result = parser.parse(ctx); assert_equals(true, result.is_success()); - ctx = parser_context("-"); + ctx = common_chat_parse_context("-"); result = parser.parse(ctx); assert_equals(true, result.is_success()); - ctx = parser_context("z"); + ctx = common_chat_parse_context("z"); result = parser.parse(ctx); assert_equals(true, result.is_success()); // Should NOT match 'b' since \- is a literal dash, not a range - ctx = parser_context("b"); + ctx = common_chat_parse_context("b"); result = parser.parse(ctx); assert_equals(true, result.is_fail()); } } static void test_recursive_references() { - auto value_parser = build_parser([](parser_builder& p) { + auto value_parser = build_combinator_parser([](common_chat_combinator_parser_builder& p) { p.add_rule("number", p.one_or_more(p.one("0-9"))); p.add_rule("list", p.sequence({ p.literal("["), @@ -247,73 +247,73 @@ static void test_recursive_references() { return p.add_rule("value", p.rule("number") | p.rule("list")); }); - parser_context ctx; - parser_result result; + common_chat_parse_context ctx; + common_chat_parse_result result; // Test simple number - ctx = parser_context("1", true); + ctx = common_chat_parse_context("1", true); result = value_parser.parse(ctx); assert_equals(true, result.is_success()); // Test simple list - ctx = parser_context("[1]", true); + ctx = common_chat_parse_context("[1]", true); result = value_parser.parse(ctx); assert_equals(true, result.is_success()); // Test nested list - ctx = parser_context("[[2]]", true); + ctx = common_chat_parse_context("[[2]]", true); result = value_parser.parse(ctx); assert_equals(true, result.is_success()); // Test deeply nested list - ctx = parser_context("[[[3]]]", true); + ctx = common_chat_parse_context("[[[3]]]", true); result = value_parser.parse(ctx); assert_equals(true, result.is_success()); // Test partial match - ctx = parser_context("[[", false); + ctx = common_chat_parse_context("[[", false); result = value_parser.parse(ctx); assert_equals(true, result.is_need_more_input()); // Test no match - ctx = parser_context("[a]", true); + ctx = common_chat_parse_context("[a]", true); result = value_parser.parse(ctx); assert_equals(true, result.is_fail()); } static void test_optional() { // Test optional with a match - auto parser = build_parser([](parser_builder& p) { + auto parser = build_combinator_parser([](common_chat_combinator_parser_builder& p) { return p.literal("hello") + p.optional(p.literal(" world")); }); // Full match with optional part present - auto ctx = parser_context("hello world"); + auto ctx = common_chat_parse_context("hello world"); auto result = parser.parse(ctx); assert_equals(true, result.is_success()); assert_equals((size_t)11, result.end); // Full match with optional part absent - ctx = parser_context("hello", true); + ctx = common_chat_parse_context("hello", true); result = parser.parse(ctx); assert_equals(true, result.is_success()); assert_equals((size_t)5, result.end); // Partial match - waiting for more input to determine if optional matches - ctx = parser_context("hello ", false); + ctx = common_chat_parse_context("hello ", false); result = parser.parse(ctx); assert_equals(true, result.is_need_more_input()); } static void test_json_parser() { - auto json = build_parser([](parser_builder & p) { + auto json = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.json(); }); { // Test parsing a simple JSON object std::string input = R"({"name": "test", "value": 42, "flag": true})"; - parser_context ctx(input); + common_chat_parse_context ctx(input); auto result = json.parse(ctx); @@ -323,7 +323,7 @@ static void test_json_parser() { { // Test parsing a JSON array with mixed types std::string input = R"([1, "hello", true, null, 3.14])"; - parser_context ctx(input); + common_chat_parse_context ctx(input); auto result = json.parse(ctx); @@ -333,7 +333,7 @@ static void test_json_parser() { { // Test parsing nested JSON with objects and arrays std::string input = R"({"users": [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}], "count": 2, "metadata": {"version": "1.0", "tags": ["admin", "user"]}})"; - parser_context ctx(input); + common_chat_parse_context ctx(input); auto result = json.parse(ctx); @@ -343,7 +343,7 @@ static void test_json_parser() { { // Test partial parsing - incomplete object std::string input = R"({"name": "test", "value": )"; - parser_context ctx(input, false); + common_chat_parse_context ctx(input, false); auto result = json.parse(ctx); @@ -352,7 +352,7 @@ static void test_json_parser() { { // Test partial parsing - incomplete array std::string input = R"([1, 2, 3, )"; - parser_context ctx(input, false); + common_chat_parse_context ctx(input, false); auto result = json.parse(ctx); @@ -361,7 +361,7 @@ static void test_json_parser() { { // Test partial parsing - incomplete nested structure std::string input = R"({"data": {"nested": )"; - parser_context ctx(input, false); + common_chat_parse_context ctx(input, false); auto result = json.parse(ctx); @@ -372,15 +372,15 @@ static void test_json_parser() { static void test_actions() { { // Test simple action - append matched text to content - auto parser = build_parser([](parser_builder& p) { + auto parser = build_combinator_parser([](common_chat_combinator_parser_builder& p) { auto word = p.chars("[a-z]+"); - return p.action(word, [](const parser_action & act) { + return p.action(word, [](const common_chat_parse_action & act) { act.env.result.content += std::string(act.match); }); }); - parser_environment env; - parser_context ctx("hello", &env); + common_chat_parse_semantics env; + common_chat_parse_context ctx("hello", &env); auto result = parser.parse(ctx); assert_equals(true, result.is_success()); @@ -388,12 +388,12 @@ static void test_actions() { } { // Test multiple sequential actions - build a sentence - auto parser = build_parser([](parser_builder& p) { - auto greeting = p.action(p.literal("hello"), [](const parser_action & act) { + auto parser = build_combinator_parser([](common_chat_combinator_parser_builder& p) { + auto greeting = p.action(p.literal("hello"), [](const common_chat_parse_action & act) { act.env.result.content += std::string(act.match) + " "; }); - auto name = p.action(p.chars("[A-Z][a-z]+"), [](const parser_action & act) { + auto name = p.action(p.chars("[A-Z][a-z]+"), [](const common_chat_parse_action & act) { act.env.result.content += std::string(act.match); act.env.captures["name"] = std::string(act.match); }); @@ -401,8 +401,8 @@ static void test_actions() { return greeting + p.literal(" ") + name; }); - parser_environment env; - parser_context ctx("hello Alice", &env); + common_chat_parse_semantics env; + common_chat_parse_context ctx("hello Alice", &env); auto result = parser.parse(ctx); assert_equals(true, result.is_success()); @@ -411,14 +411,14 @@ static void test_actions() { } { // Test actions don't run when parse fails - auto parser = build_parser([](parser_builder& p) { - return p.action(p.literal("success"), [](const parser_action & act) { + auto parser = build_combinator_parser([](common_chat_combinator_parser_builder& p) { + return p.action(p.literal("success"), [](const common_chat_parse_action & act) { act.env.result.content = "action_ran"; }); }); - parser_environment env; - parser_context ctx("failure", &env); + common_chat_parse_semantics env; + common_chat_parse_context ctx("failure", &env); auto result = parser.parse(ctx); assert_equals(true, result.is_fail()); @@ -426,32 +426,32 @@ static void test_actions() { } { // Test Actions work with partial parsing - auto parser = build_parser([](parser_builder& p) { - auto content = p.action(p.until(""), [](const parser_action & act) { + auto parser = build_combinator_parser([](common_chat_combinator_parser_builder& p) { + auto content = p.action(p.until(""), [](const common_chat_parse_action & act) { act.env.result.content += std::string(act.match); }); return "" << content << ""; }); { - parser_environment env; - parser_context ctx("hello ", &env, false); + common_chat_parse_semantics env; + common_chat_parse_context ctx("hello ", &env, false); auto result = parser.parse(ctx); assert_equals(true, result.is_need_more_input()); assert_equals("hello ", env.result.content); } { - parser_environment env; - parser_context ctx("hello world", &env, false); + common_chat_parse_semantics env; + common_chat_parse_context ctx("hello world", &env, false); auto result = parser.parse(ctx); assert_equals(true, result.is_need_more_input()); assert_equals("hello world", env.result.content); } { - parser_environment env; - parser_context ctx("hello world", &env, true); + common_chat_parse_semantics env; + common_chat_parse_context ctx("hello world", &env, true); auto result = parser.parse(ctx); assert_equals(true, result.is_success()); @@ -463,14 +463,14 @@ static void test_actions() { static void test_sax_events() { { // Test basic event firing - auto parser = build_parser([](parser_builder& p) { + auto parser = build_combinator_parser([](common_chat_combinator_parser_builder& p) { return p.add_rule("greeting", p.literal("hello")); }); - parser_environment env; - std::vector events; + common_chat_parse_semantics env; + std::vector events; - parser_context ctx("hello", &env, [&](const parse_event& evt, parser_environment&) { + common_chat_parse_context ctx("hello", &env, [&](const common_chat_parse_event& evt, common_chat_parse_semantics&) { events.push_back(evt); }); @@ -478,17 +478,17 @@ static void test_sax_events() { assert_equals(true, result.is_success()); assert_equals((size_t)2, events.size()); - assert_equals(PARSER_EVENT_NODE_START, events[0].type); + assert_equals(COMMON_CHAT_PARSE_EVENT_NODE_START, events[0].type); assert_equals("greeting", events[0].rule); assert_equals((size_t)0, events[0].start); assert_equals(0, events[0].depth); - assert_equals(PARSER_EVENT_NODE_END, events[1].type); + assert_equals(COMMON_CHAT_PARSE_EVENT_NODE_END, events[1].type); assert_equals("greeting", events[1].rule); assert_equals((size_t)0, events[1].start); assert_equals((size_t)5, events[1].end); assert_equals("hello", std::string(events[1].text)); - assert_equals(PARSER_RESULT_SUCCESS, events[1].status); + assert_equals(COMMON_CHAT_PARSE_RESULT_SUCCESS, events[1].status); assert_equals(0, events[1].depth); } } @@ -496,7 +496,7 @@ static void test_sax_events() { static void test_gbnf_generation() { { // Test literal - auto parser = build_parser([](parser_builder& p) { + auto parser = build_combinator_parser([](common_chat_combinator_parser_builder& p) { return p.literal("hello"); }); @@ -509,7 +509,7 @@ static void test_gbnf_generation() { } { // Test char class - auto parser = build_parser([](parser_builder& p) { + auto parser = build_combinator_parser([](common_chat_combinator_parser_builder& p) { return p.one("[a-z]"); }); @@ -521,7 +521,7 @@ static void test_gbnf_generation() { } { // Test sequence - auto parser = build_parser([](parser_builder& p) { + auto parser = build_combinator_parser([](common_chat_combinator_parser_builder& p) { return p.literal("hello") + p.literal(" ") + p.literal("world"); }); @@ -533,7 +533,7 @@ static void test_gbnf_generation() { } { // Test choice - auto parser = build_parser([](parser_builder& p) { + auto parser = build_combinator_parser([](common_chat_combinator_parser_builder& p) { return p.literal("cat") | p.literal("dog"); }); @@ -545,7 +545,7 @@ static void test_gbnf_generation() { } { // Test one_or_more - auto parser = build_parser([](parser_builder& p) { + auto parser = build_combinator_parser([](common_chat_combinator_parser_builder& p) { return p.one_or_more(p.one("[0-9]")); }); @@ -557,7 +557,7 @@ static void test_gbnf_generation() { } { // Test zero_or_more - auto parser = build_parser([](parser_builder& p) { + auto parser = build_combinator_parser([](common_chat_combinator_parser_builder& p) { return p.zero_or_more(p.one("[a-z]")); }); @@ -569,7 +569,7 @@ static void test_gbnf_generation() { } { // Test optional - auto parser = build_parser([](parser_builder& p) { + auto parser = build_combinator_parser([](common_chat_combinator_parser_builder& p) { return p.literal("hello") + p.optional(p.literal(" world")); }); @@ -581,7 +581,7 @@ static void test_gbnf_generation() { } { // Test until - auto parser = build_parser([](parser_builder& p) { + auto parser = build_combinator_parser([](common_chat_combinator_parser_builder& p) { return p.until(""); }); @@ -594,7 +594,7 @@ static void test_gbnf_generation() { } { // Test complex expression with parentheses - auto parser = build_parser([](parser_builder& p) { + auto parser = build_combinator_parser([](common_chat_combinator_parser_builder& p) { return p.one_or_more(p.literal("a") | p.literal("b")); }); @@ -606,7 +606,7 @@ static void test_gbnf_generation() { } { // Test rule references - auto parser = build_parser([](parser_builder& p) { + auto parser = build_combinator_parser([](common_chat_combinator_parser_builder& p) { auto digit = p.add_rule("digit", p.one("[0-9]")); return p.one_or_more(digit); }); @@ -621,7 +621,7 @@ static void test_gbnf_generation() { } { // Test escaping in literals - auto parser = build_parser([](parser_builder& p) { + auto parser = build_combinator_parser([](common_chat_combinator_parser_builder& p) { return p.literal("hello\nworld\t!"); }); @@ -633,7 +633,7 @@ static void test_gbnf_generation() { } { // Test operator<< (whitespace insertion) - auto parser = build_parser([](parser_builder& p) { + auto parser = build_combinator_parser([](common_chat_combinator_parser_builder& p) { return p.literal("hello") << p.literal("world"); }); @@ -684,7 +684,7 @@ static std::vector simple_tokenize(const std::string & input) { } static void example_qwen3_coder() { - auto parser = build_parser([](parser_builder & p) { + auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { auto thinking = p.add_rule("raw-reasoning", "" << p.add_rule("reasoning-content", p.until("")) << ""); @@ -713,7 +713,7 @@ static void example_qwen3_coder() { return thinking + p.optional(p.space() + content) + p.zero_or_more(p.space() + tool_call); }); - auto handler = [&](const parse_event & ev, parser_environment & env) { + auto handler = [&](const common_chat_parse_event & ev, common_chat_parse_semantics & env) { if (ev.rule == "reasoning-content" && ev.ending()) { env.result.reasoning_content = ev.text; } @@ -783,8 +783,8 @@ static void example_qwen3_coder() { for (auto it = tokens.begin(); it != tokens.end(); it++) { std::string in = std::accumulate(tokens.begin(), it, std::string()); - parser_environment env; - parser_context ctx(in, &env, it == tokens.end() - 1); + common_chat_parse_semantics env; + common_chat_parse_context ctx(in, &env, it == tokens.end() - 1); ctx.event_handler = handler; auto parse_result = parser.parse(ctx); @@ -825,8 +825,8 @@ static void example_qwen3_coder() { } } -static parser create_command_r7b_parser() { - auto parser = build_parser([](parser_builder & p) { +static common_chat_combinator_parser create_command_r7b_parser() { + auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { auto thinking = p.add_rule("thinking", "<|START_THINKING|>" << p.add_rule("reasoning-content", p.until("<|END_THINKING|>")) << "<|END_THINKING|>"); @@ -866,11 +866,11 @@ static parser create_command_r7b_parser() { return parser; } -static void test_command_r7b_parser(const parser & p, const std::string & input, bool partial, bool print_results = false) { - parser_environment env; - parser_context ctx(input, &env, !partial); +static void test_command_r7b_parser(const common_chat_combinator_parser & p, const std::string & input, bool partial, bool print_results = false) { + common_chat_parse_semantics env; + common_chat_parse_context ctx(input, &env, !partial); - ctx.event_handler = [&](const parse_event & ev, parser_environment & env) { + ctx.event_handler = [&](const common_chat_parse_event & ev, common_chat_parse_semantics & env) { if (ev.rule == "reasoning-content" && ev.ending()) { env.result.reasoning_content = ev.text; } From 7f92bcfe1c2d9d94b05f3129cf5e20b21a26ee8e Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Fri, 14 Nov 2025 07:58:42 -0600 Subject: [PATCH 048/183] remove is_ suffix from functions --- common/chat-parser-combinator.cpp | 16 ++-- common/chat-parser-combinator.h | 8 +- tests/test-chat-parser-combinator.cpp | 112 +++++++++++++------------- 3 files changed, 68 insertions(+), 68 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index e3c95bb6893de..75274a8a23f50 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -383,7 +383,7 @@ class sequence_parser : public common_chat_combinator_parser_base { auto pos = start; for (const auto & p : parsers_) { auto result = p->parse(ctx, pos); - if (!result.is_success()) { + if (!result.success()) { return common_chat_parse_result(result.type, start, result.end); } @@ -440,7 +440,7 @@ class choice_parser : public common_chat_combinator_parser_base { auto pos = start; for (const auto & p : parsers_) { auto result = p->parse(ctx, pos); - if (!result.is_fail()) { + if (!result.fail()) { return result; } } @@ -497,7 +497,7 @@ class repetition_parser : public common_chat_combinator_parser_base { auto result = parser_->parse(ctx, pos); - if (result.is_success()) { + if (result.success()) { // Prevent infinite loop on empty matches if (result.end == pos) { break; @@ -507,7 +507,7 @@ class repetition_parser : public common_chat_combinator_parser_base { continue; } - if (result.is_need_more_input()) { + if (result.need_more_input()) { return common_chat_parse_result(result.type, start, result.end); } @@ -609,10 +609,10 @@ class and_parser : public common_chat_combinator_parser_base { common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto result = parser_->parse(ctx, start); - if (result.is_success()) { + if (result.success()) { return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start); } - if (result.is_need_more_input()) { + if (result.need_more_input()) { return result; } return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start); @@ -647,12 +647,12 @@ class not_parser : public common_chat_combinator_parser_base { common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto result = parser_->parse(ctx, start); - if (result.is_success()) { + if (result.success()) { // Fail if the underlying parser matches return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); } - if (result.is_need_more_input()) { + if (result.need_more_input()) { // Propagate - need to know what child would match before negating return result; } diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index 6156d62128700..c4041686eafde 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -54,9 +54,9 @@ struct common_chat_parse_result { common_chat_parse_result(common_chat_parse_result_type type, size_t start, size_t end) : type(type), start(start), end(end) {} - bool is_fail() const { return type == COMMON_CHAT_PARSE_RESULT_FAIL; } - bool is_need_more_input() const { return type == COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT; } - bool is_success() const { return type == COMMON_CHAT_PARSE_RESULT_SUCCESS; } + bool fail() const { return type == COMMON_CHAT_PARSE_RESULT_FAIL; } + bool need_more_input() const { return type == COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT; } + bool success() const { return type == COMMON_CHAT_PARSE_RESULT_SUCCESS; } }; struct common_chat_parse_action { @@ -83,7 +83,7 @@ struct common_chat_parse_event { bool ending() const { return type == COMMON_CHAT_PARSE_EVENT_NODE_END; } bool success() const { return status == COMMON_CHAT_PARSE_RESULT_SUCCESS; } - bool partial() const { return status == COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT; } + bool need_more_input() const { return status == COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT; } bool fail() const { return status == COMMON_CHAT_PARSE_RESULT_FAIL; } }; diff --git a/tests/test-chat-parser-combinator.cpp b/tests/test-chat-parser-combinator.cpp index 486fe1deb34bc..a3e4496760018 100644 --- a/tests/test-chat-parser-combinator.cpp +++ b/tests/test-chat-parser-combinator.cpp @@ -42,7 +42,7 @@ static void test_partial_parsing() { ctx = common_chat_parse_context("hello"); result = parser.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.success()); } { // Test char class @@ -55,11 +55,11 @@ static void test_partial_parsing() { ctx = common_chat_parse_context("a"); result = parser.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.success()); ctx = common_chat_parse_context("A"); result = parser.parse(ctx); - assert_equals(true, result.is_fail()); + assert_equals(true, result.fail()); parser = build_combinator_parser([](common_chat_combinator_parser_builder& p) { return p.one("a-z-"); @@ -67,15 +67,15 @@ static void test_partial_parsing() { ctx = common_chat_parse_context("f"); result = parser.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.success()); ctx = common_chat_parse_context("-"); result = parser.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.success()); ctx = common_chat_parse_context("A"); result = parser.parse(ctx); - assert_equals(true, result.is_fail()); + assert_equals(true, result.fail()); } { // Test sequences and literals @@ -86,25 +86,25 @@ static void test_partial_parsing() { // Partial matches auto ctx = common_chat_parse_context("", false); result = parser.parse(ctx); - assert_equals(true, result.is_need_more_input()); + assert_equals(true, result.need_more_input()); ctx = common_chat_parse_context("", true); result = parser.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.success()); // No match, since it does not adhere to the grammar ctx = common_chat_parse_context("I am parser", false); result = parser.parse(ctx); - assert_equals(true, result.is_fail()); + assert_equals(true, result.fail()); } { // Test choices @@ -115,25 +115,25 @@ static void test_partial_parsing() { // Partial matches auto ctx = common_chat_parse_context("", true); result = parser.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.success()); ctx = common_chat_parse_context("", true); result = parser.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.success()); // No match ctx = common_chat_parse_context("", true); result = parser.parse(ctx); - assert_equals(true, result.is_fail()); + assert_equals(true, result.fail()); } { // Test zero_or_more @@ -144,16 +144,16 @@ static void test_partial_parsing() { // Partial matches auto ctx = common_chat_parse_context("a", false); auto result = parser.parse(ctx); - assert_equals(true, result.is_need_more_input()); + assert_equals(true, result.need_more_input()); ctx = common_chat_parse_context("aba", false); result = parser.parse(ctx); - assert_equals(true, result.is_need_more_input()); + assert_equals(true, result.need_more_input()); // Full match ctx = common_chat_parse_context("ab", true); result = parser.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.success()); } { // Test one_or_more @@ -164,21 +164,21 @@ static void test_partial_parsing() { // Partial matches auto ctx = common_chat_parse_context("a", false); auto result = parser.parse(ctx); - assert_equals(true, result.is_need_more_input()); + assert_equals(true, result.need_more_input()); ctx = common_chat_parse_context("aba", false); result = parser.parse(ctx); - assert_equals(true, result.is_need_more_input()); + assert_equals(true, result.need_more_input()); // Full match ctx = common_chat_parse_context("ab", true); result = parser.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.success()); // No match ctx = common_chat_parse_context("cd", true); result = parser.parse(ctx); - assert_equals(true, result.is_fail()); + assert_equals(true, result.fail()); } } @@ -194,19 +194,19 @@ static void test_one() { ctx = common_chat_parse_context("\n"); result = parser.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.success()); ctx = common_chat_parse_context("\t"); result = parser.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.success()); ctx = common_chat_parse_context("\\"); result = parser.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.success()); ctx = common_chat_parse_context(" "); result = parser.parse(ctx); - assert_equals(true, result.is_fail()); + assert_equals(true, result.fail()); } { // Test escaped dash (literal dash, not a range) @@ -219,20 +219,20 @@ static void test_one() { ctx = common_chat_parse_context("a"); result = parser.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.success()); ctx = common_chat_parse_context("-"); result = parser.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.success()); ctx = common_chat_parse_context("z"); result = parser.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.success()); // Should NOT match 'b' since \- is a literal dash, not a range ctx = common_chat_parse_context("b"); result = parser.parse(ctx); - assert_equals(true, result.is_fail()); + assert_equals(true, result.fail()); } } @@ -253,32 +253,32 @@ static void test_recursive_references() { // Test simple number ctx = common_chat_parse_context("1", true); result = value_parser.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.success()); // Test simple list ctx = common_chat_parse_context("[1]", true); result = value_parser.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.success()); // Test nested list ctx = common_chat_parse_context("[[2]]", true); result = value_parser.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.success()); // Test deeply nested list ctx = common_chat_parse_context("[[[3]]]", true); result = value_parser.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.success()); // Test partial match ctx = common_chat_parse_context("[[", false); result = value_parser.parse(ctx); - assert_equals(true, result.is_need_more_input()); + assert_equals(true, result.need_more_input()); // Test no match ctx = common_chat_parse_context("[a]", true); result = value_parser.parse(ctx); - assert_equals(true, result.is_fail()); + assert_equals(true, result.fail()); } static void test_optional() { @@ -290,19 +290,19 @@ static void test_optional() { // Full match with optional part present auto ctx = common_chat_parse_context("hello world"); auto result = parser.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.success()); assert_equals((size_t)11, result.end); // Full match with optional part absent ctx = common_chat_parse_context("hello", true); result = parser.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.success()); assert_equals((size_t)5, result.end); // Partial match - waiting for more input to determine if optional matches ctx = common_chat_parse_context("hello ", false); result = parser.parse(ctx); - assert_equals(true, result.is_need_more_input()); + assert_equals(true, result.need_more_input()); } static void test_json_parser() { @@ -317,7 +317,7 @@ static void test_json_parser() { auto result = json.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.success()); assert_equals(input.size(), result.end); } { @@ -327,7 +327,7 @@ static void test_json_parser() { auto result = json.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.success()); assert_equals(input.size(), result.end); } { @@ -337,7 +337,7 @@ static void test_json_parser() { auto result = json.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.success()); assert_equals(input.size(), result.end); } { @@ -347,7 +347,7 @@ static void test_json_parser() { auto result = json.parse(ctx); - assert_equals(true, result.is_need_more_input()); + assert_equals(true, result.need_more_input()); } { // Test partial parsing - incomplete array @@ -356,7 +356,7 @@ static void test_json_parser() { auto result = json.parse(ctx); - assert_equals(true, result.is_need_more_input()); + assert_equals(true, result.need_more_input()); } { // Test partial parsing - incomplete nested structure @@ -365,7 +365,7 @@ static void test_json_parser() { auto result = json.parse(ctx); - assert_equals(true, result.is_need_more_input()); + assert_equals(true, result.need_more_input()); } } @@ -383,7 +383,7 @@ static void test_actions() { common_chat_parse_context ctx("hello", &env); auto result = parser.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.success()); assert_equals("hello", env.result.content); } { @@ -405,7 +405,7 @@ static void test_actions() { common_chat_parse_context ctx("hello Alice", &env); auto result = parser.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.success()); assert_equals("hello Alice", env.result.content); assert_equals("Alice", env.captures["name"]); } @@ -421,7 +421,7 @@ static void test_actions() { common_chat_parse_context ctx("failure", &env); auto result = parser.parse(ctx); - assert_equals(true, result.is_fail()); + assert_equals(true, result.fail()); assert_equals("", env.result.content); // Action should not have run } { @@ -438,7 +438,7 @@ static void test_actions() { common_chat_parse_context ctx("hello ", &env, false); auto result = parser.parse(ctx); - assert_equals(true, result.is_need_more_input()); + assert_equals(true, result.need_more_input()); assert_equals("hello ", env.result.content); } { @@ -446,7 +446,7 @@ static void test_actions() { common_chat_parse_context ctx("hello world", &env, false); auto result = parser.parse(ctx); - assert_equals(true, result.is_need_more_input()); + assert_equals(true, result.need_more_input()); assert_equals("hello world", env.result.content); } { @@ -454,7 +454,7 @@ static void test_actions() { common_chat_parse_context ctx("hello world", &env, true); auto result = parser.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.success()); assert_equals("hello world", env.result.content); } } @@ -476,7 +476,7 @@ static void test_sax_events() { auto result = parser.parse(ctx); - assert_equals(true, result.is_success()); + assert_equals(true, result.success()); assert_equals((size_t)2, events.size()); assert_equals(COMMON_CHAT_PARSE_EVENT_NODE_START, events[0].type); assert_equals("greeting", events[0].rule); @@ -749,7 +749,7 @@ static void example_qwen3_coder() { tc.arguments += "\""; } - if (ev.rule == "arg-json-content" && ev.ending() && (ev.success() || ev.partial())) { + if (ev.rule == "arg-json-content" && ev.ending() && (ev.success() || ev.need_more_input())) { auto & tc = env.result.tool_calls.back(); tc.arguments += std::string(ev.text); } @@ -788,7 +788,7 @@ static void example_qwen3_coder() { ctx.event_handler = handler; auto parse_result = parser.parse(ctx); - assert_equals(false, parse_result.is_fail()); + assert_equals(false, parse_result.fail()); std::cout << "=================================\n"; std::cout << in << "\n\n"; @@ -893,7 +893,7 @@ static void test_command_r7b_parser(const common_chat_combinator_parser & p, con tc.name = nlohmann::json::parse(ev.text).get(); } - if (ev.rule == "tool-args-value" && ev.ending() && (ev.success() || ev.partial())) { + if (ev.rule == "tool-args-value" && ev.ending() && (ev.success() || ev.need_more_input())) { auto & tc = env.result.tool_calls.back(); tc.arguments = ev.text; } From 87b92affccac16062f0c47a32b2d111575c90c98 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Fri, 14 Nov 2025 08:00:51 -0600 Subject: [PATCH 049/183] rename from id_counter to just counter --- common/chat-parser-combinator.cpp | 22 +++++++++++----------- common/chat-parser-combinator.h | 8 ++++---- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index 75274a8a23f50..255fab3f52473 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -66,7 +66,7 @@ class common_chat_combinator_parser_base { // Actual parsing implementation (to be overridden by subclasses) virtual common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) = 0; - virtual void assign_id(std::shared_ptr counter) { + virtual void assign_id(std::shared_ptr counter) { if (id_ == -1) { id_ = counter->next(); } @@ -393,7 +393,7 @@ class sequence_parser : public common_chat_combinator_parser_base { return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); } - void assign_id(std::shared_ptr counter) override { + void assign_id(std::shared_ptr counter) override { common_chat_combinator_parser_base::assign_id(counter); for (auto & p : parsers_) { p->assign_id(counter); @@ -448,7 +448,7 @@ class choice_parser : public common_chat_combinator_parser_base { return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); } - void assign_id(std::shared_ptr counter) override { + void assign_id(std::shared_ptr counter) override { common_chat_combinator_parser_base::assign_id(counter); for (auto & p : parsers_) { p->assign_id(counter); @@ -523,7 +523,7 @@ class repetition_parser : public common_chat_combinator_parser_base { return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); } - void assign_id(std::shared_ptr counter) override { + void assign_id(std::shared_ptr counter) override { common_chat_combinator_parser_base::assign_id(counter); parser_->assign_id(counter); } @@ -618,7 +618,7 @@ class and_parser : public common_chat_combinator_parser_base { return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start); } - void assign_id(std::shared_ptr counter) override { + void assign_id(std::shared_ptr counter) override { common_chat_combinator_parser_base::assign_id(counter); parser_->assign_id(counter); } @@ -661,7 +661,7 @@ class not_parser : public common_chat_combinator_parser_base { return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start); } - void assign_id(std::shared_ptr counter) override { + void assign_id(std::shared_ptr counter) override { common_chat_combinator_parser_base::assign_id(counter); parser_->assign_id(counter); } @@ -1115,7 +1115,7 @@ class root_parser : public common_chat_combinator_parser_base { return root_->parse(ctx, start); } - void assign_id(std::shared_ptr counter) override { + void assign_id(std::shared_ptr counter) override { common_chat_combinator_parser_base::assign_id(counter); root_->assign_id(counter); } @@ -1165,7 +1165,7 @@ class action_parser : public common_chat_combinator_parser_base { return result; } - void assign_id(std::shared_ptr counter) override { + void assign_id(std::shared_ptr counter) override { common_chat_combinator_parser_base::assign_id(counter); parser_->assign_id(counter); } @@ -1533,9 +1533,9 @@ void common_chat_combinator_parser::build_grammar(const common_grammar_builder & common_chat_combinator_parser_builder::common_chat_combinator_parser_builder() : rules_(std::make_shared>()) - , counter_(std::make_shared(0)) {} + , counter_(std::make_shared(0)) {} -common_chat_combinator_parser_builder::common_chat_combinator_parser_builder(std::shared_ptr counter) +common_chat_combinator_parser_builder::common_chat_combinator_parser_builder(std::shared_ptr counter) : rules_(std::make_shared>()) , counter_(std::move(counter)) {} @@ -1650,7 +1650,7 @@ common_chat_combinator_parser build_combinator_parser(const std::function counter) { +static common_chat_combinator_parser json_parser(std::shared_ptr counter) { common_chat_combinator_parser_builder builder(std::move(counter)); // Whitespace: space, tab, newline, carriage return diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index c4041686eafde..ebc5321f9a244 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -171,20 +171,20 @@ common_chat_combinator_parser operator+(const char * lhs, const common_chat_comb common_chat_combinator_parser operator|(const char * lhs, const common_chat_combinator_parser & rhs); common_chat_combinator_parser operator<<(const char * lhs, const common_chat_combinator_parser & rhs); -class common_chat_combinator_parser_id_counter { +class common_chat_combinator_parser_counter { int next_id_; public: - common_chat_combinator_parser_id_counter(int start) : next_id_(start) {} + common_chat_combinator_parser_counter(int start) : next_id_(start) {} int next() { return next_id_++; } }; class common_chat_combinator_parser_builder { std::shared_ptr> rules_; - std::shared_ptr counter_; + std::shared_ptr counter_; public: common_chat_combinator_parser_builder(); - common_chat_combinator_parser_builder(std::shared_ptr counter); + common_chat_combinator_parser_builder(std::shared_ptr counter); // Matches an exact literal string. // S -> "hello" From b1aadf8dfc21f581e6a79ee40b88f8d84473b66b Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Fri, 14 Nov 2025 16:02:20 +0100 Subject: [PATCH 050/183] Final refactored tests --- tests/CMakeLists.txt | 12 +- tests/combinator/benchmark.cpp | 24 + tests/combinator/simple_tokenizer.cpp | 37 + tests/combinator/test-actions.cpp | 117 +- tests/combinator/test-combinator-all.cpp | 36 - .../test-command7-parser-compare.cpp | 238 ++++ tests/combinator/test-complete-example.cpp | 214 +-- tests/combinator/test-example-qwen3-coder.cpp | 162 +++ tests/combinator/test-gbnf-generation.cpp | 244 ++-- tests/combinator/test-json-parser.cpp | 120 +- tests/combinator/test-one.cpp | 187 ++- tests/combinator/test-optional.cpp | 68 +- tests/combinator/test-partial-parsing.cpp | 348 +++-- .../combinator/test-recursive-references.cpp | 155 +-- tests/combinator/tests.h | 97 +- tests/test-chat-parser-combinator.cpp | 1224 +---------------- tests/test-grammar-integration.cpp | 9 +- tests/testcase.hpp | 205 +-- 18 files changed, 1395 insertions(+), 2102 deletions(-) create mode 100644 tests/combinator/benchmark.cpp create mode 100644 tests/combinator/simple_tokenizer.cpp delete mode 100644 tests/combinator/test-combinator-all.cpp create mode 100644 tests/combinator/test-command7-parser-compare.cpp create mode 100644 tests/combinator/test-example-qwen3-coder.cpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a8ff1debc6661..6168a46b0bfca 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -180,18 +180,18 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS) endif() llama_build_and_test(test-chat-parser.cpp) -llama_build_and_test(test-chat-parser-combinator.cpp) # Combinator tests (modular) file(GLOB_RECURSE COMBINATOR_TEST_SOURCES combinator/*.cpp combinator/*.hpp + test-chat-parser-combinator.cpp ) -add_executable(test-combinator ${COMBINATOR_TEST_SOURCES}) -target_link_libraries(test-combinator PRIVATE common) -install(TARGETS test-combinator RUNTIME) -add_test(NAME test-combinator COMMAND test-combinator) -set_property(TEST test-combinator PROPERTY LABELS main) +add_executable(test-chat-parser-combinator ${COMBINATOR_TEST_SOURCES}) +target_link_libraries(test-chat-parser-combinator PRIVATE common) +install(TARGETS test-chat-parser-combinator RUNTIME) +add_test(NAME test-chat-parser-combinator COMMAND test-combinator) +set_property(TEST test-chat-parser-combinator PROPERTY LABELS main) llama_build_and_test(test-chat-template.cpp) llama_build_and_test(test-json-partial.cpp) diff --git a/tests/combinator/benchmark.cpp b/tests/combinator/benchmark.cpp new file mode 100644 index 0000000000000..b7c14ffbed684 --- /dev/null +++ b/tests/combinator/benchmark.cpp @@ -0,0 +1,24 @@ +#include "tests.h" +#include +#include +#include +#include + +// benchmark_test base class implementation +benchmark_test::benchmark_test(std::vector> cs): cases(std::move(cs)) {} + +long long benchmark_test::run_benchmark(size_t which, int iterations) { + if (which >= cases.size()) { + throw std::runtime_error(std::string("Invalid index for benchmark test: ") + std::to_string(which)); + } + std::chrono::microseconds duration(0); + test_case& tc = *cases.at(which); + for (int i = 0; i < iterations; i++) { + auto start = std::chrono::high_resolution_clock::now(); + tc.run(); + auto end = std::chrono::high_resolution_clock::now(); + tc.reset(); + duration += std::chrono::duration_cast(end - start); + } + return duration.count() / iterations; +} diff --git a/tests/combinator/simple_tokenizer.cpp b/tests/combinator/simple_tokenizer.cpp new file mode 100644 index 0000000000000..77beee35a346e --- /dev/null +++ b/tests/combinator/simple_tokenizer.cpp @@ -0,0 +1,37 @@ +#include "tests.h" + +std::vector uses_simple_tokenizer::simple_tokenize(const std::string & input) { + std::vector result; + std::string current; + + for (size_t i = 0; i < input.size(); i++) { + switch (input[i]) { + case ' ': + case '\n': + case '\t': + case '{': + case '}': + case ',': + case '[': + case '"': + case ']': + case '.': + case '<': + case '>': + case '=': + case '/': + if (!current.empty()) { + result.push_back(current); + current.clear(); + } + default:; + } + current += input[i]; + } + + if (!current.empty()) { + result.push_back(current); + } + + return result; +} diff --git a/tests/combinator/test-actions.cpp b/tests/combinator/test-actions.cpp index 3ec274ad19b8d..6c31c4c9b8e1f 100644 --- a/tests/combinator/test-actions.cpp +++ b/tests/combinator/test-actions.cpp @@ -1,28 +1,28 @@ #include "tests.h" -class test_actions : public compound_test { -public: - test_actions() : compound_test("test_actions") { - // Test simple action - append matched text to content - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { +test_actions::test_actions() : compound_test("test_actions") { + // Test simple action - append matched text to content + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { auto word = p.chars("[a-z]+"); - return p.action(word, [](const parser_action & act) { - act.env.result.content += std::string(act.match); - }); + return p.action(word, + [](const parser_action & act) { act.env.result.content += std::string(act.match); }); }); parser_environment env; - parser_context ctx("hello", &env); - auto result = parser.parse(ctx); + parser_context ctx("hello", &env); + auto result = parser.parse(ctx); h.assert_equals("result_is_success", true, result.is_success()); h.assert_equals("result_is_hello", std::string("hello"), env.result.content); - }, "simple action - append matched text to content"); - - // Test multiple sequential actions - build a sentence - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { + }, + "simple action - append matched text to content"); + + // Test multiple sequential actions - build a sentence + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { auto greeting = p.action(p.literal("hello"), [](const parser_action & act) { act.env.result.content += std::string(act.match) + " "; }); @@ -36,20 +36,22 @@ class test_actions : public compound_test { }); parser_environment env; - parser_context ctx("hello Alice", &env); - auto result = parser.parse(ctx); + parser_context ctx("hello Alice", &env); + auto result = parser.parse(ctx); h.assert_equals("result_is_success", true, result.is_success()); h.assert_equals("result_content", std::string("hello Alice"), env.result.content); h.assert_equals("scratchpad_name", std::string("Alice"), std::get(env.scratchpad["name"])); - }, "multiple sequential actions - build a sentence"); - - // Test using scratchpad for intermediate calculations - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { + }, + "multiple sequential actions - build a sentence"); + + // Test using scratchpad for intermediate calculations + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { auto digit = p.action(p.one("[0-9]"), [](const parser_action & act) { - auto it = act.env.scratchpad.find("sum"); - int current_sum = it != act.env.scratchpad.end() ? std::get(it->second) : 0; + auto it = act.env.scratchpad.find("sum"); + int current_sum = it != act.env.scratchpad.end() ? std::get(it->second) : 0; current_sum += (act.match[0] - '0'); act.env.scratchpad["sum"] = current_sum; }); @@ -58,32 +60,35 @@ class test_actions : public compound_test { }); parser_environment env; - parser_context ctx("1+2+3+4", &env); - auto result = parser.parse(ctx); + parser_context ctx("1+2+3+4", &env); + auto result = parser.parse(ctx); h.assert_equals("result_is_success", true, result.is_success()); h.assert_equals("scratchpad_sum", 10, std::get(env.scratchpad["sum"])); // 1+2+3+4 = 10 - }, "using scratchpad for intermediate calculations"); - - // Test actions don't run when parse fails - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.action(p.literal("success"), [](const parser_action & act) { - act.env.result.content = "action_ran"; - }); + }, + "using scratchpad for intermediate calculations"); + + // Test actions don't run when parse fails + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { + return p.action(p.literal("success"), + [](const parser_action & act) { act.env.result.content = "action_ran"; }); }); parser_environment env; - parser_context ctx("failure", &env); - auto result = parser.parse(ctx); + parser_context ctx("failure", &env); + auto result = parser.parse(ctx); h.assert_equals("result_is_fail", true, result.is_fail()); h.assert_equals("result_content_empty", std::string(""), env.result.content); // Action should not have run - }, "actions don't run when parse fails"); - - // Test Actions work with partial parsing - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { + }, + "actions don't run when parse fails"); + + // Test Actions work with partial parsing + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { auto content = p.action(p.until(""), [](const parser_action & act) { act.env.result.content += std::string(act.match); }); @@ -92,36 +97,30 @@ class test_actions : public compound_test { { parser_environment env; - parser_context ctx("hello ", &env, false); - auto result = parser.parse(ctx); + parser_context ctx("hello ", &env, false); + auto result = parser.parse(ctx); h.assert_equals("result_is_need_more_input_1", true, result.is_need_more_input()); h.assert_equals("result_content_1", std::string("hello "), env.result.content); } - + { parser_environment env; - parser_context ctx("hello world", &env, false); - auto result = parser.parse(ctx); + parser_context ctx("hello world", &env, false); + auto result = parser.parse(ctx); h.assert_equals("result_is_need_more_input_2", true, result.is_need_more_input()); h.assert_equals("result_content_2", std::string("hello world"), env.result.content); } - + { parser_environment env; - parser_context ctx("hello world", &env, true); - auto result = parser.parse(ctx); + parser_context ctx("hello world", &env, true); + auto result = parser.parse(ctx); h.assert_equals("result_is_success", true, result.is_success()); h.assert_equals("result_content_final", std::string("hello world"), env.result.content); } - }, "actions work with partial parsing"); - } - - // Provide a convenient way to run all tests - void run_all_tests() { - run_all(); - summary(); - } -}; \ No newline at end of file + }, + "actions work with partial parsing"); +} diff --git a/tests/combinator/test-combinator-all.cpp b/tests/combinator/test-combinator-all.cpp deleted file mode 100644 index 90df6daa9aba3..0000000000000 --- a/tests/combinator/test-combinator-all.cpp +++ /dev/null @@ -1,36 +0,0 @@ -#include "test-partial-parsing.cpp" -#include "test-one.cpp" -#include "test-optional.cpp" -#include "test-recursive-references.cpp" -#include "test-json-parser.cpp" -#include "test-complete-example.cpp" -#include "test-actions.cpp" -#include "test-gbnf-generation.cpp" - -int main() { - test_partial_parsing partial_parsing_test; - partial_parsing_test.run_all_tests(); - - test_one one_test; - one_test.run_all_tests(); - - test_optional optional_test; - optional_test.run_all_tests(); - - test_recursive_references recursive_references_test; - recursive_references_test.run_all_tests(); - - test_json_parser json_parser_test; - json_parser_test.run_all_tests(); - - test_complete_example complete_example_test; - complete_example_test.run_all_tests(); - - test_actions actions_test; - actions_test.run_all_tests(); - - test_gbnf_generation gbnf_generation_test; - gbnf_generation_test.run_all_tests(); - - return 0; -} \ No newline at end of file diff --git a/tests/combinator/test-command7-parser-compare.cpp b/tests/combinator/test-command7-parser-compare.cpp new file mode 100644 index 0000000000000..64b91814e7ce4 --- /dev/null +++ b/tests/combinator/test-command7-parser-compare.cpp @@ -0,0 +1,238 @@ +#include "chat-parser.h" +#include "json-schema-to-grammar.h" +#include "tests.h" +#include +#include + +class parser test_command7_parser_compare::create_command_r7b_parser() { + auto parser = build_parser([](parser_builder & p) { + auto thinking = p.add_rule( + "thinking", "<|START_THINKING|>" << p.append_reasoning(p.until("<|END_THINKING|>")) << "<|END_THINKING|>"); + + auto response = p.add_rule( + "response", "<|START_RESPONSE|>" << p.append_content(p.until("<|END_RESPONSE|>")) << "<|END_RESPONSE|>"); + + auto json = p.add_rule("json", p.json()); + + auto tool_call_id = + p.add_rule("tool-call-id", + p.json_key("tool_call_id", + "\"" + p.capture_tool_call_id(p.json_string(), /* unescape_json = */ true) + "\"")); + + auto tool_call_name = + p.add_rule("tool-name", + p.json_key("tool_name", + "\"" + p.capture_tool_call_name(p.json_string(), /* unescape_json = */ true) + "\"")); + + auto tool_call_args = p.add_rule("tool-args", p.json_key("parameters", p.capture_tool_call_args(json))); + + auto tool_call_fields = p.add_rule("tool-call-fields", tool_call_id | tool_call_name | tool_call_args); + + auto tool_call = p.add_rule( + "tool-call", + "{" << p.add_tool_call(tool_call_fields << p.zero_or_more(p.literal(",") << tool_call_fields)) << "}"); + + auto tool_calls = p.add_rule( + "tool-calls", "<|START_ACTION|>" << ("[" << tool_call << p.zero_or_more(p.literal(",") << tool_call) << "]") + << "<|END_ACTION|>"); + + return p.optional(thinking) << p.add_rule("content", tool_calls | response); + }); + + // Check if + build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); + return parser; +} + +// command7_parser_compare_test implementation +test_command7_parser_compare::test_command7_parser_compare() : + benchmark_test(std::vector>()), + parser(create_command_r7b_parser()), + reasoning("To plan an effective trip to Japan that includes both historical sites and modern attractions within a " + "budget of $4000 for a two-week stay, we need to:\n\n" + "1. Identify key historical sites and modern attractions in Japan.\n" + "2. Find affordable accommodation options that provide a balance between comfort and cost.\n" + "3. Determine the best modes of transportation for getting around Japan.\n" + "4. Create a day-by-day itinerary that ensures the user gets to see a variety of attractions without " + "overspending.\n" + "5. Provide a detailed cost breakdown that includes accommodation, transportation, meals, and entry fees " + "to attractions."), + content("For a two-week trip to Japan with a $4,000 budget, I recommend planning an itinerary that balances " + "historical sites with modern attractions. The destination will be Japan, with a duration of 14 days.\n\n" + "Given your interests in both historical sites and modern attractions, you'll want to focus on cities like " + "Kyoto for its temples and traditional culture, Tokyo for its cutting-edge technology and entertainment " + "districts, and possibly Hiroshima or Nara for additional historical significance.\n\n" + "For accommodation, I suggest looking for affordable options such as budget hotels, hostels, or " + "guesthouses that offer good value without sacrificing too much comfort. Japan has excellent mid-range " + "accommodation options that can keep your lodging costs manageable.\n\n" + "Transportation should prioritize efficiencyβ€”consider getting a JR Rail Pass for intercity travel, which " + "allows unlimited rides on most JR trains including the Shinkansen (bullet train). Within cities, use " + "local trains and subways, which are both affordable and highly reliable.\n\n" + "For meals, embrace local cuisine by eating at neighborhood restaurants, ramen shops, and izakayas rather " + "than touristy establishments. This will give you an authentic experience while keeping costs " + "reasonableβ€”you can enjoy excellent meals for $10-20 per person at local spots.\n\n"), + tool_calls({ + { "call_0", "plan_trip", nlohmann::json::parse(R"({ + "destination": "Japan", + "duration": 14, + "budget": 4000, + "interests": ["historical sites", "modern attractions"], + "accommodation_preferences": "affordable", + "transportation_preferences": "efficient", + "meal_preferences": "local cuisine" + })") } + }) + { + // Build response + if (!reasoning.empty()) { + auto tokenized = simple_tokenize(reasoning); + tokens.emplace_back("<|START_THINKING|>"); + tokens.insert(tokens.end(), tokenized.begin(), tokenized.end()); + tokens.emplace_back("<|END_THINKING|>"); + } + + if (!content.empty()) { + auto tokenized = simple_tokenize(content); + tokens.emplace_back("<|START_RESPONSE|>"); + tokens.insert(tokens.end(), tokenized.begin(), tokenized.end()); + tokens.emplace_back("<|END_RESPONSE|>"); + } + + if (!tool_calls.empty()) { + tokens.emplace_back("<|START_ACTION|>"); + + auto json = nlohmann::json::array(); + for (const auto & tc : tool_calls) { + auto tc_json = nlohmann::json::object(); + tc_json["tool_call_id"] = tc.id; + tc_json["tool_name"] = tc.name; + tc_json["parameters"] = tc.args; + json.push_back(tc_json); + } + + auto tokenized = simple_tokenize(json.dump(-1, ' ', true)); + tokens.insert(tokens.end(), tokenized.begin(), tokenized.end()); + + tokens.emplace_back("<|END_ACTION|>"); + } + + test_case legacy = test_case([this](test_harness h) { + bool no_error = true; + try { + std::string input = std::accumulate(tokens.begin(), tokens.end(), std::string()); + test_command_r7b_legacy_parser(input, false, false); + } catch (std::exception &e) { + no_error = false; + std::cerr << "Error during legacy run: " << e.what() << "\n"; + } + h.assert_equals("no_errors", true, no_error); + }, "legacy_parse"); + + test_case current = test_case([this](test_harness h) { + bool no_error = true; + try { + std::string input = std::accumulate(tokens.begin(), tokens.end(), std::string()); + test_command_r7b_parser(parser, input, false, false); + } catch (std::exception &e) { + no_error = false; + std::cerr << "Error during legacy run: " << e.what() << "\n"; + } + h.assert_equals("no_errors", true, no_error); + }, "current_parse"); + legacy.set_omit_success_msg(true); + current.set_omit_success_msg(true); + + cases.push_back(std::make_unique(legacy)); + cases.push_back(std::make_unique(current)); +} + +void test_command7_parser_compare::run_comparison(int iterations) { + long long t1 = run_benchmark(0, iterations); + long long t2 = run_benchmark(1, iterations); + + std::cout << "=== Command7 parser comparison benchmark (" << iterations << " iterations) ===\n"; + std::cout << "Legacy parser performance: " << t1 << "ms (" << (float) t1 / iterations << "ms per iteration)\n"; + std::cout << "Current parser performance: " << t2 << "ms (" << (float) t2 / iterations << "ms per iteration)\n"; +} + +void test_command7_parser_compare::test_command_r7b_parser(const class parser & p, + const std::string & input, + bool partial, + bool print_results) { + parser_environment env; + parser_context ctx(input, &env, !partial); + p.parse(ctx); + + if (print_results) { + std::cout << "== Parsed (new) ==\n"; + std::cout << "=== Reasoning ===\n"; + std::cout << env.result.reasoning_content << "\n"; + std::cout << "\n\n=== Content ===\n"; + std::cout << env.result.content << "\n"; + std::cout << "\n\n=== Tool Calls ===\n"; + for (const auto & tc : env.result.tool_calls) { + std::cout << "id: " << tc.id << "\n"; + std::cout << "name: " << tc.name << "\n"; + std::cout << "args: " << tc.arguments << "\n"; + } + } +} + +void test_command7_parser_compare::test_command_r7b_legacy_parser(const std::string & input, + bool partial, + bool print_results) { + // Original parser taken from chat.cpp + common_chat_msg_parser builder(input, + /* is_partial= */ partial, + { + /* .format = */ COMMON_CHAT_FORMAT_GENERIC, + /* .reasoning_format = */ COMMON_REASONING_FORMAT_AUTO, + /* .reasoning_in_content = */ false, + /* .thinking_forced_open = */ false, + }); + + builder.try_parse_reasoning("<|START_THINKING|>", "<|END_THINKING|>"); + + static const common_regex start_action_regex("<\\|START_ACTION\\|>"); + static const common_regex end_action_regex("<\\|END_ACTION\\|>"); + static const common_regex start_response_regex("<\\|START_RESPONSE\\|>"); + static const common_regex end_response_regex("<\\|END_RESPONSE\\|>"); + + if (auto res = builder.try_find_regex(start_action_regex)) { + // If we didn't extract thoughts, prelude includes them. + auto tool_calls = builder.consume_json_with_dumped_args({ { "parameters" } }); + for (const auto & tool_call : tool_calls.value) { + std::string name = tool_call.contains("tool_name") ? tool_call.at("tool_name") : ""; + std::string id = tool_call.contains("tool_call_id") ? tool_call.at("tool_call_id") : ""; + std::string arguments = tool_call.contains("parameters") ? tool_call.at("parameters") : ""; + if (!builder.add_tool_call(name, id, arguments) || tool_calls.is_partial) { + throw common_chat_msg_partial_exception("incomplete tool call"); + } + } + if (tool_calls.is_partial) { + throw common_chat_msg_partial_exception("incomplete tool call"); + } + builder.consume_regex(end_action_regex); + } else if (auto res = builder.try_find_regex(start_response_regex)) { + if (!builder.try_find_regex(end_response_regex)) { + builder.add_content(builder.consume_rest()); + throw common_chat_msg_partial_exception(end_response_regex.str()); + } + } else { + builder.add_content(builder.consume_rest()); + } + + if (print_results) { + std::cout << "== Parsed (legacy) ==\n"; + std::cout << "=== Reasoning ===\n"; + std::cout << builder.result().reasoning_content << "\n"; + std::cout << "\n\n=== Content ===\n"; + std::cout << builder.result().content << "\n"; + std::cout << "\n\n=== Tool Calls ===\n"; + for (const auto & tc : builder.result().tool_calls) { + std::cout << "id: " << tc.id << "\n"; + std::cout << "name: " << tc.name << "\n"; + std::cout << "args: " << tc.arguments << "\n"; + } + } +} diff --git a/tests/combinator/test-complete-example.cpp b/tests/combinator/test-complete-example.cpp index 8a09bc70cb50b..b1965de7c3991 100644 --- a/tests/combinator/test-complete-example.cpp +++ b/tests/combinator/test-complete-example.cpp @@ -1,145 +1,151 @@ #include "json-schema-to-grammar.h" #include "tests.h" + #include -class test_complete_example : public compound_test { - public: - test_complete_example() : compound_test("test_complete_example") { - /* Parser for a fictitious model that outputs: +test_complete_example::test_complete_example() : compound_test("test_complete_example") { + /* Parser for a fictitious model that outputs: * - * + * * ... reasoning content ... - * + * * ... content ... * * tool_name * { ... json args ... } * */ - auto parser = build_parser([](parser_builder & p) { - auto reasoning = - p.add_rule("reasoning", "" << p.append_reasoning(p.until("")) << ""); - - auto content = p.add_rule("content", p.append_content(p.until(""))); - - auto json = p.json(); - - auto tool_call_name = - p.add_rule("tool-call-name", "" << p.capture_tool_call_name(p.until("")) << ""); - - auto schema = nlohmann::json::parse(R"({"type": "object"})"); - - auto tool_call_args = p.add_rule( - "tool-call-args", - "" << p.capture_tool_call_args(p.schema(p.succeed(json), "get_weather", schema)) << ""); - - auto tool_call = - p.add_rule("tool-call", "" << p.add_tool_call(tool_call_name << p.succeed(tool_call_args)) - << ""); - - return reasoning << p.optional(content) << p.optional(tool_call); - }); - - // Test complete input - std::string input = - std::string(R"(I need to call get_weather with city = New Yorkget_weather{"city": "New York"})"); - parser_environment env; - parser_context ctx(input, &env); - - auto result = parser.parse(ctx); - - // Test complete input with reasoning and tool call - add_test( - [env, input, result](test_harness h) { - h.assert_equals("parse_success", true, result.is_success()); - h.assert_equals("parse_end", (size_t) input.size(), result.end); - h.assert_equals("reasoning_content", std::string("I need to call get_weather with city = New York"), - env.result.reasoning_content); - h.assert_equals("tool_calls_size", (size_t) 1, env.result.tool_calls.size()); - h.assert_equals("tool_call_id", std::string(""), env.result.tool_calls[0].id); - h.assert_equals("tool_call_name", std::string("get_weather"), env.result.tool_calls[0].name); - h.assert_equals("tool_call_args", std::string(R"({"city": "New York"})"), - env.result.tool_calls[0].arguments); - }, - "complete_tool_call_parsing"); - - // Test partial input - add_test([parser](test_harness h) { - std::string input = R"(I need to call get_weather)"; - parser_environment env = parser_environment(); - parser_context ctx = parser_context(input, &env, /* .is_input_complete = */ false); + parser = build_parser([](parser_builder & p) { + auto reasoning = p.add_rule("reasoning", "" << p.append_reasoning(p.until("")) << ""); + + auto content = p.add_rule("content", p.append_content(p.until(""))); + + auto json = p.json(); + + auto tool_call_name = + p.add_rule("tool-call-name", "" << p.capture_tool_call_name(p.until("")) << ""); + + auto schema = nlohmann::json::parse(R"({"type": "object"})"); + + auto tool_call_args = p.add_rule( + "tool-call-args", + "" << p.capture_tool_call_args(p.schema(p.succeed(json), "get_weather", schema)) << ""); + + auto tool_call = + p.add_rule("tool-call", + "" << p.add_tool_call(tool_call_name << p.succeed(tool_call_args)) << ""); + + return reasoning << p.optional(content) << p.optional(tool_call); + }); + + // Test complete input with reasoning and tool call + add_test( + [this](test_harness h) { + // Test complete input + std::string input = std::string( + R"(I need to call get_weather with city = New Yorkget_weather{"city": "New York"})"); + parser_environment env; + parser_context ctx(input, &env); auto result = parser.parse(ctx); - h.assert_equals("needs_more_input", true, result.is_need_more_input()); - h.assert_equals("reasoning_content", std::string("I need to call get_weather"), env.result.reasoning_content); - }, "partial_input"); + h.assert_equals("parse_success", true, result.is_success()); + h.assert_equals("parse_end", (size_t) input.size(), result.end); + h.assert_equals("reasoning_content", std::string("I need to call get_weather with city = New York"), + env.result.reasoning_content); + h.assert_equals("tool_calls_size", (size_t) 1, env.result.tool_calls.size()); + h.assert_equals("tool_call_id", std::string(""), env.result.tool_calls[0].id); + h.assert_equals("tool_call_name", std::string("get_weather"), env.result.tool_calls[0].name); + h.assert_equals("tool_call_args", std::string(R"({"city": "New York"})"), + env.result.tool_calls[0].arguments); + }, + "complete_tool_call_parsing"); + + // Test partial input + add_test( + [this](test_harness h) { + std::string input = R"(I need to call get_weather)"; + parser_environment env; + parser_context ctx(input, &env, /* .is_input_complete = */ false); + auto result = parser.parse(ctx); - add_test([parser](test_harness h) { - std::string input = R"(I need to call I need to call I need to call get_weatherI need to call get_weatherI need to call get_weatherget_weather)"; - parser_environment env = parser_environment(); - parser_context ctx = parser_context(input, &env, /* .is_input_complete = */ false); + }, + "input_incomplete_2"); + add_test( + [this](test_harness h) { + std::string input = R"(I need to call get_weatherget_weather)"; + parser_environment env; + parser_context ctx(input, &env, /* .is_input_complete = */ false); auto result = parser.parse(ctx); h.assert_equals("needs_more_input", true, result.is_need_more_input()); - h.assert_equals("reasoning_content", std::string("I need to call get_weather"), env.result.reasoning_content); - }, "tool_call_incomplete"); - add_test([parser](test_harness h) { - std::string input = R"(I need to call get_weatherget_weatherI need to call get_weatherget_weatherI need to call get_weatherget_weather{"cit)"; - parser_environment env = parser_environment(); - parser_context ctx = parser_context(input, &env, /* .is_input_complete = */ false); + h.assert_equals("reasoning_content", std::string("I need to call get_weather"), + env.result.reasoning_content); + }, + "tool_call_incomplete_2"); + add_test( + [this](test_harness h) { + std::string input = + R"(I need to call get_weatherget_weather{"cit)"; + parser_environment env; + parser_context ctx(input, &env, /* .is_input_complete = */ false); auto result = parser.parse(ctx); h.assert_equals("needs_more_input", true, result.is_need_more_input()); - h.assert_equals("reasoning_content", std::string("I need to call get_weather"), env.result.reasoning_content); + h.assert_equals("reasoning_content", std::string("I need to call get_weather"), + env.result.reasoning_content); h.assert_equals("tool_name", std::string("get_weather"), env.result.tool_calls[0].name); h.assert_equals("tool_incomplete_arg", std::string(R"({"cit)"), env.result.tool_calls[0].arguments); - }, "tool_call_arg_incomplete"); - - auto gbnf = build_grammar([parser](const common_grammar_builder & builder) { - parser.build_grammar(builder); - }); - - add_test([gbnf](test_harness h) { + }, + "tool_call_arg_incomplete"); + add_test( + [this](test_harness h) { + auto gbnf = + build_grammar([this](const common_grammar_builder & builder) { parser.build_grammar(builder); }); h.assert_equals("not_empty", false, gbnf.empty()); - }, "grammar_is_there"); - } - - // Provide a convenient way to run all tests - void run_all_tests() { - run_all(); - summary(); - } -}; + }, + "grammar_is_there"); +} diff --git a/tests/combinator/test-example-qwen3-coder.cpp b/tests/combinator/test-example-qwen3-coder.cpp new file mode 100644 index 0000000000000..dafea8cffdcdb --- /dev/null +++ b/tests/combinator/test-example-qwen3-coder.cpp @@ -0,0 +1,162 @@ +#include "tests.h" + +#include +#include + +test_example_qwen3_coder::test_example_qwen3_coder() : compound_test("sample_qwen3_coder_test") { + parser = build_parser([](parser_builder & p) { + auto thinking = p.add_rule("thinking", "" << p.append_reasoning(p.until("")) << ""); + + auto content = p.add_rule("content", p.append_content(p.until(""))); + + auto arg_start = p.add_rule("arg-start", p.action("", [](const parser_action & act) { act.env.tool_call_args += "\":"; })); + + auto arg_end = p.add_rule("arg-end", ""); + + auto string_arg = p.add_rule("arg-string", p.action(arg_start, [&](const parser_action & act) { + act.env.tool_call_args += "\""; + }) << p.action(p.until(""), [&](const parser_action & act) { + // TODO: add a JSON escape helper + act.env.tool_call_args += std::string(act.match); + }) << p.action(arg_end, [&](const parser_action & act) { act.env.tool_call_args += "\""; })); + + auto json = p.json(); + + auto json_arg = p.add_rule("arg-json", arg_start << p.action(json, [&](const parser_action & act) { + // JSON should already be properly formatted + act.env.tool_call_args += std::string(act.match); + + // This can be streamed by passing p.success(json), but we have + // to be mindful of the potential backtracking--it only works + // if we only keep the last value... + }) << arg_end); + + auto function = p.add_rule( + "function", + p.add_tool_call( + "", [&](const parser_action & act) { act.env.tool_call_args += "{"; }) + + p.one_or_more(p.space() + (json_arg | string_arg)) + << p.action("", [&](const parser_action & act) { act.env.tool_call_args += "}"; }))); + + auto tool_call = p.add_rule("tool-call", "" << p.one_or_more(function) << ""); + + return thinking + p.optional(p.space() + content) + p.zero_or_more(p.space() + tool_call); + }); + + add_test([this](test_harness h) { + std::string input = + "The user wants to find large log files that haven't been accessed recently. " + "I should search for files with .log extension, filter by size (over 100MB), " + "and check access time within the last 30 days. I'll need to use the search_files function." + "Based on your requirements, I'll search for log files over 100MB that haven't been " + "accessed in the last month. This will help identify candidates for cleanup or archival.\n\n" + "\n" + "\n" + "/var/log\n" + "*.log\n" + "100\n" + "5\n" + "false\n" + "30\n" + "true\n" + "size\n" + "{\"exclude_patterns\": [\"*temp*\", \"*cache*\"], \"file_types\": " + "[\"regular\"]}\n" + "\n" + ""; + + std::vector tokens = simple_tokenize(input); + + common_chat_msg prev; + int token_cnt = 0; + for (auto it = tokens.begin(); it != tokens.end(); it++) { + token_cnt++; + std::string in = std::accumulate(tokens.begin(), it, std::string()); + + parser_environment env; + parser_context ctx(in, &env, it == tokens.end() - 1); + + auto result = parser.parse(ctx); + h.assert_equals(std::string("should_not_fail_token_") + std::to_string(token_cnt), false, result.is_fail()); + /* + std::cout << "Input:\n" << in << "\n\n"; + std::cout << "Reasoning: " << prev.reasoning_content << "\n"; + std::cout << "Content : " << prev.content << "\n"; + if (!prev.tool_calls.empty()) { + std::cout << "\n=== Tool Calls ===\n"; + for (const auto & tc : prev.tool_calls) { + std::cout << "ID : " << tc.id << "\n"; + std::cout << "Name: " << tc.name << "\n"; + std::cout << "Args: " << tc.arguments << "\n"; + } + } + */ + + // This shouldn't emit any runtime errors + auto diffs = common_chat_msg_diff::compute_diffs(prev, env.result); + prev = env.result; + + /* + std::cout << "----\n"; + std::cout << "Reasoning: " << prev.reasoning_content << "\n"; + std::cout << "Content : " << prev.content << "\n"; + if (!prev.tool_calls.empty()) { + std::cout << "\n=== Tool Calls ===\n"; + for (const auto & tc : prev.tool_calls) { + std::cout << "ID : " << tc.id << "\n"; + std::cout << "Name: " << tc.name << "\n"; + std::cout << "Args: " << tc.arguments << "\n"; + } + } + std::cout << "======================\n"; + */ + + /* + std::cout << "=== Diffs ===\n\n"; + if (!diffs.empty()) { + for (size_t i = 0; i < diffs.size(); ++i) { + const auto& diff = diffs[i]; + + std::cout << "Diff #" << (i + 1) << "\n"; + + if (!diff.reasoning_content_delta.empty()) { + std::cout << " [Reasoning Content]: " << diff.reasoning_content_delta << "\n"; + } + + if (!diff.content_delta.empty()) { + std::cout << " [Content]: " << diff.content_delta << "\n"; + } + + if (diff.tool_call_index != std::string::npos) { + std::cout << " [Tool Call #" << diff.tool_call_index << "]" << "\n"; + + if (!diff.tool_call_delta.id.empty()) { + std::cout << " ID: " << diff.tool_call_delta.id << "\n"; + } + + if (!diff.tool_call_delta.name.empty()) { + std::cout << " Name: " << diff.tool_call_delta.name << "\n"; + } + + if (!diff.tool_call_delta.arguments.empty()) { + std::cout << " Arguments: " << diff.tool_call_delta.arguments << "\n"; + } + } + + std::cout << "\n"; + } + } else { + std::cout << "No changes detected.\n"; + } + */ + } + }, "accumulation_test"); +} diff --git a/tests/combinator/test-gbnf-generation.cpp b/tests/combinator/test-gbnf-generation.cpp index 685e4374db689..fa2cc93a1fc0c 100644 --- a/tests/combinator/test-gbnf-generation.cpp +++ b/tests/combinator/test-gbnf-generation.cpp @@ -1,176 +1,154 @@ -#include "tests.h" #include "json-schema-to-grammar.h" +#include "tests.h" -class test_gbnf_generation : public compound_test { -public: - test_gbnf_generation() : compound_test("test_gbnf_generation") { - // Test literal - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.literal("hello"); - }); +test_gbnf_generation::test_gbnf_generation() : compound_test("test_gbnf_generation") { + // Test literal + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.literal("hello"); }); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder); - }); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); h.assert_equals("has_root_hello", true, gbnf.find("root ::= \"hello\"") != std::string::npos); h.assert_equals("has_space", true, gbnf.find("space ::=") != std::string::npos); - }, "literal grammar generation"); - - // Test char class - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.one("[a-z]"); - }); + }, + "literal grammar generation"); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder); - }); + // Test char class + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.one("[a-z]"); }); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); h.assert_equals("has_char_class", true, gbnf.find("root ::= [a-z]") != std::string::npos); - }, "char class grammar"); - - // Test sequence - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.literal("hello") + p.literal(" ") + p.literal("world"); - }); + }, + "char class grammar"); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder); - }); + // Test sequence + add_test( + [](test_harness h) { + auto parser = build_parser( + [](parser_builder & p) { return p.literal("hello") + p.literal(" ") + p.literal("world"); }); - h.assert_equals("has_proper_sequence", true, gbnf.find("root ::= \"hello\" \" \" \"world\"") != std::string::npos); - }, "sequence grammar"); - - // Test choice - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.literal("cat") | p.literal("dog"); - }); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder); - }); + h.assert_equals("has_proper_sequence", true, + gbnf.find("root ::= \"hello\" \" \" \"world\"") != std::string::npos); + }, + "sequence grammar"); + + // Test choice + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.literal("cat") | p.literal("dog"); }); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); h.assert_equals("has_proper_choice", true, gbnf.find("root ::= \"cat\" | \"dog\"") != std::string::npos); - }, "choice grammar"); - - // Test one_or_more - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.one_or_more(p.one("[0-9]")); - }); + }, + "choice grammar"); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder); - }); + // Test one_or_more + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.one_or_more(p.one("[0-9]")); }); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); h.assert_equals("has_proper_one_or_more", true, gbnf.find("root ::= [0-9]+") != std::string::npos); - }, "one_or_more grammar"); - - // Test zero_or_more - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.zero_or_more(p.one("[a-z]")); - }); + }, + "one_or_more grammar"); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder); - }); + // Test zero_or_more + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.zero_or_more(p.one("[a-z]")); }); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); h.assert_equals("has_proper_zero_or_more", true, gbnf.find("root ::= [a-z]*") != std::string::npos); - }, "zero_or_more grammar"); - - // Test optional - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.literal("hello") + p.optional(p.literal(" world")); - }); + }, + "zero_or_more grammar"); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder); - }); + // Test optional + add_test( + [](test_harness h) { + auto parser = + build_parser([](parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); }); - h.assert_equals("has_proper_optional", true, gbnf.find("root ::= \"hello\" \" world\"?") != std::string::npos); - }, "optional grammar"); - - // Test until - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.until(""); - }); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder); - }); + h.assert_equals("has_proper_optional", true, + gbnf.find("root ::= \"hello\" \" world\"?") != std::string::npos); + }, + "optional grammar"); - // Should generate pattern that prevents matching the full delimiter - h.assert_equals("has_proper_until", true, gbnf.find("root ::= ([^<] | \"<\" [^/] | \"])*") != std::string::npos); - }, "until grammar"); - - // Test complex expression with parentheses - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.one_or_more(p.literal("a") | p.literal("b")); - }); + // Test until + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.until(""); }); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder); - }); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); + + // Should generate pattern that prevents matching the full delimiter + h.assert_equals( + "has_proper_until", true, + gbnf.find( + "root ::= ([^<] | \"<\" [^/] | \"])*") != + std::string::npos); + }, + "until grammar"); + + // Test complex expression with parentheses + add_test( + [](test_harness h) { + auto parser = + build_parser([](parser_builder & p) { return p.one_or_more(p.literal("a") | p.literal("b")); }); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); h.assert_equals("has_proper_complex", true, gbnf.find("root ::= (\"a\" | \"b\")+") != std::string::npos); - }, "complex expressions with parentheses"); - - // Test rule references - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { + }, + "complex expressions with parentheses"); + + // Test rule references + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { auto digit = p.add_rule("digit", p.one("[0-9]")); return p.one_or_more(digit); }); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder); - }); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); // Should have digit rule defined and referenced h.assert_equals("has_digit_rule", true, gbnf.find("digit ::= [0-9]") != std::string::npos); h.assert_equals("has_root_digit_ref", true, gbnf.find("root ::= digit+") != std::string::npos); - }, "rule references"); - - // Test escaping in literals - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.literal("hello\nworld\t!"); - }); + }, + "rule references"); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder); - }); + // Test escaping in literals + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.literal("hello\nworld\t!"); }); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); h.assert_equals("has_escaping", true, gbnf.find("root ::= \"hello\\nworld\\t!\"") != std::string::npos); - }, "escaping in literals"); - - // Test operator<< (whitespace insertion) - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.literal("hello") << p.literal("world"); - }); + }, + "escaping in literals"); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder); - }); + // Test operator<< (whitespace insertion) + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.literal("hello") << p.literal("world"); }); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); // Should inline the whitespace pattern h.assert_equals("has_inlined_hello", true, gbnf.find("\"hello\"") != std::string::npos); h.assert_equals("has_inlined_world", true, gbnf.find("\"world\"") != std::string::npos); - }, "operator<< (whitespace insertion)"); - } - - // Provide a convenient way to run all tests - void run_all_tests() { - run_all(); - summary(); - } -}; \ No newline at end of file + }, + "operator<< (whitespace insertion)"); +} diff --git a/tests/combinator/test-json-parser.cpp b/tests/combinator/test-json-parser.cpp index ed238270bf3d2..5f47f5d314b70 100644 --- a/tests/combinator/test-json-parser.cpp +++ b/tests/combinator/test-json-parser.cpp @@ -1,99 +1,91 @@ #include "tests.h" -class test_json_parser : public compound_test { -public: - test_json_parser() : compound_test("test_json_parser") { - // Test parsing a simple JSON object - add_test([](test_harness h) { - auto json = build_parser([](parser_builder & p) { - return p.json(); - }); - - std::string input = R"({"name": "test", "value": 42, "flag": true})"; +test_json_parser::test_json_parser() : compound_test("test_json_parser") { + // Test parsing a simple JSON object + add_test( + [](test_harness h) { + auto json = build_parser([](parser_builder & p) { return p.json(); }); + + std::string input = R"({"name": "test", "value": 42, "flag": true})"; parser_context ctx(input); auto result = json.parse(ctx); h.assert_equals("result_is_success", true, result.is_success()); h.assert_equals("result_end", input.size(), result.end); - }, "simple JSON object parsing"); - - // Test parsing a JSON array with mixed types - add_test([](test_harness h) { - auto json = build_parser([](parser_builder & p) { - return p.json(); - }); - - std::string input = R"([1, "hello", true, null, 3.14])"; + }, + "simple JSON object parsing"); + + // Test parsing a JSON array with mixed types + add_test( + [](test_harness h) { + auto json = build_parser([](parser_builder & p) { return p.json(); }); + + std::string input = R"([1, "hello", true, null, 3.14])"; parser_context ctx(input); auto result = json.parse(ctx); h.assert_equals("result_is_success", true, result.is_success()); h.assert_equals("result_end", input.size(), result.end); - }, "JSON array with mixed types"); - - // Test parsing nested JSON with objects and arrays - add_test([](test_harness h) { - auto json = build_parser([](parser_builder & p) { - return p.json(); - }); - - std::string input = R"({"users": [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}], "count": 2, "metadata": {"version": "1.0", "tags": ["admin", "user"]}})"; + }, + "JSON array with mixed types"); + + // Test parsing nested JSON with objects and arrays + add_test( + [](test_harness h) { + auto json = build_parser([](parser_builder & p) { return p.json(); }); + + std::string input = + R"({"users": [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}], "count": 2, "metadata": {"version": "1.0", "tags": ["admin", "user"]}})"; parser_context ctx(input); auto result = json.parse(ctx); h.assert_equals("result_is_success", true, result.is_success()); h.assert_equals("result_end", input.size(), result.end); - }, "nested JSON with objects and arrays"); - - // Test partial parsing - incomplete object - add_test([](test_harness h) { - auto json = build_parser([](parser_builder & p) { - return p.json(); - }); - - std::string input = R"({"name": "test", "value": )"; + }, + "nested JSON with objects and arrays"); + + // Test partial parsing - incomplete object + add_test( + [](test_harness h) { + auto json = build_parser([](parser_builder & p) { return p.json(); }); + + std::string input = R"({"name": "test", "value": )"; parser_context ctx(input, false); auto result = json.parse(ctx); h.assert_equals("result_is_need_more_input", true, result.is_need_more_input()); - }, "partial parsing - incomplete object"); - - // Test partial parsing - incomplete array - add_test([](test_harness h) { - auto json = build_parser([](parser_builder & p) { - return p.json(); - }); - - std::string input = R"([1, 2, 3, )"; + }, + "partial parsing - incomplete object"); + + // Test partial parsing - incomplete array + add_test( + [](test_harness h) { + auto json = build_parser([](parser_builder & p) { return p.json(); }); + + std::string input = R"([1, 2, 3, )"; parser_context ctx(input, false); auto result = json.parse(ctx); h.assert_equals("result_is_need_more_input", true, result.is_need_more_input()); - }, "partial parsing - incomplete array"); - - // Test partial parsing - incomplete nested structure - add_test([](test_harness h) { - auto json = build_parser([](parser_builder & p) { - return p.json(); - }); - - std::string input = R"({"data": {"nested": )"; + }, + "partial parsing - incomplete array"); + + // Test partial parsing - incomplete nested structure + add_test( + [](test_harness h) { + auto json = build_parser([](parser_builder & p) { return p.json(); }); + + std::string input = R"({"data": {"nested": )"; parser_context ctx(input, false); auto result = json.parse(ctx); h.assert_equals("result_is_need_more_input", true, result.is_need_more_input()); - }, "partial parsing - incomplete nested structure"); - } - - // Provide a convenient way to run all tests - void run_all_tests() { - run_all(); - summary(); - } -}; \ No newline at end of file + }, + "partial parsing - incomplete nested structure"); +} diff --git a/tests/combinator/test-one.cpp b/tests/combinator/test-one.cpp index a6f74c6cbfa3a..7dd44f1529642 100644 --- a/tests/combinator/test-one.cpp +++ b/tests/combinator/test-one.cpp @@ -1,124 +1,115 @@ #include "tests.h" -class test_one : public compound_test { -public: - test_one() : compound_test("test_one") { - // Test common escape sequences - newline - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.one("[\\n\\t\\\\]"); - }); - +test_one::test_one() : compound_test("test_one") { + // Test common escape sequences - newline + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); + parser_context ctx; - parser_result result; - - ctx = parser_context("\n"); + parser_result result; + + ctx = parser_context("\n"); result = parser.parse(ctx); h.assert_equals("escape_sequence_newline", true, result.is_success()); - }, "escape_sequence_newline"); - - // Test common escape sequences - tab - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.one("[\\n\\t\\\\]"); - }); - + }, + "escape_sequence_newline"); + + // Test common escape sequences - tab + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); + parser_context ctx; - parser_result result; - - ctx = parser_context("\t"); + parser_result result; + + ctx = parser_context("\t"); result = parser.parse(ctx); h.assert_equals("escape_sequence_tab", true, result.is_success()); - }, "escape_sequence_tab"); - - // Test common escape sequences - backslash - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.one("[\\n\\t\\\\]"); - }); - + }, + "escape_sequence_tab"); + + // Test common escape sequences - backslash + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); + parser_context ctx; - parser_result result; - - ctx = parser_context("\\"); + parser_result result; + + ctx = parser_context("\\"); result = parser.parse(ctx); h.assert_equals("escape_sequence_backslash", true, result.is_success()); - }, "escape_sequence_backslash"); - - // Test common escape sequences - space (should fail) - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.one("[\\n\\t\\\\]"); - }); - + }, + "escape_sequence_backslash"); + + // Test common escape sequences - space (should fail) + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); + parser_context ctx; - parser_result result; - - ctx = parser_context(" "); + parser_result result; + + ctx = parser_context(" "); result = parser.parse(ctx); h.assert_equals("escape_sequence_space_fail", true, result.is_fail()); - }, "escape_sequence_space_fail"); - - // Test escaped dash - 'a' should succeed - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.one("[a\\-z]"); - }); - + }, + "escape_sequence_space_fail"); + + // Test escaped dash - 'a' should succeed + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.one("[a\\-z]"); }); + parser_context ctx; - parser_result result; - - ctx = parser_context("a"); + parser_result result; + + ctx = parser_context("a"); result = parser.parse(ctx); h.assert_equals("escaped_dash_a", true, result.is_success()); - }, "escaped_dash_a"); - - // Test escaped dash - '-' should succeed (literal dash) - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.one("[a\\-z]"); - }); - + }, + "escaped_dash_a"); + + // Test escaped dash - '-' should succeed (literal dash) + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.one("[a\\-z]"); }); + parser_context ctx; - parser_result result; - - ctx = parser_context("-"); + parser_result result; + + ctx = parser_context("-"); result = parser.parse(ctx); h.assert_equals("escaped_dash_literal", true, result.is_success()); - }, "escaped_dash_literal"); - - // Test escaped dash - 'z' should succeed - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.one("[a\\-z]"); - }); - + }, + "escaped_dash_literal"); + + // Test escaped dash - 'z' should succeed + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.one("[a\\-z]"); }); + parser_context ctx; - parser_result result; - - ctx = parser_context("z"); + parser_result result; + + ctx = parser_context("z"); result = parser.parse(ctx); h.assert_equals("escaped_dash_z", true, result.is_success()); - }, "escaped_dash_z"); - - // Test escaped dash - 'b' should NOT match (since \- is literal dash, not range) - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.one("[a\\-z]"); - }); - + }, + "escaped_dash_z"); + + // Test escaped dash - 'b' should NOT match (since \- is literal dash, not range) + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.one("[a\\-z]"); }); + parser_context ctx; - parser_result result; - - ctx = parser_context("b"); + parser_result result; + + ctx = parser_context("b"); result = parser.parse(ctx); h.assert_equals("escaped_dash_b_fail", true, result.is_fail()); - }, "escaped_dash_b_fail"); - } - - // Provide a convenient way to run all tests - void run_all_tests() { - run_all(); - summary(); - } -}; \ No newline at end of file + }, + "escaped_dash_b_fail"); +} diff --git a/tests/combinator/test-optional.cpp b/tests/combinator/test-optional.cpp index 16a3c66f81e2f..e99e637624653 100644 --- a/tests/combinator/test-optional.cpp +++ b/tests/combinator/test-optional.cpp @@ -1,49 +1,43 @@ #include "tests.h" -class test_optional : public compound_test { -public: - test_optional() : compound_test("test_optional") { - // Full match with optional part present - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.literal("hello") + p.optional(p.literal(" world")); - }); - - auto ctx = parser_context("hello world"); +test_optional::test_optional() : compound_test("test_optional") { + // Full match with optional part present + add_test( + [](test_harness h) { + auto parser = + build_parser([](parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); }); + + auto ctx = parser_context("hello world"); auto result = parser.parse(ctx); h.assert_equals("optional_present", true, result.is_success()); int end_pos = result.end; h.assert_equals("optional_present_end", 11, end_pos); - }, "optional_present"); - - // Full match with optional part absent - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.literal("hello") + p.optional(p.literal(" world")); - }); - - auto ctx = parser_context("hello", true); + }, + "optional_present"); + + // Full match with optional part absent + add_test( + [](test_harness h) { + auto parser = + build_parser([](parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); }); + + auto ctx = parser_context("hello", true); auto result = parser.parse(ctx); h.assert_equals("optional_absent", true, result.is_success()); int end_pos = result.end; h.assert_equals("optional_absent_end", 5, end_pos); - }, "optional_absent"); - - // Partial match - waiting for more input to determine if optional matches - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.literal("hello") + p.optional(p.literal(" world")); - }); - - auto ctx = parser_context("hello ", false); + }, + "optional_absent"); + + // Partial match - waiting for more input to determine if optional matches + add_test( + [](test_harness h) { + auto parser = + build_parser([](parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); }); + + auto ctx = parser_context("hello ", false); auto result = parser.parse(ctx); h.assert_equals("partial_match_need_more", true, result.is_need_more_input()); - }, "partial_match_need_more"); - } - - // Provide a convenient way to run all tests - void run_all_tests() { - run_all(); - summary(); - } -}; \ No newline at end of file + }, + "partial_match_need_more"); +} diff --git a/tests/combinator/test-partial-parsing.cpp b/tests/combinator/test-partial-parsing.cpp index 8045ee9cd0475..0d66bf3af4615 100644 --- a/tests/combinator/test-partial-parsing.cpp +++ b/tests/combinator/test-partial-parsing.cpp @@ -1,283 +1,275 @@ #include "tests.h" -class test_partial_parsing : public compound_test { -public: - test_partial_parsing() : compound_test("test_partial_parsing") { - // Literals - Basic Success - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.literal("hello"); - }); +test_partial_parsing::test_partial_parsing() : compound_test("test_partial_parsing") { + // Literals - Basic Success + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.literal("hello"); }); parser_context ctx; - parser_result result; + parser_result result; - ctx = parser_context("hello"); + ctx = parser_context("hello"); result = parser.parse(ctx); h.assert_equals("literal_success", true, result.is_success()); - }, "literal_success"); + }, + "literal_success"); - // Char Classes - Basic Lowercase Success - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.one("a-z"); - }); + // Char Classes - Basic Lowercase Success + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.one("a-z"); }); parser_context ctx; - parser_result result; + parser_result result; - ctx = parser_context("a"); + ctx = parser_context("a"); result = parser.parse(ctx); h.assert_equals("char_class_lowercase_success", true, result.is_success()); - }, "char_class_lowercase_success"); + }, + "char_class_lowercase_success"); - // Char Classes - Uppercase Fail - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.one("a-z"); - }); + // Char Classes - Uppercase Fail + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.one("a-z"); }); parser_context ctx; - parser_result result; + parser_result result; - ctx = parser_context("A"); + ctx = parser_context("A"); result = parser.parse(ctx); h.assert_equals("char_class_uppercase_fail", true, result.is_fail()); - }, "char_class_uppercase_fail"); + }, + "char_class_uppercase_fail"); - // Char Classes with Dash - Lowercase Success - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.one("a-z-"); - }); + // Char Classes with Dash - Lowercase Success + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.one("a-z-"); }); parser_context ctx; - parser_result result; + parser_result result; - ctx = parser_context("f"); + ctx = parser_context("f"); result = parser.parse(ctx); h.assert_equals("char_class_with_dash_lowercase", true, result.is_success()); - }, "char_class_with_dash_lowercase"); + }, + "char_class_with_dash_lowercase"); - // Char Classes with Dash - Literal Dash Success - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.one("a-z-"); - }); + // Char Classes with Dash - Literal Dash Success + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.one("a-z-"); }); parser_context ctx; - parser_result result; + parser_result result; - ctx = parser_context("-"); + ctx = parser_context("-"); result = parser.parse(ctx); h.assert_equals("char_class_with_dash_literal_dash", true, result.is_success()); - }, "char_class_with_dash_literal_dash"); + }, + "char_class_with_dash_literal_dash"); - // Char Classes with Dash - Uppercase Fail - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.one("a-z-"); - }); + // Char Classes with Dash - Uppercase Fail + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.one("a-z-"); }); parser_context ctx; - parser_result result; + parser_result result; - ctx = parser_context("A"); + ctx = parser_context("A"); result = parser.parse(ctx); h.assert_equals("char_class_with_dash_uppercase_fail", true, result.is_fail()); - }, "char_class_with_dash_uppercase_fail"); + }, + "char_class_with_dash_uppercase_fail"); - // Sequences - Partial Match 1 - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.literal("") + p.literal(""); }); - auto ctx = parser_context("thi", false); + auto ctx = parser_context("") + p.literal(""); }); - auto ctx = parser_context("start/end", false); + auto ctx = parser_context("") + p.literal(""); }); - auto ctx = parser_context("foobar", false); + auto ctx = parser_context("I am parser", false); auto result = parser.parse(ctx); h.assert_equals("sequence_no_match", true, result.is_fail()); - }, "sequence_no_match"); + }, + "sequence_no_match"); - // Choices - Partial Match 1 - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.literal("option1") | p.literal("option2"); - }); + // Choices - Partial Match 1 + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.literal("option1") | p.literal("option2"); }); - auto ctx = parser_context("opt", false); + auto ctx = parser_context("opt", false); auto result = parser.parse(ctx); h.assert_equals("choices_partial_match_1", true, result.is_need_more_input()); - }, "choices_partial_match_1"); + }, + "choices_partial_match_1"); - // Choices - Partial Match 2 - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.literal("choice_a") | p.literal("choice_b"); - }); + // Choices - Partial Match 2 + add_test( + [](test_harness h) { + auto parser = + build_parser([](parser_builder & p) { return p.literal("choice_a") | p.literal("choice_b"); }); - auto ctx = parser_context("choice", false); + auto ctx = parser_context("choice", false); auto result = parser.parse(ctx); h.assert_equals("choices_partial_match_2", true, result.is_need_more_input()); - }, "choices_partial_match_2"); + }, + "choices_partial_match_2"); - // Choices - Full Match 1 - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.literal("first") | p.literal("second"); - }); + // Choices - Full Match 1 + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.literal("first") | p.literal("second"); }); - auto ctx = parser_context("first", true); + auto ctx = parser_context("first", true); auto result = parser.parse(ctx); h.assert_equals("choices_full_match_1", true, result.is_success()); - }, "choices_full_match_1"); + }, + "choices_full_match_1"); - // Choices - Full Match 2 - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.literal("alpha") | p.literal("beta"); - }); + // Choices - Full Match 2 + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.literal("alpha") | p.literal("beta"); }); - auto ctx = parser_context("beta", true); + auto ctx = parser_context("beta", true); auto result = parser.parse(ctx); h.assert_equals("choices_full_match_2", true, result.is_success()); - }, "choices_full_match_2"); + }, + "choices_full_match_2"); - // Choices - No Match - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.literal("good") | p.literal("better"); - }); + // Choices - No Match + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.literal("good") | p.literal("better"); }); - auto ctx = parser_context("best", true); + auto ctx = parser_context("best", true); auto result = parser.parse(ctx); h.assert_equals("choices_no_match", true, result.is_fail()); - }, "choices_no_match"); + }, + "choices_no_match"); - // Zero or More - Partial Match 1 - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.zero_or_more(p.literal("ab")); - }); + // Zero or More - Partial Match 1 + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.zero_or_more(p.literal("ab")); }); - auto ctx = parser_context("a", false); + auto ctx = parser_context("a", false); auto result = parser.parse(ctx); h.assert_equals("zero_or_more_partial_match_1", true, result.is_need_more_input()); - }, "zero_or_more_partial_match_1"); + }, + "zero_or_more_partial_match_1"); - // Zero or More - Partial Match 2 - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.zero_or_more(p.literal("xy")); - }); + // Zero or More - Partial Match 2 + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.zero_or_more(p.literal("xy")); }); - auto ctx = parser_context("xyx", false); + auto ctx = parser_context("xyx", false); auto result = parser.parse(ctx); h.assert_equals("zero_or_more_partial_match_2", true, result.is_need_more_input()); - }, "zero_or_more_partial_match_2"); + }, + "zero_or_more_partial_match_2"); - // Zero or More - Full Match - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.zero_or_more(p.literal("test")); - }); + // Zero or More - Full Match + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.zero_or_more(p.literal("test")); }); - auto ctx = parser_context("test", true); + auto ctx = parser_context("test", true); auto result = parser.parse(ctx); h.assert_equals("zero_or_more_full_match", true, result.is_success()); - }, "zero_or_more_full_match"); + }, + "zero_or_more_full_match"); - // One or More - Partial Match 1 - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.one_or_more(p.literal("repeat")); - }); + // One or More - Partial Match 1 + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.one_or_more(p.literal("repeat")); }); - auto ctx = parser_context("rep", false); + auto ctx = parser_context("rep", false); auto result = parser.parse(ctx); h.assert_equals("one_or_more_partial_match_1", true, result.is_need_more_input()); - }, "one_or_more_partial_match_1"); + }, + "one_or_more_partial_match_1"); - // One or More - Partial Match 2 - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.one_or_more(p.literal("again")); - }); + // One or More - Partial Match 2 + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.one_or_more(p.literal("ab")); }); - auto ctx = parser_context("againagain", false); + auto ctx = parser_context("aba", false); auto result = parser.parse(ctx); h.assert_equals("one_or_more_partial_match_2", true, result.is_need_more_input()); - }, "one_or_more_partial_match_2"); + }, + "one_or_more_partial_match_2"); - // One or More - Full Match - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.one_or_more(p.literal("single")); - }); + // One or More - Full Match + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.one_or_more(p.literal("single")); }); - auto ctx = parser_context("single", true); + auto ctx = parser_context("single", true); auto result = parser.parse(ctx); h.assert_equals("one_or_more_full_match", true, result.is_success()); - }, "one_or_more_full_match"); + }, + "one_or_more_full_match"); - // One or More - No Match - add_test([](test_harness h) { - auto parser = build_parser([](parser_builder& p) { - return p.one_or_more(p.literal("fail")); - }); + // One or More - No Match + add_test( + [](test_harness h) { + auto parser = build_parser([](parser_builder & p) { return p.one_or_more(p.literal("fail")); }); - auto ctx = parser_context("success", true); + auto ctx = parser_context("success", true); auto result = parser.parse(ctx); h.assert_equals("one_or_more_no_match", true, result.is_fail()); - }, "one_or_more_no_match"); - } - - // Provide a convenient way to run all tests - void run_all_tests() { - run_all(); - summary(); - } -}; \ No newline at end of file + }, + "one_or_more_no_match"); +} diff --git a/tests/combinator/test-recursive-references.cpp b/tests/combinator/test-recursive-references.cpp index d51e76ddee533..1d0cdea594dc2 100644 --- a/tests/combinator/test-recursive-references.cpp +++ b/tests/combinator/test-recursive-references.cpp @@ -1,120 +1,99 @@ #include "tests.h" -class test_recursive_references : public compound_test { -public: - test_recursive_references() : compound_test("test_recursive_references") { - // Test simple number - add_test([](test_harness h) { - auto value_parser = build_parser([](parser_builder& p) { +test_recursive_references::test_recursive_references() : compound_test("test_recursive_references") { + // Test simple number + add_test( + [](test_harness h) { + auto value_parser = build_parser([](parser_builder & p) { p.add_rule("number", p.one_or_more(p.one("0-9"))); - p.add_rule("list", p.sequence({ - p.literal("["), - p.rule("value"), - p.literal("]") - })); + p.add_rule("list", p.sequence({ p.literal("["), p.rule("value"), p.literal("]") })); return p.add_rule("value", p.rule("number") | p.rule("list")); }); - + parser_context ctx("1", true); - auto result = value_parser.parse(ctx); - + auto result = value_parser.parse(ctx); + h.assert_equals("result_is_success", true, result.is_success()); - }, "simple_number"); - - // Test simple list - add_test([](test_harness h) { - auto value_parser = build_parser([](parser_builder& p) { + }, + "simple_number"); + + // Test simple list + add_test( + [](test_harness h) { + auto value_parser = build_parser([](parser_builder & p) { p.add_rule("number", p.one_or_more(p.one("0-9"))); - p.add_rule("list", p.sequence({ - p.literal("["), - p.rule("value"), - p.literal("]") - })); + p.add_rule("list", p.sequence({ p.literal("["), p.rule("value"), p.literal("]") })); return p.add_rule("value", p.rule("number") | p.rule("list")); }); - + parser_context ctx("[1]", true); - auto result = value_parser.parse(ctx); - + auto result = value_parser.parse(ctx); + h.assert_equals("result_is_success", true, result.is_success()); - }, "simple_list"); - - // Test nested list - add_test([](test_harness h) { - auto value_parser = build_parser([](parser_builder& p) { + }, + "simple_list"); + + // Test nested list + add_test( + [](test_harness h) { + auto value_parser = build_parser([](parser_builder & p) { p.add_rule("number", p.one_or_more(p.one("0-9"))); - p.add_rule("list", p.sequence({ - p.literal("["), - p.rule("value"), - p.literal("]") - })); + p.add_rule("list", p.sequence({ p.literal("["), p.rule("value"), p.literal("]") })); return p.add_rule("value", p.rule("number") | p.rule("list")); }); - + parser_context ctx("[[2]]", true); - auto result = value_parser.parse(ctx); - + auto result = value_parser.parse(ctx); + h.assert_equals("result_is_success", true, result.is_success()); - }, "nested_list"); - - // Test deeply nested list - add_test([](test_harness h) { - auto value_parser = build_parser([](parser_builder& p) { + }, + "nested_list"); + + // Test deeply nested list + add_test( + [](test_harness h) { + auto value_parser = build_parser([](parser_builder & p) { p.add_rule("number", p.one_or_more(p.one("0-9"))); - p.add_rule("list", p.sequence({ - p.literal("["), - p.rule("value"), - p.literal("]") - })); + p.add_rule("list", p.sequence({ p.literal("["), p.rule("value"), p.literal("]") })); return p.add_rule("value", p.rule("number") | p.rule("list")); }); - + parser_context ctx("[[[3]]]", true); - auto result = value_parser.parse(ctx); - + auto result = value_parser.parse(ctx); + h.assert_equals("result_is_success", true, result.is_success()); - }, "deeply_nested_list"); - - // Test partial match - add_test([](test_harness h) { - auto value_parser = build_parser([](parser_builder& p) { + }, + "deeply_nested_list"); + + // Test partial match + add_test( + [](test_harness h) { + auto value_parser = build_parser([](parser_builder & p) { p.add_rule("number", p.one_or_more(p.one("0-9"))); - p.add_rule("list", p.sequence({ - p.literal("["), - p.rule("value"), - p.literal("]") - })); + p.add_rule("list", p.sequence({ p.literal("["), p.rule("value"), p.literal("]") })); return p.add_rule("value", p.rule("number") | p.rule("list")); }); - + parser_context ctx("[[", false); - auto result = value_parser.parse(ctx); - + auto result = value_parser.parse(ctx); + h.assert_equals("result_is_need_more_input", true, result.is_need_more_input()); - }, "partial_match"); - - // Test no match - add_test([](test_harness h) { - auto value_parser = build_parser([](parser_builder& p) { + }, + "partial_match"); + + // Test no match + add_test( + [](test_harness h) { + auto value_parser = build_parser([](parser_builder & p) { p.add_rule("number", p.one_or_more(p.one("0-9"))); - p.add_rule("list", p.sequence({ - p.literal("["), - p.rule("value"), - p.literal("]") - })); + p.add_rule("list", p.sequence({ p.literal("["), p.rule("value"), p.literal("]") })); return p.add_rule("value", p.rule("number") | p.rule("list")); }); - + parser_context ctx("[a]", true); - auto result = value_parser.parse(ctx); - + auto result = value_parser.parse(ctx); + h.assert_equals("result_is_fail", true, result.is_fail()); - }, "no_match"); - } - - // Provide a convenient way to run all tests - void run_all_tests() { - run_all(); - summary(); - } -}; \ No newline at end of file + }, + "no_match"); +} diff --git a/tests/combinator/tests.h b/tests/combinator/tests.h index fcaf4cfedc8df..7c3ebf45a992d 100644 --- a/tests/combinator/tests.h +++ b/tests/combinator/tests.h @@ -2,14 +2,93 @@ // Common includes for all test files #include "../testcase.hpp" +#include #include "chat-parser-combinator.h" +#include -// Forward declarations of all test classes -class test_partial_parsing; -class test_one; -class test_optional; -class test_recursive_references; -class test_json_parser; -class test_complete_example; -class test_actions; -class test_gbnf_generation; \ No newline at end of file +// Test class declarations +class test_partial_parsing : public compound_test { +public: + test_partial_parsing(); +}; + +class test_one : public compound_test { +public: + test_one(); +}; + +class test_optional : public compound_test { +public: + test_optional(); +}; + +class test_recursive_references : public compound_test { +public: + test_recursive_references(); +}; + +class test_json_parser : public compound_test { +public: + test_json_parser(); +}; + +class test_complete_example : public compound_test { +private: + class parser parser; +public: + test_complete_example(); +}; + +class test_actions : public compound_test { +public: + test_actions(); +}; + +class test_gbnf_generation : public compound_test { +public: + test_gbnf_generation(); +}; + +class uses_simple_tokenizer { +protected: + static std::vector simple_tokenize(const std::string &); +}; + +struct bench_tool_call { + std::string id; + std::string name; + nlohmann::ordered_json args; +}; + +class benchmark_test { +protected: + std::vector> cases; + long long run_benchmark(size_t which, int iterations); +public: + benchmark_test(std::vector>); +}; + +class test_command7_parser_compare : public uses_simple_tokenizer, public benchmark_test { +private: + class parser parser; + std::string reasoning; + std::string content; + std::vector tool_calls; + std::vector tokens; + // Helper methods + static class parser create_command_r7b_parser(); + static void test_command_r7b_parser(const class parser & p, const std::string & input, bool partial, bool print_results = false); + static void test_command_r7b_legacy_parser(const std::string & input, bool partial, bool print_results = false); +public: + test_command7_parser_compare(); + void run_comparison(int iterations); +}; + +class test_example_qwen3_coder : public uses_simple_tokenizer, public compound_test { +private: + // Simple tokenize function that splits by space and special chars + + class parser parser; +public: + test_example_qwen3_coder(); +}; \ No newline at end of file diff --git a/tests/test-chat-parser-combinator.cpp b/tests/test-chat-parser-combinator.cpp index f4ad8a64c3c70..ef186568fbb33 100644 --- a/tests/test-chat-parser-combinator.cpp +++ b/tests/test-chat-parser-combinator.cpp @@ -1,1211 +1,35 @@ -#include -#include -#include +#include "combinator/tests.h" -#include "nlohmann/json.hpp" - -#include "chat.h" -#include "chat-parser.h" -#include "chat-parser-combinator.h" -#include "common.h" -#include "json-schema-to-grammar.h" - -template -static void assert_equals(const std::string_view label, const T & expected, const T & actual) { - if (expected != actual) { - std::cerr << label << "\n"; - std::cerr << "Expected: " << expected << "\n"; - std::cerr << "Actual: " << actual << "\n"; - std::cerr << std::flush; - throw std::runtime_error("Test failed"); - } -} - -template -static void assert_equals(const T & expected, const T & actual) { - assert_equals("", expected, actual); -} - -static void assert_equals(const char * expected, const std::string & actual) { - assert_equals(expected, actual); -} - -static void test_partial_parsing() { - { - // Test literal - auto parser = build_parser([](parser_builder& p) { - return p.literal("hello"); - }); - - parser_context ctx; - parser_result result; - - ctx = parser_context("hello"); - result = parser.parse(ctx); - assert_equals(true, result.is_success()); - } - { - // Test char class - auto parser = build_parser([](parser_builder& p) { - return p.one("a-z"); - }); - - parser_context ctx; - parser_result result; - - ctx = parser_context("a"); - result = parser.parse(ctx); - assert_equals(true, result.is_success()); - - ctx = parser_context("A"); - result = parser.parse(ctx); - assert_equals(true, result.is_fail()); - - parser = build_parser([](parser_builder& p) { - return p.one("a-z-"); - }); - - ctx = parser_context("f"); - result = parser.parse(ctx); - assert_equals(true, result.is_success()); - - ctx = parser_context("-"); - result = parser.parse(ctx); - assert_equals(true, result.is_success()); - - ctx = parser_context("A"); - result = parser.parse(ctx); - assert_equals(true, result.is_fail()); - } - { - // Test sequences and literals - auto parser = build_parser([](parser_builder& p) { - return p.literal("") + p.literal(""); - }); - - // Partial matches - auto ctx = parser_context("", false); - result = parser.parse(ctx); - assert_equals(true, result.is_need_more_input()); - - ctx = parser_context("", true); - result = parser.parse(ctx); - assert_equals(true, result.is_success()); - - // No match, since it does not adhere to the grammar - ctx = parser_context("I am parser", false); - result = parser.parse(ctx); - assert_equals(true, result.is_fail()); - } - { - // Test choices - auto parser = build_parser([](parser_builder& p) { - return p.literal("") | p.literal(""); - }); - - // Partial matches - auto ctx = parser_context("", true); - result = parser.parse(ctx); - assert_equals(true, result.is_success()); - - ctx = parser_context("", true); - result = parser.parse(ctx); - assert_equals(true, result.is_success()); - - // No match - ctx = parser_context("", true); - result = parser.parse(ctx); - assert_equals(true, result.is_fail()); - } - { - // Test zero_or_more - auto parser = build_parser([](parser_builder& p) { - return p.zero_or_more(p.literal("ab")); - }); - - // Partial matches - auto ctx = parser_context("a", false); - auto result = parser.parse(ctx); - assert_equals(true, result.is_need_more_input()); - - ctx = parser_context("aba", false); - result = parser.parse(ctx); - assert_equals(true, result.is_need_more_input()); - - // Full match - ctx = parser_context("ab", true); - result = parser.parse(ctx); - assert_equals(true, result.is_success()); - } - { - // Test one_or_more - auto parser = build_parser([](parser_builder& p) { - return p.one_or_more(p.literal("ab")); - }); - - // Partial matches - auto ctx = parser_context("a", false); - auto result = parser.parse(ctx); - assert_equals(true, result.is_need_more_input()); - - ctx = parser_context("aba", false); - result = parser.parse(ctx); - assert_equals(true, result.is_need_more_input()); - - // Full match - ctx = parser_context("ab", true); - result = parser.parse(ctx); - assert_equals(true, result.is_success()); - - // No match - ctx = parser_context("cd", true); - result = parser.parse(ctx); - assert_equals(true, result.is_fail()); - } -} - -static void test_one() { - { - // Test common escape sequences - auto parser = build_parser([](parser_builder& p) { - return p.one("[\\n\\t\\\\]"); - }); - - parser_context ctx; - parser_result result; - - ctx = parser_context("\n"); - result = parser.parse(ctx); - assert_equals(true, result.is_success()); - - ctx = parser_context("\t"); - result = parser.parse(ctx); - assert_equals(true, result.is_success()); - - ctx = parser_context("\\"); - result = parser.parse(ctx); - assert_equals(true, result.is_success()); - - ctx = parser_context(" "); - result = parser.parse(ctx); - assert_equals(true, result.is_fail()); - } - { - // Test escaped dash (literal dash, not a range) - auto parser = build_parser([](parser_builder& p) { - return p.one("[a\\-z]"); - }); - - parser_context ctx; - parser_result result; - - ctx = parser_context("a"); - result = parser.parse(ctx); - assert_equals(true, result.is_success()); - - ctx = parser_context("-"); - result = parser.parse(ctx); - assert_equals(true, result.is_success()); - - ctx = parser_context("z"); - result = parser.parse(ctx); - assert_equals(true, result.is_success()); - - // Should NOT match 'b' since \- is a literal dash, not a range - ctx = parser_context("b"); - result = parser.parse(ctx); - assert_equals(true, result.is_fail()); - } -} - -static void test_recursive_references() { - auto value_parser = build_parser([](parser_builder& p) { - p.add_rule("number", p.one_or_more(p.one("0-9"))); - p.add_rule("list", p.sequence({ - p.literal("["), - p.rule("value"), - p.literal("]") - })); - return p.add_rule("value", p.rule("number") | p.rule("list")); - }); - - parser_context ctx; - parser_result result; - - // Test simple number - ctx = parser_context("1", true); - result = value_parser.parse(ctx); - assert_equals(true, result.is_success()); - - // Test simple list - ctx = parser_context("[1]", true); - result = value_parser.parse(ctx); - assert_equals(true, result.is_success()); - - // Test nested list - ctx = parser_context("[[2]]", true); - result = value_parser.parse(ctx); - assert_equals(true, result.is_success()); - - // Test deeply nested list - ctx = parser_context("[[[3]]]", true); - result = value_parser.parse(ctx); - assert_equals(true, result.is_success()); - - // Test partial match - ctx = parser_context("[[", false); - result = value_parser.parse(ctx); - assert_equals(true, result.is_need_more_input()); - - // Test no match - ctx = parser_context("[a]", true); - result = value_parser.parse(ctx); - assert_equals(true, result.is_fail()); -} - -static void test_optional() { - // Test optional with a match - auto parser = build_parser([](parser_builder& p) { - return p.literal("hello") + p.optional(p.literal(" world")); - }); - - // Full match with optional part present - auto ctx = parser_context("hello world"); - auto result = parser.parse(ctx); - assert_equals(true, result.is_success()); - assert_equals((size_t)11, result.end); - - // Full match with optional part absent - ctx = parser_context("hello", true); - result = parser.parse(ctx); - assert_equals(true, result.is_success()); - assert_equals((size_t)5, result.end); - - // Partial match - waiting for more input to determine if optional matches - ctx = parser_context("hello ", false); - result = parser.parse(ctx); - assert_equals(true, result.is_need_more_input()); -} - -static void test_json_parser() { - auto json = build_parser([](parser_builder & p) { - return p.json(); - }); - - { - // Test parsing a simple JSON object - std::string input = R"({"name": "test", "value": 42, "flag": true})"; - parser_context ctx(input); - - auto result = json.parse(ctx); - - assert_equals(true, result.is_success()); - assert_equals(input.size(), result.end); - } - { - // Test parsing a JSON array with mixed types - std::string input = R"([1, "hello", true, null, 3.14])"; - parser_context ctx(input); - - auto result = json.parse(ctx); - - assert_equals(true, result.is_success()); - assert_equals(input.size(), result.end); - } - { - // Test parsing nested JSON with objects and arrays - std::string input = R"({"users": [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}], "count": 2, "metadata": {"version": "1.0", "tags": ["admin", "user"]}})"; - parser_context ctx(input); - - auto result = json.parse(ctx); - - assert_equals(true, result.is_success()); - assert_equals(input.size(), result.end); - } - { - // Test partial parsing - incomplete object - std::string input = R"({"name": "test", "value": )"; - parser_context ctx(input, false); - - auto result = json.parse(ctx); - - assert_equals(true, result.is_need_more_input()); - } - { - // Test partial parsing - incomplete array - std::string input = R"([1, 2, 3, )"; - parser_context ctx(input, false); - - auto result = json.parse(ctx); - - assert_equals(true, result.is_need_more_input()); - } - { - // Test partial parsing - incomplete nested structure - std::string input = R"({"data": {"nested": )"; - parser_context ctx(input, false); - - auto result = json.parse(ctx); - - assert_equals(true, result.is_need_more_input()); - } -} - -static void test_complete_example() { - // Parser for a fictitious model that outputs: - // - // - // ... reasoning content ... - // - // ... content ... - // - // tool_name - // { ... json args ... } - // - // - auto parser = build_parser([](parser_builder & p) { - auto reasoning = p.add_rule("reasoning", - "" << p.append_reasoning(p.until("")) << ""); - - auto content = p.add_rule("content", - p.append_content(p.until(""))); - - auto json = p.json(); - - auto tool_call_name = p.add_rule("tool-call-name", - "" << p.capture_tool_call_name(p.until("")) << ""); - - auto schema = nlohmann::ordered_json::parse(R"({"type": "object"})"); - - auto tool_call_args = p.add_rule("tool-call-args", - "" << p.capture_tool_call_args(p.schema(p.succeed(json), "get_weather", schema)) << ""); - - auto tool_call = p.add_rule("tool-call", - "" << p.add_tool_call(tool_call_name << p.succeed(tool_call_args)) << ""); - - return reasoning << p.optional(content) << p.optional(tool_call); - }); - - // Test complete input - { - std::string input = R"(I need to call get_weather with city = New Yorkget_weather{"city": "New York"})"; - parser_environment env; - parser_context ctx(input, &env); - - auto result = parser.parse(ctx); - - assert_equals(true, result.is_success()); - assert_equals(input.size(), result.end); - assert_equals("I need to call get_weather with city = New York", env.result.reasoning_content); - assert_equals((size_t)1, env.result.tool_calls.size()); - assert_equals("", env.result.tool_calls[0].id); - assert_equals("get_weather", env.result.tool_calls[0].name); - assert_equals(R"({"city": "New York"})", env.result.tool_calls[0].arguments); - } - - // Test partial input - { - std::string input = R"(I need to call get_weather)"; - parser_environment env = parser_environment(); - parser_context ctx = parser_context(input, &env, /* .is_input_complete = */ false); - - auto result = parser.parse(ctx); - - assert_equals(true, result.is_need_more_input()); - assert_equals("I need to call get_weather", env.result.reasoning_content); - } - { - std::string input = R"(I need to call I need to call get_weatherI need to call get_weatherget_weather)"; - parser_environment env = parser_environment(); - parser_context ctx = parser_context(input, &env, /* .is_input_complete = */ false); - - auto result = parser.parse(ctx); - - assert_equals(true, result.is_need_more_input()); - assert_equals("I need to call get_weather", env.result.reasoning_content); - } - { - std::string input = R"(I need to call get_weatherget_weatherI need to call get_weatherget_weather{"cit)"; - parser_environment env = parser_environment(); - parser_context ctx = parser_context(input, &env, /* .is_input_complete = */ false); - - auto result = parser.parse(ctx); - - assert_equals(true, result.is_need_more_input()); - assert_equals("I need to call get_weather", env.result.reasoning_content); - assert_equals("get_weather", env.result.tool_calls[0].name); - assert_equals(R"({"cit)", env.result.tool_calls[0].arguments); - } - - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder); - }); - - std::cout << "Grammar:\n" << gbnf << "\n"; -} - -static void test_actions() { - { - // Test simple action - append matched text to content - auto parser = build_parser([](parser_builder& p) { - auto word = p.chars("[a-z]+"); - return p.action(word, [](const parser_action & act) { - act.env.result.content += std::string(act.match); - }); - }); - - parser_environment env; - parser_context ctx("hello", &env); - auto result = parser.parse(ctx); - - assert_equals(true, result.is_success()); - assert_equals("hello", env.result.content); - } - { - // Test multiple sequential actions - build a sentence - auto parser = build_parser([](parser_builder& p) { - auto greeting = p.action(p.literal("hello"), [](const parser_action & act) { - act.env.result.content += std::string(act.match) + " "; - }); - - auto name = p.action(p.chars("[A-Z][a-z]+"), [](const parser_action & act) { - act.env.result.content += std::string(act.match); - act.env.scratchpad["name"] = std::string(act.match); - }); - - return greeting + p.literal(" ") + name; - }); - - parser_environment env; - parser_context ctx("hello Alice", &env); - auto result = parser.parse(ctx); - - assert_equals(true, result.is_success()); - assert_equals("hello Alice", env.result.content); - assert_equals("Alice", std::get(env.scratchpad["name"])); - } - { - // Test using scratchpad for intermediate calculations - auto parser = build_parser([](parser_builder& p) { - auto digit = p.action(p.one("[0-9]"), [](const parser_action & act) { - auto it = act.env.scratchpad.find("sum"); - int current_sum = it != act.env.scratchpad.end() ? std::get(it->second) : 0; - current_sum += (act.match[0] - '0'); - act.env.scratchpad["sum"] = current_sum; - }); - - return p.one_or_more(digit + p.optional(p.literal("+"))); - }); - - parser_environment env; - parser_context ctx("1+2+3+4", &env); - auto result = parser.parse(ctx); - - assert_equals(true, result.is_success()); - assert_equals(10, std::get(env.scratchpad["sum"])); // 1+2+3+4 = 10 - } - { - // Test actions don't run when parse fails - auto parser = build_parser([](parser_builder& p) { - return p.action(p.literal("success"), [](const parser_action & act) { - act.env.result.content = "action_ran"; - }); - }); - - parser_environment env; - parser_context ctx("failure", &env); - auto result = parser.parse(ctx); - - assert_equals(true, result.is_fail()); - assert_equals("", env.result.content); // Action should not have run - } - { - // Test Actions work with partial parsing - auto parser = build_parser([](parser_builder& p) { - auto content = p.action(p.until(""), [](const parser_action & act) { - act.env.result.content += std::string(act.match); - }); - return "" << content << ""; - }); - - { - parser_environment env; - parser_context ctx("hello ", &env, false); - auto result = parser.parse(ctx); - - assert_equals(true, result.is_need_more_input()); - assert_equals("hello ", env.result.content); - } - { - parser_environment env; - parser_context ctx("hello world", &env, false); - auto result = parser.parse(ctx); - - assert_equals(true, result.is_need_more_input()); - assert_equals("hello world", env.result.content); - } - { - parser_environment env; - parser_context ctx("hello world", &env, true); - auto result = parser.parse(ctx); - - assert_equals(true, result.is_success()); - assert_equals("hello world", env.result.content); - } - } -} - -static void test_gbnf_generation() { - { - // Test literal - auto parser = build_parser([](parser_builder& p) { - return p.literal("hello"); - }); - - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder); - }); - - assert_equals(true, gbnf.find("root ::= \"hello\"") != std::string::npos); - assert_equals(true, gbnf.find("space ::=") != std::string::npos); - } - { - // Test char class - auto parser = build_parser([](parser_builder& p) { - return p.one("[a-z]"); - }); - - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder); - }); - - assert_equals(true, gbnf.find("root ::= [a-z]") != std::string::npos); - } - { - // Test sequence - auto parser = build_parser([](parser_builder& p) { - return p.literal("hello") + p.literal(" ") + p.literal("world"); - }); - - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder); - }); - - assert_equals(true, gbnf.find("root ::= \"hello\" \" \" \"world\"") != std::string::npos); - } - { - // Test choice - auto parser = build_parser([](parser_builder& p) { - return p.literal("cat") | p.literal("dog"); - }); - - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder); - }); - - assert_equals(true, gbnf.find("root ::= \"cat\" | \"dog\"") != std::string::npos); - } - { - // Test one_or_more - auto parser = build_parser([](parser_builder& p) { - return p.one_or_more(p.one("[0-9]")); - }); - - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder); - }); - - assert_equals(true, gbnf.find("root ::= [0-9]+") != std::string::npos); - } - { - // Test zero_or_more - auto parser = build_parser([](parser_builder& p) { - return p.zero_or_more(p.one("[a-z]")); - }); - - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder); - }); - - assert_equals(true, gbnf.find("root ::= [a-z]*") != std::string::npos); - } - { - // Test optional - auto parser = build_parser([](parser_builder& p) { - return p.literal("hello") + p.optional(p.literal(" world")); - }); - - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder); - }); - - assert_equals(true, gbnf.find("root ::= \"hello\" \" world\"?") != std::string::npos); - } - { - // Test until - auto parser = build_parser([](parser_builder& p) { - return p.until(""); - }); - - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder); - }); - - // Should generate pattern that prevents matching the full delimiter - assert_equals(true, gbnf.find("root ::= ([^<] | \"<\" [^/] | \"])*") != std::string::npos); - } - { - // Test complex expression with parentheses - auto parser = build_parser([](parser_builder& p) { - return p.one_or_more(p.literal("a") | p.literal("b")); - }); - - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder); - }); - - assert_equals(true, gbnf.find("root ::= (\"a\" | \"b\")+") != std::string::npos); - } - { - // Test rule references - auto parser = build_parser([](parser_builder& p) { - auto digit = p.add_rule("digit", p.one("[0-9]")); - return p.one_or_more(digit); - }); - - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder); - }); - - // Should have digit rule defined and referenced - assert_equals(true, gbnf.find("digit ::= [0-9]") != std::string::npos); - assert_equals(true, gbnf.find("root ::= digit+") != std::string::npos); - } - { - // Test escaping in literals - auto parser = build_parser([](parser_builder& p) { - return p.literal("hello\nworld\t!"); - }); - - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder); - }); - - assert_equals(true, gbnf.find("root ::= \"hello\\nworld\\t!\"") != std::string::npos); - } - { - // Test operator<< (whitespace insertion) - auto parser = build_parser([](parser_builder& p) { - return p.literal("hello") << p.literal("world"); - }); - - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder); - }); - - // Should inline the whitespace pattern - assert_equals(true, gbnf.find("\"hello\"") != std::string::npos); - assert_equals(true, gbnf.find("\"world\"") != std::string::npos); - } -} - -// Simple tokenize function that splits by space and special chars -static std::vector simple_tokenize(const std::string & input) { - std::vector result; - std::string current; - - for (size_t i = 0; i < input.size(); i++) { - switch (input[i]) { - case ' ': - case '\n': - case '\t': - case '{': - case '}': - case ',': - case '[': - case '"': - case ']': - case '.': - case '<': - case '>': - case '=': - case '/': - if (!current.empty()) { - result.push_back(current); - current.clear(); - } - } - current += input[i]; - } - - if (!current.empty()) { - result.push_back(current); - } - - return result; -} - -static void example_qwen3_coder() { - auto parser = build_parser([](parser_builder & p) { - auto thinking = p.add_rule("thinking", - "" << p.append_reasoning(p.until("")) << ""); - - auto content = p.add_rule("content", p.append_content(p.until(""))); - - auto arg_start = p.add_rule("arg-start", - p.action("", [](const parser_action & act) { - act.env.tool_call_args += "\":"; - })); - - auto arg_end = p.add_rule("arg-end", ""); - - auto string_arg = p.add_rule("arg-string", - p.action(arg_start, [&](const parser_action & act) { - act.env.tool_call_args += "\""; - }) - << p.action(p.until(""), [&](const parser_action & act) { - // TODO: add a JSON escape helper - act.env.tool_call_args += std::string(act.match); - }) - << p.action(arg_end, [&](const parser_action & act) { - act.env.tool_call_args += "\""; - })); - - auto json = p.json(); - - auto json_arg = p.add_rule("arg-json", - arg_start - << p.action(json, [&](const parser_action & act) { - // JSON should already be properly formatted - act.env.tool_call_args += std::string(act.match); - - // This can be streamed by passing p.success(json), but we have - // to be mindful of the potential backtracking--it only works - // if we only keep the last value... - }) - << arg_end); - - auto function = p.add_rule("function", p.add_tool_call( - "", [&](const parser_action & act) { - act.env.tool_call_args += "{"; - }) - + p.one_or_more(p.space() + (json_arg | string_arg)) - << p.action("", [&](const parser_action & act) { - act.env.tool_call_args += "}"; - }))); - - auto tool_call = p.add_rule("tool-call", - "" << p.one_or_more(function) << ""); - - - return thinking + p.optional(p.space() + content) + p.zero_or_more(p.space() + tool_call); - }); - - std::string input = - "The user wants to find large log files that haven't been accessed recently. " - "I should search for files with .log extension, filter by size (over 100MB), " - "and check access time within the last 30 days. I'll need to use the search_files function." - "Based on your requirements, I'll search for log files over 100MB that haven't been " - "accessed in the last month. This will help identify candidates for cleanup or archival.\n\n" - "\n" - "\n" - "/var/log\n" - "*.log\n" - "100\n" - "5\n" - "false\n" - "30\n" - "true\n" - "size\n" - "{\"exclude_patterns\": [\"*temp*\", \"*cache*\"], \"file_types\": [\"regular\"]}\n" - "\n" - ""; - - std::vector tokens = simple_tokenize(input); - - common_chat_msg prev; - - for (auto it = tokens.begin(); it != tokens.end(); it++) { - std::string in = std::accumulate(tokens.begin(), it, std::string()); - - parser_environment env; - parser_context ctx(in, &env, it == tokens.end() - 1); - - auto result = parser.parse(ctx); - if (result.is_fail()) { - break; - } - - /* - std::cout << "Input:\n" << in << "\n\n"; - std::cout << "Reasoning: " << prev.reasoning_content << "\n"; - std::cout << "Content : " << prev.content << "\n"; - if (!prev.tool_calls.empty()) { - std::cout << "\n=== Tool Calls ===\n"; - for (const auto & tc : prev.tool_calls) { - std::cout << "ID : " << tc.id << "\n"; - std::cout << "Name: " << tc.name << "\n"; - std::cout << "Args: " << tc.arguments << "\n"; - } - } - */ - - // This shouldn't emit any runtime errors - auto diffs = common_chat_msg_diff::compute_diffs(prev, env.result); - prev = env.result; - - /* - std::cout << "----\n"; - std::cout << "Reasoning: " << prev.reasoning_content << "\n"; - std::cout << "Content : " << prev.content << "\n"; - if (!prev.tool_calls.empty()) { - std::cout << "\n=== Tool Calls ===\n"; - for (const auto & tc : prev.tool_calls) { - std::cout << "ID : " << tc.id << "\n"; - std::cout << "Name: " << tc.name << "\n"; - std::cout << "Args: " << tc.arguments << "\n"; - } - } - std::cout << "======================\n"; - */ - - /* - std::cout << "=== Diffs ===\n\n"; - if (!diffs.empty()) { - for (size_t i = 0; i < diffs.size(); ++i) { - const auto& diff = diffs[i]; - - std::cout << "Diff #" << (i + 1) << "\n"; - - if (!diff.reasoning_content_delta.empty()) { - std::cout << " [Reasoning Content]: " << diff.reasoning_content_delta << "\n"; - } - - if (!diff.content_delta.empty()) { - std::cout << " [Content]: " << diff.content_delta << "\n"; - } - - if (diff.tool_call_index != std::string::npos) { - std::cout << " [Tool Call #" << diff.tool_call_index << "]" << "\n"; - - if (!diff.tool_call_delta.id.empty()) { - std::cout << " ID: " << diff.tool_call_delta.id << "\n"; - } - - if (!diff.tool_call_delta.name.empty()) { - std::cout << " Name: " << diff.tool_call_delta.name << "\n"; - } - - if (!diff.tool_call_delta.arguments.empty()) { - std::cout << " Arguments: " << diff.tool_call_delta.arguments << "\n"; - } - } - - std::cout << "\n"; - } - } else { - std::cout << "No changes detected.\n"; - } - */ - } -} - -static parser create_command_r7b_parser() { - auto parser = build_parser([](parser_builder & p) { - auto thinking = p.add_rule("thinking", - "<|START_THINKING|>" << p.append_reasoning(p.until("<|END_THINKING|>")) << "<|END_THINKING|>"); - - auto response = p.add_rule("response", - "<|START_RESPONSE|>" << p.append_content(p.until("<|END_RESPONSE|>")) << "<|END_RESPONSE|>"); - - auto json = p.add_rule("json", p.json()); - - auto tool_call_id = p.add_rule("tool-call-id", - p.json_key("tool_call_id", "\"" + p.capture_tool_call_id(p.json_string(), /* unescape_json = */ true) + "\"")); - - auto tool_call_name = p.add_rule("tool-name", - p.json_key("tool_name", "\"" + p.capture_tool_call_name(p.json_string(), /* unescape_json = */ true) + "\"")); - - auto tool_call_args = p.add_rule("tool-args", p.json_key("parameters", p.capture_tool_call_args(json))); - - auto tool_call_fields = p.add_rule("tool-call-fields", tool_call_id | tool_call_name | tool_call_args); - - auto tool_call = p.add_rule("tool-call", - "{" << p.add_tool_call(tool_call_fields << p.zero_or_more(p.literal(",") << tool_call_fields)) << "}"); - - auto tool_calls = p.add_rule("tool-calls", - "<|START_ACTION|>" - << ("[" << tool_call << p.zero_or_more(p.literal(",") << tool_call) << "]") - << "<|END_ACTION|>"); - - return p.optional(thinking) << p.add_rule("content", tool_calls | response); - }); - - auto grammar = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder); - }); - - std::cout << "=== Grammar ===\n\n" << grammar << "\n\n"; - - return parser; -} - -static void test_command_r7b_parser(const parser & p, const std::string & input, bool partial, bool print_results = false) { - parser_environment env; - parser_context ctx(input, &env, !partial); - p.parse(ctx); - - if (print_results) { - std::cout << "== Parsed (new) ==\n"; - std::cout << "=== Reasoning ===\n"; - std::cout << env.result.reasoning_content << "\n"; - std::cout << "\n\n=== Content ===\n"; - std::cout << env.result.content << "\n"; - std::cout << "\n\n=== Tool Calls ===\n"; - for (const auto & tc : env.result.tool_calls) { - std::cout << "id: " << tc.id << "\n"; - std::cout << "name: " << tc.name << "\n"; - std::cout << "args: " << tc.arguments << "\n"; - } - } -} - -static void test_command_r7b_legacy_parser(const std::string & input, bool partial, bool print_results = false) { - // Original parser taken from chat.cpp - common_chat_msg_parser builder(input, - /* is_partial= */ partial, { - /* .format = */ COMMON_CHAT_FORMAT_GENERIC, - /* .reasoning_format = */ COMMON_REASONING_FORMAT_AUTO, - /* .reasoning_in_content = */ false, - /* .thinking_forced_open = */ false, - }); - - builder.try_parse_reasoning("<|START_THINKING|>", "<|END_THINKING|>"); - - static const common_regex start_action_regex("<\\|START_ACTION\\|>"); - static const common_regex end_action_regex("<\\|END_ACTION\\|>"); - static const common_regex start_response_regex("<\\|START_RESPONSE\\|>"); - static const common_regex end_response_regex("<\\|END_RESPONSE\\|>"); - - if (auto res = builder.try_find_regex(start_action_regex)) { - // If we didn't extract thoughts, prelude includes them. - auto tool_calls = builder.consume_json_with_dumped_args({{"parameters"}}); - for (const auto & tool_call : tool_calls.value) { - std::string name = tool_call.contains("tool_name") ? tool_call.at("tool_name") : ""; - std::string id = tool_call.contains("tool_call_id") ? tool_call.at("tool_call_id") : ""; - std::string arguments = tool_call.contains("parameters") ? tool_call.at("parameters") : ""; - if (!builder.add_tool_call(name, id, arguments) || tool_calls.is_partial) { - throw common_chat_msg_partial_exception("incomplete tool call"); - } - } - if (tool_calls.is_partial) { - throw common_chat_msg_partial_exception("incomplete tool call"); - } - builder.consume_regex(end_action_regex); - } else if (auto res = builder.try_find_regex(start_response_regex)) { - if (!builder.try_find_regex(end_response_regex)) { - builder.add_content(builder.consume_rest()); - throw common_chat_msg_partial_exception(end_response_regex.str()); - } - } else { - builder.add_content(builder.consume_rest()); - } - - if (print_results) { - std::cout << "== Parsed (legacy) ==\n"; - std::cout << "=== Reasoning ===\n"; - std::cout << builder.result().reasoning_content << "\n"; - std::cout << "\n\n=== Content ===\n"; - std::cout << builder.result().content << "\n"; - std::cout << "\n\n=== Tool Calls ===\n"; - for (const auto & tc : builder.result().tool_calls) { - std::cout << "id: " << tc.id << "\n"; - std::cout << "name: " << tc.name << "\n"; - std::cout << "args: " << tc.arguments << "\n"; - } - } -} - -struct bench_tool_call { - std::string id; - std::string name; - nlohmann::ordered_json args; -}; - -static void benchmark_compare( - const std::string & reasoning, - const std::string & content, - const std::vector & tool_calls, - int iterations) { - - // Build response - std::vector tokens; // Since we don't have a command r7b tokenizer, we're going to "simulate" them. - - if (!reasoning.empty()) { - auto tokenized = simple_tokenize(reasoning); - tokens.emplace_back("<|START_THINKING|>"); - tokens.insert(tokens.end(), tokenized.begin(), tokenized.end()); - tokens.emplace_back("<|END_THINKING|>"); - } - - if (!content.empty()) { - auto tokenized = simple_tokenize(content); - tokens.emplace_back("<|START_RESPONSE|>"); - tokens.insert(tokens.end(), tokenized.begin(), tokenized.end()); - tokens.emplace_back("<|END_RESPONSE|>"); - } - - if (!tool_calls.empty()) { - tokens.emplace_back("<|START_ACTION|>"); - - auto json = nlohmann::json::array(); - for (const auto & tc : tool_calls) { - auto tc_json = nlohmann::json::object(); - tc_json["tool_call_id"] = tc.id; - tc_json["tool_name"] = tc.name; - tc_json["parameters"] = tc.args; - json.push_back(tc_json); - } - - auto tokenized = simple_tokenize(json.dump(-1, ' ', true)); - tokens.insert(tokens.end(), tokenized.begin(), tokenized.end()); - - tokens.emplace_back("<|END_ACTION|>"); - } - - auto run = [&](const std::function & fn) { - std::string input = std::accumulate(tokens.begin(), tokens.end(), std::string()); - - std::chrono::microseconds duration(0); - for (int i = 0; i < iterations; i++) { - auto start = std::chrono::high_resolution_clock::now(); - fn(input, false, i == 0); - auto end = std::chrono::high_resolution_clock::now(); - duration += std::chrono::duration_cast(end - start); - } - return duration.count() / iterations; - }; - - auto parser = create_command_r7b_parser(); - - auto duration_new = run([&](const std::string & input, bool partial, bool print_content) { - test_command_r7b_parser(parser, input, partial, print_content); - }); +int main() { + test_partial_parsing partial_parsing_test; + partial_parsing_test.run_all_tests(); - auto duration_legacy = run([&](const std::string & input, bool partial, bool print_content) { - try { - test_command_r7b_legacy_parser(input, partial, print_content); - } catch (const common_chat_msg_partial_exception &) { } - }); + test_one one_test; + one_test.run_all_tests(); - std::cout << " New parser avg: " << duration_new << " us\n"; - std::cout << "Legacy parser avg: " << duration_legacy << " us\n"; -} + test_optional optional_test; + optional_test.run_all_tests(); -int main() { - test_partial_parsing(); - test_one(); - test_recursive_references(); - test_optional(); - test_json_parser(); - test_complete_example(); - test_actions(); - test_gbnf_generation(); - std::cout << "All tests passed!\n"; + test_recursive_references recursive_references_test; + recursive_references_test.run_all_tests(); - example_qwen3_coder(); + test_json_parser json_parser_test; + json_parser_test.run_all_tests(); - std::cout << "\n== Benchmarks ==\n"; - std::string example_reasoning = - "To plan an effective trip to Japan that includes both historical sites and modern attractions within a budget of $4000 for a two-week stay, we need to:\n\n" - "1. Identify key historical sites and modern attractions in Japan.\n" - "2. Find affordable accommodation options that provide a balance between comfort and cost.\n" - "3. Determine the best modes of transportation for getting around Japan.\n" - "4. Create a day-by-day itinerary that ensures the user gets to see a variety of attractions without overspending.\n" - "5. Provide a detailed cost breakdown that includes accommodation, transportation, meals, and entry fees to attractions."; + test_complete_example complete_example_test; + complete_example_test.run_all_tests(); - std::string example_content = - "For a two-week trip to Japan with a $4,000 budget, I recommend planning an itinerary that balances historical sites with modern attractions. The destination will be Japan, with a duration of 14 days.\n\n" - "Given your interests in both historical sites and modern attractions, you'll want to focus on cities like Kyoto for its temples and traditional culture, Tokyo for its cutting-edge technology and entertainment districts, and possibly Hiroshima or Nara for additional historical significance.\n\n" - "For accommodation, I suggest looking for affordable options such as budget hotels, hostels, or guesthouses that offer good value without sacrificing too much comfort. Japan has excellent mid-range accommodation options that can keep your lodging costs manageable.\n\n" - "Transportation should prioritize efficiencyβ€”consider getting a JR Rail Pass for intercity travel, which allows unlimited rides on most JR trains including the Shinkansen (bullet train). Within cities, use local trains and subways, which are both affordable and highly reliable.\n\n" - "For meals, embrace local cuisine by eating at neighborhood restaurants, ramen shops, and izakayas rather than touristy establishments. This will give you an authentic experience while keeping costs reasonableβ€”you can enjoy excellent meals for $10-20 per person at local spots.\n\n"; + test_actions actions_test; + actions_test.run_all_tests(); - std::vector example_tool_calls = {{ - "call_0", - "plan_trip", - nlohmann::json::parse(R"({ - "destination": "Japan", - "duration": 14, - "budget": 4000, - "interests": ["historical sites", "modern attractions"], - "accommodation_preferences": "affordable", - "transportation_preferences": "efficient", - "meal_preferences": "local cuisine" - })") - }}; + test_gbnf_generation gbnf_generation_test; + gbnf_generation_test.run_all_tests(); - std::cout << "\nReasoning + Content:\n"; - benchmark_compare(example_reasoning, example_content, std::vector(), 100); + test_example_qwen3_coder qwen3_coder_test; + qwen3_coder_test.run_all_tests(); - std::cout << "\nReasoning + Tool Call:\n"; - benchmark_compare(example_reasoning, "", example_tool_calls, 100); + test_command7_parser_compare command7_compare_test; + command7_compare_test.run_comparison(100); + return 0; -} +} \ No newline at end of file diff --git a/tests/test-grammar-integration.cpp b/tests/test-grammar-integration.cpp index 82fae671ed00b..a1797ffd25479 100644 --- a/tests/test-grammar-integration.cpp +++ b/tests/test-grammar-integration.cpp @@ -46,14 +46,7 @@ static bool match_string(const std::string & input, llama_grammar * grammar) { } } - for (const auto & stack : stacks_cur) { - if (stack.empty()) { - // An empty stack means that the grammar has been completed - return true; - } - } - - return false; + return std::any_of(stacks_cur.begin(), stacks_cur.end(), [](auto stack) { return stack.empty(); }); } static void test(const std::string & test_desc, const std::string & grammar_str, const std::vector & passing_strings, const std::vector & failing_strings) { diff --git a/tests/testcase.hpp b/tests/testcase.hpp index a08d391ebe0b2..70b5d10acae83 100644 --- a/tests/testcase.hpp +++ b/tests/testcase.hpp @@ -1,126 +1,158 @@ #pragma once +#include +#include #include +#include #include -#include -#include #include -#include +#include #include -class test_harness { // TODO: more prototypes? -private: - int& successes_; - int& failures_; - std::ostream& error_stream_; - -public: - test_harness(int& successes, int& failures, std::ostream& error_stream = std::cerr) - : successes_(successes), failures_(failures), error_stream_(error_stream) {} - - template - bool assert_equals(const std::string &label, T expected, T actual) { - if (expected != actual) { - error_stream_ << "[" << label << "] FAILED\n"; - error_stream_ << "Expected: " << expected << "\n"; - error_stream_ << "Actual: " << actual << "\n"; - error_stream_ << std::flush; - failures_++; - return false; - } - error_stream_ << "[" << label << "] PASSED\n"; - successes_++; - return true; - } -}; - -class base_test_case { -public: - virtual ~base_test_case() = default; - virtual bool run() = 0; - virtual std::string get_name() const = 0; +class test_harness { + private: + class test_case & tc_; + std::ostream & error_stream_; + std::string test_label_; + public: + test_harness(test_case &tc, const std::string &test_label, std::ostream & error_stream = std::cerr); + template bool assert_equals(const std::string &label, T expected, T actual); }; -class test_case : public base_test_case { -private: +class test_case { + friend class test_harness; + private: std::function test_func_; - std::string name_; - int successes = 0, failures = 0; - test_harness harness; + std::string name_; + int successes = 0, failures = 0, errors = 0; + bool omit_success_msg = false; + + void inc_fail() { failures++; } -public: - test_case(std::function test_func, const std::string& name) - : test_func_(std::move(test_func)), name_(name), - harness(successes, failures) {} + void inc_suc() { successes++; } + public: + test_case(std::function test_func, const std::string & name) : + test_func_(std::move(test_func)), + name_(name) {} - bool run() override { + bool run() { // clean counters on run successes = 0; - failures = 0; + failures = 0; + test_harness harness(*this, name_); // execute run with harness - test_func_(harness); - std::cerr << "[" << get_name() << "] "; + try { + test_func_(harness); + } catch (std::exception & e) { + errors++; + std::cerr << "[" << get_name() << "] error during execution:\n" << e.what() << "\n"; + } + if (is_success()) { - std::cerr << "PASSED" << '\n'; + if (!omit_success_msg) { + std::cerr << "[" << get_name() << "] PASSED" << '\n'; + } return true; } - std::cerr << "FAILED (" << successes << "/" << (successes + failures) << ")\n"; + if (is_error()) { + std::cerr << "[" << get_name() << "] ERROR" << '\n'; + return false; + } + std::cerr << "[" << get_name() << "] FAILED (" << successes << "/" << (successes + failures) << ")\n"; return false; } - std::string get_name() const override { return name_; } - bool is_success() const { return successes > 0 && failures == 0; } + void reset() { + successes = 0; + failures = 0; + errors = 0; + } + + std::string get_name() { return name_; } + + bool is_success() const { return successes > 0 && failures == 0 && errors == 0; } + + bool is_error() const { return errors > 0; } + + void set_omit_success_msg(bool omit) { this->omit_success_msg = omit; } + + bool is_omit_success_msg() const { return this->omit_success_msg; } }; +inline test_harness::test_harness(test_case & tc, const std::string & test_label, std::ostream & error_stream) : + tc_(tc), + error_stream_(error_stream), + test_label_(test_label) {} + +template bool test_harness::assert_equals(const std::string & label, T expected, T actual) { + if (expected != actual) { + error_stream_ << "[" << label << "] FAILED\n"; + error_stream_ << "Expected: " << expected << "\n"; + error_stream_ << "Actual: " << actual << "\n"; + error_stream_ << std::flush; + tc_.inc_fail(); + return false; + } + if (!tc_.is_omit_success_msg()) { + error_stream_ << "[" << test_label_ << " -> " << label << "] PASSED\n"; + } + tc_.inc_suc(); + return true; +} + class compound_test { -private: - std::vector> test_cases_; - std::string name_; - int successes_ = 0; - int failures_ = 0; - std::unordered_map test_name_to_index_; + private: + std::vector> test_cases_; + std::string name_; + int successes_ = 0; + int failures_ = 0; + int errors_ = 0; + std::unordered_map test_name_to_index_; + + void run_test_case(std::unique_ptr & test_case) { + try { + bool result = test_case->run(); -public: - explicit compound_test(const std::string& name) : name_(name) {} + if (result) { + successes_++; + } else { + failures_++; + } + } catch (std::exception & e) { + errors_++; + std::cerr << "Error while running test " << test_case->get_name() << ":\n" << e.what() << "\n"; + } + } + + public: + explicit compound_test(const std::string & name) : name_(name) {} // Add a test case - void add_test(const std::function& test_func, const std::string& test_name) { - auto test = std::make_unique(test_func, test_name); - int index = test_cases_.size(); + void add_test(const std::function & test_func, const std::string & test_name) { + auto test = std::make_unique(test_func, test_name); + int index = test_cases_.size(); test_name_to_index_[test_name] = index; test_cases_.push_back(std::move(test)); } // Access test by name - bool operator[](const std::string& test_name) { + bool operator[](const std::string & test_name) { auto it = test_name_to_index_.find(test_name); if (it == test_name_to_index_.end()) { std::cerr << "Test case '" << test_name << "' not found in compound test '" << name_ << "'\n"; return false; } - - int index = it->second; - bool result = test_cases_[index]->run(); - - if (result) { - successes_++; - } else { - failures_++; - } - - return result; + int index = it->second; + auto & test_case = test_cases_[index]; + run_test_case(test_case); + return test_case->is_success(); } // Execute all tests void run_all() { std::cerr << "Running all tests for: " << name_ << "\n"; - for (auto& test_case : test_cases_) { - bool result = test_case->run(); - if (result) { - successes_++; - } else { - failures_++; - } + for (auto & test_case : test_cases_) { + run_test_case(test_case); } } @@ -136,12 +168,21 @@ class compound_test { std::cerr << "========================================\n"; } + // Provide a convenient way to run all tests + void run_all_tests() { + run_all(); + summary(); + } + // Get results int get_successes() const { return successes_; } + int get_failures() const { return failures_; } + int get_total() const { return successes_ + failures_; } + double get_pass_rate() const { int total = successes_ + failures_; return total > 0 ? (successes_ * 100.0 / total) : 0.0; } -}; \ No newline at end of file +}; From dcace46448522d406f1ea2eefbf90eadd9ab37bc Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Fri, 14 Nov 2025 16:13:55 +0100 Subject: [PATCH 051/183] Fix executable name and editorconfig-checker --- tests/CMakeLists.txt | 2 +- tests/combinator/test-command7-parser-compare.cpp | 6 +++--- tests/combinator/tests.h | 4 +--- tests/test-chat-parser-combinator.cpp | 4 ++-- tests/testcase.hpp | 2 +- 5 files changed, 8 insertions(+), 10 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 6168a46b0bfca..e490858d4de7c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -190,7 +190,7 @@ file(GLOB_RECURSE COMBINATOR_TEST_SOURCES add_executable(test-chat-parser-combinator ${COMBINATOR_TEST_SOURCES}) target_link_libraries(test-chat-parser-combinator PRIVATE common) install(TARGETS test-chat-parser-combinator RUNTIME) -add_test(NAME test-chat-parser-combinator COMMAND test-combinator) +add_test(NAME test-chat-parser-combinator COMMAND test-parser-combinator) set_property(TEST test-chat-parser-combinator PROPERTY LABELS main) llama_build_and_test(test-chat-template.cpp) diff --git a/tests/combinator/test-command7-parser-compare.cpp b/tests/combinator/test-command7-parser-compare.cpp index 64b91814e7ce4..e7921e742f1f2 100644 --- a/tests/combinator/test-command7-parser-compare.cpp +++ b/tests/combinator/test-command7-parser-compare.cpp @@ -38,15 +38,15 @@ class parser test_command7_parser_compare::create_command_r7b_parser() { return p.optional(thinking) << p.add_rule("content", tool_calls | response); }); - + // Check if build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); return parser; } - + // command7_parser_compare_test implementation test_command7_parser_compare::test_command7_parser_compare() : - benchmark_test(std::vector>()), + benchmark_test(std::vector>()), parser(create_command_r7b_parser()), reasoning("To plan an effective trip to Japan that includes both historical sites and modern attractions within a " "budget of $4000 for a two-week stay, we need to:\n\n" diff --git a/tests/combinator/tests.h b/tests/combinator/tests.h index 7c3ebf45a992d..d378e2fec5582 100644 --- a/tests/combinator/tests.h +++ b/tests/combinator/tests.h @@ -86,9 +86,7 @@ class test_command7_parser_compare : public uses_simple_tokenizer, public benchm class test_example_qwen3_coder : public uses_simple_tokenizer, public compound_test { private: - // Simple tokenize function that splits by space and special chars - class parser parser; public: test_example_qwen3_coder(); -}; \ No newline at end of file +}; diff --git a/tests/test-chat-parser-combinator.cpp b/tests/test-chat-parser-combinator.cpp index ef186568fbb33..636979090eb59 100644 --- a/tests/test-chat-parser-combinator.cpp +++ b/tests/test-chat-parser-combinator.cpp @@ -30,6 +30,6 @@ int main() { test_command7_parser_compare command7_compare_test; command7_compare_test.run_comparison(100); - + return 0; -} \ No newline at end of file +} diff --git a/tests/testcase.hpp b/tests/testcase.hpp index 70b5d10acae83..b874d0cdc17a5 100644 --- a/tests/testcase.hpp +++ b/tests/testcase.hpp @@ -47,7 +47,7 @@ class test_case { errors++; std::cerr << "[" << get_name() << "] error during execution:\n" << e.what() << "\n"; } - + if (is_success()) { if (!omit_success_msg) { std::cerr << "[" << get_name() << "] PASSED" << '\n'; From 107000fb2b1996bcb9c7eea407e5d55d29919a68 Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Fri, 14 Nov 2025 16:39:39 +0100 Subject: [PATCH 052/183] Third time's the charm... --- tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e490858d4de7c..5b6ea27ef3fe8 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -190,7 +190,7 @@ file(GLOB_RECURSE COMBINATOR_TEST_SOURCES add_executable(test-chat-parser-combinator ${COMBINATOR_TEST_SOURCES}) target_link_libraries(test-chat-parser-combinator PRIVATE common) install(TARGETS test-chat-parser-combinator RUNTIME) -add_test(NAME test-chat-parser-combinator COMMAND test-parser-combinator) +add_test(NAME test-chat-parser-combinator COMMAND test-chat-parser-combinator) set_property(TEST test-chat-parser-combinator PROPERTY LABELS main) llama_build_and_test(test-chat-template.cpp) From 4ebddbd2fa042ffa9d06d330a8f79891f4aa6818 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Fri, 14 Nov 2025 14:59:47 -0600 Subject: [PATCH 053/183] add trigger parser to begin lazy grammar rule generation --- common/chat-parser-combinator.cpp | 43 +++++++++++++++++++++++++++++++ common/chat-parser-combinator.h | 6 +++++ 2 files changed, 49 insertions(+) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index 255fab3f52473..88ec3fb0a5578 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -28,6 +28,7 @@ enum parser_type { ROOT, JSON_STRING, ACTION, + TRIGGER, }; class parser_visitor; @@ -1179,6 +1180,38 @@ class action_parser : public common_chat_combinator_parser_base { const common_chat_combinator_parser & child() const { return parser_; } }; +// Annotate nodes for use when generating lazy GBNF grammar rules. When built +// with lazy = true, only grammar rules reachable from trigger nodes are +// emitted. +class trigger_parser : public common_chat_combinator_parser_base { + common_chat_combinator_parser parser_; + + public: + static constexpr parser_type type_value = TRIGGER; + + trigger_parser(const common_chat_combinator_parser & parser, int id) + : common_chat_combinator_parser_base(id), parser_(parser) {} + + parser_type type() const override { return type_value; } + + common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { + return parser_->parse(ctx, start); + } + + void assign_id(std::shared_ptr counter) override { + common_chat_combinator_parser_base::assign_id(counter); + parser_->assign_id(counter); + } + + std::string dump() const override { + return "Trigger(" + parser_->dump() + ")"; + } + + void accept(parser_visitor & visitor) override; + + const common_chat_combinator_parser & child() const { return parser_; } +}; + // Base visitor class for parser tree traversal class parser_visitor { public: @@ -1202,6 +1235,7 @@ class parser_visitor { virtual void visit(rule_parser & p) = 0; virtual void visit(root_parser & p) = 0; virtual void visit(action_parser & p) = 0; + virtual void visit(trigger_parser & p) = 0; }; // Escape special characters for GBNF literals @@ -1430,6 +1464,10 @@ class gbnf_visitor : public parser_visitor { // Actions are transparent for grammar generation - just visit child p.child()->accept(*this); } + + void visit(trigger_parser & p) override { + p.child()->accept(*this); + } }; // Implement accept() methods for all parser classes @@ -1451,6 +1489,7 @@ void schema_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void rule_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void root_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void action_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } +void trigger_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } common_chat_parse_result common_chat_parse_cache::set(int id, size_t start, common_chat_parse_result result) { if (id == -1) { @@ -1626,6 +1665,10 @@ common_chat_combinator_parser common_chat_combinator_parser_builder::capture(con }, COMMON_CHAT_PARSE_RESULT_SUCCESS); } +common_chat_combinator_parser common_chat_combinator_parser_builder::trigger(const common_chat_combinator_parser & p) { + return common_chat_combinator_parser(std::make_shared(p, counter_->next())); +} + common_chat_combinator_parser common_chat_combinator_parser_builder::add_rule(const std::string & name, const common_chat_combinator_parser & p) { (*rules_)[name] = p; return rule(name); diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index ebc5321f9a244..009a9bde66c75 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -275,6 +275,12 @@ class common_chat_combinator_parser_builder { // Captures matched text to env.captures[key] common_chat_combinator_parser capture(const std::string & key, const common_chat_combinator_parser & p); + // Mark a node as a trigger for GBNF grammar generartion. This is used for + // lazy grammar evaluation by only producing GBNF grammar rules that are + // reachable from trigger nodes. + // S -> Trigger(A) + common_chat_combinator_parser trigger(const common_chat_combinator_parser & p); + common_chat_combinator_parser add_rule(const std::string & name, const common_chat_combinator_parser & p); void assign_ids(common_chat_combinator_parser & p); From 68f003b31fdb3c4f5e318ee688d57b2f311168fc Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Fri, 14 Nov 2025 15:27:56 -0600 Subject: [PATCH 054/183] working lazy grammar --- common/chat-parser-combinator.cpp | 153 +++++++++++++++++++++++++- common/chat-parser-combinator.h | 2 +- tests/test-chat-parser-combinator.cpp | 68 ++++++++++++ 3 files changed, 216 insertions(+), 7 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index 88ec3fb0a5578..817123b5c3275 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -8,6 +8,7 @@ #include #include #include +#include enum parser_type { LITERAL, @@ -1269,13 +1270,81 @@ static std::string gbnf_excluding_pattern(const std::vector & strin return generic_excluding_pattern(strings, gbnf_literal, gbnf_escape_char_class, true); } +// Visitor for collecting reachable rules from a subtree +class reachability_visitor : public parser_visitor { + std::unordered_set & reachable_rules_; + std::shared_ptr> rules_; + + public: + reachability_visitor( + std::unordered_set & reachable_rules, + std::shared_ptr> rules + ) : reachable_rules_(reachable_rules), rules_(rules) {} + + void visit(literal_parser &) override {} + void visit(any_parser &) override {} + void visit(space_parser &) override {} + void visit(json_string_parser &) override {} + void visit(chars_parser &) override {} + void visit(until_parser &) override {} + void visit(and_parser & p) override { p.child()->accept(*this); } + void visit(not_parser & p) override { p.child()->accept(*this); } + + void visit(sequence_parser & p) override { + for (const auto & child : p.parsers()) { + child->accept(*this); + } + } + + void visit(choice_parser & p) override { + for (const auto & child : p.parsers()) { + child->accept(*this); + } + } + + void visit(one_or_more_parser & p) override { p.child()->accept(*this); } + void visit(zero_or_more_parser & p) override { p.child()->accept(*this); } + void visit(optional_parser & p) override { p.child()->accept(*this); } + void visit(repetition_parser & p) override { p.child()->accept(*this); } + void visit(schema_parser & p) override { p.child()->accept(*this); } + void visit(action_parser & p) override { p.child()->accept(*this); } + void visit(trigger_parser & p) override { p.child()->accept(*this); } + + void visit(rule_parser & p) override { + const std::string & name = p.name(); + // If we've already processed this rule, skip to avoid infinite recursion + if (reachable_rules_.find(name) != reachable_rules_.end()) { + return; + } + reachable_rules_.insert(name); + + // Recursively visit the rule's definition + if (rules_) { + auto it = rules_->find(name); + if (it != rules_->end()) { + it->second->accept(*this); + } + } + } + + void visit(root_parser & p) override { + p.root()->accept(*this); + } +}; + class gbnf_visitor : public parser_visitor { const common_grammar_builder & builder_; std::unordered_map rule_name_mapping_; std::string current_result_; + bool lazy_; + std::vector trigger_names_; + std::unordered_set reachable_rules_; + int trigger_counter_; + std::vector> triggers_; public: - gbnf_visitor(const common_grammar_builder & builder) : builder_(builder) {} + gbnf_visitor(const common_grammar_builder & builder, bool lazy = false) + : builder_(builder), lazy_(lazy), trigger_counter_(0) {} const std::string& result() const { return current_result_; } @@ -1285,6 +1354,18 @@ class gbnf_visitor : public parser_visitor { return type == CHOICE || type == SEQUENCE; } + // Collect all reachable rules from the given triggers + void collect_reachable_rules( + const std::vector> & triggers, + std::shared_ptr> rules + ) { + reachable_rules_.clear(); + reachability_visitor visitor(reachable_rules_, rules); + for (const auto & trigger : triggers) { + trigger->accept(visitor); + } + } + public: void visit(literal_parser & p) override { current_result_ = gbnf_literal(p.literal()); @@ -1445,10 +1526,48 @@ class gbnf_visitor : public parser_visitor { } void visit(root_parser & p) override { - // Generate named rules first auto rules = p.rules(); + + if (!lazy_) { + // Non-lazy mode: generate all rules eagerly + if (rules) { + for (const auto & [name, rule] : *rules) { + rule->accept(*this); + auto rule_body = current_result_; + auto canonical_name = builder_.add_rule(name, rule_body); + rule_name_mapping_[name] = canonical_name; + } + } + + // Return root body for composition + p.root()->accept(*this); + return; + } + + // Lazy mode: only generate rules reachable from triggers + + // First pass: traverse root to collect triggers and generate synthetic rules + // (visit(trigger_parser) will populate triggers_ and trigger_names_) + p.root()->accept(*this); + + // Check if we found any triggers + if (triggers_.empty()) { + LOG_ERR("Lazy grammar generation enabled but no trigger nodes found\n"); + current_result_ = ""; + return; + } + + // Second pass: collect all rules reachable from triggers + collect_reachable_rules(triggers_, rules); + + // Third pass: generate only reachable rules if (rules) { for (const auto & [name, rule] : *rules) { + // Skip rules that aren't reachable + if (reachable_rules_.find(name) == reachable_rules_.end()) { + continue; + } + rule->accept(*this); auto rule_body = current_result_; auto canonical_name = builder_.add_rule(name, rule_body); @@ -1456,8 +1575,8 @@ class gbnf_visitor : public parser_visitor { } } - // Return root body for composition - p.root()->accept(*this); + // Generate root as alternation of trigger rules + current_result_ = string_join(trigger_names_, " | "); } void visit(action_parser & p) override { @@ -1466,7 +1585,29 @@ class gbnf_visitor : public parser_visitor { } void visit(trigger_parser & p) override { + if (!lazy_) { + // Non-lazy mode: transparent pass-through + p.child()->accept(*this); + return; + } + + // Lazy mode: create synthetic rule for this trigger + ++trigger_counter_; + std::string trigger_name = "trigger-" + std::to_string(trigger_counter_); + + // Visit child to generate its grammar p.child()->accept(*this); + std::string child_grammar = current_result_; + + // Add synthetic rule + builder_.add_rule(trigger_name, child_grammar); + trigger_names_.push_back(trigger_name); + + // Store trigger for reachability analysis + triggers_.push_back(p.child().ptr()); + + // Return the trigger rule reference + current_result_ = trigger_name; } }; @@ -1561,8 +1702,8 @@ std::string common_chat_combinator_parser::dump() const { return ptr_->dump(); } -void common_chat_combinator_parser::build_grammar(const common_grammar_builder & builder) const { - gbnf_visitor visitor(builder); +void common_chat_combinator_parser::build_grammar(const common_grammar_builder & builder, bool lazy) const { + gbnf_visitor visitor(builder, lazy); ptr_->accept(visitor); auto result = visitor.result(); if (!result.empty()) { diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index 009a9bde66c75..c790e0f6638a9 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -164,7 +164,7 @@ class common_chat_combinator_parser { std::string dump() const; - void build_grammar(const common_grammar_builder & builder) const; + void build_grammar(const common_grammar_builder & builder, bool lazy = false) const; }; common_chat_combinator_parser operator+(const char * lhs, const common_chat_combinator_parser & rhs); diff --git a/tests/test-chat-parser-combinator.cpp b/tests/test-chat-parser-combinator.cpp index a3e4496760018..d64ab2f5c232b 100644 --- a/tests/test-chat-parser-combinator.cpp +++ b/tests/test-chat-parser-combinator.cpp @@ -493,6 +493,73 @@ static void test_sax_events() { } } +static void test_triggers() { + { + // Test basic trigger functionality with lazy mode + auto parser = build_combinator_parser([](common_chat_combinator_parser_builder& p) { + auto greeting = p.trigger(p.literal("hello")); + auto farewell = p.trigger(p.literal("goodbye")); + return greeting | farewell; + }); + + // Non-lazy mode: triggers are transparent + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder, false); + }); + + assert_equals(true, gbnf.find("root ::= \"hello\" | \"goodbye\"") != std::string::npos); + + // Lazy mode: triggers create synthetic rules + auto gbnf_lazy = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder, true); + }); + + // Should have trigger-1 and trigger-2 synthetic rules + assert_equals(true, gbnf_lazy.find("trigger-1 ::= \"hello\"") != std::string::npos); + assert_equals(true, gbnf_lazy.find("trigger-2 ::= \"goodbye\"") != std::string::npos); + // Root should be alternation of triggers + assert_equals(true, gbnf_lazy.find("root ::= trigger-1 | trigger-2") != std::string::npos); + } + { + // Test that only reachable rules from triggers are generated + auto parser = build_combinator_parser([](common_chat_combinator_parser_builder& p) { + // Add multiple rules + auto digit = p.add_rule("digit", p.one("[0-9]")); + auto letter = p.add_rule("letter", p.one("[a-z]")); + auto word = p.add_rule("word", p.one_or_more(letter)); + auto number = p.add_rule("number", p.one_or_more(digit)); + + // Only trigger the word path, not the number path + auto triggered_word = p.trigger(word); + return triggered_word; + }); + + // Non-lazy mode: all rules generated + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder, false); + }); + + assert_equals(true, gbnf.find("digit ::=") != std::string::npos); + assert_equals(true, gbnf.find("letter ::=") != std::string::npos); + assert_equals(true, gbnf.find("word ::=") != std::string::npos); + assert_equals(true, gbnf.find("number ::=") != std::string::npos); + + // Lazy mode: only rules reachable from trigger + auto gbnf_lazy = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder, true); + }); + + // Should have letter and word (reachable from trigger) + assert_equals(true, gbnf_lazy.find("letter ::=") != std::string::npos); + assert_equals(true, gbnf_lazy.find("word ::=") != std::string::npos); + // Should NOT have digit and number (not reachable from trigger) + assert_equals(true, gbnf_lazy.find("digit ::=") == std::string::npos); + assert_equals(true, gbnf_lazy.find("number ::=") == std::string::npos); + // Should have trigger-1 synthetic rule + assert_equals(true, gbnf_lazy.find("trigger-1 ::=") != std::string::npos); + } +} + static void test_gbnf_generation() { { // Test literal @@ -1056,6 +1123,7 @@ int main() { test_json_parser(); test_actions(); test_sax_events(); + test_triggers(); test_gbnf_generation(); std::cout << "All tests passed!\n"; From 9f09c9fc6383919e7f550d400c566ec597ba39fe Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Fri, 14 Nov 2025 17:32:14 -0600 Subject: [PATCH 055/183] refactor json rules now that we check for reachability --- common/chat-parser-combinator.cpp | 129 +++++++++++++++--------------- common/chat-parser-combinator.h | 16 +++- 2 files changed, 80 insertions(+), 65 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index 817123b5c3275..e055d2eae674d 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -1306,7 +1306,10 @@ class reachability_visitor : public parser_visitor { void visit(zero_or_more_parser & p) override { p.child()->accept(*this); } void visit(optional_parser & p) override { p.child()->accept(*this); } void visit(repetition_parser & p) override { p.child()->accept(*this); } - void visit(schema_parser & p) override { p.child()->accept(*this); } + void visit(schema_parser & p) override { + // Schema parsers are opaque - don't traverse their children + // The schema system will handle rule generation via builder_.add_schema() + } void visit(action_parser & p) override { p.child()->accept(*this); } void visit(trigger_parser & p) override { p.child()->accept(*this); } @@ -1763,7 +1766,7 @@ common_chat_combinator_parser common_chat_combinator_parser_builder::one(const s return chars(classes, 1, 1); } -common_chat_combinator_parser common_chat_combinator_parser_builder::json_string() { +common_chat_combinator_parser common_chat_combinator_parser_builder::json_string_unqouted() { return common_chat_combinator_parser(std::make_shared(counter_->next())); } @@ -1815,6 +1818,17 @@ common_chat_combinator_parser common_chat_combinator_parser_builder::add_rule(co return rule(name); } +common_chat_combinator_parser common_chat_combinator_parser_builder::add_rule(const std::string & name, const std::function & builder) { + if (rules_->find(name) != rules_->end()) { + return rule(name); + } + + (*rules_)[name] = literal(""); // Placeholder + auto parser = builder(); + (*rules_)[name] = parser; + return rule(name); +} + void common_chat_combinator_parser_builder::assign_ids(common_chat_combinator_parser & p) { if (p.ptr()) { p.ptr()->assign_id(counter_); @@ -1834,74 +1848,61 @@ common_chat_combinator_parser build_combinator_parser(const std::function counter) { - common_chat_combinator_parser_builder builder(std::move(counter)); - - // Whitespace: space, tab, newline, carriage return - auto ws = builder.space(); - - // Number components - auto digit1_9 = builder.chars("[1-9]", 1, 1); - auto digits = builder.chars("[0-9]"); - - // Integer part: 0 or non-zero digit followed by more digits - auto int_part = builder.literal("0") | (digit1_9 + builder.chars("[0-9]", 0, -1)); - - // Optional fractional part - auto frac = builder.literal(".") + digits; - - // Optional exponent part - auto exp = (builder.literal("e") | builder.literal("E")) + builder.optional(builder.chars("[+\\-]", 1, 1)) + digits; - - // Complete number - auto number = builder.optional(builder.literal("-")) + int_part + builder.optional(frac) + builder.optional(exp); - - builder.add_rule("json_number", number); - - // String: specialized single-pass parser (content only, wrapped with quotes) - auto string = builder.literal("\"") + builder.json_string() + builder.literal("\""); - - builder.add_rule("json_string", string); - - // Literals - auto true_lit = builder.literal("true"); - auto false_lit = builder.literal("false"); - auto null_lit = builder.literal("null"); - - // Object: { "key": value, ... } - auto member = builder.rule("json_string") + ws + builder.literal(":") + ws + builder.rule("json_value"); - auto members = member + builder.zero_or_more(ws + builder.literal(",") + ws + member); - - // Empty object or object with members - auto object = (builder.literal("{") + ws + builder.literal("}")) | - (builder.literal("{") + ws + members + ws + builder.literal("}")); - - builder.add_rule("json_object", object); +common_chat_combinator_parser common_chat_combinator_parser_builder::json_number() { + return add_rule("json-number", [this]() { + auto digit1_9 = chars("[1-9]", 1, 1); + auto digits = chars("[0-9]"); + auto int_part = literal("0") | (digit1_9 + chars("[0-9]", 0, -1)); + auto frac = literal(".") + digits; + auto exp = (literal("e") | literal("E")) + optional(chars("[+\\-]", 1, 1)) + digits; + return optional(literal("-")) + int_part + optional(frac) + optional(exp); + }); +} - // Array: [ value, ... ] - auto elements = builder.rule("json_value") + builder.zero_or_more(ws + builder.literal(",") + ws + builder.rule("json_value")); +common_chat_combinator_parser common_chat_combinator_parser_builder::json_string() { + return add_rule("json-string", [this]() { + return literal("\"") + json_string_unqouted() + literal("\""); + }); +} - // Empty array or array with elements - auto array = (builder.literal("[") + ws + builder.literal("]")) | - (builder.literal("[") + ws + elements + ws + builder.literal("]")); +common_chat_combinator_parser common_chat_combinator_parser_builder::json_bool() { + return add_rule("json-bool", [this]() { + return literal("true") | literal("false"); + }); +} - builder.add_rule("json_array", array); +common_chat_combinator_parser common_chat_combinator_parser_builder::json_null() { + return add_rule("json-null", [this]() { + return literal("null"); + }); +} - // Value - uses forward references for recursive structures - auto root = builder.add_rule("json_value", - builder.rule("json_object") | - builder.rule("json_array") | - builder.rule("json_string") | - builder.rule("json_number") | - true_lit | - false_lit | - null_lit - ); +common_chat_combinator_parser common_chat_combinator_parser_builder::json_object() { + return add_rule("json-object", [this]() { + auto ws = space(); + auto member = json_string() + ws + literal(":") + ws + json(); + auto members = member + zero_or_more(ws + literal(",") + ws + member); + return (literal("{") + ws + literal("}")) | + (literal("{") + ws + members + ws + literal("}")); + }); +} - // Wrap in root_parser to own the rules - return common_chat_combinator_parser(std::make_shared(root, builder.rules(), -1)); +common_chat_combinator_parser common_chat_combinator_parser_builder::json_array() { + return add_rule("json-array", [this]() { + auto ws = space(); + auto elements = json() + zero_or_more(ws + literal(",") + ws + json()); + return (literal("[") + ws + literal("]")) | + (literal("[") + ws + elements + ws + literal("]")); + }); } common_chat_combinator_parser common_chat_combinator_parser_builder::json() { - return json_parser(counter_); + return add_rule("json-value", [this]() { + return json_object() | + json_array() | + json_string() | + json_number() | + json_bool() | + json_null(); + }); } diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index c790e0f6638a9..a9e06a754439c 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -259,9 +259,15 @@ class common_chat_combinator_parser_builder { // Creates a complete JSON parser supporting objects, arrays, strings, numbers, booleans, and null. // value -> object | array | string | number | true | false | null common_chat_combinator_parser json(); + common_chat_combinator_parser json_object(); + common_chat_combinator_parser json_string(); + common_chat_combinator_parser json_array(); + common_chat_combinator_parser json_number(); + common_chat_combinator_parser json_bool(); + common_chat_combinator_parser json_null(); // Specialized single-pass JSON string parser with escape sequence handling - common_chat_combinator_parser json_string(); + common_chat_combinator_parser json_string_unqouted(); // Wraps a parser with JSON schema metadata for grammar generation. // Used internally to convert JSON schemas to GBNF grammar rules. @@ -281,8 +287,16 @@ class common_chat_combinator_parser_builder { // S -> Trigger(A) common_chat_combinator_parser trigger(const common_chat_combinator_parser & p); + // Adds a named rule and returns a rule reference. common_chat_combinator_parser add_rule(const std::string & name, const common_chat_combinator_parser & p); + // Adds a named rule using a function. This handles recursive grammars by + // inserting a placeholder rule before invoking the builder, allowing the + // builder to reference the rule being defined. Use this when the rule + // definition needs to call back to itself (directly or indirectly). + // add_rule("json", [&]() { return json_object() | json_array() | ... }) + common_chat_combinator_parser add_rule(const std::string & name, const std::function & builder); + void assign_ids(common_chat_combinator_parser & p); std::shared_ptr> rules() const { return rules_; } From bee5eb480864734d22258aaf67b0860258b52950 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Fri, 14 Nov 2025 18:45:38 -0600 Subject: [PATCH 056/183] reduce pointer usage --- common/chat-parser-combinator.cpp | 238 +++++++++++++++--------------- common/chat-parser-combinator.h | 9 +- 2 files changed, 124 insertions(+), 123 deletions(-) diff --git a/common/chat-parser-combinator.cpp b/common/chat-parser-combinator.cpp index e055d2eae674d..4346751d62ba8 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-parser-combinator.cpp @@ -68,9 +68,9 @@ class common_chat_combinator_parser_base { // Actual parsing implementation (to be overridden by subclasses) virtual common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) = 0; - virtual void assign_id(std::shared_ptr counter) { + virtual void assign_id(common_chat_combinator_parser_counter & counter) { if (id_ == -1) { - id_ = counter->next(); + id_ = counter.next(); } } @@ -320,6 +320,48 @@ static std::string regex_excluding_pattern(const std::vector & stri return generic_excluding_pattern(strings, regex_escape, regex_escape_char_class); } +// Container for the root parser and all named rules in the grammar. +// Manages ownership of rule registry to enable recursive grammar definitions. +class root_parser : public common_chat_combinator_parser_base { + common_chat_combinator_parser root_; + std::unordered_map rules_; + + public: + static constexpr parser_type type_value = ROOT; + + root_parser(int id) : common_chat_combinator_parser_base(id) {} + + parser_type type() const override { return type_value; } + + common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { + return root_->parse(ctx, start); + } + + void assign_id(common_chat_combinator_parser_counter & counter) override { + common_chat_combinator_parser_base::assign_id(counter); + root_->assign_id(counter); + } + + std::string dump() const override { + return root_->dump(); + } + + void accept(parser_visitor & visitor) override; + + void add_rule(const std::string & name, const common_chat_combinator_parser & parser) { + rules_[name] = parser; + } + + void set_root(const common_chat_combinator_parser & parser) { + root_ = parser; + } + + const common_chat_combinator_parser & root() const { return root_; } + + std::unordered_map & rules() { return rules_; } + const std::unordered_map & rules() const { return rules_; } +}; + // Matches an exact literal string. // S -> "hello" class literal_parser : public common_chat_combinator_parser_base { @@ -395,7 +437,7 @@ class sequence_parser : public common_chat_combinator_parser_base { return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); } - void assign_id(std::shared_ptr counter) override { + void assign_id(common_chat_combinator_parser_counter & counter) override { common_chat_combinator_parser_base::assign_id(counter); for (auto & p : parsers_) { p->assign_id(counter); @@ -450,7 +492,7 @@ class choice_parser : public common_chat_combinator_parser_base { return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); } - void assign_id(std::shared_ptr counter) override { + void assign_id(common_chat_combinator_parser_counter & counter) override { common_chat_combinator_parser_base::assign_id(counter); for (auto & p : parsers_) { p->assign_id(counter); @@ -525,7 +567,7 @@ class repetition_parser : public common_chat_combinator_parser_base { return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); } - void assign_id(std::shared_ptr counter) override { + void assign_id(common_chat_combinator_parser_counter & counter) override { common_chat_combinator_parser_base::assign_id(counter); parser_->assign_id(counter); } @@ -620,7 +662,7 @@ class and_parser : public common_chat_combinator_parser_base { return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start); } - void assign_id(std::shared_ptr counter) override { + void assign_id(common_chat_combinator_parser_counter & counter) override { common_chat_combinator_parser_base::assign_id(counter); parser_->assign_id(counter); } @@ -663,7 +705,7 @@ class not_parser : public common_chat_combinator_parser_base { return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start); } - void assign_id(std::shared_ptr counter) override { + void assign_id(common_chat_combinator_parser_counter & counter) override { common_chat_combinator_parser_base::assign_id(counter); parser_->assign_id(counter); } @@ -1025,25 +1067,26 @@ class schema_parser : public common_chat_combinator_parser_base { // expr -> term | expr "+" term class rule_parser : public common_chat_combinator_parser_base { std::string name_; - std::weak_ptr> rules_; + std::weak_ptr root_; public: static constexpr parser_type type_value = RULE; - rule_parser(const std::string & name, const std::shared_ptr> & rules, int id) - : common_chat_combinator_parser_base(id), name_(name), rules_(rules) {} + rule_parser(const std::string & name, const std::weak_ptr & root, int id) + : common_chat_combinator_parser_base(id), name_(name), root_(root) {} parser_type type() const override { return type_value; } common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { - auto rules = rules_.lock(); - if (!rules) { - LOG_ERR("rule_parser::parse called with expired rule registry\n"); + auto root = root_.lock(); + if (!root) { + LOG_ERR("rule_parser::parse called with expired root parser\n"); return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); } - auto it = rules->find(name_); - if (it == rules->end()) { + auto & rules = root->rules(); + auto it = rules.find(name_); + if (it == rules.end()) { LOG_ERR("rule_parser::parse rule '%s' not found in registry\n", name_.c_str()); return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); } @@ -1097,42 +1140,6 @@ class rule_parser : public common_chat_combinator_parser_base { const std::string & name() const { return name_; } }; -// Container for the root parser and all named rules in the grammar. -// Manages ownership of rule registry to enable recursive grammar definitions. -class root_parser : public common_chat_combinator_parser_base { - common_chat_combinator_parser root_; - std::shared_ptr> rules_; - - friend class parser_visitor; - - public: - static constexpr parser_type type_value = ROOT; - - root_parser(const common_chat_combinator_parser & root, std::shared_ptr> rules, int id) - : common_chat_combinator_parser_base(id), root_(root), rules_(std::move(rules)) {} - - parser_type type() const override { return type_value; } - - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { - return root_->parse(ctx, start); - } - - void assign_id(std::shared_ptr counter) override { - common_chat_combinator_parser_base::assign_id(counter); - root_->assign_id(counter); - } - - std::string dump() const override { - return root_->dump(); - } - - void accept(parser_visitor & visitor) override; - - const common_chat_combinator_parser & root() const { return root_; } - - std::shared_ptr> rules() const { return rules_; } -}; - // Wraps a parser with a semantic action callback. class action_parser : public common_chat_combinator_parser_base { common_chat_combinator_parser parser_; @@ -1167,7 +1174,7 @@ class action_parser : public common_chat_combinator_parser_base { return result; } - void assign_id(std::shared_ptr counter) override { + void assign_id(common_chat_combinator_parser_counter & counter) override { common_chat_combinator_parser_base::assign_id(counter); parser_->assign_id(counter); } @@ -1199,7 +1206,7 @@ class trigger_parser : public common_chat_combinator_parser_base { return parser_->parse(ctx, start); } - void assign_id(std::shared_ptr counter) override { + void assign_id(common_chat_combinator_parser_counter & counter) override { common_chat_combinator_parser_base::assign_id(counter); parser_->assign_id(counter); } @@ -1273,12 +1280,12 @@ static std::string gbnf_excluding_pattern(const std::vector & strin // Visitor for collecting reachable rules from a subtree class reachability_visitor : public parser_visitor { std::unordered_set & reachable_rules_; - std::shared_ptr> rules_; + const std::unordered_map & rules_; public: reachability_visitor( std::unordered_set & reachable_rules, - std::shared_ptr> rules + const std::unordered_map & rules ) : reachable_rules_(reachable_rules), rules_(rules) {} void visit(literal_parser &) override {} @@ -1306,7 +1313,7 @@ class reachability_visitor : public parser_visitor { void visit(zero_or_more_parser & p) override { p.child()->accept(*this); } void visit(optional_parser & p) override { p.child()->accept(*this); } void visit(repetition_parser & p) override { p.child()->accept(*this); } - void visit(schema_parser & p) override { + void visit(schema_parser &) override { // Schema parsers are opaque - don't traverse their children // The schema system will handle rule generation via builder_.add_schema() } @@ -1322,11 +1329,9 @@ class reachability_visitor : public parser_visitor { reachable_rules_.insert(name); // Recursively visit the rule's definition - if (rules_) { - auto it = rules_->find(name); - if (it != rules_->end()) { - it->second->accept(*this); - } + auto it = rules_.find(name); + if (it != rules_.end()) { + it->second->accept(*this); } } @@ -1360,7 +1365,7 @@ class gbnf_visitor : public parser_visitor { // Collect all reachable rules from the given triggers void collect_reachable_rules( const std::vector> & triggers, - std::shared_ptr> rules + const std::unordered_map & rules ) { reachable_rules_.clear(); reachability_visitor visitor(reachable_rules_, rules); @@ -1533,13 +1538,11 @@ class gbnf_visitor : public parser_visitor { if (!lazy_) { // Non-lazy mode: generate all rules eagerly - if (rules) { - for (const auto & [name, rule] : *rules) { - rule->accept(*this); - auto rule_body = current_result_; - auto canonical_name = builder_.add_rule(name, rule_body); - rule_name_mapping_[name] = canonical_name; - } + for (const auto & [name, rule] : rules) { + rule->accept(*this); + auto rule_body = current_result_; + auto canonical_name = builder_.add_rule(name, rule_body); + rule_name_mapping_[name] = canonical_name; } // Return root body for composition @@ -1564,18 +1567,16 @@ class gbnf_visitor : public parser_visitor { collect_reachable_rules(triggers_, rules); // Third pass: generate only reachable rules - if (rules) { - for (const auto & [name, rule] : *rules) { - // Skip rules that aren't reachable - if (reachable_rules_.find(name) == reachable_rules_.end()) { - continue; - } - - rule->accept(*this); - auto rule_body = current_result_; - auto canonical_name = builder_.add_rule(name, rule_body); - rule_name_mapping_[name] = canonical_name; + for (const auto & [name, rule] : rules) { + // Skip rules that aren't reachable + if (reachable_rules_.find(name) == reachable_rules_.end()) { + continue; } + + rule->accept(*this); + auto rule_body = current_result_; + auto canonical_name = builder_.add_rule(name, rule_body); + rule_name_mapping_[name] = canonical_name; } // Generate root as alternation of trigger rules @@ -1715,51 +1716,47 @@ void common_chat_combinator_parser::build_grammar(const common_grammar_builder & } common_chat_combinator_parser_builder::common_chat_combinator_parser_builder() - : rules_(std::make_shared>()) - , counter_(std::make_shared(0)) {} - -common_chat_combinator_parser_builder::common_chat_combinator_parser_builder(std::shared_ptr counter) - : rules_(std::make_shared>()) - , counter_(std::move(counter)) {} + : root_(std::make_shared(0)) // root parser has id 0 + , counter_(1) {} common_chat_combinator_parser common_chat_combinator_parser_builder::literal(const std::string & literal) { - return common_chat_combinator_parser(std::make_shared(literal, counter_->next())); + return common_chat_combinator_parser(std::make_shared(literal, counter_.next())); } common_chat_combinator_parser common_chat_combinator_parser_builder::sequence(std::initializer_list parsers) { - return common_chat_combinator_parser(std::make_shared(parsers, counter_->next())); + return common_chat_combinator_parser(std::make_shared(parsers, counter_.next())); } common_chat_combinator_parser common_chat_combinator_parser_builder::choice(std::initializer_list parsers) { - return common_chat_combinator_parser(std::make_shared(parsers, counter_->next())); + return common_chat_combinator_parser(std::make_shared(parsers, counter_.next())); } common_chat_combinator_parser common_chat_combinator_parser_builder::one_or_more(const common_chat_combinator_parser & p) { - return common_chat_combinator_parser(std::make_shared(p, counter_->next())); + return common_chat_combinator_parser(std::make_shared(p, counter_.next())); } common_chat_combinator_parser common_chat_combinator_parser_builder::zero_or_more(const common_chat_combinator_parser & p) { - return common_chat_combinator_parser(std::make_shared(p, counter_->next())); + return common_chat_combinator_parser(std::make_shared(p, counter_.next())); } common_chat_combinator_parser common_chat_combinator_parser_builder::optional(const common_chat_combinator_parser & p) { - return common_chat_combinator_parser(std::make_shared(p, counter_->next())); + return common_chat_combinator_parser(std::make_shared(p, counter_.next())); } common_chat_combinator_parser common_chat_combinator_parser_builder::peek(const common_chat_combinator_parser & p) { - return common_chat_combinator_parser(std::make_shared(p, counter_->next())); + return common_chat_combinator_parser(std::make_shared(p, counter_.next())); } common_chat_combinator_parser common_chat_combinator_parser_builder::negate(const common_chat_combinator_parser & p) { - return common_chat_combinator_parser(std::make_shared(p, counter_->next())); + return common_chat_combinator_parser(std::make_shared(p, counter_.next())); } common_chat_combinator_parser common_chat_combinator_parser_builder::any() { - return common_chat_combinator_parser(std::make_shared(counter_->next())); + return common_chat_combinator_parser(std::make_shared(counter_.next())); } common_chat_combinator_parser common_chat_combinator_parser_builder::chars(const std::string & classes, int min, int max) { - return common_chat_combinator_parser(std::make_shared(classes, min, max, counter_->next())); + return common_chat_combinator_parser(std::make_shared(classes, min, max, counter_.next())); } common_chat_combinator_parser common_chat_combinator_parser_builder::one(const std::string & classes) { @@ -1767,27 +1764,28 @@ common_chat_combinator_parser common_chat_combinator_parser_builder::one(const s } common_chat_combinator_parser common_chat_combinator_parser_builder::json_string_unqouted() { - return common_chat_combinator_parser(std::make_shared(counter_->next())); + return common_chat_combinator_parser(std::make_shared(counter_.next())); } common_chat_combinator_parser common_chat_combinator_parser_builder::rule(const std::string & name) { - return common_chat_combinator_parser(std::make_shared(name, rules_, counter_->next())); + auto root = cast(root_); + return common_chat_combinator_parser(std::make_shared(name, std::weak_ptr(root), counter_.next())); } common_chat_combinator_parser common_chat_combinator_parser_builder::space() { - return common_chat_combinator_parser(std::make_shared(counter_->next())); + return common_chat_combinator_parser(std::make_shared(counter_.next())); } common_chat_combinator_parser common_chat_combinator_parser_builder::until(const std::string & delimiter) { - return common_chat_combinator_parser(std::make_shared(delimiter, counter_->next())); + return common_chat_combinator_parser(std::make_shared(delimiter, counter_.next())); } common_chat_combinator_parser common_chat_combinator_parser_builder::until_one_of(const std::vector & delimiters) { - return common_chat_combinator_parser(std::make_shared(delimiters, counter_->next())); + return common_chat_combinator_parser(std::make_shared(delimiters, counter_.next())); } common_chat_combinator_parser common_chat_combinator_parser_builder::repeat(const common_chat_combinator_parser & p, int min, int max) { - return common_chat_combinator_parser(std::make_shared(p, min, max, counter_->next())); + return common_chat_combinator_parser(std::make_shared(p, min, max, counter_.next())); } common_chat_combinator_parser common_chat_combinator_parser_builder::repeat(const common_chat_combinator_parser & p, int n) { @@ -1795,11 +1793,11 @@ common_chat_combinator_parser common_chat_combinator_parser_builder::repeat(cons } common_chat_combinator_parser common_chat_combinator_parser_builder::schema(const common_chat_combinator_parser & p, const std::string & name, const nlohmann::ordered_json & schema) { - return common_chat_combinator_parser(std::make_shared(p, name, schema, counter_->next())); + return common_chat_combinator_parser(std::make_shared(p, name, schema, counter_.next())); } common_chat_combinator_parser common_chat_combinator_parser_builder::action(const common_chat_combinator_parser & p, std::function fn, int when) { - return common_chat_combinator_parser(std::make_shared(p, std::move(fn), when, counter_->next())); + return common_chat_combinator_parser(std::make_shared(p, std::move(fn), when, counter_.next())); } common_chat_combinator_parser common_chat_combinator_parser_builder::capture(const std::string & key, const common_chat_combinator_parser & p) { @@ -1810,42 +1808,46 @@ common_chat_combinator_parser common_chat_combinator_parser_builder::capture(con } common_chat_combinator_parser common_chat_combinator_parser_builder::trigger(const common_chat_combinator_parser & p) { - return common_chat_combinator_parser(std::make_shared(p, counter_->next())); + return common_chat_combinator_parser(std::make_shared(p, counter_.next())); } common_chat_combinator_parser common_chat_combinator_parser_builder::add_rule(const std::string & name, const common_chat_combinator_parser & p) { - (*rules_)[name] = p; + auto root = cast(root_); + root->add_rule(name, p); return rule(name); } common_chat_combinator_parser common_chat_combinator_parser_builder::add_rule(const std::string & name, const std::function & builder) { - if (rules_->find(name) != rules_->end()) { + auto root = cast(root_); + if (root->rules().find(name) != root->rules().end()) { return rule(name); } - (*rules_)[name] = literal(""); // Placeholder + root->add_rule(name, literal("")); // Placeholder auto parser = builder(); - (*rules_)[name] = parser; + root->add_rule(name, parser); return rule(name); } -void common_chat_combinator_parser_builder::assign_ids(common_chat_combinator_parser & p) { +void common_chat_combinator_parser_builder::set_root(const common_chat_combinator_parser & p) { + auto root_container = cast(root_); + root_container->set_root(p); + + // Recursively issue IDs to reachable nodes if (p.ptr()) { p.ptr()->assign_id(counter_); } } +common_chat_combinator_parser common_chat_combinator_parser_builder::build() { + return root_; +} + common_chat_combinator_parser build_combinator_parser(const std::function & fn) { common_chat_combinator_parser_builder builder; auto root = fn(builder); - builder.assign_ids(root); // Assign IDs to rules that were created with operators - - // Wrap the root parser in a root_parser to own the rules and break circular references - auto rules = builder.rules(); - if (rules && !rules->empty()) { - return common_chat_combinator_parser(std::make_shared(root, rules, -1)); - } - return root; + builder.set_root(root); + return builder.build(); } common_chat_combinator_parser common_chat_combinator_parser_builder::json_number() { diff --git a/common/chat-parser-combinator.h b/common/chat-parser-combinator.h index a9e06a754439c..4d9eabe746ec1 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-parser-combinator.h @@ -179,12 +179,11 @@ class common_chat_combinator_parser_counter { }; class common_chat_combinator_parser_builder { - std::shared_ptr> rules_; - std::shared_ptr counter_; + common_chat_combinator_parser root_; + common_chat_combinator_parser_counter counter_; public: common_chat_combinator_parser_builder(); - common_chat_combinator_parser_builder(std::shared_ptr counter); // Matches an exact literal string. // S -> "hello" @@ -297,9 +296,9 @@ class common_chat_combinator_parser_builder { // add_rule("json", [&]() { return json_object() | json_array() | ... }) common_chat_combinator_parser add_rule(const std::string & name, const std::function & builder); - void assign_ids(common_chat_combinator_parser & p); + void set_root(const common_chat_combinator_parser & p); - std::shared_ptr> rules() const { return rules_; } + common_chat_combinator_parser build(); }; common_chat_combinator_parser build_combinator_parser(const std::function & fn); From 4228d116da0199eac38480b8c08ef2acd1d3f5ac Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Fri, 14 Nov 2025 18:46:32 -0600 Subject: [PATCH 057/183] print out grammars in example --- tests/test-chat-parser-combinator.cpp | 34 +++++++++++++-------------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/tests/test-chat-parser-combinator.cpp b/tests/test-chat-parser-combinator.cpp index d64ab2f5c232b..c176e90058535 100644 --- a/tests/test-chat-parser-combinator.cpp +++ b/tests/test-chat-parser-combinator.cpp @@ -774,12 +774,24 @@ static void example_qwen3_coder() { + p.one_or_more(json_arg | string_arg) + ""); - auto tool_call = p.add_rule("tool-call", - "" + p.one_or_more(function) + ""); + auto tool_call = p.trigger(p.add_rule("tool-call", + "" + p.one_or_more(function) + "")); return thinking + p.optional(p.space() + content) + p.zero_or_more(p.space() + tool_call); }); + std::cout << "Grammar (lazy=false):\n"; + auto grammar = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); + std::cout << grammar << "\n"; + + std::cout << "Grammar (lazy=true):\n"; + auto lazy_grammar = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder, true); + }); + std::cout << lazy_grammar << "\n"; + auto handler = [&](const common_chat_parse_event & ev, common_chat_parse_semantics & env) { if (ev.rule == "reasoning-content" && ev.ending()) { env.result.reasoning_content = ev.text; @@ -857,26 +869,13 @@ static void example_qwen3_coder() { auto parse_result = parser.parse(ctx); assert_equals(false, parse_result.fail()); - std::cout << "=================================\n"; - std::cout << in << "\n\n"; - /* - std::cout << "Reasoning: " << prev.reasoning_content << "\n"; - std::cout << "Content : " << prev.content << "\n"; - if (!prev.tool_calls.empty()) { - std::cout << "\n=== Tool Calls ===\n"; - for (const auto & tc : prev.tool_calls) { - std::cout << "ID : " << tc.id << "\n"; - std::cout << "Name: " << tc.name << "\n"; - std::cout << "Args: " << tc.arguments << "\n"; - } - } - */ - // This shouldn't emit any runtime errors auto diffs = common_chat_msg_diff::compute_diffs(prev, env.result); prev = env.result; #if 0 + std::cout << "=================================\n"; + std::cout << in << "\n\n"; std::cout << "----\n"; std::cout << "Reasoning: " << prev.reasoning_content << "\n"; std::cout << "Content : " << prev.content << "\n"; @@ -1128,7 +1127,6 @@ int main() { std::cout << "All tests passed!\n"; example_qwen3_coder(); - //return 0; std::cout << "\n== Benchmarks ==\n"; std::string example_reasoning = From ea519ca818eae4d3f895e25eb09af2ef7b2ceace Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 15 Nov 2025 01:55:17 -0600 Subject: [PATCH 058/183] rename to chat-peg-parser* and common_chat_peg_parser* --- common/CMakeLists.txt | 4 +- ...ser-combinator.cpp => chat-peg-parser.cpp} | 332 +++++++++--------- ...-parser-combinator.h => chat-peg-parser.h} | 118 +++---- tests/.gitignore | 2 +- tests/CMakeLists.txt | 30 +- .../benchmark.cpp | 0 .../simple_tokenizer.cpp | 0 .../test-actions.cpp | 8 +- .../test-command7-parser-compare.cpp | 6 +- .../test-example-qwen3-coder.cpp | 2 +- .../test-gbnf-generation.cpp | 26 +- .../test-json-parser.cpp | 12 +- .../test-one.cpp | 16 +- .../test-optional.cpp | 6 +- .../test-partial-parsing.cpp | 46 +-- .../test-recursive-references.cpp | 12 +- tests/{combinator => chat-peg-parser}/tests.h | 10 +- ...ombinator.cpp => test-chat-peg-parser.cpp} | 2 +- 18 files changed, 321 insertions(+), 311 deletions(-) rename common/{chat-parser-combinator.cpp => chat-peg-parser.cpp} (77%) rename common/{chat-parser-combinator.h => chat-peg-parser.h} (65%) rename tests/{combinator => chat-peg-parser}/benchmark.cpp (100%) rename tests/{combinator => chat-peg-parser}/simple_tokenizer.cpp (100%) rename tests/{combinator => chat-peg-parser}/test-actions.cpp (91%) rename tests/{combinator => chat-peg-parser}/test-command7-parser-compare.cpp (98%) rename tests/{combinator => chat-peg-parser}/test-example-qwen3-coder.cpp (98%) rename tests/{combinator => chat-peg-parser}/test-gbnf-generation.cpp (75%) rename tests/{combinator => chat-peg-parser}/test-json-parser.cpp (80%) rename tests/{combinator => chat-peg-parser}/test-one.cpp (74%) rename tests/{combinator => chat-peg-parser}/test-optional.cpp (74%) rename tests/{combinator => chat-peg-parser}/test-partial-parsing.cpp (71%) rename tests/{combinator => chat-peg-parser}/test-recursive-references.cpp (84%) rename tests/{combinator => chat-peg-parser}/tests.h (86%) rename tests/{test-chat-parser-combinator.cpp => test-chat-peg-parser.cpp} (95%) diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 7bdc9aab5995f..7062cc29a7064 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -48,10 +48,10 @@ add_library(${TARGET} STATIC arg.cpp arg.h base64.hpp - chat-parser-combinator.cpp - chat-parser-combinator.h chat-parser.cpp chat-parser.h + chat-peg-parser.cpp + chat-peg-parser.h chat.cpp chat.h common.cpp diff --git a/common/chat-parser-combinator.cpp b/common/chat-peg-parser.cpp similarity index 77% rename from common/chat-parser-combinator.cpp rename to common/chat-peg-parser.cpp index 4346751d62ba8..15b97bbe8051e 100644 --- a/common/chat-parser-combinator.cpp +++ b/common/chat-peg-parser.cpp @@ -1,4 +1,4 @@ -#include "chat-parser-combinator.h" +#include "chat-peg-parser.h" #include "json-schema-to-grammar.h" #include "common.h" #include "log.h" @@ -34,13 +34,13 @@ enum parser_type { class parser_visitor; -class common_chat_combinator_parser_base { +class common_chat_peg_parser_base { protected: int id_; public: - common_chat_combinator_parser_base(int id) : id_(id) {} - virtual ~common_chat_combinator_parser_base() = default; + common_chat_peg_parser_base(int id) : id_(id) {} + virtual ~common_chat_peg_parser_base() = default; int id() const { return id_; } void set_id(int id) { id_ = id; } @@ -68,7 +68,7 @@ class common_chat_combinator_parser_base { // Actual parsing implementation (to be overridden by subclasses) virtual common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) = 0; - virtual void assign_id(common_chat_combinator_parser_counter & counter) { + virtual void assign_id(common_chat_peg_parser_counter & counter) { if (id_ == -1) { id_ = counter.next(); } @@ -80,7 +80,7 @@ class common_chat_combinator_parser_base { // Convenience cast functions template -static std::shared_ptr cast(const std::shared_ptr & p) { +static std::shared_ptr cast(const std::shared_ptr & p) { if (p->type() != T::type_value) { return nullptr; } @@ -88,7 +88,7 @@ static std::shared_ptr cast(const std::shared_ptr -static std::shared_ptr cast(const common_chat_combinator_parser & p) { +static std::shared_ptr cast(const common_chat_peg_parser & p) { return cast(p.ptr()); } @@ -322,14 +322,14 @@ static std::string regex_excluding_pattern(const std::vector & stri // Container for the root parser and all named rules in the grammar. // Manages ownership of rule registry to enable recursive grammar definitions. -class root_parser : public common_chat_combinator_parser_base { - common_chat_combinator_parser root_; - std::unordered_map rules_; +class root_parser : public common_chat_peg_parser_base { + common_chat_peg_parser root_; + std::unordered_map rules_; public: static constexpr parser_type type_value = ROOT; - root_parser(int id) : common_chat_combinator_parser_base(id) {} + root_parser(int id) : common_chat_peg_parser_base(id) {} parser_type type() const override { return type_value; } @@ -337,8 +337,8 @@ class root_parser : public common_chat_combinator_parser_base { return root_->parse(ctx, start); } - void assign_id(common_chat_combinator_parser_counter & counter) override { - common_chat_combinator_parser_base::assign_id(counter); + void assign_id(common_chat_peg_parser_counter & counter) override { + common_chat_peg_parser_base::assign_id(counter); root_->assign_id(counter); } @@ -348,29 +348,29 @@ class root_parser : public common_chat_combinator_parser_base { void accept(parser_visitor & visitor) override; - void add_rule(const std::string & name, const common_chat_combinator_parser & parser) { + void add_rule(const std::string & name, const common_chat_peg_parser & parser) { rules_[name] = parser; } - void set_root(const common_chat_combinator_parser & parser) { + void set_root(const common_chat_peg_parser & parser) { root_ = parser; } - const common_chat_combinator_parser & root() const { return root_; } + const common_chat_peg_parser & root() const { return root_; } - std::unordered_map & rules() { return rules_; } - const std::unordered_map & rules() const { return rules_; } + std::unordered_map & rules() { return rules_; } + const std::unordered_map & rules() const { return rules_; } }; // Matches an exact literal string. // S -> "hello" -class literal_parser : public common_chat_combinator_parser_base { +class literal_parser : public common_chat_peg_parser_base { std::string literal_; public: static constexpr parser_type type_value = LITERAL; - literal_parser(const std::string & literal, int id) : common_chat_combinator_parser_base(id), literal_(literal) {} + literal_parser(const std::string & literal, int id) : common_chat_peg_parser_base(id), literal_(literal) {} parser_type type() const override { return type_value; } @@ -403,13 +403,13 @@ class literal_parser : public common_chat_combinator_parser_base { // Matches a sequence of parsers in order, all must succeed. // S -> A B C -class sequence_parser : public common_chat_combinator_parser_base { - std::vector parsers_; +class sequence_parser : public common_chat_peg_parser_base { + std::vector parsers_; public: static constexpr parser_type type_value = SEQUENCE; - sequence_parser(std::initializer_list parsers, int id) : common_chat_combinator_parser_base(id) { + sequence_parser(std::initializer_list parsers, int id) : common_chat_peg_parser_base(id) { for (const auto & p : parsers) { if (auto seq = cast(p)) { for (const auto & embedded : seq->parsers()) { @@ -437,8 +437,8 @@ class sequence_parser : public common_chat_combinator_parser_base { return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); } - void assign_id(common_chat_combinator_parser_counter & counter) override { - common_chat_combinator_parser_base::assign_id(counter); + void assign_id(common_chat_peg_parser_counter & counter) override { + common_chat_peg_parser_base::assign_id(counter); for (auto & p : parsers_) { p->assign_id(counter); } @@ -455,18 +455,18 @@ class sequence_parser : public common_chat_combinator_parser_base { void accept(parser_visitor & visitor) override; - const std::vector & parsers() const { return parsers_; } + const std::vector & parsers() const { return parsers_; } }; // Matches the first parser that succeeds from a list of alternatives. // S -> A | B | C -class choice_parser : public common_chat_combinator_parser_base { - std::vector parsers_; +class choice_parser : public common_chat_peg_parser_base { + std::vector parsers_; public: static constexpr parser_type type_value = CHOICE; - choice_parser(std::initializer_list parsers, int id) : common_chat_combinator_parser_base(id) { + choice_parser(std::initializer_list parsers, int id) : common_chat_peg_parser_base(id) { for (const auto & p : parsers) { if (auto choice = cast(p)) { for (const auto & embedded : choice->parsers()) { @@ -492,8 +492,8 @@ class choice_parser : public common_chat_combinator_parser_base { return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); } - void assign_id(common_chat_combinator_parser_counter & counter) override { - common_chat_combinator_parser_base::assign_id(counter); + void assign_id(common_chat_peg_parser_counter & counter) override { + common_chat_peg_parser_base::assign_id(counter); for (auto & p : parsers_) { p->assign_id(counter); } @@ -510,22 +510,22 @@ class choice_parser : public common_chat_combinator_parser_base { void accept(parser_visitor & visitor) override; - const std::vector & parsers() const { return parsers_; } + const std::vector & parsers() const { return parsers_; } }; // Matches between min and max repetitions of a parser (inclusive). // S -> A{m,n} // Use -1 for max_count to represent unbounded repetition (equivalent to {m,}) -class repetition_parser : public common_chat_combinator_parser_base { - common_chat_combinator_parser parser_; +class repetition_parser : public common_chat_peg_parser_base { + common_chat_peg_parser parser_; int min_count_; int max_count_; public: static constexpr parser_type type_value = REPETITION; - repetition_parser(const common_chat_combinator_parser & parser, int min_count, int max_count, int id) - : common_chat_combinator_parser_base(id), parser_(parser), min_count_(min_count), max_count_(max_count) {} + repetition_parser(const common_chat_peg_parser & parser, int min_count, int max_count, int id) + : common_chat_peg_parser_base(id), parser_(parser), min_count_(min_count), max_count_(max_count) {} parser_type type() const override { return type_value; } @@ -567,8 +567,8 @@ class repetition_parser : public common_chat_combinator_parser_base { return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); } - void assign_id(common_chat_combinator_parser_counter & counter) override { - common_chat_combinator_parser_base::assign_id(counter); + void assign_id(common_chat_peg_parser_counter & counter) override { + common_chat_peg_parser_base::assign_id(counter); parser_->assign_id(counter); } @@ -581,7 +581,7 @@ class repetition_parser : public common_chat_combinator_parser_base { void accept(parser_visitor & visitor) override; - const common_chat_combinator_parser & child() const { return parser_; } + const common_chat_peg_parser & child() const { return parser_; } int min_count() const { return min_count_; } @@ -594,7 +594,7 @@ class one_or_more_parser : public repetition_parser { public: static constexpr parser_type type_value = ONE_OR_MORE; - one_or_more_parser(const common_chat_combinator_parser & p, int id) : repetition_parser(p, 1, -1, id) {} + one_or_more_parser(const common_chat_peg_parser & p, int id) : repetition_parser(p, 1, -1, id) {} parser_type type() const override { return type_value; } @@ -611,7 +611,7 @@ class zero_or_more_parser : public repetition_parser { public: static constexpr parser_type type_value = ZERO_OR_MORE; - zero_or_more_parser(const common_chat_combinator_parser & p, int id) : repetition_parser(p, 0, -1, id) {} + zero_or_more_parser(const common_chat_peg_parser & p, int id) : repetition_parser(p, 0, -1, id) {} parser_type type() const override { return type_value; } @@ -628,7 +628,7 @@ class optional_parser : public repetition_parser { public: static constexpr parser_type type_value = OPTIONAL; - optional_parser(const common_chat_combinator_parser & p, int id) : repetition_parser(p, 0, 1, id) {} + optional_parser(const common_chat_peg_parser & p, int id) : repetition_parser(p, 0, 1, id) {} parser_type type() const override { return type_value; } @@ -641,13 +641,13 @@ class optional_parser : public repetition_parser { // Positive lookahead: succeeds if child parser succeeds, consumes no input. // S -> &A -class and_parser : public common_chat_combinator_parser_base { - common_chat_combinator_parser parser_; +class and_parser : public common_chat_peg_parser_base { + common_chat_peg_parser parser_; public: static constexpr parser_type type_value = AND; - and_parser(const common_chat_combinator_parser & parser, int id) : common_chat_combinator_parser_base(id), parser_(parser) {} + and_parser(const common_chat_peg_parser & parser, int id) : common_chat_peg_parser_base(id), parser_(parser) {} parser_type type() const override { return type_value; } @@ -662,8 +662,8 @@ class and_parser : public common_chat_combinator_parser_base { return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start); } - void assign_id(common_chat_combinator_parser_counter & counter) override { - common_chat_combinator_parser_base::assign_id(counter); + void assign_id(common_chat_peg_parser_counter & counter) override { + common_chat_peg_parser_base::assign_id(counter); parser_->assign_id(counter); } @@ -673,18 +673,18 @@ class and_parser : public common_chat_combinator_parser_base { void accept(parser_visitor & visitor) override; - const common_chat_combinator_parser & child() const { return parser_; } + const common_chat_peg_parser & child() const { return parser_; } }; // Negative lookahead: succeeds if child parser fails, consumes no input. // S -> !A -class not_parser : public common_chat_combinator_parser_base { - common_chat_combinator_parser parser_; +class not_parser : public common_chat_peg_parser_base { + common_chat_peg_parser parser_; public: static constexpr parser_type type_value = NOT; - not_parser(const common_chat_combinator_parser & parser, int id) : common_chat_combinator_parser_base(id), parser_(parser) {} + not_parser(const common_chat_peg_parser & parser, int id) : common_chat_peg_parser_base(id), parser_(parser) {} parser_type type() const override { return type_value; } @@ -705,8 +705,8 @@ class not_parser : public common_chat_combinator_parser_base { return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start); } - void assign_id(common_chat_combinator_parser_counter & counter) override { - common_chat_combinator_parser_base::assign_id(counter); + void assign_id(common_chat_peg_parser_counter & counter) override { + common_chat_peg_parser_base::assign_id(counter); parser_->assign_id(counter); } @@ -716,16 +716,16 @@ class not_parser : public common_chat_combinator_parser_base { void accept(parser_visitor & visitor) override; - const common_chat_combinator_parser & child() const { return parser_; } + const common_chat_peg_parser & child() const { return parser_; } }; // Matches any single character. // S -> . -class any_parser : public common_chat_combinator_parser_base { +class any_parser : public common_chat_peg_parser_base { public: static constexpr parser_type type_value = ANY; - any_parser(int id) : common_chat_combinator_parser_base(id) {} + any_parser(int id) : common_chat_peg_parser_base(id) {} parser_type type() const override { return type_value; } @@ -748,11 +748,11 @@ class any_parser : public common_chat_combinator_parser_base { // Matches zero or more whitespace characters (space, tab, newline). // S -> [ \t\n]* -class space_parser : public common_chat_combinator_parser_base { +class space_parser : public common_chat_peg_parser_base { public: static constexpr parser_type type_value = SPACE; - space_parser(int id) : common_chat_combinator_parser_base(id) {} + space_parser(int id) : common_chat_peg_parser_base(id) {} parser_type type() const override { return type_value; } @@ -779,7 +779,7 @@ class space_parser : public common_chat_combinator_parser_base { // Matches between min and max repetitions of characters from a character class. // S -> [a-z]{m,n} -class chars_parser : public common_chat_combinator_parser_base { +class chars_parser : public common_chat_peg_parser_base { struct char_range { int start; int end; @@ -795,7 +795,7 @@ class chars_parser : public common_chat_combinator_parser_base { public: chars_parser(const std::string & classes, int min_count, int max_count, int id) - : common_chat_combinator_parser_base(id), pattern_(classes), negated_(false), min_count_(min_count), max_count_(max_count) { + : common_chat_peg_parser_base(id), pattern_(classes), negated_(false), min_count_(min_count), max_count_(max_count) { std::string content = classes; if (content.front() == '[') { @@ -912,12 +912,12 @@ class chars_parser : public common_chat_combinator_parser_base { // Stops before the closing quote (doesn't consume it). // Handles escape sequences and emits NEED_MORE_INPUT for incomplete input. // S -> (regular chars and escape sequences)* until closing " -class json_string_parser : public common_chat_combinator_parser_base { +class json_string_parser : public common_chat_peg_parser_base { public: static constexpr parser_type type_value = JSON_STRING; - json_string_parser(int id) : common_chat_combinator_parser_base(id) {} + json_string_parser(int id) : common_chat_peg_parser_base(id) {} parser_type type() const override { return type_value; } @@ -1002,7 +1002,7 @@ class json_string_parser : public common_chat_combinator_parser_base { // Matches all characters until a delimiter is found (delimiter not consumed). // S -> (!delim .)* -class until_parser : public common_chat_combinator_parser_base { +class until_parser : public common_chat_peg_parser_base { std::vector delimiters_; aho_corasick_matcher matcher_; @@ -1010,7 +1010,7 @@ class until_parser : public common_chat_combinator_parser_base { static constexpr parser_type type_value = UNTIL; until_parser(const std::vector & delimiters, int id) - : common_chat_combinator_parser_base(id), delimiters_(delimiters), matcher_(delimiters) {} + : common_chat_peg_parser_base(id), delimiters_(delimiters), matcher_(delimiters) {} until_parser(const std::string & delimiter, int id) : until_parser(std::vector{delimiter}, id) {} @@ -1033,16 +1033,16 @@ class until_parser : public common_chat_combinator_parser_base { // Wraps a parser with JSON schema metadata for grammar generation. // Used internally to convert JSON schemas to GBNF grammar rules. -class schema_parser : public common_chat_combinator_parser_base { - common_chat_combinator_parser parser_; +class schema_parser : public common_chat_peg_parser_base { + common_chat_peg_parser parser_; std::string name_; nlohmann::ordered_json schema_; public: static constexpr parser_type type_value = SCHEMA; - schema_parser(const common_chat_combinator_parser & parser, const std::string & name, const nlohmann::ordered_json & schema, int id) - : common_chat_combinator_parser_base(id), parser_(parser), name_(name), schema_(schema) {} + schema_parser(const common_chat_peg_parser & parser, const std::string & name, const nlohmann::ordered_json & schema, int id) + : common_chat_peg_parser_base(id), parser_(parser), name_(name), schema_(schema) {} parser_type type() const override { return type_value; } @@ -1056,7 +1056,7 @@ class schema_parser : public common_chat_combinator_parser_base { void accept(parser_visitor & visitor) override; - const common_chat_combinator_parser & child() const { return parser_; } + const common_chat_peg_parser & child() const { return parser_; } const std::string & name() const { return name_; } @@ -1065,7 +1065,7 @@ class schema_parser : public common_chat_combinator_parser_base { // References a named rule for recursive or reusable grammar definitions. // expr -> term | expr "+" term -class rule_parser : public common_chat_combinator_parser_base { +class rule_parser : public common_chat_peg_parser_base { std::string name_; std::weak_ptr root_; @@ -1073,7 +1073,7 @@ class rule_parser : public common_chat_combinator_parser_base { static constexpr parser_type type_value = RULE; rule_parser(const std::string & name, const std::weak_ptr & root, int id) - : common_chat_combinator_parser_base(id), name_(name), root_(root) {} + : common_chat_peg_parser_base(id), name_(name), root_(root) {} parser_type type() const override { return type_value; } @@ -1141,8 +1141,8 @@ class rule_parser : public common_chat_combinator_parser_base { }; // Wraps a parser with a semantic action callback. -class action_parser : public common_chat_combinator_parser_base { - common_chat_combinator_parser parser_; +class action_parser : public common_chat_peg_parser_base { + common_chat_peg_parser parser_; std::function action_; int when_; @@ -1150,11 +1150,11 @@ class action_parser : public common_chat_combinator_parser_base { static constexpr parser_type type_value = ACTION; action_parser( - const common_chat_combinator_parser & parser, + const common_chat_peg_parser & parser, std::function action, int when, int id - ) : common_chat_combinator_parser_base(id), parser_(parser), action_(std::move(action)), when_(when) {} + ) : common_chat_peg_parser_base(id), parser_(parser), action_(std::move(action)), when_(when) {} parser_type type() const override { return type_value; } @@ -1174,8 +1174,8 @@ class action_parser : public common_chat_combinator_parser_base { return result; } - void assign_id(common_chat_combinator_parser_counter & counter) override { - common_chat_combinator_parser_base::assign_id(counter); + void assign_id(common_chat_peg_parser_counter & counter) override { + common_chat_peg_parser_base::assign_id(counter); parser_->assign_id(counter); } @@ -1185,20 +1185,20 @@ class action_parser : public common_chat_combinator_parser_base { void accept(parser_visitor & visitor) override; - const common_chat_combinator_parser & child() const { return parser_; } + const common_chat_peg_parser & child() const { return parser_; } }; // Annotate nodes for use when generating lazy GBNF grammar rules. When built // with lazy = true, only grammar rules reachable from trigger nodes are // emitted. -class trigger_parser : public common_chat_combinator_parser_base { - common_chat_combinator_parser parser_; +class trigger_parser : public common_chat_peg_parser_base { + common_chat_peg_parser parser_; public: static constexpr parser_type type_value = TRIGGER; - trigger_parser(const common_chat_combinator_parser & parser, int id) - : common_chat_combinator_parser_base(id), parser_(parser) {} + trigger_parser(const common_chat_peg_parser & parser, int id) + : common_chat_peg_parser_base(id), parser_(parser) {} parser_type type() const override { return type_value; } @@ -1206,8 +1206,8 @@ class trigger_parser : public common_chat_combinator_parser_base { return parser_->parse(ctx, start); } - void assign_id(common_chat_combinator_parser_counter & counter) override { - common_chat_combinator_parser_base::assign_id(counter); + void assign_id(common_chat_peg_parser_counter & counter) override { + common_chat_peg_parser_base::assign_id(counter); parser_->assign_id(counter); } @@ -1217,7 +1217,7 @@ class trigger_parser : public common_chat_combinator_parser_base { void accept(parser_visitor & visitor) override; - const common_chat_combinator_parser & child() const { return parser_; } + const common_chat_peg_parser & child() const { return parser_; } }; // Base visitor class for parser tree traversal @@ -1280,12 +1280,12 @@ static std::string gbnf_excluding_pattern(const std::vector & strin // Visitor for collecting reachable rules from a subtree class reachability_visitor : public parser_visitor { std::unordered_set & reachable_rules_; - const std::unordered_map & rules_; + const std::unordered_map & rules_; public: reachability_visitor( std::unordered_set & reachable_rules, - const std::unordered_map & rules + const std::unordered_map & rules ) : reachable_rules_(reachable_rules), rules_(rules) {} void visit(literal_parser &) override {} @@ -1348,7 +1348,7 @@ class gbnf_visitor : public parser_visitor { std::vector trigger_names_; std::unordered_set reachable_rules_; int trigger_counter_; - std::vector> triggers_; + std::vector> triggers_; public: gbnf_visitor(const common_grammar_builder & builder, bool lazy = false) @@ -1364,8 +1364,8 @@ class gbnf_visitor : public parser_visitor { // Collect all reachable rules from the given triggers void collect_reachable_rules( - const std::vector> & triggers, - const std::unordered_map & rules + const std::vector> & triggers, + const std::unordered_map & rules ) { reachable_rules_.clear(); reachability_visitor visitor(reachable_rules_, rules); @@ -1661,52 +1661,52 @@ void common_chat_parse_cache::clear() { results.clear(); } -common_chat_combinator_parser::common_chat_combinator_parser() {} +common_chat_peg_parser::common_chat_peg_parser() {} -common_chat_combinator_parser::common_chat_combinator_parser(std::shared_ptr parser) : ptr_(std::move(parser)) {} +common_chat_peg_parser::common_chat_peg_parser(std::shared_ptr parser) : ptr_(std::move(parser)) {} -common_chat_combinator_parser::common_chat_combinator_parser(const std::string & literal) : ptr_(std::make_shared(literal, -1)) {} +common_chat_peg_parser::common_chat_peg_parser(const std::string & literal) : ptr_(std::make_shared(literal, -1)) {} -common_chat_combinator_parser::common_chat_combinator_parser(const char * literal) : ptr_(std::make_shared(literal, -1)) {} +common_chat_peg_parser::common_chat_peg_parser(const char * literal) : ptr_(std::make_shared(literal, -1)) {} -common_chat_combinator_parser common_chat_combinator_parser::operator~() const { - return common_chat_combinator_parser(std::make_shared(*this, -1)); +common_chat_peg_parser common_chat_peg_parser::operator~() const { + return common_chat_peg_parser(std::make_shared(*this, -1)); } -common_chat_combinator_parser common_chat_combinator_parser::operator+(const common_chat_combinator_parser & other) const { - return common_chat_combinator_parser(std::make_shared(std::initializer_list{*this, other}, -1)); +common_chat_peg_parser common_chat_peg_parser::operator+(const common_chat_peg_parser & other) const { + return common_chat_peg_parser(std::make_shared(std::initializer_list{*this, other}, -1)); } -common_chat_combinator_parser common_chat_combinator_parser::operator|(const common_chat_combinator_parser & other) const { - return common_chat_combinator_parser(std::make_shared(std::initializer_list{*this, other}, -1)); +common_chat_peg_parser common_chat_peg_parser::operator|(const common_chat_peg_parser & other) const { + return common_chat_peg_parser(std::make_shared(std::initializer_list{*this, other}, -1)); } -common_chat_combinator_parser common_chat_combinator_parser::operator<<(const common_chat_combinator_parser & other) const { - auto ws = common_chat_combinator_parser(std::make_shared(-1)); - return common_chat_combinator_parser(std::make_shared(std::initializer_list{*this, ws, other}, -1)); +common_chat_peg_parser common_chat_peg_parser::operator<<(const common_chat_peg_parser & other) const { + auto ws = common_chat_peg_parser(std::make_shared(-1)); + return common_chat_peg_parser(std::make_shared(std::initializer_list{*this, ws, other}, -1)); } -common_chat_combinator_parser operator+(const char * lhs, const common_chat_combinator_parser & rhs) { return common_chat_combinator_parser(lhs) + rhs; } -common_chat_combinator_parser operator|(const char * lhs, const common_chat_combinator_parser & rhs) { return common_chat_combinator_parser(lhs) | rhs; } -common_chat_combinator_parser operator<<(const char * lhs, const common_chat_combinator_parser & rhs) { return common_chat_combinator_parser(lhs) << rhs; } +common_chat_peg_parser operator+(const char * lhs, const common_chat_peg_parser & rhs) { return common_chat_peg_parser(lhs) + rhs; } +common_chat_peg_parser operator|(const char * lhs, const common_chat_peg_parser & rhs) { return common_chat_peg_parser(lhs) | rhs; } +common_chat_peg_parser operator<<(const char * lhs, const common_chat_peg_parser & rhs) { return common_chat_peg_parser(lhs) << rhs; } -common_chat_combinator_parser_base & common_chat_combinator_parser::operator*() const { +common_chat_peg_parser_base & common_chat_peg_parser::operator*() const { return *ptr_; } -common_chat_combinator_parser_base * common_chat_combinator_parser::operator->() const { +common_chat_peg_parser_base * common_chat_peg_parser::operator->() const { return ptr_.get(); } -common_chat_parse_result common_chat_combinator_parser::parse(common_chat_parse_context & ctx, size_t start) const { +common_chat_parse_result common_chat_peg_parser::parse(common_chat_parse_context & ctx, size_t start) const { return ptr_->parse(ctx, start); } -std::string common_chat_combinator_parser::dump() const { +std::string common_chat_peg_parser::dump() const { return ptr_->dump(); } -void common_chat_combinator_parser::build_grammar(const common_grammar_builder & builder, bool lazy) const { +void common_chat_peg_parser::build_grammar(const common_grammar_builder & builder, bool lazy) const { gbnf_visitor visitor(builder, lazy); ptr_->accept(visitor); auto result = visitor.result(); @@ -1715,109 +1715,109 @@ void common_chat_combinator_parser::build_grammar(const common_grammar_builder & } } -common_chat_combinator_parser_builder::common_chat_combinator_parser_builder() +common_chat_peg_parser_builder::common_chat_peg_parser_builder() : root_(std::make_shared(0)) // root parser has id 0 , counter_(1) {} -common_chat_combinator_parser common_chat_combinator_parser_builder::literal(const std::string & literal) { - return common_chat_combinator_parser(std::make_shared(literal, counter_.next())); +common_chat_peg_parser common_chat_peg_parser_builder::literal(const std::string & literal) { + return common_chat_peg_parser(std::make_shared(literal, counter_.next())); } -common_chat_combinator_parser common_chat_combinator_parser_builder::sequence(std::initializer_list parsers) { - return common_chat_combinator_parser(std::make_shared(parsers, counter_.next())); +common_chat_peg_parser common_chat_peg_parser_builder::sequence(std::initializer_list parsers) { + return common_chat_peg_parser(std::make_shared(parsers, counter_.next())); } -common_chat_combinator_parser common_chat_combinator_parser_builder::choice(std::initializer_list parsers) { - return common_chat_combinator_parser(std::make_shared(parsers, counter_.next())); +common_chat_peg_parser common_chat_peg_parser_builder::choice(std::initializer_list parsers) { + return common_chat_peg_parser(std::make_shared(parsers, counter_.next())); } -common_chat_combinator_parser common_chat_combinator_parser_builder::one_or_more(const common_chat_combinator_parser & p) { - return common_chat_combinator_parser(std::make_shared(p, counter_.next())); +common_chat_peg_parser common_chat_peg_parser_builder::one_or_more(const common_chat_peg_parser & p) { + return common_chat_peg_parser(std::make_shared(p, counter_.next())); } -common_chat_combinator_parser common_chat_combinator_parser_builder::zero_or_more(const common_chat_combinator_parser & p) { - return common_chat_combinator_parser(std::make_shared(p, counter_.next())); +common_chat_peg_parser common_chat_peg_parser_builder::zero_or_more(const common_chat_peg_parser & p) { + return common_chat_peg_parser(std::make_shared(p, counter_.next())); } -common_chat_combinator_parser common_chat_combinator_parser_builder::optional(const common_chat_combinator_parser & p) { - return common_chat_combinator_parser(std::make_shared(p, counter_.next())); +common_chat_peg_parser common_chat_peg_parser_builder::optional(const common_chat_peg_parser & p) { + return common_chat_peg_parser(std::make_shared(p, counter_.next())); } -common_chat_combinator_parser common_chat_combinator_parser_builder::peek(const common_chat_combinator_parser & p) { - return common_chat_combinator_parser(std::make_shared(p, counter_.next())); +common_chat_peg_parser common_chat_peg_parser_builder::peek(const common_chat_peg_parser & p) { + return common_chat_peg_parser(std::make_shared(p, counter_.next())); } -common_chat_combinator_parser common_chat_combinator_parser_builder::negate(const common_chat_combinator_parser & p) { - return common_chat_combinator_parser(std::make_shared(p, counter_.next())); +common_chat_peg_parser common_chat_peg_parser_builder::negate(const common_chat_peg_parser & p) { + return common_chat_peg_parser(std::make_shared(p, counter_.next())); } -common_chat_combinator_parser common_chat_combinator_parser_builder::any() { - return common_chat_combinator_parser(std::make_shared(counter_.next())); +common_chat_peg_parser common_chat_peg_parser_builder::any() { + return common_chat_peg_parser(std::make_shared(counter_.next())); } -common_chat_combinator_parser common_chat_combinator_parser_builder::chars(const std::string & classes, int min, int max) { - return common_chat_combinator_parser(std::make_shared(classes, min, max, counter_.next())); +common_chat_peg_parser common_chat_peg_parser_builder::chars(const std::string & classes, int min, int max) { + return common_chat_peg_parser(std::make_shared(classes, min, max, counter_.next())); } -common_chat_combinator_parser common_chat_combinator_parser_builder::one(const std::string & classes) { +common_chat_peg_parser common_chat_peg_parser_builder::one(const std::string & classes) { return chars(classes, 1, 1); } -common_chat_combinator_parser common_chat_combinator_parser_builder::json_string_unqouted() { - return common_chat_combinator_parser(std::make_shared(counter_.next())); +common_chat_peg_parser common_chat_peg_parser_builder::json_string_unqouted() { + return common_chat_peg_parser(std::make_shared(counter_.next())); } -common_chat_combinator_parser common_chat_combinator_parser_builder::rule(const std::string & name) { +common_chat_peg_parser common_chat_peg_parser_builder::rule(const std::string & name) { auto root = cast(root_); - return common_chat_combinator_parser(std::make_shared(name, std::weak_ptr(root), counter_.next())); + return common_chat_peg_parser(std::make_shared(name, std::weak_ptr(root), counter_.next())); } -common_chat_combinator_parser common_chat_combinator_parser_builder::space() { - return common_chat_combinator_parser(std::make_shared(counter_.next())); +common_chat_peg_parser common_chat_peg_parser_builder::space() { + return common_chat_peg_parser(std::make_shared(counter_.next())); } -common_chat_combinator_parser common_chat_combinator_parser_builder::until(const std::string & delimiter) { - return common_chat_combinator_parser(std::make_shared(delimiter, counter_.next())); +common_chat_peg_parser common_chat_peg_parser_builder::until(const std::string & delimiter) { + return common_chat_peg_parser(std::make_shared(delimiter, counter_.next())); } -common_chat_combinator_parser common_chat_combinator_parser_builder::until_one_of(const std::vector & delimiters) { - return common_chat_combinator_parser(std::make_shared(delimiters, counter_.next())); +common_chat_peg_parser common_chat_peg_parser_builder::until_one_of(const std::vector & delimiters) { + return common_chat_peg_parser(std::make_shared(delimiters, counter_.next())); } -common_chat_combinator_parser common_chat_combinator_parser_builder::repeat(const common_chat_combinator_parser & p, int min, int max) { - return common_chat_combinator_parser(std::make_shared(p, min, max, counter_.next())); +common_chat_peg_parser common_chat_peg_parser_builder::repeat(const common_chat_peg_parser & p, int min, int max) { + return common_chat_peg_parser(std::make_shared(p, min, max, counter_.next())); } -common_chat_combinator_parser common_chat_combinator_parser_builder::repeat(const common_chat_combinator_parser & p, int n) { +common_chat_peg_parser common_chat_peg_parser_builder::repeat(const common_chat_peg_parser & p, int n) { return repeat(p, n, n); } -common_chat_combinator_parser common_chat_combinator_parser_builder::schema(const common_chat_combinator_parser & p, const std::string & name, const nlohmann::ordered_json & schema) { - return common_chat_combinator_parser(std::make_shared(p, name, schema, counter_.next())); +common_chat_peg_parser common_chat_peg_parser_builder::schema(const common_chat_peg_parser & p, const std::string & name, const nlohmann::ordered_json & schema) { + return common_chat_peg_parser(std::make_shared(p, name, schema, counter_.next())); } -common_chat_combinator_parser common_chat_combinator_parser_builder::action(const common_chat_combinator_parser & p, std::function fn, int when) { - return common_chat_combinator_parser(std::make_shared(p, std::move(fn), when, counter_.next())); +common_chat_peg_parser common_chat_peg_parser_builder::action(const common_chat_peg_parser & p, std::function fn, int when) { + return common_chat_peg_parser(std::make_shared(p, std::move(fn), when, counter_.next())); } -common_chat_combinator_parser common_chat_combinator_parser_builder::capture(const std::string & key, const common_chat_combinator_parser & p) { +common_chat_peg_parser common_chat_peg_parser_builder::capture(const std::string & key, const common_chat_peg_parser & p) { return action(p, [key](const common_chat_parse_action & act) { std::string value = std::string(act.match); act.env.captures[key] = std::move(value); }, COMMON_CHAT_PARSE_RESULT_SUCCESS); } -common_chat_combinator_parser common_chat_combinator_parser_builder::trigger(const common_chat_combinator_parser & p) { - return common_chat_combinator_parser(std::make_shared(p, counter_.next())); +common_chat_peg_parser common_chat_peg_parser_builder::trigger(const common_chat_peg_parser & p) { + return common_chat_peg_parser(std::make_shared(p, counter_.next())); } -common_chat_combinator_parser common_chat_combinator_parser_builder::add_rule(const std::string & name, const common_chat_combinator_parser & p) { +common_chat_peg_parser common_chat_peg_parser_builder::add_rule(const std::string & name, const common_chat_peg_parser & p) { auto root = cast(root_); root->add_rule(name, p); return rule(name); } -common_chat_combinator_parser common_chat_combinator_parser_builder::add_rule(const std::string & name, const std::function & builder) { +common_chat_peg_parser common_chat_peg_parser_builder::add_rule(const std::string & name, const std::function & builder) { auto root = cast(root_); if (root->rules().find(name) != root->rules().end()) { return rule(name); @@ -1829,7 +1829,7 @@ common_chat_combinator_parser common_chat_combinator_parser_builder::add_rule(co return rule(name); } -void common_chat_combinator_parser_builder::set_root(const common_chat_combinator_parser & p) { +void common_chat_peg_parser_builder::set_root(const common_chat_peg_parser & p) { auto root_container = cast(root_); root_container->set_root(p); @@ -1839,18 +1839,18 @@ void common_chat_combinator_parser_builder::set_root(const common_chat_combinato } } -common_chat_combinator_parser common_chat_combinator_parser_builder::build() { +common_chat_peg_parser common_chat_peg_parser_builder::build() { return root_; } -common_chat_combinator_parser build_combinator_parser(const std::function & fn) { - common_chat_combinator_parser_builder builder; +common_chat_peg_parser build_peg_parser(const std::function & fn) { + common_chat_peg_parser_builder builder; auto root = fn(builder); builder.set_root(root); return builder.build(); } -common_chat_combinator_parser common_chat_combinator_parser_builder::json_number() { +common_chat_peg_parser common_chat_peg_parser_builder::json_number() { return add_rule("json-number", [this]() { auto digit1_9 = chars("[1-9]", 1, 1); auto digits = chars("[0-9]"); @@ -1861,25 +1861,25 @@ common_chat_combinator_parser common_chat_combinator_parser_builder::json_number }); } -common_chat_combinator_parser common_chat_combinator_parser_builder::json_string() { +common_chat_peg_parser common_chat_peg_parser_builder::json_string() { return add_rule("json-string", [this]() { return literal("\"") + json_string_unqouted() + literal("\""); }); } -common_chat_combinator_parser common_chat_combinator_parser_builder::json_bool() { +common_chat_peg_parser common_chat_peg_parser_builder::json_bool() { return add_rule("json-bool", [this]() { return literal("true") | literal("false"); }); } -common_chat_combinator_parser common_chat_combinator_parser_builder::json_null() { +common_chat_peg_parser common_chat_peg_parser_builder::json_null() { return add_rule("json-null", [this]() { return literal("null"); }); } -common_chat_combinator_parser common_chat_combinator_parser_builder::json_object() { +common_chat_peg_parser common_chat_peg_parser_builder::json_object() { return add_rule("json-object", [this]() { auto ws = space(); auto member = json_string() + ws + literal(":") + ws + json(); @@ -1889,7 +1889,7 @@ common_chat_combinator_parser common_chat_combinator_parser_builder::json_object }); } -common_chat_combinator_parser common_chat_combinator_parser_builder::json_array() { +common_chat_peg_parser common_chat_peg_parser_builder::json_array() { return add_rule("json-array", [this]() { auto ws = space(); auto elements = json() + zero_or_more(ws + literal(",") + ws + json()); @@ -1898,7 +1898,7 @@ common_chat_combinator_parser common_chat_combinator_parser_builder::json_array( }); } -common_chat_combinator_parser common_chat_combinator_parser_builder::json() { +common_chat_peg_parser common_chat_peg_parser_builder::json() { return add_rule("json-value", [this]() { return json_object() | json_array() | diff --git a/common/chat-parser-combinator.h b/common/chat-peg-parser.h similarity index 65% rename from common/chat-parser-combinator.h rename to common/chat-peg-parser.h index 4d9eabe746ec1..aa5fd89ddc62c 100644 --- a/common/chat-parser-combinator.h +++ b/common/chat-peg-parser.h @@ -131,34 +131,34 @@ struct common_chat_parse_context { : input(input), cache(), input_is_complete(complete), env(environment), event_handler(std::move(handler)), current_depth(0) {} }; -class common_chat_combinator_parser_base; +class common_chat_peg_parser_base; -class common_chat_combinator_parser { - std::shared_ptr ptr_; +class common_chat_peg_parser { + std::shared_ptr ptr_; public: - common_chat_combinator_parser(); - common_chat_combinator_parser(std::shared_ptr parser); - common_chat_combinator_parser(const common_chat_combinator_parser & other) = default; - common_chat_combinator_parser(const std::string & literal); - common_chat_combinator_parser(const char * literal); + common_chat_peg_parser(); + common_chat_peg_parser(std::shared_ptr parser); + common_chat_peg_parser(const common_chat_peg_parser & other) = default; + common_chat_peg_parser(const std::string & literal); + common_chat_peg_parser(const char * literal); - common_chat_combinator_parser & operator=(const common_chat_combinator_parser & other) { + common_chat_peg_parser & operator=(const common_chat_peg_parser & other) { if (this != &other) { ptr_ = other.ptr_; } return *this; } - common_chat_combinator_parser operator~() const; - common_chat_combinator_parser operator+(const common_chat_combinator_parser & other) const; - common_chat_combinator_parser operator|(const common_chat_combinator_parser & other) const; - common_chat_combinator_parser operator<<(const common_chat_combinator_parser & other) const; + common_chat_peg_parser operator~() const; + common_chat_peg_parser operator+(const common_chat_peg_parser & other) const; + common_chat_peg_parser operator|(const common_chat_peg_parser & other) const; + common_chat_peg_parser operator<<(const common_chat_peg_parser & other) const; - common_chat_combinator_parser_base & operator*() const; - common_chat_combinator_parser_base * operator->() const; + common_chat_peg_parser_base & operator*() const; + common_chat_peg_parser_base * operator->() const; - std::shared_ptr ptr() const { return ptr_; } + std::shared_ptr ptr() const { return ptr_; } common_chat_parse_result parse(common_chat_parse_context & ctx, size_t start = 0) const; @@ -167,138 +167,138 @@ class common_chat_combinator_parser { void build_grammar(const common_grammar_builder & builder, bool lazy = false) const; }; -common_chat_combinator_parser operator+(const char * lhs, const common_chat_combinator_parser & rhs); -common_chat_combinator_parser operator|(const char * lhs, const common_chat_combinator_parser & rhs); -common_chat_combinator_parser operator<<(const char * lhs, const common_chat_combinator_parser & rhs); +common_chat_peg_parser operator+(const char * lhs, const common_chat_peg_parser & rhs); +common_chat_peg_parser operator|(const char * lhs, const common_chat_peg_parser & rhs); +common_chat_peg_parser operator<<(const char * lhs, const common_chat_peg_parser & rhs); -class common_chat_combinator_parser_counter { +class common_chat_peg_parser_counter { int next_id_; public: - common_chat_combinator_parser_counter(int start) : next_id_(start) {} + common_chat_peg_parser_counter(int start) : next_id_(start) {} int next() { return next_id_++; } }; -class common_chat_combinator_parser_builder { - common_chat_combinator_parser root_; - common_chat_combinator_parser_counter counter_; +class common_chat_peg_parser_builder { + common_chat_peg_parser root_; + common_chat_peg_parser_counter counter_; public: - common_chat_combinator_parser_builder(); + common_chat_peg_parser_builder(); // Matches an exact literal string. // S -> "hello" - common_chat_combinator_parser literal(const std::string & literal); + common_chat_peg_parser literal(const std::string & literal); // Matches a sequence of parsers in order, all must succeed. // S -> A B C - common_chat_combinator_parser sequence(std::initializer_list parsers); + common_chat_peg_parser sequence(std::initializer_list parsers); // Matches the first parser that succeeds from a list of alternatives. // S -> A | B | C - common_chat_combinator_parser choice(std::initializer_list parsers); + common_chat_peg_parser choice(std::initializer_list parsers); // Matches one or more repetitions of a parser. // S -> A+ - common_chat_combinator_parser one_or_more(const common_chat_combinator_parser & p); + common_chat_peg_parser one_or_more(const common_chat_peg_parser & p); // Matches zero or more repetitions of a parser, always succeeds. // S -> A* - common_chat_combinator_parser zero_or_more(const common_chat_combinator_parser & p); + common_chat_peg_parser zero_or_more(const common_chat_peg_parser & p); // Matches zero or one occurrence of a parser, always succeeds. // S -> A? - common_chat_combinator_parser optional(const common_chat_combinator_parser & p); + common_chat_peg_parser optional(const common_chat_peg_parser & p); // Negative lookahead: succeeds if child parser fails, consumes no input. // S -> !A - common_chat_combinator_parser peek(const common_chat_combinator_parser & p); + common_chat_peg_parser peek(const common_chat_peg_parser & p); // Negative lookahead: succeeds if child parser fails, consumes no input. // S -> !A - common_chat_combinator_parser negate(const common_chat_combinator_parser & p); + common_chat_peg_parser negate(const common_chat_peg_parser & p); // Matches any single character. // S -> . - common_chat_combinator_parser any(); + common_chat_peg_parser any(); // Matches between min and max repetitions of characters from a character class. // S -> [a-z]{m,n} // // Use -1 for max to represent unbounded repetition (equivalent to {m,}) - common_chat_combinator_parser chars(const std::string & classes, int min = 1, int max = -1); + common_chat_peg_parser chars(const std::string & classes, int min = 1, int max = -1); // Matches a single character from a character class or range. // S -> [a-z] or S -> [^0-9] // // Equivalent to chars(classes, 1, 1) - common_chat_combinator_parser one(const std::string & classes); + common_chat_peg_parser one(const std::string & classes); // References a named rule for recursive or reusable grammar definitions. // expr -> term | expr "+" term - common_chat_combinator_parser rule(const std::string & name); + common_chat_peg_parser rule(const std::string & name); // Matches zero or more whitespace characters (space, tab, newline). // S -> [ \t\n]* - common_chat_combinator_parser space(); + common_chat_peg_parser space(); // Matches all characters until a delimiter is found (delimiter not consumed). // S -> (!delim .)* - common_chat_combinator_parser until(const std::string & delimiter); - common_chat_combinator_parser until_one_of(const std::vector & delimiters); + common_chat_peg_parser until(const std::string & delimiter); + common_chat_peg_parser until_one_of(const std::vector & delimiters); // Matches between min and max repetitions of a parser (inclusive). // S -> A{m,n} // Use -1 for max to represent unbounded repetition (equivalent to {m,}) - common_chat_combinator_parser repeat(const common_chat_combinator_parser & p, int min, int max); + common_chat_peg_parser repeat(const common_chat_peg_parser & p, int min, int max); // Matches exactly n repetitions of a parser. // S -> A{n} - common_chat_combinator_parser repeat(const common_chat_combinator_parser & p, int n); + common_chat_peg_parser repeat(const common_chat_peg_parser & p, int n); // Creates a complete JSON parser supporting objects, arrays, strings, numbers, booleans, and null. // value -> object | array | string | number | true | false | null - common_chat_combinator_parser json(); - common_chat_combinator_parser json_object(); - common_chat_combinator_parser json_string(); - common_chat_combinator_parser json_array(); - common_chat_combinator_parser json_number(); - common_chat_combinator_parser json_bool(); - common_chat_combinator_parser json_null(); + common_chat_peg_parser json(); + common_chat_peg_parser json_object(); + common_chat_peg_parser json_string(); + common_chat_peg_parser json_array(); + common_chat_peg_parser json_number(); + common_chat_peg_parser json_bool(); + common_chat_peg_parser json_null(); // Specialized single-pass JSON string parser with escape sequence handling - common_chat_combinator_parser json_string_unqouted(); + common_chat_peg_parser json_string_unqouted(); // Wraps a parser with JSON schema metadata for grammar generation. // Used internally to convert JSON schemas to GBNF grammar rules. - common_chat_combinator_parser schema(const common_chat_combinator_parser & p, const std::string & name, const nlohmann::ordered_json & schema); + common_chat_peg_parser schema(const common_chat_peg_parser & p, const std::string & name, const nlohmann::ordered_json & schema); // Wraps a parser with a semantic action callback. // The callback is invoked on successful parse with the result, matched text, and environment. // S -> A [action] - common_chat_combinator_parser action(const common_chat_combinator_parser & p, std::function fn, int when = COMMON_CHAT_PARSE_RESULT_SUCCESS); + common_chat_peg_parser action(const common_chat_peg_parser & p, std::function fn, int when = COMMON_CHAT_PARSE_RESULT_SUCCESS); // Captures matched text to env.captures[key] - common_chat_combinator_parser capture(const std::string & key, const common_chat_combinator_parser & p); + common_chat_peg_parser capture(const std::string & key, const common_chat_peg_parser & p); // Mark a node as a trigger for GBNF grammar generartion. This is used for // lazy grammar evaluation by only producing GBNF grammar rules that are // reachable from trigger nodes. // S -> Trigger(A) - common_chat_combinator_parser trigger(const common_chat_combinator_parser & p); + common_chat_peg_parser trigger(const common_chat_peg_parser & p); // Adds a named rule and returns a rule reference. - common_chat_combinator_parser add_rule(const std::string & name, const common_chat_combinator_parser & p); + common_chat_peg_parser add_rule(const std::string & name, const common_chat_peg_parser & p); // Adds a named rule using a function. This handles recursive grammars by // inserting a placeholder rule before invoking the builder, allowing the // builder to reference the rule being defined. Use this when the rule // definition needs to call back to itself (directly or indirectly). // add_rule("json", [&]() { return json_object() | json_array() | ... }) - common_chat_combinator_parser add_rule(const std::string & name, const std::function & builder); + common_chat_peg_parser add_rule(const std::string & name, const std::function & builder); - void set_root(const common_chat_combinator_parser & p); + void set_root(const common_chat_peg_parser & p); - common_chat_combinator_parser build(); + common_chat_peg_parser build(); }; -common_chat_combinator_parser build_combinator_parser(const std::function & fn); +common_chat_peg_parser build_peg_parser(const std::function & fn); diff --git a/tests/.gitignore b/tests/.gitignore index 51b51ee6b29b6..a3b089be53e37 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1,5 +1,5 @@ * -!combinator +!chat-peg-parser !*.* *.o ggml-common.h diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 5b6ea27ef3fe8..4850e16f7e07b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -181,17 +181,27 @@ endif() llama_build_and_test(test-chat-parser.cpp) -# Combinator tests (modular) -file(GLOB_RECURSE COMBINATOR_TEST_SOURCES - combinator/*.cpp - combinator/*.hpp - test-chat-parser-combinator.cpp +# Chat PEG parser tests (modular) +file(GLOB_RECURSE CHAT_PEG_PARSER_TEST_SOURCES + chat-peg-parser/simple_tokenizer.cpp + chat-peg-parser/benchmark.cpp + chat-peg-parser/test-actions.cpp + chat-peg-parser/test-command7-parser-compare.cpp + chat-peg-parser/test-example-qwen3-coder.cpp + chat-peg-parser/test-gbnf-generation.cpp + chat-peg-parser/test-json-parser.cpp + chat-peg-parser/test-one.cpp + chat-peg-parser/test-optional.cpp + chat-peg-parser/test-partial-parsing.cpp + chat-peg-parser/test-recursive-references.cpp + chat-peg-parser/tests.h + test-chat-peg-parser.cpp ) -add_executable(test-chat-parser-combinator ${COMBINATOR_TEST_SOURCES}) -target_link_libraries(test-chat-parser-combinator PRIVATE common) -install(TARGETS test-chat-parser-combinator RUNTIME) -add_test(NAME test-chat-parser-combinator COMMAND test-chat-parser-combinator) -set_property(TEST test-chat-parser-combinator PROPERTY LABELS main) +add_executable(test-chat-peg-parser ${CHAT_PEG_PARSER_TEST_SOURCES}) +target_link_libraries(test-chat-peg-parser PRIVATE common) +install(TARGETS test-chat-peg-parser RUNTIME) +add_test(NAME test-chat-peg-parser COMMAND test-chat-peg-parser) +set_property(TEST test-chat-peg-parser PROPERTY LABELS main) llama_build_and_test(test-chat-template.cpp) llama_build_and_test(test-json-partial.cpp) diff --git a/tests/combinator/benchmark.cpp b/tests/chat-peg-parser/benchmark.cpp similarity index 100% rename from tests/combinator/benchmark.cpp rename to tests/chat-peg-parser/benchmark.cpp diff --git a/tests/combinator/simple_tokenizer.cpp b/tests/chat-peg-parser/simple_tokenizer.cpp similarity index 100% rename from tests/combinator/simple_tokenizer.cpp rename to tests/chat-peg-parser/simple_tokenizer.cpp diff --git a/tests/combinator/test-actions.cpp b/tests/chat-peg-parser/test-actions.cpp similarity index 91% rename from tests/combinator/test-actions.cpp rename to tests/chat-peg-parser/test-actions.cpp index 92a90d14c9a96..9e6966decd835 100644 --- a/tests/combinator/test-actions.cpp +++ b/tests/chat-peg-parser/test-actions.cpp @@ -4,7 +4,7 @@ test_actions::test_actions() : compound_test("test_actions") { // Test simple action - append matched text to content add_test( [](test_harness h) { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { auto word = p.chars("[a-z]+"); return p.action(word, [](const common_chat_parse_action & act) { act.env.result.content += std::string(act.match); }); @@ -22,7 +22,7 @@ test_actions::test_actions() : compound_test("test_actions") { // Test multiple sequential actions - build a sentence add_test( [](test_harness h) { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { auto greeting = p.action(p.literal("hello"), [](const common_chat_parse_action & act) { act.env.result.content += std::string(act.match) + " "; }); @@ -48,7 +48,7 @@ test_actions::test_actions() : compound_test("test_actions") { // Test actions don't run when parse fails add_test( [](test_harness h) { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.action(p.literal("success"), [](const common_chat_parse_action & act) { act.env.result.content = "action_ran"; }); }); @@ -65,7 +65,7 @@ test_actions::test_actions() : compound_test("test_actions") { // Test Actions work with partial parsing add_test( [](test_harness h) { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { auto content = p.action(p.until(""), [](const common_chat_parse_action & act) { act.env.result.content += std::string(act.match); }); diff --git a/tests/combinator/test-command7-parser-compare.cpp b/tests/chat-peg-parser/test-command7-parser-compare.cpp similarity index 98% rename from tests/combinator/test-command7-parser-compare.cpp rename to tests/chat-peg-parser/test-command7-parser-compare.cpp index ff5503e767650..34881262b281d 100644 --- a/tests/combinator/test-command7-parser-compare.cpp +++ b/tests/chat-peg-parser/test-command7-parser-compare.cpp @@ -6,8 +6,8 @@ #include #include -class common_chat_combinator_parser test_command7_parser_compare::create_command_r7b_parser() { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { +class common_chat_peg_parser test_command7_parser_compare::create_command_r7b_parser() { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { auto thinking = p.add_rule("thinking", "<|START_THINKING|>" << p.add_rule("reasoning-content", p.until("<|END_THINKING|>")) << "<|END_THINKING|>"); @@ -186,7 +186,7 @@ void test_command7_parser_compare::run_comparison(int iterations) { std::cout << "Current common_chat_combinator_parser performance: " << t2 << "us (" << (float) t2 / iterations << "us per iteration)\n"; } -void test_command7_parser_compare::test_command_r7b_parser(const class common_chat_combinator_parser & p, +void test_command7_parser_compare::test_command_r7b_parser(const class common_chat_peg_parser & p, const std::string & input, bool need_more_input, bool print_results) { diff --git a/tests/combinator/test-example-qwen3-coder.cpp b/tests/chat-peg-parser/test-example-qwen3-coder.cpp similarity index 98% rename from tests/combinator/test-example-qwen3-coder.cpp rename to tests/chat-peg-parser/test-example-qwen3-coder.cpp index a73b8f66c6d35..bf0b065060d6a 100644 --- a/tests/combinator/test-example-qwen3-coder.cpp +++ b/tests/chat-peg-parser/test-example-qwen3-coder.cpp @@ -4,7 +4,7 @@ #include test_example_qwen3_coder::test_example_qwen3_coder() : compound_test("sample_qwen3_coder_test") { - parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { + parser = build_peg_parser([](common_chat_peg_parser_builder & p) { auto thinking = p.add_rule("raw-reasoning", "" << p.add_rule("reasoning-content", p.until("")) << ""); diff --git a/tests/combinator/test-gbnf-generation.cpp b/tests/chat-peg-parser/test-gbnf-generation.cpp similarity index 75% rename from tests/combinator/test-gbnf-generation.cpp rename to tests/chat-peg-parser/test-gbnf-generation.cpp index dabde1134b1b9..2030b66da4b2f 100644 --- a/tests/combinator/test-gbnf-generation.cpp +++ b/tests/chat-peg-parser/test-gbnf-generation.cpp @@ -5,7 +5,7 @@ test_gbnf_generation::test_gbnf_generation() : compound_test("test_gbnf_generati // Test literal add_test( [](test_harness h) { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.literal("hello"); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello"); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); @@ -17,7 +17,7 @@ test_gbnf_generation::test_gbnf_generation() : compound_test("test_gbnf_generati // Test char class add_test( [](test_harness h) { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.one("[a-z]"); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[a-z]"); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); @@ -28,8 +28,8 @@ test_gbnf_generation::test_gbnf_generation() : compound_test("test_gbnf_generati // Test sequence add_test( [](test_harness h) { - auto parser = build_combinator_parser( - [](common_chat_combinator_parser_builder & p) { return p.literal("hello") + p.literal(" ") + p.literal("world"); }); + auto parser = build_peg_parser( + [](common_chat_peg_parser_builder & p) { return p.literal("hello") + p.literal(" ") + p.literal("world"); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); @@ -41,7 +41,7 @@ test_gbnf_generation::test_gbnf_generation() : compound_test("test_gbnf_generati // Test choice add_test( [](test_harness h) { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.literal("cat") | p.literal("dog"); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("cat") | p.literal("dog"); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); @@ -52,7 +52,7 @@ test_gbnf_generation::test_gbnf_generation() : compound_test("test_gbnf_generati // Test one_or_more add_test( [](test_harness h) { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.one_or_more(p.one("[0-9]")); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one_or_more(p.one("[0-9]")); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); @@ -63,7 +63,7 @@ test_gbnf_generation::test_gbnf_generation() : compound_test("test_gbnf_generati // Test zero_or_more add_test( [](test_harness h) { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.zero_or_more(p.one("[a-z]")); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.zero_or_more(p.one("[a-z]")); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); @@ -75,7 +75,7 @@ test_gbnf_generation::test_gbnf_generation() : compound_test("test_gbnf_generati add_test( [](test_harness h) { auto parser = - build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); }); + build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); @@ -87,7 +87,7 @@ test_gbnf_generation::test_gbnf_generation() : compound_test("test_gbnf_generati // Test until add_test( [](test_harness h) { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.until(""); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.until(""); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); @@ -104,7 +104,7 @@ test_gbnf_generation::test_gbnf_generation() : compound_test("test_gbnf_generati add_test( [](test_harness h) { auto parser = - build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.one_or_more(p.literal("a") | p.literal("b")); }); + build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one_or_more(p.literal("a") | p.literal("b")); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); @@ -115,7 +115,7 @@ test_gbnf_generation::test_gbnf_generation() : compound_test("test_gbnf_generati // Test rule references add_test( [](test_harness h) { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { auto digit = p.add_rule("digit", p.one("[0-9]")); return p.one_or_more(digit); }); @@ -131,7 +131,7 @@ test_gbnf_generation::test_gbnf_generation() : compound_test("test_gbnf_generati // Test escaping in literals add_test( [](test_harness h) { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.literal("hello\nworld\t!"); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello\nworld\t!"); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); @@ -142,7 +142,7 @@ test_gbnf_generation::test_gbnf_generation() : compound_test("test_gbnf_generati // Test operator<< (whitespace insertion) add_test( [](test_harness h) { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.literal("hello") << p.literal("world"); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello") << p.literal("world"); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); diff --git a/tests/combinator/test-json-parser.cpp b/tests/chat-peg-parser/test-json-parser.cpp similarity index 80% rename from tests/combinator/test-json-parser.cpp rename to tests/chat-peg-parser/test-json-parser.cpp index 1e319f8416ed2..5a1b133b5fcdf 100644 --- a/tests/combinator/test-json-parser.cpp +++ b/tests/chat-peg-parser/test-json-parser.cpp @@ -4,7 +4,7 @@ test_json_parser::test_json_parser() : compound_test("test_json_parser") { // Test parsing a simple JSON object add_test( [](test_harness h) { - auto json = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.json(); }); + auto json = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.json(); }); std::string input = R"({"name": "test", "value": 42, "flag": true})"; common_chat_parse_context ctx(input); @@ -19,7 +19,7 @@ test_json_parser::test_json_parser() : compound_test("test_json_parser") { // Test parsing a JSON array with mixed types add_test( [](test_harness h) { - auto json = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.json(); }); + auto json = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.json(); }); std::string input = R"([1, "hello", true, null, 3.14])"; common_chat_parse_context ctx(input); @@ -34,7 +34,7 @@ test_json_parser::test_json_parser() : compound_test("test_json_parser") { // Test parsing nested JSON with objects and arrays add_test( [](test_harness h) { - auto json = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.json(); }); + auto json = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.json(); }); std::string input = R"({"users": [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}], "count": 2, "metadata": {"version": "1.0", "tags": ["admin", "user"]}})"; @@ -50,7 +50,7 @@ test_json_parser::test_json_parser() : compound_test("test_json_parser") { // Test need_more_input() parsing - incomplete object add_test( [](test_harness h) { - auto json = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.json(); }); + auto json = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.json(); }); std::string input = R"({"name": "test", "value": )"; common_chat_parse_context ctx(input, false); @@ -64,7 +64,7 @@ test_json_parser::test_json_parser() : compound_test("test_json_parser") { // Test need_more_input() parsing - incomplete array add_test( [](test_harness h) { - auto json = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.json(); }); + auto json = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.json(); }); std::string input = R"([1, 2, 3, )"; common_chat_parse_context ctx(input, false); @@ -78,7 +78,7 @@ test_json_parser::test_json_parser() : compound_test("test_json_parser") { // Test need_more_input() parsing - incomplete nested structure add_test( [](test_harness h) { - auto json = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.json(); }); + auto json = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.json(); }); std::string input = R"({"data": {"nested": )"; common_chat_parse_context ctx(input, false); diff --git a/tests/combinator/test-one.cpp b/tests/chat-peg-parser/test-one.cpp similarity index 74% rename from tests/combinator/test-one.cpp rename to tests/chat-peg-parser/test-one.cpp index 7fd98074460ee..3c8a110cc3284 100644 --- a/tests/combinator/test-one.cpp +++ b/tests/chat-peg-parser/test-one.cpp @@ -4,7 +4,7 @@ test_one::test_one() : compound_test("test_one") { // Test common escape sequences - newline add_test( [](test_harness h) { - auto common_chat_combinator_parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); + auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); common_chat_parse_context ctx; common_chat_parse_result result; @@ -18,7 +18,7 @@ test_one::test_one() : compound_test("test_one") { // Test common escape sequences - tab add_test( [](test_harness h) { - auto common_chat_combinator_parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); + auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); common_chat_parse_context ctx; common_chat_parse_result result; @@ -32,7 +32,7 @@ test_one::test_one() : compound_test("test_one") { // Test common escape sequences - backslash add_test( [](test_harness h) { - auto common_chat_combinator_parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); + auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); common_chat_parse_context ctx; common_chat_parse_result result; @@ -46,7 +46,7 @@ test_one::test_one() : compound_test("test_one") { // Test common escape sequences - space (should ()) add_test( [](test_harness h) { - auto common_chat_combinator_parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); + auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); common_chat_parse_context ctx; common_chat_parse_result result; @@ -60,7 +60,7 @@ test_one::test_one() : compound_test("test_one") { // Test escaped dash - 'a' should succeed add_test( [](test_harness h) { - auto common_chat_combinator_parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.one("[a\\-z]"); }); + auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[a\\-z]"); }); common_chat_parse_context ctx; common_chat_parse_result result; @@ -74,7 +74,7 @@ test_one::test_one() : compound_test("test_one") { // Test escaped dash - '-' should succeed (literal dash) add_test( [](test_harness h) { - auto common_chat_combinator_parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.one("[a\\-z]"); }); + auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[a\\-z]"); }); common_chat_parse_context ctx; common_chat_parse_result result; @@ -88,7 +88,7 @@ test_one::test_one() : compound_test("test_one") { // Test escaped dash - 'z' should succeed add_test( [](test_harness h) { - auto common_chat_combinator_parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.one("[a\\-z]"); }); + auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[a\\-z]"); }); common_chat_parse_context ctx; common_chat_parse_result result; @@ -102,7 +102,7 @@ test_one::test_one() : compound_test("test_one") { // Test escaped dash - 'b' should NOT match (since \- is literal dash, not range) add_test( [](test_harness h) { - auto common_chat_combinator_parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.one("[a\\-z]"); }); + auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[a\\-z]"); }); common_chat_parse_context ctx; common_chat_parse_result result; diff --git a/tests/combinator/test-optional.cpp b/tests/chat-peg-parser/test-optional.cpp similarity index 74% rename from tests/combinator/test-optional.cpp rename to tests/chat-peg-parser/test-optional.cpp index 2d72b0fe3a31c..f7c794f8886dd 100644 --- a/tests/combinator/test-optional.cpp +++ b/tests/chat-peg-parser/test-optional.cpp @@ -5,7 +5,7 @@ test_optional::test_optional() : compound_test("test_optional") { add_test( [](test_harness h) { auto parser = - build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); }); + build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); }); auto ctx = common_chat_parse_context("hello world"); auto result = parser.parse(ctx); @@ -19,7 +19,7 @@ test_optional::test_optional() : compound_test("test_optional") { add_test( [](test_harness h) { auto parser = - build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); }); + build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); }); auto ctx = common_chat_parse_context("hello", true); auto result = parser.parse(ctx); @@ -33,7 +33,7 @@ test_optional::test_optional() : compound_test("test_optional") { add_test( [](test_harness h) { auto parser = - build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); }); + build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); }); auto ctx = common_chat_parse_context("hello ", false); auto result = parser.parse(ctx); diff --git a/tests/combinator/test-partial-parsing.cpp b/tests/chat-peg-parser/test-partial-parsing.cpp similarity index 71% rename from tests/combinator/test-partial-parsing.cpp rename to tests/chat-peg-parser/test-partial-parsing.cpp index 0cba4a5a5acdf..8fa58c8198e0c 100644 --- a/tests/combinator/test-partial-parsing.cpp +++ b/tests/chat-peg-parser/test-partial-parsing.cpp @@ -4,7 +4,7 @@ test_partial_parsing::test_partial_parsing() : compound_test("test_partial_parsi // Literals - Basic Success add_test( [](test_harness h) { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.literal("hello"); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello"); }); common_chat_parse_context ctx; common_chat_parse_result result; @@ -18,7 +18,7 @@ test_partial_parsing::test_partial_parsing() : compound_test("test_partial_parsi // Char Classes - Basic Lowercase Success add_test( [](test_harness h) { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.one("a-z"); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("a-z"); }); common_chat_parse_context ctx; common_chat_parse_result result; @@ -32,7 +32,7 @@ test_partial_parsing::test_partial_parsing() : compound_test("test_partial_parsi // Char Classes - Uppercase Fail add_test( [](test_harness h) { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.one("a-z"); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("a-z"); }); common_chat_parse_context ctx; common_chat_parse_result result; @@ -46,7 +46,7 @@ test_partial_parsing::test_partial_parsing() : compound_test("test_partial_parsi // Char Classes with Dash - Lowercase Success add_test( [](test_harness h) { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.one("a-z-"); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("a-z-"); }); common_chat_parse_context ctx; common_chat_parse_result result; @@ -60,7 +60,7 @@ test_partial_parsing::test_partial_parsing() : compound_test("test_partial_parsi // Char Classes with Dash - Literal Dash Success add_test( [](test_harness h) { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.one("a-z-"); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("a-z-"); }); common_chat_parse_context ctx; common_chat_parse_result result; @@ -74,7 +74,7 @@ test_partial_parsing::test_partial_parsing() : compound_test("test_partial_parsi // Char Classes with Dash - Uppercase Fail add_test( [](test_harness h) { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.one("a-z-"); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("a-z-"); }); common_chat_parse_context ctx; common_chat_parse_result result; @@ -88,7 +88,7 @@ test_partial_parsing::test_partial_parsing() : compound_test("test_partial_parsi // Sequences - Partial Match 1 add_test( [](test_harness h) { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.literal("") + p.literal(""); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("") + p.literal(""); }); auto ctx = common_chat_parse_context("") + p.literal(""); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("") + p.literal(""); }); auto ctx = common_chat_parse_context("") + p.literal(""); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("") + p.literal(""); }); auto ctx = common_chat_parse_context("I am common_chat_combinator_parser", false); auto result = parser.parse(ctx); @@ -143,7 +143,7 @@ test_partial_parsing::test_partial_parsing() : compound_test("test_partial_parsi // Choices - Partial Match 1 add_test( [](test_harness h) { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.literal("option1") | p.literal("option2"); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("option1") | p.literal("option2"); }); auto ctx = common_chat_parse_context("opt", false); auto result = parser.parse(ctx); @@ -155,7 +155,7 @@ test_partial_parsing::test_partial_parsing() : compound_test("test_partial_parsi add_test( [](test_harness h) { auto parser = - build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.literal("choice_a") | p.literal("choice_b"); }); + build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("choice_a") | p.literal("choice_b"); }); auto ctx = common_chat_parse_context("choice", false); auto result = parser.parse(ctx); @@ -166,7 +166,7 @@ test_partial_parsing::test_partial_parsing() : compound_test("test_partial_parsi // Choices - Full Match 1 add_test( [](test_harness h) { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.literal("first") | p.literal("second"); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("first") | p.literal("second"); }); auto ctx = common_chat_parse_context("first", true); auto result = parser.parse(ctx); @@ -177,7 +177,7 @@ test_partial_parsing::test_partial_parsing() : compound_test("test_partial_parsi // Choices - Full Match 2 add_test( [](test_harness h) { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.literal("alpha") | p.literal("beta"); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("alpha") | p.literal("beta"); }); auto ctx = common_chat_parse_context("beta", true); auto result = parser.parse(ctx); @@ -188,7 +188,7 @@ test_partial_parsing::test_partial_parsing() : compound_test("test_partial_parsi // Choices - No Match add_test( [](test_harness h) { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.literal("good") | p.literal("better"); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("good") | p.literal("better"); }); auto ctx = common_chat_parse_context("best", true); auto result = parser.parse(ctx); @@ -199,7 +199,7 @@ test_partial_parsing::test_partial_parsing() : compound_test("test_partial_parsi // Zero or More - Partial Match 1 add_test( [](test_harness h) { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.zero_or_more(p.literal("ab")); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.zero_or_more(p.literal("ab")); }); auto ctx = common_chat_parse_context("a", false); auto result = parser.parse(ctx); @@ -210,7 +210,7 @@ test_partial_parsing::test_partial_parsing() : compound_test("test_partial_parsi // Zero or More - Partial Match 2 add_test( [](test_harness h) { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.zero_or_more(p.literal("xy")); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.zero_or_more(p.literal("xy")); }); auto ctx = common_chat_parse_context("xyx", false); auto result = parser.parse(ctx); @@ -221,7 +221,7 @@ test_partial_parsing::test_partial_parsing() : compound_test("test_partial_parsi // Zero or More - Full Match add_test( [](test_harness h) { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.zero_or_more(p.literal("test")); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.zero_or_more(p.literal("test")); }); auto ctx = common_chat_parse_context("test", true); auto result = parser.parse(ctx); @@ -232,7 +232,7 @@ test_partial_parsing::test_partial_parsing() : compound_test("test_partial_parsi // One or More - Partial Match 1 add_test( [](test_harness h) { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.one_or_more(p.literal("repeat")); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one_or_more(p.literal("repeat")); }); auto ctx = common_chat_parse_context("rep", false); auto result = parser.parse(ctx); @@ -243,7 +243,7 @@ test_partial_parsing::test_partial_parsing() : compound_test("test_partial_parsi // One or More - Partial Match 2 add_test( [](test_harness h) { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.one_or_more(p.literal("ab")); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one_or_more(p.literal("ab")); }); auto ctx = common_chat_parse_context("aba", false); auto result = parser.parse(ctx); @@ -254,7 +254,7 @@ test_partial_parsing::test_partial_parsing() : compound_test("test_partial_parsi // One or More - Full Match add_test( [](test_harness h) { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.one_or_more(p.literal("single")); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one_or_more(p.literal("single")); }); auto ctx = common_chat_parse_context("single", true); auto result = parser.parse(ctx); @@ -265,7 +265,7 @@ test_partial_parsing::test_partial_parsing() : compound_test("test_partial_parsi // One or More - No Match add_test( [](test_harness h) { - auto parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { return p.one_or_more(p.literal("()")); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one_or_more(p.literal("()")); }); auto ctx = common_chat_parse_context("success", true); auto result = parser.parse(ctx); diff --git a/tests/combinator/test-recursive-references.cpp b/tests/chat-peg-parser/test-recursive-references.cpp similarity index 84% rename from tests/combinator/test-recursive-references.cpp rename to tests/chat-peg-parser/test-recursive-references.cpp index c3b93f3dd27e9..79aa1cf82a4ad 100644 --- a/tests/combinator/test-recursive-references.cpp +++ b/tests/chat-peg-parser/test-recursive-references.cpp @@ -4,7 +4,7 @@ test_recursive_references::test_recursive_references() : compound_test("test_rec // Test simple number add_test( [](test_harness h) { - auto value_parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { + auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { p.add_rule("number", p.one_or_more(p.one("0-9"))); p.add_rule("list", p.sequence({ p.literal("["), p.rule("value"), p.literal("]") })); return p.add_rule("value", p.rule("number") | p.rule("list")); @@ -20,7 +20,7 @@ test_recursive_references::test_recursive_references() : compound_test("test_rec // Test simple list add_test( [](test_harness h) { - auto value_parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { + auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { p.add_rule("number", p.one_or_more(p.one("0-9"))); p.add_rule("list", p.sequence({ p.literal("["), p.rule("value"), p.literal("]") })); return p.add_rule("value", p.rule("number") | p.rule("list")); @@ -36,7 +36,7 @@ test_recursive_references::test_recursive_references() : compound_test("test_rec // Test nested list add_test( [](test_harness h) { - auto value_parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { + auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { p.add_rule("number", p.one_or_more(p.one("0-9"))); p.add_rule("list", p.sequence({ p.literal("["), p.rule("value"), p.literal("]") })); return p.add_rule("value", p.rule("number") | p.rule("list")); @@ -52,7 +52,7 @@ test_recursive_references::test_recursive_references() : compound_test("test_rec // Test deeply nested list add_test( [](test_harness h) { - auto value_parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { + auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { p.add_rule("number", p.one_or_more(p.one("0-9"))); p.add_rule("list", p.sequence({ p.literal("["), p.rule("value"), p.literal("]") })); return p.add_rule("value", p.rule("number") | p.rule("list")); @@ -68,7 +68,7 @@ test_recursive_references::test_recursive_references() : compound_test("test_rec // Test need_more_input match add_test( [](test_harness h) { - auto value_parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { + auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { p.add_rule("number", p.one_or_more(p.one("0-9"))); p.add_rule("list", p.sequence({ p.literal("["), p.rule("value"), p.literal("]") })); return p.add_rule("value", p.rule("number") | p.rule("list")); @@ -84,7 +84,7 @@ test_recursive_references::test_recursive_references() : compound_test("test_rec // Test no match add_test( [](test_harness h) { - auto value_parser = build_combinator_parser([](common_chat_combinator_parser_builder & p) { + auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { p.add_rule("number", p.one_or_more(p.one("0-9"))); p.add_rule("list", p.sequence({ p.literal("["), p.rule("value"), p.literal("]") })); return p.add_rule("value", p.rule("number") | p.rule("list")); diff --git a/tests/combinator/tests.h b/tests/chat-peg-parser/tests.h similarity index 86% rename from tests/combinator/tests.h rename to tests/chat-peg-parser/tests.h index 295c572b0e286..c2db1b96bc7e6 100644 --- a/tests/combinator/tests.h +++ b/tests/chat-peg-parser/tests.h @@ -3,7 +3,7 @@ // Common includes for all test files #include "../testcase.hpp" #include -#include "chat-parser-combinator.h" +#include "chat-peg-parser.h" #include // Test class declarations @@ -63,7 +63,7 @@ class benchmark_test { class test_command7_parser_compare : public uses_simple_tokenizer, public benchmark_test { private: - class common_chat_combinator_parser parser; + class common_chat_peg_parser parser; common_chat_parse_event_handler handler; std::string reasoning; @@ -71,9 +71,9 @@ class test_command7_parser_compare : public uses_simple_tokenizer, public benchm std::vector tool_calls; std::vector tokens; // Helper methods - static class common_chat_combinator_parser create_command_r7b_parser(); + static class common_chat_peg_parser create_command_r7b_parser(); static common_chat_parse_event_handler create_command_r7b_event_handler(); - static void test_command_r7b_parser(const class common_chat_combinator_parser & p, const std::string & input, bool need_more_input, bool print_results = false); + static void test_command_r7b_parser(const class common_chat_peg_parser & p, const std::string & input, bool need_more_input, bool print_results = false); static void test_command_r7b_legacy_parser(const std::string & input, bool need_more_input, bool print_results = false); public: test_command7_parser_compare(); @@ -82,7 +82,7 @@ class test_command7_parser_compare : public uses_simple_tokenizer, public benchm class test_example_qwen3_coder : public uses_simple_tokenizer, public compound_test { private: - class common_chat_combinator_parser parser; + class common_chat_peg_parser parser; public: test_example_qwen3_coder(); }; diff --git a/tests/test-chat-parser-combinator.cpp b/tests/test-chat-peg-parser.cpp similarity index 95% rename from tests/test-chat-parser-combinator.cpp rename to tests/test-chat-peg-parser.cpp index 9b16a422ed1e0..42fd565c080f7 100644 --- a/tests/test-chat-parser-combinator.cpp +++ b/tests/test-chat-peg-parser.cpp @@ -1,4 +1,4 @@ -#include "combinator/tests.h" +#include "chat-peg-parser/tests.h" int main() { test_partial_parsing partial_parsing_test; From 15564f3a9b56b1a21bcb6f20755d6fe0c2e4a31e Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Sat, 15 Nov 2025 18:01:01 +0100 Subject: [PATCH 059/183] Revert unrelated changes --- tests/test-grammar-integration.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test-grammar-integration.cpp b/tests/test-grammar-integration.cpp index a1797ffd25479..82fae671ed00b 100644 --- a/tests/test-grammar-integration.cpp +++ b/tests/test-grammar-integration.cpp @@ -46,7 +46,14 @@ static bool match_string(const std::string & input, llama_grammar * grammar) { } } - return std::any_of(stacks_cur.begin(), stacks_cur.end(), [](auto stack) { return stack.empty(); }); + for (const auto & stack : stacks_cur) { + if (stack.empty()) { + // An empty stack means that the grammar has been completed + return true; + } + } + + return false; } static void test(const std::string & test_desc, const std::string & grammar_str, const std::vector & passing_strings, const std::vector & failing_strings) { From 6dd6cee571c3d11b5e739ceb2ee294c73602e310 Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Sat, 15 Nov 2025 19:25:59 +0100 Subject: [PATCH 060/183] New macros for CMakeLists to enable multi-file compilations --- tests/CMakeLists.txt | 172 +++++++++++++++++----------------- tests/chat-peg-parser/tests.h | 2 +- 2 files changed, 86 insertions(+), 88 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4850e16f7e07b..d0bd827ad3c74 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,73 +1,89 @@ llama_add_compile_flags() -function(llama_build source) - if (DEFINED LLAMA_TEST_NAME) - set(TEST_TARGET ${LLAMA_TEST_NAME}) - else() - get_filename_component(TEST_TARGET ${source} NAME_WE) - endif() - - add_executable(${TEST_TARGET} ${source}) - target_link_libraries(${TEST_TARGET} PRIVATE common) - install(TARGETS ${TEST_TARGET} RUNTIME) -endfunction() - -function(llama_test target) +# Helper function: parse test arguments and set default values +function(_llama_parse_test_args target_or_source) include(CMakeParseArguments) set(options) set(oneValueArgs NAME LABEL WORKING_DIRECTORY) - set(multiValueArgs ARGS) + set(multiValueArgs ARGS SOURCES) cmake_parse_arguments(LLAMA_TEST "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + # Set default values if (NOT DEFINED LLAMA_TEST_LABEL) - set(LLAMA_TEST_LABEL "main") + set(LLAMA_TEST_LABEL "main" PARENT_SCOPE) endif() if (NOT DEFINED LLAMA_TEST_WORKING_DIRECTORY) - set(LLAMA_TEST_WORKING_DIRECTORY .) + set(LLAMA_TEST_WORKING_DIRECTORY "." PARENT_SCOPE) endif() + + # Set test name and target if (DEFINED LLAMA_TEST_NAME) - set(TEST_NAME ${LLAMA_TEST_NAME}) + set(TEST_NAME ${LLAMA_TEST_NAME} PARENT_SCOPE) + set(TEST_TARGET ${LLAMA_TEST_NAME} PARENT_SCOPE) else() - set(TEST_NAME ${target}) + get_filename_component(TEST_NAME ${target_or_source} NAME_WE PARENT_SCOPE) + set(TEST_TARGET ${TEST_NAME} PARENT_SCOPE) endif() + + # Set LLAMA_TEST_SOURCES in parent scope + set(LLAMA_TEST_SOURCES "${LLAMA_TEST_SOURCES}" PARENT_SCOPE) +endfunction() - set(TEST_TARGET ${target}) - - add_test( - NAME ${TEST_NAME} - WORKING_DIRECTORY ${LLAMA_TEST_WORKING_DIRECTORY} - COMMAND $ - ${LLAMA_TEST_ARGS}) +# Helper function: build executable with optional additional sources +function(_llama_build_executable target main_source additional_sources) + if (additional_sources) + add_executable(${target} ${main_source} ${additional_sources}) + else() + add_executable(${target} ${main_source}) + endif() + target_link_libraries(${target} PRIVATE common) + install(TARGETS ${target} RUNTIME) +endfunction() - set_property(TEST ${TEST_NAME} PROPERTY LABELS ${LLAMA_TEST_LABEL}) +# Helper function: create test and set properties +function(_llama_create_test test_name test_target command_type) + if (command_type STREQUAL "TARGET") + add_test( + NAME ${test_name} + WORKING_DIRECTORY ${LLAMA_TEST_WORKING_DIRECTORY} + COMMAND $ + ${LLAMA_TEST_ARGS}) + else() + add_test( + NAME ${test_name} + WORKING_DIRECTORY ${LLAMA_TEST_WORKING_DIRECTORY} + COMMAND ${test_target} + ${LLAMA_TEST_ARGS}) + endif() + + set_property(TEST ${test_name} PROPERTY LABELS ${LLAMA_TEST_LABEL}) endfunction() -function(llama_test_cmd target) +function(llama_build source) include(CMakeParseArguments) set(options) - set(oneValueArgs NAME LABEL WORKING_DIRECTORY) - set(multiValueArgs ARGS) - cmake_parse_arguments(LLAMA_TEST "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + set(oneValueArgs) + set(multiValueArgs SOURCES) + cmake_parse_arguments(LLAMA_BUILD "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - if (NOT DEFINED LLAMA_TEST_LABEL) - set(LLAMA_TEST_LABEL "main") - endif() - if (NOT DEFINED LLAMA_TEST_WORKING_DIRECTORY) - set(LLAMA_TEST_WORKING_DIRECTORY .) - endif() if (DEFINED LLAMA_TEST_NAME) - set(TEST_NAME ${LLAMA_TEST_NAME}) + set(TEST_TARGET ${LLAMA_TEST_NAME}) else() - set(TEST_NAME ${target}) + get_filename_component(TEST_TARGET ${source} NAME_WE) endif() - add_test( - NAME ${TEST_NAME} - WORKING_DIRECTORY ${LLAMA_TEST_WORKING_DIRECTORY} - COMMAND ${target} - ${LLAMA_TEST_ARGS}) + _llama_build_executable(${TEST_TARGET} ${source} "${LLAMA_BUILD_SOURCES}") +endfunction() - set_property(TEST ${TEST_NAME} PROPERTY LABELS ${LLAMA_TEST_LABEL}) +function(llama_test target) + _llama_parse_test_args(${target} ${ARGN}) + set(TEST_TARGET ${target}) + _llama_create_test(${TEST_NAME} ${TEST_TARGET} "TARGET") +endfunction() + +function(llama_test_cmd target) + _llama_parse_test_args(${target} ${ARGN}) + _llama_create_test(${TEST_NAME} ${target} "COMMAND") endfunction() # Builds and runs a test source file. @@ -76,36 +92,23 @@ endfunction() # - LABEL: label for the test (defaults to main) # - ARGS: arguments to pass to the test executable # - WORKING_DIRECTORY +# - SOURCES: additional source files to build with the main source function(llama_build_and_test source) - include(CMakeParseArguments) - set(options) - set(oneValueArgs NAME LABEL WORKING_DIRECTORY) - set(multiValueArgs ARGS) - cmake_parse_arguments(LLAMA_TEST "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - - if (NOT DEFINED LLAMA_TEST_LABEL) - set(LLAMA_TEST_LABEL "main") - endif() - if (NOT DEFINED LLAMA_TEST_WORKING_DIRECTORY) - set(LLAMA_TEST_WORKING_DIRECTORY .) - endif() - if (DEFINED LLAMA_TEST_NAME) - set(TEST_TARGET ${LLAMA_TEST_NAME}) + _llama_parse_test_args(${source} ${ARGN}) + + # Build executable with sources + set(all_sources ${source}) + if (LLAMA_TEST_SOURCES) + list(APPEND all_sources ${LLAMA_TEST_SOURCES}) else() - get_filename_component(TEST_TARGET ${source} NAME_WE) + # Only include get-model.cpp if no additional sources are provided + list(APPEND all_sources get-model.cpp) endif() - - add_executable(${TEST_TARGET} ${source} get-model.cpp) + add_executable(${TEST_TARGET} ${all_sources}) install(TARGETS ${TEST_TARGET} RUNTIME) target_link_libraries(${TEST_TARGET} PRIVATE common) - add_test( - NAME ${TEST_TARGET} - WORKING_DIRECTORY ${LLAMA_TEST_WORKING_DIRECTORY} - COMMAND $ - ${LLAMA_TEST_ARGS}) - - set_property(TEST ${TEST_TARGET} PROPERTY LABELS ${LLAMA_TEST_LABEL}) + _llama_create_test(${TEST_TARGET} ${TEST_TARGET} "TARGET") endfunction() # build test-tokenizer-0 target once and add many tests @@ -182,26 +185,21 @@ endif() llama_build_and_test(test-chat-parser.cpp) # Chat PEG parser tests (modular) -file(GLOB_RECURSE CHAT_PEG_PARSER_TEST_SOURCES - chat-peg-parser/simple_tokenizer.cpp - chat-peg-parser/benchmark.cpp - chat-peg-parser/test-actions.cpp - chat-peg-parser/test-command7-parser-compare.cpp - chat-peg-parser/test-example-qwen3-coder.cpp - chat-peg-parser/test-gbnf-generation.cpp - chat-peg-parser/test-json-parser.cpp - chat-peg-parser/test-one.cpp - chat-peg-parser/test-optional.cpp - chat-peg-parser/test-partial-parsing.cpp - chat-peg-parser/test-recursive-references.cpp - chat-peg-parser/tests.h - test-chat-peg-parser.cpp +llama_build_and_test(test-chat-peg-parser.cpp + SOURCES + chat-peg-parser/simple_tokenizer.cpp + chat-peg-parser/benchmark.cpp + chat-peg-parser/test-actions.cpp + chat-peg-parser/test-command7-parser-compare.cpp + chat-peg-parser/test-example-qwen3-coder.cpp + chat-peg-parser/test-gbnf-generation.cpp + chat-peg-parser/test-json-parser.cpp + chat-peg-parser/test-one.cpp + chat-peg-parser/test-optional.cpp + chat-peg-parser/test-partial-parsing.cpp + chat-peg-parser/test-recursive-references.cpp + chat-peg-parser/tests.h ) -add_executable(test-chat-peg-parser ${CHAT_PEG_PARSER_TEST_SOURCES}) -target_link_libraries(test-chat-peg-parser PRIVATE common) -install(TARGETS test-chat-peg-parser RUNTIME) -add_test(NAME test-chat-peg-parser COMMAND test-chat-peg-parser) -set_property(TEST test-chat-peg-parser PROPERTY LABELS main) llama_build_and_test(test-chat-template.cpp) llama_build_and_test(test-json-partial.cpp) diff --git a/tests/chat-peg-parser/tests.h b/tests/chat-peg-parser/tests.h index c2db1b96bc7e6..5baabc86084b9 100644 --- a/tests/chat-peg-parser/tests.h +++ b/tests/chat-peg-parser/tests.h @@ -3,7 +3,7 @@ // Common includes for all test files #include "../testcase.hpp" #include -#include "chat-peg-parser.h" +#include "../common/chat-peg-parser.h" #include // Test class declarations From 9ebdd6469ddde29e596798f0c278bb0587633969 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 15 Nov 2025 03:41:04 -0600 Subject: [PATCH 061/183] starting unicode support --- common/chat-peg-parser.cpp | 218 ++++++++++++++++++++++++++++++++++++- common/chat-peg-parser.h | 8 ++ 2 files changed, 220 insertions(+), 6 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 15b97bbe8051e..b9b0e980e5104 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -11,6 +11,8 @@ #include enum parser_type { + START, + END, LITERAL, SEQUENCE, CHOICE, @@ -102,6 +104,152 @@ static bool is_hex_digit(const char c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); } +// UTF-8 parsing utilities for streaming-aware unicode support +struct utf8_parse_result { + uint32_t codepoint; // Decoded codepoint (only valid if status == SUCCESS) + size_t bytes_consumed; // How many bytes this codepoint uses (1-4) + enum status_t { SUCCESS, NEED_MORE, INVALID } status; + + utf8_parse_result(status_t s, uint32_t cp = 0, size_t bytes = 0) + : codepoint(cp), bytes_consumed(bytes), status(s) {} +}; + +// Determine the expected length of a UTF-8 sequence from its first byte +// Returns 0 for invalid first bytes +static size_t utf8_sequence_length(unsigned char first_byte) { + // Lookup table based on high 4 bits + // 0xxx xxxx = 1 byte (ASCII) + // 110x xxxx = 2 bytes + // 1110 xxxx = 3 bytes + // 1111 0xxx = 4 bytes (only 0xF0-0xF7, not 0xF8-0xFF) + static const size_t lookup[] = { + 1, 1, 1, 1, 1, 1, 1, 1, // 0000-0111 (0x00-0x7F) + 0, 0, 0, 0, // 1000-1011 (continuation bytes 0x80-0xBF, invalid as first byte) + 2, 2, // 1100-1101 (0xC0-0xDF) + 3, // 1110 (0xE0-0xEF) + 4 // 1111 (0xF0-0xFF, but need to check 0xF8-0xFF separately) + }; + size_t len = lookup[first_byte >> 4]; + + // Additional validation for invalid first bytes: + // - 0xC0-0xC1: would create overlong 2-byte sequences + // - 0xF8-0xFF: invalid 5+ byte sequences + if (first_byte >= 0xF8 || (first_byte >= 0xC0 && first_byte <= 0xC1)) { + return 0; // Invalid + } + + return len; +} + +// Parse a single UTF-8 codepoint from input, with streaming support +// Returns SUCCESS if a complete, valid codepoint is parsed +// Returns NEED_MORE if the sequence is incomplete and input_is_complete is false +// Returns INVALID if the UTF-8 encoding is malformed +static utf8_parse_result parse_utf8_codepoint( + std::string_view input, + size_t offset, + bool input_is_complete +) { + if (offset >= input.size()) { + if (input_is_complete) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + return utf8_parse_result(utf8_parse_result::NEED_MORE); + } + + const unsigned char first = static_cast(input[offset]); + + // ASCII fast path (most common case) + if (first < 0x80) { + return utf8_parse_result(utf8_parse_result::SUCCESS, first, 1); + } + + // Invalid first byte (continuation byte 10xxxxxx as first byte, or 0xF8-0xFF) + if ((first & 0xC0) == 0x80) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + + size_t seq_len = utf8_sequence_length(first); + if (seq_len == 0) { + // Invalid first byte (e.g., 0xF8-0xFF) + return utf8_parse_result(utf8_parse_result::INVALID); + } + + size_t available = input.size() - offset; + + // Check if we have enough bytes for the complete sequence + if (available < seq_len) { + if (input_is_complete) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + return utf8_parse_result(utf8_parse_result::NEED_MORE); + } + + uint32_t codepoint = 0; + + // Decode based on sequence length + if (seq_len == 2) { + // 110xxxxx 10xxxxxx + if ((first & 0xE0) != 0xC0) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + const unsigned char second = static_cast(input[offset + 1]); + if ((second & 0xC0) != 0x80) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + codepoint = ((first & 0x1F) << 6) | (second & 0x3F); + // Check for overlong encoding + if (codepoint < 0x80) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + } else if (seq_len == 3) { + // 1110xxxx 10xxxxxx 10xxxxxx + if ((first & 0xF0) != 0xE0) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + const unsigned char second = static_cast(input[offset + 1]); + const unsigned char third = static_cast(input[offset + 2]); + if ((second & 0xC0) != 0x80 || (third & 0xC0) != 0x80) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + codepoint = ((first & 0x0F) << 12) | ((second & 0x3F) << 6) | (third & 0x3F); + // Check for overlong encoding + if (codepoint < 0x800) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + // Check for surrogate pairs (0xD800-0xDFFF are invalid in UTF-8) + if (codepoint >= 0xD800 && codepoint <= 0xDFFF) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + } else if (seq_len == 4) { + // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + if ((first & 0xF8) != 0xF0) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + const unsigned char second = static_cast(input[offset + 1]); + const unsigned char third = static_cast(input[offset + 2]); + const unsigned char fourth = static_cast(input[offset + 3]); + if ((second & 0xC0) != 0x80 || (third & 0xC0) != 0x80 || (fourth & 0xC0) != 0x80) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + codepoint = ((first & 0x07) << 18) | ((second & 0x3F) << 12) | + ((third & 0x3F) << 6) | (fourth & 0x3F); + // Check for overlong encoding + if (codepoint < 0x10000) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + // Check for valid Unicode range (max is 0x10FFFF) + if (codepoint > 0x10FFFF) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + } else { + // Invalid sequence length + return utf8_parse_result(utf8_parse_result::INVALID); + } + + return utf8_parse_result(utf8_parse_result::SUCCESS, codepoint, seq_len); +} + // Unescapes a JSON string (without the surrounding quotes) // Uses nlohmann::json::parse to handle all JSON escape sequences static std::string unescape_json_string(std::string_view str) { @@ -362,6 +510,36 @@ class root_parser : public common_chat_peg_parser_base { const std::unordered_map & rules() const { return rules_; } }; +// Matches the start of the input +// S -> ^ +class start_parser : public common_chat_peg_parser_base { + public: + static constexpr parser_type type_value = START; + start_parser(int id) : common_chat_peg_parser_base(id) {} + parser_type type() const override { return type_value; } + void accept(parser_visitor & visitor) override; + std::string dump() const override { return "Start"; } + + common_chat_parse_result parse_uncached(common_chat_parse_context &, size_t start = 0) override { + return common_chat_parse_result(start == 0 ? COMMON_CHAT_PARSE_RESULT_SUCCESS : COMMON_CHAT_PARSE_RESULT_FAIL, start); + } +}; + +// Matches the end of the input +// S -> $ +class end_parser : public common_chat_peg_parser_base { + public: + static constexpr parser_type type_value = END; + end_parser(int id) : common_chat_peg_parser_base(id) {} + parser_type type() const override { return type_value; } + void accept(parser_visitor & visitor) override; + std::string dump() const override { return "End"; } + + common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { + return common_chat_parse_result(start >= ctx.input.size() ? COMMON_CHAT_PARSE_RESULT_SUCCESS : COMMON_CHAT_PARSE_RESULT_FAIL, start); + } +}; + // Matches an exact literal string. // S -> "hello" class literal_parser : public common_chat_peg_parser_base { @@ -730,13 +908,19 @@ class any_parser : public common_chat_peg_parser_base { parser_type type() const override { return type_value; } common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { - if (start >= ctx.input.size()) { - if (ctx.input_is_complete) { - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); - } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start); + // Parse a single UTF-8 codepoint (not just a single byte) + auto result = parse_utf8_codepoint(ctx.input, start, ctx.input_is_complete); + + if (result.status == utf8_parse_result::NEED_MORE) { + // Incomplete UTF-8 sequence: end position is at start (before incomplete bytes) + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start, start); } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, start + 1); + if (result.status == utf8_parse_result::INVALID) { + // Malformed UTF-8 + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); + } + // Success: advance by full codepoint (1-4 bytes) + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, start + result.bytes_consumed); } std::string dump() const override { @@ -1225,6 +1409,8 @@ class parser_visitor { public: virtual ~parser_visitor() = default; + virtual void visit(start_parser & p) = 0; + virtual void visit(end_parser & p) = 0; virtual void visit(literal_parser & p) = 0; virtual void visit(sequence_parser & p) = 0; virtual void visit(choice_parser & p) = 0; @@ -1288,6 +1474,8 @@ class reachability_visitor : public parser_visitor { const std::unordered_map & rules ) : reachable_rules_(reachable_rules), rules_(rules) {} + void visit(start_parser &) override {} + void visit(end_parser &) override {} void visit(literal_parser &) override {} void visit(any_parser &) override {} void visit(space_parser &) override {} @@ -1375,6 +1563,14 @@ class gbnf_visitor : public parser_visitor { } public: + void visit(start_parser &) override { + current_result_ = ""; + } + + void visit(end_parser &) override { + current_result_ = ""; + } + void visit(literal_parser & p) override { current_result_ = gbnf_literal(p.literal()); } @@ -1616,6 +1812,8 @@ class gbnf_visitor : public parser_visitor { }; // Implement accept() methods for all parser classes +void start_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } +void end_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void literal_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void sequence_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void choice_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } @@ -1719,6 +1917,14 @@ common_chat_peg_parser_builder::common_chat_peg_parser_builder() : root_(std::make_shared(0)) // root parser has id 0 , counter_(1) {} +common_chat_peg_parser common_chat_peg_parser_builder::start() { + return common_chat_peg_parser(std::make_shared(counter_.next())); +} + +common_chat_peg_parser common_chat_peg_parser_builder::end() { + return common_chat_peg_parser(std::make_shared(counter_.next())); +} + common_chat_peg_parser common_chat_peg_parser_builder::literal(const std::string & literal) { return common_chat_peg_parser(std::make_shared(literal, counter_.next())); } diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index aa5fd89ddc62c..299b176cdbc4d 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -185,6 +185,14 @@ class common_chat_peg_parser_builder { public: common_chat_peg_parser_builder(); + // Matches the start of the input. + // S -> ^ + common_chat_peg_parser start(); + + // Matches the end of the input. + // S -> $ + common_chat_peg_parser end(); + // Matches an exact literal string. // S -> "hello" common_chat_peg_parser literal(const std::string & literal); From c8d94d11d9aafd6c51bc1f4da16af3a4c0727568 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 15 Nov 2025 14:23:29 -0600 Subject: [PATCH 062/183] add unicode support to char_parser --- common/chat-peg-parser.cpp | 113 ++++++++++++++++++++++++++++++++++--- common/chat-peg-parser.h | 2 + 2 files changed, 106 insertions(+), 9 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index b9b0e980e5104..7d00bd16a14b7 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -34,6 +34,15 @@ enum parser_type { TRIGGER, }; +const char * common_chat_parse_result_type_name(common_chat_parse_result_type type) { + switch (type) { + case COMMON_CHAT_PARSE_RESULT_FAIL: return "fail"; + case COMMON_CHAT_PARSE_RESULT_SUCCESS: return "success"; + case COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT: return "need_more_input"; + default: return "unknown"; + } +} + class parser_visitor; class common_chat_peg_parser_base { @@ -963,12 +972,13 @@ class space_parser : public common_chat_peg_parser_base { // Matches between min and max repetitions of characters from a character class. // S -> [a-z]{m,n} +// Supports Unicode codepoint ranges and escape sequences: \xXX \uXXXX \UXXXXXXXX class chars_parser : public common_chat_peg_parser_base { struct char_range { - int start; - int end; + uint32_t start; + uint32_t end; - bool contains(char c) const { return (int)c >= start && int(c) <= end; } + bool contains(uint32_t codepoint) const { return codepoint >= start && codepoint <= end; } }; std::string pattern_; @@ -996,7 +1006,8 @@ class chars_parser : public common_chat_peg_parser_base { content = content.substr(1); } - auto parse_char = [&](size_t pos) -> std::pair { + // Parse a character or escape sequence, returning codepoint and bytes consumed + auto parse_char = [&](size_t pos) -> std::pair { if (content[pos] == '\\' && pos + 1 < content.length()) { char next = content[pos + 1]; switch (next) { @@ -1007,10 +1018,72 @@ class chars_parser : public common_chat_peg_parser_base { case ']': return {']', 2}; case '-': return {'-', 2}; case '[': return {'[', 2}; + + // \xXX - 8-bit hex escape + case 'x': { + if (pos + 3 < content.length() && + is_hex_digit(content[pos + 2]) && + is_hex_digit(content[pos + 3])) { + uint32_t value = 0; + for (int i = 0; i < 2; i++) { + char c = content[pos + 2 + i]; + value = value * 16 + (c >= 'a' ? c - 'a' + 10 : + c >= 'A' ? c - 'A' + 10 : + c - '0'); + } + return {value, 4}; // \xXX + } + return {next, 2}; // Invalid escape, treat as literal 'x' + } + + // \uXXXX - 16-bit hex escape + case 'u': { + if (pos + 5 < content.length() && + is_hex_digit(content[pos + 2]) && + is_hex_digit(content[pos + 3]) && + is_hex_digit(content[pos + 4]) && + is_hex_digit(content[pos + 5])) { + uint32_t value = 0; + for (int i = 0; i < 4; i++) { + char c = content[pos + 2 + i]; + value = value * 16 + (c >= 'a' ? c - 'a' + 10 : + c >= 'A' ? c - 'A' + 10 : + c - '0'); + } + return {value, 6}; // \uXXXX + } + return {next, 2}; // Invalid escape, treat as literal 'u' + } + + // \UXXXXXXXX - 32-bit hex escape + case 'U': { + if (pos + 9 < content.length()) { + bool all_hex = true; + for (int i = 0; i < 8; i++) { + if (!is_hex_digit(content[pos + 2 + i])) { + all_hex = false; + break; + } + } + if (all_hex) { + uint32_t value = 0; + for (int i = 0; i < 8; i++) { + char c = content[pos + 2 + i]; + value = value * 16 + (c >= 'a' ? c - 'a' + 10 : + c >= 'A' ? c - 'A' + 10 : + c - '0'); + } + return {value, 10}; // \UXXXXXXXX + } + } + return {next, 2}; // Invalid escape, treat as literal 'U' + } + default: return {next, 2}; // Treat as literal escaped character } } - return {content[pos], 1}; + // Regular character - return as codepoint + return {static_cast(static_cast(content[pos])), 1}; }; size_t i = 0; @@ -1039,13 +1112,34 @@ class chars_parser : public common_chat_peg_parser_base { // Try to match up to max_count times (or unlimited if max_count is -1) while (max_count_ == -1 || match_count < max_count_) { - if (pos >= ctx.input.size()) { - break; + // Parse UTF-8 codepoint from input + auto result = parse_utf8_codepoint(ctx.input, pos, ctx.input_is_complete); + + if (result.status == utf8_parse_result::NEED_MORE) { + // Incomplete UTF-8 sequence at current position + if (match_count >= min_count_) { + // We have enough matches, succeed with what we have + // End position is at pos (before the incomplete sequence) + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); + } + // Not enough matches yet, need more input + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start, pos); } + if (result.status == utf8_parse_result::INVALID) { + // Malformed UTF-8 in input + if (match_count >= min_count_) { + // We have enough matches, succeed up to here + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); + } + // Not enough matches, fail + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); + } + + // Check if this codepoint matches our character class bool matches = false; for (const auto & range : ranges_) { - if (range.contains(ctx.input[pos])) { + if (range.contains(result.codepoint)) { matches = true; break; } @@ -1057,9 +1151,10 @@ class chars_parser : public common_chat_peg_parser_base { } if (matches) { - ++pos; + pos += result.bytes_consumed; ++match_count; } else { + // Character doesn't match, stop matching break; } } diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index 299b176cdbc4d..200ef31a539a4 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -25,6 +25,8 @@ enum common_chat_parse_result_type { COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT = 1 << 2, }; +const char * common_chat_parse_result_type_name(common_chat_parse_result_type type); + struct common_chat_parse_cache_key { int id; size_t start; From befca676d3084b08770191adddc3ae01585bbef4 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 15 Nov 2025 14:39:40 -0600 Subject: [PATCH 063/183] use unparsed args as additional sources --- tests/CMakeLists.txt | 174 +++++++++++++++++----------------- tests/chat-peg-parser/tests.h | 2 +- 2 files changed, 87 insertions(+), 89 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d0bd827ad3c74..8fce517e8b5e8 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,89 +1,75 @@ llama_add_compile_flags() -# Helper function: parse test arguments and set default values -function(_llama_parse_test_args target_or_source) +function(llama_build source) + set(TEST_SOURCES ${source} ${ARGN}) + + if (DEFINED LLAMA_TEST_NAME) + set(TEST_TARGET ${LLAMA_TEST_NAME}) + else() + get_filename_component(TEST_TARGET ${source} NAME_WE) + endif() + + add_executable(${TEST_TARGET} ${TEST_SOURCES}) + target_link_libraries(${TEST_TARGET} PRIVATE common) + install(TARGETS ${TEST_TARGET} RUNTIME) +endfunction() + +function(llama_test target) include(CMakeParseArguments) set(options) set(oneValueArgs NAME LABEL WORKING_DIRECTORY) - set(multiValueArgs ARGS SOURCES) + set(multiValueArgs ARGS) cmake_parse_arguments(LLAMA_TEST "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - # Set default values if (NOT DEFINED LLAMA_TEST_LABEL) - set(LLAMA_TEST_LABEL "main" PARENT_SCOPE) + set(LLAMA_TEST_LABEL "main") endif() if (NOT DEFINED LLAMA_TEST_WORKING_DIRECTORY) - set(LLAMA_TEST_WORKING_DIRECTORY "." PARENT_SCOPE) + set(LLAMA_TEST_WORKING_DIRECTORY .) endif() - - # Set test name and target if (DEFINED LLAMA_TEST_NAME) - set(TEST_NAME ${LLAMA_TEST_NAME} PARENT_SCOPE) - set(TEST_TARGET ${LLAMA_TEST_NAME} PARENT_SCOPE) + set(TEST_NAME ${LLAMA_TEST_NAME}) else() - get_filename_component(TEST_NAME ${target_or_source} NAME_WE PARENT_SCOPE) - set(TEST_TARGET ${TEST_NAME} PARENT_SCOPE) + set(TEST_NAME ${target}) endif() - - # Set LLAMA_TEST_SOURCES in parent scope - set(LLAMA_TEST_SOURCES "${LLAMA_TEST_SOURCES}" PARENT_SCOPE) -endfunction() -# Helper function: build executable with optional additional sources -function(_llama_build_executable target main_source additional_sources) - if (additional_sources) - add_executable(${target} ${main_source} ${additional_sources}) - else() - add_executable(${target} ${main_source}) - endif() - target_link_libraries(${target} PRIVATE common) - install(TARGETS ${target} RUNTIME) -endfunction() + set(TEST_TARGET ${target}) -# Helper function: create test and set properties -function(_llama_create_test test_name test_target command_type) - if (command_type STREQUAL "TARGET") - add_test( - NAME ${test_name} - WORKING_DIRECTORY ${LLAMA_TEST_WORKING_DIRECTORY} - COMMAND $ - ${LLAMA_TEST_ARGS}) - else() - add_test( - NAME ${test_name} - WORKING_DIRECTORY ${LLAMA_TEST_WORKING_DIRECTORY} - COMMAND ${test_target} - ${LLAMA_TEST_ARGS}) - endif() - - set_property(TEST ${test_name} PROPERTY LABELS ${LLAMA_TEST_LABEL}) + add_test( + NAME ${TEST_NAME} + WORKING_DIRECTORY ${LLAMA_TEST_WORKING_DIRECTORY} + COMMAND $ + ${LLAMA_TEST_ARGS}) + + set_property(TEST ${TEST_NAME} PROPERTY LABELS ${LLAMA_TEST_LABEL}) endfunction() -function(llama_build source) +function(llama_test_cmd target) include(CMakeParseArguments) set(options) - set(oneValueArgs) - set(multiValueArgs SOURCES) - cmake_parse_arguments(LLAMA_BUILD "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + set(oneValueArgs NAME LABEL WORKING_DIRECTORY) + set(multiValueArgs ARGS) + cmake_parse_arguments(LLAMA_TEST "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + if (NOT DEFINED LLAMA_TEST_LABEL) + set(LLAMA_TEST_LABEL "main") + endif() + if (NOT DEFINED LLAMA_TEST_WORKING_DIRECTORY) + set(LLAMA_TEST_WORKING_DIRECTORY .) + endif() if (DEFINED LLAMA_TEST_NAME) - set(TEST_TARGET ${LLAMA_TEST_NAME}) + set(TEST_NAME ${LLAMA_TEST_NAME}) else() - get_filename_component(TEST_TARGET ${source} NAME_WE) + set(TEST_NAME ${target}) endif() - _llama_build_executable(${TEST_TARGET} ${source} "${LLAMA_BUILD_SOURCES}") -endfunction() - -function(llama_test target) - _llama_parse_test_args(${target} ${ARGN}) - set(TEST_TARGET ${target}) - _llama_create_test(${TEST_NAME} ${TEST_TARGET} "TARGET") -endfunction() + add_test( + NAME ${TEST_NAME} + WORKING_DIRECTORY ${LLAMA_TEST_WORKING_DIRECTORY} + COMMAND ${target} + ${LLAMA_TEST_ARGS}) -function(llama_test_cmd target) - _llama_parse_test_args(${target} ${ARGN}) - _llama_create_test(${TEST_NAME} ${target} "COMMAND") + set_property(TEST ${TEST_NAME} PROPERTY LABELS ${LLAMA_TEST_LABEL}) endfunction() # Builds and runs a test source file. @@ -92,23 +78,38 @@ endfunction() # - LABEL: label for the test (defaults to main) # - ARGS: arguments to pass to the test executable # - WORKING_DIRECTORY -# - SOURCES: additional source files to build with the main source function(llama_build_and_test source) - _llama_parse_test_args(${source} ${ARGN}) - - # Build executable with sources - set(all_sources ${source}) - if (LLAMA_TEST_SOURCES) - list(APPEND all_sources ${LLAMA_TEST_SOURCES}) + include(CMakeParseArguments) + set(options) + set(oneValueArgs NAME LABEL WORKING_DIRECTORY) + set(multiValueArgs ARGS) + cmake_parse_arguments(LLAMA_TEST "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + set(TEST_SOURCES ${source} ${LLAMA_TEST_UNPARSED_ARGUMENTS} get-model.cpp) + + if (NOT DEFINED LLAMA_TEST_LABEL) + set(LLAMA_TEST_LABEL "main") + endif() + if (NOT DEFINED LLAMA_TEST_WORKING_DIRECTORY) + set(LLAMA_TEST_WORKING_DIRECTORY .) + endif() + if (DEFINED LLAMA_TEST_NAME) + set(TEST_TARGET ${LLAMA_TEST_NAME}) else() - # Only include get-model.cpp if no additional sources are provided - list(APPEND all_sources get-model.cpp) + get_filename_component(TEST_TARGET ${source} NAME_WE) endif() - add_executable(${TEST_TARGET} ${all_sources}) + + add_executable(${TEST_TARGET} ${TEST_SOURCES}) install(TARGETS ${TEST_TARGET} RUNTIME) target_link_libraries(${TEST_TARGET} PRIVATE common) - _llama_create_test(${TEST_TARGET} ${TEST_TARGET} "TARGET") + add_test( + NAME ${TEST_TARGET} + WORKING_DIRECTORY ${LLAMA_TEST_WORKING_DIRECTORY} + COMMAND $ + ${LLAMA_TEST_ARGS}) + + set_property(TEST ${TEST_TARGET} PROPERTY LABELS ${LLAMA_TEST_LABEL}) endfunction() # build test-tokenizer-0 target once and add many tests @@ -183,24 +184,21 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS) endif() llama_build_and_test(test-chat-parser.cpp) - -# Chat PEG parser tests (modular) -llama_build_and_test(test-chat-peg-parser.cpp - SOURCES - chat-peg-parser/simple_tokenizer.cpp - chat-peg-parser/benchmark.cpp - chat-peg-parser/test-actions.cpp - chat-peg-parser/test-command7-parser-compare.cpp - chat-peg-parser/test-example-qwen3-coder.cpp - chat-peg-parser/test-gbnf-generation.cpp - chat-peg-parser/test-json-parser.cpp - chat-peg-parser/test-one.cpp - chat-peg-parser/test-optional.cpp - chat-peg-parser/test-partial-parsing.cpp - chat-peg-parser/test-recursive-references.cpp - chat-peg-parser/tests.h +llama_build_and_test( + test-chat-peg-parser.cpp + chat-peg-parser/simple_tokenizer.cpp + chat-peg-parser/benchmark.cpp + chat-peg-parser/test-actions.cpp + chat-peg-parser/test-command7-parser-compare.cpp + chat-peg-parser/test-example-qwen3-coder.cpp + chat-peg-parser/test-gbnf-generation.cpp + chat-peg-parser/test-json-parser.cpp + chat-peg-parser/test-one.cpp + chat-peg-parser/test-optional.cpp + chat-peg-parser/test-partial-parsing.cpp + chat-peg-parser/test-recursive-references.cpp + chat-peg-parser/tests.h ) - llama_build_and_test(test-chat-template.cpp) llama_build_and_test(test-json-partial.cpp) llama_build_and_test(test-log.cpp) diff --git a/tests/chat-peg-parser/tests.h b/tests/chat-peg-parser/tests.h index 5baabc86084b9..c2db1b96bc7e6 100644 --- a/tests/chat-peg-parser/tests.h +++ b/tests/chat-peg-parser/tests.h @@ -3,7 +3,7 @@ // Common includes for all test files #include "../testcase.hpp" #include -#include "../common/chat-peg-parser.h" +#include "chat-peg-parser.h" #include // Test class declarations From c077792bc79df6947537dafbd3d8da17afa2ee27 Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Sat, 15 Nov 2025 23:48:39 +0100 Subject: [PATCH 064/183] Refactor tests to new harness --- tests/chat-peg-parser/benchmark.cpp | 24 -- tests/chat-peg-parser/simple_tokenizer.cpp | 2 +- tests/chat-peg-parser/test-actions.cpp | 158 ++++---- .../test-command7-parser-compare.cpp | 249 ++++++------ .../test-example-qwen3-coder.cpp | 10 +- .../chat-peg-parser/test-gbnf-generation.cpp | 187 ++++------ tests/chat-peg-parser/test-json-parser.cpp | 108 +++--- tests/chat-peg-parser/test-one.cpp | 150 ++++---- tests/chat-peg-parser/test-optional.cpp | 58 ++- .../chat-peg-parser/test-partial-parsing.cpp | 353 ++++++++---------- .../test-recursive-references.cpp | 140 ++++--- tests/chat-peg-parser/test_harness.h | 181 +++++++++ tests/chat-peg-parser/tests.h | 88 +---- tests/test-chat-peg-parser.cpp | 41 +- tests/testcase.hpp | 188 ---------- 15 files changed, 854 insertions(+), 1083 deletions(-) delete mode 100644 tests/chat-peg-parser/benchmark.cpp create mode 100644 tests/chat-peg-parser/test_harness.h delete mode 100644 tests/testcase.hpp diff --git a/tests/chat-peg-parser/benchmark.cpp b/tests/chat-peg-parser/benchmark.cpp deleted file mode 100644 index b7c14ffbed684..0000000000000 --- a/tests/chat-peg-parser/benchmark.cpp +++ /dev/null @@ -1,24 +0,0 @@ -#include "tests.h" -#include -#include -#include -#include - -// benchmark_test base class implementation -benchmark_test::benchmark_test(std::vector> cs): cases(std::move(cs)) {} - -long long benchmark_test::run_benchmark(size_t which, int iterations) { - if (which >= cases.size()) { - throw std::runtime_error(std::string("Invalid index for benchmark test: ") + std::to_string(which)); - } - std::chrono::microseconds duration(0); - test_case& tc = *cases.at(which); - for (int i = 0; i < iterations; i++) { - auto start = std::chrono::high_resolution_clock::now(); - tc.run(); - auto end = std::chrono::high_resolution_clock::now(); - tc.reset(); - duration += std::chrono::duration_cast(end - start); - } - return duration.count() / iterations; -} diff --git a/tests/chat-peg-parser/simple_tokenizer.cpp b/tests/chat-peg-parser/simple_tokenizer.cpp index 77beee35a346e..7fb54f7525388 100644 --- a/tests/chat-peg-parser/simple_tokenizer.cpp +++ b/tests/chat-peg-parser/simple_tokenizer.cpp @@ -1,6 +1,6 @@ #include "tests.h" -std::vector uses_simple_tokenizer::simple_tokenize(const std::string & input) { +std::vector simple_tokenize(const std::string & input) { std::vector result; std::string current; diff --git a/tests/chat-peg-parser/test-actions.cpp b/tests/chat-peg-parser/test-actions.cpp index 9e6966decd835..feae9c41f93e3 100644 --- a/tests/chat-peg-parser/test-actions.cpp +++ b/tests/chat-peg-parser/test-actions.cpp @@ -1,103 +1,95 @@ #include "tests.h" -test_actions::test_actions() : compound_test("test_actions") { +void test_actions(testing &t) { // Test simple action - append matched text to content - add_test( - [](test_harness h) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { - auto word = p.chars("[a-z]+"); - return p.action(word, - [](const common_chat_parse_action & act) { act.env.result.content += std::string(act.match); }); - }); + t.test("simple action - append matched text to content", [](testing &t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto word = p.chars("[a-z]+"); + return p.action(word, + [](const common_chat_parse_action & act) { act.env.result.content += std::string(act.match); }); + }); - common_chat_parse_semantics env; - common_chat_parse_context ctx("hello", &env); - auto result = parser.parse(ctx); + common_chat_parse_semantics env; + common_chat_parse_context ctx("hello", &env); + auto result = parser.parse(ctx); - h.assert_equals("result_is_success", true, result.success()); - h.assert_equals("result_is_hello", std::string("hello"), env.result.content); - }, - "simple action - append matched text to content"); + t.assert_equal("result_is_success", true, result.success()); + t.assert_equal("result_is_hello", std::string("hello"), env.result.content); + }); // Test multiple sequential actions - build a sentence - add_test( - [](test_harness h) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { - auto greeting = p.action(p.literal("hello"), [](const common_chat_parse_action & act) { - act.env.result.content += std::string(act.match) + " "; - }); - - auto name = p.action(p.chars("[A-Z][a-z]+"), [](const common_chat_parse_action & act) { - act.env.result.content += std::string(act.match); - act.env.captures["name"] = std::string(act.match); - }); - - return greeting + p.literal(" ") + name; + t.test("multiple sequential actions - build a sentence", [](testing &t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto greeting = p.action(p.literal("hello"), [](const common_chat_parse_action & act) { + act.env.result.content += std::string(act.match) + " "; }); - common_chat_parse_semantics env; - common_chat_parse_context ctx("hello Alice", &env); - auto result = parser.parse(ctx); + auto name = p.action(p.chars("[A-Z][a-z]+"), [](const common_chat_parse_action & act) { + act.env.result.content += std::string(act.match); + act.env.captures["name"] = std::string(act.match); + }); + + return greeting + p.literal(" ") + name; + }); + + common_chat_parse_semantics env; + common_chat_parse_context ctx("hello Alice", &env); + auto result = parser.parse(ctx); - h.assert_equals("result_is_success", true, result.success()); - h.assert_equals("result_content", std::string("hello Alice"), env.result.content); - h.assert_equals("captured_name", std::string("Alice"), env.captures["name"]); - }, - "multiple sequential actions - build a sentence"); + t.assert_equal("result_is_success", true, result.success()); + t.assert_equal("result_content", std::string("hello Alice"), env.result.content); + t.assert_equal("captured_name", std::string("Alice"), env.captures["name"]); + }); // Test actions don't run when parse fails - add_test( - [](test_harness h) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { - return p.action(p.literal("success"), - [](const common_chat_parse_action & act) { act.env.result.content = "action_ran"; }); + t.test("actions don't run when parse fails", [](testing &t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + return p.action(p.literal("success"), + [](const common_chat_parse_action & act) { act.env.result.content = "action_ran"; }); + }); + + common_chat_parse_semantics env; + common_chat_parse_context ctx("failure", &env); + auto result = parser.parse(ctx); + + t.assert_equal("result_is_fail", true, result.fail()); + t.assert_equal("result_content_empty", std::string(""), env.result.content); // Action should not have run + }); + + // Test Actions work with partial parsing + t.test("actions work with need_more_input parsing", [](testing &t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto content = p.action(p.until(""), [](const common_chat_parse_action & act) { + act.env.result.content += std::string(act.match); }); + return "" << content << ""; + }); + { common_chat_parse_semantics env; - common_chat_parse_context ctx("failure", &env); + common_chat_parse_context ctx("hello ", &env, false); auto result = parser.parse(ctx); - h.assert_equals("result_is_fail", true, result.fail()); - h.assert_equals("result_content_empty", std::string(""), env.result.content); // Action should not have run - }, - "actions don't run when parse fails"); + t.assert_equal("result_is_need_more_input_1", true, result.need_more_input()); + t.assert_equal("result_content_1", std::string("hello "), env.result.content); + } - // Test Actions work with partial parsing - add_test( - [](test_harness h) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { - auto content = p.action(p.until(""), [](const common_chat_parse_action & act) { - act.env.result.content += std::string(act.match); - }); - return "" << content << ""; - }); + { + common_chat_parse_semantics env; + common_chat_parse_context ctx("hello world", &env, false); + auto result = parser.parse(ctx); + + t.assert_equal("result_is_need_more_input_2", true, result.need_more_input()); + t.assert_equal("result_content_2", std::string("hello world"), env.result.content); + } + + { + common_chat_parse_semantics env; + common_chat_parse_context ctx("hello world", &env, true); + auto result = parser.parse(ctx); - { - common_chat_parse_semantics env; - common_chat_parse_context ctx("hello ", &env, false); - auto result = parser.parse(ctx); - - h.assert_equals("result_is_need_more_input_1", true, result.need_more_input()); - h.assert_equals("result_content_1", std::string("hello "), env.result.content); - } - - { - common_chat_parse_semantics env; - common_chat_parse_context ctx("hello world", &env, false); - auto result = parser.parse(ctx); - - h.assert_equals("result_is_need_more_input_2", true, result.need_more_input()); - h.assert_equals("result_content_2", std::string("hello world"), env.result.content); - } - - { - common_chat_parse_semantics env; - common_chat_parse_context ctx("hello world", &env, true); - auto result = parser.parse(ctx); - - h.assert_equals("result_is_success", true, result.success()); - h.assert_equals("result_content_final", std::string("hello world"), env.result.content); - } - }, - "actions work with need_more_input parsing"); + t.assert_equal("result_is_success", true, result.success()); + t.assert_equal("result_content_final", std::string("hello world"), env.result.content); + } + }); } diff --git a/tests/chat-peg-parser/test-command7-parser-compare.cpp b/tests/chat-peg-parser/test-command7-parser-compare.cpp index 34881262b281d..fb2c29765d9fe 100644 --- a/tests/chat-peg-parser/test-command7-parser-compare.cpp +++ b/tests/chat-peg-parser/test-command7-parser-compare.cpp @@ -1,12 +1,13 @@ -#include "chat-parser.h" +#include "../common/chat-parser.h" #include "json-schema-to-grammar.h" - #include "tests.h" #include -#include +#include +#include +#include -class common_chat_peg_parser test_command7_parser_compare::create_command_r7b_parser() { +static common_chat_peg_parser create_command_r7b_parser() { auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { auto thinking = p.add_rule("thinking", "<|START_THINKING|>" << p.add_rule("reasoning-content", p.until("<|END_THINKING|>")) << "<|END_THINKING|>"); @@ -43,7 +44,7 @@ class common_chat_peg_parser test_command7_parser_compare::create_command_r7b_pa return parser; } -common_chat_parse_event_handler test_command7_parser_compare::create_command_r7b_event_handler() { +static common_chat_parse_event_handler create_command_r7b_event_handler() { return [](const common_chat_parse_event & ev, common_chat_parse_semantics & env) { if (ev.rule == "reasoning-content" && ev.ending()) { env.result.reasoning_content = ev.text; @@ -74,122 +75,10 @@ common_chat_parse_event_handler test_command7_parser_compare::create_command_r7b }; } -// command7_parser_compare_test implementation -test_command7_parser_compare::test_command7_parser_compare() : - benchmark_test(std::vector>()), - parser(create_command_r7b_parser()), - handler(create_command_r7b_event_handler()), - reasoning("To plan an effective trip to Japan that includes both historical sites and modern attractions within a " - "budget of $4000 for a two-week stay, we need to:\n\n" - "1. Identify key historical sites and modern attractions in Japan.\n" - "2. Find affordable accommodation options that provide a balance between comfort and cost.\n" - "3. Determine the best modes of transportation for getting around Japan.\n" - "4. Create a day-by-day itinerary that ensures the user gets to see a variety of attractions without " - "overspending.\n" - "5. Provide a detailed cost breakdown that includes accommodation, transportation, meals, and entry fees " - "to attractions."), - content("For a two-week trip to Japan with a $4,000 budget, I recommend planning an itinerary that balances " - "historical sites with modern attractions. The destination will be Japan, with a duration of 14 days.\n\n" - "Given your interests in both historical sites and modern attractions, you'll want to focus on cities like " - "Kyoto for its temples and traditional culture, Tokyo for its cutting-edge technology and entertainment " - "districts, and possibly Hiroshima or Nara for additional historical significance.\n\n" - "For accommodation, I suggest looking for affordable options such as budget hotels, hostels, or " - "guesthouses that offer good value without sacrificing too much comfort. Japan has excellent mid-range " - "accommodation options that can keep your lodging costs manageable.\n\n" - "Transportation should prioritize efficiencyβ€”consider getting a JR Rail Pass for intercity travel, which " - "allows unlimited rides on most JR trains including the Shinkansen (bullet train). Within cities, use " - "local trains and subways, which are both affordable and highly reliable.\n\n" - "For meals, embrace local cuisine by eating at neighborhood restaurants, ramen shops, and izakayas rather " - "than touristy establishments. This will give you an authentic experience while keeping costs " - "reasonableβ€”you can enjoy excellent meals for $10-20 per person at local spots.\n\n"), - tool_calls({ - { "call_0", "plan_trip", nlohmann::json::parse(R"({ - "destination": "Japan", - "duration": 14, - "budget": 4000, - "interests": ["historical sites", "modern attractions"], - "accommodation_preferences": "affordable", - "transportation_preferences": "efficient", - "meal_preferences": "local cuisine" - })") } - }) - { - // Build response - if (!reasoning.empty()) { - auto tokenized = simple_tokenize(reasoning); - tokens.emplace_back("<|START_THINKING|>"); - tokens.insert(tokens.end(), tokenized.begin(), tokenized.end()); - tokens.emplace_back("<|END_THINKING|>"); - } - - if (!content.empty()) { - auto tokenized = simple_tokenize(content); - tokens.emplace_back("<|START_RESPONSE|>"); - tokens.insert(tokens.end(), tokenized.begin(), tokenized.end()); - tokens.emplace_back("<|END_RESPONSE|>"); - } - - if (!tool_calls.empty()) { - tokens.emplace_back("<|START_ACTION|>"); - - auto json = nlohmann::json::array(); - for (const auto & tc : tool_calls) { - auto tc_json = nlohmann::json::object(); - tc_json["tool_call_id"] = tc.id; - tc_json["tool_name"] = tc.name; - tc_json["parameters"] = tc.args; - json.push_back(tc_json); - } - - auto tokenized = simple_tokenize(json.dump(-1, ' ', true)); - tokens.insert(tokens.end(), tokenized.begin(), tokenized.end()); - - tokens.emplace_back("<|END_ACTION|>"); - } - - test_case legacy = test_case([this](test_harness h) { - bool no_error = true; - try { - std::string input = std::accumulate(tokens.begin(), tokens.end(), std::string()); - test_command_r7b_legacy_parser(input, false, false); - } catch (std::exception &e) { - no_error = false; - std::cerr << "Error during legacy run: " << e.what() << "\n"; - } - h.assert_equals("no_errors", true, no_error); - }, "legacy_parse"); - - test_case current = test_case([this](test_harness h) { - bool no_error = true; - try { - std::string input = std::accumulate(tokens.begin(), tokens.end(), std::string()); - test_command_r7b_parser(parser, input, false, false); - } catch (std::exception &e) { - no_error = false; - std::cerr << "Error during legacy run: " << e.what() << "\n"; - } - h.assert_equals("no_errors", true, no_error); - }, "current_parse"); - legacy.set_omit_success_msg(true); - current.set_omit_success_msg(true); - - cases.push_back(std::make_unique(legacy)); - cases.push_back(std::make_unique(current)); -} - -void test_command7_parser_compare::run_comparison(int iterations) { - long long t1 = run_benchmark(0, iterations); - long long t2 = run_benchmark(1, iterations); - - std::cout << "=== Command7 common_chat_combinator_parser comparison benchmark (" << iterations << " iterations) ===\n"; - std::cout << "Legacy common_chat_combinator_parser performance: " << t1 << "us (" << (float) t1 / iterations << "us per iteration)\n"; - std::cout << "Current common_chat_combinator_parser performance: " << t2 << "us (" << (float) t2 / iterations << "us per iteration)\n"; -} - -void test_command7_parser_compare::test_command_r7b_parser(const class common_chat_peg_parser & p, - const std::string & input, - bool need_more_input, - bool print_results) { +static void test_command_r7b_parser(const common_chat_peg_parser & p, + const std::string & input, + bool need_more_input, + bool print_results) { common_chat_parse_semantics env; common_chat_parse_context ctx(input, &env, !need_more_input); p.parse(ctx); @@ -209,9 +98,9 @@ void test_command7_parser_compare::test_command_r7b_parser(const class common_ch } } -void test_command7_parser_compare::test_command_r7b_legacy_parser(const std::string & input, - bool need_more_input, - bool print_results) { +static void test_command_r7b_legacy_parser(const std::string & input, + bool need_more_input, + bool print_results) { // Original common_chat_combinator_parser taken from chat.cpp common_chat_msg_parser builder(input, /* .is_partial = */ need_more_input, @@ -267,3 +156,115 @@ void test_command7_parser_compare::test_command_r7b_legacy_parser(const std::str } } } + +void test_command7_parser_compare(testing &t) { + // Setup data + auto parser = create_command_r7b_parser(); + auto handler = create_command_r7b_event_handler(); + + std::string reasoning = "To plan an effective trip to Japan that includes both historical sites and modern attractions within a " + "budget of $4000 for a two-week stay, we need to:\n\n" + "1. Identify key historical sites and modern attractions in Japan.\n" + "2. Find affordable accommodation options that provide a balance between comfort and cost.\n" + "3. Determine the best modes of transportation for getting around Japan.\n" + "4. Create a day-by-day itinerary that ensures the user gets to see a variety of attractions without " + "overspending.\n" + "5. Provide a detailed cost breakdown that includes accommodation, transportation, meals, and entry fees " + "to attractions."; + + std::string content = "For a two-week trip to Japan with a $4,000 budget, I recommend planning an itinerary that balances " + "historical sites with modern attractions. The destination will be Japan, with a duration of 14 days.\n\n" + "Given your interests in both historical sites and modern attractions, you'll want to focus on cities like " + "Kyoto for its temples and traditional culture, Tokyo for its cutting-edge technology and entertainment " + "districts, and possibly Hiroshima or Nara for additional historical significance.\n\n" + "For accommodation, I suggest looking for affordable options such as budget hotels, hostels, or " + "guesthouses that offer good value without sacrificing too much comfort. Japan has excellent mid-range " + "accommodation options that can keep your lodging costs manageable.\n\n" + "Transportation should prioritize efficiencyβ€”consider getting a JR Rail Pass for intercity travel, which " + "allows unlimited rides on most JR trains including the Shinkansen (bullet train). Within cities, use " + "local trains and subways, which are both affordable and highly reliable.\n\n" + "For meals, embrace local cuisine by eating at neighborhood restaurants, ramen shops, and izakayas rather " + "than touristy establishments. This will give you an authentic experience while keeping costs " + "reasonableβ€”you can enjoy excellent meals for $10-20 per person at local spots.\n\n"; + + std::vector> tool_calls = { + { "call_0", "plan_trip", nlohmann::json::parse(R"({ + "destination": "Japan", + "duration": 14, + "budget": 4000, + "interests": ["historical sites", "modern attractions"], + "accommodation_preferences": "affordable", + "transportation_preferences": "efficient", + "meal_preferences": "local cuisine" + })") } + }; + + std::vector tokens; + + // Build tokens + if (!reasoning.empty()) { + auto tokenized = simple_tokenize(reasoning); + tokens.emplace_back("<|START_THINKING|>"); + tokens.insert(tokens.end(), tokenized.begin(), tokenized.end()); + tokens.emplace_back("<|END_THINKING|>"); + } + + if (!content.empty()) { + auto tokenized = simple_tokenize(content); + tokens.emplace_back("<|START_RESPONSE|>"); + tokens.insert(tokens.end(), tokenized.begin(), tokenized.end()); + tokens.emplace_back("<|END_RESPONSE|>"); + } + + if (!tool_calls.empty()) { + tokens.emplace_back("<|START_ACTION|>"); + + auto json = nlohmann::json::array(); + for (const auto & tc : tool_calls) { + auto tc_json = nlohmann::json::object(); + tc_json["tool_call_id"] = std::get<0>(tc); + tc_json["tool_name"] = std::get<1>(tc); + tc_json["parameters"] = std::get<2>(tc); + json.push_back(tc_json); + } + + auto tokenized = simple_tokenize(json.dump(-1, ' ', true)); + tokens.insert(tokens.end(), tokenized.begin(), tokenized.end()); + + tokens.emplace_back("<|END_ACTION|>"); + } + + std::string input = std::accumulate(tokens.begin(), tokens.end(), std::string()); + + // Run tests + t.test("legacy_parse", [&](testing & t) { + bool no_error = true; + try { + test_command_r7b_legacy_parser(input, false, false); + } catch (std::exception &e) { + no_error = false; + std::cerr << "Error during legacy run: " << e.what() << "\n"; + } + t.assert_equal("no_errors", true, no_error); + }); + + t.test("current_parse", [&](testing & t) { + bool no_error = true; + try { + test_command_r7b_parser(parser, input, false, false); + } catch (std::exception &e) { + no_error = false; + std::cerr << "Error during current run: " << e.what() << "\n"; + } + t.assert_equal("no_errors", true, no_error); + }); + + // Run benchmarks + t.bench("legacy_parse_benchmark", [&]() { + test_command_r7b_legacy_parser(input, false, false); + }, 1000); + + t.bench("current_parse_benchmark", [&]() { + test_command_r7b_parser(parser, input, false, false); + }, 1000); +} \ No newline at end of file diff --git a/tests/chat-peg-parser/test-example-qwen3-coder.cpp b/tests/chat-peg-parser/test-example-qwen3-coder.cpp index bf0b065060d6a..ba7aa93ceb04e 100644 --- a/tests/chat-peg-parser/test-example-qwen3-coder.cpp +++ b/tests/chat-peg-parser/test-example-qwen3-coder.cpp @@ -3,8 +3,8 @@ #include #include -test_example_qwen3_coder::test_example_qwen3_coder() : compound_test("sample_qwen3_coder_test") { - parser = build_peg_parser([](common_chat_peg_parser_builder & p) { +void test_example_qwen3_coder(testing &t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { auto thinking = p.add_rule("raw-reasoning", "" << p.add_rule("reasoning-content", p.until("")) << ""); @@ -75,7 +75,7 @@ test_example_qwen3_coder::test_example_qwen3_coder() : compound_test("sample_qwe } }; - add_test([&](test_harness h) { + t.test("accumulation_test", [&](testing &t) { std::string input = "The user wants to find large log files that haven't been accessed recently. " "I should search for files with .log extension, filter by size (over 100MB), " @@ -111,11 +111,11 @@ test_example_qwen3_coder::test_example_qwen3_coder() : compound_test("sample_qwe ctx.event_handler = handler; auto result = parser.parse(ctx); - h.assert_equals(std::string("should_not_fail_token_") + std::to_string(token_cnt), false, result.fail()); + t.assert_equal(std::string("should_not_fail_token_") + std::to_string(token_cnt), false, result.fail()); // This shouldn't emit any runtime errors auto diffs = common_chat_msg_diff::compute_diffs(prev, env.result); prev = env.result; } - }, "accumulation_test"); + }); } diff --git a/tests/chat-peg-parser/test-gbnf-generation.cpp b/tests/chat-peg-parser/test-gbnf-generation.cpp index 2030b66da4b2f..c1fae1ec7f364 100644 --- a/tests/chat-peg-parser/test-gbnf-generation.cpp +++ b/tests/chat-peg-parser/test-gbnf-generation.cpp @@ -1,154 +1,129 @@ #include "json-schema-to-grammar.h" #include "tests.h" -test_gbnf_generation::test_gbnf_generation() : compound_test("test_gbnf_generation") { +void test_gbnf_generation(testing &t) { // Test literal - add_test( - [](test_harness h) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello"); }); + t.test("literal grammar generation", [](testing &t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello"); }); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); - h.assert_equals("has_root_hello", true, gbnf.find("root ::= \"hello\"") != std::string::npos); - h.assert_equals("has_space", true, gbnf.find("space ::=") != std::string::npos); - }, - "literal grammar generation"); + t.assert_equal("has_root_hello", true, gbnf.find("root ::= \"hello\"") != std::string::npos); + t.assert_equal("has_space", true, gbnf.find("space ::=") != std::string::npos); + }); // Test char class - add_test( - [](test_harness h) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[a-z]"); }); + t.test("char class grammar", [](testing &t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[a-z]"); }); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); - h.assert_equals("has_char_class", true, gbnf.find("root ::= [a-z]") != std::string::npos); - }, - "char class grammar"); + t.assert_equal("has_char_class", true, gbnf.find("root ::= [a-z]") != std::string::npos); + }); // Test sequence - add_test( - [](test_harness h) { - auto parser = build_peg_parser( - [](common_chat_peg_parser_builder & p) { return p.literal("hello") + p.literal(" ") + p.literal("world"); }); + t.test("sequence grammar", [](testing &t) { + auto parser = build_peg_parser( + [](common_chat_peg_parser_builder & p) { return p.literal("hello") + p.literal(" ") + p.literal("world"); }); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); - h.assert_equals("has_proper_sequence", true, - gbnf.find("root ::= \"hello\" \" \" \"world\"") != std::string::npos); - }, - "sequence grammar"); + t.assert_equal("has_proper_sequence", true, + gbnf.find("root ::= \"hello\" \" \" \"world\"") != std::string::npos); + }); // Test choice - add_test( - [](test_harness h) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("cat") | p.literal("dog"); }); + t.test("choice grammar", [](testing &t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("cat") | p.literal("dog"); }); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); - h.assert_equals("has_proper_choice", true, gbnf.find("root ::= \"cat\" | \"dog\"") != std::string::npos); - }, - "choice grammar"); + t.assert_equal("has_proper_choice", true, gbnf.find("root ::= \"cat\" | \"dog\"") != std::string::npos); + }); // Test one_or_more - add_test( - [](test_harness h) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one_or_more(p.one("[0-9]")); }); + t.test("one_or_more grammar", [](testing &t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one_or_more(p.one("[0-9]")); }); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); - h.assert_equals("has_proper_one_or_more", true, gbnf.find("root ::= [0-9]+") != std::string::npos); - }, - "one_or_more grammar"); + t.assert_equal("has_proper_one_or_more", true, gbnf.find("root ::= [0-9]+") != std::string::npos); + }); // Test zero_or_more - add_test( - [](test_harness h) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.zero_or_more(p.one("[a-z]")); }); + t.test("zero_or_more grammar", [](testing &t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.zero_or_more(p.one("[a-z]")); }); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); - h.assert_equals("has_proper_zero_or_more", true, gbnf.find("root ::= [a-z]*") != std::string::npos); - }, - "zero_or_more grammar"); + t.assert_equal("has_proper_zero_or_more", true, gbnf.find("root ::= [a-z]*") != std::string::npos); + }); // Test optional - add_test( - [](test_harness h) { - auto parser = - build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); }); + t.test("optional grammar", [](testing &t) { + auto parser = + build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); }); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); - h.assert_equals("has_proper_optional", true, - gbnf.find("root ::= \"hello\" \" world\"?") != std::string::npos); - }, - "optional grammar"); + t.assert_equal("has_proper_optional", true, + gbnf.find("root ::= \"hello\" \" world\"?") != std::string::npos); + }); // Test until - add_test( - [](test_harness h) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.until(""); }); + t.test("until grammar", [](testing &t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.until(""); }); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); - // Should generate pattern that prevents matching the full delimiter - h.assert_equals( - "has_proper_until", true, - gbnf.find( - "root ::= ([^<] | \"<\" [^/] | \"])*") != - std::string::npos); - }, - "until grammar"); + // Should generate pattern that prevents matching the full delimiter + t.assert_equal("has_proper_until", true, + gbnf.find( + "root ::= ([^<] | \"<\" [^/] | \"])*") != + std::string::npos); + }); // Test complex expression with parentheses - add_test( - [](test_harness h) { - auto parser = - build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one_or_more(p.literal("a") | p.literal("b")); }); + t.test("complex expressions with parentheses", [](testing &t) { + auto parser = + build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one_or_more(p.literal("a") | p.literal("b")); }); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); - h.assert_equals("has_proper_complex", true, gbnf.find("root ::= (\"a\" | \"b\")+") != std::string::npos); - }, - "complex expressions with parentheses"); + t.assert_equal("has_proper_complex", true, gbnf.find("root ::= (\"a\" | \"b\")+") != std::string::npos); + }); // Test rule references - add_test( - [](test_harness h) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { - auto digit = p.add_rule("digit", p.one("[0-9]")); - return p.one_or_more(digit); - }); + t.test("rule references", [](testing &t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto digit = p.add_rule("digit", p.one("[0-9]")); + return p.one_or_more(digit); + }); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); - // Should have digit rule defined and referenced - h.assert_equals("has_digit_rule", true, gbnf.find("digit ::= [0-9]") != std::string::npos); - h.assert_equals("has_root_digit_ref", true, gbnf.find("root ::= digit+") != std::string::npos); - }, - "rule references"); + // Should have digit rule defined and referenced + t.assert_equal("has_digit_rule", true, gbnf.find("digit ::= [0-9]") != std::string::npos); + t.assert_equal("has_root_digit_ref", true, gbnf.find("root ::= digit+") != std::string::npos); + }); // Test escaping in literals - add_test( - [](test_harness h) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello\nworld\t!"); }); + t.test("escaping in literals", [](testing &t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello\nworld\t!"); }); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); - h.assert_equals("has_escaping", true, gbnf.find("root ::= \"hello\\nworld\\t!\"") != std::string::npos); - }, - "escaping in literals"); + t.assert_equal("has_escaping", true, gbnf.find("root ::= \"hello\\nworld\\t!\"") != std::string::npos); + }); // Test operator<< (whitespace insertion) - add_test( - [](test_harness h) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello") << p.literal("world"); }); - - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); - - // Should inline the whitespace pattern - h.assert_equals("has_inlined_hello", true, gbnf.find("\"hello\"") != std::string::npos); - h.assert_equals("has_inlined_world", true, gbnf.find("\"world\"") != std::string::npos); - }, - "operator<< (whitespace insertion)"); -} + t.test("operator<< (whitespace insertion)", [](testing &t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello") << p.literal("world"); }); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); + + // Should inline the whitespace pattern + t.assert_equal("has_inlined_hello", true, gbnf.find("\"hello\"") != std::string::npos); + t.assert_equal("has_inlined_world", true, gbnf.find("\"world\"") != std::string::npos); + }); +} \ No newline at end of file diff --git a/tests/chat-peg-parser/test-json-parser.cpp b/tests/chat-peg-parser/test-json-parser.cpp index 5a1b133b5fcdf..c10bf56c304cb 100644 --- a/tests/chat-peg-parser/test-json-parser.cpp +++ b/tests/chat-peg-parser/test-json-parser.cpp @@ -1,91 +1,79 @@ #include "tests.h" -test_json_parser::test_json_parser() : compound_test("test_json_parser") { +void test_json_parser(testing &t) { // Test parsing a simple JSON object - add_test( - [](test_harness h) { - auto json = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.json(); }); + t.test("simple JSON object parsing", [](testing &t) { + auto json = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.json(); }); - std::string input = R"({"name": "test", "value": 42, "flag": true})"; - common_chat_parse_context ctx(input); + std::string input = R"({"name": "test", "value": 42, "flag": true})"; + common_chat_parse_context ctx(input); - auto result = json.parse(ctx); + auto result = json.parse(ctx); - h.assert_equals("result_is_success", true, result.success()); - h.assert_equals("result_end", input.size(), result.end); - }, - "simple JSON object parsing"); + t.assert_equal("result_is_success", true, result.success()); + t.assert_equal("result_end", input.size(), result.end); + }); // Test parsing a JSON array with mixed types - add_test( - [](test_harness h) { - auto json = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.json(); }); + t.test("JSON array with mixed types", [](testing &t) { + auto json = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.json(); }); - std::string input = R"([1, "hello", true, null, 3.14])"; - common_chat_parse_context ctx(input); + std::string input = R"([1, "hello", true, null, 3.14])"; + common_chat_parse_context ctx(input); - auto result = json.parse(ctx); + auto result = json.parse(ctx); - h.assert_equals("result_is_success", true, result.success()); - h.assert_equals("result_end", input.size(), result.end); - }, - "JSON array with mixed types"); + t.assert_equal("result_is_success", true, result.success()); + t.assert_equal("result_end", input.size(), result.end); + }); // Test parsing nested JSON with objects and arrays - add_test( - [](test_harness h) { - auto json = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.json(); }); + t.test("nested JSON with objects and arrays", [](testing &t) { + auto json = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.json(); }); - std::string input = - R"({"users": [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}], "count": 2, "metadata": {"version": "1.0", "tags": ["admin", "user"]}})"; - common_chat_parse_context ctx(input); + std::string input = + R"({"users": [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}], "count": 2, "metadata": {"version": "1.0", "tags": ["admin", "user"]}})"; + common_chat_parse_context ctx(input); - auto result = json.parse(ctx); + auto result = json.parse(ctx); - h.assert_equals("result_is_success", true, result.success()); - h.assert_equals("result_end", input.size(), result.end); - }, - "nested JSON with objects and arrays"); + t.assert_equal("result_is_success", true, result.success()); + t.assert_equal("result_end", input.size(), result.end); + }); // Test need_more_input() parsing - incomplete object - add_test( - [](test_harness h) { - auto json = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.json(); }); + t.test("need_more_input() parsing - incomplete object", [](testing &t) { + auto json = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.json(); }); - std::string input = R"({"name": "test", "value": )"; - common_chat_parse_context ctx(input, false); + std::string input = R"({"name": "test", "value": )"; + common_chat_parse_context ctx(input, false); - auto result = json.parse(ctx); + auto result = json.parse(ctx); - h.assert_equals("result_is_need_more_input", true, result.need_more_input()); - }, - "need_more_input() parsing - incomplete object"); + t.assert_equal("result_is_need_more_input", true, result.need_more_input()); + }); // Test need_more_input() parsing - incomplete array - add_test( - [](test_harness h) { - auto json = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.json(); }); + t.test("need_more_input() parsing - incomplete array", [](testing &t) { + auto json = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.json(); }); - std::string input = R"([1, 2, 3, )"; - common_chat_parse_context ctx(input, false); + std::string input = R"([1, 2, 3, )"; + common_chat_parse_context ctx(input, false); - auto result = json.parse(ctx); + auto result = json.parse(ctx); - h.assert_equals("result_is_need_more_input", true, result.need_more_input()); - }, - "need_more_input() parsing - incomplete array"); + t.assert_equal("result_is_need_more_input", true, result.need_more_input()); + }); // Test need_more_input() parsing - incomplete nested structure - add_test( - [](test_harness h) { - auto json = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.json(); }); + t.test("need_more_input() parsing - incomplete nested structure", [](testing &t) { + auto json = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.json(); }); - std::string input = R"({"data": {"nested": )"; - common_chat_parse_context ctx(input, false); + std::string input = R"({"data": {"nested": )"; + common_chat_parse_context ctx(input, false); - auto result = json.parse(ctx); + auto result = json.parse(ctx); - h.assert_equals("result_is_need_more_input", true, result.need_more_input()); - }, - "need_more_input() parsing - incomplete nested structure"); -} + t.assert_equal("result_is_need_more_input", true, result.need_more_input()); + }); +} \ No newline at end of file diff --git a/tests/chat-peg-parser/test-one.cpp b/tests/chat-peg-parser/test-one.cpp index 3c8a110cc3284..5980f76024b77 100644 --- a/tests/chat-peg-parser/test-one.cpp +++ b/tests/chat-peg-parser/test-one.cpp @@ -1,115 +1,99 @@ #include "tests.h" -test_one::test_one() : compound_test("test_one") { +void test_one(testing &t) { // Test common escape sequences - newline - add_test( - [](test_harness h) { - auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); + t.test("escape_sequence_newline", [](testing &t) { + auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); - common_chat_parse_context ctx; - common_chat_parse_result result; + common_chat_parse_context ctx; + common_chat_parse_result result; - ctx = common_chat_parse_context("\n"); - result = common_chat_combinator_parser.parse(ctx); - h.assert_equals("escape_sequence_newline", true, result.success()); - }, - "escape_sequence_newline"); + ctx = common_chat_parse_context("\n"); + result = common_chat_combinator_parser.parse(ctx); + t.assert_equal("escape_sequence_newline", true, result.success()); + }); // Test common escape sequences - tab - add_test( - [](test_harness h) { - auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); + t.test("escape_sequence_tab", [](testing &t) { + auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); - common_chat_parse_context ctx; - common_chat_parse_result result; + common_chat_parse_context ctx; + common_chat_parse_result result; - ctx = common_chat_parse_context("\t"); - result = common_chat_combinator_parser.parse(ctx); - h.assert_equals("escape_sequence_tab", true, result.success()); - }, - "escape_sequence_tab"); + ctx = common_chat_parse_context("\t"); + result = common_chat_combinator_parser.parse(ctx); + t.assert_equal("escape_sequence_tab", true, result.success()); + }); // Test common escape sequences - backslash - add_test( - [](test_harness h) { - auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); + t.test("escape_sequence_backslash", [](testing &t) { + auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); - common_chat_parse_context ctx; - common_chat_parse_result result; + common_chat_parse_context ctx; + common_chat_parse_result result; - ctx = common_chat_parse_context("\\"); - result = common_chat_combinator_parser.parse(ctx); - h.assert_equals("escape_sequence_backslash", true, result.success()); - }, - "escape_sequence_backslash"); + ctx = common_chat_parse_context("\\"); + result = common_chat_combinator_parser.parse(ctx); + t.assert_equal("escape_sequence_backslash", true, result.success()); + }); // Test common escape sequences - space (should ()) - add_test( - [](test_harness h) { - auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); + t.test("escape_sequence_space_fail", [](testing &t) { + auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); - common_chat_parse_context ctx; - common_chat_parse_result result; + common_chat_parse_context ctx; + common_chat_parse_result result; - ctx = common_chat_parse_context(" "); - result = common_chat_combinator_parser.parse(ctx); - h.assert_equals("escape_sequence_space_fail", true, result.fail()); - }, - "escape_sequence_space_fail"); + ctx = common_chat_parse_context(" "); + result = common_chat_combinator_parser.parse(ctx); + t.assert_equal("escape_sequence_space_fail", true, result.fail()); + }); // Test escaped dash - 'a' should succeed - add_test( - [](test_harness h) { - auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[a\\-z]"); }); + t.test("escaped_dash_a", [](testing &t) { + auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[a\\-z]"); }); - common_chat_parse_context ctx; - common_chat_parse_result result; + common_chat_parse_context ctx; + common_chat_parse_result result; - ctx = common_chat_parse_context("a"); - result = common_chat_combinator_parser.parse(ctx); - h.assert_equals("escaped_dash_a", true, result.success()); - }, - "escaped_dash_a"); + ctx = common_chat_parse_context("a"); + result = common_chat_combinator_parser.parse(ctx); + t.assert_equal("escaped_dash_a", true, result.success()); + }); // Test escaped dash - '-' should succeed (literal dash) - add_test( - [](test_harness h) { - auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[a\\-z]"); }); + t.test("escaped_dash_literal", [](testing &t) { + auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[a\\-z]"); }); - common_chat_parse_context ctx; - common_chat_parse_result result; + common_chat_parse_context ctx; + common_chat_parse_result result; - ctx = common_chat_parse_context("-"); - result = common_chat_combinator_parser.parse(ctx); - h.assert_equals("escaped_dash_literal", true, result.success()); - }, - "escaped_dash_literal"); + ctx = common_chat_parse_context("-"); + result = common_chat_combinator_parser.parse(ctx); + t.assert_equal("escaped_dash_literal", true, result.success()); + }); // Test escaped dash - 'z' should succeed - add_test( - [](test_harness h) { - auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[a\\-z]"); }); + t.test("escaped_dash_z", [](testing &t) { + auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[a\\-z]"); }); - common_chat_parse_context ctx; - common_chat_parse_result result; + common_chat_parse_context ctx; + common_chat_parse_result result; - ctx = common_chat_parse_context("z"); - result = common_chat_combinator_parser.parse(ctx); - h.assert_equals("escaped_dash_z", true, result.success()); - }, - "escaped_dash_z"); + ctx = common_chat_parse_context("z"); + result = common_chat_combinator_parser.parse(ctx); + t.assert_equal("escaped_dash_z", true, result.success()); + }); // Test escaped dash - 'b' should NOT match (since \- is literal dash, not range) - add_test( - [](test_harness h) { - auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[a\\-z]"); }); - - common_chat_parse_context ctx; - common_chat_parse_result result; - - ctx = common_chat_parse_context("b"); - result = common_chat_combinator_parser.parse(ctx); - h.assert_equals("escaped_dash_b_fail", true, result.fail()); - }, - "escaped_dash_b_fail"); + t.test("escaped_dash_b_fail", [](testing &t) { + auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[a\\-z]"); }); + + common_chat_parse_context ctx; + common_chat_parse_result result; + + ctx = common_chat_parse_context("b"); + result = common_chat_combinator_parser.parse(ctx); + t.assert_equal("escaped_dash_b_fail", true, result.fail()); + }); } diff --git a/tests/chat-peg-parser/test-optional.cpp b/tests/chat-peg-parser/test-optional.cpp index f7c794f8886dd..6ea23a3abf4b5 100644 --- a/tests/chat-peg-parser/test-optional.cpp +++ b/tests/chat-peg-parser/test-optional.cpp @@ -1,43 +1,37 @@ #include "tests.h" -test_optional::test_optional() : compound_test("test_optional") { +void test_optional(testing &t) { // Full match with optional part present - add_test( - [](test_harness h) { - auto parser = - build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); }); + t.test("optional_present", [](testing &t) { + auto parser = + build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); }); - auto ctx = common_chat_parse_context("hello world"); - auto result = parser.parse(ctx); - h.assert_equals("optional_present", true, result.success()); - int end_pos = result.end; - h.assert_equals("optional_present_end", 11, end_pos); - }, - "optional_present"); + auto ctx = common_chat_parse_context("hello world"); + auto result = parser.parse(ctx); + t.assert_equal("optional_present", true, result.success()); + int end_pos = result.end; + t.assert_equal("optional_present_end", 11, end_pos); + }); // Full match with optional part absent - add_test( - [](test_harness h) { - auto parser = - build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); }); + t.test("optional_absent", [](testing &t) { + auto parser = + build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); }); - auto ctx = common_chat_parse_context("hello", true); - auto result = parser.parse(ctx); - h.assert_equals("optional_absent", true, result.success()); - int end_pos = result.end; - h.assert_equals("optional_absent_end", 5, end_pos); - }, - "optional_absent"); + auto ctx = common_chat_parse_context("hello", true); + auto result = parser.parse(ctx); + t.assert_equal("optional_absent", true, result.success()); + int end_pos = result.end; + t.assert_equal("optional_absent_end", 5, end_pos); + }); // Partial match - waiting for more input to determine if optional matches - add_test( - [](test_harness h) { - auto parser = - build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); }); + t.test("partial_match_need_more", [](testing &t) { + auto parser = + build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); }); - auto ctx = common_chat_parse_context("hello ", false); - auto result = parser.parse(ctx); - h.assert_equals("partial_match_need_more", true, result.need_more_input()); - }, - "partial_match_need_more"); + auto ctx = common_chat_parse_context("hello ", false); + auto result = parser.parse(ctx); + t.assert_equal("partial_match_need_more", true, result.need_more_input()); + }); } diff --git a/tests/chat-peg-parser/test-partial-parsing.cpp b/tests/chat-peg-parser/test-partial-parsing.cpp index 8fa58c8198e0c..0779be5e55028 100644 --- a/tests/chat-peg-parser/test-partial-parsing.cpp +++ b/tests/chat-peg-parser/test-partial-parsing.cpp @@ -1,275 +1,230 @@ #include "tests.h" +#include "test_harness.h" -test_partial_parsing::test_partial_parsing() : compound_test("test_partial_parsing") { +void test_partial_parsing(testing &t) { // Literals - Basic Success - add_test( - [](test_harness h) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello"); }); + t.test("literal_success", [&](testing & t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello"); }); - common_chat_parse_context ctx; - common_chat_parse_result result; + common_chat_parse_context ctx; + common_chat_parse_result result; - ctx = common_chat_parse_context("hello"); - result = parser.parse(ctx); - h.assert_equals("literal_success", true, result.success()); - }, - "literal_success"); + ctx = common_chat_parse_context("hello"); + result = parser.parse(ctx); + t.assert_equal("literal_success", true, result.success()); + }); // Char Classes - Basic Lowercase Success - add_test( - [](test_harness h) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("a-z"); }); + t.test("char_class_lowercase_success", [&](testing & t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("a-z"); }); - common_chat_parse_context ctx; - common_chat_parse_result result; + common_chat_parse_context ctx; + common_chat_parse_result result; - ctx = common_chat_parse_context("a"); - result = parser.parse(ctx); - h.assert_equals("char_class_lowercase_success", true, result.success()); - }, - "char_class_lowercase_success"); + ctx = common_chat_parse_context("a"); + result = parser.parse(ctx); + t.assert_equal("char_class_lowercase_success", true, result.success()); + }); // Char Classes - Uppercase Fail - add_test( - [](test_harness h) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("a-z"); }); + t.test("char_class_uppercase_fail", [&](testing & t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("a-z"); }); - common_chat_parse_context ctx; - common_chat_parse_result result; + common_chat_parse_context ctx; + common_chat_parse_result result; - ctx = common_chat_parse_context("A"); - result = parser.parse(ctx); - h.assert_equals("char_class_uppercase_fail", true, result.fail()); - }, - "char_class_uppercase_fail"); + ctx = common_chat_parse_context("A"); + result = parser.parse(ctx); + t.assert_equal("char_class_uppercase_fail", true, result.fail()); + }); // Char Classes with Dash - Lowercase Success - add_test( - [](test_harness h) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("a-z-"); }); + t.test("char_class_with_dash_lowercase", [&](testing & t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("a-z-"); }); - common_chat_parse_context ctx; - common_chat_parse_result result; + common_chat_parse_context ctx; + common_chat_parse_result result; - ctx = common_chat_parse_context("f"); - result = parser.parse(ctx); - h.assert_equals("char_class_with_dash_lowercase", true, result.success()); - }, - "char_class_with_dash_lowercase"); + ctx = common_chat_parse_context("f"); + result = parser.parse(ctx); + t.assert_equal("char_class_with_dash_lowercase", true, result.success()); + }); // Char Classes with Dash - Literal Dash Success - add_test( - [](test_harness h) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("a-z-"); }); + t.test("char_class_with_dash_literal_dash", [&](testing & t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("a-z-"); }); - common_chat_parse_context ctx; - common_chat_parse_result result; + common_chat_parse_context ctx; + common_chat_parse_result result; - ctx = common_chat_parse_context("-"); - result = parser.parse(ctx); - h.assert_equals("char_class_with_dash_literal_dash", true, result.success()); - }, - "char_class_with_dash_literal_dash"); + ctx = common_chat_parse_context("-"); + result = parser.parse(ctx); + t.assert_equal("char_class_with_dash_literal_dash", true, result.success()); + }); // Char Classes with Dash - Uppercase Fail - add_test( - [](test_harness h) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("a-z-"); }); + t.test("char_class_with_dash_uppercase_fail", [&](testing & t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("a-z-"); }); - common_chat_parse_context ctx; - common_chat_parse_result result; + common_chat_parse_context ctx; + common_chat_parse_result result; - ctx = common_chat_parse_context("A"); - result = parser.parse(ctx); - h.assert_equals("char_class_with_dash_uppercase_fail", true, result.fail()); - }, - "char_class_with_dash_uppercase_fail"); + ctx = common_chat_parse_context("A"); + result = parser.parse(ctx); + t.assert_equal("char_class_with_dash_uppercase_fail", true, result.fail()); + }); // Sequences - Partial Match 1 - add_test( - [](test_harness h) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("") + p.literal(""); }); + t.test("sequence_partial_match_1", [&](testing & t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("") + p.literal(""); }); - auto ctx = common_chat_parse_context("") + p.literal(""); }); + t.test("sequence_partial_match_3", [&](testing & t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("") + p.literal(""); }); - auto ctx = common_chat_parse_context("") + p.literal(""); }); + t.test("sequence_no_match", [&](testing & t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("") + p.literal(""); }); - auto ctx = common_chat_parse_context("I am common_chat_combinator_parser", false); - auto result = parser.parse(ctx); - h.assert_equals("sequence_no_match", true, result.fail()); - }, - "sequence_no_match"); + auto ctx = common_chat_parse_context("I am common_chat_combinator_parser", false); + auto result = parser.parse(ctx); + t.assert_equal("sequence_no_match", true, result.fail()); + }); // Choices - Partial Match 1 - add_test( - [](test_harness h) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("option1") | p.literal("option2"); }); + t.test("choices_partial_match_1", [&](testing & t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("option1") | p.literal("option2"); }); - auto ctx = common_chat_parse_context("opt", false); - auto result = parser.parse(ctx); - h.assert_equals("choices_partial_match_1", true, result.need_more_input()); - }, - "choices_partial_match_1"); + auto ctx = common_chat_parse_context("opt", false); + auto result = parser.parse(ctx); + t.assert_equal("choices_partial_match_1", true, result.need_more_input()); + }); // Choices - Partial Match 2 - add_test( - [](test_harness h) { - auto parser = - build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("choice_a") | p.literal("choice_b"); }); + t.test("choices_partial_match_2", [&](testing & t) { + auto parser = + build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("choice_a") | p.literal("choice_b"); }); - auto ctx = common_chat_parse_context("choice", false); - auto result = parser.parse(ctx); - h.assert_equals("choices_partial_match_2", true, result.need_more_input()); - }, - "choices_partial_match_2"); + auto ctx = common_chat_parse_context("choice", false); + auto result = parser.parse(ctx); + t.assert_equal("choices_partial_match_2", true, result.need_more_input()); + }); // Choices - Full Match 1 - add_test( - [](test_harness h) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("first") | p.literal("second"); }); + t.test("choices_full_match_1", [&](testing & t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("first") | p.literal("second"); }); - auto ctx = common_chat_parse_context("first", true); - auto result = parser.parse(ctx); - h.assert_equals("choices_full_match_1", true, result.success()); - }, - "choices_full_match_1"); + auto ctx = common_chat_parse_context("first", true); + auto result = parser.parse(ctx); + t.assert_equal("choices_full_match_1", true, result.success()); + }); // Choices - Full Match 2 - add_test( - [](test_harness h) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("alpha") | p.literal("beta"); }); + t.test("choices_full_match_2", [&](testing & t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("alpha") | p.literal("beta"); }); - auto ctx = common_chat_parse_context("beta", true); - auto result = parser.parse(ctx); - h.assert_equals("choices_full_match_2", true, result.success()); - }, - "choices_full_match_2"); + auto ctx = common_chat_parse_context("beta", true); + auto result = parser.parse(ctx); + t.assert_equal("choices_full_match_2", true, result.success()); + }); // Choices - No Match - add_test( - [](test_harness h) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("good") | p.literal("better"); }); + t.test("choices_no_match", [&](testing & t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("good") | p.literal("better"); }); - auto ctx = common_chat_parse_context("best", true); - auto result = parser.parse(ctx); - h.assert_equals("choices_no_match", true, result.fail()); - }, - "choices_no_match"); + auto ctx = common_chat_parse_context("best", true); + auto result = parser.parse(ctx); + t.assert_equal("choices_no_match", true, result.fail()); + }); // Zero or More - Partial Match 1 - add_test( - [](test_harness h) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.zero_or_more(p.literal("ab")); }); + t.test("zero_or_more_partial_match_1", [&](testing & t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.zero_or_more(p.literal("ab")); }); - auto ctx = common_chat_parse_context("a", false); - auto result = parser.parse(ctx); - h.assert_equals("zero_or_more_partial_match_1", true, result.need_more_input()); - }, - "zero_or_more_partial_match_1"); + auto ctx = common_chat_parse_context("a", false); + auto result = parser.parse(ctx); + t.assert_equal("zero_or_more_partial_match_1", true, result.need_more_input()); + }); // Zero or More - Partial Match 2 - add_test( - [](test_harness h) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.zero_or_more(p.literal("xy")); }); + t.test("zero_or_more_partial_match_2", [&](testing & t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.zero_or_more(p.literal("xy")); }); - auto ctx = common_chat_parse_context("xyx", false); - auto result = parser.parse(ctx); - h.assert_equals("zero_or_more_partial_match_2", true, result.need_more_input()); - }, - "zero_or_more_partial_match_2"); + auto ctx = common_chat_parse_context("xyx", false); + auto result = parser.parse(ctx); + t.assert_equal("zero_or_more_partial_match_2", true, result.need_more_input()); + }); // Zero or More - Full Match - add_test( - [](test_harness h) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.zero_or_more(p.literal("test")); }); + t.test("zero_or_more_full_match", [&](testing & t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.zero_or_more(p.literal("test")); }); - auto ctx = common_chat_parse_context("test", true); - auto result = parser.parse(ctx); - h.assert_equals("zero_or_more_full_match", true, result.success()); - }, - "zero_or_more_full_match"); + auto ctx = common_chat_parse_context("test", true); + auto result = parser.parse(ctx); + t.assert_equal("zero_or_more_full_match", true, result.success()); + }); // One or More - Partial Match 1 - add_test( - [](test_harness h) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one_or_more(p.literal("repeat")); }); + t.test("one_or_more_partial_match_1", [&](testing & t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one_or_more(p.literal("repeat")); }); - auto ctx = common_chat_parse_context("rep", false); - auto result = parser.parse(ctx); - h.assert_equals("one_or_more_partial_match_1", true, result.need_more_input()); - }, - "one_or_more_partial_match_1"); + auto ctx = common_chat_parse_context("rep", false); + auto result = parser.parse(ctx); + t.assert_equal("one_or_more_partial_match_1", true, result.need_more_input()); + }); // One or More - Partial Match 2 - add_test( - [](test_harness h) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one_or_more(p.literal("ab")); }); + t.test("one_or_more_partial_match_2", [&](testing & t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one_or_more(p.literal("ab")); }); - auto ctx = common_chat_parse_context("aba", false); - auto result = parser.parse(ctx); - h.assert_equals("one_or_more_partial_match_2", true, result.need_more_input()); - }, - "one_or_more_partial_match_2"); + auto ctx = common_chat_parse_context("aba", false); + auto result = parser.parse(ctx); + t.assert_equal("one_or_more_partial_match_2", true, result.need_more_input()); + }); // One or More - Full Match - add_test( - [](test_harness h) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one_or_more(p.literal("single")); }); + t.test("one_or_more_full_match", [&](testing & t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one_or_more(p.literal("single")); }); - auto ctx = common_chat_parse_context("single", true); - auto result = parser.parse(ctx); - h.assert_equals("one_or_more_full_match", true, result.success()); - }, - "one_or_more_full_match"); + auto ctx = common_chat_parse_context("single", true); + auto result = parser.parse(ctx); + t.assert_equal("one_or_more_full_match", true, result.success()); + }); // One or More - No Match - add_test( - [](test_harness h) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one_or_more(p.literal("()")); }); - - auto ctx = common_chat_parse_context("success", true); - auto result = parser.parse(ctx); - h.assert_equals("one_or_more_no_match", true, result.fail()); - }, - "one_or_more_no_match"); + t.test("one_or_more_no_match", [&](testing & t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one_or_more(p.literal("()")); }); + + auto ctx = common_chat_parse_context("success", true); + auto result = parser.parse(ctx); + t.assert_equal("one_or_more_no_match", true, result.fail()); + }); } diff --git a/tests/chat-peg-parser/test-recursive-references.cpp b/tests/chat-peg-parser/test-recursive-references.cpp index 79aa1cf82a4ad..6fe38f2aea71c 100644 --- a/tests/chat-peg-parser/test-recursive-references.cpp +++ b/tests/chat-peg-parser/test-recursive-references.cpp @@ -1,99 +1,87 @@ #include "tests.h" -test_recursive_references::test_recursive_references() : compound_test("test_recursive_references") { +void test_recursive_references(testing &t) { // Test simple number - add_test( - [](test_harness h) { - auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { - p.add_rule("number", p.one_or_more(p.one("0-9"))); - p.add_rule("list", p.sequence({ p.literal("["), p.rule("value"), p.literal("]") })); - return p.add_rule("value", p.rule("number") | p.rule("list")); - }); + t.test("simple_number", [](testing &t) { + auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + p.add_rule("number", p.one_or_more(p.one("0-9"))); + p.add_rule("list", p.sequence({ p.literal("["), p.rule("value"), p.literal("]") })); + return p.add_rule("value", p.rule("number") | p.rule("list")); + }); - common_chat_parse_context ctx("1", true); - auto result = value_parser.parse(ctx); + common_chat_parse_context ctx("1", true); + auto result = value_parser.parse(ctx); - h.assert_equals("result_is_success", true, result.success()); - }, - "simple_number"); + t.assert_equal("result_is_success", true, result.success()); + }); // Test simple list - add_test( - [](test_harness h) { - auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { - p.add_rule("number", p.one_or_more(p.one("0-9"))); - p.add_rule("list", p.sequence({ p.literal("["), p.rule("value"), p.literal("]") })); - return p.add_rule("value", p.rule("number") | p.rule("list")); - }); + t.test("simple_list", [](testing &t) { + auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + p.add_rule("number", p.one_or_more(p.one("0-9"))); + p.add_rule("list", p.sequence({ p.literal("["), p.rule("value"), p.literal("]") })); + return p.add_rule("value", p.rule("number") | p.rule("list")); + }); - common_chat_parse_context ctx("[1]", true); - auto result = value_parser.parse(ctx); + common_chat_parse_context ctx("[1]", true); + auto result = value_parser.parse(ctx); - h.assert_equals("result_is_success", true, result.success()); - }, - "simple_list"); + t.assert_equal("result_is_success", true, result.success()); + }); // Test nested list - add_test( - [](test_harness h) { - auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { - p.add_rule("number", p.one_or_more(p.one("0-9"))); - p.add_rule("list", p.sequence({ p.literal("["), p.rule("value"), p.literal("]") })); - return p.add_rule("value", p.rule("number") | p.rule("list")); - }); + t.test("nested_list", [](testing &t) { + auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + p.add_rule("number", p.one_or_more(p.one("0-9"))); + p.add_rule("list", p.sequence({ p.literal("["), p.rule("value"), p.literal("]") })); + return p.add_rule("value", p.rule("number") | p.rule("list")); + }); - common_chat_parse_context ctx("[[2]]", true); - auto result = value_parser.parse(ctx); + common_chat_parse_context ctx("[[2]]", true); + auto result = value_parser.parse(ctx); - h.assert_equals("result_is_success", true, result.success()); - }, - "nested_list"); + t.assert_equal("result_is_success", true, result.success()); + }); // Test deeply nested list - add_test( - [](test_harness h) { - auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { - p.add_rule("number", p.one_or_more(p.one("0-9"))); - p.add_rule("list", p.sequence({ p.literal("["), p.rule("value"), p.literal("]") })); - return p.add_rule("value", p.rule("number") | p.rule("list")); - }); + t.test("deeply_nested_list", [](testing &t) { + auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + p.add_rule("number", p.one_or_more(p.one("0-9"))); + p.add_rule("list", p.sequence({ p.literal("["), p.rule("value"), p.literal("]") })); + return p.add_rule("value", p.rule("number") | p.rule("list")); + }); - common_chat_parse_context ctx("[[[3]]]", true); - auto result = value_parser.parse(ctx); + common_chat_parse_context ctx("[[[3]]]", true); + auto result = value_parser.parse(ctx); - h.assert_equals("result_is_success", true, result.success()); - }, - "deeply_nested_list"); + t.assert_equal("result_is_success", true, result.success()); + }); // Test need_more_input match - add_test( - [](test_harness h) { - auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { - p.add_rule("number", p.one_or_more(p.one("0-9"))); - p.add_rule("list", p.sequence({ p.literal("["), p.rule("value"), p.literal("]") })); - return p.add_rule("value", p.rule("number") | p.rule("list")); - }); + t.test("need_more_input_match", [](testing &t) { + auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + p.add_rule("number", p.one_or_more(p.one("0-9"))); + p.add_rule("list", p.sequence({ p.literal("["), p.rule("value"), p.literal("]") })); + return p.add_rule("value", p.rule("number") | p.rule("list")); + }); - common_chat_parse_context ctx("[[", false); - auto result = value_parser.parse(ctx); + common_chat_parse_context ctx("[[", false); + auto result = value_parser.parse(ctx); - h.assert_equals("result_is_need_more_input", true, result.need_more_input()); - }, - "need_more_input_match"); + t.assert_equal("result_is_need_more_input", true, result.need_more_input()); + }); // Test no match - add_test( - [](test_harness h) { - auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { - p.add_rule("number", p.one_or_more(p.one("0-9"))); - p.add_rule("list", p.sequence({ p.literal("["), p.rule("value"), p.literal("]") })); - return p.add_rule("value", p.rule("number") | p.rule("list")); - }); - - common_chat_parse_context ctx("[a]", true); - auto result = value_parser.parse(ctx); - - h.assert_equals("result_is_fail", true, result.fail()); - }, - "no_match"); -} + t.test("no_match", [](testing &t) { + auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + p.add_rule("number", p.one_or_more(p.one("0-9"))); + p.add_rule("list", p.sequence({ p.literal("["), p.rule("value"), p.literal("]") })); + return p.add_rule("value", p.rule("number") | p.rule("list")); + }); + + common_chat_parse_context ctx("[a]", true); + auto result = value_parser.parse(ctx); + + t.assert_equal("result_is_fail", true, result.fail()); + }); +} \ No newline at end of file diff --git a/tests/chat-peg-parser/test_harness.h b/tests/chat-peg-parser/test_harness.h new file mode 100644 index 0000000000000..b8751badba1a6 --- /dev/null +++ b/tests/chat-peg-parser/test_harness.h @@ -0,0 +1,181 @@ +#pragma once + +#include +#include +#include +#include +#include + +struct testing { + std::ostream &out; + std::vector stack; + int tests = 0; + int assertions = 0; + int failures = 0; + int unnamed = 0; + int exceptions = 0; + + explicit testing(std::ostream &os = std::cout) : out(os) {} + + void indent() { + for (std::size_t i = 0; i < stack.size() - 1; ++i) { + out << " "; + } + } + + template + void run_with_exceptions(F &&f, const char *ctx) { + try { + f(); + } catch (const std::exception &e) { + ++failures; + ++exceptions; + indent(); + out << "UNHANDLED EXCEPTION (" << ctx << "): " << e.what() << "\n"; + } catch (...) { + ++failures; + ++exceptions; + indent(); + out << "UNHANDLED EXCEPTION (" << ctx << "): unknown\n"; + } + } + + void print_result(const std::string &label, const std::string &name, int new_failures, int new_assertions, const std::string &extra = "") { + indent(); + out << label << ": " << name << " ["; + if (new_failures == 0) { + out << "ok, "; + } else { + out << new_failures << " failed of "; + } + out << new_assertions << " assertion(s)"; + if (!extra.empty()) { + out << ", " << extra; + } + out << "]\n"; + } + + // Named test + template + void test(const std::string &name, F f) { + ++tests; + stack.push_back(name); + + indent(); + out << "BEGIN: " << name << "\n"; + + int before_failures = failures; + int before_assertions = assertions; + + run_with_exceptions([&] { f(*this); }, "test"); + + print_result("END", name, + failures - before_failures, + assertions - before_assertions); + + stack.pop_back(); + } + + // Unnamed test + template + void test(F f) { + test("test #" + std::to_string(++unnamed), f); + } + + // Named benchmark + template + void bench(const std::string &name, F f, int iterations = 100) { + ++tests; + stack.push_back(name); + + indent(); + out << "BEGIN BENCH: " << name << "\n"; + + int before_failures = failures; + int before_assertions = assertions; + + using clock = std::chrono::high_resolution_clock; + + std::chrono::microseconds duration(0); + + run_with_exceptions([&] { + for (auto i = 0; i < iterations; i++) { + auto start = clock::now(); + f(); + duration += std::chrono::duration_cast(clock::now() - start); + } + }, "bench"); + + auto avg_elapsed = duration.count() / iterations; + auto avg_elapsed_s = std::chrono::duration_cast>(duration).count(); + auto rate = (avg_elapsed_s > 0.0) ? (1.0 / avg_elapsed_s) : 0.0; + + print_result("END BENCH", name, + failures - before_failures, + assertions - before_assertions, + std::to_string(iterations) + " iteration(s), " + + "avg elapsed " + std::to_string(avg_elapsed) + + " us (" + std::to_string(rate) + " /s)"); + + stack.pop_back(); + } + + // Unnamed benchmark + template + void bench(F f, int iterations = 100) { + bench("bench #" + std::to_string(++unnamed), f, iterations); + } + + // Assertions + void assert_true(bool cond) { + assert_true("", cond); + } + + void assert_true(const std::string &msg, bool cond) { + ++assertions; + if (!cond) { + ++failures; + indent(); + out << "ASSERT TRUE FAILED"; + if (!msg.empty()) { + out << " : " << msg; + } + out << "\n"; + } + } + + template + void assert_equal(const A & expected, const B & actual) { + assert_equal("", expected, actual); + } + + template + void assert_equal(const std::string & msg, const A & expected, const B & actual) { + ++assertions; + if (!(actual == expected)) { + ++failures; + indent(); + out << "ASSERT EQUAL FAILED"; + if (!msg.empty()) { + out << " : " << msg; + } + out << "\n"; + + indent(); + out << " expected: " << expected << "\n"; + indent(); + out << " actual : " << actual << "\n"; + } + } + + // Print summary and return an exit code + int summary() { + out << "\n==== TEST SUMMARY ====\n"; + out << "tests : " << tests << "\n"; + out << "assertions : " << assertions << "\n"; + out << "failures : " << failures << "\n"; + out << "exceptions : " << exceptions << "\n"; + out << "======================\n"; + return failures == 0 ? 0 : 1; + } +}; diff --git a/tests/chat-peg-parser/tests.h b/tests/chat-peg-parser/tests.h index c2db1b96bc7e6..717578a4238b7 100644 --- a/tests/chat-peg-parser/tests.h +++ b/tests/chat-peg-parser/tests.h @@ -1,51 +1,14 @@ #pragma once // Common includes for all test files -#include "../testcase.hpp" +#include "test_harness.h" #include #include "chat-peg-parser.h" #include +#include +#include -// Test class declarations -class test_partial_parsing : public compound_test { -public: - test_partial_parsing(); -}; - -class test_one : public compound_test { -public: - test_one(); -}; - -class test_optional : public compound_test { -public: - test_optional(); -}; - -class test_recursive_references : public compound_test { -public: - test_recursive_references(); -}; - -class test_json_parser : public compound_test { -public: - test_json_parser(); -}; - -class test_actions : public compound_test { -public: - test_actions(); -}; - -class test_gbnf_generation : public compound_test { -public: - test_gbnf_generation(); -}; - -class uses_simple_tokenizer { -protected: - static std::vector simple_tokenize(const std::string &); -}; +std::vector simple_tokenize(const std::string &); struct bench_tool_call { std::string id; @@ -53,36 +16,13 @@ struct bench_tool_call { nlohmann::ordered_json args; }; -class benchmark_test { -protected: - std::vector> cases; - long long run_benchmark(size_t which, int iterations); -public: - benchmark_test(std::vector>); -}; - -class test_command7_parser_compare : public uses_simple_tokenizer, public benchmark_test { -private: - class common_chat_peg_parser parser; - common_chat_parse_event_handler handler; - - std::string reasoning; - std::string content; - std::vector tool_calls; - std::vector tokens; - // Helper methods - static class common_chat_peg_parser create_command_r7b_parser(); - static common_chat_parse_event_handler create_command_r7b_event_handler(); - static void test_command_r7b_parser(const class common_chat_peg_parser & p, const std::string & input, bool need_more_input, bool print_results = false); - static void test_command_r7b_legacy_parser(const std::string & input, bool need_more_input, bool print_results = false); -public: - test_command7_parser_compare(); - void run_comparison(int iterations); -}; - -class test_example_qwen3_coder : public uses_simple_tokenizer, public compound_test { -private: - class common_chat_peg_parser parser; -public: - test_example_qwen3_coder(); -}; +// Test function declarations +void test_partial_parsing(testing &t); +void test_one(testing &t); +void test_optional(testing &t); +void test_recursive_references(testing &t); +void test_json_parser(testing &t); +void test_actions(testing &t); +void test_gbnf_generation(testing &t); +void test_example_qwen3_coder(testing &t); +void test_command7_parser_compare(testing &t); diff --git a/tests/test-chat-peg-parser.cpp b/tests/test-chat-peg-parser.cpp index 42fd565c080f7..1b4fa4b418783 100644 --- a/tests/test-chat-peg-parser.cpp +++ b/tests/test-chat-peg-parser.cpp @@ -1,32 +1,17 @@ #include "chat-peg-parser/tests.h" int main() { - test_partial_parsing partial_parsing_test; - partial_parsing_test.run_all_tests(); - - test_one one_test; - one_test.run_all_tests(); - - test_optional optional_test; - optional_test.run_all_tests(); - - test_recursive_references recursive_references_test; - recursive_references_test.run_all_tests(); - - test_json_parser json_parser_test; - json_parser_test.run_all_tests(); - - test_actions actions_test; - actions_test.run_all_tests(); - - test_gbnf_generation gbnf_generation_test; - gbnf_generation_test.run_all_tests(); - - test_example_qwen3_coder qwen3_coder_test; - qwen3_coder_test.run_all_tests(); - - test_command7_parser_compare command7_compare_test; - command7_compare_test.run_comparison(100); - - return 0; + testing t(std::cout); + + test_partial_parsing(t); + test_one(t); + test_optional(t); + test_recursive_references(t); + test_json_parser(t); + test_actions(t); + test_gbnf_generation(t); + test_example_qwen3_coder(t); + test_command7_parser_compare(t); + + return t.summary(); } diff --git a/tests/testcase.hpp b/tests/testcase.hpp deleted file mode 100644 index b874d0cdc17a5..0000000000000 --- a/tests/testcase.hpp +++ /dev/null @@ -1,188 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -class test_harness { - private: - class test_case & tc_; - std::ostream & error_stream_; - std::string test_label_; - public: - test_harness(test_case &tc, const std::string &test_label, std::ostream & error_stream = std::cerr); - template bool assert_equals(const std::string &label, T expected, T actual); -}; - -class test_case { - friend class test_harness; - private: - std::function test_func_; - std::string name_; - int successes = 0, failures = 0, errors = 0; - bool omit_success_msg = false; - - void inc_fail() { failures++; } - - void inc_suc() { successes++; } - public: - test_case(std::function test_func, const std::string & name) : - test_func_(std::move(test_func)), - name_(name) {} - - bool run() { - // clean counters on run - successes = 0; - failures = 0; - test_harness harness(*this, name_); - // execute run with harness - try { - test_func_(harness); - } catch (std::exception & e) { - errors++; - std::cerr << "[" << get_name() << "] error during execution:\n" << e.what() << "\n"; - } - - if (is_success()) { - if (!omit_success_msg) { - std::cerr << "[" << get_name() << "] PASSED" << '\n'; - } - return true; - } - if (is_error()) { - std::cerr << "[" << get_name() << "] ERROR" << '\n'; - return false; - } - std::cerr << "[" << get_name() << "] FAILED (" << successes << "/" << (successes + failures) << ")\n"; - return false; - } - - void reset() { - successes = 0; - failures = 0; - errors = 0; - } - - std::string get_name() { return name_; } - - bool is_success() const { return successes > 0 && failures == 0 && errors == 0; } - - bool is_error() const { return errors > 0; } - - void set_omit_success_msg(bool omit) { this->omit_success_msg = omit; } - - bool is_omit_success_msg() const { return this->omit_success_msg; } -}; - -inline test_harness::test_harness(test_case & tc, const std::string & test_label, std::ostream & error_stream) : - tc_(tc), - error_stream_(error_stream), - test_label_(test_label) {} - -template bool test_harness::assert_equals(const std::string & label, T expected, T actual) { - if (expected != actual) { - error_stream_ << "[" << label << "] FAILED\n"; - error_stream_ << "Expected: " << expected << "\n"; - error_stream_ << "Actual: " << actual << "\n"; - error_stream_ << std::flush; - tc_.inc_fail(); - return false; - } - if (!tc_.is_omit_success_msg()) { - error_stream_ << "[" << test_label_ << " -> " << label << "] PASSED\n"; - } - tc_.inc_suc(); - return true; -} - -class compound_test { - private: - std::vector> test_cases_; - std::string name_; - int successes_ = 0; - int failures_ = 0; - int errors_ = 0; - std::unordered_map test_name_to_index_; - - void run_test_case(std::unique_ptr & test_case) { - try { - bool result = test_case->run(); - - if (result) { - successes_++; - } else { - failures_++; - } - } catch (std::exception & e) { - errors_++; - std::cerr << "Error while running test " << test_case->get_name() << ":\n" << e.what() << "\n"; - } - } - - public: - explicit compound_test(const std::string & name) : name_(name) {} - - // Add a test case - void add_test(const std::function & test_func, const std::string & test_name) { - auto test = std::make_unique(test_func, test_name); - int index = test_cases_.size(); - test_name_to_index_[test_name] = index; - test_cases_.push_back(std::move(test)); - } - - // Access test by name - bool operator[](const std::string & test_name) { - auto it = test_name_to_index_.find(test_name); - if (it == test_name_to_index_.end()) { - std::cerr << "Test case '" << test_name << "' not found in compound test '" << name_ << "'\n"; - return false; - } - int index = it->second; - auto & test_case = test_cases_[index]; - run_test_case(test_case); - return test_case->is_success(); - } - - // Execute all tests - void run_all() { - std::cerr << "Running all tests for: " << name_ << "\n"; - for (auto & test_case : test_cases_) { - run_test_case(test_case); - } - } - - // Display summary - void summary() { - std::cerr << "\n=== Compound Test Summary: " << name_ << " ===\n"; - std::cerr << "Successes: " << successes_ << "\n"; - std::cerr << "Failures: " << failures_ << "\n"; - std::cerr << "Total: " << (successes_ + failures_) << "\n"; - if (successes_ + failures_ > 0) { - std::cerr << "Pass Rate: " << (successes_ * 100.0 / (successes_ + failures_)) << "%\n"; - } - std::cerr << "========================================\n"; - } - - // Provide a convenient way to run all tests - void run_all_tests() { - run_all(); - summary(); - } - - // Get results - int get_successes() const { return successes_; } - - int get_failures() const { return failures_; } - - int get_total() const { return successes_ + failures_; } - - double get_pass_rate() const { - int total = successes_ + failures_; - return total > 0 ? (successes_ * 100.0 / total) : 0.0; - } -}; From 3e401ba11e23dde762bcf546f750e19117fcac6a Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Sat, 15 Nov 2025 23:50:20 +0100 Subject: [PATCH 065/183] Fix CMakeLists --- tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 8fce517e8b5e8..463b629ad8c61 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -187,7 +187,6 @@ llama_build_and_test(test-chat-parser.cpp) llama_build_and_test( test-chat-peg-parser.cpp chat-peg-parser/simple_tokenizer.cpp - chat-peg-parser/benchmark.cpp chat-peg-parser/test-actions.cpp chat-peg-parser/test-command7-parser-compare.cpp chat-peg-parser/test-example-qwen3-coder.cpp @@ -197,6 +196,7 @@ llama_build_and_test( chat-peg-parser/test-optional.cpp chat-peg-parser/test-partial-parsing.cpp chat-peg-parser/test-recursive-references.cpp + chat-peg-parser/test_harness.h chat-peg-parser/tests.h ) llama_build_and_test(test-chat-template.cpp) From 3389bb78e2c142d77af02cee8e9a05f65f3c9660 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 15 Nov 2025 17:09:54 -0600 Subject: [PATCH 066/183] fix rate calculation --- tests/chat-peg-parser/test_harness.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/chat-peg-parser/test_harness.h b/tests/chat-peg-parser/test_harness.h index b8751badba1a6..9ba56040221f3 100644 --- a/tests/chat-peg-parser/test_harness.h +++ b/tests/chat-peg-parser/test_harness.h @@ -107,7 +107,7 @@ struct testing { }, "bench"); auto avg_elapsed = duration.count() / iterations; - auto avg_elapsed_s = std::chrono::duration_cast>(duration).count(); + auto avg_elapsed_s = std::chrono::duration_cast>(duration).count() / iterations; auto rate = (avg_elapsed_s > 0.0) ? (1.0 / avg_elapsed_s) : 0.0; print_result("END BENCH", name, From 715ab569cfd895ddc23a91ee827d5a4ad1939b94 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 15 Nov 2025 17:39:50 -0600 Subject: [PATCH 067/183] add unicode tests --- tests/CMakeLists.txt | 1 + tests/chat-peg-parser/test-unicode.cpp | 208 +++++++++++++++++++++++++ tests/chat-peg-parser/tests.h | 1 + tests/test-chat-peg-parser.cpp | 5 +- 4 files changed, 213 insertions(+), 2 deletions(-) create mode 100644 tests/chat-peg-parser/test-unicode.cpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 463b629ad8c61..d73e0e145e437 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -196,6 +196,7 @@ llama_build_and_test( chat-peg-parser/test-optional.cpp chat-peg-parser/test-partial-parsing.cpp chat-peg-parser/test-recursive-references.cpp + chat-peg-parser/test-unicode.cpp chat-peg-parser/test_harness.h chat-peg-parser/tests.h ) diff --git a/tests/chat-peg-parser/test-unicode.cpp b/tests/chat-peg-parser/test-unicode.cpp new file mode 100644 index 0000000000000..6811da9342954 --- /dev/null +++ b/tests/chat-peg-parser/test-unicode.cpp @@ -0,0 +1,208 @@ +#include "tests.h" +#include "test_harness.h" + +#include "chat-peg-parser.h" + +#include +#include +#include +#include + +// Assertions specific to chat-peg-parser +static void assert_result_equal(testing & t, common_chat_parse_result_type expected, common_chat_parse_result_type actual) { + t.assert_equal(common_chat_parse_result_type_name(expected), common_chat_parse_result_type_name(actual)); +} + +// Helper function to produce hex dump for non-printable characters +static std::string hex_dump(const std::string& str) { + std::ostringstream oss; + for (unsigned char c : str) { + if (std::isprint(c)) { + oss << c; + } else { + oss << "\\x" << std::hex << std::setw(2) << std::setfill('0') << static_cast(c); + } + } + return oss.str(); +} + +void test_unicode(testing &t) { + struct test_case { + std::string input; + std::string expected_text; + common_chat_parse_result_type expected_result; + }; + + t.test("any", [](testing &t) { + std::vector test_cases { + // Valid UTF-8 sequences + {"Hello", "Hello", COMMON_CHAT_PARSE_RESULT_SUCCESS}, + {std::string("Caf\xC3\xA9"), std::string("Caf\xC3\xA9"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, + {std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, + {std::string("\xF0\x9F\x9A\x80"), std::string("\xF0\x9F\x9A\x80"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, + + // Incomplete UTF-8 sequences (partial bytes at end) + {std::string("Caf\xC3"), "Caf", COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, + {std::string("\xE4\xBD"), "", COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, + {std::string("\xF0\x9F\x9A"), "", COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, + + // Invalid/malformed UTF-8 sequences + {std::string("\xFF\xFE"), "", COMMON_CHAT_PARSE_RESULT_FAIL}, + {std::string("Hello\x80World"), "Hello", COMMON_CHAT_PARSE_RESULT_FAIL}, + {std::string("\xC3\x28"), "", COMMON_CHAT_PARSE_RESULT_FAIL}, + }; + + auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { + return p.one_or_more(p.any()) + p.end(); + }); + + for (size_t i = 0; i < test_cases.size(); i++) { + const auto & tc = test_cases[i]; + std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input); + + t.test(test_name, [&](testing &t) { + common_chat_parse_context ctx(tc.input, false); + auto result = parser.parse(ctx); + + // Assert result type matches + assert_result_equal(t, tc.expected_result, result.type); + + // Assert matched text if success or need_more_input + if (result.success() || result.need_more_input()) { + std::string matched = tc.input.substr(result.start, result.end - result.start); + t.assert_equal(tc.expected_text, matched); + } + }); + } + }); + + t.test("char classes", [](testing &t) { + t.test("unicode range U+4E00-U+9FFF (CJK)", [](testing &t) { + std::vector test_cases { + // Within range - CJK Unified Ideographs + {std::string("\xE4\xB8\x80"), std::string("\xE4\xB8\x80"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // U+4E00 + {std::string("\xE4\xBD\xA0"), std::string("\xE4\xBD\xA0"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // U+4F60 + {std::string("\xE5\xA5\xBD"), std::string("\xE5\xA5\xBD"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // U+597D + {std::string("\xE9\xBF\xBF"), std::string("\xE9\xBF\xBF"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // U+9FFF + + // Outside range - should fail + {"a", "", COMMON_CHAT_PARSE_RESULT_FAIL}, // ASCII + {std::string("\xE4\xB7\xBF"), "", COMMON_CHAT_PARSE_RESULT_FAIL}, // U+4DFF (before range) + {std::string("\xEA\x80\x80"), "", COMMON_CHAT_PARSE_RESULT_FAIL}, // U+A000 (after range) + + // Incomplete sequences in range + {std::string("\xE4\xB8"), "", COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, // Incomplete U+4E00 + {std::string("\xE5\xA5"), "", COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, // Incomplete U+597D + }; + + auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { + return p.chars(R"([\u4E00-\u9FFF])") + p.end(); + }); + + for (size_t i = 0; i < test_cases.size(); i++) { + const auto & tc = test_cases[i]; + std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input); + + t.test(test_name, [&](testing &t) { + common_chat_parse_context ctx(tc.input, false); + auto result = parser.parse(ctx); + + // Assert result type matches + assert_result_equal(t, tc.expected_result, result.type); + + // Assert matched text if success or need_more_input + if (result.success() || result.need_more_input()) { + std::string matched = tc.input.substr(result.start, result.end - result.start); + t.assert_equal(tc.expected_text, matched); + } + }); + } + }); + + t.test("unicode range U+1F600-U+1F64F (emoticons)", [](testing &t) { + std::vector test_cases { + // Within range - Emoticons (all 4-byte UTF-8) + {std::string("\xF0\x9F\x98\x80"), std::string("\xF0\x9F\x98\x80"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // U+1F600 + {std::string("\xF0\x9F\x98\x81"), std::string("\xF0\x9F\x98\x81"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // U+1F601 + {std::string("\xF0\x9F\x99\x8F"), std::string("\xF0\x9F\x99\x8F"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // U+1F64F + + // Outside range + {std::string("\xF0\x9F\x97\xBF"), "", COMMON_CHAT_PARSE_RESULT_FAIL}, // U+1F5FF (before range) + {std::string("\xF0\x9F\x99\x90"), "", COMMON_CHAT_PARSE_RESULT_FAIL}, // U+1F650 (after range) + {std::string("\xF0\x9F\x9A\x80"), "", COMMON_CHAT_PARSE_RESULT_FAIL}, // U+1F680 (outside range) + + // Incomplete sequences + {std::string("\xF0\x9F\x98"), "", COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, // Incomplete emoji + {std::string("\xF0\x9F"), "", COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, // Very incomplete + }; + + auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { + return p.chars(R"([\U0001F600-\U0001F64F])") + p.end(); + }); + + for (size_t i = 0; i < test_cases.size(); i++) { + const auto & tc = test_cases[i]; + std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input); + + t.test(test_name, [&](testing &t) { + common_chat_parse_context ctx(tc.input, false); + auto result = parser.parse(ctx); + + // Assert result type matches + assert_result_equal(t, tc.expected_result, result.type); + + // Assert matched text if success or need_more_input + if (result.success() || result.need_more_input()) { + std::string matched = tc.input.substr(result.start, result.end - result.start); + t.assert_equal(tc.expected_text, matched); + } + }); + } + }); + + t.test("mixed unicode ranges", [](testing &t) { + std::vector test_cases { + // Match CJK + {std::string("\xE4\xB8\x80"), std::string("\xE4\xB8\x80"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // U+4E00 + {std::string("\xE4\xBD\xA0"), std::string("\xE4\xBD\xA0"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // U+4F60 + + // Match emoticons + {std::string("\xF0\x9F\x98\x80"), std::string("\xF0\x9F\x98\x80"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // U+1F600 + + // Match ASCII digits + {"5", "5", COMMON_CHAT_PARSE_RESULT_SUCCESS}, + + // Don't match outside any range + {"a", "", COMMON_CHAT_PARSE_RESULT_FAIL}, + {std::string("\xF0\x9F\x9A\x80"), "", COMMON_CHAT_PARSE_RESULT_FAIL}, // U+1F680 + + // Incomplete + {std::string("\xE4\xB8"), "", COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, + {std::string("\xF0\x9F\x98"), "", COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, + }; + + auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { + return p.chars(R"([\u4E00-\u9FFF\U0001F600-\U0001F64F0-9])") + p.end(); + }); + + for (size_t i = 0; i < test_cases.size(); i++) { + const auto & tc = test_cases[i]; + std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input); + + t.test(test_name, [&](testing &t) { + common_chat_parse_context ctx(tc.input, false); + auto result = parser.parse(ctx); + + // Assert result type matches + assert_result_equal(t, tc.expected_result, result.type); + + // Assert matched text if success or need_more_input + if (result.success() || result.need_more_input()) { + std::string matched = tc.input.substr(result.start, result.end - result.start); + t.assert_equal(tc.expected_text, matched); + } + }); + } + }); + }); +} diff --git a/tests/chat-peg-parser/tests.h b/tests/chat-peg-parser/tests.h index 717578a4238b7..8ef5a28a6ba81 100644 --- a/tests/chat-peg-parser/tests.h +++ b/tests/chat-peg-parser/tests.h @@ -26,3 +26,4 @@ void test_actions(testing &t); void test_gbnf_generation(testing &t); void test_example_qwen3_coder(testing &t); void test_command7_parser_compare(testing &t); +void test_unicode(testing &t); diff --git a/tests/test-chat-peg-parser.cpp b/tests/test-chat-peg-parser.cpp index 1b4fa4b418783..895da9e4b692f 100644 --- a/tests/test-chat-peg-parser.cpp +++ b/tests/test-chat-peg-parser.cpp @@ -2,16 +2,17 @@ int main() { testing t(std::cout); - + test_partial_parsing(t); test_one(t); test_optional(t); + test_unicode(t); test_recursive_references(t); test_json_parser(t); test_actions(t); test_gbnf_generation(t); test_example_qwen3_coder(t); test_command7_parser_compare(t); - + return t.summary(); } From 600e589dbdae9d1fd0250f8ed70f8be9fa7998e3 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 15 Nov 2025 17:41:01 -0600 Subject: [PATCH 068/183] fix trailing whitespace and line endings skip-checks: true --- .../test-command7-parser-compare.cpp | 22 +++++++++---------- .../chat-peg-parser/test-gbnf-generation.cpp | 2 +- tests/chat-peg-parser/test-json-parser.cpp | 2 +- .../test-recursive-references.cpp | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/chat-peg-parser/test-command7-parser-compare.cpp b/tests/chat-peg-parser/test-command7-parser-compare.cpp index fb2c29765d9fe..5b4175897a441 100644 --- a/tests/chat-peg-parser/test-command7-parser-compare.cpp +++ b/tests/chat-peg-parser/test-command7-parser-compare.cpp @@ -161,7 +161,7 @@ void test_command7_parser_compare(testing &t) { // Setup data auto parser = create_command_r7b_parser(); auto handler = create_command_r7b_event_handler(); - + std::string reasoning = "To plan an effective trip to Japan that includes both historical sites and modern attractions within a " "budget of $4000 for a two-week stay, we need to:\n\n" "1. Identify key historical sites and modern attractions in Japan.\n" @@ -171,7 +171,7 @@ void test_command7_parser_compare(testing &t) { "overspending.\n" "5. Provide a detailed cost breakdown that includes accommodation, transportation, meals, and entry fees " "to attractions."; - + std::string content = "For a two-week trip to Japan with a $4,000 budget, I recommend planning an itinerary that balances " "historical sites with modern attractions. The destination will be Japan, with a duration of 14 days.\n\n" "Given your interests in both historical sites and modern attractions, you'll want to focus on cities like " @@ -186,7 +186,7 @@ void test_command7_parser_compare(testing &t) { "For meals, embrace local cuisine by eating at neighborhood restaurants, ramen shops, and izakayas rather " "than touristy establishments. This will give you an authentic experience while keeping costs " "reasonableβ€”you can enjoy excellent meals for $10-20 per person at local spots.\n\n"; - + std::vector> tool_calls = { { "call_0", "plan_trip", nlohmann::json::parse(R"({ "destination": "Japan", @@ -198,9 +198,9 @@ void test_command7_parser_compare(testing &t) { "meal_preferences": "local cuisine" })") } }; - + std::vector tokens; - + // Build tokens if (!reasoning.empty()) { auto tokenized = simple_tokenize(reasoning); @@ -233,9 +233,9 @@ void test_command7_parser_compare(testing &t) { tokens.emplace_back("<|END_ACTION|>"); } - + std::string input = std::accumulate(tokens.begin(), tokens.end(), std::string()); - + // Run tests t.test("legacy_parse", [&](testing & t) { bool no_error = true; @@ -247,7 +247,7 @@ void test_command7_parser_compare(testing &t) { } t.assert_equal("no_errors", true, no_error); }); - + t.test("current_parse", [&](testing & t) { bool no_error = true; try { @@ -258,13 +258,13 @@ void test_command7_parser_compare(testing &t) { } t.assert_equal("no_errors", true, no_error); }); - + // Run benchmarks t.bench("legacy_parse_benchmark", [&]() { test_command_r7b_legacy_parser(input, false, false); }, 1000); - + t.bench("current_parse_benchmark", [&]() { test_command_r7b_parser(parser, input, false, false); }, 1000); -} \ No newline at end of file +} diff --git a/tests/chat-peg-parser/test-gbnf-generation.cpp b/tests/chat-peg-parser/test-gbnf-generation.cpp index c1fae1ec7f364..17e8984538c67 100644 --- a/tests/chat-peg-parser/test-gbnf-generation.cpp +++ b/tests/chat-peg-parser/test-gbnf-generation.cpp @@ -126,4 +126,4 @@ void test_gbnf_generation(testing &t) { t.assert_equal("has_inlined_hello", true, gbnf.find("\"hello\"") != std::string::npos); t.assert_equal("has_inlined_world", true, gbnf.find("\"world\"") != std::string::npos); }); -} \ No newline at end of file +} diff --git a/tests/chat-peg-parser/test-json-parser.cpp b/tests/chat-peg-parser/test-json-parser.cpp index c10bf56c304cb..935ada3ad92f3 100644 --- a/tests/chat-peg-parser/test-json-parser.cpp +++ b/tests/chat-peg-parser/test-json-parser.cpp @@ -76,4 +76,4 @@ void test_json_parser(testing &t) { t.assert_equal("result_is_need_more_input", true, result.need_more_input()); }); -} \ No newline at end of file +} diff --git a/tests/chat-peg-parser/test-recursive-references.cpp b/tests/chat-peg-parser/test-recursive-references.cpp index 6fe38f2aea71c..224f58c94a123 100644 --- a/tests/chat-peg-parser/test-recursive-references.cpp +++ b/tests/chat-peg-parser/test-recursive-references.cpp @@ -84,4 +84,4 @@ void test_recursive_references(testing &t) { t.assert_equal("result_is_fail", true, result.fail()); }); -} \ No newline at end of file +} From ae31b328d9e6ea05f7efc32a73bccd2df5767453 Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Sun, 16 Nov 2025 01:00:43 +0100 Subject: [PATCH 069/183] Helpers + rewrite qwen3 with helpers --- common/chat-peg-parser.cpp | 56 ++++++++++++++++++- common/chat-peg-parser.h | 23 ++++++++ .../test-example-qwen3-coder.cpp | 29 +++------- 3 files changed, 84 insertions(+), 24 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 7d00bd16a14b7..c857859c0f28e 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -3,6 +3,7 @@ #include "common.h" #include "log.h" +#include #include #include @@ -269,10 +270,12 @@ static std::string unescape_json_string(std::string_view str) { if (parsed.is_string()) { return parsed.get(); } + // If not a string, return literally + return std::string(str); } catch (...) { // If parsing fails, return original string + return std::string(str); } - return std::string(str); } // Aho-Corasick automation for matching multiple literals. @@ -529,7 +532,7 @@ class start_parser : public common_chat_peg_parser_base { void accept(parser_visitor & visitor) override; std::string dump() const override { return "Start"; } - common_chat_parse_result parse_uncached(common_chat_parse_context &, size_t start = 0) override { + common_chat_parse_result parse_uncached(common_chat_parse_context & /*ctx*/, size_t start = 0) override { return common_chat_parse_result(start == 0 ? COMMON_CHAT_PARSE_RESULT_SUCCESS : COMMON_CHAT_PARSE_RESULT_FAIL, start); } }; @@ -608,6 +611,18 @@ class sequence_parser : public common_chat_peg_parser_base { } } + sequence_parser(const std::vector& parsers, int id) : common_chat_peg_parser_base(id) { + for (const auto & p : parsers) { + if (auto seq = cast(p)) { + for (const auto & embedded : seq->parsers()) { + parsers_.push_back(embedded); + } + } else { + parsers_.push_back(p); + } + } + } + parser_type type() const override { return type_value; } common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { @@ -1983,6 +1998,10 @@ common_chat_peg_parser operator+(const char * lhs, const common_chat_peg_parser common_chat_peg_parser operator|(const char * lhs, const common_chat_peg_parser & rhs) { return common_chat_peg_parser(lhs) | rhs; } common_chat_peg_parser operator<<(const char * lhs, const common_chat_peg_parser & rhs) { return common_chat_peg_parser(lhs) << rhs; } +common_chat_peg_parser operator+(const std::string & lhs, const common_chat_peg_parser & rhs) { return common_chat_peg_parser(lhs) + rhs; } +common_chat_peg_parser operator|(const std::string & lhs, const common_chat_peg_parser & rhs) { return common_chat_peg_parser(lhs) | rhs; } +common_chat_peg_parser operator<<(const std::string & lhs, const common_chat_peg_parser & rhs) { return common_chat_peg_parser(lhs) << rhs; } + common_chat_peg_parser_base & common_chat_peg_parser::operator*() const { return *ptr_; } @@ -2209,3 +2228,36 @@ common_chat_peg_parser common_chat_peg_parser_builder::json() { json_null(); }); } + +common_chat_peg_parser common_chat_peg_parser_builder::reasoning(const std::string &tag) { + return add_rule("raw-reasoning", std::string("<" + tag + ">") << add_rule("reasoning-content", until("")) << ""); +} + +common_chat_peg_parser common_chat_peg_parser_builder::content_before_tools(const std::string &tag) { + return add_rule("content", until(tag)); +} + +common_chat_peg_parser common_chat_peg_parser_builder::quasi_xml_no_attr(const std::string &function_name, const std::vector ¶meters, + const std::string &function_tag, const std::string ¶m_tag) { + std::vector args; + + for (auto it = parameters.begin(); it != parameters.end(); it++) { + auto arg_name = add_rule(std::string("arg-start-" + *it), literal("<" + param_tag + "=" + *it + ">")); + auto arg_end = add_rule("arg-end", "" + peek(literal("<" + param_tag + "=") | "")); + auto string_arg_content = add_rule("arg-string-content", + until_one_of({"<" + param_tag + "=", ""})); + auto string_arg = add_rule("arg-string-" + *it, arg_name + string_arg_content + arg_end); + auto json_sec = json(); + auto json_arg = add_rule("arg-json-" + *it, arg_name + add_rule("arg-json-content", json_sec) + arg_end); + auto arg_json_or_string = one_or_more(json_arg | string_arg); + args.push_back(arg_json_or_string); + } + + auto args_seq_raw = sequence_parser(args, counter_.next()); + auto args_sequence = common_chat_peg_parser(std::make_shared(args_seq_raw)); + auto function = add_rule("function-" + function_name, + add_rule("function-start-" + function_name, "<" + function_tag + "=" + function_name + ">") + + args_sequence + ""); + + return function; +} \ No newline at end of file diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index 200ef31a539a4..c4e7c10c51540 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -173,6 +173,10 @@ common_chat_peg_parser operator+(const char * lhs, const common_chat_peg_parser common_chat_peg_parser operator|(const char * lhs, const common_chat_peg_parser & rhs); common_chat_peg_parser operator<<(const char * lhs, const common_chat_peg_parser & rhs); +common_chat_peg_parser operator+(const std::string & lhs, const common_chat_peg_parser & rhs); +common_chat_peg_parser operator|(const std::string & lhs, const common_chat_peg_parser & rhs); +common_chat_peg_parser operator<<(const std::string & lhs, const common_chat_peg_parser & rhs); + class common_chat_peg_parser_counter { int next_id_; public: @@ -308,6 +312,25 @@ class common_chat_peg_parser_builder { void set_root(const common_chat_peg_parser & p); + // Helper methods for common patterns + + // Adds raw-reasoning for the entire reasoning block plus reasoning-content for the contents, by default thinking tag is "think" + common_chat_peg_parser reasoning(const std::string & tag = "think"); + + // Adds main content block before tool call block, due to the varied nature of tool call openers (not always XML-like) full tag is required + common_chat_peg_parser content_before_tools(const std::string &tag); + + // Adds a quasi-XML tool call spec without a separate name attribute (Qwen3 style); + // TODO: accept parameter schemas (required, value types etc.) + common_chat_peg_parser quasi_xml_no_attr(const std::string &function_name, const std::vector ¶meters, + const std::string &function_tag = "function", const std::string ¶m_tag = "parameter"); + + // Adds a quasi-XML tool call spec with a separate name attribute (Minimax-M2 style) + // TODO: accept parameter schemas (required, value types etc.) + // common_chat_peg_parser quasi_xml_attr(const std::string &function_name, const std::vector ¶meters, + // const std::string &function_tag = "invoke", const std::string ¶m_tag = "parameter", + // const std::string &name_attr = "name"); + common_chat_peg_parser build(); }; diff --git a/tests/chat-peg-parser/test-example-qwen3-coder.cpp b/tests/chat-peg-parser/test-example-qwen3-coder.cpp index ba7aa93ceb04e..e35c6f2ea2f31 100644 --- a/tests/chat-peg-parser/test-example-qwen3-coder.cpp +++ b/tests/chat-peg-parser/test-example-qwen3-coder.cpp @@ -5,28 +5,13 @@ void test_example_qwen3_coder(testing &t) { auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { - auto thinking = p.add_rule("raw-reasoning", - "" << p.add_rule("reasoning-content", p.until("")) << ""); - - auto content = p.add_rule("content", p.until("")); - - auto arg_name = p.add_rule("arg-start", ""); - auto arg_end = p.add_rule("arg-end", "" + p.peek(p.literal("")); - - auto string_arg_content = p.add_rule("arg-string-content", - p.until_one_of({""})); - - auto string_arg = p.add_rule("arg-string", arg_name + string_arg_content + arg_end); - - auto json = p.json(); - - auto json_arg = p.add_rule("arg-json", arg_name + p.add_rule("arg-json-content", json) + arg_end); - - auto function = p.add_rule("function", - p.add_rule("function-start", "") - + p.one_or_more(json_arg | string_arg) - + ""); - + auto thinking = p.reasoning(); + auto content = p.content_before_tools(""); + auto function = p.quasi_xml_no_attr("search_files", + std::vector({ + "path", "pattern", "min_size_mb", "max_depth", "include_hidden", "modified_days_ago", + "case_sensitive", "sort_by", "filters" + })); auto tool_call = p.trigger(p.add_rule("tool-call", "" + p.one_or_more(function) + "")); From ed4b1d014c6619a1acf07705607b038b25088743 Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Sun, 16 Nov 2025 01:02:00 +0100 Subject: [PATCH 070/183] Fix whitespace --- common/chat-peg-parser.cpp | 6 +++--- common/chat-peg-parser.h | 2 +- tests/chat-peg-parser/test-example-qwen3-coder.cpp | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index c857859c0f28e..678ffe0ae5e8f 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -2240,11 +2240,11 @@ common_chat_peg_parser common_chat_peg_parser_builder::content_before_tools(cons common_chat_peg_parser common_chat_peg_parser_builder::quasi_xml_no_attr(const std::string &function_name, const std::vector ¶meters, const std::string &function_tag, const std::string ¶m_tag) { std::vector args; - + for (auto it = parameters.begin(); it != parameters.end(); it++) { auto arg_name = add_rule(std::string("arg-start-" + *it), literal("<" + param_tag + "=" + *it + ">")); auto arg_end = add_rule("arg-end", "" + peek(literal("<" + param_tag + "=") | "")); - auto string_arg_content = add_rule("arg-string-content", + auto string_arg_content = add_rule("arg-string-content", until_one_of({"<" + param_tag + "=", ""})); auto string_arg = add_rule("arg-string-" + *it, arg_name + string_arg_content + arg_end); auto json_sec = json(); @@ -2260,4 +2260,4 @@ common_chat_peg_parser common_chat_peg_parser_builder::quasi_xml_no_attr(const s + args_sequence + ""); return function; -} \ No newline at end of file +} diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index c4e7c10c51540..0c7b3a1232171 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -328,7 +328,7 @@ class common_chat_peg_parser_builder { // Adds a quasi-XML tool call spec with a separate name attribute (Minimax-M2 style) // TODO: accept parameter schemas (required, value types etc.) // common_chat_peg_parser quasi_xml_attr(const std::string &function_name, const std::vector ¶meters, - // const std::string &function_tag = "invoke", const std::string ¶m_tag = "parameter", + // const std::string &function_tag = "invoke", const std::string ¶m_tag = "parameter", // const std::string &name_attr = "name"); common_chat_peg_parser build(); diff --git a/tests/chat-peg-parser/test-example-qwen3-coder.cpp b/tests/chat-peg-parser/test-example-qwen3-coder.cpp index e35c6f2ea2f31..29e8cd6a67883 100644 --- a/tests/chat-peg-parser/test-example-qwen3-coder.cpp +++ b/tests/chat-peg-parser/test-example-qwen3-coder.cpp @@ -7,8 +7,8 @@ void test_example_qwen3_coder(testing &t) { auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { auto thinking = p.reasoning(); auto content = p.content_before_tools(""); - auto function = p.quasi_xml_no_attr("search_files", - std::vector({ + auto function = p.quasi_xml_no_attr("search_files", + std::vector({ "path", "pattern", "min_size_mb", "max_depth", "include_hidden", "modified_days_ago", "case_sensitive", "sort_by", "filters" })); From 57f03e299e724a842f9b9e52f6a9dd22aa90cd5b Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 15 Nov 2025 18:38:34 -0600 Subject: [PATCH 071/183] extract unicode functions to separate file --- common/CMakeLists.txt | 2 + common/chat-peg-parser.cpp | 171 +++---------------------------------- common/unicode.cpp | 121 ++++++++++++++++++++++++++ common/unicode.h | 22 +++++ 4 files changed, 158 insertions(+), 158 deletions(-) create mode 100644 common/unicode.cpp create mode 100644 common/unicode.h diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 7062cc29a7064..77d0271e4c6e4 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -75,6 +75,8 @@ add_library(${TARGET} STATIC sampling.h speculative.cpp speculative.h + unicode.cpp + unicode.h ) if (BUILD_SHARED_LIBS) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 678ffe0ae5e8f..52eff5d1a8002 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -2,6 +2,7 @@ #include "json-schema-to-grammar.h" #include "common.h" #include "log.h" +#include "unicode.h" #include #include @@ -114,152 +115,6 @@ static bool is_hex_digit(const char c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); } -// UTF-8 parsing utilities for streaming-aware unicode support -struct utf8_parse_result { - uint32_t codepoint; // Decoded codepoint (only valid if status == SUCCESS) - size_t bytes_consumed; // How many bytes this codepoint uses (1-4) - enum status_t { SUCCESS, NEED_MORE, INVALID } status; - - utf8_parse_result(status_t s, uint32_t cp = 0, size_t bytes = 0) - : codepoint(cp), bytes_consumed(bytes), status(s) {} -}; - -// Determine the expected length of a UTF-8 sequence from its first byte -// Returns 0 for invalid first bytes -static size_t utf8_sequence_length(unsigned char first_byte) { - // Lookup table based on high 4 bits - // 0xxx xxxx = 1 byte (ASCII) - // 110x xxxx = 2 bytes - // 1110 xxxx = 3 bytes - // 1111 0xxx = 4 bytes (only 0xF0-0xF7, not 0xF8-0xFF) - static const size_t lookup[] = { - 1, 1, 1, 1, 1, 1, 1, 1, // 0000-0111 (0x00-0x7F) - 0, 0, 0, 0, // 1000-1011 (continuation bytes 0x80-0xBF, invalid as first byte) - 2, 2, // 1100-1101 (0xC0-0xDF) - 3, // 1110 (0xE0-0xEF) - 4 // 1111 (0xF0-0xFF, but need to check 0xF8-0xFF separately) - }; - size_t len = lookup[first_byte >> 4]; - - // Additional validation for invalid first bytes: - // - 0xC0-0xC1: would create overlong 2-byte sequences - // - 0xF8-0xFF: invalid 5+ byte sequences - if (first_byte >= 0xF8 || (first_byte >= 0xC0 && first_byte <= 0xC1)) { - return 0; // Invalid - } - - return len; -} - -// Parse a single UTF-8 codepoint from input, with streaming support -// Returns SUCCESS if a complete, valid codepoint is parsed -// Returns NEED_MORE if the sequence is incomplete and input_is_complete is false -// Returns INVALID if the UTF-8 encoding is malformed -static utf8_parse_result parse_utf8_codepoint( - std::string_view input, - size_t offset, - bool input_is_complete -) { - if (offset >= input.size()) { - if (input_is_complete) { - return utf8_parse_result(utf8_parse_result::INVALID); - } - return utf8_parse_result(utf8_parse_result::NEED_MORE); - } - - const unsigned char first = static_cast(input[offset]); - - // ASCII fast path (most common case) - if (first < 0x80) { - return utf8_parse_result(utf8_parse_result::SUCCESS, first, 1); - } - - // Invalid first byte (continuation byte 10xxxxxx as first byte, or 0xF8-0xFF) - if ((first & 0xC0) == 0x80) { - return utf8_parse_result(utf8_parse_result::INVALID); - } - - size_t seq_len = utf8_sequence_length(first); - if (seq_len == 0) { - // Invalid first byte (e.g., 0xF8-0xFF) - return utf8_parse_result(utf8_parse_result::INVALID); - } - - size_t available = input.size() - offset; - - // Check if we have enough bytes for the complete sequence - if (available < seq_len) { - if (input_is_complete) { - return utf8_parse_result(utf8_parse_result::INVALID); - } - return utf8_parse_result(utf8_parse_result::NEED_MORE); - } - - uint32_t codepoint = 0; - - // Decode based on sequence length - if (seq_len == 2) { - // 110xxxxx 10xxxxxx - if ((first & 0xE0) != 0xC0) { - return utf8_parse_result(utf8_parse_result::INVALID); - } - const unsigned char second = static_cast(input[offset + 1]); - if ((second & 0xC0) != 0x80) { - return utf8_parse_result(utf8_parse_result::INVALID); - } - codepoint = ((first & 0x1F) << 6) | (second & 0x3F); - // Check for overlong encoding - if (codepoint < 0x80) { - return utf8_parse_result(utf8_parse_result::INVALID); - } - } else if (seq_len == 3) { - // 1110xxxx 10xxxxxx 10xxxxxx - if ((first & 0xF0) != 0xE0) { - return utf8_parse_result(utf8_parse_result::INVALID); - } - const unsigned char second = static_cast(input[offset + 1]); - const unsigned char third = static_cast(input[offset + 2]); - if ((second & 0xC0) != 0x80 || (third & 0xC0) != 0x80) { - return utf8_parse_result(utf8_parse_result::INVALID); - } - codepoint = ((first & 0x0F) << 12) | ((second & 0x3F) << 6) | (third & 0x3F); - // Check for overlong encoding - if (codepoint < 0x800) { - return utf8_parse_result(utf8_parse_result::INVALID); - } - // Check for surrogate pairs (0xD800-0xDFFF are invalid in UTF-8) - if (codepoint >= 0xD800 && codepoint <= 0xDFFF) { - return utf8_parse_result(utf8_parse_result::INVALID); - } - } else if (seq_len == 4) { - // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx - if ((first & 0xF8) != 0xF0) { - return utf8_parse_result(utf8_parse_result::INVALID); - } - const unsigned char second = static_cast(input[offset + 1]); - const unsigned char third = static_cast(input[offset + 2]); - const unsigned char fourth = static_cast(input[offset + 3]); - if ((second & 0xC0) != 0x80 || (third & 0xC0) != 0x80 || (fourth & 0xC0) != 0x80) { - return utf8_parse_result(utf8_parse_result::INVALID); - } - codepoint = ((first & 0x07) << 18) | ((second & 0x3F) << 12) | - ((third & 0x3F) << 6) | (fourth & 0x3F); - // Check for overlong encoding - if (codepoint < 0x10000) { - return utf8_parse_result(utf8_parse_result::INVALID); - } - // Check for valid Unicode range (max is 0x10FFFF) - if (codepoint > 0x10FFFF) { - return utf8_parse_result(utf8_parse_result::INVALID); - } - } else { - // Invalid sequence length - return utf8_parse_result(utf8_parse_result::INVALID); - } - - return utf8_parse_result(utf8_parse_result::SUCCESS, codepoint, seq_len); -} - // Unescapes a JSON string (without the surrounding quotes) // Uses nlohmann::json::parse to handle all JSON escape sequences static std::string unescape_json_string(std::string_view str) { @@ -933,17 +788,17 @@ class any_parser : public common_chat_peg_parser_base { common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { // Parse a single UTF-8 codepoint (not just a single byte) - auto result = parse_utf8_codepoint(ctx.input, start, ctx.input_is_complete); + auto result = parse_utf8_codepoint(ctx.input, start); - if (result.status == utf8_parse_result::NEED_MORE) { - // Incomplete UTF-8 sequence: end position is at start (before incomplete bytes) - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start, start); + if (result.status == utf8_parse_result::INCOMPLETE) { + if (ctx.input_is_complete) { + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); + } + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start); } if (result.status == utf8_parse_result::INVALID) { - // Malformed UTF-8 return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); } - // Success: advance by full codepoint (1-4 bytes) return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, start + result.bytes_consumed); } @@ -1127,17 +982,17 @@ class chars_parser : public common_chat_peg_parser_base { // Try to match up to max_count times (or unlimited if max_count is -1) while (max_count_ == -1 || match_count < max_count_) { - // Parse UTF-8 codepoint from input - auto result = parse_utf8_codepoint(ctx.input, pos, ctx.input_is_complete); + auto result = parse_utf8_codepoint(ctx.input, pos); - if (result.status == utf8_parse_result::NEED_MORE) { - // Incomplete UTF-8 sequence at current position + if (result.status == utf8_parse_result::INCOMPLETE) { if (match_count >= min_count_) { // We have enough matches, succeed with what we have - // End position is at pos (before the incomplete sequence) return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); } - // Not enough matches yet, need more input + // Not enough matches yet + if (ctx.input_is_complete) { + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); + } return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start, pos); } diff --git a/common/unicode.cpp b/common/unicode.cpp new file mode 100644 index 0000000000000..d29bac030dd5d --- /dev/null +++ b/common/unicode.cpp @@ -0,0 +1,121 @@ +#include "unicode.h" + +size_t utf8_sequence_length(unsigned char first_byte) { + // Lookup table based on high 4 bits + // 0xxx xxxx = 1 byte (ASCII) + // 110x xxxx = 2 bytes + // 1110 xxxx = 3 bytes + // 1111 0xxx = 4 bytes (only 0xF0-0xF7, not 0xF8-0xFF) + static const size_t lookup[] = { + 1, 1, 1, 1, 1, 1, 1, 1, // 0000-0111 (0x00-0x7F) + 0, 0, 0, 0, // 1000-1011 (continuation bytes 0x80-0xBF, invalid as first byte) + 2, 2, // 1100-1101 (0xC0-0xDF) + 3, // 1110 (0xE0-0xEF) + 4 // 1111 (0xF0-0xFF, but need to check 0xF8-0xFF separately) + }; + size_t len = lookup[first_byte >> 4]; + + // Additional validation for invalid first bytes: + // - 0xC0-0xC1: would create overlong 2-byte sequences + // - 0xF8-0xFF: invalid 5+ byte sequences + if (first_byte >= 0xF8 || (first_byte >= 0xC0 && first_byte <= 0xC1)) { + return 0; // Invalid + } + + return len; +} + +utf8_parse_result parse_utf8_codepoint(std::string_view input, size_t offset) { + if (offset >= input.size()) { + return utf8_parse_result(utf8_parse_result::INCOMPLETE); + } + + const unsigned char first = static_cast(input[offset]); + + // ASCII fast path (most common case) + if (first < 0x80) { + return utf8_parse_result(utf8_parse_result::SUCCESS, first, 1); + } + + // Invalid first byte (continuation byte 10xxxxxx as first byte, or 0xF8-0xFF) + if ((first & 0xC0) == 0x80) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + + size_t seq_len = utf8_sequence_length(first); + if (seq_len == 0) { + // Invalid first byte (e.g., 0xF8-0xFF) + return utf8_parse_result(utf8_parse_result::INVALID); + } + + size_t available = input.size() - offset; + + // Check if we have enough bytes for the complete sequence + if (available < seq_len) { + return utf8_parse_result(utf8_parse_result::INCOMPLETE); + } + + uint32_t codepoint = 0; + + // Decode based on sequence length + if (seq_len == 2) { + // 110xxxxx 10xxxxxx + if ((first & 0xE0) != 0xC0) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + const unsigned char second = static_cast(input[offset + 1]); + if ((second & 0xC0) != 0x80) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + codepoint = ((first & 0x1F) << 6) | (second & 0x3F); + // Check for overlong encoding + if (codepoint < 0x80) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + } else if (seq_len == 3) { + // 1110xxxx 10xxxxxx 10xxxxxx + if ((first & 0xF0) != 0xE0) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + const unsigned char second = static_cast(input[offset + 1]); + const unsigned char third = static_cast(input[offset + 2]); + if ((second & 0xC0) != 0x80 || (third & 0xC0) != 0x80) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + codepoint = ((first & 0x0F) << 12) | ((second & 0x3F) << 6) | (third & 0x3F); + // Check for overlong encoding + if (codepoint < 0x800) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + // Check for surrogate pairs (0xD800-0xDFFF are invalid in UTF-8) + if (codepoint >= 0xD800 && codepoint <= 0xDFFF) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + } else if (seq_len == 4) { + // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + if ((first & 0xF8) != 0xF0) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + const unsigned char second = static_cast(input[offset + 1]); + const unsigned char third = static_cast(input[offset + 2]); + const unsigned char fourth = static_cast(input[offset + 3]); + if ((second & 0xC0) != 0x80 || (third & 0xC0) != 0x80 || (fourth & 0xC0) != 0x80) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + codepoint = ((first & 0x07) << 18) | ((second & 0x3F) << 12) | + ((third & 0x3F) << 6) | (fourth & 0x3F); + // Check for overlong encoding + if (codepoint < 0x10000) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + // Check for valid Unicode range (max is 0x10FFFF) + if (codepoint > 0x10FFFF) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + } else { + // Invalid sequence length + return utf8_parse_result(utf8_parse_result::INVALID); + } + + return utf8_parse_result(utf8_parse_result::SUCCESS, codepoint, seq_len); +} diff --git a/common/unicode.h b/common/unicode.h new file mode 100644 index 0000000000000..9d9e8e1227aa9 --- /dev/null +++ b/common/unicode.h @@ -0,0 +1,22 @@ +#pragma once + +#include +#include + +// UTF-8 parsing utilities for streaming-aware unicode support + +struct utf8_parse_result { + uint32_t codepoint; // Decoded codepoint (only valid if status == SUCCESS) + size_t bytes_consumed; // How many bytes this codepoint uses (1-4) + enum status { SUCCESS, INCOMPLETE, INVALID } status; + + utf8_parse_result(enum status s, uint32_t cp = 0, size_t bytes = 0) + : codepoint(cp), bytes_consumed(bytes), status(s) {} +}; + +// Determine the expected length of a UTF-8 sequence from its first byte +// Returns 0 for invalid first bytes +size_t utf8_sequence_length(unsigned char first_byte); + +// Parse a single UTF-8 codepoint from input +utf8_parse_result parse_utf8_codepoint(std::string_view input, size_t offset); From 9ff6486d56e1e03e0396cae4d0cd25d16614273a Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 15 Nov 2025 19:44:28 -0600 Subject: [PATCH 072/183] refactor parse unicode function --- common/unicode.cpp | 159 ++++++++++++++++++++++++++++----------------- 1 file changed, 99 insertions(+), 60 deletions(-) diff --git a/common/unicode.cpp b/common/unicode.cpp index d29bac030dd5d..9dbce54256f0c 100644 --- a/common/unicode.cpp +++ b/common/unicode.cpp @@ -1,25 +1,26 @@ #include "unicode.h" size_t utf8_sequence_length(unsigned char first_byte) { - // Lookup table based on high 4 bits + // Lookup table based on high 4 bits: // 0xxx xxxx = 1 byte (ASCII) // 110x xxxx = 2 bytes // 1110 xxxx = 3 bytes - // 1111 0xxx = 4 bytes (only 0xF0-0xF7, not 0xF8-0xFF) + // 1111 0xxx = 4 bytes (only 0xF0–0xF4 are valid starts) static const size_t lookup[] = { - 1, 1, 1, 1, 1, 1, 1, 1, // 0000-0111 (0x00-0x7F) - 0, 0, 0, 0, // 1000-1011 (continuation bytes 0x80-0xBF, invalid as first byte) - 2, 2, // 1100-1101 (0xC0-0xDF) - 3, // 1110 (0xE0-0xEF) - 4 // 1111 (0xF0-0xFF, but need to check 0xF8-0xFF separately) + 1, 1, 1, 1, 1, 1, 1, 1, // 0000–0111 (0x00–0x7F) ASCII + 0, 0, 0, 0, // 1000–1011 (0x80–0xBF) continuation bytes + 2, 2, // 1100–1101 (0xC0–0xDF) 2-byte sequences + 3, // 1110 (0xE0–0xEF) 3-byte sequences + 4 // 1111 (0xF0–0xFF) potential 4-byte sequences }; + size_t len = lookup[first_byte >> 4]; - // Additional validation for invalid first bytes: - // - 0xC0-0xC1: would create overlong 2-byte sequences - // - 0xF8-0xFF: invalid 5+ byte sequences - if (first_byte >= 0xF8 || (first_byte >= 0xC0 && first_byte <= 0xC1)) { - return 0; // Invalid + // Filter out invalid first bytes: + // - 0xC0–0xC1: overlong 2-byte sequences (would encode U+0000–U+007F) + // - 0xF5–0xFF: would encode beyond U+10FFFF or invalid 5+ byte sequences + if (first_byte == 0xC0 || first_byte == 0xC1 || first_byte >= 0xF5) { + return 0; } return len; @@ -32,89 +33,127 @@ utf8_parse_result parse_utf8_codepoint(std::string_view input, size_t offset) { const unsigned char first = static_cast(input[offset]); - // ASCII fast path (most common case) + // ASCII fast path (1-byte sequence) if (first < 0x80) { return utf8_parse_result(utf8_parse_result::SUCCESS, first, 1); } - // Invalid first byte (continuation byte 10xxxxxx as first byte, or 0xF8-0xFF) - if ((first & 0xC0) == 0x80) { - return utf8_parse_result(utf8_parse_result::INVALID); - } - + // Determine expected sequence length (0 means invalid first byte) size_t seq_len = utf8_sequence_length(first); if (seq_len == 0) { - // Invalid first byte (e.g., 0xF8-0xFF) return utf8_parse_result(utf8_parse_result::INVALID); } size_t available = input.size() - offset; - // Check if we have enough bytes for the complete sequence + // Handle incomplete sequences: not enough bytes for the promised length. if (available < seq_len) { + // We want INCOMPLETE only if this prefix *could* still become valid. + // So we validate as much as we can, and reject prefixes that are + // already impossible regardless of future bytes. + + if (available >= 2) { + unsigned char second = static_cast(input[offset + 1]); + + // Second byte must be a continuation byte. + if ((second & 0xC0) != 0x80) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + + // Apply lead+second byte constraints that are necessary for any + // valid UTF-8 sequence (these mirror the usual per-byte rules). + + if (seq_len == 3) { + // 3-byte sequences (first in 0xE0–0xEF): + // - E0 A0–BF .. => valid (U+0800–U+0FFF) + // - E0 80–9F .. => overlong (U+0000–U+07FF) => impossible + // - ED 80–9F .. => valid (U+D000–U+D7FF) + // - ED A0–BF .. => surrogates (U+D800–U+DFFF) => impossible + if (first == 0xE0 && second < 0xA0) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + if (first == 0xED && second > 0x9F) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + } else if (seq_len == 4) { + // 4-byte sequences (first in 0xF0–0xF4): + // - F0 90–BF .. .. => valid (U+10000–U+3FFFF) + // - F0 80–8F .. .. => overlong (U+0000–U+FFFF) => impossible + // - F4 80–8F .. .. => valid (U+100000–U+10FFFF) + // - F4 90–BF .. .. => > U+10FFFF => impossible + if (first == 0xF0 && second < 0x90) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + if (first == 0xF4 && second > 0x8F) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + } + + // For any further available bytes, just enforce the continuation pattern. + for (size_t i = 2; i < available; ++i) { + unsigned char byte = static_cast(input[offset + i]); + if ((byte & 0xC0) != 0x80) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + } + } + + // If we reach here, the prefix is syntactically and range-wise + // compatible with *some* valid UTF-8 code point; we just ran out of bytes. return utf8_parse_result(utf8_parse_result::INCOMPLETE); } + // We have at least seq_len bytes: validate all continuation bytes. + for (size_t i = 1; i < seq_len; ++i) { + unsigned char byte = static_cast(input[offset + i]); + if ((byte & 0xC0) != 0x80) { + return utf8_parse_result(utf8_parse_result::INVALID); + } + } + + // Decode based on sequence length. uint32_t codepoint = 0; - // Decode based on sequence length if (seq_len == 2) { // 110xxxxx 10xxxxxx - if ((first & 0xE0) != 0xC0) { - return utf8_parse_result(utf8_parse_result::INVALID); - } - const unsigned char second = static_cast(input[offset + 1]); - if ((second & 0xC0) != 0x80) { - return utf8_parse_result(utf8_parse_result::INVALID); - } - codepoint = ((first & 0x1F) << 6) | (second & 0x3F); - // Check for overlong encoding - if (codepoint < 0x80) { - return utf8_parse_result(utf8_parse_result::INVALID); - } + codepoint = + ((first & 0x1F) << 6) | + (static_cast(input[offset + 1]) & 0x3F); + + // 0xC0 and 0xC1 were filtered out, so this always yields U+0080–U+07FF. } else if (seq_len == 3) { // 1110xxxx 10xxxxxx 10xxxxxx - if ((first & 0xF0) != 0xE0) { - return utf8_parse_result(utf8_parse_result::INVALID); - } - const unsigned char second = static_cast(input[offset + 1]); - const unsigned char third = static_cast(input[offset + 2]); - if ((second & 0xC0) != 0x80 || (third & 0xC0) != 0x80) { - return utf8_parse_result(utf8_parse_result::INVALID); - } - codepoint = ((first & 0x0F) << 12) | ((second & 0x3F) << 6) | (third & 0x3F); - // Check for overlong encoding + codepoint = + ((first & 0x0F) << 12) | + ((static_cast(input[offset + 1]) & 0x3F) << 6) | + (static_cast(input[offset + 2]) & 0x3F); + + // Reject overlong encodings: 3-byte must encode U+0800–U+FFFF. if (codepoint < 0x800) { return utf8_parse_result(utf8_parse_result::INVALID); } - // Check for surrogate pairs (0xD800-0xDFFF are invalid in UTF-8) + + // Reject surrogate code points U+D800–U+DFFF (invalid in UTF-8). if (codepoint >= 0xD800 && codepoint <= 0xDFFF) { return utf8_parse_result(utf8_parse_result::INVALID); } } else if (seq_len == 4) { // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx - if ((first & 0xF8) != 0xF0) { - return utf8_parse_result(utf8_parse_result::INVALID); - } - const unsigned char second = static_cast(input[offset + 1]); - const unsigned char third = static_cast(input[offset + 2]); - const unsigned char fourth = static_cast(input[offset + 3]); - if ((second & 0xC0) != 0x80 || (third & 0xC0) != 0x80 || (fourth & 0xC0) != 0x80) { - return utf8_parse_result(utf8_parse_result::INVALID); - } - codepoint = ((first & 0x07) << 18) | ((second & 0x3F) << 12) | - ((third & 0x3F) << 6) | (fourth & 0x3F); - // Check for overlong encoding + codepoint = + ((first & 0x07) << 18) | + ((static_cast(input[offset + 1]) & 0x3F) << 12) | + ((static_cast(input[offset + 2]) & 0x3F) << 6) | + (static_cast(input[offset + 3]) & 0x3F); + + // Reject overlong encodings: 4-byte must encode U+10000–U+10FFFF. if (codepoint < 0x10000) { return utf8_parse_result(utf8_parse_result::INVALID); } - // Check for valid Unicode range (max is 0x10FFFF) + + // Reject code points beyond Unicode max (U+10FFFF). if (codepoint > 0x10FFFF) { return utf8_parse_result(utf8_parse_result::INVALID); } - } else { - // Invalid sequence length - return utf8_parse_result(utf8_parse_result::INVALID); } return utf8_parse_result(utf8_parse_result::SUCCESS, codepoint, seq_len); From 6f662e0cb88cbaf588ec7e52de51c6665d936a88 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 15 Nov 2025 21:38:25 -0600 Subject: [PATCH 073/183] fix compiler error --- common/chat-peg-parser.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 52eff5d1a8002..534d3963852a3 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -2098,7 +2098,7 @@ common_chat_peg_parser common_chat_peg_parser_builder::quasi_xml_no_attr(const s for (auto it = parameters.begin(); it != parameters.end(); it++) { auto arg_name = add_rule(std::string("arg-start-" + *it), literal("<" + param_tag + "=" + *it + ">")); - auto arg_end = add_rule("arg-end", "" + peek(literal("<" + param_tag + "=") | "")); + auto arg_end = add_rule("arg-end", "" + peek(literal("<" + param_tag + "=") | (""))); auto string_arg_content = add_rule("arg-string-content", until_one_of({"<" + param_tag + "=", ""})); auto string_arg = add_rule("arg-string-" + *it, arg_name + string_arg_content + arg_end); From 9c02bf5b565cdef72cd9135bdcce2ee029f8e4de Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 15 Nov 2025 21:48:36 -0600 Subject: [PATCH 074/183] improve construction of sequence/choice parsers --- common/chat-peg-parser.cpp | 67 +++++++++++++++++++++++--------------- common/chat-peg-parser.h | 12 +++++++ 2 files changed, 52 insertions(+), 27 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 534d3963852a3..21723df17dcef 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -454,29 +454,20 @@ class sequence_parser : public common_chat_peg_parser_base { public: static constexpr parser_type type_value = SEQUENCE; - sequence_parser(std::initializer_list parsers, int id) : common_chat_peg_parser_base(id) { - for (const auto & p : parsers) { - if (auto seq = cast(p)) { - for (const auto & embedded : seq->parsers()) { - parsers_.push_back(embedded); - } + template + sequence_parser(InputIt first, InputIt last, int id) : common_chat_peg_parser_base(id) { + for (auto it = first; it != last; ++it) { + if (auto seq = cast(*it)) { + parsers_.insert(parsers_.end(), seq->parsers().begin(), seq->parsers().end()); } else { - parsers_.push_back(p); + parsers_.push_back(*it); } } } - sequence_parser(const std::vector& parsers, int id) : common_chat_peg_parser_base(id) { - for (const auto & p : parsers) { - if (auto seq = cast(p)) { - for (const auto & embedded : seq->parsers()) { - parsers_.push_back(embedded); - } - } else { - parsers_.push_back(p); - } - } - } + template + sequence_parser(const T & parsers, int id) + : sequence_parser(std::begin(parsers), std::end(parsers), id) {} parser_type type() const override { return type_value; } @@ -523,18 +514,21 @@ class choice_parser : public common_chat_peg_parser_base { public: static constexpr parser_type type_value = CHOICE; - choice_parser(std::initializer_list parsers, int id) : common_chat_peg_parser_base(id) { - for (const auto & p : parsers) { - if (auto choice = cast(p)) { - for (const auto & embedded : choice->parsers()) { - parsers_.push_back(embedded); - } + template + choice_parser(InputIt first, InputIt last, int id) : common_chat_peg_parser_base(id) { + for (auto it = first; it != last; ++it) { + if (auto choice = cast(*it)) { + parsers_.insert(parsers_.end(), choice->parsers().begin(), choice->parsers().end()); } else { - parsers_.push_back(p); + parsers_.push_back(*it); } } } + template + choice_parser(const T & parsers, int id) + : choice_parser(std::begin(parsers), std::end(parsers), id) {} + parser_type type() const override { return type_value; } common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { @@ -1898,10 +1892,30 @@ common_chat_peg_parser common_chat_peg_parser_builder::literal(const std::string return common_chat_peg_parser(std::make_shared(literal, counter_.next())); } +template +common_chat_peg_parser common_chat_peg_parser_builder::sequence(InputIt first, InputIt last) { + return common_chat_peg_parser(std::make_shared(first, last, counter_.next())); +} + +template +common_chat_peg_parser common_chat_peg_parser_builder::sequence(const T & parsers) { + return common_chat_peg_parser(std::make_shared(parsers, counter_.next())); +} + common_chat_peg_parser common_chat_peg_parser_builder::sequence(std::initializer_list parsers) { return common_chat_peg_parser(std::make_shared(parsers, counter_.next())); } +template +common_chat_peg_parser common_chat_peg_parser_builder::choice(InputIt first, InputIt last) { + return common_chat_peg_parser(std::make_shared(first, last, counter_.next())); +} + +template +common_chat_peg_parser common_chat_peg_parser_builder::choice(const T & parsers) { + return common_chat_peg_parser(std::make_shared(parsers, counter_.next())); +} + common_chat_peg_parser common_chat_peg_parser_builder::choice(std::initializer_list parsers) { return common_chat_peg_parser(std::make_shared(parsers, counter_.next())); } @@ -2108,8 +2122,7 @@ common_chat_peg_parser common_chat_peg_parser_builder::quasi_xml_no_attr(const s args.push_back(arg_json_or_string); } - auto args_seq_raw = sequence_parser(args, counter_.next()); - auto args_sequence = common_chat_peg_parser(std::make_shared(args_seq_raw)); + auto args_sequence = sequence(args); auto function = add_rule("function-" + function_name, add_rule("function-start-" + function_name, "<" + function_tag + "=" + function_name + ">") + args_sequence + ""); diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index 0c7b3a1232171..f058aa6ecf8c9 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -205,10 +205,22 @@ class common_chat_peg_parser_builder { // Matches a sequence of parsers in order, all must succeed. // S -> A B C + template + common_chat_peg_parser sequence(InputIt first, InputIt last); + + template + common_chat_peg_parser sequence(const T & parsers); + common_chat_peg_parser sequence(std::initializer_list parsers); // Matches the first parser that succeeds from a list of alternatives. // S -> A | B | C + template + common_chat_peg_parser choice(InputIt first, InputIt last); + + template + common_chat_peg_parser choice(const T & parsers); + common_chat_peg_parser choice(std::initializer_list parsers); // Matches one or more repetitions of a parser. From 5ee2c2d9c308e121b7439037843fa677e815f94a Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 15 Nov 2025 22:01:22 -0600 Subject: [PATCH 075/183] be less clever --- common/chat-peg-parser.cpp | 24 ++---------------------- common/chat-peg-parser.h | 16 ++-------------- 2 files changed, 4 insertions(+), 36 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 21723df17dcef..cf4bd1b443678 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -1892,31 +1892,11 @@ common_chat_peg_parser common_chat_peg_parser_builder::literal(const std::string return common_chat_peg_parser(std::make_shared(literal, counter_.next())); } -template -common_chat_peg_parser common_chat_peg_parser_builder::sequence(InputIt first, InputIt last) { - return common_chat_peg_parser(std::make_shared(first, last, counter_.next())); -} - -template -common_chat_peg_parser common_chat_peg_parser_builder::sequence(const T & parsers) { - return common_chat_peg_parser(std::make_shared(parsers, counter_.next())); -} - -common_chat_peg_parser common_chat_peg_parser_builder::sequence(std::initializer_list parsers) { +common_chat_peg_parser common_chat_peg_parser_builder::sequence(const std::vector & parsers) { return common_chat_peg_parser(std::make_shared(parsers, counter_.next())); } -template -common_chat_peg_parser common_chat_peg_parser_builder::choice(InputIt first, InputIt last) { - return common_chat_peg_parser(std::make_shared(first, last, counter_.next())); -} - -template -common_chat_peg_parser common_chat_peg_parser_builder::choice(const T & parsers) { - return common_chat_peg_parser(std::make_shared(parsers, counter_.next())); -} - -common_chat_peg_parser common_chat_peg_parser_builder::choice(std::initializer_list parsers) { +common_chat_peg_parser common_chat_peg_parser_builder::choice(const std::vector & parsers) { return common_chat_peg_parser(std::make_shared(parsers, counter_.next())); } diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index f058aa6ecf8c9..e59f6ddbbae3d 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -205,23 +205,11 @@ class common_chat_peg_parser_builder { // Matches a sequence of parsers in order, all must succeed. // S -> A B C - template - common_chat_peg_parser sequence(InputIt first, InputIt last); - - template - common_chat_peg_parser sequence(const T & parsers); - - common_chat_peg_parser sequence(std::initializer_list parsers); + common_chat_peg_parser sequence(const std::vector & parsers); // Matches the first parser that succeeds from a list of alternatives. // S -> A | B | C - template - common_chat_peg_parser choice(InputIt first, InputIt last); - - template - common_chat_peg_parser choice(const T & parsers); - - common_chat_peg_parser choice(std::initializer_list parsers); + common_chat_peg_parser choice(const std::vector & parsers); // Matches one or more repetitions of a parser. // S -> A+ From a1f461aa0172b6f3a5b75e3ae4ef6d091c17e39a Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 15 Nov 2025 22:12:33 -0600 Subject: [PATCH 076/183] add make_parser helper function --- common/chat-peg-parser.cpp | 50 +++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index cf4bd1b443678..8446a6c600626 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -1876,97 +1876,103 @@ void common_chat_peg_parser::build_grammar(const common_grammar_builder & builde } } +// Create an internal parser and wrap it in common_chat_peg_parser +template +static common_chat_peg_parser make_parser(common_chat_peg_parser_counter & counter, Args&&... args) { + return common_chat_peg_parser(std::make_shared(std::forward(args)..., counter.next())); +} + common_chat_peg_parser_builder::common_chat_peg_parser_builder() : root_(std::make_shared(0)) // root parser has id 0 , counter_(1) {} common_chat_peg_parser common_chat_peg_parser_builder::start() { - return common_chat_peg_parser(std::make_shared(counter_.next())); + return make_parser(counter_); } common_chat_peg_parser common_chat_peg_parser_builder::end() { - return common_chat_peg_parser(std::make_shared(counter_.next())); + return make_parser(counter_); } common_chat_peg_parser common_chat_peg_parser_builder::literal(const std::string & literal) { - return common_chat_peg_parser(std::make_shared(literal, counter_.next())); + return make_parser(counter_, literal); } common_chat_peg_parser common_chat_peg_parser_builder::sequence(const std::vector & parsers) { - return common_chat_peg_parser(std::make_shared(parsers, counter_.next())); + return make_parser(counter_, parsers); } common_chat_peg_parser common_chat_peg_parser_builder::choice(const std::vector & parsers) { - return common_chat_peg_parser(std::make_shared(parsers, counter_.next())); + return make_parser(counter_, parsers); } common_chat_peg_parser common_chat_peg_parser_builder::one_or_more(const common_chat_peg_parser & p) { - return common_chat_peg_parser(std::make_shared(p, counter_.next())); + return make_parser(counter_, p); } common_chat_peg_parser common_chat_peg_parser_builder::zero_or_more(const common_chat_peg_parser & p) { - return common_chat_peg_parser(std::make_shared(p, counter_.next())); + return make_parser(counter_, p); } common_chat_peg_parser common_chat_peg_parser_builder::optional(const common_chat_peg_parser & p) { - return common_chat_peg_parser(std::make_shared(p, counter_.next())); + return make_parser(counter_, p); } common_chat_peg_parser common_chat_peg_parser_builder::peek(const common_chat_peg_parser & p) { - return common_chat_peg_parser(std::make_shared(p, counter_.next())); + return make_parser(counter_, p); } common_chat_peg_parser common_chat_peg_parser_builder::negate(const common_chat_peg_parser & p) { - return common_chat_peg_parser(std::make_shared(p, counter_.next())); + return make_parser(counter_, p); } common_chat_peg_parser common_chat_peg_parser_builder::any() { - return common_chat_peg_parser(std::make_shared(counter_.next())); + return make_parser(counter_); } common_chat_peg_parser common_chat_peg_parser_builder::chars(const std::string & classes, int min, int max) { - return common_chat_peg_parser(std::make_shared(classes, min, max, counter_.next())); + return make_parser(counter_, classes, min, max); } common_chat_peg_parser common_chat_peg_parser_builder::one(const std::string & classes) { - return chars(classes, 1, 1); + return make_parser(counter_, classes, 1, 1); } common_chat_peg_parser common_chat_peg_parser_builder::json_string_unqouted() { - return common_chat_peg_parser(std::make_shared(counter_.next())); + return make_parser(counter_); } common_chat_peg_parser common_chat_peg_parser_builder::rule(const std::string & name) { auto root = cast(root_); - return common_chat_peg_parser(std::make_shared(name, std::weak_ptr(root), counter_.next())); + return make_parser(counter_, name, std::weak_ptr(root)); } common_chat_peg_parser common_chat_peg_parser_builder::space() { - return common_chat_peg_parser(std::make_shared(counter_.next())); + return make_parser(counter_); } common_chat_peg_parser common_chat_peg_parser_builder::until(const std::string & delimiter) { - return common_chat_peg_parser(std::make_shared(delimiter, counter_.next())); + return make_parser(counter_, delimiter); } common_chat_peg_parser common_chat_peg_parser_builder::until_one_of(const std::vector & delimiters) { - return common_chat_peg_parser(std::make_shared(delimiters, counter_.next())); + return make_parser(counter_, delimiters); } common_chat_peg_parser common_chat_peg_parser_builder::repeat(const common_chat_peg_parser & p, int min, int max) { - return common_chat_peg_parser(std::make_shared(p, min, max, counter_.next())); + return make_parser(counter_, p, min, max); } common_chat_peg_parser common_chat_peg_parser_builder::repeat(const common_chat_peg_parser & p, int n) { - return repeat(p, n, n); + return make_parser(counter_, p, n, n); } common_chat_peg_parser common_chat_peg_parser_builder::schema(const common_chat_peg_parser & p, const std::string & name, const nlohmann::ordered_json & schema) { - return common_chat_peg_parser(std::make_shared(p, name, schema, counter_.next())); + return make_parser(counter_, p, name, schema); } common_chat_peg_parser common_chat_peg_parser_builder::action(const common_chat_peg_parser & p, std::function fn, int when) { - return common_chat_peg_parser(std::make_shared(p, std::move(fn), when, counter_.next())); + return make_parser(counter_, p, std::move(fn), when); } common_chat_peg_parser common_chat_peg_parser_builder::capture(const std::string & key, const common_chat_peg_parser & p) { From 086ba59d48a15f35243d965fb832690f52f6894b Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 15 Nov 2025 22:35:25 -0600 Subject: [PATCH 077/183] expand usage of make_parser, alias common_chat_msg_peg_parser_builder to builder in source --- common/chat-peg-parser.cpp | 133 +++++++++++++++++++------------------ 1 file changed, 70 insertions(+), 63 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 8446a6c600626..8ef4d5eb067f3 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -91,6 +91,17 @@ class common_chat_peg_parser_base { virtual void accept(parser_visitor & visitor) = 0; }; +// Create an internal parser +template +static std::shared_ptr make_parser(int id, Args&&... args) { + return std::make_shared(std::forward(args)..., id); +} + +template +static std::shared_ptr make_parser(common_chat_peg_parser_counter & counter, Args&&... args) { + return std::make_shared(std::forward(args)..., counter.next()); +} + // Convenience cast functions template static std::shared_ptr cast(const std::shared_ptr & p) { @@ -1822,25 +1833,25 @@ common_chat_peg_parser::common_chat_peg_parser() {} common_chat_peg_parser::common_chat_peg_parser(std::shared_ptr parser) : ptr_(std::move(parser)) {} -common_chat_peg_parser::common_chat_peg_parser(const std::string & literal) : ptr_(std::make_shared(literal, -1)) {} +common_chat_peg_parser::common_chat_peg_parser(const std::string & literal) : ptr_(make_parser(-1, literal)) {} -common_chat_peg_parser::common_chat_peg_parser(const char * literal) : ptr_(std::make_shared(literal, -1)) {} +common_chat_peg_parser::common_chat_peg_parser(const char * literal) : ptr_(make_parser(-1, literal)) {} common_chat_peg_parser common_chat_peg_parser::operator~() const { - return common_chat_peg_parser(std::make_shared(*this, -1)); + return make_parser(-1, *this); } common_chat_peg_parser common_chat_peg_parser::operator+(const common_chat_peg_parser & other) const { - return common_chat_peg_parser(std::make_shared(std::initializer_list{*this, other}, -1)); + return make_parser(-1, std::initializer_list{*this, other}); } common_chat_peg_parser common_chat_peg_parser::operator|(const common_chat_peg_parser & other) const { - return common_chat_peg_parser(std::make_shared(std::initializer_list{*this, other}, -1)); + return make_parser(-1, std::initializer_list{*this, other}); } common_chat_peg_parser common_chat_peg_parser::operator<<(const common_chat_peg_parser & other) const { - auto ws = common_chat_peg_parser(std::make_shared(-1)); - return common_chat_peg_parser(std::make_shared(std::initializer_list{*this, ws, other}, -1)); + auto ws = make_parser(-1); + return make_parser(-1, std::initializer_list{*this, ws, other}); } common_chat_peg_parser operator+(const char * lhs, const common_chat_peg_parser & rhs) { return common_chat_peg_parser(lhs) + rhs; } @@ -1876,123 +1887,119 @@ void common_chat_peg_parser::build_grammar(const common_grammar_builder & builde } } -// Create an internal parser and wrap it in common_chat_peg_parser -template -static common_chat_peg_parser make_parser(common_chat_peg_parser_counter & counter, Args&&... args) { - return common_chat_peg_parser(std::make_shared(std::forward(args)..., counter.next())); -} +using builder = common_chat_peg_parser_builder; -common_chat_peg_parser_builder::common_chat_peg_parser_builder() - : root_(std::make_shared(0)) // root parser has id 0 +builder::common_chat_peg_parser_builder() + : root_(make_parser(0)) // root parser has id 0 , counter_(1) {} -common_chat_peg_parser common_chat_peg_parser_builder::start() { +common_chat_peg_parser builder::start() { return make_parser(counter_); } -common_chat_peg_parser common_chat_peg_parser_builder::end() { +common_chat_peg_parser builder::end() { return make_parser(counter_); } -common_chat_peg_parser common_chat_peg_parser_builder::literal(const std::string & literal) { +common_chat_peg_parser builder::literal(const std::string & literal) { return make_parser(counter_, literal); } -common_chat_peg_parser common_chat_peg_parser_builder::sequence(const std::vector & parsers) { +common_chat_peg_parser builder::sequence(const std::vector & parsers) { return make_parser(counter_, parsers); } -common_chat_peg_parser common_chat_peg_parser_builder::choice(const std::vector & parsers) { +common_chat_peg_parser builder::choice(const std::vector & parsers) { return make_parser(counter_, parsers); } -common_chat_peg_parser common_chat_peg_parser_builder::one_or_more(const common_chat_peg_parser & p) { +common_chat_peg_parser builder::one_or_more(const common_chat_peg_parser & p) { return make_parser(counter_, p); } -common_chat_peg_parser common_chat_peg_parser_builder::zero_or_more(const common_chat_peg_parser & p) { +common_chat_peg_parser builder::zero_or_more(const common_chat_peg_parser & p) { return make_parser(counter_, p); } -common_chat_peg_parser common_chat_peg_parser_builder::optional(const common_chat_peg_parser & p) { +common_chat_peg_parser builder::optional(const common_chat_peg_parser & p) { return make_parser(counter_, p); } -common_chat_peg_parser common_chat_peg_parser_builder::peek(const common_chat_peg_parser & p) { +common_chat_peg_parser builder::peek(const common_chat_peg_parser & p) { return make_parser(counter_, p); } -common_chat_peg_parser common_chat_peg_parser_builder::negate(const common_chat_peg_parser & p) { +common_chat_peg_parser builder::negate(const common_chat_peg_parser & p) { return make_parser(counter_, p); } -common_chat_peg_parser common_chat_peg_parser_builder::any() { +common_chat_peg_parser builder::any() { return make_parser(counter_); } -common_chat_peg_parser common_chat_peg_parser_builder::chars(const std::string & classes, int min, int max) { +common_chat_peg_parser builder::chars(const std::string & classes, int min, int max) { return make_parser(counter_, classes, min, max); } -common_chat_peg_parser common_chat_peg_parser_builder::one(const std::string & classes) { +common_chat_peg_parser builder::one(const std::string & classes) { return make_parser(counter_, classes, 1, 1); } -common_chat_peg_parser common_chat_peg_parser_builder::json_string_unqouted() { +common_chat_peg_parser builder::json_string_unqouted() { return make_parser(counter_); } -common_chat_peg_parser common_chat_peg_parser_builder::rule(const std::string & name) { +common_chat_peg_parser builder::rule(const std::string & name) { auto root = cast(root_); return make_parser(counter_, name, std::weak_ptr(root)); } -common_chat_peg_parser common_chat_peg_parser_builder::space() { +common_chat_peg_parser builder::space() { return make_parser(counter_); } -common_chat_peg_parser common_chat_peg_parser_builder::until(const std::string & delimiter) { +common_chat_peg_parser builder::until(const std::string & delimiter) { return make_parser(counter_, delimiter); } -common_chat_peg_parser common_chat_peg_parser_builder::until_one_of(const std::vector & delimiters) { +common_chat_peg_parser builder::until_one_of(const std::vector & delimiters) { return make_parser(counter_, delimiters); } -common_chat_peg_parser common_chat_peg_parser_builder::repeat(const common_chat_peg_parser & p, int min, int max) { +common_chat_peg_parser builder::repeat(const common_chat_peg_parser & p, int min, int max) { return make_parser(counter_, p, min, max); } -common_chat_peg_parser common_chat_peg_parser_builder::repeat(const common_chat_peg_parser & p, int n) { +common_chat_peg_parser builder::repeat(const common_chat_peg_parser & p, int n) { return make_parser(counter_, p, n, n); } -common_chat_peg_parser common_chat_peg_parser_builder::schema(const common_chat_peg_parser & p, const std::string & name, const nlohmann::ordered_json & schema) { +common_chat_peg_parser builder::schema(const common_chat_peg_parser & p, const std::string & name, const nlohmann::ordered_json & schema) { return make_parser(counter_, p, name, schema); } -common_chat_peg_parser common_chat_peg_parser_builder::action(const common_chat_peg_parser & p, std::function fn, int when) { +common_chat_peg_parser builder::action(const common_chat_peg_parser & p, std::function fn, int when) { return make_parser(counter_, p, std::move(fn), when); } -common_chat_peg_parser common_chat_peg_parser_builder::capture(const std::string & key, const common_chat_peg_parser & p) { +common_chat_peg_parser builder::capture(const std::string & key, const common_chat_peg_parser & p) { return action(p, [key](const common_chat_parse_action & act) { std::string value = std::string(act.match); act.env.captures[key] = std::move(value); }, COMMON_CHAT_PARSE_RESULT_SUCCESS); } -common_chat_peg_parser common_chat_peg_parser_builder::trigger(const common_chat_peg_parser & p) { - return common_chat_peg_parser(std::make_shared(p, counter_.next())); +common_chat_peg_parser builder::trigger(const common_chat_peg_parser & p) { + return make_parser(counter_, p); } -common_chat_peg_parser common_chat_peg_parser_builder::add_rule(const std::string & name, const common_chat_peg_parser & p) { +common_chat_peg_parser builder::add_rule(const std::string & name, const common_chat_peg_parser & p) { auto root = cast(root_); root->add_rule(name, p); return rule(name); } -common_chat_peg_parser common_chat_peg_parser_builder::add_rule(const std::string & name, const std::function & builder) { +common_chat_peg_parser builder::add_rule(const std::string & name, const std::function & builder) { auto root = cast(root_); if (root->rules().find(name) != root->rules().end()) { return rule(name); @@ -2004,7 +2011,7 @@ common_chat_peg_parser common_chat_peg_parser_builder::add_rule(const std::strin return rule(name); } -void common_chat_peg_parser_builder::set_root(const common_chat_peg_parser & p) { +void builder::set_root(const common_chat_peg_parser & p) { auto root_container = cast(root_); root_container->set_root(p); @@ -2014,18 +2021,7 @@ void common_chat_peg_parser_builder::set_root(const common_chat_peg_parser & p) } } -common_chat_peg_parser common_chat_peg_parser_builder::build() { - return root_; -} - -common_chat_peg_parser build_peg_parser(const std::function & fn) { - common_chat_peg_parser_builder builder; - auto root = fn(builder); - builder.set_root(root); - return builder.build(); -} - -common_chat_peg_parser common_chat_peg_parser_builder::json_number() { +common_chat_peg_parser builder::json_number() { return add_rule("json-number", [this]() { auto digit1_9 = chars("[1-9]", 1, 1); auto digits = chars("[0-9]"); @@ -2036,25 +2032,25 @@ common_chat_peg_parser common_chat_peg_parser_builder::json_number() { }); } -common_chat_peg_parser common_chat_peg_parser_builder::json_string() { +common_chat_peg_parser builder::json_string() { return add_rule("json-string", [this]() { return literal("\"") + json_string_unqouted() + literal("\""); }); } -common_chat_peg_parser common_chat_peg_parser_builder::json_bool() { +common_chat_peg_parser builder::json_bool() { return add_rule("json-bool", [this]() { return literal("true") | literal("false"); }); } -common_chat_peg_parser common_chat_peg_parser_builder::json_null() { +common_chat_peg_parser builder::json_null() { return add_rule("json-null", [this]() { return literal("null"); }); } -common_chat_peg_parser common_chat_peg_parser_builder::json_object() { +common_chat_peg_parser builder::json_object() { return add_rule("json-object", [this]() { auto ws = space(); auto member = json_string() + ws + literal(":") + ws + json(); @@ -2064,7 +2060,7 @@ common_chat_peg_parser common_chat_peg_parser_builder::json_object() { }); } -common_chat_peg_parser common_chat_peg_parser_builder::json_array() { +common_chat_peg_parser builder::json_array() { return add_rule("json-array", [this]() { auto ws = space(); auto elements = json() + zero_or_more(ws + literal(",") + ws + json()); @@ -2073,7 +2069,7 @@ common_chat_peg_parser common_chat_peg_parser_builder::json_array() { }); } -common_chat_peg_parser common_chat_peg_parser_builder::json() { +common_chat_peg_parser builder::json() { return add_rule("json-value", [this]() { return json_object() | json_array() | @@ -2084,15 +2080,15 @@ common_chat_peg_parser common_chat_peg_parser_builder::json() { }); } -common_chat_peg_parser common_chat_peg_parser_builder::reasoning(const std::string &tag) { +common_chat_peg_parser builder::reasoning(const std::string &tag) { return add_rule("raw-reasoning", std::string("<" + tag + ">") << add_rule("reasoning-content", until("")) << ""); } -common_chat_peg_parser common_chat_peg_parser_builder::content_before_tools(const std::string &tag) { +common_chat_peg_parser builder::content_before_tools(const std::string &tag) { return add_rule("content", until(tag)); } -common_chat_peg_parser common_chat_peg_parser_builder::quasi_xml_no_attr(const std::string &function_name, const std::vector ¶meters, +common_chat_peg_parser builder::quasi_xml_no_attr(const std::string &function_name, const std::vector ¶meters, const std::string &function_tag, const std::string ¶m_tag) { std::vector args; @@ -2115,3 +2111,14 @@ common_chat_peg_parser common_chat_peg_parser_builder::quasi_xml_no_attr(const s return function; } + +common_chat_peg_parser builder::build() { + return root_; +} + +common_chat_peg_parser build_peg_parser(const std::function & fn) { + builder builder; + auto root = fn(builder); + builder.set_root(root); + return builder.build(); +} From 15c0d857c28b9a586339ba45502d922c04bb2e2a Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 15 Nov 2025 23:31:37 -0600 Subject: [PATCH 078/183] lower bench iterations --- tests/chat-peg-parser/test-command7-parser-compare.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/chat-peg-parser/test-command7-parser-compare.cpp b/tests/chat-peg-parser/test-command7-parser-compare.cpp index 5b4175897a441..ac3f313b55aee 100644 --- a/tests/chat-peg-parser/test-command7-parser-compare.cpp +++ b/tests/chat-peg-parser/test-command7-parser-compare.cpp @@ -262,9 +262,9 @@ void test_command7_parser_compare(testing &t) { // Run benchmarks t.bench("legacy_parse_benchmark", [&]() { test_command_r7b_legacy_parser(input, false, false); - }, 1000); + }, 100); t.bench("current_parse_benchmark", [&]() { test_command_r7b_parser(parser, input, false, false); - }, 1000); + }, 100); } From 68abea7f5886ff68eebcfb624cf1f427ddc53038 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 15 Nov 2025 23:31:57 -0600 Subject: [PATCH 079/183] add unicode support to until_parser --- common/chat-peg-parser.cpp | 32 +++++++- tests/chat-peg-parser/test-unicode.cpp | 108 +++++++++++++++++++++++++ 2 files changed, 139 insertions(+), 1 deletion(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 8ef4d5eb067f3..7cc596a1c6f35 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -1172,8 +1172,38 @@ class until_parser : public common_chat_peg_parser_base { parser_type type() const override { return type_value; } common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { + // First pass: byte-based Aho-Corasick search for delimiter auto search_result = matcher_.search(ctx.input, start); - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, search_result.pos); + size_t delimiter_pos = search_result.pos; + + // Second pass: validate UTF-8 from start to delimiter_pos + size_t pos = start; + size_t last_valid_pos = start; + + while (pos < delimiter_pos) { + auto utf8_result = parse_utf8_codepoint(ctx.input, pos); + + if (utf8_result.status == utf8_parse_result::INCOMPLETE) { + // Incomplete UTF-8 sequence before delimiter + if (ctx.input_is_complete) { + // Input is complete but UTF-8 is incomplete = malformed + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); + } + // Return what we have so far (before incomplete sequence) + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start, last_valid_pos); + } + + if (utf8_result.status == utf8_parse_result::INVALID) { + // Malformed UTF-8 + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); + } + + pos += utf8_result.bytes_consumed; + last_valid_pos = pos; + } + + // All UTF-8 validated up to delimiter + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, last_valid_pos); } std::string dump() const override { diff --git a/tests/chat-peg-parser/test-unicode.cpp b/tests/chat-peg-parser/test-unicode.cpp index 6811da9342954..4b9254b536585 100644 --- a/tests/chat-peg-parser/test-unicode.cpp +++ b/tests/chat-peg-parser/test-unicode.cpp @@ -205,4 +205,112 @@ void test_unicode(testing &t) { } }); }); + + t.test("until parser", [](testing &t) { + t.test("ASCII delimiter with Unicode content", [](testing &t) { + std::vector test_cases { + // CJK characters before delimiter + {std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), + std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // δ½ ε₯½ + + // Emoji before delimiter + {std::string("\xF0\x9F\x98\x80"), + std::string("\xF0\x9F\x98\x80"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // πŸ˜€ + + // Mixed content + {std::string("Hello \xE4\xB8\x96\xE7\x95\x8C!"), + std::string("Hello \xE4\xB8\x96\xE7\x95\x8C!"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // Hello δΈ–η•Œ! + }; + + auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { + return p.until(""); + }); + + for (size_t i = 0; i < test_cases.size(); i++) { + const auto & tc = test_cases[i]; + std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input); + + t.test(test_name, [&](testing &t) { + common_chat_parse_context ctx(tc.input, true); + auto result = parser.parse(ctx); + + assert_result_equal(t, tc.expected_result, result.type); + + if (result.success()) { + std::string matched = tc.input.substr(result.start, result.end - result.start); + t.assert_equal(tc.expected_text, matched); + } + }); + } + }); + + t.test("incomplete UTF-8 at end (streaming)", [](testing &t) { + std::vector test_cases { + // Incomplete emoji at end, no delimiter + {std::string("content\xF0\x9F\x98"), + std::string("content"), COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, + + // Incomplete CJK at end, no delimiter + {std::string("hello\xE4\xB8"), + std::string("hello"), COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, + + // Complete content, no delimiter (should consume all valid UTF-8) + {std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), + std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, + }; + + auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { + return p.until(""); + }); + + for (size_t i = 0; i < test_cases.size(); i++) { + const auto & tc = test_cases[i]; + std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input); + + t.test(test_name, [&](testing &t) { + common_chat_parse_context ctx(tc.input, false); // input_is_complete = false + auto result = parser.parse(ctx); + + assert_result_equal(t, tc.expected_result, result.type); + + if (result.success() || result.need_more_input()) { + std::string matched = tc.input.substr(result.start, result.end - result.start); + t.assert_equal(tc.expected_text, matched); + } + }); + } + }); + + t.test("malformed UTF-8", [](testing &t) { + std::vector test_cases { + // Invalid UTF-8 bytes + {std::string("Hello\xFF\xFE"), + "", COMMON_CHAT_PARSE_RESULT_FAIL}, + + // Continuation byte without lead byte + {std::string("Hello\x80World"), + "", COMMON_CHAT_PARSE_RESULT_FAIL}, + + // Invalid continuation byte + {std::string("\xC3\x28"), + "", COMMON_CHAT_PARSE_RESULT_FAIL}, + }; + + auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { + return p.until(""); + }); + + for (size_t i = 0; i < test_cases.size(); i++) { + const auto & tc = test_cases[i]; + std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input); + + t.test(test_name, [&](testing &t) { + common_chat_parse_context ctx(tc.input, true); + auto result = parser.parse(ctx); + + assert_result_equal(t, tc.expected_result, result.type); + }); + } + }); + }); } From 15bb3ca04dcb2143cec4ec30dcda8e32f49c7472 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 00:05:52 -0600 Subject: [PATCH 080/183] add unicode support to json_string_parser --- common/chat-peg-parser.cpp | 19 +++- tests/chat-peg-parser/test-unicode.cpp | 145 +++++++++++++++++++++++++ 2 files changed, 162 insertions(+), 2 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 7cc596a1c6f35..273d06f2b9085 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -1135,8 +1135,23 @@ class json_string_parser : public common_chat_peg_parser_base { return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); } } else { - // Regular character - ++pos; + // Regular character - validate UTF-8 + auto utf8_result = parse_utf8_codepoint(ctx.input, pos); + + if (utf8_result.status == utf8_parse_result::INCOMPLETE) { + // Incomplete UTF-8 sequence + if (ctx.input_is_complete) { + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); + } + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start, pos); + } + + if (utf8_result.status == utf8_parse_result::INVALID) { + // Malformed UTF-8 in JSON string + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); + } + + pos += utf8_result.bytes_consumed; } } diff --git a/tests/chat-peg-parser/test-unicode.cpp b/tests/chat-peg-parser/test-unicode.cpp index 4b9254b536585..42a66721e05db 100644 --- a/tests/chat-peg-parser/test-unicode.cpp +++ b/tests/chat-peg-parser/test-unicode.cpp @@ -313,4 +313,149 @@ void test_unicode(testing &t) { } }); }); + + t.test("json_string parser", [](testing &t) { + t.test("valid UTF-8 characters", [](testing &t) { + std::vector test_cases { + // ASCII only + {"Hello World\"", "Hello World", COMMON_CHAT_PARSE_RESULT_SUCCESS}, + + // 2-byte UTF-8 (accented characters) + {std::string("Caf\xC3\xA9\""), std::string("Caf\xC3\xA9"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // CafΓ© + + // 3-byte UTF-8 (CJK) + {std::string("\xE4\xBD\xA0\xE5\xA5\xBD\""), std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // δ½ ε₯½ + + // 4-byte UTF-8 (emoji) + {std::string("\xF0\x9F\x98\x80\""), std::string("\xF0\x9F\x98\x80"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // πŸ˜€ + + // Mixed content + {std::string("Hello \xE4\xB8\x96\xE7\x95\x8C!\""), std::string("Hello \xE4\xB8\x96\xE7\x95\x8C!"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // Hello δΈ–η•Œ! + }; + + for (size_t i = 0; i < test_cases.size(); i++) { + const auto & tc = test_cases[i]; + std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input); + + t.test(test_name, [&](testing &t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { + return p.json_string_unqouted() + p.literal("\""); + }); + + common_chat_parse_context ctx(tc.input, true); + auto result = parser.parse(ctx); + + assert_result_equal(t, tc.expected_result, result.type); + + if (result.success()) { + std::string matched = tc.input.substr(result.start, result.end - result.start - 1); // -1 to exclude closing quote + t.assert_equal(tc.expected_text, matched); + } + }); + } + }); + + t.test("incomplete UTF-8 (streaming mode)", [](testing &t) { + std::vector test_cases { + // Incomplete 2-byte sequence + {std::string("Caf\xC3"), std::string("Caf"), COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, + + // Incomplete 3-byte sequence + {std::string("Hello\xE4\xB8"), std::string("Hello"), COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, + + // Incomplete 4-byte sequence + {std::string("Text\xF0\x9F\x98"), std::string("Text"), COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, + + // Incomplete at very start + {std::string("\xE4\xBD"), std::string(""), COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, + }; + + for (size_t i = 0; i < test_cases.size(); i++) { + const auto & tc = test_cases[i]; + std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input); + + t.test(test_name, [&](testing &t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { + return p.json_string_unqouted(); + }); + + common_chat_parse_context ctx(tc.input, false); // input_is_complete = false + auto result = parser.parse(ctx); + + assert_result_equal(t, tc.expected_result, result.type); + + if (result.need_more_input()) { + std::string matched = tc.input.substr(result.start, result.end - result.start); + t.assert_equal(tc.expected_text, matched); + } + }); + } + }); + + t.test("malformed UTF-8", [](testing &t) { + std::vector test_cases { + // Invalid UTF-8 bytes + {std::string("Hello\xFF\xFE"), "", COMMON_CHAT_PARSE_RESULT_FAIL}, + + // Continuation byte without lead byte + {std::string("Hello\x80World"), "", COMMON_CHAT_PARSE_RESULT_FAIL}, + + // Invalid continuation byte + {std::string("\xC3\x28"), "", COMMON_CHAT_PARSE_RESULT_FAIL}, + + // Overlong encoding (security issue) + {std::string("\xC0\x80"), "", COMMON_CHAT_PARSE_RESULT_FAIL}, + }; + + for (size_t i = 0; i < test_cases.size(); i++) { + const auto & tc = test_cases[i]; + std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input); + + t.test(test_name, [&](testing &t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { + return p.json_string_unqouted(); + }); + + common_chat_parse_context ctx(tc.input, true); + auto result = parser.parse(ctx); + + assert_result_equal(t, tc.expected_result, result.type); + }); + } + }); + + t.test("escape sequences with UTF-8", [](testing &t) { + std::vector test_cases { + // Unicode escape sequence + {"Hello\\u0041\"", "Hello\\u0041", COMMON_CHAT_PARSE_RESULT_SUCCESS}, // \u0041 = 'A' + + // Mix of UTF-8 and escape sequences + {std::string("\xE4\xBD\xA0\\n\xE5\xA5\xBD\""), std::string("\xE4\xBD\xA0\\n\xE5\xA5\xBD"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // δ½ \nε₯½ + + // Escaped quote in UTF-8 string + {std::string("\xE4\xBD\xA0\\\"\xE5\xA5\xBD\""), std::string("\xE4\xBD\xA0\\\"\xE5\xA5\xBD"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // δ½ \"ε₯½ + }; + + for (size_t i = 0; i < test_cases.size(); i++) { + const auto & tc = test_cases[i]; + std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input); + + t.test(test_name, [&](testing &t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { + return p.json_string_unqouted() + p.literal("\""); + }); + + common_chat_parse_context ctx(tc.input, true); + auto result = parser.parse(ctx); + + assert_result_equal(t, tc.expected_result, result.type); + + if (result.success()) { + std::string matched = tc.input.substr(result.start, result.end - result.start - 1); // -1 to exclude closing quote + t.assert_equal(tc.expected_text, matched); + } + }); + } + }); + }); } From 8928c2abfeb22100d0a6c314a8d7ff158721765d Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 00:15:20 -0600 Subject: [PATCH 081/183] clean up unicode tests --- tests/chat-peg-parser/test-unicode.cpp | 47 ++++++++++---------------- 1 file changed, 18 insertions(+), 29 deletions(-) diff --git a/tests/chat-peg-parser/test-unicode.cpp b/tests/chat-peg-parser/test-unicode.cpp index 42a66721e05db..e807cdb4d292e 100644 --- a/tests/chat-peg-parser/test-unicode.cpp +++ b/tests/chat-peg-parser/test-unicode.cpp @@ -8,12 +8,10 @@ #include #include -// Assertions specific to chat-peg-parser static void assert_result_equal(testing & t, common_chat_parse_result_type expected, common_chat_parse_result_type actual) { t.assert_equal(common_chat_parse_result_type_name(expected), common_chat_parse_result_type_name(actual)); } -// Helper function to produce hex dump for non-printable characters static std::string hex_dump(const std::string& str) { std::ostringstream oss; for (unsigned char c : str) { @@ -210,16 +208,13 @@ void test_unicode(testing &t) { t.test("ASCII delimiter with Unicode content", [](testing &t) { std::vector test_cases { // CJK characters before delimiter - {std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), - std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // δ½ ε₯½ + {std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // Emoji before delimiter - {std::string("\xF0\x9F\x98\x80"), - std::string("\xF0\x9F\x98\x80"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // πŸ˜€ + {std::string("\xF0\x9F\x98\x80"), std::string("\xF0\x9F\x98\x80"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // Mixed content - {std::string("Hello \xE4\xB8\x96\xE7\x95\x8C!"), - std::string("Hello \xE4\xB8\x96\xE7\x95\x8C!"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // Hello δΈ–η•Œ! + {std::string("Hello \xE4\xB8\x96\xE7\x95\x8C!"), std::string("Hello \xE4\xB8\x96\xE7\x95\x8C!"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, }; auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { @@ -244,19 +239,16 @@ void test_unicode(testing &t) { } }); - t.test("incomplete UTF-8 at end (streaming)", [](testing &t) { + t.test("incomplete UTF-8 at end", [](testing &t) { std::vector test_cases { // Incomplete emoji at end, no delimiter - {std::string("content\xF0\x9F\x98"), - std::string("content"), COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, + {std::string("content\xF0\x9F\x98"), std::string("content"), COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, // Incomplete CJK at end, no delimiter - {std::string("hello\xE4\xB8"), - std::string("hello"), COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, + {std::string("hello\xE4\xB8"), std::string("hello"), COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, // Complete content, no delimiter (should consume all valid UTF-8) - {std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), - std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, + {std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, }; auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { @@ -284,16 +276,13 @@ void test_unicode(testing &t) { t.test("malformed UTF-8", [](testing &t) { std::vector test_cases { // Invalid UTF-8 bytes - {std::string("Hello\xFF\xFE"), - "", COMMON_CHAT_PARSE_RESULT_FAIL}, + {std::string("Hello\xFF\xFE"), "", COMMON_CHAT_PARSE_RESULT_FAIL}, // Continuation byte without lead byte - {std::string("Hello\x80World"), - "", COMMON_CHAT_PARSE_RESULT_FAIL}, + {std::string("Hello\x80World"), "", COMMON_CHAT_PARSE_RESULT_FAIL}, // Invalid continuation byte - {std::string("\xC3\x28"), - "", COMMON_CHAT_PARSE_RESULT_FAIL}, + {std::string("\xC3\x28"), "", COMMON_CHAT_PARSE_RESULT_FAIL}, }; auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { @@ -321,16 +310,16 @@ void test_unicode(testing &t) { {"Hello World\"", "Hello World", COMMON_CHAT_PARSE_RESULT_SUCCESS}, // 2-byte UTF-8 (accented characters) - {std::string("Caf\xC3\xA9\""), std::string("Caf\xC3\xA9"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // CafΓ© + {std::string("Caf\xC3\xA9\""), std::string("Caf\xC3\xA9"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // 3-byte UTF-8 (CJK) - {std::string("\xE4\xBD\xA0\xE5\xA5\xBD\""), std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // δ½ ε₯½ + {std::string("\xE4\xBD\xA0\xE5\xA5\xBD\""), std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // 4-byte UTF-8 (emoji) - {std::string("\xF0\x9F\x98\x80\""), std::string("\xF0\x9F\x98\x80"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // πŸ˜€ + {std::string("\xF0\x9F\x98\x80\""), std::string("\xF0\x9F\x98\x80"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // Mixed content - {std::string("Hello \xE4\xB8\x96\xE7\x95\x8C!\""), std::string("Hello \xE4\xB8\x96\xE7\x95\x8C!"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // Hello δΈ–η•Œ! + {std::string("Hello \xE4\xB8\x96\xE7\x95\x8C!\""), std::string("Hello \xE4\xB8\x96\xE7\x95\x8C!"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, }; for (size_t i = 0; i < test_cases.size(); i++) { @@ -355,7 +344,7 @@ void test_unicode(testing &t) { } }); - t.test("incomplete UTF-8 (streaming mode)", [](testing &t) { + t.test("incomplete UTF-8", [](testing &t) { std::vector test_cases { // Incomplete 2-byte sequence {std::string("Caf\xC3"), std::string("Caf"), COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, @@ -427,13 +416,13 @@ void test_unicode(testing &t) { t.test("escape sequences with UTF-8", [](testing &t) { std::vector test_cases { // Unicode escape sequence - {"Hello\\u0041\"", "Hello\\u0041", COMMON_CHAT_PARSE_RESULT_SUCCESS}, // \u0041 = 'A' + {"Hello\\u0041\"", "Hello\\u0041", COMMON_CHAT_PARSE_RESULT_SUCCESS}, // Mix of UTF-8 and escape sequences - {std::string("\xE4\xBD\xA0\\n\xE5\xA5\xBD\""), std::string("\xE4\xBD\xA0\\n\xE5\xA5\xBD"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // δ½ \nε₯½ + {std::string("\xE4\xBD\xA0\\n\xE5\xA5\xBD\""), std::string("\xE4\xBD\xA0\\n\xE5\xA5\xBD"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // Escaped quote in UTF-8 string - {std::string("\xE4\xBD\xA0\\\"\xE5\xA5\xBD\""), std::string("\xE4\xBD\xA0\\\"\xE5\xA5\xBD"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // δ½ \"ε₯½ + {std::string("\xE4\xBD\xA0\\\"\xE5\xA5\xBD\""), std::string("\xE4\xBD\xA0\\\"\xE5\xA5\xBD"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, }; for (size_t i = 0; i < test_cases.size(); i++) { From 5102b4158140f452bdeb8417ccbacd788b024ea2 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 00:24:39 -0600 Subject: [PATCH 082/183] reduce unicode details to match src/unicode.cpp --- common/unicode.cpp | 164 ++++++++++----------------------------------- 1 file changed, 37 insertions(+), 127 deletions(-) diff --git a/common/unicode.cpp b/common/unicode.cpp index 9dbce54256f0c..40df3f7ff76e9 100644 --- a/common/unicode.cpp +++ b/common/unicode.cpp @@ -1,29 +1,9 @@ #include "unicode.h" size_t utf8_sequence_length(unsigned char first_byte) { - // Lookup table based on high 4 bits: - // 0xxx xxxx = 1 byte (ASCII) - // 110x xxxx = 2 bytes - // 1110 xxxx = 3 bytes - // 1111 0xxx = 4 bytes (only 0xF0–0xF4 are valid starts) - static const size_t lookup[] = { - 1, 1, 1, 1, 1, 1, 1, 1, // 0000–0111 (0x00–0x7F) ASCII - 0, 0, 0, 0, // 1000–1011 (0x80–0xBF) continuation bytes - 2, 2, // 1100–1101 (0xC0–0xDF) 2-byte sequences - 3, // 1110 (0xE0–0xEF) 3-byte sequences - 4 // 1111 (0xF0–0xFF) potential 4-byte sequences - }; - - size_t len = lookup[first_byte >> 4]; - - // Filter out invalid first bytes: - // - 0xC0–0xC1: overlong 2-byte sequences (would encode U+0000–U+007F) - // - 0xF5–0xFF: would encode beyond U+10FFFF or invalid 5+ byte sequences - if (first_byte == 0xC0 || first_byte == 0xC1 || first_byte >= 0xF5) { - return 0; - } - - return len; + const size_t lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 }; + uint8_t highbits = static_cast(first_byte) >> 4; + return lookup[highbits]; } utf8_parse_result parse_utf8_codepoint(std::string_view input, size_t offset) { @@ -33,128 +13,58 @@ utf8_parse_result parse_utf8_codepoint(std::string_view input, size_t offset) { const unsigned char first = static_cast(input[offset]); - // ASCII fast path (1-byte sequence) - if (first < 0x80) { + // ASCII fast path + if (!(first & 0x80)) { return utf8_parse_result(utf8_parse_result::SUCCESS, first, 1); } - // Determine expected sequence length (0 means invalid first byte) - size_t seq_len = utf8_sequence_length(first); - if (seq_len == 0) { + // Invalid: continuation byte as first byte + if (!(first & 0x40)) { return utf8_parse_result(utf8_parse_result::INVALID); } - size_t available = input.size() - offset; - - // Handle incomplete sequences: not enough bytes for the promised length. - if (available < seq_len) { - // We want INCOMPLETE only if this prefix *could* still become valid. - // So we validate as much as we can, and reject prefixes that are - // already impossible regardless of future bytes. - - if (available >= 2) { - unsigned char second = static_cast(input[offset + 1]); - - // Second byte must be a continuation byte. - if ((second & 0xC0) != 0x80) { - return utf8_parse_result(utf8_parse_result::INVALID); - } - - // Apply lead+second byte constraints that are necessary for any - // valid UTF-8 sequence (these mirror the usual per-byte rules). - - if (seq_len == 3) { - // 3-byte sequences (first in 0xE0–0xEF): - // - E0 A0–BF .. => valid (U+0800–U+0FFF) - // - E0 80–9F .. => overlong (U+0000–U+07FF) => impossible - // - ED 80–9F .. => valid (U+D000–U+D7FF) - // - ED A0–BF .. => surrogates (U+D800–U+DFFF) => impossible - if (first == 0xE0 && second < 0xA0) { - return utf8_parse_result(utf8_parse_result::INVALID); - } - if (first == 0xED && second > 0x9F) { - return utf8_parse_result(utf8_parse_result::INVALID); - } - } else if (seq_len == 4) { - // 4-byte sequences (first in 0xF0–0xF4): - // - F0 90–BF .. .. => valid (U+10000–U+3FFFF) - // - F0 80–8F .. .. => overlong (U+0000–U+FFFF) => impossible - // - F4 80–8F .. .. => valid (U+100000–U+10FFFF) - // - F4 90–BF .. .. => > U+10FFFF => impossible - if (first == 0xF0 && second < 0x90) { - return utf8_parse_result(utf8_parse_result::INVALID); - } - if (first == 0xF4 && second > 0x8F) { - return utf8_parse_result(utf8_parse_result::INVALID); - } - } - - // For any further available bytes, just enforce the continuation pattern. - for (size_t i = 2; i < available; ++i) { - unsigned char byte = static_cast(input[offset + i]); - if ((byte & 0xC0) != 0x80) { - return utf8_parse_result(utf8_parse_result::INVALID); - } - } + // 2-byte sequence + if (!(first & 0x20)) { + if (offset + 1 >= input.size()) { + return utf8_parse_result(utf8_parse_result::INCOMPLETE); } - - // If we reach here, the prefix is syntactically and range-wise - // compatible with *some* valid UTF-8 code point; we just ran out of bytes. - return utf8_parse_result(utf8_parse_result::INCOMPLETE); - } - - // We have at least seq_len bytes: validate all continuation bytes. - for (size_t i = 1; i < seq_len; ++i) { - unsigned char byte = static_cast(input[offset + i]); - if ((byte & 0xC0) != 0x80) { + const unsigned char second = static_cast(input[offset + 1]); + if ((second & 0xc0) != 0x80) { return utf8_parse_result(utf8_parse_result::INVALID); } + auto result = ((first & 0x1f) << 6) | (second & 0x3f); + return utf8_parse_result(utf8_parse_result::SUCCESS, result, 2); } - // Decode based on sequence length. - uint32_t codepoint = 0; - - if (seq_len == 2) { - // 110xxxxx 10xxxxxx - codepoint = - ((first & 0x1F) << 6) | - (static_cast(input[offset + 1]) & 0x3F); - - // 0xC0 and 0xC1 were filtered out, so this always yields U+0080–U+07FF. - } else if (seq_len == 3) { - // 1110xxxx 10xxxxxx 10xxxxxx - codepoint = - ((first & 0x0F) << 12) | - ((static_cast(input[offset + 1]) & 0x3F) << 6) | - (static_cast(input[offset + 2]) & 0x3F); - - // Reject overlong encodings: 3-byte must encode U+0800–U+FFFF. - if (codepoint < 0x800) { - return utf8_parse_result(utf8_parse_result::INVALID); + // 3-byte sequence + if (!(first & 0x10)) { + if (offset + 2 >= input.size()) { + return utf8_parse_result(utf8_parse_result::INCOMPLETE); } - - // Reject surrogate code points U+D800–U+DFFF (invalid in UTF-8). - if (codepoint >= 0xD800 && codepoint <= 0xDFFF) { + const unsigned char second = static_cast(input[offset + 1]); + const unsigned char third = static_cast(input[offset + 2]); + if ((second & 0xc0) != 0x80 || (third & 0xc0) != 0x80) { return utf8_parse_result(utf8_parse_result::INVALID); } - } else if (seq_len == 4) { - // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx - codepoint = - ((first & 0x07) << 18) | - ((static_cast(input[offset + 1]) & 0x3F) << 12) | - ((static_cast(input[offset + 2]) & 0x3F) << 6) | - (static_cast(input[offset + 3]) & 0x3F); + auto result = ((first & 0x0f) << 12) | ((second & 0x3f) << 6) | (third & 0x3f); + return utf8_parse_result(utf8_parse_result::SUCCESS, result, 3); + } - // Reject overlong encodings: 4-byte must encode U+10000–U+10FFFF. - if (codepoint < 0x10000) { - return utf8_parse_result(utf8_parse_result::INVALID); + // 4-byte sequence + if (!(first & 0x08)) { + if (offset + 3 >= input.size()) { + return utf8_parse_result(utf8_parse_result::INCOMPLETE); } - - // Reject code points beyond Unicode max (U+10FFFF). - if (codepoint > 0x10FFFF) { + const unsigned char second = static_cast(input[offset + 1]); + const unsigned char third = static_cast(input[offset + 2]); + const unsigned char fourth = static_cast(input[offset + 3]); + if ((second & 0xc0) != 0x80 || (third & 0xc0) != 0x80 || (fourth & 0xc0) != 0x80) { return utf8_parse_result(utf8_parse_result::INVALID); } + auto result = ((first & 0x07) << 18) | ((second & 0x3f) << 12) | ((third & 0x3f) << 6) | (fourth & 0x3f); + return utf8_parse_result(utf8_parse_result::SUCCESS, result, 4); } - return utf8_parse_result(utf8_parse_result::SUCCESS, codepoint, seq_len); + // Invalid first byte + return utf8_parse_result(utf8_parse_result::INVALID); } From 2b1e4de7d11bd67634dcb8734b3a60ffa0dfbe5e Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 00:29:37 -0600 Subject: [PATCH 083/183] simplify even further --- common/unicode.cpp | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/common/unicode.cpp b/common/unicode.cpp index 40df3f7ff76e9..56ab0f468e038 100644 --- a/common/unicode.cpp +++ b/common/unicode.cpp @@ -1,5 +1,7 @@ #include "unicode.h" +// implementation adopted from src/unicode.cpp + size_t utf8_sequence_length(unsigned char first_byte) { const size_t lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 }; uint8_t highbits = static_cast(first_byte) >> 4; @@ -11,57 +13,49 @@ utf8_parse_result parse_utf8_codepoint(std::string_view input, size_t offset) { return utf8_parse_result(utf8_parse_result::INCOMPLETE); } - const unsigned char first = static_cast(input[offset]); - // ASCII fast path - if (!(first & 0x80)) { - return utf8_parse_result(utf8_parse_result::SUCCESS, first, 1); + if (!(input[offset] & 0x80)) { + return utf8_parse_result(utf8_parse_result::SUCCESS, input[offset], 1); } // Invalid: continuation byte as first byte - if (!(first & 0x40)) { + if (!(input[offset] & 0x40)) { return utf8_parse_result(utf8_parse_result::INVALID); } // 2-byte sequence - if (!(first & 0x20)) { + if (!(input[offset] & 0x20)) { if (offset + 1 >= input.size()) { return utf8_parse_result(utf8_parse_result::INCOMPLETE); } - const unsigned char second = static_cast(input[offset + 1]); - if ((second & 0xc0) != 0x80) { + if ((input[offset + 1] & 0xc0) != 0x80) { return utf8_parse_result(utf8_parse_result::INVALID); } - auto result = ((first & 0x1f) << 6) | (second & 0x3f); + auto result = ((input[offset] & 0x1f) << 6) | (input[offset + 1] & 0x3f); return utf8_parse_result(utf8_parse_result::SUCCESS, result, 2); } // 3-byte sequence - if (!(first & 0x10)) { + if (!(input[offset] & 0x10)) { if (offset + 2 >= input.size()) { return utf8_parse_result(utf8_parse_result::INCOMPLETE); } - const unsigned char second = static_cast(input[offset + 1]); - const unsigned char third = static_cast(input[offset + 2]); - if ((second & 0xc0) != 0x80 || (third & 0xc0) != 0x80) { + if ((input[offset + 1] & 0xc0) != 0x80 || (input[offset + 2] & 0xc0) != 0x80) { return utf8_parse_result(utf8_parse_result::INVALID); } - auto result = ((first & 0x0f) << 12) | ((second & 0x3f) << 6) | (third & 0x3f); + auto result = ((input[offset] & 0x0f) << 12) | ((input[offset + 1] & 0x3f) << 6) | (input[offset + 2] & 0x3f); return utf8_parse_result(utf8_parse_result::SUCCESS, result, 3); } // 4-byte sequence - if (!(first & 0x08)) { + if (!(input[offset] & 0x08)) { if (offset + 3 >= input.size()) { return utf8_parse_result(utf8_parse_result::INCOMPLETE); } - const unsigned char second = static_cast(input[offset + 1]); - const unsigned char third = static_cast(input[offset + 2]); - const unsigned char fourth = static_cast(input[offset + 3]); - if ((second & 0xc0) != 0x80 || (third & 0xc0) != 0x80 || (fourth & 0xc0) != 0x80) { + if ((input[offset + 1] & 0xc0) != 0x80 || (input[offset + 2] & 0xc0) != 0x80 || (input[offset + 3] & 0xc0) != 0x80) { return utf8_parse_result(utf8_parse_result::INVALID); } - auto result = ((first & 0x07) << 18) | ((second & 0x3f) << 12) | ((third & 0x3f) << 6) | (fourth & 0x3f); + auto result = ((input[offset] & 0x07) << 18) | ((input[offset + 1] & 0x3f) << 12) | ((input[offset + 2] & 0x3f) << 6) | (input[offset + 3] & 0x3f); return utf8_parse_result(utf8_parse_result::SUCCESS, result, 4); } From 045da9ea84bdd65d426387a58c8e089f4380c419 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 00:34:57 -0600 Subject: [PATCH 084/183] remove unused functions --- common/chat-peg-parser.cpp | 90 ++++++++++++++------------------------ 1 file changed, 34 insertions(+), 56 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 273d06f2b9085..554a8f982ebe0 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -292,60 +292,6 @@ class aho_corasick_matcher { } }; -// Generate an excluding pattern, with customized escaping -static std::string generic_excluding_pattern( - const std::vector & strings, - const std::function & literal, - const std::function & escape_char_class, - bool pad = false) { - - // Use the aho_corasick_matcher to grab an exhaustive list of prefixes and - // potential next characters. We can use this to build an exclusion for - // multiple strings. - aho_corasick_matcher matcher(strings); - auto pieces = matcher.collect_prefix_and_next(); - - std::string pattern; - for (size_t i = 0; i < pieces.size(); ++i) { - if (i > 0) { - pattern += pad ? " | " : "|"; - } - - const auto & pre = pieces[i].prefix; - const auto & chars = pieces[i].next_chars; - - std::string cls; - cls.reserve(chars.size()); - for (const auto & ch : chars) { - cls += escape_char_class(ch); - } - - if (!pre.empty()) { - pattern += literal(pre) + (pad ? " [^" : "[^") + cls + "]"; - } else { - pattern += "[^" + cls + "]"; - } - } - - return "(" + pattern + ")*"; -} - -// Escape a single character for use in regex character classes -static std::string regex_escape_char_class(char c) { - switch (c) { - case '\\': return "\\\\"; - case ']': return "\\]"; - case '-': return "\\-"; - case '^': return "\\^"; - default: return std::string(1, c); - } -} - -// Create a regex excluding pattern -static std::string regex_excluding_pattern(const std::vector & strings) { - return generic_excluding_pattern(strings, regex_escape, regex_escape_char_class); -} - // Container for the root parser and all named rules in the grammar. // Manages ownership of rule registry to enable recursive grammar definitions. class root_parser : public common_chat_peg_parser_base { @@ -1469,13 +1415,45 @@ static std::string gbnf_escape_char_class(char c) { case '\n': return "\\n"; case '\t': return "\\t"; case '\r': return "\\r"; - default: return regex_escape_char_class(c); // these too + case '\\': return "\\\\"; + case ']': return "\\]"; + case '-': return "\\-"; + case '^': return "\\^"; + default: return std::string(1, c); } } // Create a GBNF excluding pattern static std::string gbnf_excluding_pattern(const std::vector & strings) { - return generic_excluding_pattern(strings, gbnf_literal, gbnf_escape_char_class, true); + // Use the aho_corasick_matcher to grab an exhaustive list of prefixes and + // potential next characters. We can use this to build an exclusion for + // multiple strings. + aho_corasick_matcher matcher(strings); + auto pieces = matcher.collect_prefix_and_next(); + + std::string pattern; + for (size_t i = 0; i < pieces.size(); ++i) { + if (i > 0) { + pattern += " | "; + } + + const auto & pre = pieces[i].prefix; + const auto & chars = pieces[i].next_chars; + + std::string cls; + cls.reserve(chars.size()); + for (const auto & ch : chars) { + cls += gbnf_escape_char_class(ch); + } + + if (!pre.empty()) { + pattern += gbnf_literal(pre) + " [^" + cls + "]"; + } else { + pattern += "[^" + cls + "]"; + } + } + + return "(" + pattern + ")*"; } // Visitor for collecting reachable rules from a subtree From 3d7814451963f52138d894fa32a5091d96b7bd10 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 00:36:59 -0600 Subject: [PATCH 085/183] fix type --- common/chat-peg-parser.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 554a8f982ebe0..e40d476df9dfe 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -231,7 +231,7 @@ class aho_corasick_matcher { for (const auto & p : trie[index].children) { unsigned char ch = p.first; - int child = p.second; + auto child = p.second; prefix.push_back(ch); collect_prefix_and_next(child, prefix, out); prefix.pop_back(); From d2b4a4a78c90b79bd646a7a3eab8390af3361785 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 00:50:42 -0600 Subject: [PATCH 086/183] reformat char class parsing --- common/chat-peg-parser.cpp | 151 +++++++++++++++++-------------------- 1 file changed, 69 insertions(+), 82 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index e40d476df9dfe..e2337fef351b9 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -791,6 +791,73 @@ class space_parser : public common_chat_peg_parser_base { void accept(parser_visitor & visitor) override; }; +static std::pair parse_hex_escape(const std::string & str, size_t pos, int hex_count) { + if (pos + hex_count > str.length()) { + return {0, 0}; + } + + uint32_t value = 0; + for (int i = 0; i < hex_count; i++) { + char c = str[pos + i]; + if (!is_hex_digit(c)) { + return {0, 0}; + } + value <<= 4; + if ('a' <= c && c <= 'f') { + value += c - 'a' + 10; + } else if ('A' <= c && c <= 'F') { + value += c - 'A' + 10; + } else if ('0' <= c && c <= '9') { + value += c - '0'; + } else { + break; + } + } + return {value, static_cast(hex_count)}; +} + +static std::pair parse_char_class_char(const std::string & content, size_t pos) { + if (content[pos] == '\\' && pos + 1 < content.length()) { + switch (content[pos + 1]) { + case 'x': { + auto result = parse_hex_escape(content, pos + 2, 2); + if (result.second > 0) { + return {result.first, 2 + result.second}; + } + // Invalid escape, treat as literal 'x' + return {static_cast('x'), 2}; + } + case 'u': { + auto result = parse_hex_escape(content, pos + 2, 4); + if (result.second > 0) { + return {result.first, 2 + result.second}; + } + // Invalid escape, treat as literal 'u' + return {static_cast('u'), 2}; + } + case 'U': { + auto result = parse_hex_escape(content, pos + 2, 8); + if (result.second > 0) { + return {result.first, 2 + result.second}; + } + // Invalid escape, treat as literal 'U' + return {static_cast('U'), 2}; + } + case 'n': return {'\n', 2}; + case 't': return {'\t', 2}; + case 'r': return {'\r', 2}; + case '\\': return {'\\', 2}; + case ']': return {']', 2}; + case '-': return {'-', 2}; + case '[': return {'[', 2}; + default: return {static_cast(content[pos + 1]), 2}; + } + } + + // Regular character - return as codepoint + return {static_cast(static_cast(content[pos])), 1}; +} + // Matches between min and max repetitions of characters from a character class. // S -> [a-z]{m,n} // Supports Unicode codepoint ranges and escape sequences: \xXX \uXXXX \UXXXXXXXX @@ -827,94 +894,14 @@ class chars_parser : public common_chat_peg_parser_base { content = content.substr(1); } - // Parse a character or escape sequence, returning codepoint and bytes consumed - auto parse_char = [&](size_t pos) -> std::pair { - if (content[pos] == '\\' && pos + 1 < content.length()) { - char next = content[pos + 1]; - switch (next) { - case 'n': return {'\n', 2}; - case 't': return {'\t', 2}; - case 'r': return {'\r', 2}; - case '\\': return {'\\', 2}; - case ']': return {']', 2}; - case '-': return {'-', 2}; - case '[': return {'[', 2}; - - // \xXX - 8-bit hex escape - case 'x': { - if (pos + 3 < content.length() && - is_hex_digit(content[pos + 2]) && - is_hex_digit(content[pos + 3])) { - uint32_t value = 0; - for (int i = 0; i < 2; i++) { - char c = content[pos + 2 + i]; - value = value * 16 + (c >= 'a' ? c - 'a' + 10 : - c >= 'A' ? c - 'A' + 10 : - c - '0'); - } - return {value, 4}; // \xXX - } - return {next, 2}; // Invalid escape, treat as literal 'x' - } - - // \uXXXX - 16-bit hex escape - case 'u': { - if (pos + 5 < content.length() && - is_hex_digit(content[pos + 2]) && - is_hex_digit(content[pos + 3]) && - is_hex_digit(content[pos + 4]) && - is_hex_digit(content[pos + 5])) { - uint32_t value = 0; - for (int i = 0; i < 4; i++) { - char c = content[pos + 2 + i]; - value = value * 16 + (c >= 'a' ? c - 'a' + 10 : - c >= 'A' ? c - 'A' + 10 : - c - '0'); - } - return {value, 6}; // \uXXXX - } - return {next, 2}; // Invalid escape, treat as literal 'u' - } - - // \UXXXXXXXX - 32-bit hex escape - case 'U': { - if (pos + 9 < content.length()) { - bool all_hex = true; - for (int i = 0; i < 8; i++) { - if (!is_hex_digit(content[pos + 2 + i])) { - all_hex = false; - break; - } - } - if (all_hex) { - uint32_t value = 0; - for (int i = 0; i < 8; i++) { - char c = content[pos + 2 + i]; - value = value * 16 + (c >= 'a' ? c - 'a' + 10 : - c >= 'A' ? c - 'A' + 10 : - c - '0'); - } - return {value, 10}; // \UXXXXXXXX - } - } - return {next, 2}; // Invalid escape, treat as literal 'U' - } - - default: return {next, 2}; // Treat as literal escaped character - } - } - // Regular character - return as codepoint - return {static_cast(static_cast(content[pos])), 1}; - }; - size_t i = 0; while (i < content.length()) { - auto [start, start_len] = parse_char(i); + auto [start, start_len] = parse_char_class_char(content, i); i += start_len; if (i + 1 < content.length() && content[i] == '-') { // Range detected - auto [end, end_len] = parse_char(i + 1); + auto [end, end_len] = parse_char_class_char(content, i + 1); ranges_.push_back(char_range{start, end}); i += 1 + end_len; } else { From 3da306bf208a04e255a000a6e7b05904c3b0a2f3 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 01:08:49 -0600 Subject: [PATCH 087/183] clean up json string parser --- common/chat-peg-parser.cpp | 98 +++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index e2337fef351b9..84e53c343eb1f 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -1000,7 +1000,6 @@ class chars_parser : public common_chat_peg_parser_base { // Handles escape sequences and emits NEED_MORE_INPUT for incomplete input. // S -> (regular chars and escape sequences)* until closing " class json_string_parser : public common_chat_peg_parser_base { - public: static constexpr parser_type type_value = JSON_STRING; @@ -1021,58 +1020,14 @@ class json_string_parser : public common_chat_peg_parser_base { } if (c == '\\') { - // Handle escape sequence - ++pos; - if (pos >= ctx.input.size()) { - // Mid-escape sequence - if (ctx.input_is_complete) { - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); - } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start, pos); - } - - char escape = ctx.input[pos]; - switch (escape) { - case '"': - case '\\': - case '/': - case 'b': - case 'f': - case 'n': - case 'r': - case 't': - // Valid escape - ++pos; - break; - - case 'u': - // Unicode escape: must be followed by 4 hex digits - ++pos; - for (int i = 0; i < 4; ++i) { - if (pos >= ctx.input.size()) { - // Incomplete unicode escape - if (ctx.input_is_complete) { - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); - } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start, pos); - } - if (!is_hex_digit(ctx.input[pos])) { - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); - } - ++pos; - } - break; - - default: - // Invalid escape sequence - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); + auto result = handle_escape_sequence(ctx, start, pos); + if (!result.success()) { + return result; } } else { - // Regular character - validate UTF-8 auto utf8_result = parse_utf8_codepoint(ctx.input, pos); if (utf8_result.status == utf8_parse_result::INCOMPLETE) { - // Incomplete UTF-8 sequence if (ctx.input_is_complete) { return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); } @@ -1080,7 +1035,6 @@ class json_string_parser : public common_chat_peg_parser_base { } if (utf8_result.status == utf8_parse_result::INVALID) { - // Malformed UTF-8 in JSON string return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); } @@ -1100,6 +1054,52 @@ class json_string_parser : public common_chat_peg_parser_base { } void accept(parser_visitor & visitor) override; + + private: + static common_chat_parse_result handle_escape_sequence(common_chat_parse_context & ctx, size_t start, size_t & pos) { + ++pos; // consume '\' + if (pos >= ctx.input.size()) { + if (ctx.input_is_complete) { + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); + } + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start, pos); + } + + switch (ctx.input[pos]) { + case '"': + case '\\': + case '/': + case 'b': + case 'f': + case 'n': + case 'r': + case 't': + ++pos; + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); + case 'u': + return handle_unicode_escape(ctx, start, pos); + default: + // Invalid escape sequence + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); + } + } + + static common_chat_parse_result handle_unicode_escape(common_chat_parse_context & ctx, size_t start, size_t & pos) { + ++pos; // consume 'u' + for (int i = 0; i < 4; ++i) { + if (pos >= ctx.input.size()) { + if (ctx.input_is_complete) { + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); + } + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start, pos); + } + if (!is_hex_digit(ctx.input[pos])) { + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); + } + ++pos; + } + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); + } }; // Matches all characters until a delimiter is found (delimiter not consumed). From 26c955317539e63a1fa02884dd03dd051c6a0bc1 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 01:14:24 -0600 Subject: [PATCH 088/183] clean up + fix diagnostics --- common/chat-peg-parser.cpp | 61 +++++++++++++------------------------- 1 file changed, 20 insertions(+), 41 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 84e53c343eb1f..0eb849cec1df4 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -38,10 +38,10 @@ enum parser_type { const char * common_chat_parse_result_type_name(common_chat_parse_result_type type) { switch (type) { - case COMMON_CHAT_PARSE_RESULT_FAIL: return "fail"; - case COMMON_CHAT_PARSE_RESULT_SUCCESS: return "success"; - case COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT: return "need_more_input"; - default: return "unknown"; + case COMMON_CHAT_PARSE_RESULT_FAIL: return "fail"; + case COMMON_CHAT_PARSE_RESULT_SUCCESS: return "success"; + case COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT: return "need_more_input"; + default: return "unknown"; } } @@ -60,20 +60,17 @@ class common_chat_peg_parser_base { virtual parser_type type() const = 0; - // Template Method: handles caching, delegates to parse_uncached() virtual common_chat_parse_result parse(common_chat_parse_context & ctx, size_t start = 0) { if (id_ == -1) { // Don't cache parsers with ID -1 (from operators) return parse_uncached(ctx, start); } - // Check cache auto cached = ctx.cache.get(id_, start); if (cached) { return *cached; } - // Execute and cache auto result = parse_uncached(ctx, start); return ctx.cache.set(id_, start, result); } @@ -126,24 +123,6 @@ static bool is_hex_digit(const char c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); } -// Unescapes a JSON string (without the surrounding quotes) -// Uses nlohmann::json::parse to handle all JSON escape sequences -static std::string unescape_json_string(std::string_view str) { - try { - // Wrap in quotes and parse as JSON string - std::string quoted = "\"" + std::string(str) + "\""; - auto parsed = nlohmann::json::parse(quoted); - if (parsed.is_string()) { - return parsed.get(); - } - // If not a string, return literally - return std::string(str); - } catch (...) { - // If parsing fails, return original string - return std::string(str); - } -} - // Aho-Corasick automation for matching multiple literals. // This is used in until_parser and to build a GBNF exclusion grammar by // exploiting its trie structure. @@ -1454,14 +1433,14 @@ class reachability_visitor : public parser_visitor { const std::unordered_map & rules ) : reachable_rules_(reachable_rules), rules_(rules) {} - void visit(start_parser &) override {} - void visit(end_parser &) override {} - void visit(literal_parser &) override {} - void visit(any_parser &) override {} - void visit(space_parser &) override {} - void visit(json_string_parser &) override {} - void visit(chars_parser &) override {} - void visit(until_parser &) override {} + void visit(start_parser & /* p */) override {} + void visit(end_parser & /* p */) override {} + void visit(literal_parser & /* p */) override {} + void visit(any_parser & /* p */) override {} + void visit(space_parser & /* p */) override {} + void visit(json_string_parser & /* p */) override {} + void visit(chars_parser & /* p */) override {} + void visit(until_parser & /* p */) override {} void visit(and_parser & p) override { p.child()->accept(*this); } void visit(not_parser & p) override { p.child()->accept(*this); } @@ -1481,7 +1460,7 @@ class reachability_visitor : public parser_visitor { void visit(zero_or_more_parser & p) override { p.child()->accept(*this); } void visit(optional_parser & p) override { p.child()->accept(*this); } void visit(repetition_parser & p) override { p.child()->accept(*this); } - void visit(schema_parser &) override { + void visit(schema_parser & /* p */) override { // Schema parsers are opaque - don't traverse their children // The schema system will handle rule generation via builder_.add_schema() } @@ -1543,11 +1522,11 @@ class gbnf_visitor : public parser_visitor { } public: - void visit(start_parser &) override { + void visit(start_parser & /* p */) override { current_result_ = ""; } - void visit(end_parser &) override { + void visit(end_parser & /* p */) override { current_result_ = ""; } @@ -1642,22 +1621,22 @@ class gbnf_visitor : public parser_visitor { current_result_ = gbnf_excluding_pattern(p.delimiters()); } - void visit(and_parser &) override { + void visit(and_parser & /* p */) override { current_result_ = ""; } - void visit(not_parser &) override { + void visit(not_parser & /* p */) override { // NOT is tricky in GBNF - for now, emit error LOG_ERR("NOT operator not directly supported in GBNF generation\n"); current_result_ = ""; } - void visit(any_parser &) override { + void visit(any_parser & /* p */) override { // Match any single character current_result_ = "."; } - void visit(space_parser &) override { + void visit(space_parser & /* p */) override { // Reference the built-in space rule current_result_ = "space"; } @@ -1688,7 +1667,7 @@ class gbnf_visitor : public parser_visitor { } } - void visit(json_string_parser &) override { + void visit(json_string_parser & /* p */) override { // JSON string content (without quotes) // Pattern: (any non-quote/backslash OR escape sequences)* until closing quote current_result_ = R"(( [^"\\] | "\\" ( ["\\/ bfnrt] | "u" [0-9a-fA-F]{4} ) )*)"; From 175cb57f10b9cac65bd2a540b3e993755d594152 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 01:14:53 -0600 Subject: [PATCH 089/183] reorder includes --- common/chat-peg-parser.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 0eb849cec1df4..559fdb434e693 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -1,13 +1,13 @@ +#include "log.h" +#include "common.h" #include "chat-peg-parser.h" #include "json-schema-to-grammar.h" -#include "common.h" -#include "log.h" #include "unicode.h" -#include #include #include +#include #include #include #include From f5af89a808cda5320c1d58a45d3ad77a8aaef58b Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 01:20:17 -0600 Subject: [PATCH 090/183] compact builder functions --- common/chat-peg-parser.cpp | 116 ++++++++----------------------------- 1 file changed, 24 insertions(+), 92 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 559fdb434e693..0bfb3103e331f 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -1819,11 +1819,8 @@ void common_chat_parse_cache::clear() { } common_chat_peg_parser::common_chat_peg_parser() {} - common_chat_peg_parser::common_chat_peg_parser(std::shared_ptr parser) : ptr_(std::move(parser)) {} - common_chat_peg_parser::common_chat_peg_parser(const std::string & literal) : ptr_(make_parser(-1, literal)) {} - common_chat_peg_parser::common_chat_peg_parser(const char * literal) : ptr_(make_parser(-1, literal)) {} common_chat_peg_parser common_chat_peg_parser::operator~() const { @@ -1851,21 +1848,14 @@ common_chat_peg_parser operator+(const std::string & lhs, const common_chat_peg_ common_chat_peg_parser operator|(const std::string & lhs, const common_chat_peg_parser & rhs) { return common_chat_peg_parser(lhs) | rhs; } common_chat_peg_parser operator<<(const std::string & lhs, const common_chat_peg_parser & rhs) { return common_chat_peg_parser(lhs) << rhs; } -common_chat_peg_parser_base & common_chat_peg_parser::operator*() const { - return *ptr_; -} - -common_chat_peg_parser_base * common_chat_peg_parser::operator->() const { - return ptr_.get(); -} +common_chat_peg_parser_base & common_chat_peg_parser::operator*() const { return *ptr_; } +common_chat_peg_parser_base * common_chat_peg_parser::operator->() const { return ptr_.get(); } common_chat_parse_result common_chat_peg_parser::parse(common_chat_parse_context & ctx, size_t start) const { return ptr_->parse(ctx, start); } -std::string common_chat_peg_parser::dump() const { - return ptr_->dump(); -} +std::string common_chat_peg_parser::dump() const { return ptr_->dump(); } void common_chat_peg_parser::build_grammar(const common_grammar_builder & builder, bool lazy) const { gbnf_visitor visitor(builder, lazy); @@ -1878,91 +1868,33 @@ void common_chat_peg_parser::build_grammar(const common_grammar_builder & builde using builder = common_chat_peg_parser_builder; -builder::common_chat_peg_parser_builder() - : root_(make_parser(0)) // root parser has id 0 - , counter_(1) {} - -common_chat_peg_parser builder::start() { - return make_parser(counter_); -} - -common_chat_peg_parser builder::end() { - return make_parser(counter_); -} - -common_chat_peg_parser builder::literal(const std::string & literal) { - return make_parser(counter_, literal); -} - -common_chat_peg_parser builder::sequence(const std::vector & parsers) { - return make_parser(counter_, parsers); -} - -common_chat_peg_parser builder::choice(const std::vector & parsers) { - return make_parser(counter_, parsers); -} - -common_chat_peg_parser builder::one_or_more(const common_chat_peg_parser & p) { - return make_parser(counter_, p); -} - -common_chat_peg_parser builder::zero_or_more(const common_chat_peg_parser & p) { - return make_parser(counter_, p); -} - -common_chat_peg_parser builder::optional(const common_chat_peg_parser & p) { - return make_parser(counter_, p); -} - -common_chat_peg_parser builder::peek(const common_chat_peg_parser & p) { - return make_parser(counter_, p); -} - -common_chat_peg_parser builder::negate(const common_chat_peg_parser & p) { - return make_parser(counter_, p); -} - -common_chat_peg_parser builder::any() { - return make_parser(counter_); -} - -common_chat_peg_parser builder::chars(const std::string & classes, int min, int max) { - return make_parser(counter_, classes, min, max); -} - -common_chat_peg_parser builder::one(const std::string & classes) { - return make_parser(counter_, classes, 1, 1); -} - -common_chat_peg_parser builder::json_string_unqouted() { - return make_parser(counter_); -} +builder::common_chat_peg_parser_builder() : root_(make_parser(0)) , counter_(1) {} + +common_chat_peg_parser builder::start() { return make_parser(counter_); } +common_chat_peg_parser builder::end() { return make_parser(counter_); } +common_chat_peg_parser builder::literal(const std::string & literal) { return make_parser(counter_, literal); } +common_chat_peg_parser builder::sequence(const std::vector & parsers) { return make_parser(counter_, parsers); } +common_chat_peg_parser builder::choice(const std::vector & parsers) { return make_parser(counter_, parsers); } +common_chat_peg_parser builder::one_or_more(const common_chat_peg_parser & p) { return make_parser(counter_, p); } +common_chat_peg_parser builder::zero_or_more(const common_chat_peg_parser & p) { return make_parser(counter_, p); } +common_chat_peg_parser builder::optional(const common_chat_peg_parser & p) { return make_parser(counter_, p); } +common_chat_peg_parser builder::peek(const common_chat_peg_parser & p) { return make_parser(counter_, p); } +common_chat_peg_parser builder::negate(const common_chat_peg_parser & p) { return make_parser(counter_, p); } +common_chat_peg_parser builder::any() { return make_parser(counter_); } +common_chat_peg_parser builder::chars(const std::string & classes, int min, int max) { return make_parser(counter_, classes, min, max); } +common_chat_peg_parser builder::one(const std::string & classes) { return make_parser(counter_, classes, 1, 1); } +common_chat_peg_parser builder::json_string_unqouted() { return make_parser(counter_); } +common_chat_peg_parser builder::space() { return make_parser(counter_); } +common_chat_peg_parser builder::until(const std::string & delimiter) { return make_parser(counter_, delimiter); } +common_chat_peg_parser builder::until_one_of(const std::vector & delimiters) { return make_parser(counter_, delimiters); } +common_chat_peg_parser builder::repeat(const common_chat_peg_parser & p, int min, int max) { return make_parser(counter_, p, min, max); } +common_chat_peg_parser builder::repeat(const common_chat_peg_parser & p, int n) { return make_parser(counter_, p, n, n); } common_chat_peg_parser builder::rule(const std::string & name) { auto root = cast(root_); return make_parser(counter_, name, std::weak_ptr(root)); } -common_chat_peg_parser builder::space() { - return make_parser(counter_); -} - -common_chat_peg_parser builder::until(const std::string & delimiter) { - return make_parser(counter_, delimiter); -} - -common_chat_peg_parser builder::until_one_of(const std::vector & delimiters) { - return make_parser(counter_, delimiters); -} - -common_chat_peg_parser builder::repeat(const common_chat_peg_parser & p, int min, int max) { - return make_parser(counter_, p, min, max); -} - -common_chat_peg_parser builder::repeat(const common_chat_peg_parser & p, int n) { - return make_parser(counter_, p, n, n); -} - common_chat_peg_parser builder::schema(const common_chat_peg_parser & p, const std::string & name, const nlohmann::ordered_json & schema) { return make_parser(counter_, p, name, schema); } From 9199b00af2540cdb2edff0b377e593e3f153657e Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 01:46:03 -0600 Subject: [PATCH 091/183] replace action_parser with capture_parser, rename env to semantics --- common/chat-peg-parser.cpp | 48 ++++------ common/chat-peg-parser.h | 25 +++-- tests/CMakeLists.txt | 1 - tests/chat-peg-parser/test-actions.cpp | 95 ------------------- .../test-command7-parser-compare.cpp | 24 ++--- .../test-example-qwen3-coder.cpp | 31 +++--- tests/chat-peg-parser/tests.h | 1 - tests/test-chat-peg-parser.cpp | 1 - 8 files changed, 56 insertions(+), 170 deletions(-) delete mode 100644 tests/chat-peg-parser/test-actions.cpp diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 0bfb3103e331f..c41f0716b7b91 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -32,7 +32,7 @@ enum parser_type { SCHEMA, ROOT, JSON_STRING, - ACTION, + CAPTURE, TRIGGER, }; @@ -1251,35 +1251,27 @@ class rule_parser : public common_chat_peg_parser_base { const std::string & name() const { return name_; } }; -// Wraps a parser with a semantic action callback. -class action_parser : public common_chat_peg_parser_base { +// Capture content if child parser matches +class capture_parser : public common_chat_peg_parser_base { common_chat_peg_parser parser_; - std::function action_; - int when_; + std::string key_; public: - static constexpr parser_type type_value = ACTION; + static constexpr parser_type type_value = CAPTURE; - action_parser( - const common_chat_peg_parser & parser, - std::function action, - int when, - int id - ) : common_chat_peg_parser_base(id), parser_(parser), action_(std::move(action)), when_(when) {} + capture_parser(const common_chat_peg_parser & parser, const std::string & key, int id) + : common_chat_peg_parser_base(id), parser_(parser), key_(key) {} parser_type type() const override { return type_value; } common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto result = parser_->parse(ctx, start); - if ((result.type & when_) && ctx.env && action_) { + if (!result.fail() && ctx.env) { std::string_view matched = ctx.input; matched = matched.substr(result.start, result.end - result.start); - action_({ - result, - *ctx.env, - matched, - }); + std::string value = std::string(matched); + ctx.env->captures[key_] = std::move(value); } return result; @@ -1291,7 +1283,7 @@ class action_parser : public common_chat_peg_parser_base { } std::string dump() const override { - return "Action(" + parser_->dump() + ", when=" + std::to_string(when_) +")"; + return "Capture(" + key_ + ", " + parser_->dump() + ")"; } void accept(parser_visitor & visitor) override; @@ -1355,7 +1347,7 @@ class parser_visitor { virtual void visit(schema_parser & p) = 0; virtual void visit(rule_parser & p) = 0; virtual void visit(root_parser & p) = 0; - virtual void visit(action_parser & p) = 0; + virtual void visit(capture_parser & p) = 0; virtual void visit(trigger_parser & p) = 0; }; @@ -1464,7 +1456,7 @@ class reachability_visitor : public parser_visitor { // Schema parsers are opaque - don't traverse their children // The schema system will handle rule generation via builder_.add_schema() } - void visit(action_parser & p) override { p.child()->accept(*this); } + void visit(capture_parser & p) override { p.child()->accept(*this); } void visit(trigger_parser & p) override { p.child()->accept(*this); } void visit(rule_parser & p) override { @@ -1738,8 +1730,7 @@ class gbnf_visitor : public parser_visitor { current_result_ = string_join(trigger_names_, " | "); } - void visit(action_parser & p) override { - // Actions are transparent for grammar generation - just visit child + void visit(capture_parser & p) override { p.child()->accept(*this); } @@ -1790,7 +1781,7 @@ void json_string_parser::accept(parser_visitor & visitor) { visitor.visit(*this) void schema_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void rule_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void root_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } -void action_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } +void capture_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void trigger_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } common_chat_parse_result common_chat_parse_cache::set(int id, size_t start, common_chat_parse_result result) { @@ -1899,15 +1890,8 @@ common_chat_peg_parser builder::schema(const common_chat_peg_parser & p, const s return make_parser(counter_, p, name, schema); } -common_chat_peg_parser builder::action(const common_chat_peg_parser & p, std::function fn, int when) { - return make_parser(counter_, p, std::move(fn), when); -} - common_chat_peg_parser builder::capture(const std::string & key, const common_chat_peg_parser & p) { - return action(p, [key](const common_chat_parse_action & act) { - std::string value = std::string(act.match); - act.env.captures[key] = std::move(value); - }, COMMON_CHAT_PARSE_RESULT_SUCCESS); + return make_parser(counter_, p, key); } common_chat_peg_parser builder::trigger(const common_chat_peg_parser & p) { diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index e59f6ddbbae3d..d18c93c1be35c 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -14,9 +14,19 @@ struct common_grammar_builder; struct common_chat_parse_semantics { - common_chat_msg result; + std::string content; + std::string reasoning_content; + std::vector tool_calls; std::unordered_map captures; + + common_chat_msg to_msg() const { + common_chat_msg msg; + msg.content = content; + msg.reasoning_content = reasoning_content; + msg.tool_calls = tool_calls; + return msg; + } }; enum common_chat_parse_result_type { @@ -61,12 +71,6 @@ struct common_chat_parse_result { bool success() const { return type == COMMON_CHAT_PARSE_RESULT_SUCCESS; } }; -struct common_chat_parse_action { - common_chat_parse_result & result; - common_chat_parse_semantics & env; - std::string_view match; -}; - enum common_chat_parse_event_type { COMMON_CHAT_PARSE_EVENT_NODE_START, COMMON_CHAT_PARSE_EVENT_NODE_END, @@ -286,12 +290,7 @@ class common_chat_peg_parser_builder { // Used internally to convert JSON schemas to GBNF grammar rules. common_chat_peg_parser schema(const common_chat_peg_parser & p, const std::string & name, const nlohmann::ordered_json & schema); - // Wraps a parser with a semantic action callback. - // The callback is invoked on successful parse with the result, matched text, and environment. - // S -> A [action] - common_chat_peg_parser action(const common_chat_peg_parser & p, std::function fn, int when = COMMON_CHAT_PARSE_RESULT_SUCCESS); - - // Captures matched text to env.captures[key] + // Captures matched text to semantics.captures[key] common_chat_peg_parser capture(const std::string & key, const common_chat_peg_parser & p); // Mark a node as a trigger for GBNF grammar generartion. This is used for diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d73e0e145e437..7537b12f5800d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -187,7 +187,6 @@ llama_build_and_test(test-chat-parser.cpp) llama_build_and_test( test-chat-peg-parser.cpp chat-peg-parser/simple_tokenizer.cpp - chat-peg-parser/test-actions.cpp chat-peg-parser/test-command7-parser-compare.cpp chat-peg-parser/test-example-qwen3-coder.cpp chat-peg-parser/test-gbnf-generation.cpp diff --git a/tests/chat-peg-parser/test-actions.cpp b/tests/chat-peg-parser/test-actions.cpp deleted file mode 100644 index feae9c41f93e3..0000000000000 --- a/tests/chat-peg-parser/test-actions.cpp +++ /dev/null @@ -1,95 +0,0 @@ -#include "tests.h" - -void test_actions(testing &t) { - // Test simple action - append matched text to content - t.test("simple action - append matched text to content", [](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { - auto word = p.chars("[a-z]+"); - return p.action(word, - [](const common_chat_parse_action & act) { act.env.result.content += std::string(act.match); }); - }); - - common_chat_parse_semantics env; - common_chat_parse_context ctx("hello", &env); - auto result = parser.parse(ctx); - - t.assert_equal("result_is_success", true, result.success()); - t.assert_equal("result_is_hello", std::string("hello"), env.result.content); - }); - - // Test multiple sequential actions - build a sentence - t.test("multiple sequential actions - build a sentence", [](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { - auto greeting = p.action(p.literal("hello"), [](const common_chat_parse_action & act) { - act.env.result.content += std::string(act.match) + " "; - }); - - auto name = p.action(p.chars("[A-Z][a-z]+"), [](const common_chat_parse_action & act) { - act.env.result.content += std::string(act.match); - act.env.captures["name"] = std::string(act.match); - }); - - return greeting + p.literal(" ") + name; - }); - - common_chat_parse_semantics env; - common_chat_parse_context ctx("hello Alice", &env); - auto result = parser.parse(ctx); - - t.assert_equal("result_is_success", true, result.success()); - t.assert_equal("result_content", std::string("hello Alice"), env.result.content); - t.assert_equal("captured_name", std::string("Alice"), env.captures["name"]); - }); - - // Test actions don't run when parse fails - t.test("actions don't run when parse fails", [](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { - return p.action(p.literal("success"), - [](const common_chat_parse_action & act) { act.env.result.content = "action_ran"; }); - }); - - common_chat_parse_semantics env; - common_chat_parse_context ctx("failure", &env); - auto result = parser.parse(ctx); - - t.assert_equal("result_is_fail", true, result.fail()); - t.assert_equal("result_content_empty", std::string(""), env.result.content); // Action should not have run - }); - - // Test Actions work with partial parsing - t.test("actions work with need_more_input parsing", [](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { - auto content = p.action(p.until(""), [](const common_chat_parse_action & act) { - act.env.result.content += std::string(act.match); - }); - return "" << content << ""; - }); - - { - common_chat_parse_semantics env; - common_chat_parse_context ctx("hello ", &env, false); - auto result = parser.parse(ctx); - - t.assert_equal("result_is_need_more_input_1", true, result.need_more_input()); - t.assert_equal("result_content_1", std::string("hello "), env.result.content); - } - - { - common_chat_parse_semantics env; - common_chat_parse_context ctx("hello world", &env, false); - auto result = parser.parse(ctx); - - t.assert_equal("result_is_need_more_input_2", true, result.need_more_input()); - t.assert_equal("result_content_2", std::string("hello world"), env.result.content); - } - - { - common_chat_parse_semantics env; - common_chat_parse_context ctx("hello world", &env, true); - auto result = parser.parse(ctx); - - t.assert_equal("result_is_success", true, result.success()); - t.assert_equal("result_content_final", std::string("hello world"), env.result.content); - } - }); -} diff --git a/tests/chat-peg-parser/test-command7-parser-compare.cpp b/tests/chat-peg-parser/test-command7-parser-compare.cpp index ac3f313b55aee..8f2a84e18eb82 100644 --- a/tests/chat-peg-parser/test-command7-parser-compare.cpp +++ b/tests/chat-peg-parser/test-command7-parser-compare.cpp @@ -45,31 +45,31 @@ static common_chat_peg_parser create_command_r7b_parser() { } static common_chat_parse_event_handler create_command_r7b_event_handler() { - return [](const common_chat_parse_event & ev, common_chat_parse_semantics & env) { + return [](const common_chat_parse_event & ev, common_chat_parse_semantics & semantics) { if (ev.rule == "reasoning-content" && ev.ending()) { - env.result.reasoning_content = ev.text; + semantics.reasoning_content = ev.text; } if (ev.rule == "content" && ev.ending()) { - env.result.content = ev.text; + semantics.content = ev.text; } if (ev.rule == "tool-call" && ev.starting()) { - env.result.tool_calls.emplace_back(); + semantics.tool_calls.emplace_back(); } if (ev.rule == "tool-call-id-value" && ev.ending() && ev.success()) { - auto & tc = env.result.tool_calls.back(); + auto & tc = semantics.tool_calls.back(); tc.id = nlohmann::json::parse(ev.text).get(); } if (ev.rule == "tool-name-value" && ev.ending() && ev.success()) { - auto & tc = env.result.tool_calls.back(); + auto & tc = semantics.tool_calls.back(); tc.name = nlohmann::json::parse(ev.text).get(); } if (ev.rule == "tool-args-value" && ev.ending() && (ev.success() || ev.need_more_input())) { - auto & tc = env.result.tool_calls.back(); + auto & tc = semantics.tool_calls.back(); tc.arguments = ev.text; } }; @@ -79,18 +79,18 @@ static void test_command_r7b_parser(const common_chat_peg_parser & p, const std::string & input, bool need_more_input, bool print_results) { - common_chat_parse_semantics env; - common_chat_parse_context ctx(input, &env, !need_more_input); + common_chat_parse_semantics semantics; + common_chat_parse_context ctx(input, &semantics, !need_more_input); p.parse(ctx); if (print_results) { std::cout << "== Parsed (new) ==\n"; std::cout << "=== Reasoning ===\n"; - std::cout << env.result.reasoning_content << "\n"; + std::cout << semantics.reasoning_content << "\n"; std::cout << "\n\n=== Content ===\n"; - std::cout << env.result.content << "\n"; + std::cout << semantics.content << "\n"; std::cout << "\n\n=== Tool Calls ===\n"; - for (const auto & tc : env.result.tool_calls) { + for (const auto & tc : semantics.tool_calls) { std::cout << "id: " << tc.id << "\n"; std::cout << "name: " << tc.name << "\n"; std::cout << "args: " << tc.arguments << "\n"; diff --git a/tests/chat-peg-parser/test-example-qwen3-coder.cpp b/tests/chat-peg-parser/test-example-qwen3-coder.cpp index 29e8cd6a67883..9d5cfa7583f0d 100644 --- a/tests/chat-peg-parser/test-example-qwen3-coder.cpp +++ b/tests/chat-peg-parser/test-example-qwen3-coder.cpp @@ -18,24 +18,24 @@ void test_example_qwen3_coder(testing &t) { return thinking + p.optional(p.space() + content) + p.zero_or_more(p.space() + tool_call); }); - auto handler = [&](const common_chat_parse_event & ev, common_chat_parse_semantics & env) { + auto handler = [&](const common_chat_parse_event & ev, common_chat_parse_semantics & semantics) { if (ev.rule == "reasoning-content" && ev.ending()) { - env.result.reasoning_content = ev.text; + semantics.reasoning_content = ev.text; } if (ev.rule == "content" && ev.ending()) { - env.result.content = ev.text; + semantics.content = ev.text; } if (ev.rule == "function-start" && ev.ending() && ev.success()) { - env.result.tool_calls.emplace_back(); - auto & tc = env.result.tool_calls.back(); - tc.name = env.captures["tool-name"]; + semantics.tool_calls.emplace_back(); + auto & tc = semantics.tool_calls.back(); + tc.name = semantics.captures["tool-name"]; } if (ev.rule == "arg-start" && ev.ending() && ev.success()) { - auto & tc = env.result.tool_calls.back(); - auto name = env.captures["arg-name"]; + auto & tc = semantics.tool_calls.back(); + auto name = semantics.captures["arg-name"]; if (tc.arguments.empty()) { tc.arguments += "{"; } else { @@ -45,17 +45,17 @@ void test_example_qwen3_coder(testing &t) { } if (ev.rule == "arg-string-content" && ev.ending() && ev.success()) { - auto & tc = env.result.tool_calls.back(); + auto & tc = semantics.tool_calls.back(); tc.arguments += "\"" + std::string(ev.text); } if (ev.rule == "arg-string" && ev.ending() && ev.success()) { - auto & tc = env.result.tool_calls.back(); + auto & tc = semantics.tool_calls.back(); tc.arguments += "\""; } if (ev.rule == "arg-json-content" && ev.ending() && (ev.success() || ev.need_more_input())) { - auto & tc = env.result.tool_calls.back(); + auto & tc = semantics.tool_calls.back(); tc.arguments += std::string(ev.text); } }; @@ -90,8 +90,8 @@ void test_example_qwen3_coder(testing &t) { token_cnt++; std::string in = std::accumulate(tokens.begin(), it, std::string()); - common_chat_parse_semantics env; - common_chat_parse_context ctx(in, &env, it == tokens.end() - 1); + common_chat_parse_semantics semantics; + common_chat_parse_context ctx(in, &semantics, it == tokens.end() - 1); ctx.event_handler = handler; @@ -99,8 +99,9 @@ void test_example_qwen3_coder(testing &t) { t.assert_equal(std::string("should_not_fail_token_") + std::to_string(token_cnt), false, result.fail()); // This shouldn't emit any runtime errors - auto diffs = common_chat_msg_diff::compute_diffs(prev, env.result); - prev = env.result; + auto msg = semantics.to_msg(); + auto diffs = common_chat_msg_diff::compute_diffs(prev, msg); + prev = msg; } }); } diff --git a/tests/chat-peg-parser/tests.h b/tests/chat-peg-parser/tests.h index 8ef5a28a6ba81..5fed1b03a15c2 100644 --- a/tests/chat-peg-parser/tests.h +++ b/tests/chat-peg-parser/tests.h @@ -22,7 +22,6 @@ void test_one(testing &t); void test_optional(testing &t); void test_recursive_references(testing &t); void test_json_parser(testing &t); -void test_actions(testing &t); void test_gbnf_generation(testing &t); void test_example_qwen3_coder(testing &t); void test_command7_parser_compare(testing &t); diff --git a/tests/test-chat-peg-parser.cpp b/tests/test-chat-peg-parser.cpp index 895da9e4b692f..1595031827142 100644 --- a/tests/test-chat-peg-parser.cpp +++ b/tests/test-chat-peg-parser.cpp @@ -9,7 +9,6 @@ int main() { test_unicode(t); test_recursive_references(t); test_json_parser(t); - test_actions(t); test_gbnf_generation(t); test_example_qwen3_coder(t); test_command7_parser_compare(t); From 0c162a0ab895de1b70c7c759d9ed39497230670b Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 01:48:47 -0600 Subject: [PATCH 092/183] rename env to semantics --- common/chat-peg-parser.cpp | 12 ++++++------ common/chat-peg-parser.h | 18 +++++++++--------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index c41f0716b7b91..11b8143e37843 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -1203,7 +1203,7 @@ class rule_parser : public common_chat_peg_parser_base { } // Fire NODE_START event - if (ctx.event_handler && ctx.env) { + if (ctx.event_handler && ctx.semantics) { ctx.event_handler(common_chat_parse_event{ COMMON_CHAT_PARSE_EVENT_NODE_START, name_, @@ -1212,7 +1212,7 @@ class rule_parser : public common_chat_peg_parser_base { "", COMMON_CHAT_PARSE_RESULT_FAIL, ctx.current_depth - }, *ctx.env); + }, *ctx.semantics); ctx.current_depth++; } @@ -1220,7 +1220,7 @@ class rule_parser : public common_chat_peg_parser_base { auto result = it->second->parse(ctx, start); // Fire NODE_END event - if (ctx.event_handler && ctx.env) { + if (ctx.event_handler && ctx.semantics) { ctx.current_depth--; std::string_view text = ctx.input; if (result.start < ctx.input.size()) { @@ -1236,7 +1236,7 @@ class rule_parser : public common_chat_peg_parser_base { text, result.type, ctx.current_depth - }, *ctx.env); + }, *ctx.semantics); } return result; @@ -1267,11 +1267,11 @@ class capture_parser : public common_chat_peg_parser_base { common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto result = parser_->parse(ctx, start); - if (!result.fail() && ctx.env) { + if (!result.fail() && ctx.semantics) { std::string_view matched = ctx.input; matched = matched.substr(result.start, result.end - result.start); std::string value = std::string(matched); - ctx.env->captures[key_] = std::move(value); + ctx.semantics->captures[key_] = std::move(value); } return result; diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index d18c93c1be35c..f50754a055215 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -108,33 +108,33 @@ struct common_chat_parse_context { std::string input; common_chat_parse_cache cache; bool input_is_complete; - common_chat_parse_semantics * env; + common_chat_parse_semantics * semantics; common_chat_parse_event_handler event_handler; int current_depth; common_chat_parse_context() - : cache(), input_is_complete(true), env(nullptr), event_handler(nullptr), current_depth(0) {} + : cache(), input_is_complete(true), semantics(nullptr), event_handler(nullptr), current_depth(0) {} common_chat_parse_context(const std::string & input) - : input(input), cache(), input_is_complete(true), env(nullptr), event_handler(nullptr), current_depth(0) {} + : input(input), cache(), input_is_complete(true), semantics(nullptr), event_handler(nullptr), current_depth(0) {} common_chat_parse_context(const std::string & input, bool complete) - : input(input), cache(), input_is_complete(complete), env(nullptr), event_handler(nullptr), current_depth(0) {} + : input(input), cache(), input_is_complete(complete), semantics(nullptr), event_handler(nullptr), current_depth(0) {} common_chat_parse_context(const std::string & input, common_chat_parse_cache memo, bool complete = true) - : input(input), cache(std::move(memo)), input_is_complete(complete), env(nullptr), event_handler(nullptr), current_depth(0) {} + : input(input), cache(std::move(memo)), input_is_complete(complete), semantics(nullptr), event_handler(nullptr), current_depth(0) {} common_chat_parse_context(const std::string & input, common_chat_parse_semantics * environment) - : input(input), cache(), input_is_complete(true), env(environment), event_handler(nullptr), current_depth(0) {} + : input(input), cache(), input_is_complete(true), semantics(environment), event_handler(nullptr), current_depth(0) {} common_chat_parse_context(const std::string & input, common_chat_parse_semantics * environment, bool complete) - : input(input), cache(), input_is_complete(complete), env(environment), event_handler(nullptr), current_depth(0) {} + : input(input), cache(), input_is_complete(complete), semantics(environment), event_handler(nullptr), current_depth(0) {} common_chat_parse_context(const std::string & input, common_chat_parse_cache memo, common_chat_parse_semantics * environment, bool complete = true) - : input(input), cache(std::move(memo)), input_is_complete(complete), env(environment), event_handler(nullptr), current_depth(0) {} + : input(input), cache(std::move(memo)), input_is_complete(complete), semantics(environment), event_handler(nullptr), current_depth(0) {} common_chat_parse_context(const std::string & input, common_chat_parse_semantics * environment, common_chat_parse_event_handler handler, bool complete = true) - : input(input), cache(), input_is_complete(complete), env(environment), event_handler(std::move(handler)), current_depth(0) {} + : input(input), cache(), input_is_complete(complete), semantics(environment), event_handler(std::move(handler)), current_depth(0) {} }; class common_chat_peg_parser_base; From bea64a02e7c513d7ccfe90e2c827e4f59538933f Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 01:51:29 -0600 Subject: [PATCH 093/183] clean up common_chat_parse_context --- common/chat-peg-parser.h | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index f50754a055215..1e9fd53332bcd 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -106,35 +106,30 @@ class common_chat_parse_cache { struct common_chat_parse_context { std::string input; - common_chat_parse_cache cache; bool input_is_complete; + common_chat_parse_cache cache; common_chat_parse_semantics * semantics; common_chat_parse_event_handler event_handler; + int current_depth; common_chat_parse_context() - : cache(), input_is_complete(true), semantics(nullptr), event_handler(nullptr), current_depth(0) {} + : input_is_complete(true), cache(), semantics(nullptr), event_handler(nullptr), current_depth(0) {} common_chat_parse_context(const std::string & input) - : input(input), cache(), input_is_complete(true), semantics(nullptr), event_handler(nullptr), current_depth(0) {} + : input(input), input_is_complete(true), cache(), semantics(nullptr), event_handler(nullptr), current_depth(0) {} common_chat_parse_context(const std::string & input, bool complete) - : input(input), cache(), input_is_complete(complete), semantics(nullptr), event_handler(nullptr), current_depth(0) {} - - common_chat_parse_context(const std::string & input, common_chat_parse_cache memo, bool complete = true) - : input(input), cache(std::move(memo)), input_is_complete(complete), semantics(nullptr), event_handler(nullptr), current_depth(0) {} - - common_chat_parse_context(const std::string & input, common_chat_parse_semantics * environment) - : input(input), cache(), input_is_complete(true), semantics(environment), event_handler(nullptr), current_depth(0) {} + : input(input), input_is_complete(complete), cache(), semantics(nullptr), event_handler(nullptr), current_depth(0) {} - common_chat_parse_context(const std::string & input, common_chat_parse_semantics * environment, bool complete) - : input(input), cache(), input_is_complete(complete), semantics(environment), event_handler(nullptr), current_depth(0) {} + common_chat_parse_context(const std::string & input, common_chat_parse_semantics * semantics) + : input(input), input_is_complete(true), cache(), semantics(semantics), event_handler(nullptr), current_depth(0) {} - common_chat_parse_context(const std::string & input, common_chat_parse_cache memo, common_chat_parse_semantics * environment, bool complete = true) - : input(input), cache(std::move(memo)), input_is_complete(complete), semantics(environment), event_handler(nullptr), current_depth(0) {} + common_chat_parse_context(const std::string & input, common_chat_parse_semantics * semantics, bool complete) + : input(input), input_is_complete(complete), cache(), semantics(semantics), event_handler(nullptr), current_depth(0) {} - common_chat_parse_context(const std::string & input, common_chat_parse_semantics * environment, common_chat_parse_event_handler handler, bool complete = true) - : input(input), cache(), input_is_complete(complete), semantics(environment), event_handler(std::move(handler)), current_depth(0) {} + common_chat_parse_context(const std::string & input, common_chat_parse_semantics * semantics, common_chat_parse_event_handler handler, bool complete = true) + : input(input), input_is_complete(complete), cache(), semantics(semantics), event_handler(std::move(handler)), current_depth(0) {} }; class common_chat_peg_parser_base; From 27ffc9f12657a03d4446dc25ab07188b5171b36c Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 01:54:31 -0600 Subject: [PATCH 094/183] move type() to below constant --- common/chat-peg-parser.cpp | 69 +++++++++++++++----------------------- 1 file changed, 27 insertions(+), 42 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 11b8143e37843..ef3bfa0ed8b3b 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -279,11 +279,10 @@ class root_parser : public common_chat_peg_parser_base { public: static constexpr parser_type type_value = ROOT; + parser_type type() const override { return type_value; } root_parser(int id) : common_chat_peg_parser_base(id) {} - parser_type type() const override { return type_value; } - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { return root_->parse(ctx, start); } @@ -318,8 +317,10 @@ class root_parser : public common_chat_peg_parser_base { class start_parser : public common_chat_peg_parser_base { public: static constexpr parser_type type_value = START; - start_parser(int id) : common_chat_peg_parser_base(id) {} parser_type type() const override { return type_value; } + + start_parser(int id) : common_chat_peg_parser_base(id) {} + void accept(parser_visitor & visitor) override; std::string dump() const override { return "Start"; } @@ -333,8 +334,10 @@ class start_parser : public common_chat_peg_parser_base { class end_parser : public common_chat_peg_parser_base { public: static constexpr parser_type type_value = END; - end_parser(int id) : common_chat_peg_parser_base(id) {} parser_type type() const override { return type_value; } + + end_parser(int id) : common_chat_peg_parser_base(id) {} + void accept(parser_visitor & visitor) override; std::string dump() const override { return "End"; } @@ -350,11 +353,10 @@ class literal_parser : public common_chat_peg_parser_base { public: static constexpr parser_type type_value = LITERAL; + parser_type type() const override { return type_value; } literal_parser(const std::string & literal, int id) : common_chat_peg_parser_base(id), literal_(literal) {} - parser_type type() const override { return type_value; } - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto pos = start; for (auto i = 0u; i < literal_.size(); ++i) { @@ -389,6 +391,7 @@ class sequence_parser : public common_chat_peg_parser_base { public: static constexpr parser_type type_value = SEQUENCE; + parser_type type() const override { return type_value; } template sequence_parser(InputIt first, InputIt last, int id) : common_chat_peg_parser_base(id) { @@ -405,8 +408,6 @@ class sequence_parser : public common_chat_peg_parser_base { sequence_parser(const T & parsers, int id) : sequence_parser(std::begin(parsers), std::end(parsers), id) {} - parser_type type() const override { return type_value; } - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto pos = start; for (const auto & p : parsers_) { @@ -449,6 +450,7 @@ class choice_parser : public common_chat_peg_parser_base { public: static constexpr parser_type type_value = CHOICE; + parser_type type() const override { return type_value; } template choice_parser(InputIt first, InputIt last, int id) : common_chat_peg_parser_base(id) { @@ -465,8 +467,6 @@ class choice_parser : public common_chat_peg_parser_base { choice_parser(const T & parsers, int id) : choice_parser(std::begin(parsers), std::end(parsers), id) {} - parser_type type() const override { return type_value; } - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto pos = start; for (const auto & p : parsers_) { @@ -510,12 +510,11 @@ class repetition_parser : public common_chat_peg_parser_base { public: static constexpr parser_type type_value = REPETITION; + parser_type type() const override { return type_value; } repetition_parser(const common_chat_peg_parser & parser, int min_count, int max_count, int id) : common_chat_peg_parser_base(id), parser_(parser), min_count_(min_count), max_count_(max_count) {} - parser_type type() const override { return type_value; } - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto pos = start; int match_count = 0; @@ -580,11 +579,10 @@ class repetition_parser : public common_chat_peg_parser_base { class one_or_more_parser : public repetition_parser { public: static constexpr parser_type type_value = ONE_OR_MORE; + parser_type type() const override { return type_value; } one_or_more_parser(const common_chat_peg_parser & p, int id) : repetition_parser(p, 1, -1, id) {} - parser_type type() const override { return type_value; } - std::string dump() const override { return "OneOrMore(" + child()->dump() + ")"; } @@ -597,11 +595,10 @@ class one_or_more_parser : public repetition_parser { class zero_or_more_parser : public repetition_parser { public: static constexpr parser_type type_value = ZERO_OR_MORE; + parser_type type() const override { return type_value; } zero_or_more_parser(const common_chat_peg_parser & p, int id) : repetition_parser(p, 0, -1, id) {} - parser_type type() const override { return type_value; } - std::string dump() const override { return "ZeroOrMore(" + child()->dump() + ")"; } @@ -614,11 +611,10 @@ class zero_or_more_parser : public repetition_parser { class optional_parser : public repetition_parser { public: static constexpr parser_type type_value = OPTIONAL; + parser_type type() const override { return type_value; } optional_parser(const common_chat_peg_parser & p, int id) : repetition_parser(p, 0, 1, id) {} - parser_type type() const override { return type_value; } - std::string dump() const override { return "Optional(" + child()->dump() + ")"; } @@ -633,11 +629,10 @@ class and_parser : public common_chat_peg_parser_base { public: static constexpr parser_type type_value = AND; + parser_type type() const override { return type_value; } and_parser(const common_chat_peg_parser & parser, int id) : common_chat_peg_parser_base(id), parser_(parser) {} - parser_type type() const override { return type_value; } - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto result = parser_->parse(ctx, start); if (result.success()) { @@ -670,11 +665,10 @@ class not_parser : public common_chat_peg_parser_base { public: static constexpr parser_type type_value = NOT; + parser_type type() const override { return type_value; } not_parser(const common_chat_peg_parser & parser, int id) : common_chat_peg_parser_base(id), parser_(parser) {} - parser_type type() const override { return type_value; } - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto result = parser_->parse(ctx, start); @@ -711,11 +705,10 @@ class not_parser : public common_chat_peg_parser_base { class any_parser : public common_chat_peg_parser_base { public: static constexpr parser_type type_value = ANY; + parser_type type() const override { return type_value; } any_parser(int id) : common_chat_peg_parser_base(id) {} - parser_type type() const override { return type_value; } - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { // Parse a single UTF-8 codepoint (not just a single byte) auto result = parse_utf8_codepoint(ctx.input, start); @@ -744,11 +737,10 @@ class any_parser : public common_chat_peg_parser_base { class space_parser : public common_chat_peg_parser_base { public: static constexpr parser_type type_value = SPACE; + parser_type type() const override { return type_value; } space_parser(int id) : common_chat_peg_parser_base(id) {} - parser_type type() const override { return type_value; } - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto pos = start; while (pos < ctx.input.size()) { @@ -855,6 +847,9 @@ class chars_parser : public common_chat_peg_parser_base { int max_count_; public: + static constexpr parser_type type_value = CHARS; + parser_type type() const override { return type_value; } + chars_parser(const std::string & classes, int min_count, int max_count, int id) : common_chat_peg_parser_base(id), pattern_(classes), negated_(false), min_count_(min_count), max_count_(max_count) { @@ -889,10 +884,6 @@ class chars_parser : public common_chat_peg_parser_base { } } - static constexpr parser_type type_value = CHARS; - - parser_type type() const override { return type_value; } - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto pos = start; int match_count = 0; @@ -981,11 +972,10 @@ class chars_parser : public common_chat_peg_parser_base { class json_string_parser : public common_chat_peg_parser_base { public: static constexpr parser_type type_value = JSON_STRING; + parser_type type() const override { return type_value; } json_string_parser(int id) : common_chat_peg_parser_base(id) {} - parser_type type() const override { return type_value; } - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto pos = start; @@ -1089,6 +1079,7 @@ class until_parser : public common_chat_peg_parser_base { public: static constexpr parser_type type_value = UNTIL; + parser_type type() const override { return type_value; } until_parser(const std::vector & delimiters, int id) : common_chat_peg_parser_base(id), delimiters_(delimiters), matcher_(delimiters) {} @@ -1096,8 +1087,6 @@ class until_parser : public common_chat_peg_parser_base { until_parser(const std::string & delimiter, int id) : until_parser(std::vector{delimiter}, id) {} - parser_type type() const override { return type_value; } - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { // First pass: byte-based Aho-Corasick search for delimiter auto search_result = matcher_.search(ctx.input, start); @@ -1151,12 +1140,11 @@ class schema_parser : public common_chat_peg_parser_base { public: static constexpr parser_type type_value = SCHEMA; + parser_type type() const override { return type_value; } schema_parser(const common_chat_peg_parser & parser, const std::string & name, const nlohmann::ordered_json & schema, int id) : common_chat_peg_parser_base(id), parser_(parser), name_(name), schema_(schema) {} - parser_type type() const override { return type_value; } - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { return parser_->parse(ctx, start); } @@ -1182,12 +1170,11 @@ class rule_parser : public common_chat_peg_parser_base { public: static constexpr parser_type type_value = RULE; + parser_type type() const override { return type_value; } rule_parser(const std::string & name, const std::weak_ptr & root, int id) : common_chat_peg_parser_base(id), name_(name), root_(root) {} - parser_type type() const override { return type_value; } - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto root = root_.lock(); if (!root) { @@ -1258,12 +1245,11 @@ class capture_parser : public common_chat_peg_parser_base { public: static constexpr parser_type type_value = CAPTURE; + parser_type type() const override { return type_value; } capture_parser(const common_chat_peg_parser & parser, const std::string & key, int id) : common_chat_peg_parser_base(id), parser_(parser), key_(key) {} - parser_type type() const override { return type_value; } - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto result = parser_->parse(ctx, start); @@ -1299,12 +1285,11 @@ class trigger_parser : public common_chat_peg_parser_base { public: static constexpr parser_type type_value = TRIGGER; + parser_type type() const override { return type_value; } trigger_parser(const common_chat_peg_parser & parser, int id) : common_chat_peg_parser_base(id), parser_(parser) {} - parser_type type() const override { return type_value; } - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { return parser_->parse(ctx, start); } From 425863e5382a7cef58b18caed63356d0ad5f0d4c Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 01:55:54 -0600 Subject: [PATCH 095/183] use default constructor for common_chat_peg_parser --- common/chat-peg-parser.h | 8 -------- 1 file changed, 8 deletions(-) diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index 1e9fd53332bcd..0cb63af5f35ba 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -140,17 +140,9 @@ class common_chat_peg_parser { public: common_chat_peg_parser(); common_chat_peg_parser(std::shared_ptr parser); - common_chat_peg_parser(const common_chat_peg_parser & other) = default; common_chat_peg_parser(const std::string & literal); common_chat_peg_parser(const char * literal); - common_chat_peg_parser & operator=(const common_chat_peg_parser & other) { - if (this != &other) { - ptr_ = other.ptr_; - } - return *this; - } - common_chat_peg_parser operator~() const; common_chat_peg_parser operator+(const common_chat_peg_parser & other) const; common_chat_peg_parser operator|(const common_chat_peg_parser & other) const; From 4413c5c849c6efa1f1fe3adadf0ee6feb8b4bfc6 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 02:03:22 -0600 Subject: [PATCH 096/183] make all operators functions for consistency --- common/chat-peg-parser.cpp | 16 +++++++--------- common/chat-peg-parser.h | 11 ++++++----- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index ef3bfa0ed8b3b..950408691863f 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -1799,21 +1799,19 @@ common_chat_peg_parser::common_chat_peg_parser(std::shared_ptr(-1, literal)) {} common_chat_peg_parser::common_chat_peg_parser(const char * literal) : ptr_(make_parser(-1, literal)) {} -common_chat_peg_parser common_chat_peg_parser::operator~() const { - return make_parser(-1, *this); -} +common_chat_peg_parser operator~(const common_chat_peg_parser & p) { return make_parser(-1, p); } -common_chat_peg_parser common_chat_peg_parser::operator+(const common_chat_peg_parser & other) const { - return make_parser(-1, std::initializer_list{*this, other}); +common_chat_peg_parser operator+(const common_chat_peg_parser & lhs, const common_chat_peg_parser & rhs) { + return make_parser(-1, std::initializer_list{lhs, rhs}); } -common_chat_peg_parser common_chat_peg_parser::operator|(const common_chat_peg_parser & other) const { - return make_parser(-1, std::initializer_list{*this, other}); +common_chat_peg_parser operator|(const common_chat_peg_parser & lhs, const common_chat_peg_parser & rhs) { + return make_parser(-1, std::initializer_list{lhs, rhs}); } -common_chat_peg_parser common_chat_peg_parser::operator<<(const common_chat_peg_parser & other) const { +common_chat_peg_parser operator<<(const common_chat_peg_parser & lhs, const common_chat_peg_parser & rhs) { auto ws = make_parser(-1); - return make_parser(-1, std::initializer_list{*this, ws, other}); + return make_parser(-1, std::initializer_list{lhs, ws, rhs}); } common_chat_peg_parser operator+(const char * lhs, const common_chat_peg_parser & rhs) { return common_chat_peg_parser(lhs) + rhs; } diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index 0cb63af5f35ba..3bd6c506489c8 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -143,11 +143,6 @@ class common_chat_peg_parser { common_chat_peg_parser(const std::string & literal); common_chat_peg_parser(const char * literal); - common_chat_peg_parser operator~() const; - common_chat_peg_parser operator+(const common_chat_peg_parser & other) const; - common_chat_peg_parser operator|(const common_chat_peg_parser & other) const; - common_chat_peg_parser operator<<(const common_chat_peg_parser & other) const; - common_chat_peg_parser_base & operator*() const; common_chat_peg_parser_base * operator->() const; @@ -160,6 +155,12 @@ class common_chat_peg_parser { void build_grammar(const common_grammar_builder & builder, bool lazy = false) const; }; +common_chat_peg_parser operator~(const common_chat_peg_parser & p); + +common_chat_peg_parser operator+(const common_chat_peg_parser & lhs, const common_chat_peg_parser & rhs); +common_chat_peg_parser operator|(const common_chat_peg_parser & lhs, const common_chat_peg_parser & rhs); +common_chat_peg_parser operator<<(const common_chat_peg_parser & lhs, const common_chat_peg_parser & rhs); + common_chat_peg_parser operator+(const char * lhs, const common_chat_peg_parser & rhs); common_chat_peg_parser operator|(const char * lhs, const common_chat_peg_parser & rhs); common_chat_peg_parser operator<<(const char * lhs, const common_chat_peg_parser & rhs); From 817a0eb1f1f5292d0d4642c5f2e2e99daa7f9ac5 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 02:15:56 -0600 Subject: [PATCH 097/183] fix compilation errors in test-optional.cpp --- tests/chat-peg-parser/test-optional.cpp | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/chat-peg-parser/test-optional.cpp b/tests/chat-peg-parser/test-optional.cpp index 6ea23a3abf4b5..ea8237aca3309 100644 --- a/tests/chat-peg-parser/test-optional.cpp +++ b/tests/chat-peg-parser/test-optional.cpp @@ -3,32 +3,33 @@ void test_optional(testing &t) { // Full match with optional part present t.test("optional_present", [](testing &t) { - auto parser = - build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + return p.literal("hello") + p.optional(p.literal(" world")); + }); auto ctx = common_chat_parse_context("hello world"); auto result = parser.parse(ctx); t.assert_equal("optional_present", true, result.success()); - int end_pos = result.end; - t.assert_equal("optional_present_end", 11, end_pos); + t.assert_equal("optional_present_end", 11u, result.end); }); // Full match with optional part absent t.test("optional_absent", [](testing &t) { - auto parser = - build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + return p.literal("hello") + p.optional(p.literal(" world")); + }); auto ctx = common_chat_parse_context("hello", true); auto result = parser.parse(ctx); t.assert_equal("optional_absent", true, result.success()); - int end_pos = result.end; - t.assert_equal("optional_absent_end", 5, end_pos); + t.assert_equal("optional_absent_end", 5u, result.end); }); // Partial match - waiting for more input to determine if optional matches t.test("partial_match_need_more", [](testing &t) { - auto parser = - build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + return p.literal("hello") + p.optional(p.literal(" world")); + }); auto ctx = common_chat_parse_context("hello ", false); auto result = parser.parse(ctx); From f41539b5808519bbc84a36094f0b9488e8865fc9 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 02:20:00 -0600 Subject: [PATCH 098/183] simplify result values --- common/chat-peg-parser.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index 3bd6c506489c8..49d9e144dc178 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -30,9 +30,9 @@ struct common_chat_parse_semantics { }; enum common_chat_parse_result_type { - COMMON_CHAT_PARSE_RESULT_FAIL = 1 << 0, - COMMON_CHAT_PARSE_RESULT_SUCCESS = 1 << 1, - COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT = 1 << 2, + COMMON_CHAT_PARSE_RESULT_FAIL = 0, + COMMON_CHAT_PARSE_RESULT_SUCCESS = 1, + COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT = 2, }; const char * common_chat_parse_result_type_name(common_chat_parse_result_type type); From 7cf9b7338faaf4f1507ee9a4fdf3ed4a47a5c72c Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 02:21:27 -0600 Subject: [PATCH 099/183] rename json_string_unquoted to json_string_content --- common/chat-peg-parser.cpp | 4 ++-- common/chat-peg-parser.h | 2 +- tests/chat-peg-parser/test-unicode.cpp | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 950408691863f..8ff30a67d9b95 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -1857,7 +1857,7 @@ common_chat_peg_parser builder::negate(const common_chat_peg_parser & p) { retur common_chat_peg_parser builder::any() { return make_parser(counter_); } common_chat_peg_parser builder::chars(const std::string & classes, int min, int max) { return make_parser(counter_, classes, min, max); } common_chat_peg_parser builder::one(const std::string & classes) { return make_parser(counter_, classes, 1, 1); } -common_chat_peg_parser builder::json_string_unqouted() { return make_parser(counter_); } +common_chat_peg_parser builder::json_string_content() { return make_parser(counter_); } common_chat_peg_parser builder::space() { return make_parser(counter_); } common_chat_peg_parser builder::until(const std::string & delimiter) { return make_parser(counter_, delimiter); } common_chat_peg_parser builder::until_one_of(const std::vector & delimiters) { return make_parser(counter_, delimiters); } @@ -1922,7 +1922,7 @@ common_chat_peg_parser builder::json_number() { common_chat_peg_parser builder::json_string() { return add_rule("json-string", [this]() { - return literal("\"") + json_string_unqouted() + literal("\""); + return literal("\"") + json_string_content() + literal("\""); }); } diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index 49d9e144dc178..967c682b191d4 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -272,7 +272,7 @@ class common_chat_peg_parser_builder { common_chat_peg_parser json_null(); // Specialized single-pass JSON string parser with escape sequence handling - common_chat_peg_parser json_string_unqouted(); + common_chat_peg_parser json_string_content(); // Wraps a parser with JSON schema metadata for grammar generation. // Used internally to convert JSON schemas to GBNF grammar rules. diff --git a/tests/chat-peg-parser/test-unicode.cpp b/tests/chat-peg-parser/test-unicode.cpp index e807cdb4d292e..c0d74bf9e60fc 100644 --- a/tests/chat-peg-parser/test-unicode.cpp +++ b/tests/chat-peg-parser/test-unicode.cpp @@ -328,7 +328,7 @@ void test_unicode(testing &t) { t.test(test_name, [&](testing &t) { auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { - return p.json_string_unqouted() + p.literal("\""); + return p.json_string_content() + p.literal("\""); }); common_chat_parse_context ctx(tc.input, true); @@ -365,7 +365,7 @@ void test_unicode(testing &t) { t.test(test_name, [&](testing &t) { auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { - return p.json_string_unqouted(); + return p.json_string_content(); }); common_chat_parse_context ctx(tc.input, false); // input_is_complete = false @@ -402,7 +402,7 @@ void test_unicode(testing &t) { t.test(test_name, [&](testing &t) { auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { - return p.json_string_unqouted(); + return p.json_string_content(); }); common_chat_parse_context ctx(tc.input, true); @@ -431,7 +431,7 @@ void test_unicode(testing &t) { t.test(test_name, [&](testing &t) { auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { - return p.json_string_unqouted() + p.literal("\""); + return p.json_string_content() + p.literal("\""); }); common_chat_parse_context ctx(tc.input, true); From c0faa273db4d79561898700314877011aad371b5 Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Sun, 16 Nov 2025 12:02:48 +0100 Subject: [PATCH 100/183] Move helper to separate class, add separate explicit and helper classes --- common/CMakeLists.txt | 2 + common/chat-peg-parser-helper.cpp | 41 ++++++++ common/chat-peg-parser-helper.h | 26 +++++ common/chat-peg-parser.cpp | 32 ------ common/chat-peg-parser.h | 19 ---- .../test-example-qwen3-coder.cpp | 99 +++++++++++++++---- tests/chat-peg-parser/tests.h | 1 + 7 files changed, 149 insertions(+), 71 deletions(-) create mode 100644 common/chat-peg-parser-helper.cpp create mode 100644 common/chat-peg-parser-helper.h diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 77d0271e4c6e4..f2b52d393b68a 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -52,6 +52,8 @@ add_library(${TARGET} STATIC chat-parser.h chat-peg-parser.cpp chat-peg-parser.h + chat-peg-parser-helper.cpp + chat-peg-parser-helper.h chat.cpp chat.h common.cpp diff --git a/common/chat-peg-parser-helper.cpp b/common/chat-peg-parser-helper.cpp new file mode 100644 index 0000000000000..a1de7733e8a92 --- /dev/null +++ b/common/chat-peg-parser-helper.cpp @@ -0,0 +1,41 @@ +#include "chat-peg-parser-helper.h" +#include "chat-peg-parser.h" + +common_chat_peg_parser common_chat_peg_parser_builder_helper::reasoning(const std::string &tag) { + return add_rule("raw-reasoning", std::string("<" + tag + ">") << add_rule("reasoning-content", until("")) << ""); +} + +common_chat_peg_parser common_chat_peg_parser_builder_helper::content_before_tools(const std::string &tag) { + return add_rule("content", until(tag)); +} + +common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_no_attr(const std::string &function_name, const std::vector ¶meters, + const std::string &function_tag, const std::string ¶m_tag) { + std::vector args; + + for (auto it = parameters.begin(); it != parameters.end(); it++) { + auto arg_name = add_rule(std::string("arg-start-" + *it), literal("<" + param_tag + "=" + *it + ">")); + auto arg_end = add_rule("arg-end", "" + peek(literal("<" + param_tag + "=") | (""))); + auto string_arg_content = add_rule("arg-string-content", + until_one_of({"<" + param_tag + "=", ""})); + auto string_arg = add_rule("arg-string-" + *it, arg_name + string_arg_content + arg_end); + auto json_sec = json(); + auto json_arg = add_rule("arg-json-" + *it, arg_name + add_rule("arg-json-content", json_sec) + arg_end); + auto arg_json_or_string = one_or_more(json_arg | string_arg); + args.push_back(arg_json_or_string); + } + + auto args_sequence = sequence(args); + auto function = add_rule("function-" + function_name, + add_rule("function-start-" + function_name, "<" + function_tag + "=" + function_name + ">") + + args_sequence + ""); + + return function; +} + +common_chat_peg_parser build_peg_parser_helper(const std::function & fn) { + common_chat_peg_parser_builder_helper builder; + auto root = fn(builder); + builder.set_root(root); + return builder.build(); +} diff --git a/common/chat-peg-parser-helper.h b/common/chat-peg-parser-helper.h new file mode 100644 index 0000000000000..1613badd02f70 --- /dev/null +++ b/common/chat-peg-parser-helper.h @@ -0,0 +1,26 @@ +#include "chat-peg-parser.h" + +class common_chat_peg_parser_builder_helper : public common_chat_peg_parser_builder { + +public: + // Helper methods for common patterns + + // Adds raw-reasoning for the entire reasoning block plus reasoning-content for the contents, by default thinking tag is "think" + common_chat_peg_parser reasoning(const std::string & tag = "think"); + + // Adds main content block before tool call block, due to the varied nature of tool call openers (not always XML-like) full tag is required + common_chat_peg_parser content_before_tools(const std::string &tag); + + // Adds a quasi-XML tool call spec without a separate name attribute (Qwen3 style); + // TODO: accept parameter schemas (required, value types etc.) + common_chat_peg_parser quasi_xml_no_attr(const std::string &function_name, const std::vector ¶meters, + const std::string &function_tag = "function", const std::string ¶m_tag = "parameter"); + + // Adds a quasi-XML tool call spec with a separate name attribute (Minimax-M2 style) + // TODO: accept parameter schemas (required, value types etc.) + // common_chat_peg_parser quasi_xml_attr(const std::string &function_name, const std::vector ¶meters, + // const std::string &function_tag = "invoke", const std::string ¶m_tag = "parameter", + // const std::string &name_attr = "name"); +}; + +common_chat_peg_parser build_peg_parser_helper(const std::function & fn); diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 8ff30a67d9b95..57e71dd2defe3 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -1968,38 +1968,6 @@ common_chat_peg_parser builder::json() { }); } -common_chat_peg_parser builder::reasoning(const std::string &tag) { - return add_rule("raw-reasoning", std::string("<" + tag + ">") << add_rule("reasoning-content", until("")) << ""); -} - -common_chat_peg_parser builder::content_before_tools(const std::string &tag) { - return add_rule("content", until(tag)); -} - -common_chat_peg_parser builder::quasi_xml_no_attr(const std::string &function_name, const std::vector ¶meters, - const std::string &function_tag, const std::string ¶m_tag) { - std::vector args; - - for (auto it = parameters.begin(); it != parameters.end(); it++) { - auto arg_name = add_rule(std::string("arg-start-" + *it), literal("<" + param_tag + "=" + *it + ">")); - auto arg_end = add_rule("arg-end", "" + peek(literal("<" + param_tag + "=") | (""))); - auto string_arg_content = add_rule("arg-string-content", - until_one_of({"<" + param_tag + "=", ""})); - auto string_arg = add_rule("arg-string-" + *it, arg_name + string_arg_content + arg_end); - auto json_sec = json(); - auto json_arg = add_rule("arg-json-" + *it, arg_name + add_rule("arg-json-content", json_sec) + arg_end); - auto arg_json_or_string = one_or_more(json_arg | string_arg); - args.push_back(arg_json_or_string); - } - - auto args_sequence = sequence(args); - auto function = add_rule("function-" + function_name, - add_rule("function-start-" + function_name, "<" + function_tag + "=" + function_name + ">") - + args_sequence + ""); - - return function; -} - common_chat_peg_parser builder::build() { return root_; } diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index 967c682b191d4..76136e8654f42 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -299,25 +299,6 @@ class common_chat_peg_parser_builder { void set_root(const common_chat_peg_parser & p); - // Helper methods for common patterns - - // Adds raw-reasoning for the entire reasoning block plus reasoning-content for the contents, by default thinking tag is "think" - common_chat_peg_parser reasoning(const std::string & tag = "think"); - - // Adds main content block before tool call block, due to the varied nature of tool call openers (not always XML-like) full tag is required - common_chat_peg_parser content_before_tools(const std::string &tag); - - // Adds a quasi-XML tool call spec without a separate name attribute (Qwen3 style); - // TODO: accept parameter schemas (required, value types etc.) - common_chat_peg_parser quasi_xml_no_attr(const std::string &function_name, const std::vector ¶meters, - const std::string &function_tag = "function", const std::string ¶m_tag = "parameter"); - - // Adds a quasi-XML tool call spec with a separate name attribute (Minimax-M2 style) - // TODO: accept parameter schemas (required, value types etc.) - // common_chat_peg_parser quasi_xml_attr(const std::string &function_name, const std::vector ¶meters, - // const std::string &function_tag = "invoke", const std::string ¶m_tag = "parameter", - // const std::string &name_attr = "name"); - common_chat_peg_parser build(); }; diff --git a/tests/chat-peg-parser/test-example-qwen3-coder.cpp b/tests/chat-peg-parser/test-example-qwen3-coder.cpp index 9d5cfa7583f0d..ced43170801d0 100644 --- a/tests/chat-peg-parser/test-example-qwen3-coder.cpp +++ b/tests/chat-peg-parser/test-example-qwen3-coder.cpp @@ -4,7 +4,37 @@ #include void test_example_qwen3_coder(testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto explicit_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto thinking = p.add_rule("raw-reasoning", + "" << p.add_rule("reasoning-content", p.until("")) << ""); + + auto content = p.add_rule("content", p.until("")); + + auto arg_name = p.add_rule("arg-start", ""); + auto arg_end = p.add_rule("arg-end", "" + p.peek(p.literal("")); + + auto string_arg_content = p.add_rule("arg-string-content", + p.until_one_of({""})); + + auto string_arg = p.add_rule("arg-string", arg_name + string_arg_content + arg_end); + + auto json = p.json(); + + auto json_arg = p.add_rule("arg-json", arg_name + p.add_rule("arg-json-content", json) + arg_end); + + auto function = p.add_rule("function", + p.add_rule("function-start", "") + + p.one_or_more(json_arg | string_arg) + + ""); + + auto tool_call = p.trigger(p.add_rule("tool-call", + "" + p.one_or_more(function) + "")); + + return thinking + p.optional(p.space() + content) + p.zero_or_more(p.space() + tool_call); + }); + + + auto helper_parser = build_peg_parser_helper([](common_chat_peg_parser_builder_helper & p) { auto thinking = p.reasoning(); auto content = p.content_before_tools(""); auto function = p.quasi_xml_no_attr("search_files", @@ -60,7 +90,7 @@ void test_example_qwen3_coder(testing &t) { } }; - t.test("accumulation_test", [&](testing &t) { + t.test("qwen3_accumulation_test", [&](testing &t) { std::string input = "The user wants to find large log files that haven't been accessed recently. " "I should search for files with .log extension, filter by size (over 100MB), " @@ -85,23 +115,52 @@ void test_example_qwen3_coder(testing &t) { std::vector tokens = simple_tokenize(input); common_chat_msg prev; - int token_cnt = 0; - for (auto it = tokens.begin(); it != tokens.end(); it++) { - token_cnt++; - std::string in = std::accumulate(tokens.begin(), it, std::string()); - - common_chat_parse_semantics semantics; - common_chat_parse_context ctx(in, &semantics, it == tokens.end() - 1); - - ctx.event_handler = handler; - - auto result = parser.parse(ctx); - t.assert_equal(std::string("should_not_fail_token_") + std::to_string(token_cnt), false, result.fail()); - - // This shouldn't emit any runtime errors - auto msg = semantics.to_msg(); - auto diffs = common_chat_msg_diff::compute_diffs(prev, msg); - prev = msg; - } + t.test("explicit_builder", [&](testing &t) { + size_t token_cnt = 0; + for (auto it = tokens.begin(); it != tokens.end(); it++) { + std::string in = std::accumulate(tokens.begin(), it, std::string()); + + common_chat_parse_semantics semantics; + common_chat_parse_context ctx(in, &semantics, it == tokens.end() - 1); + + ctx.event_handler = handler; + + auto result = explicit_parser.parse(ctx); + if (result.fail()) { + break; + } + token_cnt++; + + // This shouldn't emit any runtime errors + auto msg = semantics.to_msg(); + auto diffs = common_chat_msg_diff::compute_diffs(prev, msg); + prev = msg; + } + t.assert_equal("should_parse_all_tokens_explicit", tokens.size(), token_cnt); + }); + + t.test("helper_builder", [&](testing &t) { + size_t token_cnt = 0; + for (auto it = tokens.begin(); it != tokens.end(); it++) { + token_cnt++; + std::string in = std::accumulate(tokens.begin(), it, std::string()); + + common_chat_parse_semantics semantics; + common_chat_parse_context ctx(in, &semantics, it == tokens.end() - 1); + + ctx.event_handler = handler; + + auto result = helper_parser.parse(ctx); + if (result.fail()) { + break; + } + + // This shouldn't emit any runtime errors + auto msg = semantics.to_msg(); + auto diffs = common_chat_msg_diff::compute_diffs(prev, msg); + prev = msg; + } + t.assert_equal("should_parse_all_tokens_helper", tokens.size(), token_cnt); + }); }); } diff --git a/tests/chat-peg-parser/tests.h b/tests/chat-peg-parser/tests.h index 5fed1b03a15c2..7de49b4384567 100644 --- a/tests/chat-peg-parser/tests.h +++ b/tests/chat-peg-parser/tests.h @@ -4,6 +4,7 @@ #include "test_harness.h" #include #include "chat-peg-parser.h" +#include "chat-peg-parser-helper.h" #include #include #include From 851b07099192e980808671be0edec85cf708e3ab Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Sun, 16 Nov 2025 12:03:24 +0100 Subject: [PATCH 101/183] Whitespace --- tests/chat-peg-parser/test-example-qwen3-coder.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/chat-peg-parser/test-example-qwen3-coder.cpp b/tests/chat-peg-parser/test-example-qwen3-coder.cpp index ced43170801d0..83f61993cd409 100644 --- a/tests/chat-peg-parser/test-example-qwen3-coder.cpp +++ b/tests/chat-peg-parser/test-example-qwen3-coder.cpp @@ -138,7 +138,7 @@ void test_example_qwen3_coder(testing &t) { } t.assert_equal("should_parse_all_tokens_explicit", tokens.size(), token_cnt); }); - + t.test("helper_builder", [&](testing &t) { size_t token_cnt = 0; for (auto it = tokens.begin(); it != tokens.end(); it++) { From 09976dda104fe13bd38ba48ad9c10d111974f89b Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Sun, 16 Nov 2025 12:31:30 +0100 Subject: [PATCH 102/183] Change + to append() --- common/chat-peg-parser-helper.cpp | 67 ++++++++++++++++++++++++++----- 1 file changed, 56 insertions(+), 11 deletions(-) diff --git a/common/chat-peg-parser-helper.cpp b/common/chat-peg-parser-helper.cpp index a1de7733e8a92..5a4655e90e6e9 100644 --- a/common/chat-peg-parser-helper.cpp +++ b/common/chat-peg-parser-helper.cpp @@ -2,7 +2,11 @@ #include "chat-peg-parser.h" common_chat_peg_parser common_chat_peg_parser_builder_helper::reasoning(const std::string &tag) { - return add_rule("raw-reasoning", std::string("<" + tag + ">") << add_rule("reasoning-content", until("")) << ""); + std::string open_tag; + open_tag.append("<").append(tag).append(">"); + std::string close_tag; + close_tag.append(""); + return add_rule("raw-reasoning", open_tag << add_rule("reasoning-content", until(close_tag)) << close_tag); } common_chat_peg_parser common_chat_peg_parser_builder_helper::content_before_tools(const std::string &tag) { @@ -14,21 +18,62 @@ common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_no_attr( std::vector args; for (auto it = parameters.begin(); it != parameters.end(); it++) { - auto arg_name = add_rule(std::string("arg-start-" + *it), literal("<" + param_tag + "=" + *it + ">")); - auto arg_end = add_rule("arg-end", "" + peek(literal("<" + param_tag + "=") | (""))); + std::string arg_start_name; + arg_start_name.append("arg-start-").append(*it); + + std::string param_open; + param_open.append("<").append(param_tag).append("=").append(*it).append(">"); + + auto arg_name = add_rule(arg_start_name, literal(param_open)); + + std::string param_close_end; + param_close_end.append(""); + + std::string param_close_peek; + param_close_peek.append(""); + + std::string param_peek_open; + param_peek_open.append("<").append(param_tag).append("="); + auto arg_end = add_rule("arg-end", param_close_end + peek(literal(param_peek_open) | param_close_peek)); + + std::string string_content_1; + string_content_1.append("<").append(param_tag).append("="); + + std::string string_content_2; + string_content_2.append(""); + auto string_arg_content = add_rule("arg-string-content", - until_one_of({"<" + param_tag + "=", ""})); - auto string_arg = add_rule("arg-string-" + *it, arg_name + string_arg_content + arg_end); - auto json_sec = json(); - auto json_arg = add_rule("arg-json-" + *it, arg_name + add_rule("arg-json-content", json_sec) + arg_end); + until_one_of({string_content_1, string_content_2})); + + std::string arg_string_name; + arg_string_name.append("arg-string-").append(*it); + auto string_arg = add_rule(arg_string_name, arg_name + string_arg_content + arg_end); +auto json_sec = json(); + + std::string arg_json_name; + arg_json_name.append("arg-json-").append(*it); + auto json_arg = add_rule(arg_json_name, arg_name + add_rule("arg-json-content", json_sec) + arg_end); auto arg_json_or_string = one_or_more(json_arg | string_arg); args.push_back(arg_json_or_string); + } +auto args_sequence = sequence(args); + + std::string function_start_name; + function_start_name.append("function-start-").append(function_name); + + std::string function_open; + function_open.append("<").append(function_tag).append("=").append(function_name).append(">"); + + std::string function_close; + function_close.append(""); + + std::string function_rule_name; + function_rule_name.append("function-").append(function_name); + auto function = add_rule(function_rule_name, - auto args_sequence = sequence(args); - auto function = add_rule("function-" + function_name, - add_rule("function-start-" + function_name, "<" + function_tag + "=" + function_name + ">") - + args_sequence + ""); + add_rule(function_start_name, function_open) + + args_sequence + function_close); return function; } From a1fc700f1efd64a72866a452d559640684a27221 Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Sun, 16 Nov 2025 12:32:03 +0100 Subject: [PATCH 103/183] Reformat --- common/chat-peg-parser-helper.cpp | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/common/chat-peg-parser-helper.cpp b/common/chat-peg-parser-helper.cpp index 5a4655e90e6e9..9a887d5a218b0 100644 --- a/common/chat-peg-parser-helper.cpp +++ b/common/chat-peg-parser-helper.cpp @@ -1,7 +1,7 @@ #include "chat-peg-parser-helper.h" #include "chat-peg-parser.h" -common_chat_peg_parser common_chat_peg_parser_builder_helper::reasoning(const std::string &tag) { +common_chat_peg_parser common_chat_peg_parser_builder_helper::reasoning(const std::string & tag) { std::string open_tag; open_tag.append("<").append(tag).append(">"); std::string close_tag; @@ -9,12 +9,15 @@ common_chat_peg_parser common_chat_peg_parser_builder_helper::reasoning(const st return add_rule("raw-reasoning", open_tag << add_rule("reasoning-content", until(close_tag)) << close_tag); } -common_chat_peg_parser common_chat_peg_parser_builder_helper::content_before_tools(const std::string &tag) { +common_chat_peg_parser common_chat_peg_parser_builder_helper::content_before_tools(const std::string & tag) { return add_rule("content", until(tag)); } -common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_no_attr(const std::string &function_name, const std::vector ¶meters, - const std::string &function_tag, const std::string ¶m_tag) { +common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_no_attr( + const std::string & function_name, + const std::vector & parameters, + const std::string & function_tag, + const std::string & param_tag) { std::vector args; for (auto it = parameters.begin(); it != parameters.end(); it++) { @@ -42,22 +45,20 @@ common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_no_attr( std::string string_content_2; string_content_2.append(""); - auto string_arg_content = add_rule("arg-string-content", - until_one_of({string_content_1, string_content_2})); + auto string_arg_content = add_rule("arg-string-content", until_one_of({ string_content_1, string_content_2 })); std::string arg_string_name; arg_string_name.append("arg-string-").append(*it); auto string_arg = add_rule(arg_string_name, arg_name + string_arg_content + arg_end); -auto json_sec = json(); + auto json_sec = json(); std::string arg_json_name; arg_json_name.append("arg-json-").append(*it); - auto json_arg = add_rule(arg_json_name, arg_name + add_rule("arg-json-content", json_sec) + arg_end); + auto json_arg = add_rule(arg_json_name, arg_name + add_rule("arg-json-content", json_sec) + arg_end); auto arg_json_or_string = one_or_more(json_arg | string_arg); args.push_back(arg_json_or_string); - } -auto args_sequence = sequence(args); + auto args_sequence = sequence(args); std::string function_start_name; function_start_name.append("function-start-").append(function_name); @@ -72,15 +73,15 @@ auto args_sequence = sequence(args); function_rule_name.append("function-").append(function_name); auto function = add_rule(function_rule_name, - add_rule(function_start_name, function_open) - + args_sequence + function_close); + add_rule(function_start_name, function_open) + args_sequence + function_close); return function; } -common_chat_peg_parser build_peg_parser_helper(const std::function & fn) { +common_chat_peg_parser build_peg_parser_helper( + const std::function & fn) { common_chat_peg_parser_builder_helper builder; - auto root = fn(builder); + auto root = fn(builder); builder.set_root(root); return builder.build(); } From d0c83f847afeeeea1a3f9e8acd0289a87876c7f1 Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Sun, 16 Nov 2025 13:25:18 +0100 Subject: [PATCH 104/183] Add extra helpers, tests and Minimax example --- common/chat-peg-parser-helper.cpp | 67 +++++++- common/chat-peg-parser-helper.h | 48 +++++- tests/CMakeLists.txt | 2 + tests/chat-peg-parser/convo.json | 159 ++++++++++++++++++ .../test-example-minimax-m2.cpp | 68 ++++++++ .../test-example-qwen3-coder.cpp | 46 +---- .../chat-peg-parser/test-example-seed-oss.cpp | 57 +++++++ tests/chat-peg-parser/tests.h | 2 + tests/test-chat-peg-parser.cpp | 2 + 9 files changed, 402 insertions(+), 49 deletions(-) create mode 100644 tests/chat-peg-parser/convo.json create mode 100644 tests/chat-peg-parser/test-example-minimax-m2.cpp create mode 100644 tests/chat-peg-parser/test-example-seed-oss.cpp diff --git a/common/chat-peg-parser-helper.cpp b/common/chat-peg-parser-helper.cpp index 9a887d5a218b0..e7b646f99f636 100644 --- a/common/chat-peg-parser-helper.cpp +++ b/common/chat-peg-parser-helper.cpp @@ -71,9 +71,71 @@ common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_no_attr( std::string function_rule_name; function_rule_name.append("function-").append(function_name); - auto function = add_rule(function_rule_name, + auto function = add_rule(function_rule_name, add_rule(function_start_name, function_open) + args_sequence + function_close); - add_rule(function_start_name, function_open) + args_sequence + function_close); + return function; +} + +common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_attr( + const std::string & function_name, + const std::vector & parameters, + const std::string & function_tag, + const std::string & param_tag, + const std::string & name_attr) { + std::vector args; + + for (auto it = parameters.begin(); it != parameters.end(); it++) { + std::string arg_start_name; + arg_start_name.append("arg-start-").append(*it); + + std::string param_open; + param_open.append("<").append(param_tag).append(" ").append(name_attr).append("=\"").append(*it).append("\">"); + + auto arg_name = add_rule(arg_start_name, literal(param_open)); + + std::string param_close_end; + param_close_end.append(""); + + std::string param_close_peek; + param_close_peek.append(""); + + std::string param_peek_open; + param_peek_open.append("<").append(param_tag).append(" ").append(name_attr).append("=\""); + auto arg_end = add_rule("arg-end", param_close_end + peek(literal(param_peek_open) | param_close_peek)); + + std::string string_content_1; + string_content_1.append("<").append(param_tag).append("="); + + std::string string_content_2; + string_content_2.append(""); + + auto string_arg_content = add_rule("arg-string-content", until_one_of({ string_content_1, string_content_2 })); + + std::string arg_string_name; + arg_string_name.append("arg-string-").append(*it); + auto string_arg = add_rule(arg_string_name, arg_name + string_arg_content + arg_end); + auto json_sec = json(); + + std::string arg_json_name; + arg_json_name.append("arg-json-").append(*it); + auto json_arg = add_rule(arg_json_name, arg_name + add_rule("arg-json-content", json_sec) + arg_end); + auto arg_json_or_string = one_or_more(json_arg | string_arg); + args.push_back(arg_json_or_string); + } + auto args_sequence = sequence(args); + + std::string function_start_name; + function_start_name.append("function-start-").append(function_name); + + std::string function_open; + function_open.append("<").append(function_tag).append(" ").append(name_attr).append("=\"").append(function_name).append("\">"); + + std::string function_close; + function_close.append(""); + + std::string function_rule_name; + function_rule_name.append("function-").append(function_name); + auto function = add_rule(function_rule_name, add_rule(function_start_name, function_open) + args_sequence + function_close); return function; } @@ -85,3 +147,4 @@ common_chat_peg_parser build_peg_parser_helper( builder.set_root(root); return builder.build(); } + diff --git a/common/chat-peg-parser-helper.h b/common/chat-peg-parser-helper.h index 1613badd02f70..d288dcc19962a 100644 --- a/common/chat-peg-parser-helper.h +++ b/common/chat-peg-parser-helper.h @@ -18,9 +18,51 @@ class common_chat_peg_parser_builder_helper : public common_chat_peg_parser_buil // Adds a quasi-XML tool call spec with a separate name attribute (Minimax-M2 style) // TODO: accept parameter schemas (required, value types etc.) - // common_chat_peg_parser quasi_xml_attr(const std::string &function_name, const std::vector ¶meters, - // const std::string &function_tag = "invoke", const std::string ¶m_tag = "parameter", - // const std::string &name_attr = "name"); + common_chat_peg_parser quasi_xml_attr(const std::string &function_name, const std::vector ¶meters, + const std::string &function_tag = "invoke", const std::string ¶m_tag = "parameter", + const std::string &name_attr = "name"); }; common_chat_peg_parser build_peg_parser_helper(const std::function & fn); + +inline void parser_semantic_handler(const common_chat_parse_event & ev, common_chat_parse_semantics & semantics) { + if (ev.rule == "reasoning-content" && ev.ending()) { + semantics.reasoning_content = ev.text; + } + + if (ev.rule == "content" && ev.ending()) { + semantics.content = ev.text; + } + + if (ev.rule.find("function-start") != std::string::npos && ev.ending() && ev.success()) { + semantics.tool_calls.emplace_back(); + auto & tc = semantics.tool_calls.back(); + tc.name = semantics.captures["tool-name"]; + } + + if (ev.rule.find("arg-start") != std::string::npos && ev.ending() && ev.success()) { + auto & tc = semantics.tool_calls.back(); + auto name = semantics.captures["arg-name"]; + if (tc.arguments.empty()) { + tc.arguments += "{"; + } else { + tc.arguments += ", "; + } + tc.arguments += "\"" + name + "\": "; + } + + if (ev.rule == "arg-string-content" && ev.ending() && ev.success()) { + auto & tc = semantics.tool_calls.back(); + tc.arguments += "\"" + std::string(ev.text); + } + + if (ev.rule.find("arg-string") != std::string::npos && ev.ending() && ev.success()) { + auto & tc = semantics.tool_calls.back(); + tc.arguments += "\""; + } + + if (ev.rule == "arg-json-content" && ev.ending() && (ev.success() || ev.need_more_input())) { + auto & tc = semantics.tool_calls.back(); + tc.arguments += std::string(ev.text); + } +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7537b12f5800d..c83fc844af3b1 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -189,6 +189,8 @@ llama_build_and_test( chat-peg-parser/simple_tokenizer.cpp chat-peg-parser/test-command7-parser-compare.cpp chat-peg-parser/test-example-qwen3-coder.cpp + chat-peg-parser/test-example-minimax-m2.cpp + chat-peg-parser/test-example-seed-oss.cpp chat-peg-parser/test-gbnf-generation.cpp chat-peg-parser/test-json-parser.cpp chat-peg-parser/test-one.cpp diff --git a/tests/chat-peg-parser/convo.json b/tests/chat-peg-parser/convo.json new file mode 100644 index 0000000000000..d165fdb1a9ebd --- /dev/null +++ b/tests/chat-peg-parser/convo.json @@ -0,0 +1,159 @@ +{ + "model": "testing", + "messages": [ + { + "role": "system", + "content": "You are an assistant that can retrieve information using tools and provide concise, friendly summaries." + }, + { + "role": "user", + "content": "Please give me a summary of the latest news about renewable energy, the current weather in Berlin, and a random joke." + }, + + { + "role": "assistant", + "reasoning_content": "I need to fetch recent news about renewable energy. I'll call the `fetch_news` tool first." + }, + + { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "fetch_news", + "arguments": {"query":"renewable energy","language":"en","region":"global","max_results":5} + } + } + ] + }, + + { + "role": "assistant", + "reasoning_content": "Next I need the current weather for Berlin. I'll call the `get_weather` tool." + }, + + { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_2", + "type": "function", + "function": { + "name": "get_weather", + "arguments":{"location":"Berlin","units":"metric"} + } + } + ] + }, + + { + "role": "user", + "content": "Thanks! Also, can you give me a fun fact about cats?" + }, + + { + "role": "assistant", + "reasoning_content": "To keep the reply light I’ll fetch a random joke using the `generate_joke` tool." + }, + + { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_3", + "type": "function", + "function": { + "name": "generate_joke", + "arguments": {"category":"funny"} + } + } + ] + }, + + { + "role": "assistant", + "content": "**Renewable Energy News Summary**\n- *[Placeholder for news summary from `fetch_news`]*\n\n**Current Weather in Berlin**\n- *[Placeholder for weather data from `get_weather`]*\n\n**Joke**\n- *[Placeholder for joke from `generate_joke`]*\n\nI used three tool calls (β€―`fetch_news`,β€―`get_weather`,β€―`generate_joke`β€―) to gather this information." + } + ], + + "tools": [ + { + "type": "function", + "function": { + "name": "fetch_news", + "description": "Retrieve recent news articles based on a query.", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Search term (e.g., \"renewable energy\")." + }, + "language": { + "type": "string", + "description": "Two‑letter language code, e.g., \"en\"." + }, + "region": { + "type": "string", + "description": "Region or country code, e.g., \"global\"." + }, + "max_results": { + "type": "integer", + "description": "Maximum number of articles to return." + }, + "date": { + "type": "string", + "format": "date", + "description": "Specific date for news in ISO format (optional)." + } + }, + "required": ["query", "language", "region", "max_results"] + } + } + }, + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a location.", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "City name or geographic coordinates." + }, + "units": { + "type": "string", + "enum": ["metric", "imperial"], + "description": "Units for temperature." + } + }, + "required": ["location", "units"] + } + } + }, + { + "type": "function", + "function": { + "name": "generate_joke", + "description": "Generate a random joke.", + "parameters": { + "type": "object", + "properties": { + "category": { + "type": "string", + "description": "Optional joke category (e.g., \"funny\", \"dad\")." + } + }, + "required": [] + } + } + } + ] +} diff --git a/tests/chat-peg-parser/test-example-minimax-m2.cpp b/tests/chat-peg-parser/test-example-minimax-m2.cpp new file mode 100644 index 0000000000000..753ff51e6608c --- /dev/null +++ b/tests/chat-peg-parser/test-example-minimax-m2.cpp @@ -0,0 +1,68 @@ +#include "chat-peg-parser.h" +#include "nlohmann/json.hpp" +#include "tests.h" + +#include +#include + +void test_example_minimax_m2(testing &t) { + auto helper_parser = build_peg_parser_helper([](common_chat_peg_parser_builder_helper & p) { + auto thinking = p.reasoning(); + auto content = p.content_before_tools(""); + auto function = p.quasi_xml_attr("get_weather", + std::vector({ + "location", "units" + })); + auto tool_call = p.trigger(p.add_rule("tool-call", + "" + p.one_or_more(function) + "")); + + return thinking + p.optional(p.space() + content) + p.zero_or_more(p.space() + tool_call); + }); + + + t.test("minimax_m2_accumulation_test", [&](testing &t) { + std::string input = + "" + "To keep the reply light I’ll fetch a random joke using the `generate_joke` tool." + "" + "" + "" + "" + "" + "funny" + ""; + + std::vector tokens = simple_tokenize(input); + + common_chat_msg prev; + common_chat_parse_result last_result; + t.test("helper_builder", [&](testing &t) { + size_t token_cnt = 0; + for (auto it = tokens.begin(); it != tokens.end(); it++) { + token_cnt++; + std::string in = std::accumulate(tokens.begin(), it, std::string()); + + common_chat_parse_semantics semantics; + common_chat_parse_context ctx(in, &semantics, it == tokens.end() - 1); + + ctx.event_handler = parser_semantic_handler; + + auto result = helper_parser.parse(ctx); + if (result.fail()) { + break; + } + + last_result = result; + // This shouldn't emit any runtime errors + auto msg = semantics.to_msg(); + auto diffs = common_chat_msg_diff::compute_diffs(prev, msg); + prev = msg; + } + t.assert_true("last_result_should_be_success", last_result.success()); + + std::cout << "Final message:\n" << prev.to_json_oaicompat().dump(); + + t.assert_equal("should_parse_all_tokens_helper", tokens.size(), token_cnt); + }); + }); +} diff --git a/tests/chat-peg-parser/test-example-qwen3-coder.cpp b/tests/chat-peg-parser/test-example-qwen3-coder.cpp index 83f61993cd409..112a796c13137 100644 --- a/tests/chat-peg-parser/test-example-qwen3-coder.cpp +++ b/tests/chat-peg-parser/test-example-qwen3-coder.cpp @@ -48,48 +48,6 @@ void test_example_qwen3_coder(testing &t) { return thinking + p.optional(p.space() + content) + p.zero_or_more(p.space() + tool_call); }); - auto handler = [&](const common_chat_parse_event & ev, common_chat_parse_semantics & semantics) { - if (ev.rule == "reasoning-content" && ev.ending()) { - semantics.reasoning_content = ev.text; - } - - if (ev.rule == "content" && ev.ending()) { - semantics.content = ev.text; - } - - if (ev.rule == "function-start" && ev.ending() && ev.success()) { - semantics.tool_calls.emplace_back(); - auto & tc = semantics.tool_calls.back(); - tc.name = semantics.captures["tool-name"]; - } - - if (ev.rule == "arg-start" && ev.ending() && ev.success()) { - auto & tc = semantics.tool_calls.back(); - auto name = semantics.captures["arg-name"]; - if (tc.arguments.empty()) { - tc.arguments += "{"; - } else { - tc.arguments += ", "; - } - tc.arguments += "\"" + name + "\": "; - } - - if (ev.rule == "arg-string-content" && ev.ending() && ev.success()) { - auto & tc = semantics.tool_calls.back(); - tc.arguments += "\"" + std::string(ev.text); - } - - if (ev.rule == "arg-string" && ev.ending() && ev.success()) { - auto & tc = semantics.tool_calls.back(); - tc.arguments += "\""; - } - - if (ev.rule == "arg-json-content" && ev.ending() && (ev.success() || ev.need_more_input())) { - auto & tc = semantics.tool_calls.back(); - tc.arguments += std::string(ev.text); - } - }; - t.test("qwen3_accumulation_test", [&](testing &t) { std::string input = "The user wants to find large log files that haven't been accessed recently. " @@ -123,7 +81,7 @@ void test_example_qwen3_coder(testing &t) { common_chat_parse_semantics semantics; common_chat_parse_context ctx(in, &semantics, it == tokens.end() - 1); - ctx.event_handler = handler; + ctx.event_handler = parser_semantic_handler; auto result = explicit_parser.parse(ctx); if (result.fail()) { @@ -148,7 +106,7 @@ void test_example_qwen3_coder(testing &t) { common_chat_parse_semantics semantics; common_chat_parse_context ctx(in, &semantics, it == tokens.end() - 1); - ctx.event_handler = handler; + ctx.event_handler = parser_semantic_handler; auto result = helper_parser.parse(ctx); if (result.fail()) { diff --git a/tests/chat-peg-parser/test-example-seed-oss.cpp b/tests/chat-peg-parser/test-example-seed-oss.cpp new file mode 100644 index 0000000000000..c9f47ec55b41e --- /dev/null +++ b/tests/chat-peg-parser/test-example-seed-oss.cpp @@ -0,0 +1,57 @@ +#include "tests.h" + +#include +#include + +void test_example_seed_oss(testing &t) { + auto helper_parser = build_peg_parser_helper([](common_chat_peg_parser_builder_helper & p) { + auto thinking = p.reasoning("seed:think"); + auto content = p.content_before_tools(""); + auto function = p.quasi_xml_no_attr("get_weather", + std::vector({ + "location", "units" + })); + auto tool_call = p.trigger(p.add_rule("tool-call", + "" + p.one_or_more(function) + "")); + + return thinking + p.optional(p.space() + content) + p.zero_or_more(p.space() + tool_call); + }); + + + t.test("seed_oss_accumulation_test", [&](testing &t) { + std::string input = + "Next I need the current weather for Berlin. I'll call the `get_weather` tool.assistant" + "" + "" + "Berlin" + "metric" + "" + ""; + std::vector tokens = simple_tokenize(input); + + common_chat_msg prev; + t.test("helper_builder", [&](testing &t) { + size_t token_cnt = 0; + for (auto it = tokens.begin(); it != tokens.end(); it++) { + token_cnt++; + std::string in = std::accumulate(tokens.begin(), it, std::string()); + + common_chat_parse_semantics semantics; + common_chat_parse_context ctx(in, &semantics, it == tokens.end() - 1); + + ctx.event_handler = parser_semantic_handler; + + auto result = helper_parser.parse(ctx); + if (result.fail()) { + break; + } + + // This shouldn't emit any runtime errors + auto msg = semantics.to_msg(); + auto diffs = common_chat_msg_diff::compute_diffs(prev, msg); + prev = msg; + } + t.assert_equal("should_parse_all_tokens_helper", tokens.size(), token_cnt); + }); + }); +} diff --git a/tests/chat-peg-parser/tests.h b/tests/chat-peg-parser/tests.h index 7de49b4384567..489da8944eb59 100644 --- a/tests/chat-peg-parser/tests.h +++ b/tests/chat-peg-parser/tests.h @@ -25,5 +25,7 @@ void test_recursive_references(testing &t); void test_json_parser(testing &t); void test_gbnf_generation(testing &t); void test_example_qwen3_coder(testing &t); +void test_example_seed_oss(testing &t); +void test_example_minimax_m2(testing &t); void test_command7_parser_compare(testing &t); void test_unicode(testing &t); diff --git a/tests/test-chat-peg-parser.cpp b/tests/test-chat-peg-parser.cpp index 1595031827142..9e51b9e2dfe38 100644 --- a/tests/test-chat-peg-parser.cpp +++ b/tests/test-chat-peg-parser.cpp @@ -11,6 +11,8 @@ int main() { test_json_parser(t); test_gbnf_generation(t); test_example_qwen3_coder(t); + test_example_seed_oss(t); + test_example_minimax_m2(t); test_command7_parser_compare(t); return t.summary(); From bbcf1f605e2d1888a1930f20f8f5712a33c0734a Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Sun, 16 Nov 2025 14:46:00 +0100 Subject: [PATCH 105/183] Add some extra optional debugging prints + real example of how to use them --- common/chat-peg-parser-helper.h | 50 ++++++++++++++++++ common/chat-peg-parser.cpp | 4 ++ .../test-example-minimax-m2.cpp | 51 ++++++++++++++----- 3 files changed, 93 insertions(+), 12 deletions(-) diff --git a/common/chat-peg-parser-helper.h b/common/chat-peg-parser-helper.h index d288dcc19962a..a795a41bffedb 100644 --- a/common/chat-peg-parser-helper.h +++ b/common/chat-peg-parser-helper.h @@ -1,4 +1,6 @@ #include "chat-peg-parser.h" +#include "log.h" +#include class common_chat_peg_parser_builder_helper : public common_chat_peg_parser_builder { @@ -66,3 +68,51 @@ inline void parser_semantic_handler(const common_chat_parse_event & ev, common_c tc.arguments += std::string(ev.text); } } + +inline void parser_semantic_handler_with_printout(const common_chat_parse_event & ev, common_chat_parse_semantics & semantics) { + LOG_ERR("\n===============\nEvent type: %s\n", (ev.type == COMMON_CHAT_PARSE_EVENT_NODE_START ? "START" : "END")); + LOG_ERR("Event rule: %s\nEvent text: %s\nEvent status: %s\n", ev.rule.c_str(), ev.text.data(), (ev.status == COMMON_CHAT_PARSE_RESULT_SUCCESS ? "SUCCESS" : (ev.status == COMMON_CHAT_PARSE_RESULT_FAIL ? "FAIL" : "NEED_MORE_INPUT"))); + + if (ev.rule == "reasoning-content" && ev.ending()) { + semantics.reasoning_content = ev.text; + } + + if (ev.rule == "content" && ev.ending()) { + semantics.content = ev.text; + } + + if (ev.rule.find("function-start") != std::string::npos && ev.ending() && ev.success()) { + semantics.tool_calls.emplace_back(); + auto & tc = semantics.tool_calls.back(); + tc.name = semantics.captures["tool-name"]; + } + + if (ev.rule.find("arg-start") != std::string::npos && ev.ending() && ev.success()) { + auto & tc = semantics.tool_calls.back(); + auto name = semantics.captures["arg-name"]; + if (tc.arguments.empty()) { + tc.arguments += "{"; + } else { + tc.arguments += ", "; + } + tc.arguments += "\"" + name + "\": "; + } + + if (ev.rule == "arg-string-content" && ev.ending() && ev.success()) { + auto & tc = semantics.tool_calls.back(); + tc.arguments += "\"" + std::string(ev.text); + } + + if (ev.rule.find("arg-string") != std::string::npos && ev.ending() && ev.success()) { + auto & tc = semantics.tool_calls.back(); + tc.arguments += "\""; + } + + if (ev.rule == "arg-json-content" && ev.ending() && (ev.success() || ev.need_more_input())) { + auto & tc = semantics.tool_calls.back(); + tc.arguments += std::string(ev.text); + } + + LOG_ERR("Content: %s\nReasoning: %s\nTool calls: %lu\n", semantics.content.c_str(), semantics.reasoning_content.c_str(), semantics.tool_calls.size()); +} + diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 57e71dd2defe3..11c592e436793 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -61,17 +61,21 @@ class common_chat_peg_parser_base { virtual parser_type type() const = 0; virtual common_chat_parse_result parse(common_chat_parse_context & ctx, size_t start = 0) { + LOG_DBG("[CCPP type %d] Trying to parse: %s\n", type(), ctx.input.substr(start).c_str()); if (id_ == -1) { // Don't cache parsers with ID -1 (from operators) + LOG_DBG("[CCPP type %d] Parsing uncached due to operator\n", type()); return parse_uncached(ctx, start); } auto cached = ctx.cache.get(id_, start); if (cached) { + LOG_DBG("[CCPP type %d] Found cached result, returning\n", type()); return *cached; } auto result = parse_uncached(ctx, start); + LOG_DBG("[CCPP type %d] Parse result is: %s\n", type(), result.type == COMMON_CHAT_PARSE_RESULT_FAIL ? "FAIL" : (result.type == COMMON_CHAT_PARSE_RESULT_SUCCESS ? "SUCCESS" : "NEED_MORE_INPUT")); return ctx.cache.set(id_, start, result); } diff --git a/tests/chat-peg-parser/test-example-minimax-m2.cpp b/tests/chat-peg-parser/test-example-minimax-m2.cpp index 753ff51e6608c..9848965e5b13d 100644 --- a/tests/chat-peg-parser/test-example-minimax-m2.cpp +++ b/tests/chat-peg-parser/test-example-minimax-m2.cpp @@ -1,20 +1,43 @@ #include "chat-peg-parser.h" +#include "ggml.h" +#include "log.h" #include "nlohmann/json.hpp" #include "tests.h" +#include #include +#include #include +static inline std::string join(const std::vector& parts, + const std::string& sep = ", ") { + if (parts.empty()) { return {}; } + + // Reserve an approximate size to avoid many reallocations. + std::size_t total_len = sep.size() * (parts.size() - 1); + for (const auto& s : parts) { total_len += s.size(); } + + std::string result; + result.reserve(total_len); + result += parts[0]; + + for (std::size_t i = 1; i < parts.size(); ++i) { + result += sep; + result += parts[i]; + } + return result; +} + void test_example_minimax_m2(testing &t) { auto helper_parser = build_peg_parser_helper([](common_chat_peg_parser_builder_helper & p) { auto thinking = p.reasoning(); auto content = p.content_before_tools(""); - auto function = p.quasi_xml_attr("get_weather", + auto function = p.quasi_xml_attr("generate_joke", std::vector({ - "location", "units" + "category" })); auto tool_call = p.trigger(p.add_rule("tool-call", - "" + p.one_or_more(function) + "")); + "" + p.one_or_more(function) + "")); return thinking + p.optional(p.space() + content) + p.zero_or_more(p.space() + tool_call); }); @@ -25,14 +48,13 @@ void test_example_minimax_m2(testing &t) { "" "To keep the reply light I’ll fetch a random joke using the `generate_joke` tool." "" - "" - "" "" "" "funny" ""; std::vector tokens = simple_tokenize(input); + LOG_ERR("Tokens: %s\n", join(tokens).c_str()); common_chat_msg prev; common_chat_parse_result last_result; @@ -40,29 +62,34 @@ void test_example_minimax_m2(testing &t) { size_t token_cnt = 0; for (auto it = tokens.begin(); it != tokens.end(); it++) { token_cnt++; - std::string in = std::accumulate(tokens.begin(), it, std::string()); + std::string in = std::accumulate(tokens.begin(), it + 1, std::string()); + LOG_ERR("Current input: %s\n", in.c_str()); common_chat_parse_semantics semantics; - common_chat_parse_context ctx(in, &semantics, it == tokens.end() - 1); + common_chat_parse_context ctx(in, &semantics, it + 1 == tokens.end()); - ctx.event_handler = parser_semantic_handler; + if (it + 1 == tokens.end()) { + common_log_set_verbosity_thold(LOG_DEFAULT_DEBUG); + } + + ctx.event_handler = it + 1 == tokens.end() ? parser_semantic_handler_with_printout : parser_semantic_handler; auto result = helper_parser.parse(ctx); + last_result = result; if (result.fail()) { + LOG_ERR("Parsing failure!"); break; } - last_result = result; // This shouldn't emit any runtime errors auto msg = semantics.to_msg(); auto diffs = common_chat_msg_diff::compute_diffs(prev, msg); prev = msg; } + LOG_ERR("Last message: %s\n", prev.to_json_oaicompat().dump().c_str()); t.assert_true("last_result_should_be_success", last_result.success()); - - std::cout << "Final message:\n" << prev.to_json_oaicompat().dump(); - t.assert_equal("should_parse_all_tokens_helper", tokens.size(), token_cnt); }); + common_log_set_verbosity_thold(LOG_DEFAULT_LLAMA); }); } From 8b1c3061890c2ea7f85cd55f45692c752df7169b Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 11:08:46 -0600 Subject: [PATCH 106/183] fix bug in repetitions when min_count = 0 reports failures --- common/chat-peg-parser.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 11c592e436793..cf85aff625c7a 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -550,7 +550,7 @@ class repetition_parser : public common_chat_peg_parser_base { } // Check if we got enough matches - if (match_count < min_count_) { + if (min_count_ > 0 && match_count < min_count_) { return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start, pos); } From b890bc724b54959458ac9041e81a430103d03776 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 11:08:59 -0600 Subject: [PATCH 107/183] dump rule in debug --- common/chat-peg-parser.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index cf85aff625c7a..70174614bba11 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -61,6 +61,7 @@ class common_chat_peg_parser_base { virtual parser_type type() const = 0; virtual common_chat_parse_result parse(common_chat_parse_context & ctx, size_t start = 0) { + LOG_DBG("[CCPP type %d] Rule: %s", type(), dump().c_str()); LOG_DBG("[CCPP type %d] Trying to parse: %s\n", type(), ctx.input.substr(start).c_str()); if (id_ == -1) { // Don't cache parsers with ID -1 (from operators) @@ -75,7 +76,7 @@ class common_chat_peg_parser_base { } auto result = parse_uncached(ctx, start); - LOG_DBG("[CCPP type %d] Parse result is: %s\n", type(), result.type == COMMON_CHAT_PARSE_RESULT_FAIL ? "FAIL" : (result.type == COMMON_CHAT_PARSE_RESULT_SUCCESS ? "SUCCESS" : "NEED_MORE_INPUT")); + LOG_DBG("[CCPP type %d] Parse result is: %s\n", type(), common_chat_parse_result_type_name(result.type)); return ctx.cache.set(id_, start, result); } From c54cac7cadc408b292dd54bbcda3b1220eba09a9 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 11:11:07 -0600 Subject: [PATCH 108/183] fix token accumulation and assert parsing never fails --- .../test-example-minimax-m2.cpp | 8 +------- .../test-example-qwen3-coder.cpp | 19 ++++++------------- .../chat-peg-parser/test-example-seed-oss.cpp | 9 ++------- tests/chat-peg-parser/test_harness.h | 16 ++++++++++------ 4 files changed, 19 insertions(+), 33 deletions(-) diff --git a/tests/chat-peg-parser/test-example-minimax-m2.cpp b/tests/chat-peg-parser/test-example-minimax-m2.cpp index 9848965e5b13d..b880acfe05eab 100644 --- a/tests/chat-peg-parser/test-example-minimax-m2.cpp +++ b/tests/chat-peg-parser/test-example-minimax-m2.cpp @@ -59,9 +59,7 @@ void test_example_minimax_m2(testing &t) { common_chat_msg prev; common_chat_parse_result last_result; t.test("helper_builder", [&](testing &t) { - size_t token_cnt = 0; for (auto it = tokens.begin(); it != tokens.end(); it++) { - token_cnt++; std::string in = std::accumulate(tokens.begin(), it + 1, std::string()); LOG_ERR("Current input: %s\n", in.c_str()); @@ -76,10 +74,7 @@ void test_example_minimax_m2(testing &t) { auto result = helper_parser.parse(ctx); last_result = result; - if (result.fail()) { - LOG_ERR("Parsing failure!"); - break; - } + t.assert_equal("not fail", false, result.fail()); // This shouldn't emit any runtime errors auto msg = semantics.to_msg(); @@ -88,7 +83,6 @@ void test_example_minimax_m2(testing &t) { } LOG_ERR("Last message: %s\n", prev.to_json_oaicompat().dump().c_str()); t.assert_true("last_result_should_be_success", last_result.success()); - t.assert_equal("should_parse_all_tokens_helper", tokens.size(), token_cnt); }); common_log_set_verbosity_thold(LOG_DEFAULT_LLAMA); }); diff --git a/tests/chat-peg-parser/test-example-qwen3-coder.cpp b/tests/chat-peg-parser/test-example-qwen3-coder.cpp index 112a796c13137..f4687175142b9 100644 --- a/tests/chat-peg-parser/test-example-qwen3-coder.cpp +++ b/tests/chat-peg-parser/test-example-qwen3-coder.cpp @@ -74,9 +74,8 @@ void test_example_qwen3_coder(testing &t) { common_chat_msg prev; t.test("explicit_builder", [&](testing &t) { - size_t token_cnt = 0; for (auto it = tokens.begin(); it != tokens.end(); it++) { - std::string in = std::accumulate(tokens.begin(), it, std::string()); + std::string in = std::accumulate(tokens.begin(), it + 1, std::string()); common_chat_parse_semantics semantics; common_chat_parse_context ctx(in, &semantics, it == tokens.end() - 1); @@ -84,24 +83,21 @@ void test_example_qwen3_coder(testing &t) { ctx.event_handler = parser_semantic_handler; auto result = explicit_parser.parse(ctx); - if (result.fail()) { - break; + if (!t.assert_equal("not fail", false, result.fail())) { + t.indent(); + t.out << in.substr(0, result.end) << "[failed-->]" << in.substr(result.end) << "\n"; } - token_cnt++; // This shouldn't emit any runtime errors auto msg = semantics.to_msg(); auto diffs = common_chat_msg_diff::compute_diffs(prev, msg); prev = msg; } - t.assert_equal("should_parse_all_tokens_explicit", tokens.size(), token_cnt); }); t.test("helper_builder", [&](testing &t) { - size_t token_cnt = 0; for (auto it = tokens.begin(); it != tokens.end(); it++) { - token_cnt++; - std::string in = std::accumulate(tokens.begin(), it, std::string()); + std::string in = std::accumulate(tokens.begin(), it + 1, std::string()); common_chat_parse_semantics semantics; common_chat_parse_context ctx(in, &semantics, it == tokens.end() - 1); @@ -109,16 +105,13 @@ void test_example_qwen3_coder(testing &t) { ctx.event_handler = parser_semantic_handler; auto result = helper_parser.parse(ctx); - if (result.fail()) { - break; - } + t.assert_equal("not fail", false, result.fail()); // This shouldn't emit any runtime errors auto msg = semantics.to_msg(); auto diffs = common_chat_msg_diff::compute_diffs(prev, msg); prev = msg; } - t.assert_equal("should_parse_all_tokens_helper", tokens.size(), token_cnt); }); }); } diff --git a/tests/chat-peg-parser/test-example-seed-oss.cpp b/tests/chat-peg-parser/test-example-seed-oss.cpp index c9f47ec55b41e..a0aa9d5493c37 100644 --- a/tests/chat-peg-parser/test-example-seed-oss.cpp +++ b/tests/chat-peg-parser/test-example-seed-oss.cpp @@ -31,10 +31,8 @@ void test_example_seed_oss(testing &t) { common_chat_msg prev; t.test("helper_builder", [&](testing &t) { - size_t token_cnt = 0; for (auto it = tokens.begin(); it != tokens.end(); it++) { - token_cnt++; - std::string in = std::accumulate(tokens.begin(), it, std::string()); + std::string in = std::accumulate(tokens.begin(), it + 1, std::string()); common_chat_parse_semantics semantics; common_chat_parse_context ctx(in, &semantics, it == tokens.end() - 1); @@ -42,16 +40,13 @@ void test_example_seed_oss(testing &t) { ctx.event_handler = parser_semantic_handler; auto result = helper_parser.parse(ctx); - if (result.fail()) { - break; - } + t.assert_equal("not fail", false, result.fail()); // This shouldn't emit any runtime errors auto msg = semantics.to_msg(); auto diffs = common_chat_msg_diff::compute_diffs(prev, msg); prev = msg; } - t.assert_equal("should_parse_all_tokens_helper", tokens.size(), token_cnt); }); }); } diff --git a/tests/chat-peg-parser/test_harness.h b/tests/chat-peg-parser/test_harness.h index 9ba56040221f3..3dc63800b2cfb 100644 --- a/tests/chat-peg-parser/test_harness.h +++ b/tests/chat-peg-parser/test_harness.h @@ -127,11 +127,11 @@ struct testing { } // Assertions - void assert_true(bool cond) { - assert_true("", cond); + bool assert_true(bool cond) { + return assert_true("", cond); } - void assert_true(const std::string &msg, bool cond) { + bool assert_true(const std::string &msg, bool cond) { ++assertions; if (!cond) { ++failures; @@ -141,16 +141,18 @@ struct testing { out << " : " << msg; } out << "\n"; + return false; } + return true; } template - void assert_equal(const A & expected, const B & actual) { - assert_equal("", expected, actual); + bool assert_equal(const A & expected, const B & actual) { + return assert_equal("", expected, actual); } template - void assert_equal(const std::string & msg, const A & expected, const B & actual) { + bool assert_equal(const std::string & msg, const A & expected, const B & actual) { ++assertions; if (!(actual == expected)) { ++failures; @@ -165,7 +167,9 @@ struct testing { out << " expected: " << expected << "\n"; indent(); out << " actual : " << actual << "\n"; + return false; } + return true; } // Print summary and return an exit code From 8756a3e98da95da9de568d9e9deeae3900b4c6f8 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 13:44:22 -0600 Subject: [PATCH 109/183] indent debug by depth --- common/chat-peg-parser.cpp | 55 +++++++++++++++++++++++++++++--------- common/chat-peg-parser.h | 13 ++++----- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 70174614bba11..56b9094dce6c9 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -45,6 +45,33 @@ const char * common_chat_parse_result_type_name(common_chat_parse_result_type ty } } +static const char * common_chat_parser_type_name(parser_type type) { + switch (type) { + case START: return "start"; + case END: return "end"; + case LITERAL: return "literal"; + case SEQUENCE: return "sequence"; + case CHOICE: return "choice"; + case REPETITION: return "repetition"; + case OPTIONAL: return "optional"; + case ZERO_OR_MORE: return "zero_or_more"; + case ONE_OR_MORE: return "one_or_more"; + case AND: return "and"; + case NOT: return "not"; + case ANY: return "any"; + case CHARS: return "chars"; + case RULE: return "rule"; + case UNTIL: return "until"; + case SPACE: return "space"; + case SCHEMA: return "schema"; + case ROOT: return "root"; + case JSON_STRING: return "json_string"; + case CAPTURE: return "capture"; + case TRIGGER: return "trigger"; + default: return "unknown"; + } +} + class parser_visitor; class common_chat_peg_parser_base { @@ -61,22 +88,28 @@ class common_chat_peg_parser_base { virtual parser_type type() const = 0; virtual common_chat_parse_result parse(common_chat_parse_context & ctx, size_t start = 0) { - LOG_DBG("[CCPP type %d] Rule: %s", type(), dump().c_str()); - LOG_DBG("[CCPP type %d] Trying to parse: %s\n", type(), ctx.input.substr(start).c_str()); + std::string indent(ctx.parse_depth * 2, ' '); + ctx.parse_depth++; + + LOG_DBG("%s[CCPP type %d] Rule: %s\n", indent.c_str(), type(), dump().c_str()); + LOG_DBG("%s[CCPP type %d] Trying to parse: %s\n", indent.c_str(), type(), ctx.input.substr(start).c_str()); if (id_ == -1) { // Don't cache parsers with ID -1 (from operators) - LOG_DBG("[CCPP type %d] Parsing uncached due to operator\n", type()); + LOG_DBG("%s[CCPP type %d] Parsing uncached due to operator\n", indent.c_str(), type()); + ctx.parse_depth--; return parse_uncached(ctx, start); } auto cached = ctx.cache.get(id_, start); if (cached) { - LOG_DBG("[CCPP type %d] Found cached result, returning\n", type()); + LOG_DBG("%s[CCPP type %d] Found cached result, returning\n", indent.c_str(), type()); + ctx.parse_depth--; return *cached; } auto result = parse_uncached(ctx, start); - LOG_DBG("[CCPP type %d] Parse result is: %s\n", type(), common_chat_parse_result_type_name(result.type)); + LOG_DBG("%s[CCPP type %d] Parse result is: %s\n", indent.c_str(), type(), common_chat_parse_result_type_name(result.type)); + ctx.parse_depth--; return ctx.cache.set(id_, start, result); } @@ -552,6 +585,9 @@ class repetition_parser : public common_chat_peg_parser_base { // Check if we got enough matches if (min_count_ > 0 && match_count < min_count_) { + if (pos >= ctx.input.size() && !ctx.input_is_complete) { + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start, pos); + } return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start, pos); } @@ -640,13 +676,8 @@ class and_parser : public common_chat_peg_parser_base { common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto result = parser_->parse(ctx, start); - if (result.success()) { - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start); - } - if (result.need_more_input()) { - return result; - } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start); + // Pass result but don't consume input + return common_chat_parse_result(result.type, start); } void assign_id(common_chat_peg_parser_counter & counter) override { diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index 76136e8654f42..483bca7d09e1d 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -112,24 +112,25 @@ struct common_chat_parse_context { common_chat_parse_event_handler event_handler; int current_depth; + int parse_depth; common_chat_parse_context() - : input_is_complete(true), cache(), semantics(nullptr), event_handler(nullptr), current_depth(0) {} + : input_is_complete(true), cache(), semantics(nullptr), event_handler(nullptr), current_depth(0), parse_depth(0) {} common_chat_parse_context(const std::string & input) - : input(input), input_is_complete(true), cache(), semantics(nullptr), event_handler(nullptr), current_depth(0) {} + : input(input), input_is_complete(true), cache(), semantics(nullptr), event_handler(nullptr), current_depth(0), parse_depth(0) {} common_chat_parse_context(const std::string & input, bool complete) - : input(input), input_is_complete(complete), cache(), semantics(nullptr), event_handler(nullptr), current_depth(0) {} + : input(input), input_is_complete(complete), cache(), semantics(nullptr), event_handler(nullptr), current_depth(0), parse_depth(0) {} common_chat_parse_context(const std::string & input, common_chat_parse_semantics * semantics) - : input(input), input_is_complete(true), cache(), semantics(semantics), event_handler(nullptr), current_depth(0) {} + : input(input), input_is_complete(true), cache(), semantics(semantics), event_handler(nullptr), current_depth(0), parse_depth(0) {} common_chat_parse_context(const std::string & input, common_chat_parse_semantics * semantics, bool complete) - : input(input), input_is_complete(complete), cache(), semantics(semantics), event_handler(nullptr), current_depth(0) {} + : input(input), input_is_complete(complete), cache(), semantics(semantics), event_handler(nullptr), current_depth(0), parse_depth(0) {} common_chat_parse_context(const std::string & input, common_chat_parse_semantics * semantics, common_chat_parse_event_handler handler, bool complete = true) - : input(input), input_is_complete(complete), cache(), semantics(semantics), event_handler(std::move(handler)), current_depth(0) {} + : input(input), input_is_complete(complete), cache(), semantics(semantics), event_handler(std::move(handler)), current_depth(0), parse_depth(0) {} }; class common_chat_peg_parser_base; From 1239e1035e93f3a43bd2d9dcda76e9cc0598b44d Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 13:47:12 -0600 Subject: [PATCH 110/183] use LOG_* in tests so logs sync up with test logs --- .../test-example-qwen3-coder.cpp | 52 +++++++----- tests/chat-peg-parser/test_harness.h | 83 +++++++++---------- tests/test-chat-peg-parser.cpp | 6 +- 3 files changed, 73 insertions(+), 68 deletions(-) diff --git a/tests/chat-peg-parser/test-example-qwen3-coder.cpp b/tests/chat-peg-parser/test-example-qwen3-coder.cpp index f4687175142b9..840913a8fc6fb 100644 --- a/tests/chat-peg-parser/test-example-qwen3-coder.cpp +++ b/tests/chat-peg-parser/test-example-qwen3-coder.cpp @@ -1,3 +1,4 @@ +#include "log.h" #include "tests.h" #include @@ -13,8 +14,10 @@ void test_example_qwen3_coder(testing &t) { auto arg_name = p.add_rule("arg-start", ""); auto arg_end = p.add_rule("arg-end", "" + p.peek(p.literal("")); - auto string_arg_content = p.add_rule("arg-string-content", - p.until_one_of({""})); + auto string_arg_content = p.add_rule("arg-string-content", p.until_one_of({ + "", + })); auto string_arg = p.add_rule("arg-string", arg_name + string_arg_content + arg_end); @@ -30,7 +33,7 @@ void test_example_qwen3_coder(testing &t) { auto tool_call = p.trigger(p.add_rule("tool-call", "" + p.one_or_more(function) + "")); - return thinking + p.optional(p.space() + content) + p.zero_or_more(p.space() + tool_call); + return thinking + p.optional(p.space() + content) + p.zero_or_more(p.space() + tool_call) + p.end(); }); @@ -55,22 +58,23 @@ void test_example_qwen3_coder(testing &t) { "and check access time within the last 30 days. I'll need to use the search_files function." "Based on your requirements, I'll search for log files over 100MB that haven't been " "accessed in the last month. This will help identify candidates for cleanup or archival.\n\n" - "\n" - "\n" - "/var/log\n" - "*.log\n" - "100\n" - "5\n" - "false\n" - "30\n" - "true\n" - "size\n" + "" + "" + "/var/log" + "*.log" + "100" + "5" + "false" + "30" + "true" + "size" "{\"exclude_patterns\": [\"*temp*\", \"*cache*\"], \"file_types\": " - "[\"regular\"]}\n" - "\n" + "[\"regular\"]}" + "" ""; std::vector tokens = simple_tokenize(input); + common_log_set_verbosity_thold(LOG_DEFAULT_DEBUG); common_chat_msg prev; t.test("explicit_builder", [&](testing &t) { @@ -84,14 +88,20 @@ void test_example_qwen3_coder(testing &t) { auto result = explicit_parser.parse(ctx); if (!t.assert_equal("not fail", false, result.fail())) { - t.indent(); - t.out << in.substr(0, result.end) << "[failed-->]" << in.substr(result.end) << "\n"; + LOG_ERR("%s[failed-->]%s\n", in.substr(0, result.end).c_str(), in.substr(result.end).c_str()); } - // This shouldn't emit any runtime errors - auto msg = semantics.to_msg(); - auto diffs = common_chat_msg_diff::compute_diffs(prev, msg); - prev = msg; + auto msg = semantics.to_msg(); + + try { + // This shouldn't emit any runtime errors + auto diffs = common_chat_msg_diff::compute_diffs(prev, msg); + } catch(const std::exception & e) { + LOG_ERR("%s[failed-->]%s\n", in.substr(0, result.end).c_str(), in.substr(result.end).c_str()); + t.assert_true(std::string("failed with ") + e.what(), false); + } + + prev = msg; } }); diff --git a/tests/chat-peg-parser/test_harness.h b/tests/chat-peg-parser/test_harness.h index 3dc63800b2cfb..ed74d33851c49 100644 --- a/tests/chat-peg-parser/test_harness.h +++ b/tests/chat-peg-parser/test_harness.h @@ -1,13 +1,14 @@ #pragma once +#include "log.h" + #include #include -#include +#include #include #include struct testing { - std::ostream &out; std::vector stack; int tests = 0; int assertions = 0; @@ -15,12 +16,8 @@ struct testing { int unnamed = 0; int exceptions = 0; - explicit testing(std::ostream &os = std::cout) : out(os) {} - - void indent() { - for (std::size_t i = 0; i < stack.size() - 1; ++i) { - out << " "; - } + std::string indent() { + return std::string((stack.size() - 1) * 2, ' '); } template @@ -30,29 +27,24 @@ struct testing { } catch (const std::exception &e) { ++failures; ++exceptions; - indent(); - out << "UNHANDLED EXCEPTION (" << ctx << "): " << e.what() << "\n"; + LOG_ERR("%sUNHANDLED EXCEPTION (%s): %s\n", indent().c_str(), ctx, e.what()); } catch (...) { ++failures; ++exceptions; - indent(); - out << "UNHANDLED EXCEPTION (" << ctx << "): unknown\n"; + LOG_ERR("%sUNHANDLED EXCEPTION (%s): unknown\n", indent().c_str(), ctx); } } void print_result(const std::string &label, const std::string &name, int new_failures, int new_assertions, const std::string &extra = "") { - indent(); - out << label << ": " << name << " ["; + std::string ind = indent(); + std::string status = (new_failures == 0) ? "ok" : (std::to_string(new_failures) + " failed of"); + std::string extra_str = extra.empty() ? "" : (", " + extra); + if (new_failures == 0) { - out << "ok, "; + LOG_INF("%s%s: %s [ok, %d assertion(s)%s]\n", ind.c_str(), label.c_str(), name.c_str(), new_assertions, extra_str.c_str()); } else { - out << new_failures << " failed of "; + LOG_ERR("%s%s: %s [%d failed of %d assertion(s)%s]\n", ind.c_str(), label.c_str(), name.c_str(), new_failures, new_assertions, extra_str.c_str()); } - out << new_assertions << " assertion(s)"; - if (!extra.empty()) { - out << ", " << extra; - } - out << "]\n"; } // Named test @@ -61,8 +53,7 @@ struct testing { ++tests; stack.push_back(name); - indent(); - out << "BEGIN: " << name << "\n"; + LOG_INF("%sBEGIN: %s\n", indent().c_str(), name.c_str()); int before_failures = failures; int before_assertions = assertions; @@ -88,8 +79,7 @@ struct testing { ++tests; stack.push_back(name); - indent(); - out << "BEGIN BENCH: " << name << "\n"; + LOG_INF("%sBEGIN BENCH: %s\n", indent().c_str(), name.c_str()); int before_failures = failures; int before_assertions = assertions; @@ -135,12 +125,11 @@ struct testing { ++assertions; if (!cond) { ++failures; - indent(); - out << "ASSERT TRUE FAILED"; - if (!msg.empty()) { - out << " : " << msg; + if (msg.empty()) { + LOG_ERR("%sASSERT TRUE FAILED\n", indent().c_str()); + } else { + LOG_ERR("%sASSERT TRUE FAILED : %s\n", indent().c_str(), msg.c_str()); } - out << "\n"; return false; } return true; @@ -156,17 +145,19 @@ struct testing { ++assertions; if (!(actual == expected)) { ++failures; - indent(); - out << "ASSERT EQUAL FAILED"; - if (!msg.empty()) { - out << " : " << msg; - } - out << "\n"; + std::string ind = indent(); - indent(); - out << " expected: " << expected << "\n"; - indent(); - out << " actual : " << actual << "\n"; + std::ostringstream exp_ss, act_ss; + exp_ss << expected; + act_ss << actual; + + if (msg.empty()) { + LOG_ERR("%sASSERT EQUAL FAILED\n", ind.c_str()); + } else { + LOG_ERR("%sASSERT EQUAL FAILED : %s\n", ind.c_str(), msg.c_str()); + } + LOG_ERR("%s expected: %s\n", ind.c_str(), exp_ss.str().c_str()); + LOG_ERR("%s actual : %s\n", ind.c_str(), act_ss.str().c_str()); return false; } return true; @@ -174,12 +165,12 @@ struct testing { // Print summary and return an exit code int summary() { - out << "\n==== TEST SUMMARY ====\n"; - out << "tests : " << tests << "\n"; - out << "assertions : " << assertions << "\n"; - out << "failures : " << failures << "\n"; - out << "exceptions : " << exceptions << "\n"; - out << "======================\n"; + LOG_INF("\n==== TEST SUMMARY ====\n"); + LOG_INF("tests : %d\n", tests); + LOG_INF("assertions : %d\n", assertions); + LOG_INF("failures : %d\n", failures); + LOG_INF("exceptions : %d\n", exceptions); + LOG_INF("======================\n"); return failures == 0 ? 0 : 1; } }; diff --git a/tests/test-chat-peg-parser.cpp b/tests/test-chat-peg-parser.cpp index 9e51b9e2dfe38..b77d5f4afbb8d 100644 --- a/tests/test-chat-peg-parser.cpp +++ b/tests/test-chat-peg-parser.cpp @@ -1,7 +1,11 @@ +#include "log.h" + #include "chat-peg-parser/tests.h" int main() { - testing t(std::cout); + common_log_set_verbosity_thold(LOG_DEFAULT_DEBUG); + + testing t; test_partial_parsing(t); test_one(t); From 7d30b275ebe5a0b44ed2bb28c27fa741476708d6 Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Sun, 16 Nov 2025 21:37:29 +0100 Subject: [PATCH 111/183] - Add selective testing - Refactor all messaging to use LOG_ERR - Fix lack of argument / tool name capturing - Temporary fix for double event capture --- common/chat-peg-parser-helper.cpp | 32 ++++--- common/chat-peg-parser-helper.h | 7 +- .../test-example-minimax-m2.cpp | 3 - .../test-example-qwen3-coder.cpp | 6 +- .../chat-peg-parser/test-example-seed-oss.cpp | 4 +- tests/chat-peg-parser/test_harness.h | 55 ++++++----- tests/test-chat-peg-parser.cpp | 94 ++++++++++++++++--- 7 files changed, 144 insertions(+), 57 deletions(-) diff --git a/common/chat-peg-parser-helper.cpp b/common/chat-peg-parser-helper.cpp index e7b646f99f636..dc421336420af 100644 --- a/common/chat-peg-parser-helper.cpp +++ b/common/chat-peg-parser-helper.cpp @@ -25,9 +25,11 @@ common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_no_attr( arg_start_name.append("arg-start-").append(*it); std::string param_open; - param_open.append("<").append(param_tag).append("=").append(*it).append(">"); + param_open.append("<").append(param_tag).append("="); - auto arg_name = add_rule(arg_start_name, literal(param_open)); + std::string param_open_after_name = ">"; + + auto arg_name = add_rule(arg_start_name, literal(param_open) + capture("arg-name", *it) + literal(param_open_after_name)); std::string param_close_end; param_close_end.append(""); @@ -45,7 +47,7 @@ common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_no_attr( std::string string_content_2; string_content_2.append(""); - auto string_arg_content = add_rule("arg-string-content", until_one_of({ string_content_1, string_content_2 })); + auto string_arg_content = add_rule("arg-str-content", until_one_of({ string_content_1, string_content_2 })); std::string arg_string_name; arg_string_name.append("arg-string-").append(*it); @@ -64,14 +66,17 @@ common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_no_attr( function_start_name.append("function-start-").append(function_name); std::string function_open; - function_open.append("<").append(function_tag).append("=").append(function_name).append(">"); + function_open.append("<").append(function_tag).append("="); + + std::string function_open_after_name; + function_open_after_name = ">"; std::string function_close; function_close.append(""); std::string function_rule_name; function_rule_name.append("function-").append(function_name); - auto function = add_rule(function_rule_name, add_rule(function_start_name, function_open) + args_sequence + function_close); + auto function = add_rule(function_rule_name, add_rule(function_start_name, function_open + capture("tool-name", function_name) + function_open_after_name) + args_sequence + function_close); return function; } @@ -89,9 +94,11 @@ common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_attr( arg_start_name.append("arg-start-").append(*it); std::string param_open; - param_open.append("<").append(param_tag).append(" ").append(name_attr).append("=\"").append(*it).append("\">"); + param_open.append("<").append(param_tag).append(" ").append(name_attr).append("=\""); + + std::string param_open_after_name ="\">"; - auto arg_name = add_rule(arg_start_name, literal(param_open)); + auto arg_name = add_rule(arg_start_name, literal(param_open) + capture("arg-name", literal(*it)) + literal(param_open_after_name)); std::string param_close_end; param_close_end.append(""); @@ -109,7 +116,7 @@ common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_attr( std::string string_content_2; string_content_2.append(""); - auto string_arg_content = add_rule("arg-string-content", until_one_of({ string_content_1, string_content_2 })); + auto string_arg_content = add_rule("arg-str-content", until_one_of({ string_content_1, string_content_2 })); std::string arg_string_name; arg_string_name.append("arg-string-").append(*it); @@ -128,14 +135,18 @@ common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_attr( function_start_name.append("function-start-").append(function_name); std::string function_open; - function_open.append("<").append(function_tag).append(" ").append(name_attr).append("=\"").append(function_name).append("\">"); + function_open.append("<").append(function_tag).append(" ").append(name_attr).append("=\""); + + std::string function_open_after_name = "\">"; std::string function_close; function_close.append(""); std::string function_rule_name; function_rule_name.append("function-").append(function_name); - auto function = add_rule(function_rule_name, add_rule(function_start_name, function_open) + args_sequence + function_close); + auto function = add_rule(function_rule_name, + add_rule(function_start_name, function_open + capture("tool-name", literal(function_name)) + + function_open_after_name) + args_sequence + function_close); return function; } @@ -147,4 +158,3 @@ common_chat_peg_parser build_peg_parser_helper( builder.set_root(root); return builder.build(); } - diff --git a/common/chat-peg-parser-helper.h b/common/chat-peg-parser-helper.h index a795a41bffedb..e9751f2d27ed7 100644 --- a/common/chat-peg-parser-helper.h +++ b/common/chat-peg-parser-helper.h @@ -1,6 +1,5 @@ #include "chat-peg-parser.h" #include "log.h" -#include class common_chat_peg_parser_builder_helper : public common_chat_peg_parser_builder { @@ -53,7 +52,7 @@ inline void parser_semantic_handler(const common_chat_parse_event & ev, common_c tc.arguments += "\"" + name + "\": "; } - if (ev.rule == "arg-string-content" && ev.ending() && ev.success()) { + if (ev.rule == "arg-str-content" && ev.ending() && ev.success()) { auto & tc = semantics.tool_calls.back(); tc.arguments += "\"" + std::string(ev.text); } @@ -71,7 +70,7 @@ inline void parser_semantic_handler(const common_chat_parse_event & ev, common_c inline void parser_semantic_handler_with_printout(const common_chat_parse_event & ev, common_chat_parse_semantics & semantics) { LOG_ERR("\n===============\nEvent type: %s\n", (ev.type == COMMON_CHAT_PARSE_EVENT_NODE_START ? "START" : "END")); - LOG_ERR("Event rule: %s\nEvent text: %s\nEvent status: %s\n", ev.rule.c_str(), ev.text.data(), (ev.status == COMMON_CHAT_PARSE_RESULT_SUCCESS ? "SUCCESS" : (ev.status == COMMON_CHAT_PARSE_RESULT_FAIL ? "FAIL" : "NEED_MORE_INPUT"))); + LOG_ERR("Event rule: %s\nEvent text: %s\nEvent status: %s\n", ev.rule.c_str(), std::string(ev.text.data(), ev.text.size()).c_str(), (ev.status == COMMON_CHAT_PARSE_RESULT_SUCCESS ? "SUCCESS" : (ev.status == COMMON_CHAT_PARSE_RESULT_FAIL ? "FAIL" : "NEED_MORE_INPUT"))); if (ev.rule == "reasoning-content" && ev.ending()) { semantics.reasoning_content = ev.text; @@ -98,7 +97,7 @@ inline void parser_semantic_handler_with_printout(const common_chat_parse_event tc.arguments += "\"" + name + "\": "; } - if (ev.rule == "arg-string-content" && ev.ending() && ev.success()) { + if (ev.rule == "arg-str-content" && ev.ending() && ev.success()) { auto & tc = semantics.tool_calls.back(); tc.arguments += "\"" + std::string(ev.text); } diff --git a/tests/chat-peg-parser/test-example-minimax-m2.cpp b/tests/chat-peg-parser/test-example-minimax-m2.cpp index 9848965e5b13d..d627bddff4e76 100644 --- a/tests/chat-peg-parser/test-example-minimax-m2.cpp +++ b/tests/chat-peg-parser/test-example-minimax-m2.cpp @@ -1,12 +1,9 @@ #include "chat-peg-parser.h" -#include "ggml.h" #include "log.h" #include "nlohmann/json.hpp" #include "tests.h" -#include #include -#include #include static inline std::string join(const std::vector& parts, diff --git a/tests/chat-peg-parser/test-example-qwen3-coder.cpp b/tests/chat-peg-parser/test-example-qwen3-coder.cpp index 112a796c13137..0068dcd807407 100644 --- a/tests/chat-peg-parser/test-example-qwen3-coder.cpp +++ b/tests/chat-peg-parser/test-example-qwen3-coder.cpp @@ -13,7 +13,7 @@ void test_example_qwen3_coder(testing &t) { auto arg_name = p.add_rule("arg-start", ""); auto arg_end = p.add_rule("arg-end", "" + p.peek(p.literal("")); - auto string_arg_content = p.add_rule("arg-string-content", + auto string_arg_content = p.add_rule("arg-str-content", p.until_one_of({""})); auto string_arg = p.add_rule("arg-string", arg_name + string_arg_content + arg_end); @@ -101,10 +101,10 @@ void test_example_qwen3_coder(testing &t) { size_t token_cnt = 0; for (auto it = tokens.begin(); it != tokens.end(); it++) { token_cnt++; - std::string in = std::accumulate(tokens.begin(), it, std::string()); + std::string in = std::accumulate(tokens.begin(), it + 1, std::string()); common_chat_parse_semantics semantics; - common_chat_parse_context ctx(in, &semantics, it == tokens.end() - 1); + common_chat_parse_context ctx(in, &semantics, it == tokens.end()); ctx.event_handler = parser_semantic_handler; diff --git a/tests/chat-peg-parser/test-example-seed-oss.cpp b/tests/chat-peg-parser/test-example-seed-oss.cpp index c9f47ec55b41e..136da39cf2779 100644 --- a/tests/chat-peg-parser/test-example-seed-oss.cpp +++ b/tests/chat-peg-parser/test-example-seed-oss.cpp @@ -34,10 +34,10 @@ void test_example_seed_oss(testing &t) { size_t token_cnt = 0; for (auto it = tokens.begin(); it != tokens.end(); it++) { token_cnt++; - std::string in = std::accumulate(tokens.begin(), it, std::string()); + std::string in = std::accumulate(tokens.begin(), it + 1, std::string()); common_chat_parse_semantics semantics; - common_chat_parse_context ctx(in, &semantics, it == tokens.end() - 1); + common_chat_parse_context ctx(in, &semantics, it == tokens.end()); ctx.event_handler = parser_semantic_handler; diff --git a/tests/chat-peg-parser/test_harness.h b/tests/chat-peg-parser/test_harness.h index 9ba56040221f3..a1f51e49d9d10 100644 --- a/tests/chat-peg-parser/test_harness.h +++ b/tests/chat-peg-parser/test_harness.h @@ -1,8 +1,10 @@ #pragma once +#include "log.h" #include #include #include +#include #include #include @@ -17,9 +19,9 @@ struct testing { explicit testing(std::ostream &os = std::cout) : out(os) {} - void indent() { + void indent() const { for (std::size_t i = 0; i < stack.size() - 1; ++i) { - out << " "; + LOG_ERR(" "); } } @@ -40,19 +42,19 @@ struct testing { } } - void print_result(const std::string &label, const std::string &name, int new_failures, int new_assertions, const std::string &extra = "") { + void print_result(const std::string &label, const std::string &name, int new_failures, int new_assertions, const std::string &extra = "") const { indent(); - out << label << ": " << name << " ["; + LOG_ERR("%s: %s [", label.c_str(), name.c_str()); if (new_failures == 0) { - out << "ok, "; + LOG_ERR("ok, "); } else { - out << new_failures << " failed of "; + LOG_ERR("%d failed of ", new_failures); } - out << new_assertions << " assertion(s)"; + LOG_ERR("%d assertion(s)", new_assertions); if (!extra.empty()) { - out << ", " << extra; + LOG_ERR(", %s", extra.c_str()); } - out << "]\n"; + LOG_ERR("]\n"); } // Named test @@ -62,7 +64,7 @@ struct testing { stack.push_back(name); indent(); - out << "BEGIN: " << name << "\n"; + LOG_ERR("BEGIN: %s\n", name.c_str()); int before_failures = failures; int before_assertions = assertions; @@ -155,27 +157,36 @@ struct testing { if (!(actual == expected)) { ++failures; indent(); - out << "ASSERT EQUAL FAILED"; + LOG_ERR("ASSERT EQUAL FAILED"); if (!msg.empty()) { - out << " : " << msg; + LOG_ERR(" : %s", msg.c_str()); } - out << "\n"; + LOG_ERR("\n"); indent(); - out << " expected: " << expected << "\n"; + LOG_ERR(" expected: %s\n", to_string_convert(expected).c_str()); indent(); - out << " actual : " << actual << "\n"; + LOG_ERR(" actual : %s\n", to_string_convert(actual).c_str()); } } // Print summary and return an exit code - int summary() { - out << "\n==== TEST SUMMARY ====\n"; - out << "tests : " << tests << "\n"; - out << "assertions : " << assertions << "\n"; - out << "failures : " << failures << "\n"; - out << "exceptions : " << exceptions << "\n"; - out << "======================\n"; + int summary() const { + LOG_ERR("\n==== TEST SUMMARY ====\n"); + LOG_ERR("tests : %d\n", tests); + LOG_ERR("assertions : %d\n", assertions); + LOG_ERR("failures : %d\n", failures); + LOG_ERR("exceptions : %d\n", exceptions); + LOG_ERR("======================\n"); return failures == 0 ? 0 : 1; } + +private: + template + std::string to_string_convert(const T & value) const { + std::ostringstream oss; + oss << value; + return oss.str(); + } + }; diff --git a/tests/test-chat-peg-parser.cpp b/tests/test-chat-peg-parser.cpp index 9e51b9e2dfe38..948df1e4f11a4 100644 --- a/tests/test-chat-peg-parser.cpp +++ b/tests/test-chat-peg-parser.cpp @@ -1,19 +1,89 @@ #include "chat-peg-parser/tests.h" +#include "log.h" +#include +#include +#include +#include + +// Struct to hold test information +struct TestEntry { + std::string codename; + std::string function_name; + void (*test_func)(testing&); +}; + +// Dynamic list of all available tests +static const std::vector all_tests = { + {"partial", "test_partial_parsing", test_partial_parsing}, + {"one", "test_one", test_one}, + {"optional", "test_optional", test_optional}, + {"unicode", "test_unicode", test_unicode}, + {"recursive", "test_recursive_references", test_recursive_references}, + {"json", "test_json_parser", test_json_parser}, + {"gbnf", "test_gbnf_generation", test_gbnf_generation}, + {"qwen3_coder", "test_example_qwen3_coder", test_example_qwen3_coder}, + {"seed_oss", "test_example_seed_oss", test_example_seed_oss}, + {"minimax_m2", "test_example_minimax_m2", test_example_minimax_m2}, + {"command7_parser_compare", "test_command7_parser_compare", test_command7_parser_compare} +}; + +// Function to list all available tests +static void list_available_tests() { + LOG_ERR("Available tests:\n"); + for (const auto& test : all_tests) { + LOG_ERR(" %s", test.codename.c_str()); + // Format spacing for alignment + for (size_t i = test.codename.length(); i < 25; ++i) { + LOG_ERR(" "); + } + LOG_ERR("- %s\n", test.function_name.c_str()); + } + LOG_ERR("\n"); + LOG_ERR("Usage:\n"); + LOG_ERR(" test-chat-peg-parser # Run all tests\n"); + LOG_ERR(" test-chat-peg-parser test1 test2 # Run specific tests\n"); + LOG_ERR(" test-chat-peg-parser --tests # List available tests\n"); +} + +// Function to check if a codename matches the provided arguments +static bool should_run_test(const std::vector& args, const std::string& codename) { + // If no arguments provided, run all tests + if (args.size() <= 1) { + return true; + } + + // Check if codename matches any of the provided arguments + return std::find(args.begin() + 1, args.end(), codename) != args.end(); +} + +// Helper to run a test conditionally +static void run_test_conditionally(testing& t, const std::vector& args, + const std::string& codename, void (*test_func)(testing&)) { + if (should_run_test(args, codename)) { + test_func(t); + } +} + +int main(int argc, char *argv[]) { + // Convert argv to vector of strings for easier handling + std::vector args; + args.reserve(argc); + for (int i = 0; i < argc; ++i) { + args.push_back(argv[i]); + } + + // Special case: list available tests and exit + if (argc == 2 && args[1] == "--tests") { + list_available_tests(); + return 0; + } -int main() { testing t(std::cout); - test_partial_parsing(t); - test_one(t); - test_optional(t); - test_unicode(t); - test_recursive_references(t); - test_json_parser(t); - test_gbnf_generation(t); - test_example_qwen3_coder(t); - test_example_seed_oss(t); - test_example_minimax_m2(t); - test_command7_parser_compare(t); + // Dynamically process all tests from the data structure + for (const auto& test : all_tests) { + run_test_conditionally(t, args, test.codename, test.test_func); + } return t.summary(); } From 9e787d7d2c0861473ff263f500e0ed280f53e131 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 15:42:37 -0600 Subject: [PATCH 112/183] refactor rule() and introduce ref() --- common/chat-peg-parser-helper.cpp | 28 +- common/chat-peg-parser.cpp | 395 +++++++++++------- common/chat-peg-parser.h | 28 +- .../test-command7-parser-compare.cpp | 28 +- .../test-example-minimax-m2.cpp | 4 +- .../test-example-qwen3-coder.cpp | 28 +- .../chat-peg-parser/test-example-seed-oss.cpp | 4 +- .../chat-peg-parser/test-gbnf-generation.cpp | 2 +- .../test-recursive-references.cpp | 36 +- 9 files changed, 327 insertions(+), 226 deletions(-) diff --git a/common/chat-peg-parser-helper.cpp b/common/chat-peg-parser-helper.cpp index e7b646f99f636..a84d47b442dd5 100644 --- a/common/chat-peg-parser-helper.cpp +++ b/common/chat-peg-parser-helper.cpp @@ -6,11 +6,11 @@ common_chat_peg_parser common_chat_peg_parser_builder_helper::reasoning(const st open_tag.append("<").append(tag).append(">"); std::string close_tag; close_tag.append(""); - return add_rule("raw-reasoning", open_tag << add_rule("reasoning-content", until(close_tag)) << close_tag); + return rule("raw-reasoning", open_tag << rule("reasoning-content", until(close_tag)) << close_tag); } common_chat_peg_parser common_chat_peg_parser_builder_helper::content_before_tools(const std::string & tag) { - return add_rule("content", until(tag)); + return rule("content", until(tag)); } common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_no_attr( @@ -27,7 +27,7 @@ common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_no_attr( std::string param_open; param_open.append("<").append(param_tag).append("=").append(*it).append(">"); - auto arg_name = add_rule(arg_start_name, literal(param_open)); + auto arg_name = rule(arg_start_name, literal(param_open)); std::string param_close_end; param_close_end.append(""); @@ -37,7 +37,7 @@ common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_no_attr( std::string param_peek_open; param_peek_open.append("<").append(param_tag).append("="); - auto arg_end = add_rule("arg-end", param_close_end + peek(literal(param_peek_open) | param_close_peek)); + auto arg_end = rule("arg-end", param_close_end + peek(literal(param_peek_open) | param_close_peek)); std::string string_content_1; string_content_1.append("<").append(param_tag).append("="); @@ -45,16 +45,16 @@ common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_no_attr( std::string string_content_2; string_content_2.append(""); - auto string_arg_content = add_rule("arg-string-content", until_one_of({ string_content_1, string_content_2 })); + auto string_arg_content = rule("arg-string-content", until_one_of({ string_content_1, string_content_2 })); std::string arg_string_name; arg_string_name.append("arg-string-").append(*it); - auto string_arg = add_rule(arg_string_name, arg_name + string_arg_content + arg_end); + auto string_arg = rule(arg_string_name, arg_name + string_arg_content + arg_end); auto json_sec = json(); std::string arg_json_name; arg_json_name.append("arg-json-").append(*it); - auto json_arg = add_rule(arg_json_name, arg_name + add_rule("arg-json-content", json_sec) + arg_end); + auto json_arg = rule(arg_json_name, arg_name + rule("arg-json-content", json_sec) + arg_end); auto arg_json_or_string = one_or_more(json_arg | string_arg); args.push_back(arg_json_or_string); } @@ -71,7 +71,7 @@ common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_no_attr( std::string function_rule_name; function_rule_name.append("function-").append(function_name); - auto function = add_rule(function_rule_name, add_rule(function_start_name, function_open) + args_sequence + function_close); + auto function = rule(function_rule_name, rule(function_start_name, function_open) + args_sequence + function_close); return function; } @@ -91,7 +91,7 @@ common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_attr( std::string param_open; param_open.append("<").append(param_tag).append(" ").append(name_attr).append("=\"").append(*it).append("\">"); - auto arg_name = add_rule(arg_start_name, literal(param_open)); + auto arg_name = rule(arg_start_name, literal(param_open)); std::string param_close_end; param_close_end.append(""); @@ -101,7 +101,7 @@ common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_attr( std::string param_peek_open; param_peek_open.append("<").append(param_tag).append(" ").append(name_attr).append("=\""); - auto arg_end = add_rule("arg-end", param_close_end + peek(literal(param_peek_open) | param_close_peek)); + auto arg_end = rule("arg-end", param_close_end + peek(literal(param_peek_open) | param_close_peek)); std::string string_content_1; string_content_1.append("<").append(param_tag).append("="); @@ -109,16 +109,16 @@ common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_attr( std::string string_content_2; string_content_2.append(""); - auto string_arg_content = add_rule("arg-string-content", until_one_of({ string_content_1, string_content_2 })); + auto string_arg_content = rule("arg-string-content", until_one_of({ string_content_1, string_content_2 })); std::string arg_string_name; arg_string_name.append("arg-string-").append(*it); - auto string_arg = add_rule(arg_string_name, arg_name + string_arg_content + arg_end); + auto string_arg = rule(arg_string_name, arg_name + string_arg_content + arg_end); auto json_sec = json(); std::string arg_json_name; arg_json_name.append("arg-json-").append(*it); - auto json_arg = add_rule(arg_json_name, arg_name + add_rule("arg-json-content", json_sec) + arg_end); + auto json_arg = rule(arg_json_name, arg_name + rule("arg-json-content", json_sec) + arg_end); auto arg_json_or_string = one_or_more(json_arg | string_arg); args.push_back(arg_json_or_string); } @@ -135,7 +135,7 @@ common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_attr( std::string function_rule_name; function_rule_name.append("function-").append(function_name); - auto function = add_rule(function_rule_name, add_rule(function_start_name, function_open) + args_sequence + function_close); + auto function = rule(function_rule_name, rule(function_start_name, function_open) + args_sequence + function_close); return function; } diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 56b9094dce6c9..a11c45a270b71 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include enum parser_type { @@ -27,13 +28,13 @@ enum parser_type { ANY, CHARS, RULE, + REF, UNTIL, SPACE, SCHEMA, ROOT, JSON_STRING, CAPTURE, - TRIGGER, }; const char * common_chat_parse_result_type_name(common_chat_parse_result_type type) { @@ -61,13 +62,13 @@ static const char * common_chat_parser_type_name(parser_type type) { case ANY: return "any"; case CHARS: return "chars"; case RULE: return "rule"; + case REF: return "ref"; case UNTIL: return "until"; case SPACE: return "space"; case SCHEMA: return "schema"; case ROOT: return "root"; case JSON_STRING: return "json_string"; case CAPTURE: return "capture"; - case TRIGGER: return "trigger"; default: return "unknown"; } } @@ -327,7 +328,12 @@ class root_parser : public common_chat_peg_parser_base { void assign_id(common_chat_peg_parser_counter & counter) override { common_chat_peg_parser_base::assign_id(counter); - root_->assign_id(counter); + for (auto & [name, rule] : rules_) { + rule->assign_id(counter); + } + if (root_.ptr()) { + root_->assign_id(counter); + } } std::string dump() const override { @@ -1198,33 +1204,22 @@ class schema_parser : public common_chat_peg_parser_base { const nlohmann::ordered_json & schema() const { return schema_; } }; -// References a named rule for recursive or reusable grammar definitions. +// Defines a named rule for recursive or reusable grammar definitions. +// Owns the implementation and fires NODE_START/END events. // expr -> term | expr "+" term -class rule_parser : public common_chat_peg_parser_base { +class rule_parser : public common_chat_peg_parser_base, public std::enable_shared_from_this { std::string name_; - std::weak_ptr root_; + common_chat_peg_parser child_; + bool trigger_; public: static constexpr parser_type type_value = RULE; parser_type type() const override { return type_value; } - rule_parser(const std::string & name, const std::weak_ptr & root, int id) - : common_chat_peg_parser_base(id), name_(name), root_(root) {} + rule_parser(const std::string & name, const common_chat_peg_parser & child, bool trigger, int id) + : common_chat_peg_parser_base(id), name_(name), child_(child), trigger_(trigger) {} common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { - auto root = root_.lock(); - if (!root) { - LOG_ERR("rule_parser::parse called with expired root parser\n"); - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); - } - - auto & rules = root->rules(); - auto it = rules.find(name_); - if (it == rules.end()) { - LOG_ERR("rule_parser::parse rule '%s' not found in registry\n", name_.c_str()); - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); - } - // Fire NODE_START event if (ctx.event_handler && ctx.semantics) { ctx.event_handler(common_chat_parse_event{ @@ -1239,8 +1234,8 @@ class rule_parser : public common_chat_peg_parser_base { ctx.current_depth++; } - // Parse the referenced rule - auto result = it->second->parse(ctx, start); + // Parse the child + auto result = child_->parse(ctx, start); // Fire NODE_END event if (ctx.event_handler && ctx.semantics) { @@ -1265,69 +1260,80 @@ class rule_parser : public common_chat_peg_parser_base { return result; } + void assign_id(common_chat_peg_parser_counter & counter) override { + common_chat_peg_parser_base::assign_id(counter); + child_->assign_id(counter); + } + std::string dump() const override { - return "Rule(" + name_ + ")"; + return "Rule(" + name_ + ", " + child_->dump() + ")"; } void accept(parser_visitor & visitor) override; const std::string & name() const { return name_; } + const common_chat_peg_parser & child() const { return child_; } + bool is_trigger() const { return trigger_; } }; -// Capture content if child parser matches -class capture_parser : public common_chat_peg_parser_base { - common_chat_peg_parser parser_; - std::string key_; +// References a named rule (lightweight reference, resolved during resolution phase) +// expr_ref -> expr +class ref_parser : public common_chat_peg_parser_base, public std::enable_shared_from_this { + std::string name_; + std::weak_ptr target_; public: - static constexpr parser_type type_value = CAPTURE; + static constexpr parser_type type_value = REF; parser_type type() const override { return type_value; } - capture_parser(const common_chat_peg_parser & parser, const std::string & key, int id) - : common_chat_peg_parser_base(id), parser_(parser), key_(key) {} + ref_parser(const std::string & name, int id) + : common_chat_peg_parser_base(id), name_(name) {} common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { - auto result = parser_->parse(ctx, start); - - if (!result.fail() && ctx.semantics) { - std::string_view matched = ctx.input; - matched = matched.substr(result.start, result.end - result.start); - std::string value = std::string(matched); - ctx.semantics->captures[key_] = std::move(value); + auto target = target_.lock(); + if (!target) { + LOG_ERR("ref_parser::parse called with unresolved reference '%s'\n", name_.c_str()); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); } - return result; - } - - void assign_id(common_chat_peg_parser_counter & counter) override { - common_chat_peg_parser_base::assign_id(counter); - parser_->assign_id(counter); + // Delegate to the target rule parser + return target->parse(ctx, start); } std::string dump() const override { - return "Capture(" + key_ + ", " + parser_->dump() + ")"; + return "Ref(" + name_ + ")"; } void accept(parser_visitor & visitor) override; - const common_chat_peg_parser & child() const { return parser_; } + const std::string & name() const { return name_; } + void set_target(const std::weak_ptr & target) { target_ = target; } + std::weak_ptr target() const { return target_; } }; -// Annotate nodes for use when generating lazy GBNF grammar rules. When built -// with lazy = true, only grammar rules reachable from trigger nodes are -// emitted. -class trigger_parser : public common_chat_peg_parser_base { +// Capture content if child parser matches +class capture_parser : public common_chat_peg_parser_base { common_chat_peg_parser parser_; + std::string key_; public: - static constexpr parser_type type_value = TRIGGER; + static constexpr parser_type type_value = CAPTURE; parser_type type() const override { return type_value; } - trigger_parser(const common_chat_peg_parser & parser, int id) - : common_chat_peg_parser_base(id), parser_(parser) {} + capture_parser(const common_chat_peg_parser & parser, const std::string & key, int id) + : common_chat_peg_parser_base(id), parser_(parser), key_(key) {} common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { - return parser_->parse(ctx, start); + auto result = parser_->parse(ctx, start); + + if (!result.fail() && ctx.semantics) { + std::string_view matched = ctx.input; + matched = matched.substr(result.start, result.end - result.start); + std::string value = std::string(matched); + ctx.semantics->captures[key_] = std::move(value); + } + + return result; } void assign_id(common_chat_peg_parser_counter & counter) override { @@ -1336,7 +1342,7 @@ class trigger_parser : public common_chat_peg_parser_base { } std::string dump() const override { - return "Trigger(" + parser_->dump() + ")"; + return "Capture(" + key_ + ", " + parser_->dump() + ")"; } void accept(parser_visitor & visitor) override; @@ -1367,9 +1373,9 @@ class parser_visitor { virtual void visit(json_string_parser & p) = 0; virtual void visit(schema_parser & p) = 0; virtual void visit(rule_parser & p) = 0; + virtual void visit(ref_parser & p) = 0; virtual void visit(root_parser & p) = 0; virtual void visit(capture_parser & p) = 0; - virtual void visit(trigger_parser & p) = 0; }; // Escape special characters for GBNF literals @@ -1435,16 +1441,102 @@ static std::string gbnf_excluding_pattern(const std::vector & strin return "(" + pattern + ")*"; } +// Visitor for resolving rule references and collecting rules +class resolution_visitor : public parser_visitor { + std::unordered_map> rules_; + std::vector> refs_; + + public: + resolution_visitor() = default; + + void visit(start_parser & /* p */) override {} + void visit(end_parser & /* p */) override {} + void visit(literal_parser & /* p */) override {} + void visit(any_parser & /* p */) override {} + void visit(space_parser & /* p */) override {} + void visit(json_string_parser & /* p */) override {} + void visit(chars_parser & /* p */) override {} + void visit(until_parser & /* p */) override {} + void visit(and_parser & p) override { p.child()->accept(*this); } + void visit(not_parser & p) override { p.child()->accept(*this); } + + void visit(sequence_parser & p) override { + for (const auto & child : p.parsers()) { + child->accept(*this); + } + } + + void visit(choice_parser & p) override { + for (const auto & child : p.parsers()) { + child->accept(*this); + } + } + + void visit(one_or_more_parser & p) override { p.child()->accept(*this); } + void visit(zero_or_more_parser & p) override { p.child()->accept(*this); } + void visit(optional_parser & p) override { p.child()->accept(*this); } + void visit(repetition_parser & p) override { p.child()->accept(*this); } + void visit(schema_parser & p) override { p.child()->accept(*this); } + void visit(capture_parser & p) override { p.child()->accept(*this); } + + void visit(rule_parser & p) override { + const std::string & name = p.name(); + + // Check for duplicate rule names + if (rules_.find(name) != rules_.end()) { + throw std::runtime_error("Duplicate rule name: " + name); + } + + // Collect this rule + auto rule_ptr = cast(p.shared_from_this()); + if (rule_ptr) { + rules_[name] = rule_ptr; + } + + // Recursively visit the child + p.child()->accept(*this); + } + + void visit(ref_parser & p) override { + // Collect this ref for later resolution + auto ref_ptr = cast(p.shared_from_this()); + if (ref_ptr) { + refs_.push_back(ref_ptr); + } + } + + void visit(root_parser & p) override { + // Visit all rules stored in the map + for (const auto & [name, rule] : p.rules()) { + rule->accept(*this); + } + + // Visit the root tree + p.root()->accept(*this); + } + + // Resolve all collected refs + void resolve() { + for (const auto & ref : refs_) { + const std::string & name = ref->name(); + auto it = rules_.find(name); + if (it == rules_.end()) { + throw std::runtime_error("Unresolved reference: " + name); + } + ref->set_target(it->second); + } + } + + const std::unordered_map> & rules() const { return rules_; } +}; + // Visitor for collecting reachable rules from a subtree class reachability_visitor : public parser_visitor { std::unordered_set & reachable_rules_; - const std::unordered_map & rules_; public: - reachability_visitor( - std::unordered_set & reachable_rules, - const std::unordered_map & rules - ) : reachable_rules_(reachable_rules), rules_(rules) {} + reachability_visitor(std::unordered_set & reachable_rules) + : reachable_rules_(reachable_rules) {} void visit(start_parser & /* p */) override {} void visit(end_parser & /* p */) override {} @@ -1478,7 +1570,6 @@ class reachability_visitor : public parser_visitor { // The schema system will handle rule generation via builder_.add_schema() } void visit(capture_parser & p) override { p.child()->accept(*this); } - void visit(trigger_parser & p) override { p.child()->accept(*this); } void visit(rule_parser & p) override { const std::string & name = p.name(); @@ -1488,10 +1579,22 @@ class reachability_visitor : public parser_visitor { } reachable_rules_.insert(name); - // Recursively visit the rule's definition - auto it = rules_.find(name); - if (it != rules_.end()) { - it->second->accept(*this); + // Recursively visit the rule's child + p.child()->accept(*this); + } + + void visit(ref_parser & p) override { + const std::string & name = p.name(); + // Mark as reachable + if (reachable_rules_.find(name) != reachable_rules_.end()) { + return; + } + reachable_rules_.insert(name); + + // Follow the reference to the target rule + auto target = p.target().lock(); + if (target) { + target->accept(*this); } } @@ -1505,14 +1608,12 @@ class gbnf_visitor : public parser_visitor { std::unordered_map rule_name_mapping_; std::string current_result_; bool lazy_; - std::vector trigger_names_; + std::unordered_map> all_rules_; std::unordered_set reachable_rules_; - int trigger_counter_; - std::vector> triggers_; public: gbnf_visitor(const common_grammar_builder & builder, bool lazy = false) - : builder_(builder), lazy_(lazy), trigger_counter_(0) {} + : builder_(builder), lazy_(lazy) {} const std::string& result() const { return current_result_; } @@ -1522,18 +1623,26 @@ class gbnf_visitor : public parser_visitor { return type == CHOICE || type == SEQUENCE; } - // Collect all reachable rules from the given triggers - void collect_reachable_rules( - const std::vector> & triggers, - const std::unordered_map & rules - ) { + // Collect all reachable rules from trigger rules + void collect_reachable_rules() { reachable_rules_.clear(); - reachability_visitor visitor(reachable_rules_, rules); - for (const auto & trigger : triggers) { - trigger->accept(visitor); + reachability_visitor visitor(reachable_rules_); + + // Find all trigger rules and traverse from them + for (const auto & [name, rule] : all_rules_) { + if (rule->is_trigger()) { + rule->accept(visitor); + } } } + // Collect all rules from the tree + void collect_all_rules(common_chat_peg_parser_base & root) { + resolution_visitor resolver; + root.accept(resolver); + all_rules_ = resolver.rules(); + } + public: void visit(start_parser & /* p */) override { current_result_ = ""; @@ -1691,6 +1800,11 @@ class gbnf_visitor : public parser_visitor { } void visit(rule_parser & p) override { + // When visiting a rule, generate its definition + p.child()->accept(*this); + } + + void visit(ref_parser & p) override { // Return canonical rule reference auto it = rule_name_mapping_.find(p.name()); if (it != rule_name_mapping_.end()) { @@ -1702,11 +1816,12 @@ class gbnf_visitor : public parser_visitor { } void visit(root_parser & p) override { - auto rules = p.rules(); + // Collect all rules from the tree + collect_all_rules(p); if (!lazy_) { // Non-lazy mode: generate all rules eagerly - for (const auto & [name, rule] : rules) { + for (const auto & [name, rule] : all_rules_) { rule->accept(*this); auto rule_body = current_result_; auto canonical_name = builder_.add_rule(name, rule_body); @@ -1720,22 +1835,18 @@ class gbnf_visitor : public parser_visitor { // Lazy mode: only generate rules reachable from triggers - // First pass: traverse root to collect triggers and generate synthetic rules - // (visit(trigger_parser) will populate triggers_ and trigger_names_) - p.root()->accept(*this); + // Collect all rules reachable from triggers + collect_reachable_rules(); - // Check if we found any triggers - if (triggers_.empty()) { - LOG_ERR("Lazy grammar generation enabled but no trigger nodes found\n"); + // Check if we found any trigger rules + if (reachable_rules_.empty()) { + LOG_ERR("Lazy grammar generation enabled but no trigger rules found\n"); current_result_ = ""; return; } - // Second pass: collect all rules reachable from triggers - collect_reachable_rules(triggers_, rules); - - // Third pass: generate only reachable rules - for (const auto & [name, rule] : rules) { + // Generate only reachable rules + for (const auto & [name, rule] : all_rules_) { // Skip rules that aren't reachable if (reachable_rules_.find(name) == reachable_rules_.end()) { continue; @@ -1747,39 +1858,22 @@ class gbnf_visitor : public parser_visitor { rule_name_mapping_[name] = canonical_name; } - // Generate root as alternation of trigger rules - current_result_ = string_join(trigger_names_, " | "); + // Generate root as alternation of trigger rule names + std::vector trigger_names; + for (const auto & [name, rule] : all_rules_) { + if (rule->is_trigger()) { + auto it = rule_name_mapping_.find(name); + if (it != rule_name_mapping_.end()) { + trigger_names.push_back(it->second); + } + } + } + current_result_ = string_join(trigger_names, " | "); } void visit(capture_parser & p) override { p.child()->accept(*this); } - - void visit(trigger_parser & p) override { - if (!lazy_) { - // Non-lazy mode: transparent pass-through - p.child()->accept(*this); - return; - } - - // Lazy mode: create synthetic rule for this trigger - ++trigger_counter_; - std::string trigger_name = "trigger-" + std::to_string(trigger_counter_); - - // Visit child to generate its grammar - p.child()->accept(*this); - std::string child_grammar = current_result_; - - // Add synthetic rule - builder_.add_rule(trigger_name, child_grammar); - trigger_names_.push_back(trigger_name); - - // Store trigger for reachability analysis - triggers_.push_back(p.child().ptr()); - - // Return the trigger rule reference - current_result_ = trigger_name; - } }; // Implement accept() methods for all parser classes @@ -1801,9 +1895,9 @@ void chars_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void json_string_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void schema_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void rule_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } +void ref_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void root_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } void capture_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } -void trigger_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } common_chat_parse_result common_chat_parse_cache::set(int id, size_t start, common_chat_parse_result result) { if (id == -1) { @@ -1900,9 +1994,8 @@ common_chat_peg_parser builder::until_one_of(const std::vector & de common_chat_peg_parser builder::repeat(const common_chat_peg_parser & p, int min, int max) { return make_parser(counter_, p, min, max); } common_chat_peg_parser builder::repeat(const common_chat_peg_parser & p, int n) { return make_parser(counter_, p, n, n); } -common_chat_peg_parser builder::rule(const std::string & name) { - auto root = cast(root_); - return make_parser(counter_, name, std::weak_ptr(root)); +common_chat_peg_parser builder::ref(const std::string & name) { + return make_parser(counter_, name); } common_chat_peg_parser builder::schema(const common_chat_peg_parser & p, const std::string & name, const nlohmann::ordered_json & schema) { @@ -1913,26 +2006,31 @@ common_chat_peg_parser builder::capture(const std::string & key, const common_ch return make_parser(counter_, p, key); } -common_chat_peg_parser builder::trigger(const common_chat_peg_parser & p) { - return make_parser(counter_, p); -} - -common_chat_peg_parser builder::add_rule(const std::string & name, const common_chat_peg_parser & p) { - auto root = cast(root_); - root->add_rule(name, p); - return rule(name); +common_chat_peg_parser builder::rule(const std::string & name, const common_chat_peg_parser & p, bool trigger) { + auto root_container = cast(root_); + auto rule_node = make_parser(counter_, name, p, trigger); + root_container->add_rule(name, rule_node); + return make_parser(counter_, name); } -common_chat_peg_parser builder::add_rule(const std::string & name, const std::function & builder) { - auto root = cast(root_); - if (root->rules().find(name) != root->rules().end()) { - return rule(name); +common_chat_peg_parser builder::rule(const std::string & name, const std::function & builder_fn, bool trigger) { + auto root_container = cast(root_); + if (root_container->rules().find(name) != root_container->rules().end()) { + return ref(name); } - root->add_rule(name, literal("")); // Placeholder - auto parser = builder(); - root->add_rule(name, parser); - return rule(name); + // Create placeholder rule to allow recursive references + auto placeholder = make_parser(counter_, name, literal(""), trigger); + root_container->add_rule(name, placeholder); + + // Build the actual parser + auto parser = builder_fn(); + + // Replace placeholder with actual rule + auto rule_node = make_parser(counter_, name, parser, trigger); + root_container->add_rule(name, rule_node); + + return make_parser(counter_, name); } void builder::set_root(const common_chat_peg_parser & p) { @@ -1946,7 +2044,7 @@ void builder::set_root(const common_chat_peg_parser & p) { } common_chat_peg_parser builder::json_number() { - return add_rule("json-number", [this]() { + return rule("json-number", [this]() { auto digit1_9 = chars("[1-9]", 1, 1); auto digits = chars("[0-9]"); auto int_part = literal("0") | (digit1_9 + chars("[0-9]", 0, -1)); @@ -1957,25 +2055,25 @@ common_chat_peg_parser builder::json_number() { } common_chat_peg_parser builder::json_string() { - return add_rule("json-string", [this]() { + return rule("json-string", [this]() { return literal("\"") + json_string_content() + literal("\""); }); } common_chat_peg_parser builder::json_bool() { - return add_rule("json-bool", [this]() { + return rule("json-bool", [this]() { return literal("true") | literal("false"); }); } common_chat_peg_parser builder::json_null() { - return add_rule("json-null", [this]() { + return rule("json-null", [this]() { return literal("null"); }); } common_chat_peg_parser builder::json_object() { - return add_rule("json-object", [this]() { + return rule("json-object", [this]() { auto ws = space(); auto member = json_string() + ws + literal(":") + ws + json(); auto members = member + zero_or_more(ws + literal(",") + ws + member); @@ -1985,7 +2083,7 @@ common_chat_peg_parser builder::json_object() { } common_chat_peg_parser builder::json_array() { - return add_rule("json-array", [this]() { + return rule("json-array", [this]() { auto ws = space(); auto elements = json() + zero_or_more(ws + literal(",") + ws + json()); return (literal("[") + ws + literal("]")) | @@ -1994,7 +2092,7 @@ common_chat_peg_parser builder::json_array() { } common_chat_peg_parser builder::json() { - return add_rule("json-value", [this]() { + return rule("json-value", [this]() { return json_object() | json_array() | json_string() | @@ -2005,6 +2103,11 @@ common_chat_peg_parser builder::json() { } common_chat_peg_parser builder::build() { + // Resolve all references + resolution_visitor resolver; + root_->accept(resolver); + resolver.resolve(); + return root_; } diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index 483bca7d09e1d..e5b1f8fd2fcd6 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -240,9 +240,10 @@ class common_chat_peg_parser_builder { // Equivalent to chars(classes, 1, 1) common_chat_peg_parser one(const std::string & classes); - // References a named rule for recursive or reusable grammar definitions. - // expr -> term | expr "+" term - common_chat_peg_parser rule(const std::string & name); + // Creates a lightweight reference to a named rule (resolved during build()). + // Use this for forward references in recursive grammars. + // expr_ref -> expr + common_chat_peg_parser ref(const std::string & name); // Matches zero or more whitespace characters (space, tab, newline). // S -> [ \t\n]* @@ -282,21 +283,18 @@ class common_chat_peg_parser_builder { // Captures matched text to semantics.captures[key] common_chat_peg_parser capture(const std::string & key, const common_chat_peg_parser & p); - // Mark a node as a trigger for GBNF grammar generartion. This is used for - // lazy grammar evaluation by only producing GBNF grammar rules that are - // reachable from trigger nodes. - // S -> Trigger(A) - common_chat_peg_parser trigger(const common_chat_peg_parser & p); + // Creates a named rule, stores it in the grammar, and returns a reference to it. + // If trigger=true, marks this rule as an entry point for lazy grammar generation. + // auto json = p.rule("json", json_obj | json_arr | ...) + common_chat_peg_parser rule(const std::string & name, const common_chat_peg_parser & p, bool trigger = false); - // Adds a named rule and returns a rule reference. - common_chat_peg_parser add_rule(const std::string & name, const common_chat_peg_parser & p); - - // Adds a named rule using a function. This handles recursive grammars by + // Creates a named rule using a builder function. This handles recursive grammars by // inserting a placeholder rule before invoking the builder, allowing the - // builder to reference the rule being defined. Use this when the rule + // builder to reference the rule being defined via ref(). Use this when the rule // definition needs to call back to itself (directly or indirectly). - // add_rule("json", [&]() { return json_object() | json_array() | ... }) - common_chat_peg_parser add_rule(const std::string & name, const std::function & builder); + // If trigger=true, marks this rule as an entry point for lazy grammar generation. + // auto json = p.rule("json", [&]() { return json_object() | json_array() | ... }) + common_chat_peg_parser rule(const std::string & name, const std::function & builder, bool trigger = false); void set_root(const common_chat_peg_parser & p); diff --git a/tests/chat-peg-parser/test-command7-parser-compare.cpp b/tests/chat-peg-parser/test-command7-parser-compare.cpp index 8f2a84e18eb82..03f0a4ee36ea8 100644 --- a/tests/chat-peg-parser/test-command7-parser-compare.cpp +++ b/tests/chat-peg-parser/test-command7-parser-compare.cpp @@ -9,29 +9,29 @@ static common_chat_peg_parser create_command_r7b_parser() { auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { - auto thinking = p.add_rule("thinking", - "<|START_THINKING|>" << p.add_rule("reasoning-content", p.until("<|END_THINKING|>")) << "<|END_THINKING|>"); + auto thinking = p.rule("thinking", + "<|START_THINKING|>" << p.rule("reasoning-content", p.until("<|END_THINKING|>")) << "<|END_THINKING|>"); - auto response = p.add_rule("response", - "<|START_RESPONSE|>" << p.add_rule("content", p.until("<|END_RESPONSE|>")) << "<|END_RESPONSE|>"); + auto response = p.rule("response", + "<|START_RESPONSE|>" << p.rule("content", p.until("<|END_RESPONSE|>")) << "<|END_RESPONSE|>"); - auto json = p.add_rule("json", p.json()); + auto json = p.rule("json", p.json()); - auto tool_call_id = p.add_rule("tool-call-id", - "\"tool_call_id\"" << (":" << p.add_rule("tool-call-id-value", "\"" + p.json_string() + "\""))); + auto tool_call_id = p.rule("tool-call-id", + "\"tool_call_id\"" << (":" << p.rule("tool-call-id-value", "\"" + p.json_string() + "\""))); - auto tool_call_name = p.add_rule("tool-name", - "\"tool_name\"" << (":" << p.add_rule("tool-name-value", "\"" + p.json_string() + "\""))); + auto tool_call_name = p.rule("tool-name", + "\"tool_name\"" << (":" << p.rule("tool-name-value", "\"" + p.json_string() + "\""))); - auto tool_call_args = p.add_rule("tool-args", - "\"parameters\"" << (":" << p.add_rule("tool-args-value", json))); + auto tool_call_args = p.rule("tool-args", + "\"parameters\"" << (":" << p.rule("tool-args-value", json))); - auto tool_call_fields = p.add_rule("tool-call-fields", tool_call_id | tool_call_name | tool_call_args); + auto tool_call_fields = p.rule("tool-call-fields", tool_call_id | tool_call_name | tool_call_args); - auto tool_call = p.add_rule("tool-call", + auto tool_call = p.rule("tool-call", "{" << tool_call_fields << p.zero_or_more(p.literal(",") << tool_call_fields) << "}"); - auto tool_calls = p.add_rule("tool-calls", + auto tool_calls = p.rule("tool-calls", "<|START_ACTION|>" << ("[" << tool_call << p.zero_or_more(p.literal(",") << tool_call) << "]") << "<|END_ACTION|>"); diff --git a/tests/chat-peg-parser/test-example-minimax-m2.cpp b/tests/chat-peg-parser/test-example-minimax-m2.cpp index b880acfe05eab..f3f464f655252 100644 --- a/tests/chat-peg-parser/test-example-minimax-m2.cpp +++ b/tests/chat-peg-parser/test-example-minimax-m2.cpp @@ -36,8 +36,8 @@ void test_example_minimax_m2(testing &t) { std::vector({ "category" })); - auto tool_call = p.trigger(p.add_rule("tool-call", - "" + p.one_or_more(function) + "")); + auto tool_call = p.rule("tool-call", + "" + p.one_or_more(function) + "", true); return thinking + p.optional(p.space() + content) + p.zero_or_more(p.space() + tool_call); }); diff --git a/tests/chat-peg-parser/test-example-qwen3-coder.cpp b/tests/chat-peg-parser/test-example-qwen3-coder.cpp index 840913a8fc6fb..229341516a48c 100644 --- a/tests/chat-peg-parser/test-example-qwen3-coder.cpp +++ b/tests/chat-peg-parser/test-example-qwen3-coder.cpp @@ -6,32 +6,32 @@ void test_example_qwen3_coder(testing &t) { auto explicit_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { - auto thinking = p.add_rule("raw-reasoning", - "" << p.add_rule("reasoning-content", p.until("")) << ""); + auto thinking = p.rule("raw-reasoning", + "" << p.rule("reasoning-content", p.until("")) << ""); - auto content = p.add_rule("content", p.until("")); + auto content = p.rule("content", p.until("")); - auto arg_name = p.add_rule("arg-start", ""); - auto arg_end = p.add_rule("arg-end", "" + p.peek(p.literal("")); + auto arg_name = p.rule("arg-start", ""); + auto arg_end = p.rule("arg-end", "" + p.peek(p.literal("")); - auto string_arg_content = p.add_rule("arg-string-content", p.until_one_of({ + auto string_arg_content = p.rule("arg-string-content", p.until_one_of({ "", })); - auto string_arg = p.add_rule("arg-string", arg_name + string_arg_content + arg_end); + auto string_arg = p.rule("arg-string", arg_name + string_arg_content + arg_end); auto json = p.json(); - auto json_arg = p.add_rule("arg-json", arg_name + p.add_rule("arg-json-content", json) + arg_end); + auto json_arg = p.rule("arg-json", arg_name + p.rule("arg-json-content", json) + arg_end); - auto function = p.add_rule("function", - p.add_rule("function-start", "") + auto function = p.rule("function", + p.rule("function-start", "") + p.one_or_more(json_arg | string_arg) + ""); - auto tool_call = p.trigger(p.add_rule("tool-call", - "" + p.one_or_more(function) + "")); + auto tool_call = p.rule("tool-call", + "" + p.one_or_more(function) + "", true); return thinking + p.optional(p.space() + content) + p.zero_or_more(p.space() + tool_call) + p.end(); }); @@ -45,8 +45,8 @@ void test_example_qwen3_coder(testing &t) { "path", "pattern", "min_size_mb", "max_depth", "include_hidden", "modified_days_ago", "case_sensitive", "sort_by", "filters" })); - auto tool_call = p.trigger(p.add_rule("tool-call", - "" + p.one_or_more(function) + "")); + auto tool_call = p.rule("tool-call", + "" + p.one_or_more(function) + "", true); return thinking + p.optional(p.space() + content) + p.zero_or_more(p.space() + tool_call); }); diff --git a/tests/chat-peg-parser/test-example-seed-oss.cpp b/tests/chat-peg-parser/test-example-seed-oss.cpp index a0aa9d5493c37..d7b90de160425 100644 --- a/tests/chat-peg-parser/test-example-seed-oss.cpp +++ b/tests/chat-peg-parser/test-example-seed-oss.cpp @@ -11,8 +11,8 @@ void test_example_seed_oss(testing &t) { std::vector({ "location", "units" })); - auto tool_call = p.trigger(p.add_rule("tool-call", - "" + p.one_or_more(function) + "")); + auto tool_call = p.rule("tool-call", + "" + p.one_or_more(function) + "", true); return thinking + p.optional(p.space() + content) + p.zero_or_more(p.space() + tool_call); }); diff --git a/tests/chat-peg-parser/test-gbnf-generation.cpp b/tests/chat-peg-parser/test-gbnf-generation.cpp index 17e8984538c67..26433c5aac0b3 100644 --- a/tests/chat-peg-parser/test-gbnf-generation.cpp +++ b/tests/chat-peg-parser/test-gbnf-generation.cpp @@ -96,7 +96,7 @@ void test_gbnf_generation(testing &t) { // Test rule references t.test("rule references", [](testing &t) { auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { - auto digit = p.add_rule("digit", p.one("[0-9]")); + auto digit = p.rule("digit", p.one("[0-9]")); return p.one_or_more(digit); }); diff --git a/tests/chat-peg-parser/test-recursive-references.cpp b/tests/chat-peg-parser/test-recursive-references.cpp index 224f58c94a123..b1bb19e35b614 100644 --- a/tests/chat-peg-parser/test-recursive-references.cpp +++ b/tests/chat-peg-parser/test-recursive-references.cpp @@ -4,9 +4,9 @@ void test_recursive_references(testing &t) { // Test simple number t.test("simple_number", [](testing &t) { auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { - p.add_rule("number", p.one_or_more(p.one("0-9"))); - p.add_rule("list", p.sequence({ p.literal("["), p.rule("value"), p.literal("]") })); - return p.add_rule("value", p.rule("number") | p.rule("list")); + p.rule("number", p.one_or_more(p.one("0-9"))); + p.rule("list", p.sequence({ p.literal("["), p.ref("value"), p.literal("]") })); + return p.rule("value", p.ref("number") | p.ref("list")); }); common_chat_parse_context ctx("1", true); @@ -18,9 +18,9 @@ void test_recursive_references(testing &t) { // Test simple list t.test("simple_list", [](testing &t) { auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { - p.add_rule("number", p.one_or_more(p.one("0-9"))); - p.add_rule("list", p.sequence({ p.literal("["), p.rule("value"), p.literal("]") })); - return p.add_rule("value", p.rule("number") | p.rule("list")); + p.rule("number", p.one_or_more(p.one("0-9"))); + p.rule("list", p.sequence({ p.literal("["), p.ref("value"), p.literal("]") })); + return p.rule("value", p.ref("number") | p.ref("list")); }); common_chat_parse_context ctx("[1]", true); @@ -32,9 +32,9 @@ void test_recursive_references(testing &t) { // Test nested list t.test("nested_list", [](testing &t) { auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { - p.add_rule("number", p.one_or_more(p.one("0-9"))); - p.add_rule("list", p.sequence({ p.literal("["), p.rule("value"), p.literal("]") })); - return p.add_rule("value", p.rule("number") | p.rule("list")); + p.rule("number", p.one_or_more(p.one("0-9"))); + p.rule("list", p.sequence({ p.literal("["), p.ref("value"), p.literal("]") })); + return p.rule("value", p.ref("number") | p.ref("list")); }); common_chat_parse_context ctx("[[2]]", true); @@ -46,9 +46,9 @@ void test_recursive_references(testing &t) { // Test deeply nested list t.test("deeply_nested_list", [](testing &t) { auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { - p.add_rule("number", p.one_or_more(p.one("0-9"))); - p.add_rule("list", p.sequence({ p.literal("["), p.rule("value"), p.literal("]") })); - return p.add_rule("value", p.rule("number") | p.rule("list")); + p.rule("number", p.one_or_more(p.one("0-9"))); + p.rule("list", p.sequence({ p.literal("["), p.ref("value"), p.literal("]") })); + return p.rule("value", p.ref("number") | p.ref("list")); }); common_chat_parse_context ctx("[[[3]]]", true); @@ -60,9 +60,9 @@ void test_recursive_references(testing &t) { // Test need_more_input match t.test("need_more_input_match", [](testing &t) { auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { - p.add_rule("number", p.one_or_more(p.one("0-9"))); - p.add_rule("list", p.sequence({ p.literal("["), p.rule("value"), p.literal("]") })); - return p.add_rule("value", p.rule("number") | p.rule("list")); + p.rule("number", p.one_or_more(p.one("0-9"))); + p.rule("list", p.sequence({ p.literal("["), p.ref("value"), p.literal("]") })); + return p.rule("value", p.ref("number") | p.ref("list")); }); common_chat_parse_context ctx("[[", false); @@ -74,9 +74,9 @@ void test_recursive_references(testing &t) { // Test no match t.test("no_match", [](testing &t) { auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { - p.add_rule("number", p.one_or_more(p.one("0-9"))); - p.add_rule("list", p.sequence({ p.literal("["), p.rule("value"), p.literal("]") })); - return p.add_rule("value", p.rule("number") | p.rule("list")); + p.rule("number", p.one_or_more(p.one("0-9"))); + p.rule("list", p.sequence({ p.literal("["), p.ref("value"), p.literal("]") })); + return p.rule("value", p.ref("number") | p.ref("list")); }); common_chat_parse_context ctx("[a]", true); From 38a8fd6c9607412138056fc3aeceac660b623b52 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 17:34:05 -0600 Subject: [PATCH 113/183] clean up visitor --- common/chat-peg-parser.cpp | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index a11c45a270b71..59749bde07082 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -1566,14 +1566,14 @@ class reachability_visitor : public parser_visitor { void visit(optional_parser & p) override { p.child()->accept(*this); } void visit(repetition_parser & p) override { p.child()->accept(*this); } void visit(schema_parser & /* p */) override { - // Schema parsers are opaque - don't traverse their children // The schema system will handle rule generation via builder_.add_schema() } void visit(capture_parser & p) override { p.child()->accept(*this); } void visit(rule_parser & p) override { const std::string & name = p.name(); - // If we've already processed this rule, skip to avoid infinite recursion + // Check if we arleady reached this rule. Technically we shouldn't, + // since we don't traverse ref_parser if (reachable_rules_.find(name) != reachable_rules_.end()) { return; } @@ -1584,18 +1584,8 @@ class reachability_visitor : public parser_visitor { } void visit(ref_parser & p) override { - const std::string & name = p.name(); // Mark as reachable - if (reachable_rules_.find(name) != reachable_rules_.end()) { - return; - } - reachable_rules_.insert(name); - - // Follow the reference to the target rule - auto target = p.target().lock(); - if (target) { - target->accept(*this); - } + reachable_rules_.insert(p.name()); } void visit(root_parser & p) override { From 362cb6a997c13291b5a37c9bde5adbff39aae19e Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 17:46:07 -0600 Subject: [PATCH 114/183] clean up indirection in root parser w.r.t rules --- common/chat-peg-parser.cpp | 145 ++++++++++++++++--------------------- 1 file changed, 64 insertions(+), 81 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 59749bde07082..3626efd64dc23 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -310,52 +310,6 @@ class aho_corasick_matcher { } }; -// Container for the root parser and all named rules in the grammar. -// Manages ownership of rule registry to enable recursive grammar definitions. -class root_parser : public common_chat_peg_parser_base { - common_chat_peg_parser root_; - std::unordered_map rules_; - - public: - static constexpr parser_type type_value = ROOT; - parser_type type() const override { return type_value; } - - root_parser(int id) : common_chat_peg_parser_base(id) {} - - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { - return root_->parse(ctx, start); - } - - void assign_id(common_chat_peg_parser_counter & counter) override { - common_chat_peg_parser_base::assign_id(counter); - for (auto & [name, rule] : rules_) { - rule->assign_id(counter); - } - if (root_.ptr()) { - root_->assign_id(counter); - } - } - - std::string dump() const override { - return root_->dump(); - } - - void accept(parser_visitor & visitor) override; - - void add_rule(const std::string & name, const common_chat_peg_parser & parser) { - rules_[name] = parser; - } - - void set_root(const common_chat_peg_parser & parser) { - root_ = parser; - } - - const common_chat_peg_parser & root() const { return root_; } - - std::unordered_map & rules() { return rules_; } - const std::unordered_map & rules() const { return rules_; } -}; - // Matches the start of the input // S -> ^ class start_parser : public common_chat_peg_parser_base { @@ -1350,6 +1304,51 @@ class capture_parser : public common_chat_peg_parser_base { const common_chat_peg_parser & child() const { return parser_; } }; +// Container for the root parser and all named rules in the grammar. +// Manages ownership of rule registry to enable recursive grammar definitions. +class root_parser : public common_chat_peg_parser_base { + common_chat_peg_parser root_; + std::unordered_map> rules_; + + public: + static constexpr parser_type type_value = ROOT; + parser_type type() const override { return type_value; } + + root_parser(int id) : common_chat_peg_parser_base(id) {} + + common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { + return root_->parse(ctx, start); + } + + void assign_id(common_chat_peg_parser_counter & counter) override { + common_chat_peg_parser_base::assign_id(counter); + for (auto & [name, rule] : rules_) { + rule->assign_id(counter); + } + if (root_.ptr()) { + root_->assign_id(counter); + } + } + + std::string dump() const override { + return root_->dump(); + } + + void accept(parser_visitor & visitor) override; + + void add_rule(const std::string & name, const std::shared_ptr & rule) { + rules_[name] = rule; + } + + void set_root(const common_chat_peg_parser & parser) { + root_ = parser; + } + + const common_chat_peg_parser & root() const { return root_; } + + const std::unordered_map> & rules() const { return rules_; } +}; + // Base visitor class for parser tree traversal class parser_visitor { public: @@ -1599,7 +1598,6 @@ class gbnf_visitor : public parser_visitor { std::string current_result_; bool lazy_; std::unordered_map> all_rules_; - std::unordered_set reachable_rules_; public: gbnf_visitor(const common_grammar_builder & builder, bool lazy = false) @@ -1613,26 +1611,6 @@ class gbnf_visitor : public parser_visitor { return type == CHOICE || type == SEQUENCE; } - // Collect all reachable rules from trigger rules - void collect_reachable_rules() { - reachable_rules_.clear(); - reachability_visitor visitor(reachable_rules_); - - // Find all trigger rules and traverse from them - for (const auto & [name, rule] : all_rules_) { - if (rule->is_trigger()) { - rule->accept(visitor); - } - } - } - - // Collect all rules from the tree - void collect_all_rules(common_chat_peg_parser_base & root) { - resolution_visitor resolver; - root.accept(resolver); - all_rules_ = resolver.rules(); - } - public: void visit(start_parser & /* p */) override { current_result_ = ""; @@ -1806,39 +1784,44 @@ class gbnf_visitor : public parser_visitor { } void visit(root_parser & p) override { - // Collect all rules from the tree - collect_all_rules(p); - if (!lazy_) { // Non-lazy mode: generate all rules eagerly - for (const auto & [name, rule] : all_rules_) { + for (const auto & [name, rule] : p.rules()) { rule->accept(*this); auto rule_body = current_result_; auto canonical_name = builder_.add_rule(name, rule_body); rule_name_mapping_[name] = canonical_name; } - // Return root body for composition + // Return root body p.root()->accept(*this); return; } - // Lazy mode: only generate rules reachable from triggers + // Lazy mode // Collect all rules reachable from triggers - collect_reachable_rules(); + std::unordered_set reachable; + reachability_visitor visitor(reachable); + + // Find all trigger rules and traverse from them + for (const auto & [name, rule] : all_rules_) { + if (rule->is_trigger()) { + rule->accept(visitor); + } + } // Check if we found any trigger rules - if (reachable_rules_.empty()) { + if (reachable.empty()) { LOG_ERR("Lazy grammar generation enabled but no trigger rules found\n"); current_result_ = ""; return; } // Generate only reachable rules - for (const auto & [name, rule] : all_rules_) { + for (const auto & [name, rule] : p.rules()) { // Skip rules that aren't reachable - if (reachable_rules_.find(name) == reachable_rules_.end()) { + if (reachable.find(name) == reachable.end()) { continue; } @@ -1850,7 +1833,7 @@ class gbnf_visitor : public parser_visitor { // Generate root as alternation of trigger rule names std::vector trigger_names; - for (const auto & [name, rule] : all_rules_) { + for (const auto & [name, rule] : p.rules()) { if (rule->is_trigger()) { auto it = rule_name_mapping_.find(name); if (it != rule_name_mapping_.end()) { @@ -1998,7 +1981,7 @@ common_chat_peg_parser builder::capture(const std::string & key, const common_ch common_chat_peg_parser builder::rule(const std::string & name, const common_chat_peg_parser & p, bool trigger) { auto root_container = cast(root_); - auto rule_node = make_parser(counter_, name, p, trigger); + auto rule_node = std::make_shared(name, p, trigger, counter_.next()); root_container->add_rule(name, rule_node); return make_parser(counter_, name); } @@ -2010,14 +1993,14 @@ common_chat_peg_parser builder::rule(const std::string & name, const std::functi } // Create placeholder rule to allow recursive references - auto placeholder = make_parser(counter_, name, literal(""), trigger); + auto placeholder = std::make_shared(name, any(), trigger, counter_.next()); root_container->add_rule(name, placeholder); // Build the actual parser auto parser = builder_fn(); // Replace placeholder with actual rule - auto rule_node = make_parser(counter_, name, parser, trigger); + auto rule_node = std::make_shared(name, parser, trigger, counter_.next()); root_container->add_rule(name, rule_node); return make_parser(counter_, name); From 0482db6bb3c4a823df00b838782111b76304b43f Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 20:19:14 -0600 Subject: [PATCH 115/183] store shared ptr directly in parser classes --- common/chat-peg-parser.cpp | 93 +++++++++++++++++++++++++++----------- 1 file changed, 67 insertions(+), 26 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 3626efd64dc23..f5e0c294a3f1a 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -385,7 +385,7 @@ class literal_parser : public common_chat_peg_parser_base { // Matches a sequence of parsers in order, all must succeed. // S -> A B C class sequence_parser : public common_chat_peg_parser_base { - std::vector parsers_; + std::vector> parsers_; public: static constexpr parser_type type_value = SEQUENCE; @@ -394,10 +394,11 @@ class sequence_parser : public common_chat_peg_parser_base { template sequence_parser(InputIt first, InputIt last, int id) : common_chat_peg_parser_base(id) { for (auto it = first; it != last; ++it) { - if (auto seq = cast(*it)) { + auto ptr = it->ptr(); + if (auto seq = cast(ptr)) { parsers_.insert(parsers_.end(), seq->parsers().begin(), seq->parsers().end()); } else { - parsers_.push_back(*it); + parsers_.push_back(ptr); } } } @@ -406,6 +407,17 @@ class sequence_parser : public common_chat_peg_parser_base { sequence_parser(const T & parsers, int id) : sequence_parser(std::begin(parsers), std::end(parsers), id) {} + sequence_parser(const std::vector> & parsers, int id) + : common_chat_peg_parser_base(id) { + for (const auto & ptr : parsers) { + if (auto seq = cast(ptr)) { + parsers_.insert(parsers_.end(), seq->parsers().begin(), seq->parsers().end()); + } else { + parsers_.push_back(ptr); + } + } + } + common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto pos = start; for (const auto & p : parsers_) { @@ -438,13 +450,13 @@ class sequence_parser : public common_chat_peg_parser_base { void accept(parser_visitor & visitor) override; - const std::vector & parsers() const { return parsers_; } + const std::vector> & parsers() const { return parsers_; } }; // Matches the first parser that succeeds from a list of alternatives. // S -> A | B | C class choice_parser : public common_chat_peg_parser_base { - std::vector parsers_; + std::vector> parsers_; public: static constexpr parser_type type_value = CHOICE; @@ -453,10 +465,11 @@ class choice_parser : public common_chat_peg_parser_base { template choice_parser(InputIt first, InputIt last, int id) : common_chat_peg_parser_base(id) { for (auto it = first; it != last; ++it) { - if (auto choice = cast(*it)) { + auto ptr = it->ptr(); + if (auto choice = cast(ptr)) { parsers_.insert(parsers_.end(), choice->parsers().begin(), choice->parsers().end()); } else { - parsers_.push_back(*it); + parsers_.push_back(ptr); } } } @@ -465,6 +478,17 @@ class choice_parser : public common_chat_peg_parser_base { choice_parser(const T & parsers, int id) : choice_parser(std::begin(parsers), std::end(parsers), id) {} + choice_parser(const std::vector> & parsers, int id) + : common_chat_peg_parser_base(id) { + for (const auto & ptr : parsers) { + if (auto choice = cast(ptr)) { + parsers_.insert(parsers_.end(), choice->parsers().begin(), choice->parsers().end()); + } else { + parsers_.push_back(ptr); + } + } + } + common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto pos = start; for (const auto & p : parsers_) { @@ -495,14 +519,14 @@ class choice_parser : public common_chat_peg_parser_base { void accept(parser_visitor & visitor) override; - const std::vector & parsers() const { return parsers_; } + const std::vector> & parsers() const { return parsers_; } }; // Matches between min and max repetitions of a parser (inclusive). // S -> A{m,n} // Use -1 for max_count to represent unbounded repetition (equivalent to {m,}) class repetition_parser : public common_chat_peg_parser_base { - common_chat_peg_parser parser_; + std::shared_ptr parser_; int min_count_; int max_count_; @@ -511,6 +535,9 @@ class repetition_parser : public common_chat_peg_parser_base { parser_type type() const override { return type_value; } repetition_parser(const common_chat_peg_parser & parser, int min_count, int max_count, int id) + : common_chat_peg_parser_base(id), parser_(parser.ptr()), min_count_(min_count), max_count_(max_count) {} + + repetition_parser(const std::shared_ptr & parser, int min_count, int max_count, int id) : common_chat_peg_parser_base(id), parser_(parser), min_count_(min_count), max_count_(max_count) {} common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { @@ -568,7 +595,7 @@ class repetition_parser : public common_chat_peg_parser_base { void accept(parser_visitor & visitor) override; - const common_chat_peg_parser & child() const { return parser_; } + const std::shared_ptr & child() const { return parser_; } int min_count() const { return min_count_; } @@ -583,6 +610,7 @@ class one_or_more_parser : public repetition_parser { parser_type type() const override { return type_value; } one_or_more_parser(const common_chat_peg_parser & p, int id) : repetition_parser(p, 1, -1, id) {} + one_or_more_parser(const std::shared_ptr & p, int id) : repetition_parser(p, 1, -1, id) {} std::string dump() const override { return "OneOrMore(" + child()->dump() + ")"; @@ -599,6 +627,7 @@ class zero_or_more_parser : public repetition_parser { parser_type type() const override { return type_value; } zero_or_more_parser(const common_chat_peg_parser & p, int id) : repetition_parser(p, 0, -1, id) {} + zero_or_more_parser(const std::shared_ptr & p, int id) : repetition_parser(p, 0, -1, id) {} std::string dump() const override { return "ZeroOrMore(" + child()->dump() + ")"; @@ -615,6 +644,7 @@ class optional_parser : public repetition_parser { parser_type type() const override { return type_value; } optional_parser(const common_chat_peg_parser & p, int id) : repetition_parser(p, 0, 1, id) {} + optional_parser(const std::shared_ptr & p, int id) : repetition_parser(p, 0, 1, id) {} std::string dump() const override { return "Optional(" + child()->dump() + ")"; @@ -626,13 +656,14 @@ class optional_parser : public repetition_parser { // Positive lookahead: succeeds if child parser succeeds, consumes no input. // S -> &A class and_parser : public common_chat_peg_parser_base { - common_chat_peg_parser parser_; + std::shared_ptr parser_; public: static constexpr parser_type type_value = AND; parser_type type() const override { return type_value; } - and_parser(const common_chat_peg_parser & parser, int id) : common_chat_peg_parser_base(id), parser_(parser) {} + and_parser(const common_chat_peg_parser & parser, int id) : common_chat_peg_parser_base(id), parser_(parser.ptr()) {} + and_parser(const std::shared_ptr & parser, int id) : common_chat_peg_parser_base(id), parser_(parser) {} common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto result = parser_->parse(ctx, start); @@ -651,19 +682,20 @@ class and_parser : public common_chat_peg_parser_base { void accept(parser_visitor & visitor) override; - const common_chat_peg_parser & child() const { return parser_; } + const std::shared_ptr & child() const { return parser_; } }; // Negative lookahead: succeeds if child parser fails, consumes no input. // S -> !A class not_parser : public common_chat_peg_parser_base { - common_chat_peg_parser parser_; + std::shared_ptr parser_; public: static constexpr parser_type type_value = NOT; parser_type type() const override { return type_value; } - not_parser(const common_chat_peg_parser & parser, int id) : common_chat_peg_parser_base(id), parser_(parser) {} + not_parser(const common_chat_peg_parser & parser, int id) : common_chat_peg_parser_base(id), parser_(parser.ptr()) {} + not_parser(const std::shared_ptr & parser, int id) : common_chat_peg_parser_base(id), parser_(parser) {} common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { auto result = parser_->parse(ctx, start); @@ -693,7 +725,7 @@ class not_parser : public common_chat_peg_parser_base { void accept(parser_visitor & visitor) override; - const common_chat_peg_parser & child() const { return parser_; } + const std::shared_ptr & child() const { return parser_; } }; // Matches any single character. @@ -1130,7 +1162,7 @@ class until_parser : public common_chat_peg_parser_base { // Wraps a parser with JSON schema metadata for grammar generation. // Used internally to convert JSON schemas to GBNF grammar rules. class schema_parser : public common_chat_peg_parser_base { - common_chat_peg_parser parser_; + std::shared_ptr parser_; std::string name_; nlohmann::ordered_json schema_; @@ -1139,6 +1171,9 @@ class schema_parser : public common_chat_peg_parser_base { parser_type type() const override { return type_value; } schema_parser(const common_chat_peg_parser & parser, const std::string & name, const nlohmann::ordered_json & schema, int id) + : common_chat_peg_parser_base(id), parser_(parser.ptr()), name_(name), schema_(schema) {} + + schema_parser(const std::shared_ptr & parser, const std::string & name, const nlohmann::ordered_json & schema, int id) : common_chat_peg_parser_base(id), parser_(parser), name_(name), schema_(schema) {} common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { @@ -1151,7 +1186,7 @@ class schema_parser : public common_chat_peg_parser_base { void accept(parser_visitor & visitor) override; - const common_chat_peg_parser & child() const { return parser_; } + const std::shared_ptr & child() const { return parser_; } const std::string & name() const { return name_; } @@ -1163,7 +1198,7 @@ class schema_parser : public common_chat_peg_parser_base { // expr -> term | expr "+" term class rule_parser : public common_chat_peg_parser_base, public std::enable_shared_from_this { std::string name_; - common_chat_peg_parser child_; + std::shared_ptr child_; bool trigger_; public: @@ -1171,6 +1206,9 @@ class rule_parser : public common_chat_peg_parser_base, public std::enable_share parser_type type() const override { return type_value; } rule_parser(const std::string & name, const common_chat_peg_parser & child, bool trigger, int id) + : common_chat_peg_parser_base(id), name_(name), child_(child.ptr()), trigger_(trigger) {} + + rule_parser(const std::string & name, const std::shared_ptr & child, bool trigger, int id) : common_chat_peg_parser_base(id), name_(name), child_(child), trigger_(trigger) {} common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { @@ -1226,7 +1264,7 @@ class rule_parser : public common_chat_peg_parser_base, public std::enable_share void accept(parser_visitor & visitor) override; const std::string & name() const { return name_; } - const common_chat_peg_parser & child() const { return child_; } + const std::shared_ptr & child() const { return child_; } bool is_trigger() const { return trigger_; } }; @@ -1267,7 +1305,7 @@ class ref_parser : public common_chat_peg_parser_base, public std::enable_shared // Capture content if child parser matches class capture_parser : public common_chat_peg_parser_base { - common_chat_peg_parser parser_; + std::shared_ptr parser_; std::string key_; public: @@ -1275,6 +1313,9 @@ class capture_parser : public common_chat_peg_parser_base { parser_type type() const override { return type_value; } capture_parser(const common_chat_peg_parser & parser, const std::string & key, int id) + : common_chat_peg_parser_base(id), parser_(parser.ptr()), key_(key) {} + + capture_parser(const std::shared_ptr & parser, const std::string & key, int id) : common_chat_peg_parser_base(id), parser_(parser), key_(key) {} common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { @@ -1301,13 +1342,13 @@ class capture_parser : public common_chat_peg_parser_base { void accept(parser_visitor & visitor) override; - const common_chat_peg_parser & child() const { return parser_; } + const std::shared_ptr & child() const { return parser_; } }; // Container for the root parser and all named rules in the grammar. // Manages ownership of rule registry to enable recursive grammar definitions. class root_parser : public common_chat_peg_parser_base { - common_chat_peg_parser root_; + std::shared_ptr root_; std::unordered_map> rules_; public: @@ -1325,7 +1366,7 @@ class root_parser : public common_chat_peg_parser_base { for (auto & [name, rule] : rules_) { rule->assign_id(counter); } - if (root_.ptr()) { + if (root_) { root_->assign_id(counter); } } @@ -1341,10 +1382,10 @@ class root_parser : public common_chat_peg_parser_base { } void set_root(const common_chat_peg_parser & parser) { - root_ = parser; + root_ = parser.ptr(); } - const common_chat_peg_parser & root() const { return root_; } + const std::shared_ptr & root() const { return root_; } const std::unordered_map> & rules() const { return rules_; } }; From 677c17dafe34cd9661c065814a10899d49c2fbd6 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 19:52:16 -0600 Subject: [PATCH 116/183] replace aho-corasick automation with a simple trie --- common/chat-peg-parser.cpp | 135 +++++++++++++++---------------------- 1 file changed, 53 insertions(+), 82 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index f5e0c294a3f1a..a91665beeca53 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -162,12 +162,10 @@ static bool is_hex_digit(const char c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); } -// Aho-Corasick automation for matching multiple literals. -// This is used in until_parser and to build a GBNF exclusion grammar by -// exploiting its trie structure. -class aho_corasick_matcher { +// Trie for matching multiple literals. +// This is used in until_parser and to build a GBNF exclusion grammar +class trie_matcher { struct node { - size_t fail = 0; size_t depth = 0; std::map children; std::vector word_lengths; @@ -176,50 +174,46 @@ class aho_corasick_matcher { std::vector trie; public: - aho_corasick_matcher(const std::vector & words) { + trie_matcher(const std::vector & words) { create_node(); // root node for (const auto & w : words) { insert(w); } - build_fail_links(); } - struct search_result { - size_t pos; - bool found; - bool is_partial; + struct match_result { + enum match_type { NO_MATCH, PARTIAL_MATCH, COMPLETE_MATCH } type; }; - search_result search(std::string_view sv, size_t start = 0) { - size_t current = 0; - - for (auto i = start; i < sv.size(); ++i) { - // Aho-Corasick transition - while (current != 0 && trie[current].children.find(sv[i]) == trie[current].children.end()) { - current = trie[current].fail; - } - - auto it = trie[current].children.find(sv[i]); - if (it != trie[current].children.end()) { - current = it->second; - } else { - current = 0; - } - - if (!trie[current].word_lengths.empty()) { - // Return back the longest word - size_t pos = sv.size(); - for (const auto & len : trie[current].word_lengths) { - pos = std::min(pos, i - len + 1); - } - return search_result{pos, true, false}; - } - } + // Check if a delimiter starts at the given position + match_result check_at(std::string_view sv, size_t start_pos) const { + size_t current = 0; // Start at root + size_t pos = start_pos; - if (trie[current].depth > 0) { - return search_result{sv.size() - trie[current].depth, true, true}; - } - return search_result{sv.size(), false, false}; + while (pos < sv.size()) { + auto it = trie[current].children.find(sv[pos]); + if (it == trie[current].children.end()) { + // Can't continue matching + return match_result{match_result::NO_MATCH}; + } + + current = it->second; + pos++; + + // Check if we've matched a complete word + if (!trie[current].word_lengths.empty()) { + return match_result{match_result::COMPLETE_MATCH}; + } + } + + // Reached end of input while still in the trie (not at root) + if (current != 0) { + // We're in the middle of a potential match + return match_result{match_result::PARTIAL_MATCH}; + } + + // Reached end at root (no match) + return match_result{match_result::NO_MATCH}; } struct prefix_and_next { @@ -277,37 +271,6 @@ class aho_corasick_matcher { } trie[current].word_lengths.push_back(word.length()); } - - void build_fail_links() { - std::deque queue; - - size_t root = 0; - trie[root].fail = 0; - for (const auto & it : trie[root].children) { - size_t child = it.second; - trie[child].fail = 0; - queue.push_back(child); - } - - while (!queue.empty()) { - size_t current = queue.front(); - queue.pop_front(); - - for (const auto & p : trie[current].children) { - unsigned char ch = p.first; - size_t child = p.second; - queue.push_back(child); - - auto fail = trie[current].fail; - while (fail != 0 && trie[fail].children.find(p.first) == trie[fail].children.end()) { - fail = trie[fail].fail; - } - - auto fail_it = trie[fail].children.find(ch); - trie[child].fail = fail_it != trie[fail].children.end() ? fail_it->second : 0; - } - } - } }; // Matches the start of the input @@ -1103,7 +1066,7 @@ class json_string_parser : public common_chat_peg_parser_base { // S -> (!delim .)* class until_parser : public common_chat_peg_parser_base { std::vector delimiters_; - aho_corasick_matcher matcher_; + trie_matcher matcher_; public: static constexpr parser_type type_value = UNTIL; @@ -1116,19 +1079,15 @@ class until_parser : public common_chat_peg_parser_base { : until_parser(std::vector{delimiter}, id) {} common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { - // First pass: byte-based Aho-Corasick search for delimiter - auto search_result = matcher_.search(ctx.input, start); - size_t delimiter_pos = search_result.pos; - - // Second pass: validate UTF-8 from start to delimiter_pos + // Scan input and check for delimiters size_t pos = start; size_t last_valid_pos = start; - while (pos < delimiter_pos) { + while (pos < ctx.input.size()) { auto utf8_result = parse_utf8_codepoint(ctx.input, pos); if (utf8_result.status == utf8_parse_result::INCOMPLETE) { - // Incomplete UTF-8 sequence before delimiter + // Incomplete UTF-8 sequence if (ctx.input_is_complete) { // Input is complete but UTF-8 is incomplete = malformed return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); @@ -1142,11 +1101,23 @@ class until_parser : public common_chat_peg_parser_base { return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); } + // Check if a delimiter starts at this position + auto match = matcher_.check_at(ctx.input, pos); + + if (match.type == trie_matcher::match_result::COMPLETE_MATCH) { + // Found a complete delimiter, return everything before it + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); + } + + if (match.type == trie_matcher::match_result::PARTIAL_MATCH) { + // Found a partial match extending to end of input, return everything before it + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); + } + pos += utf8_result.bytes_consumed; last_valid_pos = pos; } - // All UTF-8 validated up to delimiter return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, last_valid_pos); } @@ -1450,10 +1421,10 @@ static std::string gbnf_escape_char_class(char c) { // Create a GBNF excluding pattern static std::string gbnf_excluding_pattern(const std::vector & strings) { - // Use the aho_corasick_matcher to grab an exhaustive list of prefixes and + // Use the trie_matcher to grab an exhaustive list of prefixes and // potential next characters. We can use this to build an exclusion for // multiple strings. - aho_corasick_matcher matcher(strings); + trie_matcher matcher(strings); auto pieces = matcher.collect_prefix_and_next(); std::string pattern; From c50f2bcbefd9ca32a2b8b5338789e0c75b985b4a Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 23:55:08 -0600 Subject: [PATCH 117/183] Reset prev for qwen3 helper example variant --- tests/chat-peg-parser/test-example-qwen3-coder.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/chat-peg-parser/test-example-qwen3-coder.cpp b/tests/chat-peg-parser/test-example-qwen3-coder.cpp index c042dff95daa1..cb48d7e2b1a8a 100644 --- a/tests/chat-peg-parser/test-example-qwen3-coder.cpp +++ b/tests/chat-peg-parser/test-example-qwen3-coder.cpp @@ -74,8 +74,8 @@ void test_example_qwen3_coder(testing &t) { std::vector tokens = simple_tokenize(input); common_log_set_verbosity_thold(LOG_DEFAULT_DEBUG); - common_chat_msg prev; t.test("explicit_builder", [&](testing &t) { + common_chat_msg prev; for (auto it = tokens.begin(); it != tokens.end(); it++) { std::string in = std::accumulate(tokens.begin(), it + 1, std::string()); @@ -104,6 +104,7 @@ void test_example_qwen3_coder(testing &t) { }); t.test("helper_builder", [&](testing &t) { + common_chat_msg prev; for (auto it = tokens.begin(); it != tokens.end(); it++) { std::string in = std::accumulate(tokens.begin(), it + 1, std::string()); From 0e907b999e140b8c649eba0819fb18e5b38fe2c6 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 16 Nov 2025 23:35:54 -0600 Subject: [PATCH 118/183] refactor to use value semantics with std::variant/std::visit --- common/chat-peg-parser-helper.cpp | 21 +- common/chat-peg-parser-helper.h | 9 +- common/chat-peg-parser.cpp | 2420 ++++++----------- common/chat-peg-parser.h | 278 +- .../test-command7-parser-compare.cpp | 8 +- .../test-example-qwen3-coder.cpp | 17 +- .../test-recursive-references.cpp | 12 +- tests/chat-peg-parser/test-unicode.cpp | 12 +- 8 files changed, 1142 insertions(+), 1635 deletions(-) diff --git a/common/chat-peg-parser-helper.cpp b/common/chat-peg-parser-helper.cpp index 38c7af402fdee..43ce524a3d521 100644 --- a/common/chat-peg-parser-helper.cpp +++ b/common/chat-peg-parser-helper.cpp @@ -6,7 +6,7 @@ common_chat_peg_parser common_chat_peg_parser_builder_helper::reasoning(const st open_tag.append("<").append(tag).append(">"); std::string close_tag; close_tag.append(""); - return rule("raw-reasoning", open_tag << rule("reasoning-content", until(close_tag)) << close_tag); + return rule("raw-reasoning", literal(open_tag) << rule("reasoning-content", until(close_tag)) << literal(close_tag)); } common_chat_peg_parser common_chat_peg_parser_builder_helper::content_before_tools(const std::string & tag) { @@ -29,7 +29,7 @@ common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_no_attr( std::string param_open_after_name = ">"; - auto arg_name = rule(arg_start_name, literal(param_open) + capture("arg-name", *it) + literal(param_open_after_name)); + auto arg_name = rule(arg_start_name, literal(param_open) + capture("arg-name", literal(*it)) + literal(param_open_after_name)); std::string param_close_end; param_close_end.append(""); @@ -39,7 +39,7 @@ common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_no_attr( std::string param_peek_open; param_peek_open.append("<").append(param_tag).append("="); - auto arg_end = rule("arg-end", param_close_end + peek(literal(param_peek_open) | param_close_peek)); + auto arg_end = rule("arg-end", literal(param_close_end) + peek(literal(param_peek_open) | literal(param_close_peek))); std::string string_content_1; string_content_1.append("<").append(param_tag).append("="); @@ -76,7 +76,7 @@ common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_no_attr( std::string function_rule_name; function_rule_name.append("function-").append(function_name); - auto function = rule(function_rule_name, rule(function_start_name, function_open + capture("tool-name", function_name) + function_open_after_name) + args_sequence + function_close); + auto function = rule(function_rule_name, rule(function_start_name, literal(function_open) + capture("tool-name", literal(function_name)) + literal(function_open_after_name)) + args_sequence + literal(function_close)); return function; } @@ -108,7 +108,7 @@ common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_attr( std::string param_peek_open; param_peek_open.append("<").append(param_tag).append(" ").append(name_attr).append("=\""); - auto arg_end = rule("arg-end", param_close_end + peek(literal(param_peek_open) | param_close_peek)); + auto arg_end = rule("arg-end", literal(param_close_end) + peek(literal(param_peek_open) | literal(param_close_peek))); std::string string_content_1; string_content_1.append("<").append(param_tag).append("="); @@ -145,16 +145,9 @@ common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_attr( std::string function_rule_name; function_rule_name.append("function-").append(function_name); auto function = rule(function_rule_name, - rule(function_start_name, function_open + capture("tool-name", literal(function_name)) + - function_open_after_name) + args_sequence + function_close); + rule(function_start_name, literal(function_open) + capture("tool-name", literal(function_name)) + + literal(function_open_after_name)) + args_sequence + literal(function_close)); return function; } -common_chat_peg_parser build_peg_parser_helper( - const std::function & fn) { - common_chat_peg_parser_builder_helper builder; - auto root = fn(builder); - builder.set_root(root); - return builder.build(); -} diff --git a/common/chat-peg-parser-helper.h b/common/chat-peg-parser-helper.h index e9751f2d27ed7..6fec93d3bd941 100644 --- a/common/chat-peg-parser-helper.h +++ b/common/chat-peg-parser-helper.h @@ -24,7 +24,13 @@ class common_chat_peg_parser_builder_helper : public common_chat_peg_parser_buil const std::string &name_attr = "name"); }; -common_chat_peg_parser build_peg_parser_helper(const std::function & fn); +template +common_chat_peg_arena build_peg_parser_helper(F && fn) { + common_chat_peg_parser_builder_helper builder; + auto root = fn(builder); + builder.set_root(root); + return builder.build(); +} inline void parser_semantic_handler(const common_chat_parse_event & ev, common_chat_parse_semantics & semantics) { if (ev.rule == "reasoning-content" && ev.ending()) { @@ -114,4 +120,3 @@ inline void parser_semantic_handler_with_printout(const common_chat_parse_event LOG_ERR("Content: %s\nReasoning: %s\nTool calls: %lu\n", semantics.content.c_str(), semantics.reasoning_content.c_str(), semantics.tool_calls.size()); } - diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index a91665beeca53..01891b889cdaf 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -12,30 +12,7 @@ #include #include #include - -enum parser_type { - START, - END, - LITERAL, - SEQUENCE, - CHOICE, - REPETITION, - OPTIONAL, - ZERO_OR_MORE, - ONE_OR_MORE, - AND, - NOT, - ANY, - CHARS, - RULE, - REF, - UNTIL, - SPACE, - SCHEMA, - ROOT, - JSON_STRING, - CAPTURE, -}; +#include const char * common_chat_parse_result_type_name(common_chat_parse_result_type type) { switch (type) { @@ -46,112 +23,6 @@ const char * common_chat_parse_result_type_name(common_chat_parse_result_type ty } } -static const char * common_chat_parser_type_name(parser_type type) { - switch (type) { - case START: return "start"; - case END: return "end"; - case LITERAL: return "literal"; - case SEQUENCE: return "sequence"; - case CHOICE: return "choice"; - case REPETITION: return "repetition"; - case OPTIONAL: return "optional"; - case ZERO_OR_MORE: return "zero_or_more"; - case ONE_OR_MORE: return "one_or_more"; - case AND: return "and"; - case NOT: return "not"; - case ANY: return "any"; - case CHARS: return "chars"; - case RULE: return "rule"; - case REF: return "ref"; - case UNTIL: return "until"; - case SPACE: return "space"; - case SCHEMA: return "schema"; - case ROOT: return "root"; - case JSON_STRING: return "json_string"; - case CAPTURE: return "capture"; - default: return "unknown"; - } -} - -class parser_visitor; - -class common_chat_peg_parser_base { - protected: - int id_; - - public: - common_chat_peg_parser_base(int id) : id_(id) {} - virtual ~common_chat_peg_parser_base() = default; - - int id() const { return id_; } - void set_id(int id) { id_ = id; } - - virtual parser_type type() const = 0; - - virtual common_chat_parse_result parse(common_chat_parse_context & ctx, size_t start = 0) { - std::string indent(ctx.parse_depth * 2, ' '); - ctx.parse_depth++; - - LOG_DBG("%s[CCPP type %d] Rule: %s\n", indent.c_str(), type(), dump().c_str()); - LOG_DBG("%s[CCPP type %d] Trying to parse: %s\n", indent.c_str(), type(), ctx.input.substr(start).c_str()); - if (id_ == -1) { - // Don't cache parsers with ID -1 (from operators) - LOG_DBG("%s[CCPP type %d] Parsing uncached due to operator\n", indent.c_str(), type()); - ctx.parse_depth--; - return parse_uncached(ctx, start); - } - - auto cached = ctx.cache.get(id_, start); - if (cached) { - LOG_DBG("%s[CCPP type %d] Found cached result, returning\n", indent.c_str(), type()); - ctx.parse_depth--; - return *cached; - } - - auto result = parse_uncached(ctx, start); - LOG_DBG("%s[CCPP type %d] Parse result is: %s\n", indent.c_str(), type(), common_chat_parse_result_type_name(result.type)); - ctx.parse_depth--; - return ctx.cache.set(id_, start, result); - } - - // Actual parsing implementation (to be overridden by subclasses) - virtual common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) = 0; - - virtual void assign_id(common_chat_peg_parser_counter & counter) { - if (id_ == -1) { - id_ = counter.next(); - } - } - - virtual std::string dump() const = 0; - virtual void accept(parser_visitor & visitor) = 0; -}; - -// Create an internal parser -template -static std::shared_ptr make_parser(int id, Args&&... args) { - return std::make_shared(std::forward(args)..., id); -} - -template -static std::shared_ptr make_parser(common_chat_peg_parser_counter & counter, Args&&... args) { - return std::make_shared(std::forward(args)..., counter.next()); -} - -// Convenience cast functions -template -static std::shared_ptr cast(const std::shared_ptr & p) { - if (p->type() != T::type_value) { - return nullptr; - } - return std::static_pointer_cast(p); -} - -template -static std::shared_ptr cast(const common_chat_peg_parser & p) { - return cast(p.ptr()); -} - // We define our own space function because MSVC's std::isspace() // crashes for non-printable characters in Debug builds. static bool is_space(const char c) { @@ -163,7 +34,7 @@ static bool is_hex_digit(const char c) { } // Trie for matching multiple literals. -// This is used in until_parser and to build a GBNF exclusion grammar +// This is used in common_chat_peg_until_parser and to build a GBNF exclusion grammar class trie_matcher { struct node { size_t depth = 0; @@ -273,247 +144,231 @@ class trie_matcher { } }; -// Matches the start of the input -// S -> ^ -class start_parser : public common_chat_peg_parser_base { - public: - static constexpr parser_type type_value = START; - parser_type type() const override { return type_value; } - - start_parser(int id) : common_chat_peg_parser_base(id) {} - - void accept(parser_visitor & visitor) override; - std::string dump() const override { return "Start"; } - - common_chat_parse_result parse_uncached(common_chat_parse_context & /*ctx*/, size_t start = 0) override { - return common_chat_parse_result(start == 0 ? COMMON_CHAT_PARSE_RESULT_SUCCESS : COMMON_CHAT_PARSE_RESULT_FAIL, start); +static std::pair parse_hex_escape(const std::string & str, size_t pos, int hex_count) { + if (pos + hex_count > str.length()) { + return {0, 0}; } -}; - -// Matches the end of the input -// S -> $ -class end_parser : public common_chat_peg_parser_base { - public: - static constexpr parser_type type_value = END; - parser_type type() const override { return type_value; } - end_parser(int id) : common_chat_peg_parser_base(id) {} - - void accept(parser_visitor & visitor) override; - std::string dump() const override { return "End"; } - - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { - return common_chat_parse_result(start >= ctx.input.size() ? COMMON_CHAT_PARSE_RESULT_SUCCESS : COMMON_CHAT_PARSE_RESULT_FAIL, start); + uint32_t value = 0; + for (int i = 0; i < hex_count; i++) { + char c = str[pos + i]; + if (!is_hex_digit(c)) { + return {0, 0}; + } + value <<= 4; + if ('a' <= c && c <= 'f') { + value += c - 'a' + 10; + } else if ('A' <= c && c <= 'F') { + value += c - 'A' + 10; + } else if ('0' <= c && c <= '9') { + value += c - '0'; + } else { + break; + } } -}; - -// Matches an exact literal string. -// S -> "hello" -class literal_parser : public common_chat_peg_parser_base { - std::string literal_; - - public: - static constexpr parser_type type_value = LITERAL; - parser_type type() const override { return type_value; } - - literal_parser(const std::string & literal, int id) : common_chat_peg_parser_base(id), literal_(literal) {} + return {value, static_cast(hex_count)}; +} - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { - auto pos = start; - for (auto i = 0u; i < literal_.size(); ++i) { - if (pos >= ctx.input.size()) { - if (ctx.input_is_complete) { - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); +static std::pair parse_char_class_char(const std::string & content, size_t pos) { + if (content[pos] == '\\' && pos + 1 < content.length()) { + switch (content[pos + 1]) { + case 'x': { + auto result = parse_hex_escape(content, pos + 2, 2); + if (result.second > 0) { + return {result.first, 2 + result.second}; } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start, pos); + // Invalid escape, treat as literal 'x' + return {static_cast('x'), 2}; } - if (ctx.input[pos] != literal_[i]) { - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); + case 'u': { + auto result = parse_hex_escape(content, pos + 2, 4); + if (result.second > 0) { + return {result.first, 2 + result.second}; + } + // Invalid escape, treat as literal 'u' + return {static_cast('u'), 2}; } - ++pos; + case 'U': { + auto result = parse_hex_escape(content, pos + 2, 8); + if (result.second > 0) { + return {result.first, 2 + result.second}; + } + // Invalid escape, treat as literal 'U' + return {static_cast('U'), 2}; + } + case 'n': return {'\n', 2}; + case 't': return {'\t', 2}; + case 'r': return {'\r', 2}; + case '\\': return {'\\', 2}; + case ']': return {']', 2}; + case '-': return {'-', 2}; + case '[': return {'[', 2}; + default: return {static_cast(content[pos + 1]), 2}; } - - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); } - std::string dump() const override { - return "Literal(" + literal_ + ")"; - } + // Regular character - return as codepoint + return {static_cast(static_cast(content[pos])), 1}; +} - void accept(parser_visitor & visitor) override; +// Helper to parse common_chat_peg_chars_parser pattern and build ranges +static std::pair, bool> parse_char_classes(const std::string & classes) { + std::vector ranges; + bool negated = false; - const std::string & literal() const { return literal_; } -}; + std::string content = classes; + if (content.front() == '[') { + content = content.substr(1); + } -// Matches a sequence of parsers in order, all must succeed. -// S -> A B C -class sequence_parser : public common_chat_peg_parser_base { - std::vector> parsers_; + if (content.back() == ']') { + content.pop_back(); + } - public: - static constexpr parser_type type_value = SEQUENCE; - parser_type type() const override { return type_value; } - - template - sequence_parser(InputIt first, InputIt last, int id) : common_chat_peg_parser_base(id) { - for (auto it = first; it != last; ++it) { - auto ptr = it->ptr(); - if (auto seq = cast(ptr)) { - parsers_.insert(parsers_.end(), seq->parsers().begin(), seq->parsers().end()); - } else { - parsers_.push_back(ptr); - } - } + // Check for negation + if (!content.empty() && content.front() == '^') { + negated = true; + content = content.substr(1); } - template - sequence_parser(const T & parsers, int id) - : sequence_parser(std::begin(parsers), std::end(parsers), id) {} + size_t i = 0; + while (i < content.length()) { + auto [start, start_len] = parse_char_class_char(content, i); + i += start_len; - sequence_parser(const std::vector> & parsers, int id) - : common_chat_peg_parser_base(id) { - for (const auto & ptr : parsers) { - if (auto seq = cast(ptr)) { - parsers_.insert(parsers_.end(), seq->parsers().begin(), seq->parsers().end()); - } else { - parsers_.push_back(ptr); - } + if (i + 1 < content.length() && content[i] == '-') { + // Range detected + auto [end, end_len] = parse_char_class_char(content, i + 1); + ranges.push_back(common_chat_peg_chars_parser::char_range{start, end}); + i += 1 + end_len; + } else { + ranges.push_back(common_chat_peg_chars_parser::char_range{start, start}); } } - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { - auto pos = start; - for (const auto & p : parsers_) { - auto result = p->parse(ctx, pos); - if (!result.success()) { - return common_chat_parse_result(result.type, start, result.end); - } + return {ranges, negated}; +} - pos = result.end; - } +// Parse cache implementation +common_chat_parse_result common_chat_parse_cache::set(common_chat_peg_parser_id id, size_t start, common_chat_parse_result result) { + results[common_chat_parse_cache_key{id, start}] = result; + return result; +} - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); +std::optional common_chat_parse_cache::get(common_chat_peg_parser_id id, size_t start) { + auto it = results.find(common_chat_parse_cache_key{id, start}); + if (it != results.end()) { + return it->second; } + return std::nullopt; +} - void assign_id(common_chat_peg_parser_counter & counter) override { - common_chat_peg_parser_base::assign_id(counter); - for (auto & p : parsers_) { - p->assign_id(counter); - } - } +void common_chat_parse_cache::clear() { + results.clear(); +} - std::string dump() const override { - std::vector parts; - parts.reserve(parsers_.size()); - for (const auto & p : parsers_) { - parts.push_back(p->dump()); - } - return "Sequence(" + string_join(parts, ", ") + ")"; - } +// Forward declaration of parser +struct parser_executor; - void accept(parser_visitor & visitor) override; +// Arena implementation +common_chat_peg_arena::common_chat_peg_arena() : root_(COMMON_CHAT_PEG_INVALID_PARSER_ID) {} - const std::vector> & parsers() const { return parsers_; } -}; +common_chat_peg_parser_id common_chat_peg_arena::add_parser(common_chat_peg_parser_variant parser) { + common_chat_peg_parser_id id = parsers_.size(); + parsers_.push_back(std::move(parser)); + return id; +} -// Matches the first parser that succeeds from a list of alternatives. -// S -> A | B | C -class choice_parser : public common_chat_peg_parser_base { - std::vector> parsers_; +void common_chat_peg_arena::add_rule(const std::string & name, common_chat_peg_parser_id id) { + rules_[name] = id; +} - public: - static constexpr parser_type type_value = CHOICE; - parser_type type() const override { return type_value; } - - template - choice_parser(InputIt first, InputIt last, int id) : common_chat_peg_parser_base(id) { - for (auto it = first; it != last; ++it) { - auto ptr = it->ptr(); - if (auto choice = cast(ptr)) { - parsers_.insert(parsers_.end(), choice->parsers().begin(), choice->parsers().end()); - } else { - parsers_.push_back(ptr); - } - } +common_chat_peg_parser_id common_chat_peg_arena::get_rule(const std::string & name) const { + auto it = rules_.find(name); + if (it == rules_.end()) { + throw std::runtime_error("Rule not found: " + name); } + return it->second; +} - template - choice_parser(const T & parsers, int id) - : choice_parser(std::begin(parsers), std::end(parsers), id) {} - - choice_parser(const std::vector> & parsers, int id) - : common_chat_peg_parser_base(id) { - for (const auto & ptr : parsers) { - if (auto choice = cast(ptr)) { - parsers_.insert(parsers_.end(), choice->parsers().begin(), choice->parsers().end()); - } else { - parsers_.push_back(ptr); - } - } - } +// Parsing executor - uses std::visit to dispatch to appropriate parser +struct parser_executor { + const common_chat_peg_arena & arena; + common_chat_parse_context & ctx; + size_t start_pos; - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { - auto pos = start; - for (const auto & p : parsers_) { - auto result = p->parse(ctx, pos); - if (!result.fail()) { - return result; - } - } + parser_executor(const common_chat_peg_arena & arena, common_chat_parse_context & ctx, size_t start) + : arena(arena), ctx(ctx), start_pos(start) {} - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); + common_chat_parse_result operator()(const common_chat_peg_start_parser & /* p */) { + return common_chat_parse_result( + start_pos == 0 ? COMMON_CHAT_PARSE_RESULT_SUCCESS : COMMON_CHAT_PARSE_RESULT_FAIL, + start_pos + ); } - void assign_id(common_chat_peg_parser_counter & counter) override { - common_chat_peg_parser_base::assign_id(counter); - for (auto & p : parsers_) { - p->assign_id(counter); - } + common_chat_parse_result operator()(const common_chat_peg_end_parser & /* p */) { + return common_chat_parse_result( + start_pos >= ctx.input.size() ? COMMON_CHAT_PARSE_RESULT_SUCCESS : COMMON_CHAT_PARSE_RESULT_FAIL, + start_pos + ); } - std::string dump() const override { - std::vector parts; - parts.reserve(parsers_.size()); - for (const auto & p : parsers_) { - parts.push_back(p->dump()); + common_chat_parse_result operator()(const common_chat_peg_literal_parser & p) { + auto pos = start_pos; + for (auto i = 0u; i < p.literal.size(); ++i) { + if (pos >= ctx.input.size()) { + if (ctx.input_is_complete) { + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start_pos); + } + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos); + } + if (ctx.input[pos] != p.literal[i]) { + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start_pos); + } + ++pos; } - return "Choice(" + string_join(parts, ", ") + ")"; - } - void accept(parser_visitor & visitor) override; + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos, pos); + } - const std::vector> & parsers() const { return parsers_; } -}; + common_chat_parse_result operator()(const common_chat_peg_sequence_parser & p) { + auto pos = start_pos; + for (const auto & child_id : p.children) { + auto result = arena.parse(child_id, ctx, pos); + if (!result.success()) { + return common_chat_parse_result(result.type, start_pos, result.end); + } -// Matches between min and max repetitions of a parser (inclusive). -// S -> A{m,n} -// Use -1 for max_count to represent unbounded repetition (equivalent to {m,}) -class repetition_parser : public common_chat_peg_parser_base { - std::shared_ptr parser_; - int min_count_; - int max_count_; + pos = result.end; + } - public: - static constexpr parser_type type_value = REPETITION; - parser_type type() const override { return type_value; } + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos, pos); + } - repetition_parser(const common_chat_peg_parser & parser, int min_count, int max_count, int id) - : common_chat_peg_parser_base(id), parser_(parser.ptr()), min_count_(min_count), max_count_(max_count) {} + common_chat_parse_result operator()(const common_chat_peg_choice_parser & p) { + auto pos = start_pos; + for (const auto & child_id : p.children) { + auto result = arena.parse(child_id, ctx, pos); + if (!result.fail()) { + return result; + } + } - repetition_parser(const std::shared_ptr & parser, int min_count, int max_count, int id) - : common_chat_peg_parser_base(id), parser_(parser), min_count_(min_count), max_count_(max_count) {} + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start_pos); + } - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { - auto pos = start; + common_chat_parse_result operator()(const common_chat_peg_repetition_parser & p) { + auto pos = start_pos; int match_count = 0; // Try to match up to max_count times (or unlimited if max_count is -1) - while (max_count_ == -1 || match_count < max_count_) { + while (p.max_count == -1 || match_count < p.max_count) { if (pos >= ctx.input.size()) { break; } - auto result = parser_->parse(ctx, pos); + auto result = arena.parse(p.child, ctx, pos); if (result.success()) { // Prevent infinite loop on empty matches @@ -526,7 +381,7 @@ class repetition_parser : public common_chat_peg_parser_base { } if (result.need_more_input()) { - return common_chat_parse_result(result.type, start, result.end); + return common_chat_parse_result(result.type, start_pos, result.end); } // Child failed - stop trying @@ -534,138 +389,40 @@ class repetition_parser : public common_chat_peg_parser_base { } // Check if we got enough matches - if (min_count_ > 0 && match_count < min_count_) { + if (p.min_count > 0 && match_count < p.min_count) { if (pos >= ctx.input.size() && !ctx.input_is_complete) { - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start, pos); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos); } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start, pos); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start_pos, pos); } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); - } - - void assign_id(common_chat_peg_parser_counter & counter) override { - common_chat_peg_parser_base::assign_id(counter); - parser_->assign_id(counter); - } - - std::string dump() const override { - if (max_count_ == -1) { - return "Repetition(" + parser_->dump() + ", " + std::to_string(min_count_) + ", unbounded)"; - } - return "Repetition(" + parser_->dump() + ", " + std::to_string(min_count_) + ", " + std::to_string(max_count_) + ")"; + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos, pos); } - void accept(parser_visitor & visitor) override; - - const std::shared_ptr & child() const { return parser_; } - - int min_count() const { return min_count_; } - - int max_count() const { return max_count_; } -}; - -// Matches one or more repetitions of a parser. -// S -> A+ -class one_or_more_parser : public repetition_parser { - public: - static constexpr parser_type type_value = ONE_OR_MORE; - parser_type type() const override { return type_value; } - - one_or_more_parser(const common_chat_peg_parser & p, int id) : repetition_parser(p, 1, -1, id) {} - one_or_more_parser(const std::shared_ptr & p, int id) : repetition_parser(p, 1, -1, id) {} - - std::string dump() const override { - return "OneOrMore(" + child()->dump() + ")"; + common_chat_parse_result operator()(const common_chat_peg_one_or_more_parser & p) { + return (*this)(common_chat_peg_repetition_parser{p.child, 1, -1}); } - void accept(parser_visitor & visitor) override; -}; - -// Matches zero or more repetitions of a parser, always succeeds. -// S -> A* -class zero_or_more_parser : public repetition_parser { - public: - static constexpr parser_type type_value = ZERO_OR_MORE; - parser_type type() const override { return type_value; } - - zero_or_more_parser(const common_chat_peg_parser & p, int id) : repetition_parser(p, 0, -1, id) {} - zero_or_more_parser(const std::shared_ptr & p, int id) : repetition_parser(p, 0, -1, id) {} - - std::string dump() const override { - return "ZeroOrMore(" + child()->dump() + ")"; + common_chat_parse_result operator()(const common_chat_peg_zero_or_more_parser & p) { + return (*this)(common_chat_peg_repetition_parser{p.child, 0, -1}); } - void accept(parser_visitor & visitor) override; -}; - -// Matches zero or one occurrence of a parser, always succeeds. -// S -> A? -class optional_parser : public repetition_parser { - public: - static constexpr parser_type type_value = OPTIONAL; - parser_type type() const override { return type_value; } - - optional_parser(const common_chat_peg_parser & p, int id) : repetition_parser(p, 0, 1, id) {} - optional_parser(const std::shared_ptr & p, int id) : repetition_parser(p, 0, 1, id) {} - - std::string dump() const override { - return "Optional(" + child()->dump() + ")"; + common_chat_parse_result operator()(const common_chat_peg_optional_parser & p) { + return (*this)(common_chat_peg_repetition_parser{p.child, 0, 1}); } - void accept(parser_visitor & visitor) override; -}; - -// Positive lookahead: succeeds if child parser succeeds, consumes no input. -// S -> &A -class and_parser : public common_chat_peg_parser_base { - std::shared_ptr parser_; - - public: - static constexpr parser_type type_value = AND; - parser_type type() const override { return type_value; } - - and_parser(const common_chat_peg_parser & parser, int id) : common_chat_peg_parser_base(id), parser_(parser.ptr()) {} - and_parser(const std::shared_ptr & parser, int id) : common_chat_peg_parser_base(id), parser_(parser) {} - - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { - auto result = parser_->parse(ctx, start); + common_chat_parse_result operator()(const common_chat_peg_and_parser & p) { + auto result = arena.parse(p.child, ctx, start_pos); // Pass result but don't consume input - return common_chat_parse_result(result.type, start); - } - - void assign_id(common_chat_peg_parser_counter & counter) override { - common_chat_peg_parser_base::assign_id(counter); - parser_->assign_id(counter); - } - - std::string dump() const override { - return "And(" + parser_->dump() + ")"; + return common_chat_parse_result(result.type, start_pos); } - void accept(parser_visitor & visitor) override; - - const std::shared_ptr & child() const { return parser_; } -}; - -// Negative lookahead: succeeds if child parser fails, consumes no input. -// S -> !A -class not_parser : public common_chat_peg_parser_base { - std::shared_ptr parser_; - - public: - static constexpr parser_type type_value = NOT; - parser_type type() const override { return type_value; } - - not_parser(const common_chat_peg_parser & parser, int id) : common_chat_peg_parser_base(id), parser_(parser.ptr()) {} - not_parser(const std::shared_ptr & parser, int id) : common_chat_peg_parser_base(id), parser_(parser) {} - - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { - auto result = parser_->parse(ctx, start); + common_chat_parse_result operator()(const common_chat_peg_not_parser & p) { + auto result = arena.parse(p.child, ctx, start_pos); if (result.success()) { // Fail if the underlying parser matches - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start_pos); } if (result.need_more_input()) { @@ -674,66 +431,27 @@ class not_parser : public common_chat_peg_parser_base { } // Child failed, so negation succeeds - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos); } - void assign_id(common_chat_peg_parser_counter & counter) override { - common_chat_peg_parser_base::assign_id(counter); - parser_->assign_id(counter); - } - - std::string dump() const override { - return "Not(" + parser_->dump() + ")"; - } - - void accept(parser_visitor & visitor) override; - - const std::shared_ptr & child() const { return parser_; } -}; - -// Matches any single character. -// S -> . -class any_parser : public common_chat_peg_parser_base { - public: - static constexpr parser_type type_value = ANY; - parser_type type() const override { return type_value; } - - any_parser(int id) : common_chat_peg_parser_base(id) {} - - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { + common_chat_parse_result operator()(const common_chat_peg_any_parser & /* p */) { // Parse a single UTF-8 codepoint (not just a single byte) - auto result = parse_utf8_codepoint(ctx.input, start); + auto result = parse_utf8_codepoint(ctx.input, start_pos); if (result.status == utf8_parse_result::INCOMPLETE) { if (ctx.input_is_complete) { - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start_pos); } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start_pos); } if (result.status == utf8_parse_result::INVALID) { - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start_pos); } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, start + result.bytes_consumed); - } - - std::string dump() const override { - return "Any"; + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos, start_pos + result.bytes_consumed); } - void accept(parser_visitor & visitor) override; -}; - -// Matches zero or more whitespace characters (space, tab, newline). -// S -> [ \t\n]* -class space_parser : public common_chat_peg_parser_base { - public: - static constexpr parser_type type_value = SPACE; - parser_type type() const override { return type_value; } - - space_parser(int id) : common_chat_peg_parser_base(id) {} - - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { - auto pos = start; + common_chat_parse_result operator()(const common_chat_peg_space_parser & /* p */) { + auto pos = start_pos; while (pos < ctx.input.size()) { char c = ctx.input[pos]; if (is_space(c)) { @@ -743,171 +461,42 @@ class space_parser : public common_chat_peg_parser_base { } } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); - } - - std::string dump() const override { - return "Space"; + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos, pos); } - void accept(parser_visitor & visitor) override; -}; + common_chat_parse_result operator()(const common_chat_peg_chars_parser & p) { + auto pos = start_pos; + int match_count = 0; -static std::pair parse_hex_escape(const std::string & str, size_t pos, int hex_count) { - if (pos + hex_count > str.length()) { - return {0, 0}; - } - - uint32_t value = 0; - for (int i = 0; i < hex_count; i++) { - char c = str[pos + i]; - if (!is_hex_digit(c)) { - return {0, 0}; - } - value <<= 4; - if ('a' <= c && c <= 'f') { - value += c - 'a' + 10; - } else if ('A' <= c && c <= 'F') { - value += c - 'A' + 10; - } else if ('0' <= c && c <= '9') { - value += c - '0'; - } else { - break; - } - } - return {value, static_cast(hex_count)}; -} - -static std::pair parse_char_class_char(const std::string & content, size_t pos) { - if (content[pos] == '\\' && pos + 1 < content.length()) { - switch (content[pos + 1]) { - case 'x': { - auto result = parse_hex_escape(content, pos + 2, 2); - if (result.second > 0) { - return {result.first, 2 + result.second}; - } - // Invalid escape, treat as literal 'x' - return {static_cast('x'), 2}; - } - case 'u': { - auto result = parse_hex_escape(content, pos + 2, 4); - if (result.second > 0) { - return {result.first, 2 + result.second}; - } - // Invalid escape, treat as literal 'u' - return {static_cast('u'), 2}; - } - case 'U': { - auto result = parse_hex_escape(content, pos + 2, 8); - if (result.second > 0) { - return {result.first, 2 + result.second}; - } - // Invalid escape, treat as literal 'U' - return {static_cast('U'), 2}; - } - case 'n': return {'\n', 2}; - case 't': return {'\t', 2}; - case 'r': return {'\r', 2}; - case '\\': return {'\\', 2}; - case ']': return {']', 2}; - case '-': return {'-', 2}; - case '[': return {'[', 2}; - default: return {static_cast(content[pos + 1]), 2}; - } - } - - // Regular character - return as codepoint - return {static_cast(static_cast(content[pos])), 1}; -} - -// Matches between min and max repetitions of characters from a character class. -// S -> [a-z]{m,n} -// Supports Unicode codepoint ranges and escape sequences: \xXX \uXXXX \UXXXXXXXX -class chars_parser : public common_chat_peg_parser_base { - struct char_range { - uint32_t start; - uint32_t end; - - bool contains(uint32_t codepoint) const { return codepoint >= start && codepoint <= end; } - }; - - std::string pattern_; - std::vector ranges_; - bool negated_; - int min_count_; - int max_count_; - - public: - static constexpr parser_type type_value = CHARS; - parser_type type() const override { return type_value; } - - chars_parser(const std::string & classes, int min_count, int max_count, int id) - : common_chat_peg_parser_base(id), pattern_(classes), negated_(false), min_count_(min_count), max_count_(max_count) { - - std::string content = classes; - if (content.front() == '[') { - content = content.substr(1); - } - - if (content.back() == ']') { - content.pop_back(); - } - - // Check for negation - if (!content.empty() && content.front() == '^') { - negated_ = true; - content = content.substr(1); - } - - size_t i = 0; - while (i < content.length()) { - auto [start, start_len] = parse_char_class_char(content, i); - i += start_len; - - if (i + 1 < content.length() && content[i] == '-') { - // Range detected - auto [end, end_len] = parse_char_class_char(content, i + 1); - ranges_.push_back(char_range{start, end}); - i += 1 + end_len; - } else { - ranges_.push_back(char_range{start, start}); - } - } - } - - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { - auto pos = start; - int match_count = 0; - - // Try to match up to max_count times (or unlimited if max_count is -1) - while (max_count_ == -1 || match_count < max_count_) { - auto result = parse_utf8_codepoint(ctx.input, pos); + // Try to match up to max_count times (or unlimited if max_count is -1) + while (p.max_count == -1 || match_count < p.max_count) { + auto result = parse_utf8_codepoint(ctx.input, pos); if (result.status == utf8_parse_result::INCOMPLETE) { - if (match_count >= min_count_) { + if (match_count >= p.min_count) { // We have enough matches, succeed with what we have - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos, pos); } // Not enough matches yet if (ctx.input_is_complete) { - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start_pos); } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start, pos); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos); } if (result.status == utf8_parse_result::INVALID) { // Malformed UTF-8 in input - if (match_count >= min_count_) { + if (match_count >= p.min_count) { // We have enough matches, succeed up to here - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos, pos); } // Not enough matches, fail - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start_pos); } // Check if this codepoint matches our character class bool matches = false; - for (const auto & range : ranges_) { + for (const auto & range : p.ranges) { if (range.contains(result.codepoint)) { matches = true; break; @@ -915,7 +504,7 @@ class chars_parser : public common_chat_peg_parser_base { } // If negated, invert the match result - if (negated_) { + if (p.negated) { matches = !matches; } @@ -929,93 +518,16 @@ class chars_parser : public common_chat_peg_parser_base { } // Check if we got enough matches - if (match_count < min_count_) { + if (match_count < p.min_count) { if (pos >= ctx.input.size() && !ctx.input_is_complete) { - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start, pos); - } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start, pos); - } - - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); - } - - std::string dump() const override { - if (max_count_ == -1) { - return "CharRepeat(" + pattern_ + ", " + std::to_string(min_count_) + ", unbounded)"; - } - return "CharRepeat(" + pattern_ + ", " + std::to_string(min_count_) + ", " + std::to_string(max_count_) + ")"; - } - - void accept(parser_visitor & visitor) override; - - const std::string & pattern() const { return pattern_; } - - int min_count() const { return min_count_; } - - int max_count() const { return max_count_; } -}; - -// Specialized parser for JSON string content (without quotes). -// Parses the content between quotes with single-pass streaming support. -// Stops before the closing quote (doesn't consume it). -// Handles escape sequences and emits NEED_MORE_INPUT for incomplete input. -// S -> (regular chars and escape sequences)* until closing " -class json_string_parser : public common_chat_peg_parser_base { - public: - static constexpr parser_type type_value = JSON_STRING; - parser_type type() const override { return type_value; } - - json_string_parser(int id) : common_chat_peg_parser_base(id) {} - - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { - auto pos = start; - - // Parse string content (without quotes) - while (pos < ctx.input.size()) { - char c = ctx.input[pos]; - - if (c == '"') { - // Found closing quote - success (don't consume it) - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); - } - - if (c == '\\') { - auto result = handle_escape_sequence(ctx, start, pos); - if (!result.success()) { - return result; - } - } else { - auto utf8_result = parse_utf8_codepoint(ctx.input, pos); - - if (utf8_result.status == utf8_parse_result::INCOMPLETE) { - if (ctx.input_is_complete) { - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); - } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start, pos); - } - - if (utf8_result.status == utf8_parse_result::INVALID) { - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); - } - - pos += utf8_result.bytes_consumed; + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos); } + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start_pos, pos); } - // Reached end without finding closing quote - if (ctx.input_is_complete) { - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start, pos); - } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start, pos); - } - - std::string dump() const override { - return "JsonString()"; + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos, pos); } - void accept(parser_visitor & visitor) override; - - private: static common_chat_parse_result handle_escape_sequence(common_chat_parse_context & ctx, size_t start, size_t & pos) { ++pos; // consume '\' if (pos >= ctx.input.size()) { @@ -1060,28 +572,55 @@ class json_string_parser : public common_chat_peg_parser_base { } return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); } -}; -// Matches all characters until a delimiter is found (delimiter not consumed). -// S -> (!delim .)* -class until_parser : public common_chat_peg_parser_base { - std::vector delimiters_; - trie_matcher matcher_; + common_chat_parse_result operator()(const common_chat_peg_json_string_parser & /* p */) { + auto pos = start_pos; - public: - static constexpr parser_type type_value = UNTIL; - parser_type type() const override { return type_value; } + // Parse string content (without quotes) + while (pos < ctx.input.size()) { + char c = ctx.input[pos]; + + if (c == '"') { + // Found closing quote - success (don't consume it) + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos, pos); + } + + if (c == '\\') { + auto result = handle_escape_sequence(ctx, start_pos, pos); + if (!result.success()) { + return result; + } + } else { + auto utf8_result = parse_utf8_codepoint(ctx.input, pos); + + if (utf8_result.status == utf8_parse_result::INCOMPLETE) { + if (ctx.input_is_complete) { + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start_pos); + } + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos); + } + + if (utf8_result.status == utf8_parse_result::INVALID) { + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start_pos); + } + + pos += utf8_result.bytes_consumed; + } + } - until_parser(const std::vector & delimiters, int id) - : common_chat_peg_parser_base(id), delimiters_(delimiters), matcher_(delimiters) {} + // Reached end without finding closing quote + if (ctx.input_is_complete) { + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start_pos, pos); + } + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos); + } - until_parser(const std::string & delimiter, int id) - : until_parser(std::vector{delimiter}, id) {} + common_chat_parse_result operator()(const common_chat_peg_until_parser & p) { + trie_matcher matcher(p.delimiters); - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { // Scan input and check for delimiters - size_t pos = start; - size_t last_valid_pos = start; + size_t pos = start_pos; + size_t last_valid_pos = start_pos; while (pos < ctx.input.size()) { auto utf8_result = parse_utf8_codepoint(ctx.input, pos); @@ -1090,106 +629,49 @@ class until_parser : public common_chat_peg_parser_base { // Incomplete UTF-8 sequence if (ctx.input_is_complete) { // Input is complete but UTF-8 is incomplete = malformed - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start_pos); } // Return what we have so far (before incomplete sequence) - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start, last_valid_pos); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start_pos, last_valid_pos); } if (utf8_result.status == utf8_parse_result::INVALID) { // Malformed UTF-8 - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start_pos); } // Check if a delimiter starts at this position - auto match = matcher_.check_at(ctx.input, pos); + auto match = matcher.check_at(ctx.input, pos); if (match.type == trie_matcher::match_result::COMPLETE_MATCH) { // Found a complete delimiter, return everything before it - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos, pos); } if (match.type == trie_matcher::match_result::PARTIAL_MATCH) { // Found a partial match extending to end of input, return everything before it - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos, pos); } pos += utf8_result.bytes_consumed; last_valid_pos = pos; } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, last_valid_pos); - } - - std::string dump() const override { - return "Until(" + string_join(delimiters_, " | ") + ")"; - } - - void accept(parser_visitor & visitor) override; - - std::vector delimiters() const { return delimiters_; } -}; - -// Wraps a parser with JSON schema metadata for grammar generation. -// Used internally to convert JSON schemas to GBNF grammar rules. -class schema_parser : public common_chat_peg_parser_base { - std::shared_ptr parser_; - std::string name_; - nlohmann::ordered_json schema_; - - public: - static constexpr parser_type type_value = SCHEMA; - parser_type type() const override { return type_value; } - - schema_parser(const common_chat_peg_parser & parser, const std::string & name, const nlohmann::ordered_json & schema, int id) - : common_chat_peg_parser_base(id), parser_(parser.ptr()), name_(name), schema_(schema) {} - - schema_parser(const std::shared_ptr & parser, const std::string & name, const nlohmann::ordered_json & schema, int id) - : common_chat_peg_parser_base(id), parser_(parser), name_(name), schema_(schema) {} - - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { - return parser_->parse(ctx, start); + return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos, last_valid_pos); } - std::string dump() const override { - return "Schema(" + parser_->dump() + ", " + schema_.dump() + ")"; + common_chat_parse_result operator()(const common_chat_peg_schema_parser & p) { + return arena.parse(p.child, ctx, start_pos); } - void accept(parser_visitor & visitor) override; - - const std::shared_ptr & child() const { return parser_; } - - const std::string & name() const { return name_; } - - const nlohmann::ordered_json & schema() const { return schema_; } -}; - -// Defines a named rule for recursive or reusable grammar definitions. -// Owns the implementation and fires NODE_START/END events. -// expr -> term | expr "+" term -class rule_parser : public common_chat_peg_parser_base, public std::enable_shared_from_this { - std::string name_; - std::shared_ptr child_; - bool trigger_; - - public: - static constexpr parser_type type_value = RULE; - parser_type type() const override { return type_value; } - - rule_parser(const std::string & name, const common_chat_peg_parser & child, bool trigger, int id) - : common_chat_peg_parser_base(id), name_(name), child_(child.ptr()), trigger_(trigger) {} - - rule_parser(const std::string & name, const std::shared_ptr & child, bool trigger, int id) - : common_chat_peg_parser_base(id), name_(name), child_(child), trigger_(trigger) {} - - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { + common_chat_parse_result operator()(const common_chat_peg_rule_parser & p) { // Fire NODE_START event if (ctx.event_handler && ctx.semantics) { ctx.event_handler(common_chat_parse_event{ COMMON_CHAT_PARSE_EVENT_NODE_START, - name_, - start, - start, + p.name, + start_pos, + start_pos, "", COMMON_CHAT_PARSE_RESULT_FAIL, ctx.current_depth @@ -1198,7 +680,7 @@ class rule_parser : public common_chat_peg_parser_base, public std::enable_share } // Parse the child - auto result = child_->parse(ctx, start); + auto result = arena.parse(p.child, ctx, start_pos); // Fire NODE_END event if (ctx.event_handler && ctx.semantics) { @@ -1211,7 +693,7 @@ class rule_parser : public common_chat_peg_parser_base, public std::enable_share } ctx.event_handler(common_chat_parse_event{ COMMON_CHAT_PARSE_EVENT_NODE_END, - name_, + p.name, result.start, result.end, text, @@ -1223,173 +705,428 @@ class rule_parser : public common_chat_peg_parser_base, public std::enable_share return result; } - void assign_id(common_chat_peg_parser_counter & counter) override { - common_chat_peg_parser_base::assign_id(counter); - child_->assign_id(counter); + common_chat_parse_result operator()(const common_chat_peg_ref_parser & p) { + auto rule_id = arena.get_rule(p.name); + return arena.parse(rule_id, ctx, start_pos); } - std::string dump() const override { - return "Rule(" + name_ + ", " + child_->dump() + ")"; - } + common_chat_parse_result operator()(const common_chat_peg_capture_parser & p) { + auto result = arena.parse(p.child, ctx, start_pos); - void accept(parser_visitor & visitor) override; + if (!result.fail() && ctx.semantics) { + std::string_view matched = ctx.input; + matched = matched.substr(result.start, result.end - result.start); + std::string value = std::string(matched); + ctx.semantics->captures[p.key] = std::move(value); + } - const std::string & name() const { return name_; } - const std::shared_ptr & child() const { return child_; } - bool is_trigger() const { return trigger_; } + return result; + } }; -// References a named rule (lightweight reference, resolved during resolution phase) -// expr_ref -> expr -class ref_parser : public common_chat_peg_parser_base, public std::enable_shared_from_this { - std::string name_; - std::weak_ptr target_; - - public: - static constexpr parser_type type_value = REF; - parser_type type() const override { return type_value; } +common_chat_parse_result common_chat_peg_arena::parse(common_chat_parse_context & ctx, size_t start) const { + if (root_ == COMMON_CHAT_PEG_INVALID_PARSER_ID) { + throw std::runtime_error("No root parser set"); + } + return parse(root_, ctx, start); +} - ref_parser(const std::string & name, int id) - : common_chat_peg_parser_base(id), name_(name) {} +common_chat_parse_result common_chat_peg_arena::parse(common_chat_peg_parser_id id, common_chat_parse_context & ctx, size_t start) const { + // Check cache + auto cached = ctx.cache.get(id, start); + if (cached) { + return *cached; + } - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { - auto target = target_.lock(); - if (!target) { - LOG_ERR("ref_parser::parse called with unresolved reference '%s'\n", name_.c_str()); - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); - } + // Execute parser + const auto & parser = parsers_.at(id); + parser_executor exec(*this, ctx, start); + auto result = std::visit(exec, parser); - // Delegate to the target rule parser - return target->parse(ctx, start); - } + // Cache result + return ctx.cache.set(id, start, result); +} - std::string dump() const override { - return "Ref(" + name_ + ")"; - } +// Dump implementation (for debugging) +std::string common_chat_peg_arena::dump(common_chat_peg_parser_id id) const { + const auto & parser = parsers_.at(id); + + return std::visit([this](const auto & p) -> std::string { + using T = std::decay_t; + + if constexpr (std::is_same_v) { + return "Start"; + } else if constexpr (std::is_same_v) { + return "End"; + } else if constexpr (std::is_same_v) { + return "Literal(" + p.literal + ")"; + } else if constexpr (std::is_same_v) { + std::vector parts; + for (const auto & child : p.children) { + parts.push_back(dump(child)); + } + return "Sequence(" + string_join(parts, ", ") + ")"; + } else if constexpr (std::is_same_v) { + std::vector parts; + for (const auto & child : p.children) { + parts.push_back(dump(child)); + } + return "Choice(" + string_join(parts, ", ") + ")"; + } else if constexpr (std::is_same_v) { + if (p.max_count == -1) { + return "Repetition(" + dump(p.child) + ", " + std::to_string(p.min_count) + ", unbounded)"; + } + return "Repetition(" + dump(p.child) + ", " + std::to_string(p.min_count) + ", " + std::to_string(p.max_count) + ")"; + } else if constexpr (std::is_same_v) { + return "OneOrMore(" + dump(p.child) + ")"; + } else if constexpr (std::is_same_v) { + return "ZeroOrMore(" + dump(p.child) + ")"; + } else if constexpr (std::is_same_v) { + return "Optional(" + dump(p.child) + ")"; + } else if constexpr (std::is_same_v) { + return "And(" + dump(p.child) + ")"; + } else if constexpr (std::is_same_v) { + return "Not(" + dump(p.child) + ")"; + } else if constexpr (std::is_same_v) { + return "Any"; + } else if constexpr (std::is_same_v) { + return "Space"; + } else if constexpr (std::is_same_v) { + if (p.max_count == -1) { + return "CharRepeat(" + p.pattern + ", " + std::to_string(p.min_count) + ", unbounded)"; + } + return "CharRepeat(" + p.pattern + ", " + std::to_string(p.min_count) + ", " + std::to_string(p.max_count) + ")"; + } else if constexpr (std::is_same_v) { + return "JsonString()"; + } else if constexpr (std::is_same_v) { + return "Until(" + string_join(p.delimiters, " | ") + ")"; + } else if constexpr (std::is_same_v) { + return "Schema(" + dump(p.child) + ", " + (p.schema ? p.schema->dump() : "null") + ")"; + } else if constexpr (std::is_same_v) { + return "Rule(" + p.name + ", " + dump(p.child) + ")"; + } else if constexpr (std::is_same_v) { + return "Ref(" + p.name + ")"; + } else if constexpr (std::is_same_v) { + return "Capture(" + p.key + ", " + dump(p.child) + ")"; + } else { + return "Unknown"; + } + }, parser); +} - void accept(parser_visitor & visitor) override; +// Parser wrapper operator implementations +common_chat_peg_parser common_chat_peg_parser::operator+(const common_chat_peg_parser & other) const { + return builder_->sequence({id_, other.id_}); +} - const std::string & name() const { return name_; } - void set_target(const std::weak_ptr & target) { target_ = target; } - std::weak_ptr target() const { return target_; } -}; +common_chat_peg_parser common_chat_peg_parser::operator|(const common_chat_peg_parser & other) const { + return builder_->choice({id_, other.id_}); +} -// Capture content if child parser matches -class capture_parser : public common_chat_peg_parser_base { - std::shared_ptr parser_; - std::string key_; +common_chat_peg_parser common_chat_peg_parser::operator<<(const common_chat_peg_parser & other) const { + return builder_->sequence({id_, builder_->space(), other.id_}); +} - public: - static constexpr parser_type type_value = CAPTURE; - parser_type type() const override { return type_value; } +// String literal overloads +common_chat_peg_parser common_chat_peg_parser::operator+(const char * str) const { + return *this + builder_->literal(str); +} - capture_parser(const common_chat_peg_parser & parser, const std::string & key, int id) - : common_chat_peg_parser_base(id), parser_(parser.ptr()), key_(key) {} +common_chat_peg_parser common_chat_peg_parser::operator+(const std::string & str) const { + return *this + builder_->literal(str); +} - capture_parser(const std::shared_ptr & parser, const std::string & key, int id) - : common_chat_peg_parser_base(id), parser_(parser), key_(key) {} +common_chat_peg_parser common_chat_peg_parser::operator|(const char * str) const { + return *this | builder_->literal(str); +} - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { - auto result = parser_->parse(ctx, start); +common_chat_peg_parser common_chat_peg_parser::operator|(const std::string & str) const { + return *this | builder_->literal(str); +} - if (!result.fail() && ctx.semantics) { - std::string_view matched = ctx.input; - matched = matched.substr(result.start, result.end - result.start); - std::string value = std::string(matched); - ctx.semantics->captures[key_] = std::move(value); - } +common_chat_peg_parser common_chat_peg_parser::operator<<(const char * str) const { + return *this << builder_->literal(str); +} - return result; - } +common_chat_peg_parser common_chat_peg_parser::operator<<(const std::string & str) const { + return *this << builder_->literal(str); +} - void assign_id(common_chat_peg_parser_counter & counter) override { - common_chat_peg_parser_base::assign_id(counter); - parser_->assign_id(counter); - } +// Free function operators for string + parser +common_chat_peg_parser operator+(const char * str, const common_chat_peg_parser & p) { + return p.builder()->literal(str) + p; +} - std::string dump() const override { - return "Capture(" + key_ + ", " + parser_->dump() + ")"; - } +common_chat_peg_parser operator+(const std::string & str, const common_chat_peg_parser & p) { + return operator+(str.c_str(), p); +} - void accept(parser_visitor & visitor) override; +common_chat_peg_parser operator<<(const char * str, const common_chat_peg_parser & p) { + return p.builder()->literal(str) << p; +} - const std::shared_ptr & child() const { return parser_; } -}; +common_chat_peg_parser operator<<(const std::string & str, const common_chat_peg_parser & p) { + return operator<<(str.c_str(), p); +} -// Container for the root parser and all named rules in the grammar. -// Manages ownership of rule registry to enable recursive grammar definitions. -class root_parser : public common_chat_peg_parser_base { - std::shared_ptr root_; - std::unordered_map> rules_; +// Builder implementation +common_chat_peg_parser_builder::common_chat_peg_parser_builder() {} - public: - static constexpr parser_type type_value = ROOT; - parser_type type() const override { return type_value; } +common_chat_peg_parser common_chat_peg_parser_builder::start() { + return wrap(arena_.add_parser(common_chat_peg_start_parser{})); +} - root_parser(int id) : common_chat_peg_parser_base(id) {} +common_chat_peg_parser common_chat_peg_parser_builder::end() { + return wrap(arena_.add_parser(common_chat_peg_end_parser{})); +} - common_chat_parse_result parse_uncached(common_chat_parse_context & ctx, size_t start = 0) override { - return root_->parse(ctx, start); - } +common_chat_peg_parser common_chat_peg_parser_builder::literal(const std::string & literal) { + return wrap(arena_.add_parser(common_chat_peg_literal_parser{literal})); +} - void assign_id(common_chat_peg_parser_counter & counter) override { - common_chat_peg_parser_base::assign_id(counter); - for (auto & [name, rule] : rules_) { - rule->assign_id(counter); - } - if (root_) { - root_->assign_id(counter); +common_chat_peg_parser common_chat_peg_parser_builder::sequence(const std::vector & parsers) { + // Flatten nested sequences + std::vector flattened; + for (const auto & p : parsers) { + const auto & parser = arena_.get(p); + if (auto seq = std::get_if(&parser)) { + flattened.insert(flattened.end(), seq->children.begin(), seq->children.end()); + } else { + flattened.push_back(p); } } + return wrap(arena_.add_parser(common_chat_peg_sequence_parser{flattened})); +} - std::string dump() const override { - return root_->dump(); +common_chat_peg_parser common_chat_peg_parser_builder::sequence(const std::vector & parsers) { + std::vector ids; + ids.reserve(parsers.size()); + for (const auto & p : parsers) { + ids.push_back(p.id()); } + return sequence(ids); +} - void accept(parser_visitor & visitor) override; - - void add_rule(const std::string & name, const std::shared_ptr & rule) { - rules_[name] = rule; +common_chat_peg_parser common_chat_peg_parser_builder::sequence(std::initializer_list parsers) { + std::vector ids; + ids.reserve(parsers.size()); + for (const auto & p : parsers) { + ids.push_back(p.id()); } + return sequence(ids); +} - void set_root(const common_chat_peg_parser & parser) { - root_ = parser.ptr(); +common_chat_peg_parser common_chat_peg_parser_builder::choice(const std::vector & parsers) { + // Flatten nested choices + std::vector flattened; + for (const auto & p : parsers) { + const auto & parser = arena_.get(p); + if (auto choice = std::get_if(&parser)) { + flattened.insert(flattened.end(), choice->children.begin(), choice->children.end()); + } else { + flattened.push_back(p); + } } + return wrap(arena_.add_parser(common_chat_peg_choice_parser{flattened})); +} - const std::shared_ptr & root() const { return root_; } +common_chat_peg_parser common_chat_peg_parser_builder::choice(const std::vector & parsers) { + std::vector ids; + ids.reserve(parsers.size()); + for (const auto & p : parsers) { + ids.push_back(p.id()); + } + return choice(ids); +} - const std::unordered_map> & rules() const { return rules_; } -}; +common_chat_peg_parser common_chat_peg_parser_builder::choice(std::initializer_list parsers) { + std::vector ids; + ids.reserve(parsers.size()); + for (const auto & p : parsers) { + ids.push_back(p.id()); + } + return choice(ids); +} -// Base visitor class for parser tree traversal -class parser_visitor { - public: - virtual ~parser_visitor() = default; - - virtual void visit(start_parser & p) = 0; - virtual void visit(end_parser & p) = 0; - virtual void visit(literal_parser & p) = 0; - virtual void visit(sequence_parser & p) = 0; - virtual void visit(choice_parser & p) = 0; - virtual void visit(one_or_more_parser & p) = 0; - virtual void visit(zero_or_more_parser & p) = 0; - virtual void visit(optional_parser & p) = 0; - virtual void visit(repetition_parser & p) = 0; - virtual void visit(until_parser & p) = 0; - virtual void visit(and_parser & p) = 0; - virtual void visit(not_parser & p) = 0; - virtual void visit(any_parser & p) = 0; - virtual void visit(space_parser & p) = 0; - virtual void visit(chars_parser & p) = 0; - virtual void visit(json_string_parser & p) = 0; - virtual void visit(schema_parser & p) = 0; - virtual void visit(rule_parser & p) = 0; - virtual void visit(ref_parser & p) = 0; - virtual void visit(root_parser & p) = 0; - virtual void visit(capture_parser & p) = 0; -}; +common_chat_peg_parser common_chat_peg_parser_builder::one_or_more(common_chat_peg_parser p) { + return wrap(arena_.add_parser(common_chat_peg_one_or_more_parser{p.id()})); +} + +common_chat_peg_parser common_chat_peg_parser_builder::zero_or_more(common_chat_peg_parser p) { + return wrap(arena_.add_parser(common_chat_peg_zero_or_more_parser{p.id()})); +} + +common_chat_peg_parser common_chat_peg_parser_builder::optional(common_chat_peg_parser p) { + return wrap(arena_.add_parser(common_chat_peg_optional_parser{p.id()})); +} + +common_chat_peg_parser common_chat_peg_parser_builder::peek(common_chat_peg_parser p) { + return wrap(arena_.add_parser(common_chat_peg_and_parser{p.id()})); +} + +common_chat_peg_parser common_chat_peg_parser_builder::negate(common_chat_peg_parser p) { + return wrap(arena_.add_parser(common_chat_peg_not_parser{p.id()})); +} + +common_chat_peg_parser common_chat_peg_parser_builder::any() { + return wrap(arena_.add_parser(common_chat_peg_any_parser{})); +} + +common_chat_peg_parser common_chat_peg_parser_builder::chars(const std::string & classes, int min, int max) { + auto [ranges, negated] = parse_char_classes(classes); + return wrap(arena_.add_parser(common_chat_peg_chars_parser{classes, ranges, negated, min, max})); +} + +common_chat_peg_parser common_chat_peg_parser_builder::one(const std::string & classes) { + return chars(classes, 1, 1); +} + +common_chat_peg_parser common_chat_peg_parser_builder::ref(const std::string & name) { + return wrap(arena_.add_parser(common_chat_peg_ref_parser{name})); +} + +common_chat_peg_parser common_chat_peg_parser_builder::space() { + return wrap(arena_.add_parser(common_chat_peg_space_parser{})); +} + +common_chat_peg_parser common_chat_peg_parser_builder::until(const std::string & delimiter) { + return wrap(arena_.add_parser(common_chat_peg_until_parser{std::vector{delimiter}})); +} + +common_chat_peg_parser common_chat_peg_parser_builder::until_one_of(const std::vector & delimiters) { + return wrap(arena_.add_parser(common_chat_peg_until_parser{delimiters})); +} + +common_chat_peg_parser common_chat_peg_parser_builder::repeat(common_chat_peg_parser p, int min, int max) { + return wrap(arena_.add_parser(common_chat_peg_repetition_parser{p.id(), min, max})); +} + +common_chat_peg_parser common_chat_peg_parser_builder::repeat(common_chat_peg_parser p, int n) { + return wrap(arena_.add_parser(common_chat_peg_repetition_parser{p.id(), n, n})); +} + +common_chat_peg_parser common_chat_peg_parser_builder::json_string_content() { + return wrap(arena_.add_parser(common_chat_peg_json_string_parser{})); +} + +common_chat_peg_parser common_chat_peg_parser_builder::schema(common_chat_peg_parser p, const std::string & name, const nlohmann::ordered_json & schema) { + return wrap(arena_.add_parser(common_chat_peg_schema_parser{p.id(), name, std::make_shared(schema)})); +} + +common_chat_peg_parser common_chat_peg_parser_builder::capture(const std::string & key, common_chat_peg_parser p) { + return wrap(arena_.add_parser(common_chat_peg_capture_parser{p.id(), key})); +} + +common_chat_peg_parser common_chat_peg_parser_builder::rule(const std::string & name, common_chat_peg_parser p, bool trigger) { + auto rule_id = arena_.add_parser(common_chat_peg_rule_parser{name, p.id(), trigger}); + arena_.add_rule(name, rule_id); + return ref(name); +} + +common_chat_peg_parser common_chat_peg_parser_builder::rule(const std::string & name, const std::function & builder_fn, bool trigger) { + // Check if rule already exists + if (arena_.has_rule(name)) { + return ref(name); + } + + // Create placeholder rule to allow recursive references + auto placeholder = any(); // Temporary placeholder + auto placeholder_rule_id = arena_.add_parser(common_chat_peg_rule_parser{name, placeholder.id(), trigger}); + arena_.add_rule(name, placeholder_rule_id); + + // Build the actual parser + auto parser = builder_fn(); + + // Replace placeholder with actual rule + auto rule_id = arena_.add_parser(common_chat_peg_rule_parser{name, parser.id(), trigger}); + arena_.rules_[name] = rule_id; + + return ref(name); +} + +void common_chat_peg_parser_builder::set_root(common_chat_peg_parser p) { + arena_.set_root(p.id()); +} + +common_chat_peg_arena common_chat_peg_parser_builder::build() { + return std::move(arena_); +} + +// JSON parsers +common_chat_peg_parser common_chat_peg_parser_builder::json_number() { + std::function builder = [this]() { + auto digit1_9 = chars("[1-9]", 1, 1); + auto digits = chars("[0-9]"); + auto int_part = choice({literal("0"), sequence({digit1_9, chars("[0-9]", 0, -1)})}); + auto frac = sequence({literal("."), digits}); + auto exp = sequence({choice({literal("e"), literal("E")}), optional(chars("[+\\-]", 1, 1)), digits}); + return sequence({optional(literal("-")), int_part, optional(frac), optional(exp)}); + }; + return rule("json-number", builder); +} + +common_chat_peg_parser common_chat_peg_parser_builder::json_string() { + std::function builder = [this]() { + return sequence({literal("\""), json_string_content(), literal("\"")}); + }; + return rule("json-string", builder); +} + +common_chat_peg_parser common_chat_peg_parser_builder::json_bool() { + std::function builder = [this]() { + return choice({literal("true"), literal("false")}); + }; + return rule("json-bool", builder); +} + +common_chat_peg_parser common_chat_peg_parser_builder::json_null() { + std::function builder = [this]() { + return literal("null"); + }; + return rule("json-null", builder); +} + +common_chat_peg_parser common_chat_peg_parser_builder::json_object() { + std::function builder = [this]() { + auto ws = space(); + auto member = sequence({json_string(), ws, literal(":"), ws, json()}); + auto members = sequence({member, zero_or_more(sequence({ws, literal(","), ws, member}))}); + return choice({ + sequence({literal("{"), ws, literal("}")}), + sequence({literal("{"), ws, members, ws, literal("}")}) + }); + }; + return rule("json-object", builder); +} -// Escape special characters for GBNF literals +common_chat_peg_parser common_chat_peg_parser_builder::json_array() { + std::function builder = [this]() { + auto ws = space(); + auto elements = sequence({json(), zero_or_more(sequence({ws, literal(","), ws, json()}))}); + return choice({ + sequence({literal("["), ws, literal("]")}), + sequence({literal("["), ws, elements, ws, literal("]")}) + }); + }; + return rule("json-array", builder); +} + +common_chat_peg_parser common_chat_peg_parser_builder::json() { + std::function builder = [this]() { + return choice({ + json_object(), + json_array(), + json_string(), + json_number(), + json_bool(), + json_null() + }); + }; + return rule("json-value", builder); +} + + +// GBNF generation helper functions static std::string gbnf_literal(const std::string & s) { std::string escaped; for (char c : s) { @@ -1405,7 +1142,6 @@ static std::string gbnf_literal(const std::string & s) { return "\"" + escaped + "\""; } -// Escape a single character for use in gbnf character classes static std::string gbnf_escape_char_class(char c) { switch (c) { case '\n': return "\\n"; @@ -1419,11 +1155,7 @@ static std::string gbnf_escape_char_class(char c) { } } -// Create a GBNF excluding pattern static std::string gbnf_excluding_pattern(const std::vector & strings) { - // Use the trie_matcher to grab an exhaustive list of prefixes and - // potential next characters. We can use this to build an exclusion for - // multiple strings. trie_matcher matcher(strings); auto pieces = matcher.collect_prefix_and_next(); @@ -1452,653 +1184,243 @@ static std::string gbnf_excluding_pattern(const std::vector & strin return "(" + pattern + ")*"; } -// Visitor for resolving rule references and collecting rules -class resolution_visitor : public parser_visitor { - std::unordered_map> rules_; - std::vector> refs_; - - public: - resolution_visitor() = default; - - void visit(start_parser & /* p */) override {} - void visit(end_parser & /* p */) override {} - void visit(literal_parser & /* p */) override {} - void visit(any_parser & /* p */) override {} - void visit(space_parser & /* p */) override {} - void visit(json_string_parser & /* p */) override {} - void visit(chars_parser & /* p */) override {} - void visit(until_parser & /* p */) override {} - void visit(and_parser & p) override { p.child()->accept(*this); } - void visit(not_parser & p) override { p.child()->accept(*this); } - - void visit(sequence_parser & p) override { - for (const auto & child : p.parsers()) { - child->accept(*this); - } - } - - void visit(choice_parser & p) override { - for (const auto & child : p.parsers()) { - child->accept(*this); - } - } - - void visit(one_or_more_parser & p) override { p.child()->accept(*this); } - void visit(zero_or_more_parser & p) override { p.child()->accept(*this); } - void visit(optional_parser & p) override { p.child()->accept(*this); } - void visit(repetition_parser & p) override { p.child()->accept(*this); } - void visit(schema_parser & p) override { p.child()->accept(*this); } - void visit(capture_parser & p) override { p.child()->accept(*this); } - - void visit(rule_parser & p) override { - const std::string & name = p.name(); - - // Check for duplicate rule names - if (rules_.find(name) != rules_.end()) { - throw std::runtime_error("Duplicate rule name: " + name); - } - - // Collect this rule - auto rule_ptr = cast(p.shared_from_this()); - if (rule_ptr) { - rules_[name] = rule_ptr; - } - - // Recursively visit the child - p.child()->accept(*this); - } - - void visit(ref_parser & p) override { - // Collect this ref for later resolution - auto ref_ptr = cast(p.shared_from_this()); - if (ref_ptr) { - refs_.push_back(ref_ptr); - } - } - - void visit(root_parser & p) override { - // Visit all rules stored in the map - for (const auto & [name, rule] : p.rules()) { - rule->accept(*this); - } - - // Visit the root tree - p.root()->accept(*this); - } - - // Resolve all collected refs - void resolve() { - for (const auto & ref : refs_) { - const std::string & name = ref->name(); - auto it = rules_.find(name); - if (it == rules_.end()) { - throw std::runtime_error("Unresolved reference: " + name); - } - ref->set_target(it->second); - } - } - - const std::unordered_map> & rules() const { return rules_; } -}; - -// Visitor for collecting reachable rules from a subtree -class reachability_visitor : public parser_visitor { - std::unordered_set & reachable_rules_; - - public: - reachability_visitor(std::unordered_set & reachable_rules) - : reachable_rules_(reachable_rules) {} - - void visit(start_parser & /* p */) override {} - void visit(end_parser & /* p */) override {} - void visit(literal_parser & /* p */) override {} - void visit(any_parser & /* p */) override {} - void visit(space_parser & /* p */) override {} - void visit(json_string_parser & /* p */) override {} - void visit(chars_parser & /* p */) override {} - void visit(until_parser & /* p */) override {} - void visit(and_parser & p) override { p.child()->accept(*this); } - void visit(not_parser & p) override { p.child()->accept(*this); } - - void visit(sequence_parser & p) override { - for (const auto & child : p.parsers()) { - child->accept(*this); - } - } - - void visit(choice_parser & p) override { - for (const auto & child : p.parsers()) { - child->accept(*this); - } - } - - void visit(one_or_more_parser & p) override { p.child()->accept(*this); } - void visit(zero_or_more_parser & p) override { p.child()->accept(*this); } - void visit(optional_parser & p) override { p.child()->accept(*this); } - void visit(repetition_parser & p) override { p.child()->accept(*this); } - void visit(schema_parser & /* p */) override { - // The schema system will handle rule generation via builder_.add_schema() - } - void visit(capture_parser & p) override { p.child()->accept(*this); } - - void visit(rule_parser & p) override { - const std::string & name = p.name(); - // Check if we arleady reached this rule. Technically we shouldn't, - // since we don't traverse ref_parser - if (reachable_rules_.find(name) != reachable_rules_.end()) { - return; - } - reachable_rules_.insert(name); - - // Recursively visit the rule's child - p.child()->accept(*this); - } - - void visit(ref_parser & p) override { - // Mark as reachable - reachable_rules_.insert(p.name()); - } - - void visit(root_parser & p) override { - p.root()->accept(*this); - } -}; - -class gbnf_visitor : public parser_visitor { - const common_grammar_builder & builder_; - std::unordered_map rule_name_mapping_; - std::string current_result_; - bool lazy_; - std::unordered_map> all_rules_; - - public: - gbnf_visitor(const common_grammar_builder & builder, bool lazy = false) - : builder_(builder), lazy_(lazy) {} - - const std::string& result() const { return current_result_; } +// Collect all rule parsers reachable from triggers (for lazy mode) +static std::unordered_set collect_reachable_rules( + const common_chat_peg_arena & arena, + const std::unordered_map & all_rules +) { + std::unordered_set reachable; + std::unordered_set visited; - private: - // Check if expression needs parentheses - static bool needs_parens(parser_type type) { - return type == CHOICE || type == SEQUENCE; - } - - public: - void visit(start_parser & /* p */) override { - current_result_ = ""; - } - - void visit(end_parser & /* p */) override { - current_result_ = ""; - } - - void visit(literal_parser & p) override { - current_result_ = gbnf_literal(p.literal()); - } - - void visit(sequence_parser & p) override { - std::string s; - for (const auto & child : p.parsers()) { - if (!s.empty()) { - s += " "; - } - child->accept(*this); + std::function visit = [&](common_chat_peg_parser_id id) { + const auto & parser = arena.get(id); - // Parenthesize choices - if (needs_parens(child->type())) { - s += "(" + current_result_ + ")"; - } else { - s += current_result_; - } - } - current_result_ = s; - } + std::visit([&](const auto & p) { + using T = std::decay_t; - void visit(choice_parser & p) override { - std::string s; - for (const auto & child : p.parsers()) { - if (!s.empty()) { - s += " | "; + if constexpr (std::is_same_v) { + for (auto child : p.children) { + visit(child); + } + } else if constexpr (std::is_same_v) { + for (auto child : p.children) { + visit(child); + } + } else if constexpr (std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v) { + visit(p.child); + } else if constexpr (std::is_same_v) { + if (visited.find(p.name) == visited.end()) { + visited.insert(p.name); + reachable.insert(p.name); + visit(p.child); + } + } else if constexpr (std::is_same_v) { + reachable.insert(p.name); } + }, parser); + }; - child->accept(*this); - - // Parenthesize choices - if (child->type() == CHOICE) { - s += "(" + current_result_ + ")"; - } else { - s += current_result_; + // Find trigger rules and traverse from them + for (const auto & [name, rule_id] : all_rules) { + const auto & parser = arena.get(rule_id); + if (auto rule = std::get_if(&parser)) { + if (rule->trigger) { + visit(rule_id); } } - current_result_ = s; - } - - void visit(one_or_more_parser & p) override { - p.child()->accept(*this); - if (needs_parens(p.child()->type())) { - current_result_ = "(" + current_result_ + ")+"; - } else { - current_result_ = current_result_ + "+"; - } } - void visit(zero_or_more_parser & p) override { - p.child()->accept(*this); - if (needs_parens(p.child()->type())) { - current_result_ = "(" + current_result_ + ")*"; - } else { - current_result_ = current_result_ + "*"; - } - } - - void visit(optional_parser & p) override { - p.child()->accept(*this); - if (needs_parens(p.child()->type())) { - current_result_ = "(" + current_result_ + ")?"; - } else { - current_result_ = current_result_ + "?"; - } - } + return reachable; +} - void visit(repetition_parser & p) override { - p.child()->accept(*this); - std::string child_result = current_result_; +// GBNF generation implementation +void common_chat_peg_arena::build_grammar(const common_grammar_builder & builder, bool lazy) const { + std::unordered_map rule_name_mapping; + std::unordered_set reachable_rules; - if (needs_parens(p.child()->type())) { - child_result = "(" + child_result + ")"; - } - - if (p.max_count() == -1) { - // Unbounded: {n,} - current_result_ = child_result + "{" + std::to_string(p.min_count()) + ",}"; - } else { - // Bounded: {n,m} - current_result_ = child_result + "{" + std::to_string(p.min_count()) + "," + - std::to_string(p.max_count()) + "}"; + // Collect all rules + if (lazy) { + reachable_rules = collect_reachable_rules(*this, rules_); + if (reachable_rules.empty()) { + LOG_ERR("Lazy grammar generation enabled but no trigger rules found\n"); + return; } } - void visit(until_parser & p) override { - // Generate pattern that matches prefixes but prevents full delimiter match - current_result_ = gbnf_excluding_pattern(p.delimiters()); - } - - void visit(and_parser & /* p */) override { - current_result_ = ""; - } - - void visit(not_parser & /* p */) override { - // NOT is tricky in GBNF - for now, emit error - LOG_ERR("NOT operator not directly supported in GBNF generation\n"); - current_result_ = ""; - } - - void visit(any_parser & /* p */) override { - // Match any single character - current_result_ = "."; - } - - void visit(space_parser & /* p */) override { - // Reference the built-in space rule - current_result_ = "space"; - } - - void visit(chars_parser & p) override { - const std::string & pattern = p.pattern(); - - if (p.min_count() == 0 && p.max_count() == -1) { - // Zero or more: * - current_result_ = pattern + "*"; - } else if (p.min_count() == 1 && p.max_count() == -1) { - // One or more: + - current_result_ = pattern + "+"; - } else if (p.max_count() == -1) { - // Unbounded: {n,} - current_result_ = pattern + "{" + std::to_string(p.min_count()) + ",}"; - } else if (p.min_count() == p.max_count()) { - // Exact count: {n} or just pattern for n=1 - if (p.min_count() == 1) { - current_result_ = pattern; + // Generate GBNF for a parser + std::function to_gbnf = [&](common_chat_peg_parser_id id) -> std::string { + const auto & parser = parsers_.at(id); + + return std::visit([&](const auto & p) -> std::string { + using T = std::decay_t; + + if constexpr (std::is_same_v || std::is_same_v) { + return ""; + } else if constexpr (std::is_same_v) { + return gbnf_literal(p.literal); + } else if constexpr (std::is_same_v) { + std::string s; + for (const auto & child : p.children) { + if (!s.empty()) s += " "; + auto child_gbnf = to_gbnf(child); + const auto & child_parser = parsers_.at(child); + if (std::holds_alternative(child_parser) || + std::holds_alternative(child_parser)) { + s += "(" + child_gbnf + ")"; + } else { + s += child_gbnf; + } + } + return s; + } else if constexpr (std::is_same_v) { + std::string s; + for (const auto & child : p.children) { + if (!s.empty()) s += " | "; + auto child_gbnf = to_gbnf(child); + const auto & child_parser = parsers_.at(child); + if (std::holds_alternative(child_parser)) { + s += "(" + child_gbnf + ")"; + } else { + s += child_gbnf; + } + } + return s; + } else if constexpr (std::is_same_v) { + auto child_gbnf = to_gbnf(p.child); + const auto & child_parser = parsers_.at(p.child); + if (std::holds_alternative(child_parser) || + std::holds_alternative(child_parser)) { + return "(" + child_gbnf + ")+"; + } + return child_gbnf + "+"; + } else if constexpr (std::is_same_v) { + auto child_gbnf = to_gbnf(p.child); + const auto & child_parser = parsers_.at(p.child); + if (std::holds_alternative(child_parser) || + std::holds_alternative(child_parser)) { + return "(" + child_gbnf + ")*"; + } + return child_gbnf + "*"; + } else if constexpr (std::is_same_v) { + auto child_gbnf = to_gbnf(p.child); + const auto & child_parser = parsers_.at(p.child); + if (std::holds_alternative(child_parser) || + std::holds_alternative(child_parser)) { + return "(" + child_gbnf + ")?"; + } + return child_gbnf + "?"; + } else if constexpr (std::is_same_v) { + auto child_gbnf = to_gbnf(p.child); + const auto & child_parser = parsers_.at(p.child); + if (std::holds_alternative(child_parser) || + std::holds_alternative(child_parser)) { + child_gbnf = "(" + child_gbnf + ")"; + } + if (p.max_count == -1) { + return child_gbnf + "{" + std::to_string(p.min_count) + ",}"; + } + return child_gbnf + "{" + std::to_string(p.min_count) + "," + std::to_string(p.max_count) + "}"; + } else if constexpr (std::is_same_v || std::is_same_v) { + return ""; // Lookahead not supported in GBNF + } else if constexpr (std::is_same_v) { + return "."; + } else if constexpr (std::is_same_v) { + return "space"; + } else if constexpr (std::is_same_v) { + std::string result = p.pattern; + if (p.min_count == 0 && p.max_count == -1) { + return result + "*"; + } else if (p.min_count == 1 && p.max_count == -1) { + return result + "+"; + } else if (p.max_count == -1) { + return result + "{" + std::to_string(p.min_count) + ",}"; + } else if (p.min_count == p.max_count) { + if (p.min_count == 1) { + return result; + } + return result + "{" + std::to_string(p.min_count) + "}"; + } else { + return result + "{" + std::to_string(p.min_count) + "," + std::to_string(p.max_count) + "}"; + } + } else if constexpr (std::is_same_v) { + return R"(( [^"\\] | "\\" ( ["\\/ bfnrt] | "u" [0-9a-fA-F]{4} ) )*)"; + } else if constexpr (std::is_same_v) { + return gbnf_excluding_pattern(p.delimiters); + } else if constexpr (std::is_same_v) { + if (p.schema) { + return builder.add_schema(p.name, *p.schema); + } + return to_gbnf(p.child); + } else if constexpr (std::is_same_v) { + return to_gbnf(p.child); + } else if constexpr (std::is_same_v) { + auto it = rule_name_mapping.find(p.name); + if (it != rule_name_mapping.end()) { + return it->second; + } + return p.name; + } else if constexpr (std::is_same_v) { + return to_gbnf(p.child); } else { - current_result_ = pattern + "{" + std::to_string(p.min_count()) + "}"; + return ""; } - } else { - // Bounded: {n,m} - current_result_ = pattern + "{" + std::to_string(p.min_count()) + "," + - std::to_string(p.max_count()) + "}"; - } - } - - void visit(json_string_parser & /* p */) override { - // JSON string content (without quotes) - // Pattern: (any non-quote/backslash OR escape sequences)* until closing quote - current_result_ = R"(( [^"\\] | "\\" ( ["\\/ bfnrt] | "u" [0-9a-fA-F]{4} ) )*)"; - } - - void visit(schema_parser & p) override { - current_result_ = builder_.add_schema(p.name(), p.schema()); - } - - void visit(rule_parser & p) override { - // When visiting a rule, generate its definition - p.child()->accept(*this); - } - - void visit(ref_parser & p) override { - // Return canonical rule reference - auto it = rule_name_mapping_.find(p.name()); - if (it != rule_name_mapping_.end()) { - current_result_ = it->second; - } else { - // Fallback to original name if not in mapping (shouldn't happen in valid usage) - current_result_ = p.name(); - } - } - - void visit(root_parser & p) override { - if (!lazy_) { - // Non-lazy mode: generate all rules eagerly - for (const auto & [name, rule] : p.rules()) { - rule->accept(*this); - auto rule_body = current_result_; - auto canonical_name = builder_.add_rule(name, rule_body); - rule_name_mapping_[name] = canonical_name; - } - - // Return root body - p.root()->accept(*this); - return; - } - - // Lazy mode - - // Collect all rules reachable from triggers - std::unordered_set reachable; - reachability_visitor visitor(reachable); - - // Find all trigger rules and traverse from them - for (const auto & [name, rule] : all_rules_) { - if (rule->is_trigger()) { - rule->accept(visitor); - } - } - - // Check if we found any trigger rules - if (reachable.empty()) { - LOG_ERR("Lazy grammar generation enabled but no trigger rules found\n"); - current_result_ = ""; - return; - } + }, parser); + }; - // Generate only reachable rules - for (const auto & [name, rule] : p.rules()) { - // Skip rules that aren't reachable - if (reachable.find(name) == reachable.end()) { + // Generate rules + if (lazy) { + // Lazy mode: only generate reachable rules + for (const auto & [name, rule_id] : rules_) { + if (reachable_rules.find(name) == reachable_rules.end()) { continue; } - rule->accept(*this); - auto rule_body = current_result_; - auto canonical_name = builder_.add_rule(name, rule_body); - rule_name_mapping_[name] = canonical_name; + const auto & parser = parsers_.at(rule_id); + if (auto rule = std::get_if(&parser)) { + auto rule_body = to_gbnf(rule->child); + auto canonical_name = builder.add_rule(name, rule_body); + rule_name_mapping[name] = canonical_name; + } } // Generate root as alternation of trigger rule names std::vector trigger_names; - for (const auto & [name, rule] : p.rules()) { - if (rule->is_trigger()) { - auto it = rule_name_mapping_.find(name); - if (it != rule_name_mapping_.end()) { - trigger_names.push_back(it->second); + for (const auto & [name, rule_id] : rules_) { + const auto & parser = parsers_.at(rule_id); + if (auto rule = std::get_if(&parser)) { + if (rule->trigger) { + auto it = rule_name_mapping.find(name); + if (it != rule_name_mapping.end()) { + trigger_names.push_back(it->second); + } } } } - current_result_ = string_join(trigger_names, " | "); - } - - void visit(capture_parser & p) override { - p.child()->accept(*this); - } -}; - -// Implement accept() methods for all parser classes -void start_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } -void end_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } -void literal_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } -void sequence_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } -void choice_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } -void one_or_more_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } -void zero_or_more_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } -void optional_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } -void repetition_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } -void until_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } -void and_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } -void not_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } -void any_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } -void space_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } -void chars_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } -void json_string_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } -void schema_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } -void rule_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } -void ref_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } -void root_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } -void capture_parser::accept(parser_visitor & visitor) { visitor.visit(*this); } - -common_chat_parse_result common_chat_parse_cache::set(int id, size_t start, common_chat_parse_result result) { - if (id == -1) { - // Don't cache parsers with ID -1 (from operators and global factory functions) - return result; - } - results[common_chat_parse_cache_key{id, start}] = result; - return result; -} - -std::optional common_chat_parse_cache::get(int id, size_t start) { - if (id == -1) { - // Don't cache parsers with ID -1 (from operators and global factory functions) - return std::nullopt; - } - auto it = results.find(common_chat_parse_cache_key{id, start}); - if (it != results.end()) { - return it->second; - } - return std::nullopt; -} - -void common_chat_parse_cache::clear() { - results.clear(); -} - -common_chat_peg_parser::common_chat_peg_parser() {} -common_chat_peg_parser::common_chat_peg_parser(std::shared_ptr parser) : ptr_(std::move(parser)) {} -common_chat_peg_parser::common_chat_peg_parser(const std::string & literal) : ptr_(make_parser(-1, literal)) {} -common_chat_peg_parser::common_chat_peg_parser(const char * literal) : ptr_(make_parser(-1, literal)) {} - -common_chat_peg_parser operator~(const common_chat_peg_parser & p) { return make_parser(-1, p); } - -common_chat_peg_parser operator+(const common_chat_peg_parser & lhs, const common_chat_peg_parser & rhs) { - return make_parser(-1, std::initializer_list{lhs, rhs}); -} - -common_chat_peg_parser operator|(const common_chat_peg_parser & lhs, const common_chat_peg_parser & rhs) { - return make_parser(-1, std::initializer_list{lhs, rhs}); -} - -common_chat_peg_parser operator<<(const common_chat_peg_parser & lhs, const common_chat_peg_parser & rhs) { - auto ws = make_parser(-1); - return make_parser(-1, std::initializer_list{lhs, ws, rhs}); -} - -common_chat_peg_parser operator+(const char * lhs, const common_chat_peg_parser & rhs) { return common_chat_peg_parser(lhs) + rhs; } -common_chat_peg_parser operator|(const char * lhs, const common_chat_peg_parser & rhs) { return common_chat_peg_parser(lhs) | rhs; } -common_chat_peg_parser operator<<(const char * lhs, const common_chat_peg_parser & rhs) { return common_chat_peg_parser(lhs) << rhs; } - -common_chat_peg_parser operator+(const std::string & lhs, const common_chat_peg_parser & rhs) { return common_chat_peg_parser(lhs) + rhs; } -common_chat_peg_parser operator|(const std::string & lhs, const common_chat_peg_parser & rhs) { return common_chat_peg_parser(lhs) | rhs; } -common_chat_peg_parser operator<<(const std::string & lhs, const common_chat_peg_parser & rhs) { return common_chat_peg_parser(lhs) << rhs; } - -common_chat_peg_parser_base & common_chat_peg_parser::operator*() const { return *ptr_; } -common_chat_peg_parser_base * common_chat_peg_parser::operator->() const { return ptr_.get(); } - -common_chat_parse_result common_chat_peg_parser::parse(common_chat_parse_context & ctx, size_t start) const { - return ptr_->parse(ctx, start); -} - -std::string common_chat_peg_parser::dump() const { return ptr_->dump(); } - -void common_chat_peg_parser::build_grammar(const common_grammar_builder & builder, bool lazy) const { - gbnf_visitor visitor(builder, lazy); - ptr_->accept(visitor); - auto result = visitor.result(); - if (!result.empty()) { - builder.add_rule("root", result); - } -} - -using builder = common_chat_peg_parser_builder; - -builder::common_chat_peg_parser_builder() : root_(make_parser(0)) , counter_(1) {} - -common_chat_peg_parser builder::start() { return make_parser(counter_); } -common_chat_peg_parser builder::end() { return make_parser(counter_); } -common_chat_peg_parser builder::literal(const std::string & literal) { return make_parser(counter_, literal); } -common_chat_peg_parser builder::sequence(const std::vector & parsers) { return make_parser(counter_, parsers); } -common_chat_peg_parser builder::choice(const std::vector & parsers) { return make_parser(counter_, parsers); } -common_chat_peg_parser builder::one_or_more(const common_chat_peg_parser & p) { return make_parser(counter_, p); } -common_chat_peg_parser builder::zero_or_more(const common_chat_peg_parser & p) { return make_parser(counter_, p); } -common_chat_peg_parser builder::optional(const common_chat_peg_parser & p) { return make_parser(counter_, p); } -common_chat_peg_parser builder::peek(const common_chat_peg_parser & p) { return make_parser(counter_, p); } -common_chat_peg_parser builder::negate(const common_chat_peg_parser & p) { return make_parser(counter_, p); } -common_chat_peg_parser builder::any() { return make_parser(counter_); } -common_chat_peg_parser builder::chars(const std::string & classes, int min, int max) { return make_parser(counter_, classes, min, max); } -common_chat_peg_parser builder::one(const std::string & classes) { return make_parser(counter_, classes, 1, 1); } -common_chat_peg_parser builder::json_string_content() { return make_parser(counter_); } -common_chat_peg_parser builder::space() { return make_parser(counter_); } -common_chat_peg_parser builder::until(const std::string & delimiter) { return make_parser(counter_, delimiter); } -common_chat_peg_parser builder::until_one_of(const std::vector & delimiters) { return make_parser(counter_, delimiters); } -common_chat_peg_parser builder::repeat(const common_chat_peg_parser & p, int min, int max) { return make_parser(counter_, p, min, max); } -common_chat_peg_parser builder::repeat(const common_chat_peg_parser & p, int n) { return make_parser(counter_, p, n, n); } - -common_chat_peg_parser builder::ref(const std::string & name) { - return make_parser(counter_, name); -} - -common_chat_peg_parser builder::schema(const common_chat_peg_parser & p, const std::string & name, const nlohmann::ordered_json & schema) { - return make_parser(counter_, p, name, schema); -} - -common_chat_peg_parser builder::capture(const std::string & key, const common_chat_peg_parser & p) { - return make_parser(counter_, p, key); -} - -common_chat_peg_parser builder::rule(const std::string & name, const common_chat_peg_parser & p, bool trigger) { - auto root_container = cast(root_); - auto rule_node = std::make_shared(name, p, trigger, counter_.next()); - root_container->add_rule(name, rule_node); - return make_parser(counter_, name); -} - -common_chat_peg_parser builder::rule(const std::string & name, const std::function & builder_fn, bool trigger) { - auto root_container = cast(root_); - if (root_container->rules().find(name) != root_container->rules().end()) { - return ref(name); - } - - // Create placeholder rule to allow recursive references - auto placeholder = std::make_shared(name, any(), trigger, counter_.next()); - root_container->add_rule(name, placeholder); - - // Build the actual parser - auto parser = builder_fn(); - - // Replace placeholder with actual rule - auto rule_node = std::make_shared(name, parser, trigger, counter_.next()); - root_container->add_rule(name, rule_node); - - return make_parser(counter_, name); -} - -void builder::set_root(const common_chat_peg_parser & p) { - auto root_container = cast(root_); - root_container->set_root(p); + if (!trigger_names.empty()) { + builder.add_rule("root", string_join(trigger_names, " | ")); + } + } else { + // Non-lazy mode: generate all rules + for (const auto & [name, rule_id] : rules_) { + const auto & parser = parsers_.at(rule_id); + if (auto rule = std::get_if(&parser)) { + auto rule_body = to_gbnf(rule->child); + auto canonical_name = builder.add_rule(name, rule_body); + rule_name_mapping[name] = canonical_name; + } + } - // Recursively issue IDs to reachable nodes - if (p.ptr()) { - p.ptr()->assign_id(counter_); + // Generate root + if (root_ != COMMON_CHAT_PEG_INVALID_PARSER_ID) { + auto root_body = to_gbnf(root_); + builder.add_rule("root", root_body); + } } } - -common_chat_peg_parser builder::json_number() { - return rule("json-number", [this]() { - auto digit1_9 = chars("[1-9]", 1, 1); - auto digits = chars("[0-9]"); - auto int_part = literal("0") | (digit1_9 + chars("[0-9]", 0, -1)); - auto frac = literal(".") + digits; - auto exp = (literal("e") | literal("E")) + optional(chars("[+\\-]", 1, 1)) + digits; - return optional(literal("-")) + int_part + optional(frac) + optional(exp); - }); -} - -common_chat_peg_parser builder::json_string() { - return rule("json-string", [this]() { - return literal("\"") + json_string_content() + literal("\""); - }); -} - -common_chat_peg_parser builder::json_bool() { - return rule("json-bool", [this]() { - return literal("true") | literal("false"); - }); -} - -common_chat_peg_parser builder::json_null() { - return rule("json-null", [this]() { - return literal("null"); - }); -} - -common_chat_peg_parser builder::json_object() { - return rule("json-object", [this]() { - auto ws = space(); - auto member = json_string() + ws + literal(":") + ws + json(); - auto members = member + zero_or_more(ws + literal(",") + ws + member); - return (literal("{") + ws + literal("}")) | - (literal("{") + ws + members + ws + literal("}")); - }); -} - -common_chat_peg_parser builder::json_array() { - return rule("json-array", [this]() { - auto ws = space(); - auto elements = json() + zero_or_more(ws + literal(",") + ws + json()); - return (literal("[") + ws + literal("]")) | - (literal("[") + ws + elements + ws + literal("]")); - }); -} - -common_chat_peg_parser builder::json() { - return rule("json-value", [this]() { - return json_object() | - json_array() | - json_string() | - json_number() | - json_bool() | - json_null(); - }); -} - -common_chat_peg_parser builder::build() { - // Resolve all references - resolution_visitor resolver; - root_->accept(resolver); - resolver.resolve(); - - return root_; -} - -common_chat_peg_parser build_peg_parser(const std::function & fn) { - builder builder; - auto root = fn(builder); - builder.set_root(root); - return builder.build(); -} diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index e5b1f8fd2fcd6..9d9c7c685daff 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -10,9 +10,57 @@ #include #include #include +#include +#include struct common_grammar_builder; +// Forward declarations +using common_chat_peg_parser_id = size_t; +constexpr common_chat_peg_parser_id COMMON_CHAT_PEG_INVALID_PARSER_ID = static_cast(-1); + +// Forward declare builder for parser wrapper +class common_chat_peg_parser_builder; + +// Lightweight wrapper around common_chat_peg_parser_id that enables operator overloading +// and implicit conversions from strings/literals +class common_chat_peg_parser { + common_chat_peg_parser_id id_; + common_chat_peg_parser_builder * builder_; + + public: + // Construct from common_chat_peg_parser_id + common_chat_peg_parser(common_chat_peg_parser_id id, common_chat_peg_parser_builder * builder) : id_(id), builder_(builder) {} + + // Implicit conversion to common_chat_peg_parser_id + operator common_chat_peg_parser_id() const { return id_; } + + // Get the underlying ID + common_chat_peg_parser_id id() const { return id_; } + + // Get builder (for free function operators) + common_chat_peg_parser_builder * builder() const { return builder_; } + + // Operator overloads + common_chat_peg_parser operator+(const common_chat_peg_parser & other) const; + common_chat_peg_parser operator|(const common_chat_peg_parser & other) const; + common_chat_peg_parser operator<<(const common_chat_peg_parser & other) const; // sequence with space + + // Overloads for string literals + common_chat_peg_parser operator+(const char * str) const; + common_chat_peg_parser operator+(const std::string & str) const; + common_chat_peg_parser operator|(const char * str) const; + common_chat_peg_parser operator|(const std::string & str) const; + common_chat_peg_parser operator<<(const char * str) const; + common_chat_peg_parser operator<<(const std::string & str) const; +}; + +// Free function operators for string + parser +common_chat_peg_parser operator+(const char * str, const common_chat_peg_parser & p); +common_chat_peg_parser operator+(const std::string & str, const common_chat_peg_parser & p); +common_chat_peg_parser operator<<(const char * str, const common_chat_peg_parser & p); +common_chat_peg_parser operator<<(const std::string & str, const common_chat_peg_parser & p); + struct common_chat_parse_semantics { std::string content; std::string reasoning_content; @@ -38,7 +86,7 @@ enum common_chat_parse_result_type { const char * common_chat_parse_result_type_name(common_chat_parse_result_type type); struct common_chat_parse_cache_key { - int id; + common_chat_peg_parser_id id; size_t start; bool operator==(const common_chat_parse_cache_key & other) const { @@ -49,7 +97,7 @@ struct common_chat_parse_cache_key { template <> struct std::hash { std::size_t operator()(const common_chat_parse_cache_key & k) const { - return std::hash{}(((size_t)k.id << 32) | k.start); + return std::hash{}((k.id << 32) | k.start); } }; @@ -99,8 +147,8 @@ class common_chat_parse_cache { std::unordered_map results; public: - common_chat_parse_result set(int id, size_t start, common_chat_parse_result result); - std::optional get(int id, size_t start); + common_chat_parse_result set(common_chat_peg_parser_id id, size_t start, common_chat_parse_result result); + std::optional get(common_chat_peg_parser_id id, size_t start); void clear(); }; @@ -133,53 +181,168 @@ struct common_chat_parse_context { : input(input), input_is_complete(complete), cache(), semantics(semantics), event_handler(std::move(handler)), current_depth(0), parse_depth(0) {} }; -class common_chat_peg_parser_base; +// Forward declaration +class common_chat_peg_arena; -class common_chat_peg_parser { - std::shared_ptr ptr_; +// Parser variant structs (value-based, no inheritance) +struct common_chat_peg_start_parser {}; - public: - common_chat_peg_parser(); - common_chat_peg_parser(std::shared_ptr parser); - common_chat_peg_parser(const std::string & literal); - common_chat_peg_parser(const char * literal); +struct common_chat_peg_end_parser {}; - common_chat_peg_parser_base & operator*() const; - common_chat_peg_parser_base * operator->() const; +struct common_chat_peg_literal_parser { + std::string literal; +}; - std::shared_ptr ptr() const { return ptr_; } +struct common_chat_peg_sequence_parser { + std::vector children; +}; - common_chat_parse_result parse(common_chat_parse_context & ctx, size_t start = 0) const; +struct common_chat_peg_choice_parser { + std::vector children; +}; - std::string dump() const; +struct common_chat_peg_repetition_parser { + common_chat_peg_parser_id child; + int min_count; + int max_count; // -1 for unbounded +}; - void build_grammar(const common_grammar_builder & builder, bool lazy = false) const; +struct common_chat_peg_one_or_more_parser { + common_chat_peg_parser_id child; }; -common_chat_peg_parser operator~(const common_chat_peg_parser & p); +struct common_chat_peg_zero_or_more_parser { + common_chat_peg_parser_id child; +}; -common_chat_peg_parser operator+(const common_chat_peg_parser & lhs, const common_chat_peg_parser & rhs); -common_chat_peg_parser operator|(const common_chat_peg_parser & lhs, const common_chat_peg_parser & rhs); -common_chat_peg_parser operator<<(const common_chat_peg_parser & lhs, const common_chat_peg_parser & rhs); +struct common_chat_peg_optional_parser { + common_chat_peg_parser_id child; +}; -common_chat_peg_parser operator+(const char * lhs, const common_chat_peg_parser & rhs); -common_chat_peg_parser operator|(const char * lhs, const common_chat_peg_parser & rhs); -common_chat_peg_parser operator<<(const char * lhs, const common_chat_peg_parser & rhs); +struct common_chat_peg_and_parser { + common_chat_peg_parser_id child; +}; + +struct common_chat_peg_not_parser { + common_chat_peg_parser_id child; +}; -common_chat_peg_parser operator+(const std::string & lhs, const common_chat_peg_parser & rhs); -common_chat_peg_parser operator|(const std::string & lhs, const common_chat_peg_parser & rhs); -common_chat_peg_parser operator<<(const std::string & lhs, const common_chat_peg_parser & rhs); +struct common_chat_peg_any_parser {}; + +struct common_chat_peg_space_parser {}; + +struct common_chat_peg_chars_parser { + struct char_range { + uint32_t start; + uint32_t end; + bool contains(uint32_t codepoint) const { return codepoint >= start && codepoint <= end; } + }; + + std::string pattern; + std::vector ranges; + bool negated; + int min_count; + int max_count; // -1 for unbounded +}; + +struct common_chat_peg_json_string_parser {}; + +struct common_chat_peg_until_parser { + std::vector delimiters; +}; + +struct common_chat_peg_schema_parser { + common_chat_peg_parser_id child; + std::string name; + std::shared_ptr schema; +}; + +struct common_chat_peg_rule_parser { + std::string name; + common_chat_peg_parser_id child; + bool trigger; +}; + +struct common_chat_peg_ref_parser { + std::string name; +}; + +struct common_chat_peg_capture_parser { + common_chat_peg_parser_id child; + std::string key; +}; + +// Variant holding all parser types +using common_chat_peg_parser_variant = std::variant< + common_chat_peg_start_parser, + common_chat_peg_end_parser, + common_chat_peg_literal_parser, + common_chat_peg_sequence_parser, + common_chat_peg_choice_parser, + common_chat_peg_repetition_parser, + common_chat_peg_one_or_more_parser, + common_chat_peg_zero_or_more_parser, + common_chat_peg_optional_parser, + common_chat_peg_and_parser, + common_chat_peg_not_parser, + common_chat_peg_any_parser, + common_chat_peg_space_parser, + common_chat_peg_chars_parser, + common_chat_peg_json_string_parser, + common_chat_peg_until_parser, + common_chat_peg_schema_parser, + common_chat_peg_rule_parser, + common_chat_peg_ref_parser, + common_chat_peg_capture_parser +>; + +// Arena owns all parsers +class common_chat_peg_arena { + std::vector parsers_; + std::unordered_map rules_; + common_chat_peg_parser_id root_; -class common_chat_peg_parser_counter { - int next_id_; public: - common_chat_peg_parser_counter(int start) : next_id_(start) {} - int next() { return next_id_++; } + common_chat_peg_arena(); + + // Access + const common_chat_peg_parser_variant & get(common_chat_peg_parser_id id) const { return parsers_.at(id); } + common_chat_peg_parser_variant & get(common_chat_peg_parser_id id) { return parsers_.at(id); } + + size_t size() const { return parsers_.size(); } + + // Rule lookup + common_chat_peg_parser_id get_rule(const std::string & name) const; + bool has_rule(const std::string & name) const { return rules_.find(name) != rules_.end(); } + + // Root + common_chat_peg_parser_id root() const { return root_; } + void set_root(common_chat_peg_parser_id id) { root_ = id; } + + // Parse + common_chat_parse_result parse(common_chat_parse_context & ctx, size_t start = 0) const; + common_chat_parse_result parse(common_chat_peg_parser_id id, common_chat_parse_context & ctx, size_t start) const; + + // Grammar generation + void build_grammar(const common_grammar_builder & builder, bool lazy = false) const; + + // Dump for debugging + std::string dump(common_chat_peg_parser_id id) const; + + // Builder access (for adding parsers) + friend class common_chat_peg_parser_builder; + + private: + common_chat_peg_parser_id add_parser(common_chat_peg_parser_variant parser); + void add_rule(const std::string & name, common_chat_peg_parser_id id); }; +// Builder for constructing parsers class common_chat_peg_parser_builder { - common_chat_peg_parser root_; - common_chat_peg_parser_counter counter_; + common_chat_peg_arena arena_; + + // Helper to wrap common_chat_peg_parser_id with this builder + common_chat_peg_parser wrap(common_chat_peg_parser_id id) { return common_chat_peg_parser(id, this); } public: common_chat_peg_parser_builder(); @@ -196,33 +359,43 @@ class common_chat_peg_parser_builder { // S -> "hello" common_chat_peg_parser literal(const std::string & literal); + // Implicit conversion: const char* -> parser (literal) + common_chat_peg_parser operator()(const char * str) { return literal(str); } + + // Implicit conversion: std::string -> parser (literal) + common_chat_peg_parser operator()(const std::string & str) { return literal(str); } + // Matches a sequence of parsers in order, all must succeed. // S -> A B C + common_chat_peg_parser sequence(const std::vector & parsers); common_chat_peg_parser sequence(const std::vector & parsers); + common_chat_peg_parser sequence(std::initializer_list parsers); // Matches the first parser that succeeds from a list of alternatives. // S -> A | B | C + common_chat_peg_parser choice(const std::vector & parsers); common_chat_peg_parser choice(const std::vector & parsers); + common_chat_peg_parser choice(std::initializer_list parsers); // Matches one or more repetitions of a parser. // S -> A+ - common_chat_peg_parser one_or_more(const common_chat_peg_parser & p); + common_chat_peg_parser one_or_more(common_chat_peg_parser p); // Matches zero or more repetitions of a parser, always succeeds. // S -> A* - common_chat_peg_parser zero_or_more(const common_chat_peg_parser & p); + common_chat_peg_parser zero_or_more(common_chat_peg_parser p); // Matches zero or one occurrence of a parser, always succeeds. // S -> A? - common_chat_peg_parser optional(const common_chat_peg_parser & p); + common_chat_peg_parser optional(common_chat_peg_parser p); - // Negative lookahead: succeeds if child parser fails, consumes no input. - // S -> !A - common_chat_peg_parser peek(const common_chat_peg_parser & p); + // Positive lookahead: succeeds if child parser succeeds, consumes no input. + // S -> &A + common_chat_peg_parser peek(common_chat_peg_parser p); // Negative lookahead: succeeds if child parser fails, consumes no input. // S -> !A - common_chat_peg_parser negate(const common_chat_peg_parser & p); + common_chat_peg_parser negate(common_chat_peg_parser p); // Matches any single character. // S -> . @@ -257,11 +430,11 @@ class common_chat_peg_parser_builder { // Matches between min and max repetitions of a parser (inclusive). // S -> A{m,n} // Use -1 for max to represent unbounded repetition (equivalent to {m,}) - common_chat_peg_parser repeat(const common_chat_peg_parser & p, int min, int max); + common_chat_peg_parser repeat(common_chat_peg_parser p, int min, int max); // Matches exactly n repetitions of a parser. // S -> A{n} - common_chat_peg_parser repeat(const common_chat_peg_parser & p, int n); + common_chat_peg_parser repeat(common_chat_peg_parser p, int n); // Creates a complete JSON parser supporting objects, arrays, strings, numbers, booleans, and null. // value -> object | array | string | number | true | false | null @@ -278,15 +451,15 @@ class common_chat_peg_parser_builder { // Wraps a parser with JSON schema metadata for grammar generation. // Used internally to convert JSON schemas to GBNF grammar rules. - common_chat_peg_parser schema(const common_chat_peg_parser & p, const std::string & name, const nlohmann::ordered_json & schema); + common_chat_peg_parser schema(common_chat_peg_parser p, const std::string & name, const nlohmann::ordered_json & schema); // Captures matched text to semantics.captures[key] - common_chat_peg_parser capture(const std::string & key, const common_chat_peg_parser & p); + common_chat_peg_parser capture(const std::string & key, common_chat_peg_parser p); // Creates a named rule, stores it in the grammar, and returns a reference to it. // If trigger=true, marks this rule as an entry point for lazy grammar generation. // auto json = p.rule("json", json_obj | json_arr | ...) - common_chat_peg_parser rule(const std::string & name, const common_chat_peg_parser & p, bool trigger = false); + common_chat_peg_parser rule(const std::string & name, common_chat_peg_parser p, bool trigger = false); // Creates a named rule using a builder function. This handles recursive grammars by // inserting a placeholder rule before invoking the builder, allowing the @@ -296,9 +469,16 @@ class common_chat_peg_parser_builder { // auto json = p.rule("json", [&]() { return json_object() | json_array() | ... }) common_chat_peg_parser rule(const std::string & name, const std::function & builder, bool trigger = false); - void set_root(const common_chat_peg_parser & p); + void set_root(common_chat_peg_parser p); - common_chat_peg_parser build(); + common_chat_peg_arena build(); }; -common_chat_peg_parser build_peg_parser(const std::function & fn); +// Helper function for building parsers +template +common_chat_peg_arena build_peg_parser(F && fn) { + common_chat_peg_parser_builder builder; + auto root = fn(builder); + builder.set_root(root); + return builder.build(); +} diff --git a/tests/chat-peg-parser/test-command7-parser-compare.cpp b/tests/chat-peg-parser/test-command7-parser-compare.cpp index 03f0a4ee36ea8..00f9bf0198e32 100644 --- a/tests/chat-peg-parser/test-command7-parser-compare.cpp +++ b/tests/chat-peg-parser/test-command7-parser-compare.cpp @@ -7,7 +7,7 @@ #include #include -static common_chat_peg_parser create_command_r7b_parser() { +static common_chat_peg_arena create_command_r7b_parser() { auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { auto thinking = p.rule("thinking", "<|START_THINKING|>" << p.rule("reasoning-content", p.until("<|END_THINKING|>")) << "<|END_THINKING|>"); @@ -18,10 +18,10 @@ static common_chat_peg_parser create_command_r7b_parser() { auto json = p.rule("json", p.json()); auto tool_call_id = p.rule("tool-call-id", - "\"tool_call_id\"" << (":" << p.rule("tool-call-id-value", "\"" + p.json_string() + "\""))); + "\"tool_call_id\"" << (":" << p.rule("tool-call-id-value", "\"" + p.json_string_content() + "\""))); auto tool_call_name = p.rule("tool-name", - "\"tool_name\"" << (":" << p.rule("tool-name-value", "\"" + p.json_string() + "\""))); + "\"tool_name\"" << (":" << p.rule("tool-name-value", "\"" + p.json_string_content() + "\""))); auto tool_call_args = p.rule("tool-args", "\"parameters\"" << (":" << p.rule("tool-args-value", json))); @@ -75,7 +75,7 @@ static common_chat_parse_event_handler create_command_r7b_event_handler() { }; } -static void test_command_r7b_parser(const common_chat_peg_parser & p, +static void test_command_r7b_parser(const common_chat_peg_arena & p, const std::string & input, bool need_more_input, bool print_results) { diff --git a/tests/chat-peg-parser/test-example-qwen3-coder.cpp b/tests/chat-peg-parser/test-example-qwen3-coder.cpp index cb48d7e2b1a8a..6c690c4e5e52a 100644 --- a/tests/chat-peg-parser/test-example-qwen3-coder.cpp +++ b/tests/chat-peg-parser/test-example-qwen3-coder.cpp @@ -109,7 +109,7 @@ void test_example_qwen3_coder(testing &t) { std::string in = std::accumulate(tokens.begin(), it + 1, std::string()); common_chat_parse_semantics semantics; - common_chat_parse_context ctx(in, &semantics, it == tokens.end()); + common_chat_parse_context ctx(in, &semantics, it + 1 == tokens.end()); ctx.event_handler = parser_semantic_handler; @@ -118,10 +118,17 @@ void test_example_qwen3_coder(testing &t) { LOG_ERR("%s[failed-->]%s\n", in.substr(0, result.end).c_str(), in.substr(result.end).c_str()); } - // This shouldn't emit any runtime errors - auto msg = semantics.to_msg(); - auto diffs = common_chat_msg_diff::compute_diffs(prev, msg); - prev = msg; + auto msg = semantics.to_msg(); + + try { + // This shouldn't emit any runtime errors + auto diffs = common_chat_msg_diff::compute_diffs(prev, msg); + } catch(const std::exception & e) { + LOG_ERR("%s[failed-->]%s\n", in.substr(0, result.end).c_str(), in.substr(result.end).c_str()); + t.assert_true(std::string("failed with ") + e.what(), false); + } + + prev = msg; } }); }); diff --git a/tests/chat-peg-parser/test-recursive-references.cpp b/tests/chat-peg-parser/test-recursive-references.cpp index b1bb19e35b614..3824f70cf8fdf 100644 --- a/tests/chat-peg-parser/test-recursive-references.cpp +++ b/tests/chat-peg-parser/test-recursive-references.cpp @@ -5,7 +5,7 @@ void test_recursive_references(testing &t) { t.test("simple_number", [](testing &t) { auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { p.rule("number", p.one_or_more(p.one("0-9"))); - p.rule("list", p.sequence({ p.literal("["), p.ref("value"), p.literal("]") })); + p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); return p.rule("value", p.ref("number") | p.ref("list")); }); @@ -19,7 +19,7 @@ void test_recursive_references(testing &t) { t.test("simple_list", [](testing &t) { auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { p.rule("number", p.one_or_more(p.one("0-9"))); - p.rule("list", p.sequence({ p.literal("["), p.ref("value"), p.literal("]") })); + p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); return p.rule("value", p.ref("number") | p.ref("list")); }); @@ -33,7 +33,7 @@ void test_recursive_references(testing &t) { t.test("nested_list", [](testing &t) { auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { p.rule("number", p.one_or_more(p.one("0-9"))); - p.rule("list", p.sequence({ p.literal("["), p.ref("value"), p.literal("]") })); + p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); return p.rule("value", p.ref("number") | p.ref("list")); }); @@ -47,7 +47,7 @@ void test_recursive_references(testing &t) { t.test("deeply_nested_list", [](testing &t) { auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { p.rule("number", p.one_or_more(p.one("0-9"))); - p.rule("list", p.sequence({ p.literal("["), p.ref("value"), p.literal("]") })); + p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); return p.rule("value", p.ref("number") | p.ref("list")); }); @@ -61,7 +61,7 @@ void test_recursive_references(testing &t) { t.test("need_more_input_match", [](testing &t) { auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { p.rule("number", p.one_or_more(p.one("0-9"))); - p.rule("list", p.sequence({ p.literal("["), p.ref("value"), p.literal("]") })); + p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); return p.rule("value", p.ref("number") | p.ref("list")); }); @@ -75,7 +75,7 @@ void test_recursive_references(testing &t) { t.test("no_match", [](testing &t) { auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { p.rule("number", p.one_or_more(p.one("0-9"))); - p.rule("list", p.sequence({ p.literal("["), p.ref("value"), p.literal("]") })); + p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); return p.rule("value", p.ref("number") | p.ref("list")); }); diff --git a/tests/chat-peg-parser/test-unicode.cpp b/tests/chat-peg-parser/test-unicode.cpp index c0d74bf9e60fc..7352fd57edcbd 100644 --- a/tests/chat-peg-parser/test-unicode.cpp +++ b/tests/chat-peg-parser/test-unicode.cpp @@ -51,7 +51,7 @@ void test_unicode(testing &t) { }; auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { - return p.one_or_more(p.any()) + p.end(); + return p.sequence({p.one_or_more(p.any()), p.end()}); }); for (size_t i = 0; i < test_cases.size(); i++) { @@ -94,7 +94,7 @@ void test_unicode(testing &t) { }; auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { - return p.chars(R"([\u4E00-\u9FFF])") + p.end(); + return p.sequence({p.chars(R"([\u4E00-\u9FFF])"), p.end()}); }); for (size_t i = 0; i < test_cases.size(); i++) { @@ -135,7 +135,7 @@ void test_unicode(testing &t) { }; auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { - return p.chars(R"([\U0001F600-\U0001F64F])") + p.end(); + return p.sequence({p.chars(R"([\U0001F600-\U0001F64F])"), p.end()}); }); for (size_t i = 0; i < test_cases.size(); i++) { @@ -180,7 +180,7 @@ void test_unicode(testing &t) { }; auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { - return p.chars(R"([\u4E00-\u9FFF\U0001F600-\U0001F64F0-9])") + p.end(); + return p.sequence({p.chars(R"([\u4E00-\u9FFF\U0001F600-\U0001F64F0-9])"), p.end()}); }); for (size_t i = 0; i < test_cases.size(); i++) { @@ -328,7 +328,7 @@ void test_unicode(testing &t) { t.test(test_name, [&](testing &t) { auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { - return p.json_string_content() + p.literal("\""); + return p.sequence({p.json_string_content(), p.literal("\"")}); }); common_chat_parse_context ctx(tc.input, true); @@ -431,7 +431,7 @@ void test_unicode(testing &t) { t.test(test_name, [&](testing &t) { auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { - return p.json_string_content() + p.literal("\""); + return p.sequence({p.json_string_content(), p.literal("\"")}); }); common_chat_parse_context ctx(tc.input, true); From 7daf9b28533f42fe3ddcd84d2bd2e3554bb0db1e Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Mon, 17 Nov 2025 02:48:21 -0600 Subject: [PATCH 119/183] simplify trie_matcher result --- common/chat-peg-parser.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 01891b889cdaf..109a9b6a0d728 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -52,9 +52,7 @@ class trie_matcher { } } - struct match_result { - enum match_type { NO_MATCH, PARTIAL_MATCH, COMPLETE_MATCH } type; - }; + enum match_result { NO_MATCH, PARTIAL_MATCH, COMPLETE_MATCH }; // Check if a delimiter starts at the given position match_result check_at(std::string_view sv, size_t start_pos) const { @@ -643,12 +641,12 @@ struct parser_executor { // Check if a delimiter starts at this position auto match = matcher.check_at(ctx.input, pos); - if (match.type == trie_matcher::match_result::COMPLETE_MATCH) { + if (match == trie_matcher::COMPLETE_MATCH) { // Found a complete delimiter, return everything before it return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos, pos); } - if (match.type == trie_matcher::match_result::PARTIAL_MATCH) { + if (match == trie_matcher::PARTIAL_MATCH) { // Found a partial match extending to end of input, return everything before it return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos, pos); } From 96a69800a89bbc7d68e7f2b059e405f246c84470 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Mon, 17 Nov 2025 02:54:35 -0600 Subject: [PATCH 120/183] fix linting issues --- common/chat-peg-parser.cpp | 44 ++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 109a9b6a0d728..a3c70e5f2baf6 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -298,14 +298,14 @@ struct parser_executor { parser_executor(const common_chat_peg_arena & arena, common_chat_parse_context & ctx, size_t start) : arena(arena), ctx(ctx), start_pos(start) {} - common_chat_parse_result operator()(const common_chat_peg_start_parser & /* p */) { + common_chat_parse_result operator()(const common_chat_peg_start_parser & /* p */) const { return common_chat_parse_result( start_pos == 0 ? COMMON_CHAT_PARSE_RESULT_SUCCESS : COMMON_CHAT_PARSE_RESULT_FAIL, start_pos ); } - common_chat_parse_result operator()(const common_chat_peg_end_parser & /* p */) { + common_chat_parse_result operator()(const common_chat_peg_end_parser & /* p */) const { return common_chat_parse_result( start_pos >= ctx.input.size() ? COMMON_CHAT_PARSE_RESULT_SUCCESS : COMMON_CHAT_PARSE_RESULT_FAIL, start_pos @@ -432,7 +432,7 @@ struct parser_executor { return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos); } - common_chat_parse_result operator()(const common_chat_peg_any_parser & /* p */) { + common_chat_parse_result operator()(const common_chat_peg_any_parser & /* p */) const { // Parse a single UTF-8 codepoint (not just a single byte) auto result = parse_utf8_codepoint(ctx.input, start_pos); @@ -462,7 +462,7 @@ struct parser_executor { return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos, pos); } - common_chat_parse_result operator()(const common_chat_peg_chars_parser & p) { + common_chat_parse_result operator()(const common_chat_peg_chars_parser & p) const { auto pos = start_pos; int match_count = 0; @@ -613,7 +613,7 @@ struct parser_executor { return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos); } - common_chat_parse_result operator()(const common_chat_peg_until_parser & p) { + common_chat_parse_result operator()(const common_chat_peg_until_parser & p) const { trie_matcher matcher(p.delimiters); // Scan input and check for delimiters @@ -1205,13 +1205,13 @@ static std::unordered_set collect_reachable_rules( visit(child); } } else if constexpr (std::is_same_v || - std::is_same_v || - std::is_same_v || - std::is_same_v || - std::is_same_v || - std::is_same_v || - std::is_same_v || - std::is_same_v) { + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v) { visit(p.child); } else if constexpr (std::is_same_v) { if (visited.find(p.name) == visited.end()) { @@ -1266,7 +1266,9 @@ void common_chat_peg_arena::build_grammar(const common_grammar_builder & builder } else if constexpr (std::is_same_v) { std::string s; for (const auto & child : p.children) { - if (!s.empty()) s += " "; + if (!s.empty()) { + s += " "; + } auto child_gbnf = to_gbnf(child); const auto & child_parser = parsers_.at(child); if (std::holds_alternative(child_parser) || @@ -1280,7 +1282,9 @@ void common_chat_peg_arena::build_grammar(const common_grammar_builder & builder } else if constexpr (std::is_same_v) { std::string s; for (const auto & child : p.children) { - if (!s.empty()) s += " | "; + if (!s.empty()) { + s += " | "; + } auto child_gbnf = to_gbnf(child); const auto & child_parser = parsers_.at(child); if (std::holds_alternative(child_parser)) { @@ -1335,18 +1339,20 @@ void common_chat_peg_arena::build_grammar(const common_grammar_builder & builder std::string result = p.pattern; if (p.min_count == 0 && p.max_count == -1) { return result + "*"; - } else if (p.min_count == 1 && p.max_count == -1) { + } + if (p.min_count == 1 && p.max_count == -1) { return result + "+"; - } else if (p.max_count == -1) { + } + if (p.max_count == -1) { return result + "{" + std::to_string(p.min_count) + ",}"; - } else if (p.min_count == p.max_count) { + } + if (p.min_count == p.max_count) { if (p.min_count == 1) { return result; } return result + "{" + std::to_string(p.min_count) + "}"; - } else { - return result + "{" + std::to_string(p.min_count) + "," + std::to_string(p.max_count) + "}"; } + return result + "{" + std::to_string(p.min_count) + "," + std::to_string(p.max_count) + "}"; } else if constexpr (std::is_same_v) { return R"(( [^"\\] | "\\" ( ["\\/ bfnrt] | "u" [0-9a-fA-F]{4} ) )*)"; } else if constexpr (std::is_same_v) { From 841bd629f54c97caf272ffb65532ea661f0c110e Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Mon, 17 Nov 2025 03:05:17 -0600 Subject: [PATCH 121/183] add annotations to rules --- common/chat-peg-parser-helper.cpp | 8 ++++---- common/chat-peg-parser-helper.h | 8 ++++---- common/chat-peg-parser.cpp | 16 +++++++++++++--- common/chat-peg-parser.h | 4 ++++ .../chat-peg-parser/test-example-qwen3-coder.cpp | 4 ++-- tests/chat-peg-parser/test_harness.h | 4 +++- 6 files changed, 30 insertions(+), 14 deletions(-) diff --git a/common/chat-peg-parser-helper.cpp b/common/chat-peg-parser-helper.cpp index 43ce524a3d521..38838cced575b 100644 --- a/common/chat-peg-parser-helper.cpp +++ b/common/chat-peg-parser-helper.cpp @@ -47,11 +47,11 @@ common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_no_attr( std::string string_content_2; string_content_2.append(""); - auto string_arg_content = rule("arg-str-content", until_one_of({ string_content_1, string_content_2 })); + auto string_arg_content = rule("arg-string-content", until_one_of({ string_content_1, string_content_2 })); std::string arg_string_name; arg_string_name.append("arg-string-").append(*it); - auto string_arg = rule(arg_string_name, arg_name + string_arg_content + arg_end); + auto string_arg = rule(arg_string_name, "arg-string", arg_name + string_arg_content + arg_end); auto json_sec = json(); std::string arg_json_name; @@ -116,11 +116,11 @@ common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_attr( std::string string_content_2; string_content_2.append(""); - auto string_arg_content = rule("arg-str-content", until_one_of({ string_content_1, string_content_2 })); + auto string_arg_content = rule("arg-string-content", until_one_of({ string_content_1, string_content_2 })); std::string arg_string_name; arg_string_name.append("arg-string-").append(*it); - auto string_arg = rule(arg_string_name, arg_name + string_arg_content + arg_end); + auto string_arg = rule(arg_string_name, "arg-string", arg_name + string_arg_content + arg_end); auto json_sec = json(); std::string arg_json_name; diff --git a/common/chat-peg-parser-helper.h b/common/chat-peg-parser-helper.h index 6fec93d3bd941..5062896702964 100644 --- a/common/chat-peg-parser-helper.h +++ b/common/chat-peg-parser-helper.h @@ -58,12 +58,12 @@ inline void parser_semantic_handler(const common_chat_parse_event & ev, common_c tc.arguments += "\"" + name + "\": "; } - if (ev.rule == "arg-str-content" && ev.ending() && ev.success()) { + if (ev.rule == "arg-string-content" && ev.ending() && ev.success()) { auto & tc = semantics.tool_calls.back(); tc.arguments += "\"" + std::string(ev.text); } - if (ev.rule.find("arg-string") != std::string::npos && ev.ending() && ev.success()) { + if (ev.annotation == "arg-string" && ev.ending() && ev.success()) { auto & tc = semantics.tool_calls.back(); tc.arguments += "\""; } @@ -103,12 +103,12 @@ inline void parser_semantic_handler_with_printout(const common_chat_parse_event tc.arguments += "\"" + name + "\": "; } - if (ev.rule == "arg-str-content" && ev.ending() && ev.success()) { + if (ev.rule == "arg-string-content" && ev.ending() && ev.success()) { auto & tc = semantics.tool_calls.back(); tc.arguments += "\"" + std::string(ev.text); } - if (ev.rule.find("arg-string") != std::string::npos && ev.ending() && ev.success()) { + if (ev.annotation == "arg-string" && ev.ending() && ev.success()) { auto & tc = semantics.tool_calls.back(); tc.arguments += "\""; } diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index a3c70e5f2baf6..d023008f79afa 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -668,6 +668,7 @@ struct parser_executor { ctx.event_handler(common_chat_parse_event{ COMMON_CHAT_PARSE_EVENT_NODE_START, p.name, + p.annotation, start_pos, start_pos, "", @@ -692,6 +693,7 @@ struct parser_executor { ctx.event_handler(common_chat_parse_event{ COMMON_CHAT_PARSE_EVENT_NODE_END, p.name, + p.annotation, result.start, result.end, text, @@ -1016,12 +1018,20 @@ common_chat_peg_parser common_chat_peg_parser_builder::capture(const std::string } common_chat_peg_parser common_chat_peg_parser_builder::rule(const std::string & name, common_chat_peg_parser p, bool trigger) { - auto rule_id = arena_.add_parser(common_chat_peg_rule_parser{name, p.id(), trigger}); + return rule(name, "", p, trigger); +} + +common_chat_peg_parser common_chat_peg_parser_builder::rule(const std::string & name, const std::string & annotation, common_chat_peg_parser p, bool trigger) { + auto rule_id = arena_.add_parser(common_chat_peg_rule_parser{name, annotation, p.id(), trigger}); arena_.add_rule(name, rule_id); return ref(name); } common_chat_peg_parser common_chat_peg_parser_builder::rule(const std::string & name, const std::function & builder_fn, bool trigger) { + return rule(name, "", builder_fn, trigger); +} + +common_chat_peg_parser common_chat_peg_parser_builder::rule(const std::string & name, const std::string & annotation, const std::function & builder_fn, bool trigger) { // Check if rule already exists if (arena_.has_rule(name)) { return ref(name); @@ -1029,14 +1039,14 @@ common_chat_peg_parser common_chat_peg_parser_builder::rule(const std::string & // Create placeholder rule to allow recursive references auto placeholder = any(); // Temporary placeholder - auto placeholder_rule_id = arena_.add_parser(common_chat_peg_rule_parser{name, placeholder.id(), trigger}); + auto placeholder_rule_id = arena_.add_parser(common_chat_peg_rule_parser{name, annotation, placeholder.id(), trigger}); arena_.add_rule(name, placeholder_rule_id); // Build the actual parser auto parser = builder_fn(); // Replace placeholder with actual rule - auto rule_id = arena_.add_parser(common_chat_peg_rule_parser{name, parser.id(), trigger}); + auto rule_id = arena_.add_parser(common_chat_peg_rule_parser{name, annotation, parser.id(), trigger}); arena_.rules_[name] = rule_id; return ref(name); diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index 9d9c7c685daff..873a844ebe6d7 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -127,6 +127,7 @@ enum common_chat_parse_event_type { struct common_chat_parse_event { common_chat_parse_event_type type; std::string rule; + std::string annotation; size_t start; size_t end; std::string_view text; @@ -259,6 +260,7 @@ struct common_chat_peg_schema_parser { struct common_chat_peg_rule_parser { std::string name; + std::string annotation; common_chat_peg_parser_id child; bool trigger; }; @@ -460,6 +462,7 @@ class common_chat_peg_parser_builder { // If trigger=true, marks this rule as an entry point for lazy grammar generation. // auto json = p.rule("json", json_obj | json_arr | ...) common_chat_peg_parser rule(const std::string & name, common_chat_peg_parser p, bool trigger = false); + common_chat_peg_parser rule(const std::string & name, const std::string & annotation, common_chat_peg_parser p, bool trigger = false); // Creates a named rule using a builder function. This handles recursive grammars by // inserting a placeholder rule before invoking the builder, allowing the @@ -468,6 +471,7 @@ class common_chat_peg_parser_builder { // If trigger=true, marks this rule as an entry point for lazy grammar generation. // auto json = p.rule("json", [&]() { return json_object() | json_array() | ... }) common_chat_peg_parser rule(const std::string & name, const std::function & builder, bool trigger = false); + common_chat_peg_parser rule(const std::string & name, const std::string & annotation, const std::function & builder, bool trigger = false); void set_root(common_chat_peg_parser p); diff --git a/tests/chat-peg-parser/test-example-qwen3-coder.cpp b/tests/chat-peg-parser/test-example-qwen3-coder.cpp index 6c690c4e5e52a..8488f1e476b0e 100644 --- a/tests/chat-peg-parser/test-example-qwen3-coder.cpp +++ b/tests/chat-peg-parser/test-example-qwen3-coder.cpp @@ -14,10 +14,10 @@ void test_example_qwen3_coder(testing &t) { auto arg_name = p.rule("arg-start", ""); auto arg_end = p.rule("arg-end", "" + p.peek(p.literal("")); - auto string_arg_content = p.rule("arg-str-content", + auto string_arg_content = p.rule("arg-string-content", p.until_one_of({""})); - auto string_arg = p.rule("arg-string", arg_name + string_arg_content + arg_end); + auto string_arg = p.rule("arg-string", "arg-string", arg_name + string_arg_content + arg_end); auto json = p.json(); diff --git a/tests/chat-peg-parser/test_harness.h b/tests/chat-peg-parser/test_harness.h index 1cf99fe2e42b5..f26a7ca2ad290 100644 --- a/tests/chat-peg-parser/test_harness.h +++ b/tests/chat-peg-parser/test_harness.h @@ -27,18 +27,20 @@ struct testing { template void run_with_exceptions(F &&f, const char *ctx) { + f(); try { - f(); } catch (const std::exception &e) { ++failures; ++exceptions; indent(); out << "UNHANDLED EXCEPTION (" << ctx << "): " << e.what() << "\n"; + throw e; } catch (...) { ++failures; ++exceptions; indent(); out << "UNHANDLED EXCEPTION (" << ctx << "): unknown\n"; + throw; } } From 9b3d4f21b6b266066af89c4761b98099695d512c Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Mon, 17 Nov 2025 03:12:26 -0600 Subject: [PATCH 122/183] revert test workaround --- tests/chat-peg-parser/test_harness.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/chat-peg-parser/test_harness.h b/tests/chat-peg-parser/test_harness.h index f26a7ca2ad290..adc541acf01f4 100644 --- a/tests/chat-peg-parser/test_harness.h +++ b/tests/chat-peg-parser/test_harness.h @@ -27,8 +27,8 @@ struct testing { template void run_with_exceptions(F &&f, const char *ctx) { - f(); try { + f(); } catch (const std::exception &e) { ++failures; ++exceptions; From 305ed3eeaae07cd8f4fedc5ecd7aaf31fb8ed931 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Mon, 17 Nov 2025 19:45:29 -0600 Subject: [PATCH 123/183] implement serializing the parser --- common/chat-peg-parser.cpp | 290 ++++++++++++++++++ common/chat-peg-parser.h | 4 + tests/CMakeLists.txt | 1 + .../test-json-serialization.cpp | 243 +++++++++++++++ tests/chat-peg-parser/tests.h | 1 + tests/test-chat-peg-parser.cpp | 3 +- 6 files changed, 541 insertions(+), 1 deletion(-) create mode 100644 tests/chat-peg-parser/test-json-serialization.cpp diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index d023008f79afa..144f4b88cefe0 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -1438,3 +1438,293 @@ void common_chat_peg_arena::build_grammar(const common_grammar_builder & builder } } } + +// Serialization helper: convert parser variant to JSON +static nlohmann::json serialize_parser_variant(const common_chat_peg_parser_variant & variant) { + return std::visit([](const auto & p) -> nlohmann::json { + using T = std::decay_t; + + nlohmann::json j; + + if constexpr (std::is_same_v) { + j["type"] = "start"; + } else if constexpr (std::is_same_v) { + j["type"] = "end"; + } else if constexpr (std::is_same_v) { + j["type"] = "literal"; + j["literal"] = p.literal; + } else if constexpr (std::is_same_v) { + j["type"] = "sequence"; + j["children"] = p.children; + } else if constexpr (std::is_same_v) { + j["type"] = "choice"; + j["children"] = p.children; + } else if constexpr (std::is_same_v) { + j["type"] = "repetition"; + j["child"] = p.child; + j["min_count"] = p.min_count; + j["max_count"] = p.max_count; + } else if constexpr (std::is_same_v) { + j["type"] = "one_or_more"; + j["child"] = p.child; + } else if constexpr (std::is_same_v) { + j["type"] = "zero_or_more"; + j["child"] = p.child; + } else if constexpr (std::is_same_v) { + j["type"] = "optional"; + j["child"] = p.child; + } else if constexpr (std::is_same_v) { + j["type"] = "and"; + j["child"] = p.child; + } else if constexpr (std::is_same_v) { + j["type"] = "not"; + j["child"] = p.child; + } else if constexpr (std::is_same_v) { + j["type"] = "any"; + } else if constexpr (std::is_same_v) { + j["type"] = "space"; + } else if constexpr (std::is_same_v) { + j["type"] = "chars"; + j["pattern"] = p.pattern; + nlohmann::json ranges = nlohmann::json::array(); + for (const auto & range : p.ranges) { + ranges.push_back({ + {"start", range.start}, + {"end", range.end} + }); + } + j["ranges"] = ranges; + j["negated"] = p.negated; + j["min_count"] = p.min_count; + j["max_count"] = p.max_count; + } else if constexpr (std::is_same_v) { + j["type"] = "json_string"; + } else if constexpr (std::is_same_v) { + j["type"] = "until"; + j["delimiters"] = p.delimiters; + } else if constexpr (std::is_same_v) { + j["type"] = "schema"; + j["child"] = p.child; + j["name"] = p.name; + if (p.schema) { + j["schema"] = *p.schema; + } else { + j["schema"] = nullptr; + } + } else if constexpr (std::is_same_v) { + j["type"] = "rule"; + j["name"] = p.name; + j["annotation"] = p.annotation; + j["child"] = p.child; + j["trigger"] = p.trigger; + } else if constexpr (std::is_same_v) { + j["type"] = "ref"; + j["name"] = p.name; + } else if constexpr (std::is_same_v) { + j["type"] = "capture"; + j["child"] = p.child; + j["key"] = p.key; + } + + return j; + }, variant); +} + +nlohmann::json common_chat_peg_arena::to_json() const { + nlohmann::json j; + + auto parsers = nlohmann::json::array(); + for (const auto & parser : parsers_) { + parsers.push_back(serialize_parser_variant(parser)); + } + + j["parsers"] = parsers; + j["rules"] = rules_; + j["root"] = root_; + return j; +} + +// Deserialization helper: convert JSON to parser variant +static common_chat_peg_parser_variant deserialize_parser_variant(const nlohmann::json & j) { + if (!j.contains("type") || !j["type"].is_string()) { + throw std::runtime_error("Parser variant JSON missing or invalid 'type' field"); + } + + std::string type = j["type"]; + + if (type == "start") { + return common_chat_peg_start_parser{}; + } + if (type == "end") { + return common_chat_peg_end_parser{}; + } + if (type == "literal") { + if (!j.contains("literal") || !j["literal"].is_string()) { + throw std::runtime_error("literal parser missing or invalid 'literal' field"); + } + return common_chat_peg_literal_parser{j["literal"]}; + } + if (type == "sequence") { + if (!j.contains("children") || !j["children"].is_array()) { + throw std::runtime_error("sequence parser missing or invalid 'children' field"); + } + return common_chat_peg_sequence_parser{j["children"].get>()}; + } + if (type == "choice") { + if (!j.contains("children") || !j["children"].is_array()) { + throw std::runtime_error("choice parser missing or invalid 'children' field"); + } + return common_chat_peg_choice_parser{j["children"].get>()}; + } + if (type == "repetition") { + if (!j.contains("child") || !j.contains("min_count") || !j.contains("max_count")) { + throw std::runtime_error("repetition parser missing required fields"); + } + return common_chat_peg_repetition_parser{ + j["child"].get(), + j["min_count"].get(), + j["max_count"].get() + }; + } + if (type == "one_or_more") { + if (!j.contains("child")) { + throw std::runtime_error("one_or_more parser missing 'child' field"); + } + return common_chat_peg_one_or_more_parser{j["child"].get()}; + } + if (type == "zero_or_more") { + if (!j.contains("child")) { + throw std::runtime_error("zero_or_more parser missing 'child' field"); + } + return common_chat_peg_zero_or_more_parser{j["child"].get()}; + } + if (type == "optional") { + if (!j.contains("child")) { + throw std::runtime_error("optional parser missing 'child' field"); + } + return common_chat_peg_optional_parser{j["child"].get()}; + } + if (type == "and") { + if (!j.contains("child")) { + throw std::runtime_error("and parser missing 'child' field"); + } + return common_chat_peg_and_parser{j["child"].get()}; + } + if (type == "not") { + if (!j.contains("child")) { + throw std::runtime_error("not parser missing 'child' field"); + } + return common_chat_peg_not_parser{j["child"].get()}; + } + if (type == "any") { + return common_chat_peg_any_parser{}; + } + if (type == "space") { + return common_chat_peg_space_parser{}; + } + if (type == "chars") { + if (!j.contains("pattern") || !j.contains("ranges") || !j.contains("negated") || + !j.contains("min_count") || !j.contains("max_count")) { + throw std::runtime_error("chars parser missing required fields"); + } + common_chat_peg_chars_parser parser; + parser.pattern = j["pattern"]; + parser.negated = j["negated"]; + parser.min_count = j["min_count"]; + parser.max_count = j["max_count"]; + for (const auto & range_json : j["ranges"]) { + if (!range_json.contains("start") || !range_json.contains("end")) { + throw std::runtime_error("char_range missing 'start' or 'end' field"); + } + parser.ranges.push_back({ + range_json["start"].get(), + range_json["end"].get() + }); + } + return parser; + } + if (type == "json_string") { + return common_chat_peg_json_string_parser{}; + } + if (type == "until") { + if (!j.contains("delimiters") || !j["delimiters"].is_array()) { + throw std::runtime_error("until parser missing or invalid 'delimiters' field"); + } + return common_chat_peg_until_parser{j["delimiters"].get>()}; + } + if (type == "schema") { + if (!j.contains("child") || !j.contains("name") || !j.contains("schema")) { + throw std::runtime_error("schema parser missing required fields"); + } + common_chat_peg_schema_parser parser; + parser.child = j["child"].get(); + parser.name = j["name"]; + if (!j["schema"].is_null()) { + parser.schema = std::make_shared(j["schema"]); + } + return parser; + } + if (type == "rule") { + if (!j.contains("name") || !j.contains("annotation") || !j.contains("child") || !j.contains("trigger")) { + throw std::runtime_error("rule parser missing required fields"); + } + return common_chat_peg_rule_parser{ + j["name"].get(), + j["annotation"].get(), + j["child"].get(), + j["trigger"].get() + }; + } + if (type == "ref") { + if (!j.contains("name") || !j["name"].is_string()) { + throw std::runtime_error("ref parser missing or invalid 'name' field"); + } + return common_chat_peg_ref_parser{j["name"]}; + } + if (type == "capture") { + if (!j.contains("child") || !j.contains("key")) { + throw std::runtime_error("capture parser missing required fields"); + } + return common_chat_peg_capture_parser{ + j["child"].get(), + j["key"].get() + }; + } + + throw std::runtime_error("Unknown parser type: " + type); +} + +common_chat_peg_arena common_chat_peg_arena::from_json(const nlohmann::json & j) { + if (!j.contains("parsers") || !j["parsers"].is_array()) { + throw std::runtime_error("JSON missing or invalid 'parsers' array"); + } + if (!j.contains("rules") || !j["rules"].is_object()) { + throw std::runtime_error("JSON missing or invalid 'rules' object"); + } + if (!j.contains("root")) { + throw std::runtime_error("JSON missing 'root' field"); + } + + common_chat_peg_arena arena; + + const auto & parsers_json = j["parsers"]; + arena.parsers_.reserve(parsers_json.size()); + for (const auto & parser_json : parsers_json) { + arena.parsers_.push_back(deserialize_parser_variant(parser_json)); + } + + arena.rules_ = j["rules"].get>(); + + for (const auto & [name, id] : arena.rules_) { + if (id >= arena.parsers_.size()) { + throw std::runtime_error("Rule '" + name + "' references invalid parser ID: " + std::to_string(id)); + } + } + + arena.root_ = j["root"].get(); + if (arena.root_ != COMMON_CHAT_PEG_INVALID_PARSER_ID && arena.root_ >= arena.parsers_.size()) { + throw std::runtime_error("Root references invalid parser ID: " + std::to_string(arena.root_)); + } + + return arena; +} diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index 873a844ebe6d7..cede4e31360ef 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -331,6 +331,10 @@ class common_chat_peg_arena { // Dump for debugging std::string dump(common_chat_peg_parser_id id) const; + // Serialization + nlohmann::json to_json() const; + static common_chat_peg_arena from_json(const nlohmann::json & j); + // Builder access (for adding parsers) friend class common_chat_peg_parser_builder; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c83fc844af3b1..033dcacee560f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -193,6 +193,7 @@ llama_build_and_test( chat-peg-parser/test-example-seed-oss.cpp chat-peg-parser/test-gbnf-generation.cpp chat-peg-parser/test-json-parser.cpp + chat-peg-parser/test-json-serialization.cpp chat-peg-parser/test-one.cpp chat-peg-parser/test-optional.cpp chat-peg-parser/test-partial-parsing.cpp diff --git a/tests/chat-peg-parser/test-json-serialization.cpp b/tests/chat-peg-parser/test-json-serialization.cpp new file mode 100644 index 0000000000000..df771f29de7f9 --- /dev/null +++ b/tests/chat-peg-parser/test-json-serialization.cpp @@ -0,0 +1,243 @@ +#include "tests.h" + +void test_json_serialization(testing &t) { + t.test("simple literal parser round-trip", [](testing &t) { + auto original = build_peg_parser([](common_chat_peg_parser_builder & p) { + return p.literal("hello"); + }); + + auto json = original.to_json(); + auto deserialized = common_chat_peg_arena::from_json(json); + + // Test that both parsers produce identical results + std::string input = "hello world"; + common_chat_parse_context ctx1(input); + common_chat_parse_context ctx2(input); + + auto result1 = original.parse(ctx1); + auto result2 = deserialized.parse(ctx2); + + t.assert_equal("both_succeed", result1.success(), result2.success()); + t.assert_equal("same_end_pos", result1.end, result2.end); + }); + + t.test("complex parser round-trip", [](testing &t) { + auto original = build_peg_parser([](common_chat_peg_parser_builder & p) { + return p.choice({ + p.sequence({p.literal("hello"), p.space(), p.literal("world")}), + p.literal("goodbye") + }); + }); + + auto json = original.to_json(); + auto deserialized = common_chat_peg_arena::from_json(json); + + // Test both branches work identically + std::string input1 = "hello world"; + common_chat_parse_context ctx1a(input1); + common_chat_parse_context ctx1b(input1); + + auto result1a = original.parse(ctx1a); + auto result1b = deserialized.parse(ctx1b); + + t.assert_equal("hello_both_succeed", result1a.success(), result1b.success()); + t.assert_equal("hello_same_end", result1a.end, result1b.end); + + std::string input2 = "goodbye"; + common_chat_parse_context ctx2a(input2); + common_chat_parse_context ctx2b(input2); + + auto result2a = original.parse(ctx2a); + auto result2b = deserialized.parse(ctx2b); + + t.assert_equal("goodbye_both_succeed", result2a.success(), result2b.success()); + t.assert_equal("goodbye_same_end", result2a.end, result2b.end); + }); + + // Test round-trip serialization of recursive grammar + t.test("recursive grammar round-trip", [](testing &t) { + auto original = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto expr = p.rule("expr", [&]() { + return p.choice({ + p.sequence({p.literal("("), p.space(), p.ref("expr"), p.space(), p.literal(")")}), + p.one("[a-z]+") + }); + }); + return expr; + }); + + // Serialize + auto json = original.to_json(); + + // Deserialize + auto deserialized = common_chat_peg_arena::from_json(json); + + // Test nested expressions + std::string input = "(( abc ))"; + common_chat_parse_context ctx1(input); + common_chat_parse_context ctx2(input); + + auto result1 = original.parse(ctx1); + auto result2 = deserialized.parse(ctx2); + + t.assert_equal("both_succeed", result1.success(), result2.success()); + t.assert_equal("same_end_pos", result1.end, result2.end); + }); + + // Test round-trip serialization of JSON parser + t.test("JSON parser round-trip", [](testing &t) { + auto original = build_peg_parser([](common_chat_peg_parser_builder & p) { + return p.json(); + }); + + // Serialize + auto json_serialized = original.to_json(); + + // Deserialize + auto deserialized = common_chat_peg_arena::from_json(json_serialized); + + // Test complex JSON + std::string input = R"({"name": "test", "values": [1, 2, 3], "nested": {"a": true}})"; + common_chat_parse_context ctx1(input); + common_chat_parse_context ctx2(input); + + auto result1 = original.parse(ctx1); + auto result2 = deserialized.parse(ctx2); + + t.assert_equal("both_succeed", result1.success(), result2.success()); + t.assert_equal("same_end_pos", result1.end, result2.end); + }); + + // Test round-trip with captures + t.test("parser with captures round-trip", [](testing &t) { + auto original = build_peg_parser([](common_chat_peg_parser_builder & p) { + return p.sequence({ + p.capture("greeting", p.literal("hello")), + p.space(), + p.capture("name", p.one("[a-z]+")) + }); + }); + + // Serialize + auto json = original.to_json(); + + // Deserialize + auto deserialized = common_chat_peg_arena::from_json(json); + + // Test with semantics + std::string input = "hello alice"; + common_chat_parse_semantics sem1; + common_chat_parse_semantics sem2; + common_chat_parse_context ctx1(input, &sem1); + common_chat_parse_context ctx2(input, &sem2); + + auto result1 = original.parse(ctx1); + auto result2 = deserialized.parse(ctx2); + + t.assert_equal("both_succeed", result1.success(), result2.success()); + t.assert_equal("both_capture_greeting", sem1.captures.count("greeting") > 0, sem2.captures.count("greeting") > 0); + t.assert_equal("both_capture_name", sem1.captures.count("name") > 0, sem2.captures.count("name") > 0); + if (sem1.captures.count("greeting") && sem2.captures.count("greeting")) { + t.assert_equal("same_greeting", sem1.captures["greeting"], sem2.captures["greeting"]); + } + if (sem1.captures.count("name") && sem2.captures.count("name")) { + t.assert_equal("same_name", sem1.captures["name"], sem2.captures["name"]); + } + }); + + // Test serialization with repetitions + t.test("parser with repetitions round-trip", [](testing &t) { + auto original = build_peg_parser([](common_chat_peg_parser_builder & p) { + return p.sequence({ + p.one_or_more(p.one("[a-z]")), + p.optional(p.one("[0-9]")), + p.zero_or_more(p.literal("!")) + }); + }); + + // Serialize + auto json = original.to_json(); + + // Deserialize + auto deserialized = common_chat_peg_arena::from_json(json); + + // Test various inputs + std::vector test_inputs = {"abc", "abc5", "xyz!!!", "test9!"}; + + for (const auto& input : test_inputs) { + common_chat_parse_context ctx1(input); + common_chat_parse_context ctx2(input); + + auto result1 = original.parse(ctx1); + auto result2 = deserialized.parse(ctx2); + + t.assert_equal("input_" + input + "_both_succeed", result1.success(), result2.success()); + t.assert_equal("input_" + input + "_same_end", result1.end, result2.end); + } + }); + + // Test serialization with chars parser + t.test("chars parser round-trip", [](testing &t) { + auto original = build_peg_parser([](common_chat_peg_parser_builder & p) { + return p.chars("[a-zA-Z0-9_]", 3, 10); + }); + + // Serialize + auto json = original.to_json(); + + // Deserialize + auto deserialized = common_chat_peg_arena::from_json(json); + + // Test + std::string input = "hello123"; + common_chat_parse_context ctx1(input); + common_chat_parse_context ctx2(input); + + auto result1 = original.parse(ctx1); + auto result2 = deserialized.parse(ctx2); + + t.assert_equal("both_succeed", result1.success(), result2.success()); + t.assert_equal("same_end_pos", result1.end, result2.end); + }); + + // Test serialization with lookahead + t.test("lookahead parser round-trip", [](testing &t) { + auto original = build_peg_parser([](common_chat_peg_parser_builder & p) { + return p.sequence({ + p.peek(p.literal("test")), + p.literal("test") + }); + }); + + // Serialize + auto json = original.to_json(); + + // Deserialize + auto deserialized = common_chat_peg_arena::from_json(json); + + // Test + std::string input = "test"; + common_chat_parse_context ctx1(input); + common_chat_parse_context ctx2(input); + + auto result1 = original.parse(ctx1); + auto result2 = deserialized.parse(ctx2); + + t.assert_equal("both_succeed", result1.success(), result2.success()); + t.assert_equal("same_end_pos", result1.end, result2.end); + }); + + // Benchmark: deserialize JSON parser + t.test("deserialize JSON parser", [&](testing & t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + return p.json(); + }); + + auto json = parser.to_json(); + auto json_str = json.dump(); + + t.bench("deserialize json", [&]() { + auto deserialized = common_chat_peg_arena::from_json(nlohmann::json::parse(json_str)); + }, 1000); + }); +} diff --git a/tests/chat-peg-parser/tests.h b/tests/chat-peg-parser/tests.h index 489da8944eb59..156ca0b37f3c2 100644 --- a/tests/chat-peg-parser/tests.h +++ b/tests/chat-peg-parser/tests.h @@ -29,3 +29,4 @@ void test_example_seed_oss(testing &t); void test_example_minimax_m2(testing &t); void test_command7_parser_compare(testing &t); void test_unicode(testing &t); +void test_json_serialization(testing &t); diff --git a/tests/test-chat-peg-parser.cpp b/tests/test-chat-peg-parser.cpp index 948df1e4f11a4..ca5c14ba5eeb0 100644 --- a/tests/test-chat-peg-parser.cpp +++ b/tests/test-chat-peg-parser.cpp @@ -24,7 +24,8 @@ static const std::vector all_tests = { {"qwen3_coder", "test_example_qwen3_coder", test_example_qwen3_coder}, {"seed_oss", "test_example_seed_oss", test_example_seed_oss}, {"minimax_m2", "test_example_minimax_m2", test_example_minimax_m2}, - {"command7_parser_compare", "test_command7_parser_compare", test_command7_parser_compare} + {"command7_parser_compare", "test_command7_parser_compare", test_command7_parser_compare}, + {"serialization", "test_json_serialization", test_json_serialization} }; // Function to list all available tests From 3948cdf2d2846880f1188ea0ddf4497ca00f5b0a Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Mon, 17 Nov 2025 20:20:08 -0600 Subject: [PATCH 124/183] remove redundant parsers --- common/chat-peg-parser.cpp | 103 +++++++------------------------------ common/chat-peg-parser.h | 15 ------ 2 files changed, 20 insertions(+), 98 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 144f4b88cefe0..d38bdd0b936ae 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -397,18 +397,6 @@ struct parser_executor { return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos, pos); } - common_chat_parse_result operator()(const common_chat_peg_one_or_more_parser & p) { - return (*this)(common_chat_peg_repetition_parser{p.child, 1, -1}); - } - - common_chat_parse_result operator()(const common_chat_peg_zero_or_more_parser & p) { - return (*this)(common_chat_peg_repetition_parser{p.child, 0, -1}); - } - - common_chat_parse_result operator()(const common_chat_peg_optional_parser & p) { - return (*this)(common_chat_peg_repetition_parser{p.child, 0, 1}); - } - common_chat_parse_result operator()(const common_chat_peg_and_parser & p) { auto result = arena.parse(p.child, ctx, start_pos); // Pass result but don't consume input @@ -777,12 +765,6 @@ std::string common_chat_peg_arena::dump(common_chat_peg_parser_id id) const { return "Repetition(" + dump(p.child) + ", " + std::to_string(p.min_count) + ", unbounded)"; } return "Repetition(" + dump(p.child) + ", " + std::to_string(p.min_count) + ", " + std::to_string(p.max_count) + ")"; - } else if constexpr (std::is_same_v) { - return "OneOrMore(" + dump(p.child) + ")"; - } else if constexpr (std::is_same_v) { - return "ZeroOrMore(" + dump(p.child) + ")"; - } else if constexpr (std::is_same_v) { - return "Optional(" + dump(p.child) + ")"; } else if constexpr (std::is_same_v) { return "And(" + dump(p.child) + ")"; } else if constexpr (std::is_same_v) { @@ -948,18 +930,6 @@ common_chat_peg_parser common_chat_peg_parser_builder::choice(std::initializer_l return choice(ids); } -common_chat_peg_parser common_chat_peg_parser_builder::one_or_more(common_chat_peg_parser p) { - return wrap(arena_.add_parser(common_chat_peg_one_or_more_parser{p.id()})); -} - -common_chat_peg_parser common_chat_peg_parser_builder::zero_or_more(common_chat_peg_parser p) { - return wrap(arena_.add_parser(common_chat_peg_zero_or_more_parser{p.id()})); -} - -common_chat_peg_parser common_chat_peg_parser_builder::optional(common_chat_peg_parser p) { - return wrap(arena_.add_parser(common_chat_peg_optional_parser{p.id()})); -} - common_chat_peg_parser common_chat_peg_parser_builder::peek(common_chat_peg_parser p) { return wrap(arena_.add_parser(common_chat_peg_and_parser{p.id()})); } @@ -1005,6 +975,18 @@ common_chat_peg_parser common_chat_peg_parser_builder::repeat(common_chat_peg_pa return wrap(arena_.add_parser(common_chat_peg_repetition_parser{p.id(), n, n})); } +common_chat_peg_parser common_chat_peg_parser_builder::optional(common_chat_peg_parser p) { + return repeat(p, 0, 1); +} + +common_chat_peg_parser common_chat_peg_parser_builder::zero_or_more(common_chat_peg_parser p) { + return repeat(p, 0, -1); +} + +common_chat_peg_parser common_chat_peg_parser_builder::one_or_more(common_chat_peg_parser p) { + return repeat(p, 1, -1); +} + common_chat_peg_parser common_chat_peg_parser_builder::json_string_content() { return wrap(arena_.add_parser(common_chat_peg_json_string_parser{})); } @@ -1215,9 +1197,6 @@ static std::unordered_set collect_reachable_rules( visit(child); } } else if constexpr (std::is_same_v || - std::is_same_v || - std::is_same_v || - std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || @@ -1304,36 +1283,21 @@ void common_chat_peg_arena::build_grammar(const common_grammar_builder & builder } } return s; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { auto child_gbnf = to_gbnf(p.child); const auto & child_parser = parsers_.at(p.child); if (std::holds_alternative(child_parser) || std::holds_alternative(child_parser)) { - return "(" + child_gbnf + ")+"; + child_gbnf = "(" + child_gbnf + ")"; } - return child_gbnf + "+"; - } else if constexpr (std::is_same_v) { - auto child_gbnf = to_gbnf(p.child); - const auto & child_parser = parsers_.at(p.child); - if (std::holds_alternative(child_parser) || - std::holds_alternative(child_parser)) { - return "(" + child_gbnf + ")*"; + if (p.min_count == 0 && p.max_count == 1) { + return child_gbnf + "?"; } - return child_gbnf + "*"; - } else if constexpr (std::is_same_v) { - auto child_gbnf = to_gbnf(p.child); - const auto & child_parser = parsers_.at(p.child); - if (std::holds_alternative(child_parser) || - std::holds_alternative(child_parser)) { - return "(" + child_gbnf + ")?"; + if (p.min_count == 0 && p.max_count == -1) { + return child_gbnf + "*"; } - return child_gbnf + "?"; - } else if constexpr (std::is_same_v) { - auto child_gbnf = to_gbnf(p.child); - const auto & child_parser = parsers_.at(p.child); - if (std::holds_alternative(child_parser) || - std::holds_alternative(child_parser)) { - child_gbnf = "(" + child_gbnf + ")"; + if (p.min_count == 1 && p.max_count == -1) { + return child_gbnf + "+"; } if (p.max_count == -1) { return child_gbnf + "{" + std::to_string(p.min_count) + ",}"; @@ -1464,15 +1428,6 @@ static nlohmann::json serialize_parser_variant(const common_chat_peg_parser_vari j["child"] = p.child; j["min_count"] = p.min_count; j["max_count"] = p.max_count; - } else if constexpr (std::is_same_v) { - j["type"] = "one_or_more"; - j["child"] = p.child; - } else if constexpr (std::is_same_v) { - j["type"] = "zero_or_more"; - j["child"] = p.child; - } else if constexpr (std::is_same_v) { - j["type"] = "optional"; - j["child"] = p.child; } else if constexpr (std::is_same_v) { j["type"] = "and"; j["child"] = p.child; @@ -1586,24 +1541,6 @@ static common_chat_peg_parser_variant deserialize_parser_variant(const nlohmann: j["max_count"].get() }; } - if (type == "one_or_more") { - if (!j.contains("child")) { - throw std::runtime_error("one_or_more parser missing 'child' field"); - } - return common_chat_peg_one_or_more_parser{j["child"].get()}; - } - if (type == "zero_or_more") { - if (!j.contains("child")) { - throw std::runtime_error("zero_or_more parser missing 'child' field"); - } - return common_chat_peg_zero_or_more_parser{j["child"].get()}; - } - if (type == "optional") { - if (!j.contains("child")) { - throw std::runtime_error("optional parser missing 'child' field"); - } - return common_chat_peg_optional_parser{j["child"].get()}; - } if (type == "and") { if (!j.contains("child")) { throw std::runtime_error("and parser missing 'child' field"); diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index cede4e31360ef..779d3884cdea9 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -208,18 +208,6 @@ struct common_chat_peg_repetition_parser { int max_count; // -1 for unbounded }; -struct common_chat_peg_one_or_more_parser { - common_chat_peg_parser_id child; -}; - -struct common_chat_peg_zero_or_more_parser { - common_chat_peg_parser_id child; -}; - -struct common_chat_peg_optional_parser { - common_chat_peg_parser_id child; -}; - struct common_chat_peg_and_parser { common_chat_peg_parser_id child; }; @@ -282,9 +270,6 @@ using common_chat_peg_parser_variant = std::variant< common_chat_peg_sequence_parser, common_chat_peg_choice_parser, common_chat_peg_repetition_parser, - common_chat_peg_one_or_more_parser, - common_chat_peg_zero_or_more_parser, - common_chat_peg_optional_parser, common_chat_peg_and_parser, common_chat_peg_not_parser, common_chat_peg_any_parser, From 0de56d65d9214df0511ad7e2b1fcf760995f138e Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Mon, 17 Nov 2025 20:20:19 -0600 Subject: [PATCH 125/183] remove tests --- .../test-json-serialization.cpp | 231 +----------------- 1 file changed, 8 insertions(+), 223 deletions(-) diff --git a/tests/chat-peg-parser/test-json-serialization.cpp b/tests/chat-peg-parser/test-json-serialization.cpp index df771f29de7f9..487f346347af7 100644 --- a/tests/chat-peg-parser/test-json-serialization.cpp +++ b/tests/chat-peg-parser/test-json-serialization.cpp @@ -1,100 +1,14 @@ #include "tests.h" void test_json_serialization(testing &t) { - t.test("simple literal parser round-trip", [](testing &t) { - auto original = build_peg_parser([](common_chat_peg_parser_builder & p) { - return p.literal("hello"); - }); - - auto json = original.to_json(); - auto deserialized = common_chat_peg_arena::from_json(json); - - // Test that both parsers produce identical results - std::string input = "hello world"; - common_chat_parse_context ctx1(input); - common_chat_parse_context ctx2(input); - - auto result1 = original.parse(ctx1); - auto result2 = deserialized.parse(ctx2); - - t.assert_equal("both_succeed", result1.success(), result2.success()); - t.assert_equal("same_end_pos", result1.end, result2.end); - }); - - t.test("complex parser round-trip", [](testing &t) { - auto original = build_peg_parser([](common_chat_peg_parser_builder & p) { - return p.choice({ - p.sequence({p.literal("hello"), p.space(), p.literal("world")}), - p.literal("goodbye") - }); - }); - - auto json = original.to_json(); - auto deserialized = common_chat_peg_arena::from_json(json); - - // Test both branches work identically - std::string input1 = "hello world"; - common_chat_parse_context ctx1a(input1); - common_chat_parse_context ctx1b(input1); - - auto result1a = original.parse(ctx1a); - auto result1b = deserialized.parse(ctx1b); - - t.assert_equal("hello_both_succeed", result1a.success(), result1b.success()); - t.assert_equal("hello_same_end", result1a.end, result1b.end); - - std::string input2 = "goodbye"; - common_chat_parse_context ctx2a(input2); - common_chat_parse_context ctx2b(input2); - - auto result2a = original.parse(ctx2a); - auto result2b = deserialized.parse(ctx2b); - - t.assert_equal("goodbye_both_succeed", result2a.success(), result2b.success()); - t.assert_equal("goodbye_same_end", result2a.end, result2b.end); - }); - - // Test round-trip serialization of recursive grammar - t.test("recursive grammar round-trip", [](testing &t) { - auto original = build_peg_parser([](common_chat_peg_parser_builder & p) { - auto expr = p.rule("expr", [&]() { - return p.choice({ - p.sequence({p.literal("("), p.space(), p.ref("expr"), p.space(), p.literal(")")}), - p.one("[a-z]+") - }); - }); - return expr; - }); - - // Serialize - auto json = original.to_json(); - - // Deserialize - auto deserialized = common_chat_peg_arena::from_json(json); - - // Test nested expressions - std::string input = "(( abc ))"; - common_chat_parse_context ctx1(input); - common_chat_parse_context ctx2(input); - - auto result1 = original.parse(ctx1); - auto result2 = deserialized.parse(ctx2); - - t.assert_equal("both_succeed", result1.success(), result2.success()); - t.assert_equal("same_end_pos", result1.end, result2.end); + auto original = build_peg_parser([](common_chat_peg_parser_builder & p) { + return "" + p.json() + ""; }); - // Test round-trip serialization of JSON parser - t.test("JSON parser round-trip", [](testing &t) { - auto original = build_peg_parser([](common_chat_peg_parser_builder & p) { - return p.json(); - }); - - // Serialize - auto json_serialized = original.to_json(); + auto json_serialized = original.to_json().dump(); - // Deserialize - auto deserialized = common_chat_peg_arena::from_json(json_serialized); + t.test("compare before/after", [&](testing &t) { + auto deserialized = common_chat_peg_arena::from_json(nlohmann::json::parse(json_serialized)); // Test complex JSON std::string input = R"({"name": "test", "values": [1, 2, 3], "nested": {"a": true}})"; @@ -108,136 +22,7 @@ void test_json_serialization(testing &t) { t.assert_equal("same_end_pos", result1.end, result2.end); }); - // Test round-trip with captures - t.test("parser with captures round-trip", [](testing &t) { - auto original = build_peg_parser([](common_chat_peg_parser_builder & p) { - return p.sequence({ - p.capture("greeting", p.literal("hello")), - p.space(), - p.capture("name", p.one("[a-z]+")) - }); - }); - - // Serialize - auto json = original.to_json(); - - // Deserialize - auto deserialized = common_chat_peg_arena::from_json(json); - - // Test with semantics - std::string input = "hello alice"; - common_chat_parse_semantics sem1; - common_chat_parse_semantics sem2; - common_chat_parse_context ctx1(input, &sem1); - common_chat_parse_context ctx2(input, &sem2); - - auto result1 = original.parse(ctx1); - auto result2 = deserialized.parse(ctx2); - - t.assert_equal("both_succeed", result1.success(), result2.success()); - t.assert_equal("both_capture_greeting", sem1.captures.count("greeting") > 0, sem2.captures.count("greeting") > 0); - t.assert_equal("both_capture_name", sem1.captures.count("name") > 0, sem2.captures.count("name") > 0); - if (sem1.captures.count("greeting") && sem2.captures.count("greeting")) { - t.assert_equal("same_greeting", sem1.captures["greeting"], sem2.captures["greeting"]); - } - if (sem1.captures.count("name") && sem2.captures.count("name")) { - t.assert_equal("same_name", sem1.captures["name"], sem2.captures["name"]); - } - }); - - // Test serialization with repetitions - t.test("parser with repetitions round-trip", [](testing &t) { - auto original = build_peg_parser([](common_chat_peg_parser_builder & p) { - return p.sequence({ - p.one_or_more(p.one("[a-z]")), - p.optional(p.one("[0-9]")), - p.zero_or_more(p.literal("!")) - }); - }); - - // Serialize - auto json = original.to_json(); - - // Deserialize - auto deserialized = common_chat_peg_arena::from_json(json); - - // Test various inputs - std::vector test_inputs = {"abc", "abc5", "xyz!!!", "test9!"}; - - for (const auto& input : test_inputs) { - common_chat_parse_context ctx1(input); - common_chat_parse_context ctx2(input); - - auto result1 = original.parse(ctx1); - auto result2 = deserialized.parse(ctx2); - - t.assert_equal("input_" + input + "_both_succeed", result1.success(), result2.success()); - t.assert_equal("input_" + input + "_same_end", result1.end, result2.end); - } - }); - - // Test serialization with chars parser - t.test("chars parser round-trip", [](testing &t) { - auto original = build_peg_parser([](common_chat_peg_parser_builder & p) { - return p.chars("[a-zA-Z0-9_]", 3, 10); - }); - - // Serialize - auto json = original.to_json(); - - // Deserialize - auto deserialized = common_chat_peg_arena::from_json(json); - - // Test - std::string input = "hello123"; - common_chat_parse_context ctx1(input); - common_chat_parse_context ctx2(input); - - auto result1 = original.parse(ctx1); - auto result2 = deserialized.parse(ctx2); - - t.assert_equal("both_succeed", result1.success(), result2.success()); - t.assert_equal("same_end_pos", result1.end, result2.end); - }); - - // Test serialization with lookahead - t.test("lookahead parser round-trip", [](testing &t) { - auto original = build_peg_parser([](common_chat_peg_parser_builder & p) { - return p.sequence({ - p.peek(p.literal("test")), - p.literal("test") - }); - }); - - // Serialize - auto json = original.to_json(); - - // Deserialize - auto deserialized = common_chat_peg_arena::from_json(json); - - // Test - std::string input = "test"; - common_chat_parse_context ctx1(input); - common_chat_parse_context ctx2(input); - - auto result1 = original.parse(ctx1); - auto result2 = deserialized.parse(ctx2); - - t.assert_equal("both_succeed", result1.success(), result2.success()); - t.assert_equal("same_end_pos", result1.end, result2.end); - }); - - // Benchmark: deserialize JSON parser - t.test("deserialize JSON parser", [&](testing & t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { - return p.json(); - }); - - auto json = parser.to_json(); - auto json_str = json.dump(); - - t.bench("deserialize json", [&]() { - auto deserialized = common_chat_peg_arena::from_json(nlohmann::json::parse(json_str)); - }, 1000); - }); + t.bench("deserialize", [&]() { + auto deserialized = common_chat_peg_arena::from_json(nlohmann::json::parse(json_serialized)); + }, 100); } From 37d5faf4cb69401c054b59e5ee5889e25338976d Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Tue, 18 Nov 2025 02:09:43 -0600 Subject: [PATCH 126/183] gbnf generation fixes --- common/chat-peg-parser.cpp | 43 ++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index d38bdd0b936ae..67aeac08d0e34 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -199,7 +200,6 @@ static std::pair parse_char_class_char(const std::string & con case 'r': return {'\r', 2}; case '\\': return {'\\', 2}; case ']': return {']', 2}; - case '-': return {'-', 2}; case '[': return {'[', 2}; default: return {static_cast(content[pos + 1]), 2}; } @@ -851,6 +851,12 @@ common_chat_peg_parser operator<<(const std::string & str, const common_chat_peg return operator<<(str.c_str(), p); } +// Rule name helper, intended to produce valid GBNF rule names +static std::string rule_name(const std::string & name) { + static const std::regex invalid_rule_chars_re("[^a-zA-Z0-9-]+"); + return std::regex_replace(name, invalid_rule_chars_re, "-"); +} + // Builder implementation common_chat_peg_parser_builder::common_chat_peg_parser_builder() {} @@ -952,7 +958,7 @@ common_chat_peg_parser common_chat_peg_parser_builder::one(const std::string & c } common_chat_peg_parser common_chat_peg_parser_builder::ref(const std::string & name) { - return wrap(arena_.add_parser(common_chat_peg_ref_parser{name})); + return wrap(arena_.add_parser(common_chat_peg_ref_parser{rule_name(name)})); } common_chat_peg_parser common_chat_peg_parser_builder::space() { @@ -1004,9 +1010,10 @@ common_chat_peg_parser common_chat_peg_parser_builder::rule(const std::string & } common_chat_peg_parser common_chat_peg_parser_builder::rule(const std::string & name, const std::string & annotation, common_chat_peg_parser p, bool trigger) { - auto rule_id = arena_.add_parser(common_chat_peg_rule_parser{name, annotation, p.id(), trigger}); - arena_.add_rule(name, rule_id); - return ref(name); + auto clean_name = rule_name(name); + auto rule_id = arena_.add_parser(common_chat_peg_rule_parser{clean_name, annotation, p.id(), trigger}); + arena_.add_rule(clean_name, rule_id); + return ref(clean_name); } common_chat_peg_parser common_chat_peg_parser_builder::rule(const std::string & name, const std::function & builder_fn, bool trigger) { @@ -1014,24 +1021,24 @@ common_chat_peg_parser common_chat_peg_parser_builder::rule(const std::string & } common_chat_peg_parser common_chat_peg_parser_builder::rule(const std::string & name, const std::string & annotation, const std::function & builder_fn, bool trigger) { - // Check if rule already exists - if (arena_.has_rule(name)) { - return ref(name); + auto clean_name = rule_name(name); + if (arena_.has_rule(clean_name)) { + return ref(clean_name); } // Create placeholder rule to allow recursive references auto placeholder = any(); // Temporary placeholder - auto placeholder_rule_id = arena_.add_parser(common_chat_peg_rule_parser{name, annotation, placeholder.id(), trigger}); - arena_.add_rule(name, placeholder_rule_id); + auto placeholder_rule_id = arena_.add_parser(common_chat_peg_rule_parser{clean_name, annotation, placeholder.id(), trigger}); + arena_.add_rule(clean_name, placeholder_rule_id); // Build the actual parser auto parser = builder_fn(); // Replace placeholder with actual rule - auto rule_id = arena_.add_parser(common_chat_peg_rule_parser{name, annotation, parser.id(), trigger}); - arena_.rules_[name] = rule_id; + auto rule_id = arena_.add_parser(common_chat_peg_rule_parser{clean_name, annotation, parser.id(), trigger}); + arena_.rules_[clean_name] = rule_id; - return ref(name); + return ref(clean_name); } void common_chat_peg_parser_builder::set_root(common_chat_peg_parser p) { @@ -1049,7 +1056,7 @@ common_chat_peg_parser common_chat_peg_parser_builder::json_number() { auto digits = chars("[0-9]"); auto int_part = choice({literal("0"), sequence({digit1_9, chars("[0-9]", 0, -1)})}); auto frac = sequence({literal("."), digits}); - auto exp = sequence({choice({literal("e"), literal("E")}), optional(chars("[+\\-]", 1, 1)), digits}); + auto exp = sequence({choice({literal("e"), literal("E")}), optional(chars("[+-]", 1, 1)), digits}); return sequence({optional(literal("-")), int_part, optional(frac), optional(exp)}); }; return rule("json-number", builder); @@ -1139,8 +1146,7 @@ static std::string gbnf_escape_char_class(char c) { case '\r': return "\\r"; case '\\': return "\\\\"; case ']': return "\\]"; - case '-': return "\\-"; - case '^': return "\\^"; + case '[': return "\\["; default: return std::string(1, c); } } @@ -1333,6 +1339,7 @@ void common_chat_peg_arena::build_grammar(const common_grammar_builder & builder return gbnf_excluding_pattern(p.delimiters); } else if constexpr (std::is_same_v) { if (p.schema) { + builder.resolve_refs(*p.schema); return builder.add_schema(p.name, *p.schema); } return to_gbnf(p.child); @@ -1364,7 +1371,7 @@ void common_chat_peg_arena::build_grammar(const common_grammar_builder & builder if (auto rule = std::get_if(&parser)) { auto rule_body = to_gbnf(rule->child); auto canonical_name = builder.add_rule(name, rule_body); - rule_name_mapping[name] = canonical_name; + //rule_name_mapping[name] = canonical_name; } } @@ -1391,7 +1398,7 @@ void common_chat_peg_arena::build_grammar(const common_grammar_builder & builder if (auto rule = std::get_if(&parser)) { auto rule_body = to_gbnf(rule->child); auto canonical_name = builder.add_rule(name, rule_body); - rule_name_mapping[name] = canonical_name; + //rule_name_mapping[name] = canonical_name; } } From e09378a0c87038b0d44f0993369b538c0cb4dfd7 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Tue, 18 Nov 2025 17:57:59 -0600 Subject: [PATCH 127/183] remove LOG_* use in tests --- .../test-example-minimax-m2.cpp | 5 -- .../test-example-qwen3-coder.cpp | 1 - tests/chat-peg-parser/test_harness.h | 58 +++++++++---------- tests/test-chat-peg-parser.cpp | 18 ++---- 4 files changed, 32 insertions(+), 50 deletions(-) diff --git a/tests/chat-peg-parser/test-example-minimax-m2.cpp b/tests/chat-peg-parser/test-example-minimax-m2.cpp index 94794723a9e07..17ec5a94b598a 100644 --- a/tests/chat-peg-parser/test-example-minimax-m2.cpp +++ b/tests/chat-peg-parser/test-example-minimax-m2.cpp @@ -63,10 +63,6 @@ void test_example_minimax_m2(testing &t) { common_chat_parse_semantics semantics; common_chat_parse_context ctx(in, &semantics, it + 1 == tokens.end()); - if (it + 1 == tokens.end()) { - common_log_set_verbosity_thold(LOG_DEFAULT_DEBUG); - } - ctx.event_handler = it + 1 == tokens.end() ? parser_semantic_handler_with_printout : parser_semantic_handler; auto result = helper_parser.parse(ctx); @@ -81,6 +77,5 @@ void test_example_minimax_m2(testing &t) { LOG_ERR("Last message: %s\n", prev.to_json_oaicompat().dump().c_str()); t.assert_true("last_result_should_be_success", last_result.success()); }); - common_log_set_verbosity_thold(LOG_DEFAULT_LLAMA); }); } diff --git a/tests/chat-peg-parser/test-example-qwen3-coder.cpp b/tests/chat-peg-parser/test-example-qwen3-coder.cpp index 8488f1e476b0e..dc125d4963f76 100644 --- a/tests/chat-peg-parser/test-example-qwen3-coder.cpp +++ b/tests/chat-peg-parser/test-example-qwen3-coder.cpp @@ -72,7 +72,6 @@ void test_example_qwen3_coder(testing &t) { ""; std::vector tokens = simple_tokenize(input); - common_log_set_verbosity_thold(LOG_DEFAULT_DEBUG); t.test("explicit_builder", [&](testing &t) { common_chat_msg prev; diff --git a/tests/chat-peg-parser/test_harness.h b/tests/chat-peg-parser/test_harness.h index adc541acf01f4..6e11b17510277 100644 --- a/tests/chat-peg-parser/test_harness.h +++ b/tests/chat-peg-parser/test_harness.h @@ -1,16 +1,15 @@ #pragma once -#include "log.h" #include #include #include -#include #include #include struct testing { std::ostream &out; std::vector stack; + bool throw_exception = false; int tests = 0; int assertions = 0; int failures = 0; @@ -21,7 +20,7 @@ struct testing { void indent() const { for (std::size_t i = 0; i < stack.size() - 1; ++i) { - LOG_ERR(" "); + out << " "; } } @@ -34,29 +33,33 @@ struct testing { ++exceptions; indent(); out << "UNHANDLED EXCEPTION (" << ctx << "): " << e.what() << "\n"; - throw e; + if (throw_exception) { + throw; + } } catch (...) { ++failures; ++exceptions; indent(); out << "UNHANDLED EXCEPTION (" << ctx << "): unknown\n"; - throw; + if (throw_exception) { + throw; + } } } void print_result(const std::string &label, const std::string &name, int new_failures, int new_assertions, const std::string &extra = "") const { indent(); - LOG_ERR("%s: %s [", label.c_str(), name.c_str()); + out << label << ": " << name; if (new_failures == 0) { - LOG_ERR("ok, "); + out << "ok, "; } else { - LOG_ERR("%d failed of ", new_failures); + out << new_failures << " failed of "; } - LOG_ERR("%d assertion(s)", new_assertions); + out << new_assertions << " assertion(s)"; if (!extra.empty()) { - LOG_ERR(", %s", extra.c_str()); + out << ", " << extra; } - LOG_ERR("]\n"); + out << "\n"; } // Named test @@ -66,7 +69,7 @@ struct testing { stack.push_back(name); indent(); - LOG_ERR("BEGIN: %s\n", name.c_str()); + out << "BEGIN: " << name << "\n"; int before_failures = failures; int before_assertions = assertions; @@ -161,16 +164,16 @@ struct testing { if (!(actual == expected)) { ++failures; indent(); - LOG_ERR("ASSERT EQUAL FAILED"); + out << "ASSERT EQUAL FAILED"; if (!msg.empty()) { - LOG_ERR(" : %s", msg.c_str()); + out << " : " << msg; } - LOG_ERR("\n"); + out << "\n"; indent(); - LOG_ERR(" expected: %s\n", to_string_convert(expected).c_str()); + out << " expected: " << expected << "\n"; indent(); - LOG_ERR(" actual : %s\n", to_string_convert(actual).c_str()); + out << " actual : " << actual << "\n"; return false; } return true; @@ -178,21 +181,12 @@ struct testing { // Print summary and return an exit code int summary() const { - LOG_ERR("\n==== TEST SUMMARY ====\n"); - LOG_ERR("tests : %d\n", tests); - LOG_ERR("assertions : %d\n", assertions); - LOG_ERR("failures : %d\n", failures); - LOG_ERR("exceptions : %d\n", exceptions); - LOG_ERR("======================\n"); + out << "\n==== TEST SUMMARY ====\n"; + out << "tests : " << tests << "\n"; + out << "assertions : " << assertions << "\n"; + out << "failures : " << failures << "\n"; + out << "exceptions : " << exceptions << "\n"; + out << "======================\n"; return failures == 0 ? 0 : 1; } - -private: - template - std::string to_string_convert(const T & value) const { - std::ostringstream oss; - oss << value; - return oss.str(); - } - }; diff --git a/tests/test-chat-peg-parser.cpp b/tests/test-chat-peg-parser.cpp index ca5c14ba5eeb0..27d880d7a6fd9 100644 --- a/tests/test-chat-peg-parser.cpp +++ b/tests/test-chat-peg-parser.cpp @@ -30,20 +30,14 @@ static const std::vector all_tests = { // Function to list all available tests static void list_available_tests() { - LOG_ERR("Available tests:\n"); + std::cout << "Available tests:\n"; for (const auto& test : all_tests) { - LOG_ERR(" %s", test.codename.c_str()); - // Format spacing for alignment - for (size_t i = test.codename.length(); i < 25; ++i) { - LOG_ERR(" "); - } - LOG_ERR("- %s\n", test.function_name.c_str()); + std::cout << std::left << std::setw(25) << test.codename << "- " << test.function_name << "\n"; } - LOG_ERR("\n"); - LOG_ERR("Usage:\n"); - LOG_ERR(" test-chat-peg-parser # Run all tests\n"); - LOG_ERR(" test-chat-peg-parser test1 test2 # Run specific tests\n"); - LOG_ERR(" test-chat-peg-parser --tests # List available tests\n"); + std::cout << "\nUsage:\n"; + std::cout << " test-chat-peg-parser # Run all tests\n"; + std::cout << " test-chat-peg-parser test1 test2 # Run specific tests\n"; + std::cout << " test-chat-peg-parser --tests # List available tests\n"; } // Function to check if a codename matches the provided arguments From 92d38eaa94ff8431428db6832456aa1bdd4d5919 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Tue, 18 Nov 2025 18:24:08 -0600 Subject: [PATCH 128/183] update gbnf tests to test entire grammar --- .../chat-peg-parser/test-gbnf-generation.cpp | 193 ++++++++++++------ 1 file changed, 131 insertions(+), 62 deletions(-) diff --git a/tests/chat-peg-parser/test-gbnf-generation.cpp b/tests/chat-peg-parser/test-gbnf-generation.cpp index 26433c5aac0b3..e444d28b5a482 100644 --- a/tests/chat-peg-parser/test-gbnf-generation.cpp +++ b/tests/chat-peg-parser/test-gbnf-generation.cpp @@ -1,129 +1,198 @@ -#include "json-schema-to-grammar.h" #include "tests.h" +#include "json-schema-to-grammar.h" + +#include + +static std::string trim_leading_space(const std::string & s) { + static const std::regex leading_ws_re = std::regex(R"((^|\n)\s+)"); + return std::regex_replace(s, leading_ws_re, "$1"); +} + +static void assert_gbnf_equal(testing & t, const std::string & expected, const std::string & actual) { + t.assert_equal("gbnf are equal", trim_leading_space(expected), trim_leading_space(actual)); +} + void test_gbnf_generation(testing &t) { - // Test literal t.test("literal grammar generation", [](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello"); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + return p.literal("hello"); + }); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); - t.assert_equal("has_root_hello", true, gbnf.find("root ::= \"hello\"") != std::string::npos); - t.assert_equal("has_space", true, gbnf.find("space ::=") != std::string::npos); + assert_gbnf_equal(t, R"""( + root ::= "hello" + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""", gbnf); }); - // Test char class t.test("char class grammar", [](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[a-z]"); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + return p.one("[a-z]"); + }); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); - t.assert_equal("has_char_class", true, gbnf.find("root ::= [a-z]") != std::string::npos); + assert_gbnf_equal(t, R"""( + root ::= [a-z] + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""", gbnf); }); - // Test sequence t.test("sequence grammar", [](testing &t) { - auto parser = build_peg_parser( - [](common_chat_peg_parser_builder & p) { return p.literal("hello") + p.literal(" ") + p.literal("world"); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + return p.literal("hello") + p.literal(" ") + p.literal("world"); + }); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); - t.assert_equal("has_proper_sequence", true, - gbnf.find("root ::= \"hello\" \" \" \"world\"") != std::string::npos); + assert_gbnf_equal(t, R"""( + root ::= "hello" " " "world" + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""", gbnf); }); - // Test choice t.test("choice grammar", [](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("cat") | p.literal("dog"); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + return p.literal("cat") | p.literal("dog"); + }); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); - t.assert_equal("has_proper_choice", true, gbnf.find("root ::= \"cat\" | \"dog\"") != std::string::npos); + assert_gbnf_equal(t, R"""( + root ::= "cat" | "dog" + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""", gbnf); }); - // Test one_or_more t.test("one_or_more grammar", [](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one_or_more(p.one("[0-9]")); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + return p.one_or_more(p.one("[0-9]")); + }); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); - t.assert_equal("has_proper_one_or_more", true, gbnf.find("root ::= [0-9]+") != std::string::npos); + assert_gbnf_equal(t, R"""( + root ::= [0-9]+ + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""", gbnf); }); - // Test zero_or_more t.test("zero_or_more grammar", [](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.zero_or_more(p.one("[a-z]")); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + return p.zero_or_more(p.one("[a-z]")); + }); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); - t.assert_equal("has_proper_zero_or_more", true, gbnf.find("root ::= [a-z]*") != std::string::npos); + assert_gbnf_equal(t, R"""( + root ::= [a-z]* + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""", gbnf); }); - // Test optional t.test("optional grammar", [](testing &t) { - auto parser = - build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + return p.literal("hello") + p.optional(p.literal(" world")); + }); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); - t.assert_equal("has_proper_optional", true, - gbnf.find("root ::= \"hello\" \" world\"?") != std::string::npos); + assert_gbnf_equal(t, R"""( + root ::= "hello" " world"? + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""", gbnf); }); - // Test until t.test("until grammar", [](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.until(""); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + return p.until(""); + }); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); - // Should generate pattern that prevents matching the full delimiter - t.assert_equal("has_proper_until", true, - gbnf.find( - "root ::= ([^<] | \"<\" [^/] | \"])*") != - std::string::npos); + assert_gbnf_equal(t, R"""( + root ::= ([^<] | "<" [^/] | "])* + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""", gbnf); }); - // Test complex expression with parentheses t.test("complex expressions with parentheses", [](testing &t) { - auto parser = - build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one_or_more(p.literal("a") | p.literal("b")); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + return p.one_or_more(p.literal("a") | p.literal("b")); + }); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); - t.assert_equal("has_proper_complex", true, gbnf.find("root ::= (\"a\" | \"b\")+") != std::string::npos); + assert_gbnf_equal(t, R"""( + root ::= ("a" | "b")+ + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""", gbnf); }); - // Test rule references t.test("rule references", [](testing &t) { auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { auto digit = p.rule("digit", p.one("[0-9]")); return p.one_or_more(digit); }); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); - // Should have digit rule defined and referenced - t.assert_equal("has_digit_rule", true, gbnf.find("digit ::= [0-9]") != std::string::npos); - t.assert_equal("has_root_digit_ref", true, gbnf.find("root ::= digit+") != std::string::npos); + assert_gbnf_equal(t, R"""( + digit ::= [0-9] + root ::= digit+ + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""", gbnf); }); - // Test escaping in literals t.test("escaping in literals", [](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello\nworld\t!"); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + return p.literal("hello\nworld\t!"); + }); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); - t.assert_equal("has_escaping", true, gbnf.find("root ::= \"hello\\nworld\\t!\"") != std::string::npos); + assert_gbnf_equal(t, R"""( + root ::= "hello\nworld\t!" + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""", gbnf); }); - // Test operator<< (whitespace insertion) t.test("operator<< (whitespace insertion)", [](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello") << p.literal("world"); }); + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + return p.literal("hello") << p.literal("world"); + }); - auto gbnf = build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); - // Should inline the whitespace pattern - t.assert_equal("has_inlined_hello", true, gbnf.find("\"hello\"") != std::string::npos); - t.assert_equal("has_inlined_world", true, gbnf.find("\"world\"") != std::string::npos); + assert_gbnf_equal(t, R"""( + root ::= "hello" space "world" + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""", gbnf); }); } From 0e1989e34f11febe582845f57e82a33c4ce27201 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Tue, 18 Nov 2025 19:08:57 -0600 Subject: [PATCH 129/183] clean up gbnf generation and fix a few bugs --- common/chat-peg-parser.cpp | 104 +++++++----------- .../chat-peg-parser/test-gbnf-generation.cpp | 52 +++++++++ 2 files changed, 92 insertions(+), 64 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 67aeac08d0e34..29261e65a4c60 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -1180,10 +1180,10 @@ static std::string gbnf_excluding_pattern(const std::vector & strin return "(" + pattern + ")*"; } -// Collect all rule parsers reachable from triggers (for lazy mode) +// Collect reachable rules from a given rule static std::unordered_set collect_reachable_rules( const common_chat_peg_arena & arena, - const std::unordered_map & all_rules + const common_chat_peg_parser_id & rule ) { std::unordered_set reachable; std::unordered_set visited; @@ -1215,38 +1215,19 @@ static std::unordered_set collect_reachable_rules( visit(p.child); } } else if constexpr (std::is_same_v) { - reachable.insert(p.name); + // Traverse rules so we pick up everything + auto referenced_rule = arena.get_rule(p.name); + visit(referenced_rule); } }, parser); }; - // Find trigger rules and traverse from them - for (const auto & [name, rule_id] : all_rules) { - const auto & parser = arena.get(rule_id); - if (auto rule = std::get_if(&parser)) { - if (rule->trigger) { - visit(rule_id); - } - } - } - + visit(rule); return reachable; } // GBNF generation implementation void common_chat_peg_arena::build_grammar(const common_grammar_builder & builder, bool lazy) const { - std::unordered_map rule_name_mapping; - std::unordered_set reachable_rules; - - // Collect all rules - if (lazy) { - reachable_rules = collect_reachable_rules(*this, rules_); - if (reachable_rules.empty()) { - LOG_ERR("Lazy grammar generation enabled but no trigger rules found\n"); - return; - } - } - // Generate GBNF for a parser std::function to_gbnf = [&](common_chat_peg_parser_id id) -> std::string { const auto & parser = parsers_.at(id); @@ -1339,17 +1320,12 @@ void common_chat_peg_arena::build_grammar(const common_grammar_builder & builder return gbnf_excluding_pattern(p.delimiters); } else if constexpr (std::is_same_v) { if (p.schema) { - builder.resolve_refs(*p.schema); return builder.add_schema(p.name, *p.schema); } return to_gbnf(p.child); } else if constexpr (std::is_same_v) { return to_gbnf(p.child); } else if constexpr (std::is_same_v) { - auto it = rule_name_mapping.find(p.name); - if (it != rule_name_mapping.end()) { - return it->second; - } return p.name; } else if constexpr (std::is_same_v) { return to_gbnf(p.child); @@ -1359,54 +1335,54 @@ void common_chat_peg_arena::build_grammar(const common_grammar_builder & builder }, parser); }; - // Generate rules - if (lazy) { - // Lazy mode: only generate reachable rules - for (const auto & [name, rule_id] : rules_) { - if (reachable_rules.find(name) == reachable_rules.end()) { - continue; - } + // Collect reachable rules + std::unordered_set reachable_rules; - const auto & parser = parsers_.at(rule_id); + if (lazy) { + // Collect rules reachable from trigger rules + for (const auto & [name, id] : rules_) { + const auto & parser = parsers_.at(id); if (auto rule = std::get_if(&parser)) { - auto rule_body = to_gbnf(rule->child); - auto canonical_name = builder.add_rule(name, rule_body); - //rule_name_mapping[name] = canonical_name; + if (rule->trigger) { + // Mark trigger as reachable and visit it + reachable_rules.insert(name); + auto add_rules = collect_reachable_rules(*this, id); + reachable_rules.insert(add_rules.begin(), add_rules.end()); + } } } + } else { + // Collect rules reachable from root + reachable_rules = collect_reachable_rules(*this, root_); + } + + // Create GBNF rules for all reachable rules + for (const auto & [name, rule_id] : rules_) { + if (reachable_rules.find(name) == reachable_rules.end()) { + continue; + } - // Generate root as alternation of trigger rule names + const auto & parser = parsers_.at(rule_id); + if (auto rule = std::get_if(&parser)) { + builder.add_rule(rule->name, to_gbnf(rule->child)); + } + } + + if (lazy) { + // Generate root rule from trigger rules only std::vector trigger_names; for (const auto & [name, rule_id] : rules_) { const auto & parser = parsers_.at(rule_id); if (auto rule = std::get_if(&parser)) { if (rule->trigger) { - auto it = rule_name_mapping.find(name); - if (it != rule_name_mapping.end()) { - trigger_names.push_back(it->second); - } + trigger_names.push_back(rule->name); } } } - if (!trigger_names.empty()) { - builder.add_rule("root", string_join(trigger_names, " | ")); - } - } else { - // Non-lazy mode: generate all rules - for (const auto & [name, rule_id] : rules_) { - const auto & parser = parsers_.at(rule_id); - if (auto rule = std::get_if(&parser)) { - auto rule_body = to_gbnf(rule->child); - auto canonical_name = builder.add_rule(name, rule_body); - //rule_name_mapping[name] = canonical_name; - } - } - // Generate root - if (root_ != COMMON_CHAT_PEG_INVALID_PARSER_ID) { - auto root_body = to_gbnf(root_); - builder.add_rule("root", root_body); - } + builder.add_rule("root", string_join(trigger_names, " | ")); + } else if (root_ != COMMON_CHAT_PEG_INVALID_PARSER_ID) { + builder.add_rule("root", to_gbnf(root_)); } } diff --git a/tests/chat-peg-parser/test-gbnf-generation.cpp b/tests/chat-peg-parser/test-gbnf-generation.cpp index e444d28b5a482..655738aecb795 100644 --- a/tests/chat-peg-parser/test-gbnf-generation.cpp +++ b/tests/chat-peg-parser/test-gbnf-generation.cpp @@ -195,4 +195,56 @@ void test_gbnf_generation(testing &t) { space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); + + t.test("emit only reachable rules", [](testing &t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + p.rule("orphan", p.literal("orphan")); + return p.literal("hello") + p.rule("child", p.literal(" world")); + }); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); + + assert_gbnf_equal(t, R"""( + child ::= " world" + root ::= "hello" child + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""", gbnf); + }); + + t.test("emit only trigger rules (and references)", [](testing &t) { + auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto rule1 = p.rule("rule-1", p.literal("a") + p.ref("rule-2")); + p.rule("rule-2", p.literal("b") + p.ref("rule-3"), true); + p.rule("rule-3", p.literal("c") + p.ref("rule-4")); + p.rule("rule-4", p.literal("d"), true); + return rule1; + }); + + auto gbnf = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); + + assert_gbnf_equal(t, R"""( + root ::= rule-1 + rule-1 ::= "a" rule-2 + rule-2 ::= "b" rule-3 + rule-3 ::= "c" rule-4 + rule-4 ::= "d" + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""", gbnf); + + auto gbnf_lazy = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder, true); + }); + + assert_gbnf_equal(t, R"""( + root ::= rule-2 | rule-4 + rule-2 ::= "b" rule-3 + rule-3 ::= "c" rule-4 + rule-4 ::= "d" + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""", gbnf_lazy); + }); } From ce15c640596dfe66b08e66d799b43ab15a3b1253 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Tue, 18 Nov 2025 19:20:27 -0600 Subject: [PATCH 130/183] fix typo in test output --- tests/chat-peg-parser/test_harness.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/chat-peg-parser/test_harness.h b/tests/chat-peg-parser/test_harness.h index 6e11b17510277..da002a8e2938d 100644 --- a/tests/chat-peg-parser/test_harness.h +++ b/tests/chat-peg-parser/test_harness.h @@ -51,7 +51,7 @@ struct testing { indent(); out << label << ": " << name; if (new_failures == 0) { - out << "ok, "; + out << " ok, "; } else { out << new_failures << " failed of "; } From 3cd2af458b5b944f9d9f480d81906052224b1c14 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Tue, 18 Nov 2025 19:28:41 -0600 Subject: [PATCH 131/183] remove implicit conversion rules --- common/chat-peg-parser.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index 779d3884cdea9..02647cf088326 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -350,12 +350,6 @@ class common_chat_peg_parser_builder { // S -> "hello" common_chat_peg_parser literal(const std::string & literal); - // Implicit conversion: const char* -> parser (literal) - common_chat_peg_parser operator()(const char * str) { return literal(str); } - - // Implicit conversion: std::string -> parser (literal) - common_chat_peg_parser operator()(const std::string & str) { return literal(str); } - // Matches a sequence of parsers in order, all must succeed. // S -> A B C common_chat_peg_parser sequence(const std::vector & parsers); From fe092360e3857956df200673673b1d7f31fed97e Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Tue, 18 Nov 2025 20:59:34 -0600 Subject: [PATCH 132/183] improve test output --- common/chat-peg-parser-helper.cpp | 67 ++++++++++ common/chat-peg-parser-helper.h | 92 +------------- common/chat-peg-parser.h | 7 ++ .../test-command7-parser-compare.cpp | 18 +-- .../test-example-minimax-m2.cpp | 30 +---- .../test-example-qwen3-coder.cpp | 10 +- .../chat-peg-parser/test-example-seed-oss.cpp | 3 +- tests/chat-peg-parser/test_harness.h | 115 ++++++++++-------- 8 files changed, 162 insertions(+), 180 deletions(-) diff --git a/common/chat-peg-parser-helper.cpp b/common/chat-peg-parser-helper.cpp index 38838cced575b..eec52199c54c4 100644 --- a/common/chat-peg-parser-helper.cpp +++ b/common/chat-peg-parser-helper.cpp @@ -1,5 +1,6 @@ #include "chat-peg-parser-helper.h" #include "chat-peg-parser.h" +#include common_chat_peg_parser common_chat_peg_parser_builder_helper::reasoning(const std::string & tag) { std::string open_tag; @@ -151,3 +152,69 @@ common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_attr( return function; } +void common_chat_parse_simple_handler::operator()(const common_chat_parse_event & ev, common_chat_parse_semantics & semantics) const { + if (log) { + std::stringstream ss; + ss << "Event: type=" << (ev.type == COMMON_CHAT_PARSE_EVENT_NODE_START ? "start" : "end "); + ss << " rule=" << ev.rule; + ss << " result=" << common_chat_parse_result_type_name(ev.status); + ss << " text=" << ev.text; + log(ss.str()); + } + + if (ev.rule == "reasoning-content" && ev.ending()) { + semantics.reasoning_content = ev.text; + if (log) { + log(" reasoning_content=" + semantics.reasoning_content); + } + } + + if (ev.rule == "content" && ev.ending()) { + semantics.content = ev.text; + if (log) { + log(" content=" + semantics.content); + } + } + + if (ev.rule.find("function-start") != std::string::npos && ev.ending() && ev.success()) { + semantics.tool_calls.emplace_back(); + auto & tc = semantics.tool_calls.back(); + tc.name = semantics.captures["tool-name"]; + if (log) { + log(" tool call added"); + log(" name=" + tc.name); + } + } + + if (ev.rule.find("arg-start") != std::string::npos && ev.ending() && ev.success()) { + auto & tc = semantics.tool_calls.back(); + auto name = semantics.captures["arg-name"]; + if (tc.arguments.empty()) { + tc.arguments += "{"; + } else { + tc.arguments += ", "; + } + tc.arguments += "\"" + name + "\": "; + } + + if (ev.rule == "arg-string-content" && ev.ending() && ev.success()) { + auto & tc = semantics.tool_calls.back(); + tc.arguments += "\"" + std::string(ev.text); + } + + if (ev.annotation == "arg-string" && ev.ending() && ev.success()) { + auto & tc = semantics.tool_calls.back(); + tc.arguments += "\""; + if (log) { + log(" args=" + tc.arguments); + } + } + + if (ev.rule == "arg-json-content" && ev.ending() && (ev.success() || ev.need_more_input())) { + auto & tc = semantics.tool_calls.back(); + tc.arguments += std::string(ev.text); + if (log) { + log(" args=" + tc.arguments); + } + } +} diff --git a/common/chat-peg-parser-helper.h b/common/chat-peg-parser-helper.h index 5062896702964..71157ea426630 100644 --- a/common/chat-peg-parser-helper.h +++ b/common/chat-peg-parser-helper.h @@ -32,91 +32,7 @@ common_chat_peg_arena build_peg_parser_helper(F && fn) { return builder.build(); } -inline void parser_semantic_handler(const common_chat_parse_event & ev, common_chat_parse_semantics & semantics) { - if (ev.rule == "reasoning-content" && ev.ending()) { - semantics.reasoning_content = ev.text; - } - - if (ev.rule == "content" && ev.ending()) { - semantics.content = ev.text; - } - - if (ev.rule.find("function-start") != std::string::npos && ev.ending() && ev.success()) { - semantics.tool_calls.emplace_back(); - auto & tc = semantics.tool_calls.back(); - tc.name = semantics.captures["tool-name"]; - } - - if (ev.rule.find("arg-start") != std::string::npos && ev.ending() && ev.success()) { - auto & tc = semantics.tool_calls.back(); - auto name = semantics.captures["arg-name"]; - if (tc.arguments.empty()) { - tc.arguments += "{"; - } else { - tc.arguments += ", "; - } - tc.arguments += "\"" + name + "\": "; - } - - if (ev.rule == "arg-string-content" && ev.ending() && ev.success()) { - auto & tc = semantics.tool_calls.back(); - tc.arguments += "\"" + std::string(ev.text); - } - - if (ev.annotation == "arg-string" && ev.ending() && ev.success()) { - auto & tc = semantics.tool_calls.back(); - tc.arguments += "\""; - } - - if (ev.rule == "arg-json-content" && ev.ending() && (ev.success() || ev.need_more_input())) { - auto & tc = semantics.tool_calls.back(); - tc.arguments += std::string(ev.text); - } -} - -inline void parser_semantic_handler_with_printout(const common_chat_parse_event & ev, common_chat_parse_semantics & semantics) { - LOG_ERR("\n===============\nEvent type: %s\n", (ev.type == COMMON_CHAT_PARSE_EVENT_NODE_START ? "START" : "END")); - LOG_ERR("Event rule: %s\nEvent text: %s\nEvent status: %s\n", ev.rule.c_str(), std::string(ev.text.data(), ev.text.size()).c_str(), (ev.status == COMMON_CHAT_PARSE_RESULT_SUCCESS ? "SUCCESS" : (ev.status == COMMON_CHAT_PARSE_RESULT_FAIL ? "FAIL" : "NEED_MORE_INPUT"))); - - if (ev.rule == "reasoning-content" && ev.ending()) { - semantics.reasoning_content = ev.text; - } - - if (ev.rule == "content" && ev.ending()) { - semantics.content = ev.text; - } - - if (ev.rule.find("function-start") != std::string::npos && ev.ending() && ev.success()) { - semantics.tool_calls.emplace_back(); - auto & tc = semantics.tool_calls.back(); - tc.name = semantics.captures["tool-name"]; - } - - if (ev.rule.find("arg-start") != std::string::npos && ev.ending() && ev.success()) { - auto & tc = semantics.tool_calls.back(); - auto name = semantics.captures["arg-name"]; - if (tc.arguments.empty()) { - tc.arguments += "{"; - } else { - tc.arguments += ", "; - } - tc.arguments += "\"" + name + "\": "; - } - - if (ev.rule == "arg-string-content" && ev.ending() && ev.success()) { - auto & tc = semantics.tool_calls.back(); - tc.arguments += "\"" + std::string(ev.text); - } - - if (ev.annotation == "arg-string" && ev.ending() && ev.success()) { - auto & tc = semantics.tool_calls.back(); - tc.arguments += "\""; - } - - if (ev.rule == "arg-json-content" && ev.ending() && (ev.success() || ev.need_more_input())) { - auto & tc = semantics.tool_calls.back(); - tc.arguments += std::string(ev.text); - } - - LOG_ERR("Content: %s\nReasoning: %s\nTool calls: %lu\n", semantics.content.c_str(), semantics.reasoning_content.c_str(), semantics.tool_calls.size()); -} +struct common_chat_parse_simple_handler { + std::function log; + void operator()(const common_chat_parse_event & ev, common_chat_parse_semantics & semantics) const; +}; diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index 02647cf088326..165b02fcbc697 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -180,6 +180,13 @@ struct common_chat_parse_context { common_chat_parse_context(const std::string & input, common_chat_parse_semantics * semantics, common_chat_parse_event_handler handler, bool complete = true) : input(input), input_is_complete(complete), cache(), semantics(semantics), event_handler(std::move(handler)), current_depth(0), parse_depth(0) {} + + template + void set_event_handler(const T & handler) { + event_handler = [&](const common_chat_parse_event & ev, common_chat_parse_semantics & semantics) { + handler(ev, semantics); + }; + } }; // Forward declaration diff --git a/tests/chat-peg-parser/test-command7-parser-compare.cpp b/tests/chat-peg-parser/test-command7-parser-compare.cpp index 00f9bf0198e32..078f5cb156e88 100644 --- a/tests/chat-peg-parser/test-command7-parser-compare.cpp +++ b/tests/chat-peg-parser/test-command7-parser-compare.cpp @@ -238,25 +238,11 @@ void test_command7_parser_compare(testing &t) { // Run tests t.test("legacy_parse", [&](testing & t) { - bool no_error = true; - try { - test_command_r7b_legacy_parser(input, false, false); - } catch (std::exception &e) { - no_error = false; - std::cerr << "Error during legacy run: " << e.what() << "\n"; - } - t.assert_equal("no_errors", true, no_error); + test_command_r7b_legacy_parser(input, false, false); }); t.test("current_parse", [&](testing & t) { - bool no_error = true; - try { - test_command_r7b_parser(parser, input, false, false); - } catch (std::exception &e) { - no_error = false; - std::cerr << "Error during current run: " << e.what() << "\n"; - } - t.assert_equal("no_errors", true, no_error); + test_command_r7b_parser(parser, input, false, false); }); // Run benchmarks diff --git a/tests/chat-peg-parser/test-example-minimax-m2.cpp b/tests/chat-peg-parser/test-example-minimax-m2.cpp index 17ec5a94b598a..f254feb392587 100644 --- a/tests/chat-peg-parser/test-example-minimax-m2.cpp +++ b/tests/chat-peg-parser/test-example-minimax-m2.cpp @@ -1,30 +1,11 @@ +#include "common.h" #include "chat-peg-parser.h" -#include "log.h" #include "nlohmann/json.hpp" #include "tests.h" #include #include -static inline std::string join(const std::vector& parts, - const std::string& sep = ", ") { - if (parts.empty()) { return {}; } - - // Reserve an approximate size to avoid many reallocations. - std::size_t total_len = sep.size() * (parts.size() - 1); - for (const auto& s : parts) { total_len += s.size(); } - - std::string result; - result.reserve(total_len); - result += parts[0]; - - for (std::size_t i = 1; i < parts.size(); ++i) { - result += sep; - result += parts[i]; - } - return result; -} - void test_example_minimax_m2(testing &t) { auto helper_parser = build_peg_parser_helper([](common_chat_peg_parser_builder_helper & p) { auto thinking = p.reasoning(); @@ -51,19 +32,20 @@ void test_example_minimax_m2(testing &t) { ""; std::vector tokens = simple_tokenize(input); - LOG_ERR("Tokens: %s\n", join(tokens).c_str()); + // t.log("Tokens: " + string_join(tokens, ", ")); common_chat_msg prev; common_chat_parse_result last_result; t.test("helper_builder", [&](testing &t) { for (auto it = tokens.begin(); it != tokens.end(); it++) { std::string in = std::accumulate(tokens.begin(), it + 1, std::string()); - LOG_ERR("Current input: %s\n", in.c_str()); + // t.log("Current input: " + in); common_chat_parse_semantics semantics; common_chat_parse_context ctx(in, &semantics, it + 1 == tokens.end()); - ctx.event_handler = it + 1 == tokens.end() ? parser_semantic_handler_with_printout : parser_semantic_handler; + common_chat_parse_simple_handler handler; + ctx.set_event_handler(handler); auto result = helper_parser.parse(ctx); last_result = result; @@ -74,7 +56,7 @@ void test_example_minimax_m2(testing &t) { auto diffs = common_chat_msg_diff::compute_diffs(prev, msg); prev = msg; } - LOG_ERR("Last message: %s\n", prev.to_json_oaicompat().dump().c_str()); + // t.log("Last message: " + prev.to_json_oaicompat().dump()); t.assert_true("last_result_should_be_success", last_result.success()); }); }); diff --git a/tests/chat-peg-parser/test-example-qwen3-coder.cpp b/tests/chat-peg-parser/test-example-qwen3-coder.cpp index dc125d4963f76..8fc3e73fa5177 100644 --- a/tests/chat-peg-parser/test-example-qwen3-coder.cpp +++ b/tests/chat-peg-parser/test-example-qwen3-coder.cpp @@ -81,7 +81,12 @@ void test_example_qwen3_coder(testing &t) { common_chat_parse_semantics semantics; common_chat_parse_context ctx(in, &semantics, it == tokens.end() - 1); - ctx.event_handler = parser_semantic_handler; + common_chat_parse_simple_handler handler; + // handler.log = [&](const std::string & msg) { + // t.log(msg); + // }; + + ctx.set_event_handler(handler); auto result = explicit_parser.parse(ctx); if (!t.assert_equal("not fail", false, result.fail())) { @@ -110,7 +115,8 @@ void test_example_qwen3_coder(testing &t) { common_chat_parse_semantics semantics; common_chat_parse_context ctx(in, &semantics, it + 1 == tokens.end()); - ctx.event_handler = parser_semantic_handler; + common_chat_parse_simple_handler handler; + ctx.set_event_handler(handler); auto result = helper_parser.parse(ctx); if (!t.assert_equal("not fail", false, result.fail())) { diff --git a/tests/chat-peg-parser/test-example-seed-oss.cpp b/tests/chat-peg-parser/test-example-seed-oss.cpp index a9b2a3fdbbfa4..709c1e8b4c15d 100644 --- a/tests/chat-peg-parser/test-example-seed-oss.cpp +++ b/tests/chat-peg-parser/test-example-seed-oss.cpp @@ -37,7 +37,8 @@ void test_example_seed_oss(testing &t) { common_chat_parse_semantics semantics; common_chat_parse_context ctx(in, &semantics, it == tokens.end()); - ctx.event_handler = parser_semantic_handler; + common_chat_parse_simple_handler handler; + ctx.set_event_handler(handler); auto result = helper_parser.parse(ctx); t.assert_equal("not fail", false, result.fail()); diff --git a/tests/chat-peg-parser/test_harness.h b/tests/chat-peg-parser/test_harness.h index da002a8e2938d..e65819b2edea3 100644 --- a/tests/chat-peg-parser/test_harness.h +++ b/tests/chat-peg-parser/test_harness.h @@ -16,12 +16,19 @@ struct testing { int unnamed = 0; int exceptions = 0; + static constexpr std::size_t status_column = 80; + explicit testing(std::ostream &os = std::cout) : out(os) {} - void indent() const { - for (std::size_t i = 0; i < stack.size() - 1; ++i) { - out << " "; + std::string indent() const { + if (stack.empty()) { + return ""; } + return std::string((stack.size() - 1) * 2, ' '); + } + + void log(const std::string & msg) { + out << indent() << " " << msg << "\n"; } template @@ -31,74 +38,87 @@ struct testing { } catch (const std::exception &e) { ++failures; ++exceptions; - indent(); - out << "UNHANDLED EXCEPTION (" << ctx << "): " << e.what() << "\n"; + out << indent() << "UNHANDLED EXCEPTION (" << ctx << "): " << e.what() << "\n"; if (throw_exception) { throw; } } catch (...) { ++failures; ++exceptions; - indent(); - out << "UNHANDLED EXCEPTION (" << ctx << "): unknown\n"; + out << indent() << "UNHANDLED EXCEPTION (" << ctx << "): unknown\n"; if (throw_exception) { throw; } } } - void print_result(const std::string &label, const std::string &name, int new_failures, int new_assertions, const std::string &extra = "") const { - indent(); - out << label << ": " << name; - if (new_failures == 0) { - out << " ok, "; - } else { - out << new_failures << " failed of "; + void print_result(const std::string &label, int new_failures, int new_assertions, const std::string &extra = "") const { + std::string line = indent() + label; + + std::string details; + if (new_assertions > 0) { + if (new_failures == 0) { + details = std::to_string(new_assertions) + " assertion(s)"; + } else { + details = std::to_string(new_failures) + " of " + + std::to_string(new_assertions) + " assertion(s) failed"; + } } - out << new_assertions << " assertion(s)"; if (!extra.empty()) { - out << ", " << extra; + if (!details.empty()) { + details += ", "; + } + details += extra; } - out << "\n"; + + if (!details.empty()) { + line += " (" + details + ")"; + } + + std::string status = (new_failures == 0) ? "[PASS]" : "[FAIL]"; + + if (line.size() + 1 < status_column) { + line.append(status_column - line.size(), ' '); + } else { + line.push_back(' '); + } + + out << line << status << "\n"; } - // Named test template void test(const std::string &name, F f) { ++tests; stack.push_back(name); - indent(); - out << "BEGIN: " << name << "\n"; + out << indent() << name << "\n"; - int before_failures = failures; + int before_failures = failures; int before_assertions = assertions; run_with_exceptions([&] { f(*this); }, "test"); - print_result("END", name, - failures - before_failures, - assertions - before_assertions); + int new_failures = failures - before_failures; + int new_assertions = assertions - before_assertions; + + print_result(name, new_failures, new_assertions); stack.pop_back(); } - // Unnamed test template void test(F f) { test("test #" + std::to_string(++unnamed), f); } - // Named benchmark template void bench(const std::string &name, F f, int iterations = 100) { ++tests; stack.push_back(name); - indent(); - out << "BEGIN BENCH: " << name << "\n"; + out << indent() << "[bench] " << name << "\n"; - int before_failures = failures; + int before_failures = failures; int before_assertions = assertions; using clock = std::chrono::high_resolution_clock; @@ -113,21 +133,23 @@ struct testing { } }, "bench"); - auto avg_elapsed = duration.count() / iterations; + auto avg_elapsed = duration.count() / iterations; auto avg_elapsed_s = std::chrono::duration_cast>(duration).count() / iterations; auto rate = (avg_elapsed_s > 0.0) ? (1.0 / avg_elapsed_s) : 0.0; - print_result("END BENCH", name, - failures - before_failures, - assertions - before_assertions, - std::to_string(iterations) + " iteration(s), " + - "avg elapsed " + std::to_string(avg_elapsed) + - " us (" + std::to_string(rate) + " /s)"); + int new_failures = failures - before_failures; + int new_assertions = assertions - before_assertions; + + std::string extra = + "n=" + std::to_string(iterations) + + " avg=" + std::to_string(avg_elapsed) + "us" + + " rate=" + std::to_string(int(rate)) + "/s"; + + print_result("[bench] " + name, new_failures, new_assertions, extra); stack.pop_back(); } - // Unnamed benchmark template void bench(F f, int iterations = 100) { bench("bench #" + std::to_string(++unnamed), f, iterations); @@ -142,8 +164,7 @@ struct testing { ++assertions; if (!cond) { ++failures; - indent(); - out << "ASSERT TRUE FAILED"; + out << indent() << "ASSERT TRUE FAILED"; if (!msg.empty()) { out << " : " << msg; } @@ -154,26 +175,23 @@ struct testing { } template - bool assert_equal(const A & expected, const B & actual) { + bool assert_equal(const A &expected, const B &actual) { return assert_equal("", expected, actual); } template - bool assert_equal(const std::string & msg, const A & expected, const B & actual) { + bool assert_equal(const std::string &msg, const A &expected, const B &actual) { ++assertions; if (!(actual == expected)) { ++failures; - indent(); - out << "ASSERT EQUAL FAILED"; + out << indent() << "ASSERT EQUAL FAILED"; if (!msg.empty()) { out << " : " << msg; } out << "\n"; - indent(); - out << " expected: " << expected << "\n"; - indent(); - out << " actual : " << actual << "\n"; + out << indent() << " expected: " << expected << "\n"; + out << indent() << " actual : " << actual << "\n"; return false; } return true; @@ -181,12 +199,11 @@ struct testing { // Print summary and return an exit code int summary() const { - out << "\n==== TEST SUMMARY ====\n"; + out << "\n"; out << "tests : " << tests << "\n"; out << "assertions : " << assertions << "\n"; out << "failures : " << failures << "\n"; out << "exceptions : " << exceptions << "\n"; - out << "======================\n"; return failures == 0 ? 0 : 1; } }; From b3424574270fc42d96869d7cabad9092822040b7 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Wed, 19 Nov 2025 00:59:02 -0600 Subject: [PATCH 133/183] rename trie_matcher to trie --- common/chat-peg-parser.cpp | 45 +++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 29261e65a4c60..777eeb425b369 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -36,17 +36,16 @@ static bool is_hex_digit(const char c) { // Trie for matching multiple literals. // This is used in common_chat_peg_until_parser and to build a GBNF exclusion grammar -class trie_matcher { +struct trie { struct node { size_t depth = 0; std::map children; std::vector word_lengths; }; - std::vector trie; + std::vector nodes; - public: - trie_matcher(const std::vector & words) { + trie(const std::vector & words) { create_node(); // root node for (const auto & w : words) { insert(w); @@ -61,8 +60,8 @@ class trie_matcher { size_t pos = start_pos; while (pos < sv.size()) { - auto it = trie[current].children.find(sv[pos]); - if (it == trie[current].children.end()) { + auto it = nodes[current].children.find(sv[pos]); + if (it == nodes[current].children.end()) { // Can't continue matching return match_result{match_result::NO_MATCH}; } @@ -71,7 +70,7 @@ class trie_matcher { pos++; // Check if we've matched a complete word - if (!trie[current].word_lengths.empty()) { + if (!nodes[current].word_lengths.empty()) { return match_result{match_result::COMPLETE_MATCH}; } } @@ -100,18 +99,18 @@ class trie_matcher { private: void collect_prefix_and_next(size_t index, std::string & prefix, std::vector & out) { - if (trie[index].word_lengths.empty()) { - if (!trie[index].children.empty()) { + if (nodes[index].word_lengths.empty()) { + if (!nodes[index].children.empty()) { std::string chars; - chars.reserve(trie[index].children.size()); - for (const auto & p : trie[index].children) { + chars.reserve(nodes[index].children.size()); + for (const auto & p : nodes[index].children) { chars.push_back(p.first); } out.emplace_back(prefix_and_next{prefix, chars}); } } - for (const auto & p : trie[index].children) { + for (const auto & p : nodes[index].children) { unsigned char ch = p.first; auto child = p.second; prefix.push_back(ch); @@ -121,25 +120,25 @@ class trie_matcher { } size_t create_node() { - size_t index = trie.size(); - trie.emplace_back(); + size_t index = nodes.size(); + nodes.emplace_back(); return index; } void insert(const std::string & word) { size_t current = 0; for (unsigned char ch : word) { - auto it = trie[current].children.find(ch); - if (it == trie[current].children.end()) { + auto it = nodes[current].children.find(ch); + if (it == nodes[current].children.end()) { size_t child = create_node(); - trie[child].depth = trie[current].depth + 1; - trie[current].children[ch] = child; + nodes[child].depth = nodes[current].depth + 1; + nodes[current].children[ch] = child; current = child; } else { current = it->second; } } - trie[current].word_lengths.push_back(word.length()); + nodes[current].word_lengths.push_back(word.length()); } }; @@ -602,7 +601,7 @@ struct parser_executor { } common_chat_parse_result operator()(const common_chat_peg_until_parser & p) const { - trie_matcher matcher(p.delimiters); + trie matcher(p.delimiters); // Scan input and check for delimiters size_t pos = start_pos; @@ -629,12 +628,12 @@ struct parser_executor { // Check if a delimiter starts at this position auto match = matcher.check_at(ctx.input, pos); - if (match == trie_matcher::COMPLETE_MATCH) { + if (match == trie::COMPLETE_MATCH) { // Found a complete delimiter, return everything before it return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos, pos); } - if (match == trie_matcher::PARTIAL_MATCH) { + if (match == trie::PARTIAL_MATCH) { // Found a partial match extending to end of input, return everything before it return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos, pos); } @@ -1152,7 +1151,7 @@ static std::string gbnf_escape_char_class(char c) { } static std::string gbnf_excluding_pattern(const std::vector & strings) { - trie_matcher matcher(strings); + trie matcher(strings); auto pieces = matcher.collect_prefix_and_next(); std::string pattern; From 4fb8b9d0a489a0b52e0b5bf52629878e07ecf5c8 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Wed, 19 Nov 2025 01:00:54 -0600 Subject: [PATCH 134/183] simplify trie to just know if a node is the end of a word --- common/chat-peg-parser.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 777eeb425b369..1edff367cf25f 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -40,7 +40,7 @@ struct trie { struct node { size_t depth = 0; std::map children; - std::vector word_lengths; + bool is_word; }; std::vector nodes; @@ -70,7 +70,7 @@ struct trie { pos++; // Check if we've matched a complete word - if (!nodes[current].word_lengths.empty()) { + if (nodes[current].is_word) { return match_result{match_result::COMPLETE_MATCH}; } } @@ -99,7 +99,7 @@ struct trie { private: void collect_prefix_and_next(size_t index, std::string & prefix, std::vector & out) { - if (nodes[index].word_lengths.empty()) { + if (!nodes[index].is_word) { if (!nodes[index].children.empty()) { std::string chars; chars.reserve(nodes[index].children.size()); @@ -138,7 +138,7 @@ struct trie { current = it->second; } } - nodes[current].word_lengths.push_back(word.length()); + nodes[current].is_word = true; } }; From fc835023933590c322a4c0c69f2e61e48018a736 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Wed, 19 Nov 2025 20:00:50 -0600 Subject: [PATCH 135/183] remove common_chat_ prefix and ensure a common_peg_ prefix to all types --- common/chat-peg-parser-helper.cpp | 18 +- common/chat-peg-parser-helper.h | 18 +- common/chat-peg-parser.cpp | 584 +++++++++--------- common/chat-peg-parser.h | 358 +++++------ .../test-command7-parser-compare.cpp | 14 +- .../test-example-minimax-m2.cpp | 10 +- .../test-example-qwen3-coder.cpp | 16 +- .../chat-peg-parser/test-example-seed-oss.cpp | 8 +- .../chat-peg-parser/test-gbnf-generation.cpp | 28 +- tests/chat-peg-parser/test-json-parser.cpp | 24 +- .../test-json-serialization.cpp | 10 +- tests/chat-peg-parser/test-one.cpp | 64 +- tests/chat-peg-parser/test-optional.cpp | 12 +- .../chat-peg-parser/test-partial-parsing.cpp | 116 ++-- .../test-recursive-references.cpp | 24 +- tests/chat-peg-parser/test-unicode.cpp | 170 ++--- 16 files changed, 737 insertions(+), 737 deletions(-) diff --git a/common/chat-peg-parser-helper.cpp b/common/chat-peg-parser-helper.cpp index eec52199c54c4..9561619c2a96c 100644 --- a/common/chat-peg-parser-helper.cpp +++ b/common/chat-peg-parser-helper.cpp @@ -2,7 +2,7 @@ #include "chat-peg-parser.h" #include -common_chat_peg_parser common_chat_peg_parser_builder_helper::reasoning(const std::string & tag) { +common_peg_parser common_peg_parser_builder_helper::reasoning(const std::string & tag) { std::string open_tag; open_tag.append("<").append(tag).append(">"); std::string close_tag; @@ -10,16 +10,16 @@ common_chat_peg_parser common_chat_peg_parser_builder_helper::reasoning(const st return rule("raw-reasoning", literal(open_tag) << rule("reasoning-content", until(close_tag)) << literal(close_tag)); } -common_chat_peg_parser common_chat_peg_parser_builder_helper::content_before_tools(const std::string & tag) { +common_peg_parser common_peg_parser_builder_helper::content_before_tools(const std::string & tag) { return rule("content", until(tag)); } -common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_no_attr( +common_peg_parser common_peg_parser_builder_helper::quasi_xml_no_attr( const std::string & function_name, const std::vector & parameters, const std::string & function_tag, const std::string & param_tag) { - std::vector args; + std::vector args; for (auto it = parameters.begin(); it != parameters.end(); it++) { std::string arg_start_name; @@ -82,13 +82,13 @@ common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_no_attr( return function; } -common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_attr( +common_peg_parser common_peg_parser_builder_helper::quasi_xml_attr( const std::string & function_name, const std::vector & parameters, const std::string & function_tag, const std::string & param_tag, const std::string & name_attr) { - std::vector args; + std::vector args; for (auto it = parameters.begin(); it != parameters.end(); it++) { std::string arg_start_name; @@ -152,12 +152,12 @@ common_chat_peg_parser common_chat_peg_parser_builder_helper::quasi_xml_attr( return function; } -void common_chat_parse_simple_handler::operator()(const common_chat_parse_event & ev, common_chat_parse_semantics & semantics) const { +void common_peg_parse_simple_handler::operator()(const common_peg_parse_event & ev, common_peg_parse_semantics & semantics) const { if (log) { std::stringstream ss; - ss << "Event: type=" << (ev.type == COMMON_CHAT_PARSE_EVENT_NODE_START ? "start" : "end "); + ss << "Event: type=" << (ev.type == COMMON_PEG_PARSE_EVENT_NODE_START ? "start" : "end "); ss << " rule=" << ev.rule; - ss << " result=" << common_chat_parse_result_type_name(ev.status); + ss << " result=" << common_peg_parse_result_type_name(ev.status); ss << " text=" << ev.text; log(ss.str()); } diff --git a/common/chat-peg-parser-helper.h b/common/chat-peg-parser-helper.h index 71157ea426630..1879834414b00 100644 --- a/common/chat-peg-parser-helper.h +++ b/common/chat-peg-parser-helper.h @@ -1,38 +1,38 @@ #include "chat-peg-parser.h" #include "log.h" -class common_chat_peg_parser_builder_helper : public common_chat_peg_parser_builder { +class common_peg_parser_builder_helper : public common_peg_parser_builder { public: // Helper methods for common patterns // Adds raw-reasoning for the entire reasoning block plus reasoning-content for the contents, by default thinking tag is "think" - common_chat_peg_parser reasoning(const std::string & tag = "think"); + common_peg_parser reasoning(const std::string & tag = "think"); // Adds main content block before tool call block, due to the varied nature of tool call openers (not always XML-like) full tag is required - common_chat_peg_parser content_before_tools(const std::string &tag); + common_peg_parser content_before_tools(const std::string &tag); // Adds a quasi-XML tool call spec without a separate name attribute (Qwen3 style); // TODO: accept parameter schemas (required, value types etc.) - common_chat_peg_parser quasi_xml_no_attr(const std::string &function_name, const std::vector ¶meters, + common_peg_parser quasi_xml_no_attr(const std::string &function_name, const std::vector ¶meters, const std::string &function_tag = "function", const std::string ¶m_tag = "parameter"); // Adds a quasi-XML tool call spec with a separate name attribute (Minimax-M2 style) // TODO: accept parameter schemas (required, value types etc.) - common_chat_peg_parser quasi_xml_attr(const std::string &function_name, const std::vector ¶meters, + common_peg_parser quasi_xml_attr(const std::string &function_name, const std::vector ¶meters, const std::string &function_tag = "invoke", const std::string ¶m_tag = "parameter", const std::string &name_attr = "name"); }; template -common_chat_peg_arena build_peg_parser_helper(F && fn) { - common_chat_peg_parser_builder_helper builder; +common_peg_arena build_peg_parser_helper(F && fn) { + common_peg_parser_builder_helper builder; auto root = fn(builder); builder.set_root(root); return builder.build(); } -struct common_chat_parse_simple_handler { +struct common_peg_parse_simple_handler { std::function log; - void operator()(const common_chat_parse_event & ev, common_chat_parse_semantics & semantics) const; + void operator()(const common_peg_parse_event & ev, common_peg_parse_semantics & semantics) const; }; diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 1edff367cf25f..cd98ad89ded57 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -15,11 +15,11 @@ #include #include -const char * common_chat_parse_result_type_name(common_chat_parse_result_type type) { +const char * common_peg_parse_result_type_name(common_peg_parse_result_type type) { switch (type) { - case COMMON_CHAT_PARSE_RESULT_FAIL: return "fail"; - case COMMON_CHAT_PARSE_RESULT_SUCCESS: return "success"; - case COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT: return "need_more_input"; + case COMMON_PEG_PARSE_RESULT_FAIL: return "fail"; + case COMMON_PEG_PARSE_RESULT_SUCCESS: return "success"; + case COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT: return "need_more_input"; default: return "unknown"; } } @@ -35,7 +35,7 @@ static bool is_hex_digit(const char c) { } // Trie for matching multiple literals. -// This is used in common_chat_peg_until_parser and to build a GBNF exclusion grammar +// This is used in common_peg_until_parser and to build a GBNF exclusion grammar struct trie { struct node { size_t depth = 0; @@ -208,9 +208,9 @@ static std::pair parse_char_class_char(const std::string & con return {static_cast(static_cast(content[pos])), 1}; } -// Helper to parse common_chat_peg_chars_parser pattern and build ranges -static std::pair, bool> parse_char_classes(const std::string & classes) { - std::vector ranges; +// Helper to parse common_peg_chars_parser pattern and build ranges +static std::pair, bool> parse_char_classes(const std::string & classes) { + std::vector ranges; bool negated = false; std::string content = classes; @@ -236,10 +236,10 @@ static std::pair, bool> pa if (i + 1 < content.length() && content[i] == '-') { // Range detected auto [end, end_len] = parse_char_class_char(content, i + 1); - ranges.push_back(common_chat_peg_chars_parser::char_range{start, end}); + ranges.push_back(common_peg_chars_parser::char_range{start, end}); i += 1 + end_len; } else { - ranges.push_back(common_chat_peg_chars_parser::char_range{start, start}); + ranges.push_back(common_peg_chars_parser::char_range{start, start}); } } @@ -247,20 +247,20 @@ static std::pair, bool> pa } // Parse cache implementation -common_chat_parse_result common_chat_parse_cache::set(common_chat_peg_parser_id id, size_t start, common_chat_parse_result result) { - results[common_chat_parse_cache_key{id, start}] = result; +common_peg_parse_result common_peg_parse_cache::set(common_peg_parser_id id, size_t start, common_peg_parse_result result) { + results[common_peg_parse_cache_key{id, start}] = result; return result; } -std::optional common_chat_parse_cache::get(common_chat_peg_parser_id id, size_t start) { - auto it = results.find(common_chat_parse_cache_key{id, start}); +std::optional common_peg_parse_cache::get(common_peg_parser_id id, size_t start) { + auto it = results.find(common_peg_parse_cache_key{id, start}); if (it != results.end()) { return it->second; } return std::nullopt; } -void common_chat_parse_cache::clear() { +void common_peg_parse_cache::clear() { results.clear(); } @@ -268,19 +268,19 @@ void common_chat_parse_cache::clear() { struct parser_executor; // Arena implementation -common_chat_peg_arena::common_chat_peg_arena() : root_(COMMON_CHAT_PEG_INVALID_PARSER_ID) {} +common_peg_arena::common_peg_arena() : root_(COMMON_PEG_INVALID_PARSER_ID) {} -common_chat_peg_parser_id common_chat_peg_arena::add_parser(common_chat_peg_parser_variant parser) { - common_chat_peg_parser_id id = parsers_.size(); +common_peg_parser_id common_peg_arena::add_parser(common_peg_parser_variant parser) { + common_peg_parser_id id = parsers_.size(); parsers_.push_back(std::move(parser)); return id; } -void common_chat_peg_arena::add_rule(const std::string & name, common_chat_peg_parser_id id) { +void common_peg_arena::add_rule(const std::string & name, common_peg_parser_id id) { rules_[name] = id; } -common_chat_peg_parser_id common_chat_peg_arena::get_rule(const std::string & name) const { +common_peg_parser_id common_peg_arena::get_rule(const std::string & name) const { auto it = rules_.find(name); if (it == rules_.end()) { throw std::runtime_error("Rule not found: " + name); @@ -290,60 +290,60 @@ common_chat_peg_parser_id common_chat_peg_arena::get_rule(const std::string & na // Parsing executor - uses std::visit to dispatch to appropriate parser struct parser_executor { - const common_chat_peg_arena & arena; - common_chat_parse_context & ctx; + const common_peg_arena & arena; + common_peg_parse_context & ctx; size_t start_pos; - parser_executor(const common_chat_peg_arena & arena, common_chat_parse_context & ctx, size_t start) + parser_executor(const common_peg_arena & arena, common_peg_parse_context & ctx, size_t start) : arena(arena), ctx(ctx), start_pos(start) {} - common_chat_parse_result operator()(const common_chat_peg_start_parser & /* p */) const { - return common_chat_parse_result( - start_pos == 0 ? COMMON_CHAT_PARSE_RESULT_SUCCESS : COMMON_CHAT_PARSE_RESULT_FAIL, + common_peg_parse_result operator()(const common_peg_start_parser & /* p */) const { + return common_peg_parse_result( + start_pos == 0 ? COMMON_PEG_PARSE_RESULT_SUCCESS : COMMON_PEG_PARSE_RESULT_FAIL, start_pos ); } - common_chat_parse_result operator()(const common_chat_peg_end_parser & /* p */) const { - return common_chat_parse_result( - start_pos >= ctx.input.size() ? COMMON_CHAT_PARSE_RESULT_SUCCESS : COMMON_CHAT_PARSE_RESULT_FAIL, + common_peg_parse_result operator()(const common_peg_end_parser & /* p */) const { + return common_peg_parse_result( + start_pos >= ctx.input.size() ? COMMON_PEG_PARSE_RESULT_SUCCESS : COMMON_PEG_PARSE_RESULT_FAIL, start_pos ); } - common_chat_parse_result operator()(const common_chat_peg_literal_parser & p) { + common_peg_parse_result operator()(const common_peg_literal_parser & p) { auto pos = start_pos; for (auto i = 0u; i < p.literal.size(); ++i) { if (pos >= ctx.input.size()) { if (ctx.input_is_complete) { - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start_pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos); } if (ctx.input[pos] != p.literal[i]) { - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start_pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); } ++pos; } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos, pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos); } - common_chat_parse_result operator()(const common_chat_peg_sequence_parser & p) { + common_peg_parse_result operator()(const common_peg_sequence_parser & p) { auto pos = start_pos; for (const auto & child_id : p.children) { auto result = arena.parse(child_id, ctx, pos); if (!result.success()) { - return common_chat_parse_result(result.type, start_pos, result.end); + return common_peg_parse_result(result.type, start_pos, result.end); } pos = result.end; } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos, pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos); } - common_chat_parse_result operator()(const common_chat_peg_choice_parser & p) { + common_peg_parse_result operator()(const common_peg_choice_parser & p) { auto pos = start_pos; for (const auto & child_id : p.children) { auto result = arena.parse(child_id, ctx, pos); @@ -352,10 +352,10 @@ struct parser_executor { } } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start_pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); } - common_chat_parse_result operator()(const common_chat_peg_repetition_parser & p) { + common_peg_parse_result operator()(const common_peg_repetition_parser & p) { auto pos = start_pos; int match_count = 0; @@ -378,7 +378,7 @@ struct parser_executor { } if (result.need_more_input()) { - return common_chat_parse_result(result.type, start_pos, result.end); + return common_peg_parse_result(result.type, start_pos, result.end); } // Child failed - stop trying @@ -388,26 +388,26 @@ struct parser_executor { // Check if we got enough matches if (p.min_count > 0 && match_count < p.min_count) { if (pos >= ctx.input.size() && !ctx.input_is_complete) { - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos); } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start_pos, pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos, pos); } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos, pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos); } - common_chat_parse_result operator()(const common_chat_peg_and_parser & p) { + common_peg_parse_result operator()(const common_peg_and_parser & p) { auto result = arena.parse(p.child, ctx, start_pos); // Pass result but don't consume input - return common_chat_parse_result(result.type, start_pos); + return common_peg_parse_result(result.type, start_pos); } - common_chat_parse_result operator()(const common_chat_peg_not_parser & p) { + common_peg_parse_result operator()(const common_peg_not_parser & p) { auto result = arena.parse(p.child, ctx, start_pos); if (result.success()) { // Fail if the underlying parser matches - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start_pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); } if (result.need_more_input()) { @@ -416,26 +416,26 @@ struct parser_executor { } // Child failed, so negation succeeds - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos); } - common_chat_parse_result operator()(const common_chat_peg_any_parser & /* p */) const { + common_peg_parse_result operator()(const common_peg_any_parser & /* p */) const { // Parse a single UTF-8 codepoint (not just a single byte) auto result = parse_utf8_codepoint(ctx.input, start_pos); if (result.status == utf8_parse_result::INCOMPLETE) { if (ctx.input_is_complete) { - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start_pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start_pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos); } if (result.status == utf8_parse_result::INVALID) { - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start_pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos, start_pos + result.bytes_consumed); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, start_pos + result.bytes_consumed); } - common_chat_parse_result operator()(const common_chat_peg_space_parser & /* p */) { + common_peg_parse_result operator()(const common_peg_space_parser & /* p */) { auto pos = start_pos; while (pos < ctx.input.size()) { char c = ctx.input[pos]; @@ -446,10 +446,10 @@ struct parser_executor { } } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos, pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos); } - common_chat_parse_result operator()(const common_chat_peg_chars_parser & p) const { + common_peg_parse_result operator()(const common_peg_chars_parser & p) const { auto pos = start_pos; int match_count = 0; @@ -460,23 +460,23 @@ struct parser_executor { if (result.status == utf8_parse_result::INCOMPLETE) { if (match_count >= p.min_count) { // We have enough matches, succeed with what we have - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos, pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos); } // Not enough matches yet if (ctx.input_is_complete) { - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start_pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos); } if (result.status == utf8_parse_result::INVALID) { // Malformed UTF-8 in input if (match_count >= p.min_count) { // We have enough matches, succeed up to here - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos, pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos); } // Not enough matches, fail - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start_pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); } // Check if this codepoint matches our character class @@ -505,21 +505,21 @@ struct parser_executor { // Check if we got enough matches if (match_count < p.min_count) { if (pos >= ctx.input.size() && !ctx.input_is_complete) { - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos); } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start_pos, pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos, pos); } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos, pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos); } - static common_chat_parse_result handle_escape_sequence(common_chat_parse_context & ctx, size_t start, size_t & pos) { + static common_peg_parse_result handle_escape_sequence(common_peg_parse_context & ctx, size_t start, size_t & pos) { ++pos; // consume '\' if (pos >= ctx.input.size()) { if (ctx.input_is_complete) { - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start); } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start, pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start, pos); } switch (ctx.input[pos]) { @@ -532,33 +532,33 @@ struct parser_executor { case 'r': case 't': ++pos; - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start, pos); case 'u': return handle_unicode_escape(ctx, start, pos); default: // Invalid escape sequence - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start); } } - static common_chat_parse_result handle_unicode_escape(common_chat_parse_context & ctx, size_t start, size_t & pos) { + static common_peg_parse_result handle_unicode_escape(common_peg_parse_context & ctx, size_t start, size_t & pos) { ++pos; // consume 'u' for (int i = 0; i < 4; ++i) { if (pos >= ctx.input.size()) { if (ctx.input_is_complete) { - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start); } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start, pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start, pos); } if (!is_hex_digit(ctx.input[pos])) { - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start); } ++pos; } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start, pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start, pos); } - common_chat_parse_result operator()(const common_chat_peg_json_string_parser & /* p */) { + common_peg_parse_result operator()(const common_peg_json_string_parser & /* p */) { auto pos = start_pos; // Parse string content (without quotes) @@ -567,7 +567,7 @@ struct parser_executor { if (c == '"') { // Found closing quote - success (don't consume it) - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos, pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos); } if (c == '\\') { @@ -580,13 +580,13 @@ struct parser_executor { if (utf8_result.status == utf8_parse_result::INCOMPLETE) { if (ctx.input_is_complete) { - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start_pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos); } if (utf8_result.status == utf8_parse_result::INVALID) { - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start_pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); } pos += utf8_result.bytes_consumed; @@ -595,12 +595,12 @@ struct parser_executor { // Reached end without finding closing quote if (ctx.input_is_complete) { - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start_pos, pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos, pos); } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos); } - common_chat_parse_result operator()(const common_chat_peg_until_parser & p) const { + common_peg_parse_result operator()(const common_peg_until_parser & p) const { trie matcher(p.delimiters); // Scan input and check for delimiters @@ -614,15 +614,15 @@ struct parser_executor { // Incomplete UTF-8 sequence if (ctx.input_is_complete) { // Input is complete but UTF-8 is incomplete = malformed - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start_pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); } // Return what we have so far (before incomplete sequence) - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT, start_pos, last_valid_pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, last_valid_pos); } if (utf8_result.status == utf8_parse_result::INVALID) { // Malformed UTF-8 - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_FAIL, start_pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); } // Check if a delimiter starts at this position @@ -630,36 +630,36 @@ struct parser_executor { if (match == trie::COMPLETE_MATCH) { // Found a complete delimiter, return everything before it - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos, pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos); } if (match == trie::PARTIAL_MATCH) { // Found a partial match extending to end of input, return everything before it - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos, pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos); } pos += utf8_result.bytes_consumed; last_valid_pos = pos; } - return common_chat_parse_result(COMMON_CHAT_PARSE_RESULT_SUCCESS, start_pos, last_valid_pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, last_valid_pos); } - common_chat_parse_result operator()(const common_chat_peg_schema_parser & p) { + common_peg_parse_result operator()(const common_peg_schema_parser & p) { return arena.parse(p.child, ctx, start_pos); } - common_chat_parse_result operator()(const common_chat_peg_rule_parser & p) { + common_peg_parse_result operator()(const common_peg_rule_parser & p) { // Fire NODE_START event if (ctx.event_handler && ctx.semantics) { - ctx.event_handler(common_chat_parse_event{ - COMMON_CHAT_PARSE_EVENT_NODE_START, + ctx.event_handler(common_peg_parse_event{ + COMMON_PEG_PARSE_EVENT_NODE_START, p.name, p.annotation, start_pos, start_pos, "", - COMMON_CHAT_PARSE_RESULT_FAIL, + COMMON_PEG_PARSE_RESULT_FAIL, ctx.current_depth }, *ctx.semantics); ctx.current_depth++; @@ -677,8 +677,8 @@ struct parser_executor { } else { text = ""; } - ctx.event_handler(common_chat_parse_event{ - COMMON_CHAT_PARSE_EVENT_NODE_END, + ctx.event_handler(common_peg_parse_event{ + COMMON_PEG_PARSE_EVENT_NODE_END, p.name, p.annotation, result.start, @@ -692,12 +692,12 @@ struct parser_executor { return result; } - common_chat_parse_result operator()(const common_chat_peg_ref_parser & p) { + common_peg_parse_result operator()(const common_peg_ref_parser & p) { auto rule_id = arena.get_rule(p.name); return arena.parse(rule_id, ctx, start_pos); } - common_chat_parse_result operator()(const common_chat_peg_capture_parser & p) { + common_peg_parse_result operator()(const common_peg_capture_parser & p) { auto result = arena.parse(p.child, ctx, start_pos); if (!result.fail() && ctx.semantics) { @@ -711,14 +711,14 @@ struct parser_executor { } }; -common_chat_parse_result common_chat_peg_arena::parse(common_chat_parse_context & ctx, size_t start) const { - if (root_ == COMMON_CHAT_PEG_INVALID_PARSER_ID) { +common_peg_parse_result common_peg_arena::parse(common_peg_parse_context & ctx, size_t start) const { + if (root_ == COMMON_PEG_INVALID_PARSER_ID) { throw std::runtime_error("No root parser set"); } return parse(root_, ctx, start); } -common_chat_parse_result common_chat_peg_arena::parse(common_chat_peg_parser_id id, common_chat_parse_context & ctx, size_t start) const { +common_peg_parse_result common_peg_arena::parse(common_peg_parser_id id, common_peg_parse_context & ctx, size_t start) const { // Check cache auto cached = ctx.cache.get(id, start); if (cached) { @@ -735,59 +735,59 @@ common_chat_parse_result common_chat_peg_arena::parse(common_chat_peg_parser_id } // Dump implementation (for debugging) -std::string common_chat_peg_arena::dump(common_chat_peg_parser_id id) const { +std::string common_peg_arena::dump(common_peg_parser_id id) const { const auto & parser = parsers_.at(id); return std::visit([this](const auto & p) -> std::string { using T = std::decay_t; - if constexpr (std::is_same_v) { + if constexpr (std::is_same_v) { return "Start"; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { return "End"; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { return "Literal(" + p.literal + ")"; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { std::vector parts; for (const auto & child : p.children) { parts.push_back(dump(child)); } return "Sequence(" + string_join(parts, ", ") + ")"; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { std::vector parts; for (const auto & child : p.children) { parts.push_back(dump(child)); } return "Choice(" + string_join(parts, ", ") + ")"; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { if (p.max_count == -1) { return "Repetition(" + dump(p.child) + ", " + std::to_string(p.min_count) + ", unbounded)"; } return "Repetition(" + dump(p.child) + ", " + std::to_string(p.min_count) + ", " + std::to_string(p.max_count) + ")"; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { return "And(" + dump(p.child) + ")"; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { return "Not(" + dump(p.child) + ")"; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { return "Any"; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { return "Space"; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { if (p.max_count == -1) { return "CharRepeat(" + p.pattern + ", " + std::to_string(p.min_count) + ", unbounded)"; } return "CharRepeat(" + p.pattern + ", " + std::to_string(p.min_count) + ", " + std::to_string(p.max_count) + ")"; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { return "JsonString()"; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { return "Until(" + string_join(p.delimiters, " | ") + ")"; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { return "Schema(" + dump(p.child) + ", " + (p.schema ? p.schema->dump() : "null") + ")"; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { return "Rule(" + p.name + ", " + dump(p.child) + ")"; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { return "Ref(" + p.name + ")"; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { return "Capture(" + p.key + ", " + dump(p.child) + ")"; } else { return "Unknown"; @@ -796,57 +796,57 @@ std::string common_chat_peg_arena::dump(common_chat_peg_parser_id id) const { } // Parser wrapper operator implementations -common_chat_peg_parser common_chat_peg_parser::operator+(const common_chat_peg_parser & other) const { +common_peg_parser common_peg_parser::operator+(const common_peg_parser & other) const { return builder_->sequence({id_, other.id_}); } -common_chat_peg_parser common_chat_peg_parser::operator|(const common_chat_peg_parser & other) const { +common_peg_parser common_peg_parser::operator|(const common_peg_parser & other) const { return builder_->choice({id_, other.id_}); } -common_chat_peg_parser common_chat_peg_parser::operator<<(const common_chat_peg_parser & other) const { +common_peg_parser common_peg_parser::operator<<(const common_peg_parser & other) const { return builder_->sequence({id_, builder_->space(), other.id_}); } // String literal overloads -common_chat_peg_parser common_chat_peg_parser::operator+(const char * str) const { +common_peg_parser common_peg_parser::operator+(const char * str) const { return *this + builder_->literal(str); } -common_chat_peg_parser common_chat_peg_parser::operator+(const std::string & str) const { +common_peg_parser common_peg_parser::operator+(const std::string & str) const { return *this + builder_->literal(str); } -common_chat_peg_parser common_chat_peg_parser::operator|(const char * str) const { +common_peg_parser common_peg_parser::operator|(const char * str) const { return *this | builder_->literal(str); } -common_chat_peg_parser common_chat_peg_parser::operator|(const std::string & str) const { +common_peg_parser common_peg_parser::operator|(const std::string & str) const { return *this | builder_->literal(str); } -common_chat_peg_parser common_chat_peg_parser::operator<<(const char * str) const { +common_peg_parser common_peg_parser::operator<<(const char * str) const { return *this << builder_->literal(str); } -common_chat_peg_parser common_chat_peg_parser::operator<<(const std::string & str) const { +common_peg_parser common_peg_parser::operator<<(const std::string & str) const { return *this << builder_->literal(str); } // Free function operators for string + parser -common_chat_peg_parser operator+(const char * str, const common_chat_peg_parser & p) { +common_peg_parser operator+(const char * str, const common_peg_parser & p) { return p.builder()->literal(str) + p; } -common_chat_peg_parser operator+(const std::string & str, const common_chat_peg_parser & p) { +common_peg_parser operator+(const std::string & str, const common_peg_parser & p) { return operator+(str.c_str(), p); } -common_chat_peg_parser operator<<(const char * str, const common_chat_peg_parser & p) { +common_peg_parser operator<<(const char * str, const common_peg_parser & p) { return p.builder()->literal(str) << p; } -common_chat_peg_parser operator<<(const std::string & str, const common_chat_peg_parser & p) { +common_peg_parser operator<<(const std::string & str, const common_peg_parser & p) { return operator<<(str.c_str(), p); } @@ -857,36 +857,36 @@ static std::string rule_name(const std::string & name) { } // Builder implementation -common_chat_peg_parser_builder::common_chat_peg_parser_builder() {} +common_peg_parser_builder::common_peg_parser_builder() {} -common_chat_peg_parser common_chat_peg_parser_builder::start() { - return wrap(arena_.add_parser(common_chat_peg_start_parser{})); +common_peg_parser common_peg_parser_builder::start() { + return wrap(arena_.add_parser(common_peg_start_parser{})); } -common_chat_peg_parser common_chat_peg_parser_builder::end() { - return wrap(arena_.add_parser(common_chat_peg_end_parser{})); +common_peg_parser common_peg_parser_builder::end() { + return wrap(arena_.add_parser(common_peg_end_parser{})); } -common_chat_peg_parser common_chat_peg_parser_builder::literal(const std::string & literal) { - return wrap(arena_.add_parser(common_chat_peg_literal_parser{literal})); +common_peg_parser common_peg_parser_builder::literal(const std::string & literal) { + return wrap(arena_.add_parser(common_peg_literal_parser{literal})); } -common_chat_peg_parser common_chat_peg_parser_builder::sequence(const std::vector & parsers) { +common_peg_parser common_peg_parser_builder::sequence(const std::vector & parsers) { // Flatten nested sequences - std::vector flattened; + std::vector flattened; for (const auto & p : parsers) { const auto & parser = arena_.get(p); - if (auto seq = std::get_if(&parser)) { + if (auto seq = std::get_if(&parser)) { flattened.insert(flattened.end(), seq->children.begin(), seq->children.end()); } else { flattened.push_back(p); } } - return wrap(arena_.add_parser(common_chat_peg_sequence_parser{flattened})); + return wrap(arena_.add_parser(common_peg_sequence_parser{flattened})); } -common_chat_peg_parser common_chat_peg_parser_builder::sequence(const std::vector & parsers) { - std::vector ids; +common_peg_parser common_peg_parser_builder::sequence(const std::vector & parsers) { + std::vector ids; ids.reserve(parsers.size()); for (const auto & p : parsers) { ids.push_back(p.id()); @@ -894,8 +894,8 @@ common_chat_peg_parser common_chat_peg_parser_builder::sequence(const std::vecto return sequence(ids); } -common_chat_peg_parser common_chat_peg_parser_builder::sequence(std::initializer_list parsers) { - std::vector ids; +common_peg_parser common_peg_parser_builder::sequence(std::initializer_list parsers) { + std::vector ids; ids.reserve(parsers.size()); for (const auto & p : parsers) { ids.push_back(p.id()); @@ -903,22 +903,22 @@ common_chat_peg_parser common_chat_peg_parser_builder::sequence(std::initializer return sequence(ids); } -common_chat_peg_parser common_chat_peg_parser_builder::choice(const std::vector & parsers) { +common_peg_parser common_peg_parser_builder::choice(const std::vector & parsers) { // Flatten nested choices - std::vector flattened; + std::vector flattened; for (const auto & p : parsers) { const auto & parser = arena_.get(p); - if (auto choice = std::get_if(&parser)) { + if (auto choice = std::get_if(&parser)) { flattened.insert(flattened.end(), choice->children.begin(), choice->children.end()); } else { flattened.push_back(p); } } - return wrap(arena_.add_parser(common_chat_peg_choice_parser{flattened})); + return wrap(arena_.add_parser(common_peg_choice_parser{flattened})); } -common_chat_peg_parser common_chat_peg_parser_builder::choice(const std::vector & parsers) { - std::vector ids; +common_peg_parser common_peg_parser_builder::choice(const std::vector & parsers) { + std::vector ids; ids.reserve(parsers.size()); for (const auto & p : parsers) { ids.push_back(p.id()); @@ -926,8 +926,8 @@ common_chat_peg_parser common_chat_peg_parser_builder::choice(const std::vector< return choice(ids); } -common_chat_peg_parser common_chat_peg_parser_builder::choice(std::initializer_list parsers) { - std::vector ids; +common_peg_parser common_peg_parser_builder::choice(std::initializer_list parsers) { + std::vector ids; ids.reserve(parsers.size()); for (const auto & p : parsers) { ids.push_back(p.id()); @@ -935,91 +935,91 @@ common_chat_peg_parser common_chat_peg_parser_builder::choice(std::initializer_l return choice(ids); } -common_chat_peg_parser common_chat_peg_parser_builder::peek(common_chat_peg_parser p) { - return wrap(arena_.add_parser(common_chat_peg_and_parser{p.id()})); +common_peg_parser common_peg_parser_builder::peek(common_peg_parser p) { + return wrap(arena_.add_parser(common_peg_and_parser{p.id()})); } -common_chat_peg_parser common_chat_peg_parser_builder::negate(common_chat_peg_parser p) { - return wrap(arena_.add_parser(common_chat_peg_not_parser{p.id()})); +common_peg_parser common_peg_parser_builder::negate(common_peg_parser p) { + return wrap(arena_.add_parser(common_peg_not_parser{p.id()})); } -common_chat_peg_parser common_chat_peg_parser_builder::any() { - return wrap(arena_.add_parser(common_chat_peg_any_parser{})); +common_peg_parser common_peg_parser_builder::any() { + return wrap(arena_.add_parser(common_peg_any_parser{})); } -common_chat_peg_parser common_chat_peg_parser_builder::chars(const std::string & classes, int min, int max) { +common_peg_parser common_peg_parser_builder::chars(const std::string & classes, int min, int max) { auto [ranges, negated] = parse_char_classes(classes); - return wrap(arena_.add_parser(common_chat_peg_chars_parser{classes, ranges, negated, min, max})); + return wrap(arena_.add_parser(common_peg_chars_parser{classes, ranges, negated, min, max})); } -common_chat_peg_parser common_chat_peg_parser_builder::one(const std::string & classes) { +common_peg_parser common_peg_parser_builder::one(const std::string & classes) { return chars(classes, 1, 1); } -common_chat_peg_parser common_chat_peg_parser_builder::ref(const std::string & name) { - return wrap(arena_.add_parser(common_chat_peg_ref_parser{rule_name(name)})); +common_peg_parser common_peg_parser_builder::ref(const std::string & name) { + return wrap(arena_.add_parser(common_peg_ref_parser{rule_name(name)})); } -common_chat_peg_parser common_chat_peg_parser_builder::space() { - return wrap(arena_.add_parser(common_chat_peg_space_parser{})); +common_peg_parser common_peg_parser_builder::space() { + return wrap(arena_.add_parser(common_peg_space_parser{})); } -common_chat_peg_parser common_chat_peg_parser_builder::until(const std::string & delimiter) { - return wrap(arena_.add_parser(common_chat_peg_until_parser{std::vector{delimiter}})); +common_peg_parser common_peg_parser_builder::until(const std::string & delimiter) { + return wrap(arena_.add_parser(common_peg_until_parser{std::vector{delimiter}})); } -common_chat_peg_parser common_chat_peg_parser_builder::until_one_of(const std::vector & delimiters) { - return wrap(arena_.add_parser(common_chat_peg_until_parser{delimiters})); +common_peg_parser common_peg_parser_builder::until_one_of(const std::vector & delimiters) { + return wrap(arena_.add_parser(common_peg_until_parser{delimiters})); } -common_chat_peg_parser common_chat_peg_parser_builder::repeat(common_chat_peg_parser p, int min, int max) { - return wrap(arena_.add_parser(common_chat_peg_repetition_parser{p.id(), min, max})); +common_peg_parser common_peg_parser_builder::repeat(common_peg_parser p, int min, int max) { + return wrap(arena_.add_parser(common_peg_repetition_parser{p.id(), min, max})); } -common_chat_peg_parser common_chat_peg_parser_builder::repeat(common_chat_peg_parser p, int n) { - return wrap(arena_.add_parser(common_chat_peg_repetition_parser{p.id(), n, n})); +common_peg_parser common_peg_parser_builder::repeat(common_peg_parser p, int n) { + return wrap(arena_.add_parser(common_peg_repetition_parser{p.id(), n, n})); } -common_chat_peg_parser common_chat_peg_parser_builder::optional(common_chat_peg_parser p) { +common_peg_parser common_peg_parser_builder::optional(common_peg_parser p) { return repeat(p, 0, 1); } -common_chat_peg_parser common_chat_peg_parser_builder::zero_or_more(common_chat_peg_parser p) { +common_peg_parser common_peg_parser_builder::zero_or_more(common_peg_parser p) { return repeat(p, 0, -1); } -common_chat_peg_parser common_chat_peg_parser_builder::one_or_more(common_chat_peg_parser p) { +common_peg_parser common_peg_parser_builder::one_or_more(common_peg_parser p) { return repeat(p, 1, -1); } -common_chat_peg_parser common_chat_peg_parser_builder::json_string_content() { - return wrap(arena_.add_parser(common_chat_peg_json_string_parser{})); +common_peg_parser common_peg_parser_builder::json_string_content() { + return wrap(arena_.add_parser(common_peg_json_string_parser{})); } -common_chat_peg_parser common_chat_peg_parser_builder::schema(common_chat_peg_parser p, const std::string & name, const nlohmann::ordered_json & schema) { - return wrap(arena_.add_parser(common_chat_peg_schema_parser{p.id(), name, std::make_shared(schema)})); +common_peg_parser common_peg_parser_builder::schema(common_peg_parser p, const std::string & name, const nlohmann::ordered_json & schema) { + return wrap(arena_.add_parser(common_peg_schema_parser{p.id(), name, std::make_shared(schema)})); } -common_chat_peg_parser common_chat_peg_parser_builder::capture(const std::string & key, common_chat_peg_parser p) { - return wrap(arena_.add_parser(common_chat_peg_capture_parser{p.id(), key})); +common_peg_parser common_peg_parser_builder::capture(const std::string & key, common_peg_parser p) { + return wrap(arena_.add_parser(common_peg_capture_parser{p.id(), key})); } -common_chat_peg_parser common_chat_peg_parser_builder::rule(const std::string & name, common_chat_peg_parser p, bool trigger) { +common_peg_parser common_peg_parser_builder::rule(const std::string & name, common_peg_parser p, bool trigger) { return rule(name, "", p, trigger); } -common_chat_peg_parser common_chat_peg_parser_builder::rule(const std::string & name, const std::string & annotation, common_chat_peg_parser p, bool trigger) { +common_peg_parser common_peg_parser_builder::rule(const std::string & name, const std::string & annotation, common_peg_parser p, bool trigger) { auto clean_name = rule_name(name); - auto rule_id = arena_.add_parser(common_chat_peg_rule_parser{clean_name, annotation, p.id(), trigger}); + auto rule_id = arena_.add_parser(common_peg_rule_parser{clean_name, annotation, p.id(), trigger}); arena_.add_rule(clean_name, rule_id); return ref(clean_name); } -common_chat_peg_parser common_chat_peg_parser_builder::rule(const std::string & name, const std::function & builder_fn, bool trigger) { +common_peg_parser common_peg_parser_builder::rule(const std::string & name, const std::function & builder_fn, bool trigger) { return rule(name, "", builder_fn, trigger); } -common_chat_peg_parser common_chat_peg_parser_builder::rule(const std::string & name, const std::string & annotation, const std::function & builder_fn, bool trigger) { +common_peg_parser common_peg_parser_builder::rule(const std::string & name, const std::string & annotation, const std::function & builder_fn, bool trigger) { auto clean_name = rule_name(name); if (arena_.has_rule(clean_name)) { return ref(clean_name); @@ -1027,30 +1027,30 @@ common_chat_peg_parser common_chat_peg_parser_builder::rule(const std::string & // Create placeholder rule to allow recursive references auto placeholder = any(); // Temporary placeholder - auto placeholder_rule_id = arena_.add_parser(common_chat_peg_rule_parser{clean_name, annotation, placeholder.id(), trigger}); + auto placeholder_rule_id = arena_.add_parser(common_peg_rule_parser{clean_name, annotation, placeholder.id(), trigger}); arena_.add_rule(clean_name, placeholder_rule_id); // Build the actual parser auto parser = builder_fn(); // Replace placeholder with actual rule - auto rule_id = arena_.add_parser(common_chat_peg_rule_parser{clean_name, annotation, parser.id(), trigger}); + auto rule_id = arena_.add_parser(common_peg_rule_parser{clean_name, annotation, parser.id(), trigger}); arena_.rules_[clean_name] = rule_id; return ref(clean_name); } -void common_chat_peg_parser_builder::set_root(common_chat_peg_parser p) { +void common_peg_parser_builder::set_root(common_peg_parser p) { arena_.set_root(p.id()); } -common_chat_peg_arena common_chat_peg_parser_builder::build() { +common_peg_arena common_peg_parser_builder::build() { return std::move(arena_); } // JSON parsers -common_chat_peg_parser common_chat_peg_parser_builder::json_number() { - std::function builder = [this]() { +common_peg_parser common_peg_parser_builder::json_number() { + std::function builder = [this]() { auto digit1_9 = chars("[1-9]", 1, 1); auto digits = chars("[0-9]"); auto int_part = choice({literal("0"), sequence({digit1_9, chars("[0-9]", 0, -1)})}); @@ -1061,29 +1061,29 @@ common_chat_peg_parser common_chat_peg_parser_builder::json_number() { return rule("json-number", builder); } -common_chat_peg_parser common_chat_peg_parser_builder::json_string() { - std::function builder = [this]() { +common_peg_parser common_peg_parser_builder::json_string() { + std::function builder = [this]() { return sequence({literal("\""), json_string_content(), literal("\"")}); }; return rule("json-string", builder); } -common_chat_peg_parser common_chat_peg_parser_builder::json_bool() { - std::function builder = [this]() { +common_peg_parser common_peg_parser_builder::json_bool() { + std::function builder = [this]() { return choice({literal("true"), literal("false")}); }; return rule("json-bool", builder); } -common_chat_peg_parser common_chat_peg_parser_builder::json_null() { - std::function builder = [this]() { +common_peg_parser common_peg_parser_builder::json_null() { + std::function builder = [this]() { return literal("null"); }; return rule("json-null", builder); } -common_chat_peg_parser common_chat_peg_parser_builder::json_object() { - std::function builder = [this]() { +common_peg_parser common_peg_parser_builder::json_object() { + std::function builder = [this]() { auto ws = space(); auto member = sequence({json_string(), ws, literal(":"), ws, json()}); auto members = sequence({member, zero_or_more(sequence({ws, literal(","), ws, member}))}); @@ -1095,8 +1095,8 @@ common_chat_peg_parser common_chat_peg_parser_builder::json_object() { return rule("json-object", builder); } -common_chat_peg_parser common_chat_peg_parser_builder::json_array() { - std::function builder = [this]() { +common_peg_parser common_peg_parser_builder::json_array() { + std::function builder = [this]() { auto ws = space(); auto elements = sequence({json(), zero_or_more(sequence({ws, literal(","), ws, json()}))}); return choice({ @@ -1107,8 +1107,8 @@ common_chat_peg_parser common_chat_peg_parser_builder::json_array() { return rule("json-array", builder); } -common_chat_peg_parser common_chat_peg_parser_builder::json() { - std::function builder = [this]() { +common_peg_parser common_peg_parser_builder::json() { + std::function builder = [this]() { return choice({ json_object(), json_array(), @@ -1181,39 +1181,39 @@ static std::string gbnf_excluding_pattern(const std::vector & strin // Collect reachable rules from a given rule static std::unordered_set collect_reachable_rules( - const common_chat_peg_arena & arena, - const common_chat_peg_parser_id & rule + const common_peg_arena & arena, + const common_peg_parser_id & rule ) { std::unordered_set reachable; std::unordered_set visited; - std::function visit = [&](common_chat_peg_parser_id id) { + std::function visit = [&](common_peg_parser_id id) { const auto & parser = arena.get(id); std::visit([&](const auto & p) { using T = std::decay_t; - if constexpr (std::is_same_v) { + if constexpr (std::is_same_v) { for (auto child : p.children) { visit(child); } - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { for (auto child : p.children) { visit(child); } - } else if constexpr (std::is_same_v || - std::is_same_v || - std::is_same_v || - std::is_same_v || - std::is_same_v) { + } else if constexpr (std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v) { visit(p.child); - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { if (visited.find(p.name) == visited.end()) { visited.insert(p.name); reachable.insert(p.name); visit(p.child); } - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { // Traverse rules so we pick up everything auto referenced_rule = arena.get_rule(p.name); visit(referenced_rule); @@ -1226,19 +1226,19 @@ static std::unordered_set collect_reachable_rules( } // GBNF generation implementation -void common_chat_peg_arena::build_grammar(const common_grammar_builder & builder, bool lazy) const { +void common_peg_arena::build_grammar(const common_grammar_builder & builder, bool lazy) const { // Generate GBNF for a parser - std::function to_gbnf = [&](common_chat_peg_parser_id id) -> std::string { + std::function to_gbnf = [&](common_peg_parser_id id) -> std::string { const auto & parser = parsers_.at(id); return std::visit([&](const auto & p) -> std::string { using T = std::decay_t; - if constexpr (std::is_same_v || std::is_same_v) { + if constexpr (std::is_same_v || std::is_same_v) { return ""; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { return gbnf_literal(p.literal); - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { std::string s; for (const auto & child : p.children) { if (!s.empty()) { @@ -1246,15 +1246,15 @@ void common_chat_peg_arena::build_grammar(const common_grammar_builder & builder } auto child_gbnf = to_gbnf(child); const auto & child_parser = parsers_.at(child); - if (std::holds_alternative(child_parser) || - std::holds_alternative(child_parser)) { + if (std::holds_alternative(child_parser) || + std::holds_alternative(child_parser)) { s += "(" + child_gbnf + ")"; } else { s += child_gbnf; } } return s; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { std::string s; for (const auto & child : p.children) { if (!s.empty()) { @@ -1262,18 +1262,18 @@ void common_chat_peg_arena::build_grammar(const common_grammar_builder & builder } auto child_gbnf = to_gbnf(child); const auto & child_parser = parsers_.at(child); - if (std::holds_alternative(child_parser)) { + if (std::holds_alternative(child_parser)) { s += "(" + child_gbnf + ")"; } else { s += child_gbnf; } } return s; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { auto child_gbnf = to_gbnf(p.child); const auto & child_parser = parsers_.at(p.child); - if (std::holds_alternative(child_parser) || - std::holds_alternative(child_parser)) { + if (std::holds_alternative(child_parser) || + std::holds_alternative(child_parser)) { child_gbnf = "(" + child_gbnf + ")"; } if (p.min_count == 0 && p.max_count == 1) { @@ -1289,13 +1289,13 @@ void common_chat_peg_arena::build_grammar(const common_grammar_builder & builder return child_gbnf + "{" + std::to_string(p.min_count) + ",}"; } return child_gbnf + "{" + std::to_string(p.min_count) + "," + std::to_string(p.max_count) + "}"; - } else if constexpr (std::is_same_v || std::is_same_v) { + } else if constexpr (std::is_same_v || std::is_same_v) { return ""; // Lookahead not supported in GBNF - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { return "."; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { return "space"; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { std::string result = p.pattern; if (p.min_count == 0 && p.max_count == -1) { return result + "*"; @@ -1313,20 +1313,20 @@ void common_chat_peg_arena::build_grammar(const common_grammar_builder & builder return result + "{" + std::to_string(p.min_count) + "}"; } return result + "{" + std::to_string(p.min_count) + "," + std::to_string(p.max_count) + "}"; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { return R"(( [^"\\] | "\\" ( ["\\/ bfnrt] | "u" [0-9a-fA-F]{4} ) )*)"; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { return gbnf_excluding_pattern(p.delimiters); - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { if (p.schema) { return builder.add_schema(p.name, *p.schema); } return to_gbnf(p.child); - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { return to_gbnf(p.child); - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { return p.name; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { return to_gbnf(p.child); } else { return ""; @@ -1341,7 +1341,7 @@ void common_chat_peg_arena::build_grammar(const common_grammar_builder & builder // Collect rules reachable from trigger rules for (const auto & [name, id] : rules_) { const auto & parser = parsers_.at(id); - if (auto rule = std::get_if(&parser)) { + if (auto rule = std::get_if(&parser)) { if (rule->trigger) { // Mark trigger as reachable and visit it reachable_rules.insert(name); @@ -1362,7 +1362,7 @@ void common_chat_peg_arena::build_grammar(const common_grammar_builder & builder } const auto & parser = parsers_.at(rule_id); - if (auto rule = std::get_if(&parser)) { + if (auto rule = std::get_if(&parser)) { builder.add_rule(rule->name, to_gbnf(rule->child)); } } @@ -1372,7 +1372,7 @@ void common_chat_peg_arena::build_grammar(const common_grammar_builder & builder std::vector trigger_names; for (const auto & [name, rule_id] : rules_) { const auto & parser = parsers_.at(rule_id); - if (auto rule = std::get_if(&parser)) { + if (auto rule = std::get_if(&parser)) { if (rule->trigger) { trigger_names.push_back(rule->name); } @@ -1380,47 +1380,47 @@ void common_chat_peg_arena::build_grammar(const common_grammar_builder & builder } builder.add_rule("root", string_join(trigger_names, " | ")); - } else if (root_ != COMMON_CHAT_PEG_INVALID_PARSER_ID) { + } else if (root_ != COMMON_PEG_INVALID_PARSER_ID) { builder.add_rule("root", to_gbnf(root_)); } } // Serialization helper: convert parser variant to JSON -static nlohmann::json serialize_parser_variant(const common_chat_peg_parser_variant & variant) { +static nlohmann::json serialize_parser_variant(const common_peg_parser_variant & variant) { return std::visit([](const auto & p) -> nlohmann::json { using T = std::decay_t; nlohmann::json j; - if constexpr (std::is_same_v) { + if constexpr (std::is_same_v) { j["type"] = "start"; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { j["type"] = "end"; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { j["type"] = "literal"; j["literal"] = p.literal; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { j["type"] = "sequence"; j["children"] = p.children; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { j["type"] = "choice"; j["children"] = p.children; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { j["type"] = "repetition"; j["child"] = p.child; j["min_count"] = p.min_count; j["max_count"] = p.max_count; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { j["type"] = "and"; j["child"] = p.child; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { j["type"] = "not"; j["child"] = p.child; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { j["type"] = "any"; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { j["type"] = "space"; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { j["type"] = "chars"; j["pattern"] = p.pattern; nlohmann::json ranges = nlohmann::json::array(); @@ -1434,12 +1434,12 @@ static nlohmann::json serialize_parser_variant(const common_chat_peg_parser_vari j["negated"] = p.negated; j["min_count"] = p.min_count; j["max_count"] = p.max_count; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { j["type"] = "json_string"; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { j["type"] = "until"; j["delimiters"] = p.delimiters; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { j["type"] = "schema"; j["child"] = p.child; j["name"] = p.name; @@ -1448,16 +1448,16 @@ static nlohmann::json serialize_parser_variant(const common_chat_peg_parser_vari } else { j["schema"] = nullptr; } - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { j["type"] = "rule"; j["name"] = p.name; j["annotation"] = p.annotation; j["child"] = p.child; j["trigger"] = p.trigger; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { j["type"] = "ref"; j["name"] = p.name; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { j["type"] = "capture"; j["child"] = p.child; j["key"] = p.key; @@ -1467,7 +1467,7 @@ static nlohmann::json serialize_parser_variant(const common_chat_peg_parser_vari }, variant); } -nlohmann::json common_chat_peg_arena::to_json() const { +nlohmann::json common_peg_arena::to_json() const { nlohmann::json j; auto parsers = nlohmann::json::array(); @@ -1482,7 +1482,7 @@ nlohmann::json common_chat_peg_arena::to_json() const { } // Deserialization helper: convert JSON to parser variant -static common_chat_peg_parser_variant deserialize_parser_variant(const nlohmann::json & j) { +static common_peg_parser_variant deserialize_parser_variant(const nlohmann::json & j) { if (!j.contains("type") || !j["type"].is_string()) { throw std::runtime_error("Parser variant JSON missing or invalid 'type' field"); } @@ -1490,35 +1490,35 @@ static common_chat_peg_parser_variant deserialize_parser_variant(const nlohmann: std::string type = j["type"]; if (type == "start") { - return common_chat_peg_start_parser{}; + return common_peg_start_parser{}; } if (type == "end") { - return common_chat_peg_end_parser{}; + return common_peg_end_parser{}; } if (type == "literal") { if (!j.contains("literal") || !j["literal"].is_string()) { throw std::runtime_error("literal parser missing or invalid 'literal' field"); } - return common_chat_peg_literal_parser{j["literal"]}; + return common_peg_literal_parser{j["literal"]}; } if (type == "sequence") { if (!j.contains("children") || !j["children"].is_array()) { throw std::runtime_error("sequence parser missing or invalid 'children' field"); } - return common_chat_peg_sequence_parser{j["children"].get>()}; + return common_peg_sequence_parser{j["children"].get>()}; } if (type == "choice") { if (!j.contains("children") || !j["children"].is_array()) { throw std::runtime_error("choice parser missing or invalid 'children' field"); } - return common_chat_peg_choice_parser{j["children"].get>()}; + return common_peg_choice_parser{j["children"].get>()}; } if (type == "repetition") { if (!j.contains("child") || !j.contains("min_count") || !j.contains("max_count")) { throw std::runtime_error("repetition parser missing required fields"); } - return common_chat_peg_repetition_parser{ - j["child"].get(), + return common_peg_repetition_parser{ + j["child"].get(), j["min_count"].get(), j["max_count"].get() }; @@ -1527,26 +1527,26 @@ static common_chat_peg_parser_variant deserialize_parser_variant(const nlohmann: if (!j.contains("child")) { throw std::runtime_error("and parser missing 'child' field"); } - return common_chat_peg_and_parser{j["child"].get()}; + return common_peg_and_parser{j["child"].get()}; } if (type == "not") { if (!j.contains("child")) { throw std::runtime_error("not parser missing 'child' field"); } - return common_chat_peg_not_parser{j["child"].get()}; + return common_peg_not_parser{j["child"].get()}; } if (type == "any") { - return common_chat_peg_any_parser{}; + return common_peg_any_parser{}; } if (type == "space") { - return common_chat_peg_space_parser{}; + return common_peg_space_parser{}; } if (type == "chars") { if (!j.contains("pattern") || !j.contains("ranges") || !j.contains("negated") || !j.contains("min_count") || !j.contains("max_count")) { throw std::runtime_error("chars parser missing required fields"); } - common_chat_peg_chars_parser parser; + common_peg_chars_parser parser; parser.pattern = j["pattern"]; parser.negated = j["negated"]; parser.min_count = j["min_count"]; @@ -1563,20 +1563,20 @@ static common_chat_peg_parser_variant deserialize_parser_variant(const nlohmann: return parser; } if (type == "json_string") { - return common_chat_peg_json_string_parser{}; + return common_peg_json_string_parser{}; } if (type == "until") { if (!j.contains("delimiters") || !j["delimiters"].is_array()) { throw std::runtime_error("until parser missing or invalid 'delimiters' field"); } - return common_chat_peg_until_parser{j["delimiters"].get>()}; + return common_peg_until_parser{j["delimiters"].get>()}; } if (type == "schema") { if (!j.contains("child") || !j.contains("name") || !j.contains("schema")) { throw std::runtime_error("schema parser missing required fields"); } - common_chat_peg_schema_parser parser; - parser.child = j["child"].get(); + common_peg_schema_parser parser; + parser.child = j["child"].get(); parser.name = j["name"]; if (!j["schema"].is_null()) { parser.schema = std::make_shared(j["schema"]); @@ -1587,10 +1587,10 @@ static common_chat_peg_parser_variant deserialize_parser_variant(const nlohmann: if (!j.contains("name") || !j.contains("annotation") || !j.contains("child") || !j.contains("trigger")) { throw std::runtime_error("rule parser missing required fields"); } - return common_chat_peg_rule_parser{ + return common_peg_rule_parser{ j["name"].get(), j["annotation"].get(), - j["child"].get(), + j["child"].get(), j["trigger"].get() }; } @@ -1598,14 +1598,14 @@ static common_chat_peg_parser_variant deserialize_parser_variant(const nlohmann: if (!j.contains("name") || !j["name"].is_string()) { throw std::runtime_error("ref parser missing or invalid 'name' field"); } - return common_chat_peg_ref_parser{j["name"]}; + return common_peg_ref_parser{j["name"]}; } if (type == "capture") { if (!j.contains("child") || !j.contains("key")) { throw std::runtime_error("capture parser missing required fields"); } - return common_chat_peg_capture_parser{ - j["child"].get(), + return common_peg_capture_parser{ + j["child"].get(), j["key"].get() }; } @@ -1613,7 +1613,7 @@ static common_chat_peg_parser_variant deserialize_parser_variant(const nlohmann: throw std::runtime_error("Unknown parser type: " + type); } -common_chat_peg_arena common_chat_peg_arena::from_json(const nlohmann::json & j) { +common_peg_arena common_peg_arena::from_json(const nlohmann::json & j) { if (!j.contains("parsers") || !j["parsers"].is_array()) { throw std::runtime_error("JSON missing or invalid 'parsers' array"); } @@ -1624,7 +1624,7 @@ common_chat_peg_arena common_chat_peg_arena::from_json(const nlohmann::json & j) throw std::runtime_error("JSON missing 'root' field"); } - common_chat_peg_arena arena; + common_peg_arena arena; const auto & parsers_json = j["parsers"]; arena.parsers_.reserve(parsers_json.size()); @@ -1632,7 +1632,7 @@ common_chat_peg_arena common_chat_peg_arena::from_json(const nlohmann::json & j) arena.parsers_.push_back(deserialize_parser_variant(parser_json)); } - arena.rules_ = j["rules"].get>(); + arena.rules_ = j["rules"].get>(); for (const auto & [name, id] : arena.rules_) { if (id >= arena.parsers_.size()) { @@ -1640,8 +1640,8 @@ common_chat_peg_arena common_chat_peg_arena::from_json(const nlohmann::json & j) } } - arena.root_ = j["root"].get(); - if (arena.root_ != COMMON_CHAT_PEG_INVALID_PARSER_ID && arena.root_ >= arena.parsers_.size()) { + arena.root_ = j["root"].get(); + if (arena.root_ != COMMON_PEG_INVALID_PARSER_ID && arena.root_ >= arena.parsers_.size()) { throw std::runtime_error("Root references invalid parser ID: " + std::to_string(arena.root_)); } diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index 165b02fcbc697..792d08618c0e4 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -16,52 +16,52 @@ struct common_grammar_builder; // Forward declarations -using common_chat_peg_parser_id = size_t; -constexpr common_chat_peg_parser_id COMMON_CHAT_PEG_INVALID_PARSER_ID = static_cast(-1); +using common_peg_parser_id = size_t; +constexpr common_peg_parser_id COMMON_PEG_INVALID_PARSER_ID = static_cast(-1); // Forward declare builder for parser wrapper -class common_chat_peg_parser_builder; +class common_peg_parser_builder; -// Lightweight wrapper around common_chat_peg_parser_id that enables operator overloading +// Lightweight wrapper around common_peg_parser_id that enables operator overloading // and implicit conversions from strings/literals -class common_chat_peg_parser { - common_chat_peg_parser_id id_; - common_chat_peg_parser_builder * builder_; +class common_peg_parser { + common_peg_parser_id id_; + common_peg_parser_builder * builder_; public: - // Construct from common_chat_peg_parser_id - common_chat_peg_parser(common_chat_peg_parser_id id, common_chat_peg_parser_builder * builder) : id_(id), builder_(builder) {} + // Construct from common_peg_parser_id + common_peg_parser(common_peg_parser_id id, common_peg_parser_builder * builder) : id_(id), builder_(builder) {} - // Implicit conversion to common_chat_peg_parser_id - operator common_chat_peg_parser_id() const { return id_; } + // Implicit conversion to common_peg_parser_id + operator common_peg_parser_id() const { return id_; } // Get the underlying ID - common_chat_peg_parser_id id() const { return id_; } + common_peg_parser_id id() const { return id_; } // Get builder (for free function operators) - common_chat_peg_parser_builder * builder() const { return builder_; } + common_peg_parser_builder * builder() const { return builder_; } // Operator overloads - common_chat_peg_parser operator+(const common_chat_peg_parser & other) const; - common_chat_peg_parser operator|(const common_chat_peg_parser & other) const; - common_chat_peg_parser operator<<(const common_chat_peg_parser & other) const; // sequence with space + common_peg_parser operator+(const common_peg_parser & other) const; + common_peg_parser operator|(const common_peg_parser & other) const; + common_peg_parser operator<<(const common_peg_parser & other) const; // sequence with space // Overloads for string literals - common_chat_peg_parser operator+(const char * str) const; - common_chat_peg_parser operator+(const std::string & str) const; - common_chat_peg_parser operator|(const char * str) const; - common_chat_peg_parser operator|(const std::string & str) const; - common_chat_peg_parser operator<<(const char * str) const; - common_chat_peg_parser operator<<(const std::string & str) const; + common_peg_parser operator+(const char * str) const; + common_peg_parser operator+(const std::string & str) const; + common_peg_parser operator|(const char * str) const; + common_peg_parser operator|(const std::string & str) const; + common_peg_parser operator<<(const char * str) const; + common_peg_parser operator<<(const std::string & str) const; }; // Free function operators for string + parser -common_chat_peg_parser operator+(const char * str, const common_chat_peg_parser & p); -common_chat_peg_parser operator+(const std::string & str, const common_chat_peg_parser & p); -common_chat_peg_parser operator<<(const char * str, const common_chat_peg_parser & p); -common_chat_peg_parser operator<<(const std::string & str, const common_chat_peg_parser & p); +common_peg_parser operator+(const char * str, const common_peg_parser & p); +common_peg_parser operator+(const std::string & str, const common_peg_parser & p); +common_peg_parser operator<<(const char * str, const common_peg_parser & p); +common_peg_parser operator<<(const std::string & str, const common_peg_parser & p); -struct common_chat_parse_semantics { +struct common_peg_parse_semantics { std::string content; std::string reasoning_content; std::vector tool_calls; @@ -77,157 +77,157 @@ struct common_chat_parse_semantics { } }; -enum common_chat_parse_result_type { - COMMON_CHAT_PARSE_RESULT_FAIL = 0, - COMMON_CHAT_PARSE_RESULT_SUCCESS = 1, - COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT = 2, +enum common_peg_parse_result_type { + COMMON_PEG_PARSE_RESULT_FAIL = 0, + COMMON_PEG_PARSE_RESULT_SUCCESS = 1, + COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT = 2, }; -const char * common_chat_parse_result_type_name(common_chat_parse_result_type type); +const char * common_peg_parse_result_type_name(common_peg_parse_result_type type); -struct common_chat_parse_cache_key { - common_chat_peg_parser_id id; +struct common_peg_parse_cache_key { + common_peg_parser_id id; size_t start; - bool operator==(const common_chat_parse_cache_key & other) const { + bool operator==(const common_peg_parse_cache_key & other) const { return id == other.id && start == other.start; } }; template <> -struct std::hash { - std::size_t operator()(const common_chat_parse_cache_key & k) const { +struct std::hash { + std::size_t operator()(const common_peg_parse_cache_key & k) const { return std::hash{}((k.id << 32) | k.start); } }; -struct common_chat_parse_result { - common_chat_parse_result_type type = COMMON_CHAT_PARSE_RESULT_FAIL; +struct common_peg_parse_result { + common_peg_parse_result_type type = COMMON_PEG_PARSE_RESULT_FAIL; size_t start = 0; size_t end = 0; - common_chat_parse_result() : type(COMMON_CHAT_PARSE_RESULT_FAIL) {} + common_peg_parse_result() : type(COMMON_PEG_PARSE_RESULT_FAIL) {} - common_chat_parse_result(common_chat_parse_result_type type, size_t start) + common_peg_parse_result(common_peg_parse_result_type type, size_t start) : type(type), start(start), end(start) {} - common_chat_parse_result(common_chat_parse_result_type type, size_t start, size_t end) + common_peg_parse_result(common_peg_parse_result_type type, size_t start, size_t end) : type(type), start(start), end(end) {} - bool fail() const { return type == COMMON_CHAT_PARSE_RESULT_FAIL; } - bool need_more_input() const { return type == COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT; } - bool success() const { return type == COMMON_CHAT_PARSE_RESULT_SUCCESS; } + bool fail() const { return type == COMMON_PEG_PARSE_RESULT_FAIL; } + bool need_more_input() const { return type == COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT; } + bool success() const { return type == COMMON_PEG_PARSE_RESULT_SUCCESS; } }; -enum common_chat_parse_event_type { - COMMON_CHAT_PARSE_EVENT_NODE_START, - COMMON_CHAT_PARSE_EVENT_NODE_END, +enum common_peg_parse_event_type { + COMMON_PEG_PARSE_EVENT_NODE_START, + COMMON_PEG_PARSE_EVENT_NODE_END, }; -struct common_chat_parse_event { - common_chat_parse_event_type type; +struct common_peg_parse_event { + common_peg_parse_event_type type; std::string rule; std::string annotation; size_t start; size_t end; std::string_view text; - common_chat_parse_result_type status; + common_peg_parse_result_type status; int depth; - bool starting() const { return type == COMMON_CHAT_PARSE_EVENT_NODE_START; } - bool ending() const { return type == COMMON_CHAT_PARSE_EVENT_NODE_END; } + bool starting() const { return type == COMMON_PEG_PARSE_EVENT_NODE_START; } + bool ending() const { return type == COMMON_PEG_PARSE_EVENT_NODE_END; } - bool success() const { return status == COMMON_CHAT_PARSE_RESULT_SUCCESS; } - bool need_more_input() const { return status == COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT; } - bool fail() const { return status == COMMON_CHAT_PARSE_RESULT_FAIL; } + bool success() const { return status == COMMON_PEG_PARSE_RESULT_SUCCESS; } + bool need_more_input() const { return status == COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT; } + bool fail() const { return status == COMMON_PEG_PARSE_RESULT_FAIL; } }; -using common_chat_parse_event_handler = std::function; +using common_peg_parse_event_handler = std::function; -class common_chat_parse_cache { - std::unordered_map results; +class common_peg_parse_cache { + std::unordered_map results; public: - common_chat_parse_result set(common_chat_peg_parser_id id, size_t start, common_chat_parse_result result); - std::optional get(common_chat_peg_parser_id id, size_t start); + common_peg_parse_result set(common_peg_parser_id id, size_t start, common_peg_parse_result result); + std::optional get(common_peg_parser_id id, size_t start); void clear(); }; -struct common_chat_parse_context { +struct common_peg_parse_context { std::string input; bool input_is_complete; - common_chat_parse_cache cache; - common_chat_parse_semantics * semantics; - common_chat_parse_event_handler event_handler; + common_peg_parse_cache cache; + common_peg_parse_semantics * semantics; + common_peg_parse_event_handler event_handler; int current_depth; int parse_depth; - common_chat_parse_context() + common_peg_parse_context() : input_is_complete(true), cache(), semantics(nullptr), event_handler(nullptr), current_depth(0), parse_depth(0) {} - common_chat_parse_context(const std::string & input) + common_peg_parse_context(const std::string & input) : input(input), input_is_complete(true), cache(), semantics(nullptr), event_handler(nullptr), current_depth(0), parse_depth(0) {} - common_chat_parse_context(const std::string & input, bool complete) + common_peg_parse_context(const std::string & input, bool complete) : input(input), input_is_complete(complete), cache(), semantics(nullptr), event_handler(nullptr), current_depth(0), parse_depth(0) {} - common_chat_parse_context(const std::string & input, common_chat_parse_semantics * semantics) + common_peg_parse_context(const std::string & input, common_peg_parse_semantics * semantics) : input(input), input_is_complete(true), cache(), semantics(semantics), event_handler(nullptr), current_depth(0), parse_depth(0) {} - common_chat_parse_context(const std::string & input, common_chat_parse_semantics * semantics, bool complete) + common_peg_parse_context(const std::string & input, common_peg_parse_semantics * semantics, bool complete) : input(input), input_is_complete(complete), cache(), semantics(semantics), event_handler(nullptr), current_depth(0), parse_depth(0) {} - common_chat_parse_context(const std::string & input, common_chat_parse_semantics * semantics, common_chat_parse_event_handler handler, bool complete = true) + common_peg_parse_context(const std::string & input, common_peg_parse_semantics * semantics, common_peg_parse_event_handler handler, bool complete = true) : input(input), input_is_complete(complete), cache(), semantics(semantics), event_handler(std::move(handler)), current_depth(0), parse_depth(0) {} template void set_event_handler(const T & handler) { - event_handler = [&](const common_chat_parse_event & ev, common_chat_parse_semantics & semantics) { + event_handler = [&](const common_peg_parse_event & ev, common_peg_parse_semantics & semantics) { handler(ev, semantics); }; } }; // Forward declaration -class common_chat_peg_arena; +class common_peg_arena; // Parser variant structs (value-based, no inheritance) -struct common_chat_peg_start_parser {}; +struct common_peg_start_parser {}; -struct common_chat_peg_end_parser {}; +struct common_peg_end_parser {}; -struct common_chat_peg_literal_parser { +struct common_peg_literal_parser { std::string literal; }; -struct common_chat_peg_sequence_parser { - std::vector children; +struct common_peg_sequence_parser { + std::vector children; }; -struct common_chat_peg_choice_parser { - std::vector children; +struct common_peg_choice_parser { + std::vector children; }; -struct common_chat_peg_repetition_parser { - common_chat_peg_parser_id child; +struct common_peg_repetition_parser { + common_peg_parser_id child; int min_count; int max_count; // -1 for unbounded }; -struct common_chat_peg_and_parser { - common_chat_peg_parser_id child; +struct common_peg_and_parser { + common_peg_parser_id child; }; -struct common_chat_peg_not_parser { - common_chat_peg_parser_id child; +struct common_peg_not_parser { + common_peg_parser_id child; }; -struct common_chat_peg_any_parser {}; +struct common_peg_any_parser {}; -struct common_chat_peg_space_parser {}; +struct common_peg_space_parser {}; -struct common_chat_peg_chars_parser { +struct common_peg_chars_parser { struct char_range { uint32_t start; uint32_t end; @@ -241,218 +241,218 @@ struct common_chat_peg_chars_parser { int max_count; // -1 for unbounded }; -struct common_chat_peg_json_string_parser {}; +struct common_peg_json_string_parser {}; -struct common_chat_peg_until_parser { +struct common_peg_until_parser { std::vector delimiters; }; -struct common_chat_peg_schema_parser { - common_chat_peg_parser_id child; +struct common_peg_schema_parser { + common_peg_parser_id child; std::string name; std::shared_ptr schema; }; -struct common_chat_peg_rule_parser { +struct common_peg_rule_parser { std::string name; std::string annotation; - common_chat_peg_parser_id child; + common_peg_parser_id child; bool trigger; }; -struct common_chat_peg_ref_parser { +struct common_peg_ref_parser { std::string name; }; -struct common_chat_peg_capture_parser { - common_chat_peg_parser_id child; +struct common_peg_capture_parser { + common_peg_parser_id child; std::string key; }; // Variant holding all parser types -using common_chat_peg_parser_variant = std::variant< - common_chat_peg_start_parser, - common_chat_peg_end_parser, - common_chat_peg_literal_parser, - common_chat_peg_sequence_parser, - common_chat_peg_choice_parser, - common_chat_peg_repetition_parser, - common_chat_peg_and_parser, - common_chat_peg_not_parser, - common_chat_peg_any_parser, - common_chat_peg_space_parser, - common_chat_peg_chars_parser, - common_chat_peg_json_string_parser, - common_chat_peg_until_parser, - common_chat_peg_schema_parser, - common_chat_peg_rule_parser, - common_chat_peg_ref_parser, - common_chat_peg_capture_parser +using common_peg_parser_variant = std::variant< + common_peg_start_parser, + common_peg_end_parser, + common_peg_literal_parser, + common_peg_sequence_parser, + common_peg_choice_parser, + common_peg_repetition_parser, + common_peg_and_parser, + common_peg_not_parser, + common_peg_any_parser, + common_peg_space_parser, + common_peg_chars_parser, + common_peg_json_string_parser, + common_peg_until_parser, + common_peg_schema_parser, + common_peg_rule_parser, + common_peg_ref_parser, + common_peg_capture_parser >; // Arena owns all parsers -class common_chat_peg_arena { - std::vector parsers_; - std::unordered_map rules_; - common_chat_peg_parser_id root_; +class common_peg_arena { + std::vector parsers_; + std::unordered_map rules_; + common_peg_parser_id root_; public: - common_chat_peg_arena(); + common_peg_arena(); // Access - const common_chat_peg_parser_variant & get(common_chat_peg_parser_id id) const { return parsers_.at(id); } - common_chat_peg_parser_variant & get(common_chat_peg_parser_id id) { return parsers_.at(id); } + const common_peg_parser_variant & get(common_peg_parser_id id) const { return parsers_.at(id); } + common_peg_parser_variant & get(common_peg_parser_id id) { return parsers_.at(id); } size_t size() const { return parsers_.size(); } // Rule lookup - common_chat_peg_parser_id get_rule(const std::string & name) const; + common_peg_parser_id get_rule(const std::string & name) const; bool has_rule(const std::string & name) const { return rules_.find(name) != rules_.end(); } // Root - common_chat_peg_parser_id root() const { return root_; } - void set_root(common_chat_peg_parser_id id) { root_ = id; } + common_peg_parser_id root() const { return root_; } + void set_root(common_peg_parser_id id) { root_ = id; } // Parse - common_chat_parse_result parse(common_chat_parse_context & ctx, size_t start = 0) const; - common_chat_parse_result parse(common_chat_peg_parser_id id, common_chat_parse_context & ctx, size_t start) const; + common_peg_parse_result parse(common_peg_parse_context & ctx, size_t start = 0) const; + common_peg_parse_result parse(common_peg_parser_id id, common_peg_parse_context & ctx, size_t start) const; // Grammar generation void build_grammar(const common_grammar_builder & builder, bool lazy = false) const; // Dump for debugging - std::string dump(common_chat_peg_parser_id id) const; + std::string dump(common_peg_parser_id id) const; // Serialization nlohmann::json to_json() const; - static common_chat_peg_arena from_json(const nlohmann::json & j); + static common_peg_arena from_json(const nlohmann::json & j); // Builder access (for adding parsers) - friend class common_chat_peg_parser_builder; + friend class common_peg_parser_builder; private: - common_chat_peg_parser_id add_parser(common_chat_peg_parser_variant parser); - void add_rule(const std::string & name, common_chat_peg_parser_id id); + common_peg_parser_id add_parser(common_peg_parser_variant parser); + void add_rule(const std::string & name, common_peg_parser_id id); }; // Builder for constructing parsers -class common_chat_peg_parser_builder { - common_chat_peg_arena arena_; +class common_peg_parser_builder { + common_peg_arena arena_; - // Helper to wrap common_chat_peg_parser_id with this builder - common_chat_peg_parser wrap(common_chat_peg_parser_id id) { return common_chat_peg_parser(id, this); } + // Helper to wrap common_peg_parser_id with this builder + common_peg_parser wrap(common_peg_parser_id id) { return common_peg_parser(id, this); } public: - common_chat_peg_parser_builder(); + common_peg_parser_builder(); // Matches the start of the input. // S -> ^ - common_chat_peg_parser start(); + common_peg_parser start(); // Matches the end of the input. // S -> $ - common_chat_peg_parser end(); + common_peg_parser end(); // Matches an exact literal string. // S -> "hello" - common_chat_peg_parser literal(const std::string & literal); + common_peg_parser literal(const std::string & literal); // Matches a sequence of parsers in order, all must succeed. // S -> A B C - common_chat_peg_parser sequence(const std::vector & parsers); - common_chat_peg_parser sequence(const std::vector & parsers); - common_chat_peg_parser sequence(std::initializer_list parsers); + common_peg_parser sequence(const std::vector & parsers); + common_peg_parser sequence(const std::vector & parsers); + common_peg_parser sequence(std::initializer_list parsers); // Matches the first parser that succeeds from a list of alternatives. // S -> A | B | C - common_chat_peg_parser choice(const std::vector & parsers); - common_chat_peg_parser choice(const std::vector & parsers); - common_chat_peg_parser choice(std::initializer_list parsers); + common_peg_parser choice(const std::vector & parsers); + common_peg_parser choice(const std::vector & parsers); + common_peg_parser choice(std::initializer_list parsers); // Matches one or more repetitions of a parser. // S -> A+ - common_chat_peg_parser one_or_more(common_chat_peg_parser p); + common_peg_parser one_or_more(common_peg_parser p); // Matches zero or more repetitions of a parser, always succeeds. // S -> A* - common_chat_peg_parser zero_or_more(common_chat_peg_parser p); + common_peg_parser zero_or_more(common_peg_parser p); // Matches zero or one occurrence of a parser, always succeeds. // S -> A? - common_chat_peg_parser optional(common_chat_peg_parser p); + common_peg_parser optional(common_peg_parser p); // Positive lookahead: succeeds if child parser succeeds, consumes no input. // S -> &A - common_chat_peg_parser peek(common_chat_peg_parser p); + common_peg_parser peek(common_peg_parser p); // Negative lookahead: succeeds if child parser fails, consumes no input. // S -> !A - common_chat_peg_parser negate(common_chat_peg_parser p); + common_peg_parser negate(common_peg_parser p); // Matches any single character. // S -> . - common_chat_peg_parser any(); + common_peg_parser any(); // Matches between min and max repetitions of characters from a character class. // S -> [a-z]{m,n} // // Use -1 for max to represent unbounded repetition (equivalent to {m,}) - common_chat_peg_parser chars(const std::string & classes, int min = 1, int max = -1); + common_peg_parser chars(const std::string & classes, int min = 1, int max = -1); // Matches a single character from a character class or range. // S -> [a-z] or S -> [^0-9] // // Equivalent to chars(classes, 1, 1) - common_chat_peg_parser one(const std::string & classes); + common_peg_parser one(const std::string & classes); // Creates a lightweight reference to a named rule (resolved during build()). // Use this for forward references in recursive grammars. // expr_ref -> expr - common_chat_peg_parser ref(const std::string & name); + common_peg_parser ref(const std::string & name); // Matches zero or more whitespace characters (space, tab, newline). // S -> [ \t\n]* - common_chat_peg_parser space(); + common_peg_parser space(); // Matches all characters until a delimiter is found (delimiter not consumed). // S -> (!delim .)* - common_chat_peg_parser until(const std::string & delimiter); - common_chat_peg_parser until_one_of(const std::vector & delimiters); + common_peg_parser until(const std::string & delimiter); + common_peg_parser until_one_of(const std::vector & delimiters); // Matches between min and max repetitions of a parser (inclusive). // S -> A{m,n} // Use -1 for max to represent unbounded repetition (equivalent to {m,}) - common_chat_peg_parser repeat(common_chat_peg_parser p, int min, int max); + common_peg_parser repeat(common_peg_parser p, int min, int max); // Matches exactly n repetitions of a parser. // S -> A{n} - common_chat_peg_parser repeat(common_chat_peg_parser p, int n); + common_peg_parser repeat(common_peg_parser p, int n); // Creates a complete JSON parser supporting objects, arrays, strings, numbers, booleans, and null. // value -> object | array | string | number | true | false | null - common_chat_peg_parser json(); - common_chat_peg_parser json_object(); - common_chat_peg_parser json_string(); - common_chat_peg_parser json_array(); - common_chat_peg_parser json_number(); - common_chat_peg_parser json_bool(); - common_chat_peg_parser json_null(); + common_peg_parser json(); + common_peg_parser json_object(); + common_peg_parser json_string(); + common_peg_parser json_array(); + common_peg_parser json_number(); + common_peg_parser json_bool(); + common_peg_parser json_null(); // Specialized single-pass JSON string parser with escape sequence handling - common_chat_peg_parser json_string_content(); + common_peg_parser json_string_content(); // Wraps a parser with JSON schema metadata for grammar generation. // Used internally to convert JSON schemas to GBNF grammar rules. - common_chat_peg_parser schema(common_chat_peg_parser p, const std::string & name, const nlohmann::ordered_json & schema); + common_peg_parser schema(common_peg_parser p, const std::string & name, const nlohmann::ordered_json & schema); // Captures matched text to semantics.captures[key] - common_chat_peg_parser capture(const std::string & key, common_chat_peg_parser p); + common_peg_parser capture(const std::string & key, common_peg_parser p); // Creates a named rule, stores it in the grammar, and returns a reference to it. // If trigger=true, marks this rule as an entry point for lazy grammar generation. // auto json = p.rule("json", json_obj | json_arr | ...) - common_chat_peg_parser rule(const std::string & name, common_chat_peg_parser p, bool trigger = false); - common_chat_peg_parser rule(const std::string & name, const std::string & annotation, common_chat_peg_parser p, bool trigger = false); + common_peg_parser rule(const std::string & name, common_peg_parser p, bool trigger = false); + common_peg_parser rule(const std::string & name, const std::string & annotation, common_peg_parser p, bool trigger = false); // Creates a named rule using a builder function. This handles recursive grammars by // inserting a placeholder rule before invoking the builder, allowing the @@ -460,18 +460,18 @@ class common_chat_peg_parser_builder { // definition needs to call back to itself (directly or indirectly). // If trigger=true, marks this rule as an entry point for lazy grammar generation. // auto json = p.rule("json", [&]() { return json_object() | json_array() | ... }) - common_chat_peg_parser rule(const std::string & name, const std::function & builder, bool trigger = false); - common_chat_peg_parser rule(const std::string & name, const std::string & annotation, const std::function & builder, bool trigger = false); + common_peg_parser rule(const std::string & name, const std::function & builder, bool trigger = false); + common_peg_parser rule(const std::string & name, const std::string & annotation, const std::function & builder, bool trigger = false); - void set_root(common_chat_peg_parser p); + void set_root(common_peg_parser p); - common_chat_peg_arena build(); + common_peg_arena build(); }; // Helper function for building parsers template -common_chat_peg_arena build_peg_parser(F && fn) { - common_chat_peg_parser_builder builder; +common_peg_arena build_peg_parser(F && fn) { + common_peg_parser_builder builder; auto root = fn(builder); builder.set_root(root); return builder.build(); diff --git a/tests/chat-peg-parser/test-command7-parser-compare.cpp b/tests/chat-peg-parser/test-command7-parser-compare.cpp index 078f5cb156e88..ccfcb9ea6a336 100644 --- a/tests/chat-peg-parser/test-command7-parser-compare.cpp +++ b/tests/chat-peg-parser/test-command7-parser-compare.cpp @@ -7,8 +7,8 @@ #include #include -static common_chat_peg_arena create_command_r7b_parser() { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { +static common_peg_arena create_command_r7b_parser() { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { auto thinking = p.rule("thinking", "<|START_THINKING|>" << p.rule("reasoning-content", p.until("<|END_THINKING|>")) << "<|END_THINKING|>"); @@ -44,8 +44,8 @@ static common_chat_peg_arena create_command_r7b_parser() { return parser; } -static common_chat_parse_event_handler create_command_r7b_event_handler() { - return [](const common_chat_parse_event & ev, common_chat_parse_semantics & semantics) { +static common_peg_parse_event_handler create_command_r7b_event_handler() { + return [](const common_peg_parse_event & ev, common_peg_parse_semantics & semantics) { if (ev.rule == "reasoning-content" && ev.ending()) { semantics.reasoning_content = ev.text; } @@ -75,12 +75,12 @@ static common_chat_parse_event_handler create_command_r7b_event_handler() { }; } -static void test_command_r7b_parser(const common_chat_peg_arena & p, +static void test_command_r7b_parser(const common_peg_arena & p, const std::string & input, bool need_more_input, bool print_results) { - common_chat_parse_semantics semantics; - common_chat_parse_context ctx(input, &semantics, !need_more_input); + common_peg_parse_semantics semantics; + common_peg_parse_context ctx(input, &semantics, !need_more_input); p.parse(ctx); if (print_results) { diff --git a/tests/chat-peg-parser/test-example-minimax-m2.cpp b/tests/chat-peg-parser/test-example-minimax-m2.cpp index f254feb392587..52c39e2267632 100644 --- a/tests/chat-peg-parser/test-example-minimax-m2.cpp +++ b/tests/chat-peg-parser/test-example-minimax-m2.cpp @@ -7,7 +7,7 @@ #include void test_example_minimax_m2(testing &t) { - auto helper_parser = build_peg_parser_helper([](common_chat_peg_parser_builder_helper & p) { + auto helper_parser = build_peg_parser_helper([](common_peg_parser_builder_helper & p) { auto thinking = p.reasoning(); auto content = p.content_before_tools(""); auto function = p.quasi_xml_attr("generate_joke", @@ -35,16 +35,16 @@ void test_example_minimax_m2(testing &t) { // t.log("Tokens: " + string_join(tokens, ", ")); common_chat_msg prev; - common_chat_parse_result last_result; + common_peg_parse_result last_result; t.test("helper_builder", [&](testing &t) { for (auto it = tokens.begin(); it != tokens.end(); it++) { std::string in = std::accumulate(tokens.begin(), it + 1, std::string()); // t.log("Current input: " + in); - common_chat_parse_semantics semantics; - common_chat_parse_context ctx(in, &semantics, it + 1 == tokens.end()); + common_peg_parse_semantics semantics; + common_peg_parse_context ctx(in, &semantics, it + 1 == tokens.end()); - common_chat_parse_simple_handler handler; + common_peg_parse_simple_handler handler; ctx.set_event_handler(handler); auto result = helper_parser.parse(ctx); diff --git a/tests/chat-peg-parser/test-example-qwen3-coder.cpp b/tests/chat-peg-parser/test-example-qwen3-coder.cpp index 8fc3e73fa5177..9b078f8ff2268 100644 --- a/tests/chat-peg-parser/test-example-qwen3-coder.cpp +++ b/tests/chat-peg-parser/test-example-qwen3-coder.cpp @@ -5,7 +5,7 @@ #include void test_example_qwen3_coder(testing &t) { - auto explicit_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto explicit_parser = build_peg_parser([](common_peg_parser_builder & p) { auto thinking = p.rule("raw-reasoning", "" << p.rule("reasoning-content", p.until("")) << ""); @@ -35,7 +35,7 @@ void test_example_qwen3_coder(testing &t) { }); - auto helper_parser = build_peg_parser_helper([](common_chat_peg_parser_builder_helper & p) { + auto helper_parser = build_peg_parser_helper([](common_peg_parser_builder_helper & p) { auto thinking = p.reasoning(); auto content = p.content_before_tools(""); auto function = p.quasi_xml_no_attr("search_files", @@ -78,10 +78,10 @@ void test_example_qwen3_coder(testing &t) { for (auto it = tokens.begin(); it != tokens.end(); it++) { std::string in = std::accumulate(tokens.begin(), it + 1, std::string()); - common_chat_parse_semantics semantics; - common_chat_parse_context ctx(in, &semantics, it == tokens.end() - 1); + common_peg_parse_semantics semantics; + common_peg_parse_context ctx(in, &semantics, it == tokens.end() - 1); - common_chat_parse_simple_handler handler; + common_peg_parse_simple_handler handler; // handler.log = [&](const std::string & msg) { // t.log(msg); // }; @@ -112,10 +112,10 @@ void test_example_qwen3_coder(testing &t) { for (auto it = tokens.begin(); it != tokens.end(); it++) { std::string in = std::accumulate(tokens.begin(), it + 1, std::string()); - common_chat_parse_semantics semantics; - common_chat_parse_context ctx(in, &semantics, it + 1 == tokens.end()); + common_peg_parse_semantics semantics; + common_peg_parse_context ctx(in, &semantics, it + 1 == tokens.end()); - common_chat_parse_simple_handler handler; + common_peg_parse_simple_handler handler; ctx.set_event_handler(handler); auto result = helper_parser.parse(ctx); diff --git a/tests/chat-peg-parser/test-example-seed-oss.cpp b/tests/chat-peg-parser/test-example-seed-oss.cpp index 709c1e8b4c15d..d789fc312c086 100644 --- a/tests/chat-peg-parser/test-example-seed-oss.cpp +++ b/tests/chat-peg-parser/test-example-seed-oss.cpp @@ -4,7 +4,7 @@ #include void test_example_seed_oss(testing &t) { - auto helper_parser = build_peg_parser_helper([](common_chat_peg_parser_builder_helper & p) { + auto helper_parser = build_peg_parser_helper([](common_peg_parser_builder_helper & p) { auto thinking = p.reasoning("seed:think"); auto content = p.content_before_tools(""); auto function = p.quasi_xml_no_attr("get_weather", @@ -34,10 +34,10 @@ void test_example_seed_oss(testing &t) { for (auto it = tokens.begin(); it != tokens.end(); it++) { std::string in = std::accumulate(tokens.begin(), it + 1, std::string()); - common_chat_parse_semantics semantics; - common_chat_parse_context ctx(in, &semantics, it == tokens.end()); + common_peg_parse_semantics semantics; + common_peg_parse_context ctx(in, &semantics, it == tokens.end()); - common_chat_parse_simple_handler handler; + common_peg_parse_simple_handler handler; ctx.set_event_handler(handler); auto result = helper_parser.parse(ctx); diff --git a/tests/chat-peg-parser/test-gbnf-generation.cpp b/tests/chat-peg-parser/test-gbnf-generation.cpp index 655738aecb795..7623d85f86168 100644 --- a/tests/chat-peg-parser/test-gbnf-generation.cpp +++ b/tests/chat-peg-parser/test-gbnf-generation.cpp @@ -15,7 +15,7 @@ static void assert_gbnf_equal(testing & t, const std::string & expected, const s void test_gbnf_generation(testing &t) { t.test("literal grammar generation", [](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("hello"); }); @@ -30,7 +30,7 @@ void test_gbnf_generation(testing &t) { }); t.test("char class grammar", [](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[a-z]"); }); @@ -45,7 +45,7 @@ void test_gbnf_generation(testing &t) { }); t.test("sequence grammar", [](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("hello") + p.literal(" ") + p.literal("world"); }); @@ -60,7 +60,7 @@ void test_gbnf_generation(testing &t) { }); t.test("choice grammar", [](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("cat") | p.literal("dog"); }); @@ -75,7 +75,7 @@ void test_gbnf_generation(testing &t) { }); t.test("one_or_more grammar", [](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.one("[0-9]")); }); @@ -90,7 +90,7 @@ void test_gbnf_generation(testing &t) { }); t.test("zero_or_more grammar", [](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.one("[a-z]")); }); @@ -105,7 +105,7 @@ void test_gbnf_generation(testing &t) { }); t.test("optional grammar", [](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); }); @@ -120,7 +120,7 @@ void test_gbnf_generation(testing &t) { }); t.test("until grammar", [](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.until(""); }); @@ -135,7 +135,7 @@ void test_gbnf_generation(testing &t) { }); t.test("complex expressions with parentheses", [](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("a") | p.literal("b")); }); @@ -150,7 +150,7 @@ void test_gbnf_generation(testing &t) { }); t.test("rule references", [](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { auto digit = p.rule("digit", p.one("[0-9]")); return p.one_or_more(digit); }); @@ -167,7 +167,7 @@ void test_gbnf_generation(testing &t) { }); t.test("escaping in literals", [](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("hello\nworld\t!"); }); @@ -182,7 +182,7 @@ void test_gbnf_generation(testing &t) { }); t.test("operator<< (whitespace insertion)", [](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("hello") << p.literal("world"); }); @@ -197,7 +197,7 @@ void test_gbnf_generation(testing &t) { }); t.test("emit only reachable rules", [](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { p.rule("orphan", p.literal("orphan")); return p.literal("hello") + p.rule("child", p.literal(" world")); }); @@ -214,7 +214,7 @@ void test_gbnf_generation(testing &t) { }); t.test("emit only trigger rules (and references)", [](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { auto rule1 = p.rule("rule-1", p.literal("a") + p.ref("rule-2")); p.rule("rule-2", p.literal("b") + p.ref("rule-3"), true); p.rule("rule-3", p.literal("c") + p.ref("rule-4")); diff --git a/tests/chat-peg-parser/test-json-parser.cpp b/tests/chat-peg-parser/test-json-parser.cpp index 935ada3ad92f3..096bcde90ddc1 100644 --- a/tests/chat-peg-parser/test-json-parser.cpp +++ b/tests/chat-peg-parser/test-json-parser.cpp @@ -3,10 +3,10 @@ void test_json_parser(testing &t) { // Test parsing a simple JSON object t.test("simple JSON object parsing", [](testing &t) { - auto json = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.json(); }); + auto json = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); }); std::string input = R"({"name": "test", "value": 42, "flag": true})"; - common_chat_parse_context ctx(input); + common_peg_parse_context ctx(input); auto result = json.parse(ctx); @@ -16,10 +16,10 @@ void test_json_parser(testing &t) { // Test parsing a JSON array with mixed types t.test("JSON array with mixed types", [](testing &t) { - auto json = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.json(); }); + auto json = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); }); std::string input = R"([1, "hello", true, null, 3.14])"; - common_chat_parse_context ctx(input); + common_peg_parse_context ctx(input); auto result = json.parse(ctx); @@ -29,11 +29,11 @@ void test_json_parser(testing &t) { // Test parsing nested JSON with objects and arrays t.test("nested JSON with objects and arrays", [](testing &t) { - auto json = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.json(); }); + auto json = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); }); std::string input = R"({"users": [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}], "count": 2, "metadata": {"version": "1.0", "tags": ["admin", "user"]}})"; - common_chat_parse_context ctx(input); + common_peg_parse_context ctx(input); auto result = json.parse(ctx); @@ -43,10 +43,10 @@ void test_json_parser(testing &t) { // Test need_more_input() parsing - incomplete object t.test("need_more_input() parsing - incomplete object", [](testing &t) { - auto json = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.json(); }); + auto json = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); }); std::string input = R"({"name": "test", "value": )"; - common_chat_parse_context ctx(input, false); + common_peg_parse_context ctx(input, false); auto result = json.parse(ctx); @@ -55,10 +55,10 @@ void test_json_parser(testing &t) { // Test need_more_input() parsing - incomplete array t.test("need_more_input() parsing - incomplete array", [](testing &t) { - auto json = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.json(); }); + auto json = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); }); std::string input = R"([1, 2, 3, )"; - common_chat_parse_context ctx(input, false); + common_peg_parse_context ctx(input, false); auto result = json.parse(ctx); @@ -67,10 +67,10 @@ void test_json_parser(testing &t) { // Test need_more_input() parsing - incomplete nested structure t.test("need_more_input() parsing - incomplete nested structure", [](testing &t) { - auto json = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.json(); }); + auto json = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); }); std::string input = R"({"data": {"nested": )"; - common_chat_parse_context ctx(input, false); + common_peg_parse_context ctx(input, false); auto result = json.parse(ctx); diff --git a/tests/chat-peg-parser/test-json-serialization.cpp b/tests/chat-peg-parser/test-json-serialization.cpp index 487f346347af7..a85801060c0b5 100644 --- a/tests/chat-peg-parser/test-json-serialization.cpp +++ b/tests/chat-peg-parser/test-json-serialization.cpp @@ -1,19 +1,19 @@ #include "tests.h" void test_json_serialization(testing &t) { - auto original = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto original = build_peg_parser([](common_peg_parser_builder & p) { return "" + p.json() + ""; }); auto json_serialized = original.to_json().dump(); t.test("compare before/after", [&](testing &t) { - auto deserialized = common_chat_peg_arena::from_json(nlohmann::json::parse(json_serialized)); + auto deserialized = common_peg_arena::from_json(nlohmann::json::parse(json_serialized)); // Test complex JSON std::string input = R"({"name": "test", "values": [1, 2, 3], "nested": {"a": true}})"; - common_chat_parse_context ctx1(input); - common_chat_parse_context ctx2(input); + common_peg_parse_context ctx1(input); + common_peg_parse_context ctx2(input); auto result1 = original.parse(ctx1); auto result2 = deserialized.parse(ctx2); @@ -23,6 +23,6 @@ void test_json_serialization(testing &t) { }); t.bench("deserialize", [&]() { - auto deserialized = common_chat_peg_arena::from_json(nlohmann::json::parse(json_serialized)); + auto deserialized = common_peg_arena::from_json(nlohmann::json::parse(json_serialized)); }, 100); } diff --git a/tests/chat-peg-parser/test-one.cpp b/tests/chat-peg-parser/test-one.cpp index 5980f76024b77..a2cdfe78eb2a6 100644 --- a/tests/chat-peg-parser/test-one.cpp +++ b/tests/chat-peg-parser/test-one.cpp @@ -3,96 +3,96 @@ void test_one(testing &t) { // Test common escape sequences - newline t.test("escape_sequence_newline", [](testing &t) { - auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); + auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); - common_chat_parse_context ctx; - common_chat_parse_result result; + common_peg_parse_context ctx; + common_peg_parse_result result; - ctx = common_chat_parse_context("\n"); + ctx = common_peg_parse_context("\n"); result = common_chat_combinator_parser.parse(ctx); t.assert_equal("escape_sequence_newline", true, result.success()); }); // Test common escape sequences - tab t.test("escape_sequence_tab", [](testing &t) { - auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); + auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); - common_chat_parse_context ctx; - common_chat_parse_result result; + common_peg_parse_context ctx; + common_peg_parse_result result; - ctx = common_chat_parse_context("\t"); + ctx = common_peg_parse_context("\t"); result = common_chat_combinator_parser.parse(ctx); t.assert_equal("escape_sequence_tab", true, result.success()); }); // Test common escape sequences - backslash t.test("escape_sequence_backslash", [](testing &t) { - auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); + auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); - common_chat_parse_context ctx; - common_chat_parse_result result; + common_peg_parse_context ctx; + common_peg_parse_result result; - ctx = common_chat_parse_context("\\"); + ctx = common_peg_parse_context("\\"); result = common_chat_combinator_parser.parse(ctx); t.assert_equal("escape_sequence_backslash", true, result.success()); }); // Test common escape sequences - space (should ()) t.test("escape_sequence_space_fail", [](testing &t) { - auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); + auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); - common_chat_parse_context ctx; - common_chat_parse_result result; + common_peg_parse_context ctx; + common_peg_parse_result result; - ctx = common_chat_parse_context(" "); + ctx = common_peg_parse_context(" "); result = common_chat_combinator_parser.parse(ctx); t.assert_equal("escape_sequence_space_fail", true, result.fail()); }); // Test escaped dash - 'a' should succeed t.test("escaped_dash_a", [](testing &t) { - auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[a\\-z]"); }); + auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[a\\-z]"); }); - common_chat_parse_context ctx; - common_chat_parse_result result; + common_peg_parse_context ctx; + common_peg_parse_result result; - ctx = common_chat_parse_context("a"); + ctx = common_peg_parse_context("a"); result = common_chat_combinator_parser.parse(ctx); t.assert_equal("escaped_dash_a", true, result.success()); }); // Test escaped dash - '-' should succeed (literal dash) t.test("escaped_dash_literal", [](testing &t) { - auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[a\\-z]"); }); + auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[a\\-z]"); }); - common_chat_parse_context ctx; - common_chat_parse_result result; + common_peg_parse_context ctx; + common_peg_parse_result result; - ctx = common_chat_parse_context("-"); + ctx = common_peg_parse_context("-"); result = common_chat_combinator_parser.parse(ctx); t.assert_equal("escaped_dash_literal", true, result.success()); }); // Test escaped dash - 'z' should succeed t.test("escaped_dash_z", [](testing &t) { - auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[a\\-z]"); }); + auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[a\\-z]"); }); - common_chat_parse_context ctx; - common_chat_parse_result result; + common_peg_parse_context ctx; + common_peg_parse_result result; - ctx = common_chat_parse_context("z"); + ctx = common_peg_parse_context("z"); result = common_chat_combinator_parser.parse(ctx); t.assert_equal("escaped_dash_z", true, result.success()); }); // Test escaped dash - 'b' should NOT match (since \- is literal dash, not range) t.test("escaped_dash_b_fail", [](testing &t) { - auto common_chat_combinator_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("[a\\-z]"); }); + auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[a\\-z]"); }); - common_chat_parse_context ctx; - common_chat_parse_result result; + common_peg_parse_context ctx; + common_peg_parse_result result; - ctx = common_chat_parse_context("b"); + ctx = common_peg_parse_context("b"); result = common_chat_combinator_parser.parse(ctx); t.assert_equal("escaped_dash_b_fail", true, result.fail()); }); diff --git a/tests/chat-peg-parser/test-optional.cpp b/tests/chat-peg-parser/test-optional.cpp index ea8237aca3309..e47bb4b118e1a 100644 --- a/tests/chat-peg-parser/test-optional.cpp +++ b/tests/chat-peg-parser/test-optional.cpp @@ -3,11 +3,11 @@ void test_optional(testing &t) { // Full match with optional part present t.test("optional_present", [](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); }); - auto ctx = common_chat_parse_context("hello world"); + auto ctx = common_peg_parse_context("hello world"); auto result = parser.parse(ctx); t.assert_equal("optional_present", true, result.success()); t.assert_equal("optional_present_end", 11u, result.end); @@ -15,11 +15,11 @@ void test_optional(testing &t) { // Full match with optional part absent t.test("optional_absent", [](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); }); - auto ctx = common_chat_parse_context("hello", true); + auto ctx = common_peg_parse_context("hello", true); auto result = parser.parse(ctx); t.assert_equal("optional_absent", true, result.success()); t.assert_equal("optional_absent_end", 5u, result.end); @@ -27,11 +27,11 @@ void test_optional(testing &t) { // Partial match - waiting for more input to determine if optional matches t.test("partial_match_need_more", [](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("hello") + p.optional(p.literal(" world")); }); - auto ctx = common_chat_parse_context("hello ", false); + auto ctx = common_peg_parse_context("hello ", false); auto result = parser.parse(ctx); t.assert_equal("partial_match_need_more", true, result.need_more_input()); }); diff --git a/tests/chat-peg-parser/test-partial-parsing.cpp b/tests/chat-peg-parser/test-partial-parsing.cpp index 0779be5e55028..35d38e35b1cb4 100644 --- a/tests/chat-peg-parser/test-partial-parsing.cpp +++ b/tests/chat-peg-parser/test-partial-parsing.cpp @@ -4,126 +4,126 @@ void test_partial_parsing(testing &t) { // Literals - Basic Success t.test("literal_success", [&](testing & t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("hello"); }); + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("hello"); }); - common_chat_parse_context ctx; - common_chat_parse_result result; + common_peg_parse_context ctx; + common_peg_parse_result result; - ctx = common_chat_parse_context("hello"); + ctx = common_peg_parse_context("hello"); result = parser.parse(ctx); t.assert_equal("literal_success", true, result.success()); }); // Char Classes - Basic Lowercase Success t.test("char_class_lowercase_success", [&](testing & t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("a-z"); }); + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("a-z"); }); - common_chat_parse_context ctx; - common_chat_parse_result result; + common_peg_parse_context ctx; + common_peg_parse_result result; - ctx = common_chat_parse_context("a"); + ctx = common_peg_parse_context("a"); result = parser.parse(ctx); t.assert_equal("char_class_lowercase_success", true, result.success()); }); // Char Classes - Uppercase Fail t.test("char_class_uppercase_fail", [&](testing & t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("a-z"); }); + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("a-z"); }); - common_chat_parse_context ctx; - common_chat_parse_result result; + common_peg_parse_context ctx; + common_peg_parse_result result; - ctx = common_chat_parse_context("A"); + ctx = common_peg_parse_context("A"); result = parser.parse(ctx); t.assert_equal("char_class_uppercase_fail", true, result.fail()); }); // Char Classes with Dash - Lowercase Success t.test("char_class_with_dash_lowercase", [&](testing & t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("a-z-"); }); + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("a-z-"); }); - common_chat_parse_context ctx; - common_chat_parse_result result; + common_peg_parse_context ctx; + common_peg_parse_result result; - ctx = common_chat_parse_context("f"); + ctx = common_peg_parse_context("f"); result = parser.parse(ctx); t.assert_equal("char_class_with_dash_lowercase", true, result.success()); }); // Char Classes with Dash - Literal Dash Success t.test("char_class_with_dash_literal_dash", [&](testing & t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("a-z-"); }); + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("a-z-"); }); - common_chat_parse_context ctx; - common_chat_parse_result result; + common_peg_parse_context ctx; + common_peg_parse_result result; - ctx = common_chat_parse_context("-"); + ctx = common_peg_parse_context("-"); result = parser.parse(ctx); t.assert_equal("char_class_with_dash_literal_dash", true, result.success()); }); // Char Classes with Dash - Uppercase Fail t.test("char_class_with_dash_uppercase_fail", [&](testing & t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one("a-z-"); }); + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("a-z-"); }); - common_chat_parse_context ctx; - common_chat_parse_result result; + common_peg_parse_context ctx; + common_peg_parse_result result; - ctx = common_chat_parse_context("A"); + ctx = common_peg_parse_context("A"); result = parser.parse(ctx); t.assert_equal("char_class_with_dash_uppercase_fail", true, result.fail()); }); // Sequences - Partial Match 1 t.test("sequence_partial_match_1", [&](testing & t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("") + p.literal(""); }); + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("") + p.literal(""); }); - auto ctx = common_chat_parse_context("") + p.literal(""); }); + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("") + p.literal(""); }); - auto ctx = common_chat_parse_context("") + p.literal(""); }); + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("") + p.literal(""); }); - auto ctx = common_chat_parse_context("I am common_chat_combinator_parser", false); + auto ctx = common_peg_parse_context("I am common_chat_combinator_parser", false); auto result = parser.parse(ctx); t.assert_equal("sequence_no_match", true, result.fail()); }); // Choices - Partial Match 1 t.test("choices_partial_match_1", [&](testing & t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("option1") | p.literal("option2"); }); + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("option1") | p.literal("option2"); }); - auto ctx = common_chat_parse_context("opt", false); + auto ctx = common_peg_parse_context("opt", false); auto result = parser.parse(ctx); t.assert_equal("choices_partial_match_1", true, result.need_more_input()); }); @@ -131,99 +131,99 @@ void test_partial_parsing(testing &t) { // Choices - Partial Match 2 t.test("choices_partial_match_2", [&](testing & t) { auto parser = - build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("choice_a") | p.literal("choice_b"); }); + build_peg_parser([](common_peg_parser_builder & p) { return p.literal("choice_a") | p.literal("choice_b"); }); - auto ctx = common_chat_parse_context("choice", false); + auto ctx = common_peg_parse_context("choice", false); auto result = parser.parse(ctx); t.assert_equal("choices_partial_match_2", true, result.need_more_input()); }); // Choices - Full Match 1 t.test("choices_full_match_1", [&](testing & t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("first") | p.literal("second"); }); + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("first") | p.literal("second"); }); - auto ctx = common_chat_parse_context("first", true); + auto ctx = common_peg_parse_context("first", true); auto result = parser.parse(ctx); t.assert_equal("choices_full_match_1", true, result.success()); }); // Choices - Full Match 2 t.test("choices_full_match_2", [&](testing & t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("alpha") | p.literal("beta"); }); + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("alpha") | p.literal("beta"); }); - auto ctx = common_chat_parse_context("beta", true); + auto ctx = common_peg_parse_context("beta", true); auto result = parser.parse(ctx); t.assert_equal("choices_full_match_2", true, result.success()); }); // Choices - No Match t.test("choices_no_match", [&](testing & t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.literal("good") | p.literal("better"); }); + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("good") | p.literal("better"); }); - auto ctx = common_chat_parse_context("best", true); + auto ctx = common_peg_parse_context("best", true); auto result = parser.parse(ctx); t.assert_equal("choices_no_match", true, result.fail()); }); // Zero or More - Partial Match 1 t.test("zero_or_more_partial_match_1", [&](testing & t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.zero_or_more(p.literal("ab")); }); + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.literal("ab")); }); - auto ctx = common_chat_parse_context("a", false); + auto ctx = common_peg_parse_context("a", false); auto result = parser.parse(ctx); t.assert_equal("zero_or_more_partial_match_1", true, result.need_more_input()); }); // Zero or More - Partial Match 2 t.test("zero_or_more_partial_match_2", [&](testing & t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.zero_or_more(p.literal("xy")); }); + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.literal("xy")); }); - auto ctx = common_chat_parse_context("xyx", false); + auto ctx = common_peg_parse_context("xyx", false); auto result = parser.parse(ctx); t.assert_equal("zero_or_more_partial_match_2", true, result.need_more_input()); }); // Zero or More - Full Match t.test("zero_or_more_full_match", [&](testing & t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.zero_or_more(p.literal("test")); }); + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.literal("test")); }); - auto ctx = common_chat_parse_context("test", true); + auto ctx = common_peg_parse_context("test", true); auto result = parser.parse(ctx); t.assert_equal("zero_or_more_full_match", true, result.success()); }); // One or More - Partial Match 1 t.test("one_or_more_partial_match_1", [&](testing & t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one_or_more(p.literal("repeat")); }); + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("repeat")); }); - auto ctx = common_chat_parse_context("rep", false); + auto ctx = common_peg_parse_context("rep", false); auto result = parser.parse(ctx); t.assert_equal("one_or_more_partial_match_1", true, result.need_more_input()); }); // One or More - Partial Match 2 t.test("one_or_more_partial_match_2", [&](testing & t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one_or_more(p.literal("ab")); }); + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("ab")); }); - auto ctx = common_chat_parse_context("aba", false); + auto ctx = common_peg_parse_context("aba", false); auto result = parser.parse(ctx); t.assert_equal("one_or_more_partial_match_2", true, result.need_more_input()); }); // One or More - Full Match t.test("one_or_more_full_match", [&](testing & t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one_or_more(p.literal("single")); }); + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("single")); }); - auto ctx = common_chat_parse_context("single", true); + auto ctx = common_peg_parse_context("single", true); auto result = parser.parse(ctx); t.assert_equal("one_or_more_full_match", true, result.success()); }); // One or More - No Match t.test("one_or_more_no_match", [&](testing & t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder & p) { return p.one_or_more(p.literal("()")); }); + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("()")); }); - auto ctx = common_chat_parse_context("success", true); + auto ctx = common_peg_parse_context("success", true); auto result = parser.parse(ctx); t.assert_equal("one_or_more_no_match", true, result.fail()); }); diff --git a/tests/chat-peg-parser/test-recursive-references.cpp b/tests/chat-peg-parser/test-recursive-references.cpp index 3824f70cf8fdf..8f7e8b0a191fd 100644 --- a/tests/chat-peg-parser/test-recursive-references.cpp +++ b/tests/chat-peg-parser/test-recursive-references.cpp @@ -3,13 +3,13 @@ void test_recursive_references(testing &t) { // Test simple number t.test("simple_number", [](testing &t) { - auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto value_parser = build_peg_parser([](common_peg_parser_builder & p) { p.rule("number", p.one_or_more(p.one("0-9"))); p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); return p.rule("value", p.ref("number") | p.ref("list")); }); - common_chat_parse_context ctx("1", true); + common_peg_parse_context ctx("1", true); auto result = value_parser.parse(ctx); t.assert_equal("result_is_success", true, result.success()); @@ -17,13 +17,13 @@ void test_recursive_references(testing &t) { // Test simple list t.test("simple_list", [](testing &t) { - auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto value_parser = build_peg_parser([](common_peg_parser_builder & p) { p.rule("number", p.one_or_more(p.one("0-9"))); p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); return p.rule("value", p.ref("number") | p.ref("list")); }); - common_chat_parse_context ctx("[1]", true); + common_peg_parse_context ctx("[1]", true); auto result = value_parser.parse(ctx); t.assert_equal("result_is_success", true, result.success()); @@ -31,13 +31,13 @@ void test_recursive_references(testing &t) { // Test nested list t.test("nested_list", [](testing &t) { - auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto value_parser = build_peg_parser([](common_peg_parser_builder & p) { p.rule("number", p.one_or_more(p.one("0-9"))); p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); return p.rule("value", p.ref("number") | p.ref("list")); }); - common_chat_parse_context ctx("[[2]]", true); + common_peg_parse_context ctx("[[2]]", true); auto result = value_parser.parse(ctx); t.assert_equal("result_is_success", true, result.success()); @@ -45,13 +45,13 @@ void test_recursive_references(testing &t) { // Test deeply nested list t.test("deeply_nested_list", [](testing &t) { - auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto value_parser = build_peg_parser([](common_peg_parser_builder & p) { p.rule("number", p.one_or_more(p.one("0-9"))); p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); return p.rule("value", p.ref("number") | p.ref("list")); }); - common_chat_parse_context ctx("[[[3]]]", true); + common_peg_parse_context ctx("[[[3]]]", true); auto result = value_parser.parse(ctx); t.assert_equal("result_is_success", true, result.success()); @@ -59,13 +59,13 @@ void test_recursive_references(testing &t) { // Test need_more_input match t.test("need_more_input_match", [](testing &t) { - auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto value_parser = build_peg_parser([](common_peg_parser_builder & p) { p.rule("number", p.one_or_more(p.one("0-9"))); p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); return p.rule("value", p.ref("number") | p.ref("list")); }); - common_chat_parse_context ctx("[[", false); + common_peg_parse_context ctx("[[", false); auto result = value_parser.parse(ctx); t.assert_equal("result_is_need_more_input", true, result.need_more_input()); @@ -73,13 +73,13 @@ void test_recursive_references(testing &t) { // Test no match t.test("no_match", [](testing &t) { - auto value_parser = build_peg_parser([](common_chat_peg_parser_builder & p) { + auto value_parser = build_peg_parser([](common_peg_parser_builder & p) { p.rule("number", p.one_or_more(p.one("0-9"))); p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); return p.rule("value", p.ref("number") | p.ref("list")); }); - common_chat_parse_context ctx("[a]", true); + common_peg_parse_context ctx("[a]", true); auto result = value_parser.parse(ctx); t.assert_equal("result_is_fail", true, result.fail()); diff --git a/tests/chat-peg-parser/test-unicode.cpp b/tests/chat-peg-parser/test-unicode.cpp index 7352fd57edcbd..1b777810c8b54 100644 --- a/tests/chat-peg-parser/test-unicode.cpp +++ b/tests/chat-peg-parser/test-unicode.cpp @@ -8,8 +8,8 @@ #include #include -static void assert_result_equal(testing & t, common_chat_parse_result_type expected, common_chat_parse_result_type actual) { - t.assert_equal(common_chat_parse_result_type_name(expected), common_chat_parse_result_type_name(actual)); +static void assert_result_equal(testing & t, common_peg_parse_result_type expected, common_peg_parse_result_type actual) { + t.assert_equal(common_peg_parse_result_type_name(expected), common_peg_parse_result_type_name(actual)); } static std::string hex_dump(const std::string& str) { @@ -28,29 +28,29 @@ void test_unicode(testing &t) { struct test_case { std::string input; std::string expected_text; - common_chat_parse_result_type expected_result; + common_peg_parse_result_type expected_result; }; t.test("any", [](testing &t) { std::vector test_cases { // Valid UTF-8 sequences - {"Hello", "Hello", COMMON_CHAT_PARSE_RESULT_SUCCESS}, - {std::string("Caf\xC3\xA9"), std::string("Caf\xC3\xA9"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, - {std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, - {std::string("\xF0\x9F\x9A\x80"), std::string("\xF0\x9F\x9A\x80"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, + {"Hello", "Hello", COMMON_PEG_PARSE_RESULT_SUCCESS}, + {std::string("Caf\xC3\xA9"), std::string("Caf\xC3\xA9"), COMMON_PEG_PARSE_RESULT_SUCCESS}, + {std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), COMMON_PEG_PARSE_RESULT_SUCCESS}, + {std::string("\xF0\x9F\x9A\x80"), std::string("\xF0\x9F\x9A\x80"), COMMON_PEG_PARSE_RESULT_SUCCESS}, // Incomplete UTF-8 sequences (partial bytes at end) - {std::string("Caf\xC3"), "Caf", COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, - {std::string("\xE4\xBD"), "", COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, - {std::string("\xF0\x9F\x9A"), "", COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, + {std::string("Caf\xC3"), "Caf", COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT}, + {std::string("\xE4\xBD"), "", COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT}, + {std::string("\xF0\x9F\x9A"), "", COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT}, // Invalid/malformed UTF-8 sequences - {std::string("\xFF\xFE"), "", COMMON_CHAT_PARSE_RESULT_FAIL}, - {std::string("Hello\x80World"), "Hello", COMMON_CHAT_PARSE_RESULT_FAIL}, - {std::string("\xC3\x28"), "", COMMON_CHAT_PARSE_RESULT_FAIL}, + {std::string("\xFF\xFE"), "", COMMON_PEG_PARSE_RESULT_FAIL}, + {std::string("Hello\x80World"), "Hello", COMMON_PEG_PARSE_RESULT_FAIL}, + {std::string("\xC3\x28"), "", COMMON_PEG_PARSE_RESULT_FAIL}, }; - auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { + auto parser = build_peg_parser([](common_peg_parser_builder& p) { return p.sequence({p.one_or_more(p.any()), p.end()}); }); @@ -59,7 +59,7 @@ void test_unicode(testing &t) { std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input); t.test(test_name, [&](testing &t) { - common_chat_parse_context ctx(tc.input, false); + common_peg_parse_context ctx(tc.input, false); auto result = parser.parse(ctx); // Assert result type matches @@ -78,22 +78,22 @@ void test_unicode(testing &t) { t.test("unicode range U+4E00-U+9FFF (CJK)", [](testing &t) { std::vector test_cases { // Within range - CJK Unified Ideographs - {std::string("\xE4\xB8\x80"), std::string("\xE4\xB8\x80"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // U+4E00 - {std::string("\xE4\xBD\xA0"), std::string("\xE4\xBD\xA0"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // U+4F60 - {std::string("\xE5\xA5\xBD"), std::string("\xE5\xA5\xBD"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // U+597D - {std::string("\xE9\xBF\xBF"), std::string("\xE9\xBF\xBF"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // U+9FFF + {std::string("\xE4\xB8\x80"), std::string("\xE4\xB8\x80"), COMMON_PEG_PARSE_RESULT_SUCCESS}, // U+4E00 + {std::string("\xE4\xBD\xA0"), std::string("\xE4\xBD\xA0"), COMMON_PEG_PARSE_RESULT_SUCCESS}, // U+4F60 + {std::string("\xE5\xA5\xBD"), std::string("\xE5\xA5\xBD"), COMMON_PEG_PARSE_RESULT_SUCCESS}, // U+597D + {std::string("\xE9\xBF\xBF"), std::string("\xE9\xBF\xBF"), COMMON_PEG_PARSE_RESULT_SUCCESS}, // U+9FFF // Outside range - should fail - {"a", "", COMMON_CHAT_PARSE_RESULT_FAIL}, // ASCII - {std::string("\xE4\xB7\xBF"), "", COMMON_CHAT_PARSE_RESULT_FAIL}, // U+4DFF (before range) - {std::string("\xEA\x80\x80"), "", COMMON_CHAT_PARSE_RESULT_FAIL}, // U+A000 (after range) + {"a", "", COMMON_PEG_PARSE_RESULT_FAIL}, // ASCII + {std::string("\xE4\xB7\xBF"), "", COMMON_PEG_PARSE_RESULT_FAIL}, // U+4DFF (before range) + {std::string("\xEA\x80\x80"), "", COMMON_PEG_PARSE_RESULT_FAIL}, // U+A000 (after range) // Incomplete sequences in range - {std::string("\xE4\xB8"), "", COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, // Incomplete U+4E00 - {std::string("\xE5\xA5"), "", COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, // Incomplete U+597D + {std::string("\xE4\xB8"), "", COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT}, // Incomplete U+4E00 + {std::string("\xE5\xA5"), "", COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT}, // Incomplete U+597D }; - auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { + auto parser = build_peg_parser([](common_peg_parser_builder& p) { return p.sequence({p.chars(R"([\u4E00-\u9FFF])"), p.end()}); }); @@ -102,7 +102,7 @@ void test_unicode(testing &t) { std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input); t.test(test_name, [&](testing &t) { - common_chat_parse_context ctx(tc.input, false); + common_peg_parse_context ctx(tc.input, false); auto result = parser.parse(ctx); // Assert result type matches @@ -120,21 +120,21 @@ void test_unicode(testing &t) { t.test("unicode range U+1F600-U+1F64F (emoticons)", [](testing &t) { std::vector test_cases { // Within range - Emoticons (all 4-byte UTF-8) - {std::string("\xF0\x9F\x98\x80"), std::string("\xF0\x9F\x98\x80"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // U+1F600 - {std::string("\xF0\x9F\x98\x81"), std::string("\xF0\x9F\x98\x81"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // U+1F601 - {std::string("\xF0\x9F\x99\x8F"), std::string("\xF0\x9F\x99\x8F"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // U+1F64F + {std::string("\xF0\x9F\x98\x80"), std::string("\xF0\x9F\x98\x80"), COMMON_PEG_PARSE_RESULT_SUCCESS}, // U+1F600 + {std::string("\xF0\x9F\x98\x81"), std::string("\xF0\x9F\x98\x81"), COMMON_PEG_PARSE_RESULT_SUCCESS}, // U+1F601 + {std::string("\xF0\x9F\x99\x8F"), std::string("\xF0\x9F\x99\x8F"), COMMON_PEG_PARSE_RESULT_SUCCESS}, // U+1F64F // Outside range - {std::string("\xF0\x9F\x97\xBF"), "", COMMON_CHAT_PARSE_RESULT_FAIL}, // U+1F5FF (before range) - {std::string("\xF0\x9F\x99\x90"), "", COMMON_CHAT_PARSE_RESULT_FAIL}, // U+1F650 (after range) - {std::string("\xF0\x9F\x9A\x80"), "", COMMON_CHAT_PARSE_RESULT_FAIL}, // U+1F680 (outside range) + {std::string("\xF0\x9F\x97\xBF"), "", COMMON_PEG_PARSE_RESULT_FAIL}, // U+1F5FF (before range) + {std::string("\xF0\x9F\x99\x90"), "", COMMON_PEG_PARSE_RESULT_FAIL}, // U+1F650 (after range) + {std::string("\xF0\x9F\x9A\x80"), "", COMMON_PEG_PARSE_RESULT_FAIL}, // U+1F680 (outside range) // Incomplete sequences - {std::string("\xF0\x9F\x98"), "", COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, // Incomplete emoji - {std::string("\xF0\x9F"), "", COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, // Very incomplete + {std::string("\xF0\x9F\x98"), "", COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT}, // Incomplete emoji + {std::string("\xF0\x9F"), "", COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT}, // Very incomplete }; - auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { + auto parser = build_peg_parser([](common_peg_parser_builder& p) { return p.sequence({p.chars(R"([\U0001F600-\U0001F64F])"), p.end()}); }); @@ -143,7 +143,7 @@ void test_unicode(testing &t) { std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input); t.test(test_name, [&](testing &t) { - common_chat_parse_context ctx(tc.input, false); + common_peg_parse_context ctx(tc.input, false); auto result = parser.parse(ctx); // Assert result type matches @@ -161,25 +161,25 @@ void test_unicode(testing &t) { t.test("mixed unicode ranges", [](testing &t) { std::vector test_cases { // Match CJK - {std::string("\xE4\xB8\x80"), std::string("\xE4\xB8\x80"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // U+4E00 - {std::string("\xE4\xBD\xA0"), std::string("\xE4\xBD\xA0"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // U+4F60 + {std::string("\xE4\xB8\x80"), std::string("\xE4\xB8\x80"), COMMON_PEG_PARSE_RESULT_SUCCESS}, // U+4E00 + {std::string("\xE4\xBD\xA0"), std::string("\xE4\xBD\xA0"), COMMON_PEG_PARSE_RESULT_SUCCESS}, // U+4F60 // Match emoticons - {std::string("\xF0\x9F\x98\x80"), std::string("\xF0\x9F\x98\x80"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, // U+1F600 + {std::string("\xF0\x9F\x98\x80"), std::string("\xF0\x9F\x98\x80"), COMMON_PEG_PARSE_RESULT_SUCCESS}, // U+1F600 // Match ASCII digits - {"5", "5", COMMON_CHAT_PARSE_RESULT_SUCCESS}, + {"5", "5", COMMON_PEG_PARSE_RESULT_SUCCESS}, // Don't match outside any range - {"a", "", COMMON_CHAT_PARSE_RESULT_FAIL}, - {std::string("\xF0\x9F\x9A\x80"), "", COMMON_CHAT_PARSE_RESULT_FAIL}, // U+1F680 + {"a", "", COMMON_PEG_PARSE_RESULT_FAIL}, + {std::string("\xF0\x9F\x9A\x80"), "", COMMON_PEG_PARSE_RESULT_FAIL}, // U+1F680 // Incomplete - {std::string("\xE4\xB8"), "", COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, - {std::string("\xF0\x9F\x98"), "", COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, + {std::string("\xE4\xB8"), "", COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT}, + {std::string("\xF0\x9F\x98"), "", COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT}, }; - auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { + auto parser = build_peg_parser([](common_peg_parser_builder& p) { return p.sequence({p.chars(R"([\u4E00-\u9FFF\U0001F600-\U0001F64F0-9])"), p.end()}); }); @@ -188,7 +188,7 @@ void test_unicode(testing &t) { std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input); t.test(test_name, [&](testing &t) { - common_chat_parse_context ctx(tc.input, false); + common_peg_parse_context ctx(tc.input, false); auto result = parser.parse(ctx); // Assert result type matches @@ -208,16 +208,16 @@ void test_unicode(testing &t) { t.test("ASCII delimiter with Unicode content", [](testing &t) { std::vector test_cases { // CJK characters before delimiter - {std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, + {std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), COMMON_PEG_PARSE_RESULT_SUCCESS}, // Emoji before delimiter - {std::string("\xF0\x9F\x98\x80"), std::string("\xF0\x9F\x98\x80"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, + {std::string("\xF0\x9F\x98\x80"), std::string("\xF0\x9F\x98\x80"), COMMON_PEG_PARSE_RESULT_SUCCESS}, // Mixed content - {std::string("Hello \xE4\xB8\x96\xE7\x95\x8C!"), std::string("Hello \xE4\xB8\x96\xE7\x95\x8C!"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, + {std::string("Hello \xE4\xB8\x96\xE7\x95\x8C!"), std::string("Hello \xE4\xB8\x96\xE7\x95\x8C!"), COMMON_PEG_PARSE_RESULT_SUCCESS}, }; - auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { + auto parser = build_peg_parser([](common_peg_parser_builder& p) { return p.until(""); }); @@ -226,7 +226,7 @@ void test_unicode(testing &t) { std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input); t.test(test_name, [&](testing &t) { - common_chat_parse_context ctx(tc.input, true); + common_peg_parse_context ctx(tc.input, true); auto result = parser.parse(ctx); assert_result_equal(t, tc.expected_result, result.type); @@ -242,16 +242,16 @@ void test_unicode(testing &t) { t.test("incomplete UTF-8 at end", [](testing &t) { std::vector test_cases { // Incomplete emoji at end, no delimiter - {std::string("content\xF0\x9F\x98"), std::string("content"), COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, + {std::string("content\xF0\x9F\x98"), std::string("content"), COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT}, // Incomplete CJK at end, no delimiter - {std::string("hello\xE4\xB8"), std::string("hello"), COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, + {std::string("hello\xE4\xB8"), std::string("hello"), COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT}, // Complete content, no delimiter (should consume all valid UTF-8) - {std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, + {std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), COMMON_PEG_PARSE_RESULT_SUCCESS}, }; - auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { + auto parser = build_peg_parser([](common_peg_parser_builder& p) { return p.until(""); }); @@ -260,7 +260,7 @@ void test_unicode(testing &t) { std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input); t.test(test_name, [&](testing &t) { - common_chat_parse_context ctx(tc.input, false); // input_is_complete = false + common_peg_parse_context ctx(tc.input, false); // input_is_complete = false auto result = parser.parse(ctx); assert_result_equal(t, tc.expected_result, result.type); @@ -276,16 +276,16 @@ void test_unicode(testing &t) { t.test("malformed UTF-8", [](testing &t) { std::vector test_cases { // Invalid UTF-8 bytes - {std::string("Hello\xFF\xFE"), "", COMMON_CHAT_PARSE_RESULT_FAIL}, + {std::string("Hello\xFF\xFE"), "", COMMON_PEG_PARSE_RESULT_FAIL}, // Continuation byte without lead byte - {std::string("Hello\x80World"), "", COMMON_CHAT_PARSE_RESULT_FAIL}, + {std::string("Hello\x80World"), "", COMMON_PEG_PARSE_RESULT_FAIL}, // Invalid continuation byte - {std::string("\xC3\x28"), "", COMMON_CHAT_PARSE_RESULT_FAIL}, + {std::string("\xC3\x28"), "", COMMON_PEG_PARSE_RESULT_FAIL}, }; - auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { + auto parser = build_peg_parser([](common_peg_parser_builder& p) { return p.until(""); }); @@ -294,7 +294,7 @@ void test_unicode(testing &t) { std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input); t.test(test_name, [&](testing &t) { - common_chat_parse_context ctx(tc.input, true); + common_peg_parse_context ctx(tc.input, true); auto result = parser.parse(ctx); assert_result_equal(t, tc.expected_result, result.type); @@ -307,19 +307,19 @@ void test_unicode(testing &t) { t.test("valid UTF-8 characters", [](testing &t) { std::vector test_cases { // ASCII only - {"Hello World\"", "Hello World", COMMON_CHAT_PARSE_RESULT_SUCCESS}, + {"Hello World\"", "Hello World", COMMON_PEG_PARSE_RESULT_SUCCESS}, // 2-byte UTF-8 (accented characters) - {std::string("Caf\xC3\xA9\""), std::string("Caf\xC3\xA9"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, + {std::string("Caf\xC3\xA9\""), std::string("Caf\xC3\xA9"), COMMON_PEG_PARSE_RESULT_SUCCESS}, // 3-byte UTF-8 (CJK) - {std::string("\xE4\xBD\xA0\xE5\xA5\xBD\""), std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, + {std::string("\xE4\xBD\xA0\xE5\xA5\xBD\""), std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), COMMON_PEG_PARSE_RESULT_SUCCESS}, // 4-byte UTF-8 (emoji) - {std::string("\xF0\x9F\x98\x80\""), std::string("\xF0\x9F\x98\x80"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, + {std::string("\xF0\x9F\x98\x80\""), std::string("\xF0\x9F\x98\x80"), COMMON_PEG_PARSE_RESULT_SUCCESS}, // Mixed content - {std::string("Hello \xE4\xB8\x96\xE7\x95\x8C!\""), std::string("Hello \xE4\xB8\x96\xE7\x95\x8C!"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, + {std::string("Hello \xE4\xB8\x96\xE7\x95\x8C!\""), std::string("Hello \xE4\xB8\x96\xE7\x95\x8C!"), COMMON_PEG_PARSE_RESULT_SUCCESS}, }; for (size_t i = 0; i < test_cases.size(); i++) { @@ -327,11 +327,11 @@ void test_unicode(testing &t) { std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input); t.test(test_name, [&](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { + auto parser = build_peg_parser([](common_peg_parser_builder& p) { return p.sequence({p.json_string_content(), p.literal("\"")}); }); - common_chat_parse_context ctx(tc.input, true); + common_peg_parse_context ctx(tc.input, true); auto result = parser.parse(ctx); assert_result_equal(t, tc.expected_result, result.type); @@ -347,16 +347,16 @@ void test_unicode(testing &t) { t.test("incomplete UTF-8", [](testing &t) { std::vector test_cases { // Incomplete 2-byte sequence - {std::string("Caf\xC3"), std::string("Caf"), COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, + {std::string("Caf\xC3"), std::string("Caf"), COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT}, // Incomplete 3-byte sequence - {std::string("Hello\xE4\xB8"), std::string("Hello"), COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, + {std::string("Hello\xE4\xB8"), std::string("Hello"), COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT}, // Incomplete 4-byte sequence - {std::string("Text\xF0\x9F\x98"), std::string("Text"), COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, + {std::string("Text\xF0\x9F\x98"), std::string("Text"), COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT}, // Incomplete at very start - {std::string("\xE4\xBD"), std::string(""), COMMON_CHAT_PARSE_RESULT_NEED_MORE_INPUT}, + {std::string("\xE4\xBD"), std::string(""), COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT}, }; for (size_t i = 0; i < test_cases.size(); i++) { @@ -364,11 +364,11 @@ void test_unicode(testing &t) { std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input); t.test(test_name, [&](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { + auto parser = build_peg_parser([](common_peg_parser_builder& p) { return p.json_string_content(); }); - common_chat_parse_context ctx(tc.input, false); // input_is_complete = false + common_peg_parse_context ctx(tc.input, false); // input_is_complete = false auto result = parser.parse(ctx); assert_result_equal(t, tc.expected_result, result.type); @@ -384,16 +384,16 @@ void test_unicode(testing &t) { t.test("malformed UTF-8", [](testing &t) { std::vector test_cases { // Invalid UTF-8 bytes - {std::string("Hello\xFF\xFE"), "", COMMON_CHAT_PARSE_RESULT_FAIL}, + {std::string("Hello\xFF\xFE"), "", COMMON_PEG_PARSE_RESULT_FAIL}, // Continuation byte without lead byte - {std::string("Hello\x80World"), "", COMMON_CHAT_PARSE_RESULT_FAIL}, + {std::string("Hello\x80World"), "", COMMON_PEG_PARSE_RESULT_FAIL}, // Invalid continuation byte - {std::string("\xC3\x28"), "", COMMON_CHAT_PARSE_RESULT_FAIL}, + {std::string("\xC3\x28"), "", COMMON_PEG_PARSE_RESULT_FAIL}, // Overlong encoding (security issue) - {std::string("\xC0\x80"), "", COMMON_CHAT_PARSE_RESULT_FAIL}, + {std::string("\xC0\x80"), "", COMMON_PEG_PARSE_RESULT_FAIL}, }; for (size_t i = 0; i < test_cases.size(); i++) { @@ -401,11 +401,11 @@ void test_unicode(testing &t) { std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input); t.test(test_name, [&](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { + auto parser = build_peg_parser([](common_peg_parser_builder& p) { return p.json_string_content(); }); - common_chat_parse_context ctx(tc.input, true); + common_peg_parse_context ctx(tc.input, true); auto result = parser.parse(ctx); assert_result_equal(t, tc.expected_result, result.type); @@ -416,13 +416,13 @@ void test_unicode(testing &t) { t.test("escape sequences with UTF-8", [](testing &t) { std::vector test_cases { // Unicode escape sequence - {"Hello\\u0041\"", "Hello\\u0041", COMMON_CHAT_PARSE_RESULT_SUCCESS}, + {"Hello\\u0041\"", "Hello\\u0041", COMMON_PEG_PARSE_RESULT_SUCCESS}, // Mix of UTF-8 and escape sequences - {std::string("\xE4\xBD\xA0\\n\xE5\xA5\xBD\""), std::string("\xE4\xBD\xA0\\n\xE5\xA5\xBD"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, + {std::string("\xE4\xBD\xA0\\n\xE5\xA5\xBD\""), std::string("\xE4\xBD\xA0\\n\xE5\xA5\xBD"), COMMON_PEG_PARSE_RESULT_SUCCESS}, // Escaped quote in UTF-8 string - {std::string("\xE4\xBD\xA0\\\"\xE5\xA5\xBD\""), std::string("\xE4\xBD\xA0\\\"\xE5\xA5\xBD"), COMMON_CHAT_PARSE_RESULT_SUCCESS}, + {std::string("\xE4\xBD\xA0\\\"\xE5\xA5\xBD\""), std::string("\xE4\xBD\xA0\\\"\xE5\xA5\xBD"), COMMON_PEG_PARSE_RESULT_SUCCESS}, }; for (size_t i = 0; i < test_cases.size(); i++) { @@ -430,11 +430,11 @@ void test_unicode(testing &t) { std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input); t.test(test_name, [&](testing &t) { - auto parser = build_peg_parser([](common_chat_peg_parser_builder& p) { + auto parser = build_peg_parser([](common_peg_parser_builder& p) { return p.sequence({p.json_string_content(), p.literal("\"")}); }); - common_chat_parse_context ctx(tc.input, true); + common_peg_parse_context ctx(tc.input, true); auto result = parser.parse(ctx); assert_result_equal(t, tc.expected_result, result.type); From 8c2465348bf311538f46fd959d8e6ee125274c56 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Wed, 19 Nov 2025 20:03:38 -0600 Subject: [PATCH 136/183] rename chat-peg-parser -> peg-parser --- common/CMakeLists.txt | 4 ++-- common/chat-peg-parser-helper.cpp | 2 +- common/chat-peg-parser-helper.h | 2 +- common/{chat-peg-parser.cpp => peg-parser.cpp} | 2 +- common/{chat-peg-parser.h => peg-parser.h} | 0 tests/chat-peg-parser/test-example-minimax-m2.cpp | 2 +- tests/chat-peg-parser/test-unicode.cpp | 2 +- tests/chat-peg-parser/tests.h | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) rename common/{chat-peg-parser.cpp => peg-parser.cpp} (99%) rename common/{chat-peg-parser.h => peg-parser.h} (100%) diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index f2b52d393b68a..7f14b83ac839c 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -50,8 +50,6 @@ add_library(${TARGET} STATIC base64.hpp chat-parser.cpp chat-parser.h - chat-peg-parser.cpp - chat-peg-parser.h chat-peg-parser-helper.cpp chat-peg-parser-helper.h chat.cpp @@ -71,6 +69,8 @@ add_library(${TARGET} STATIC log.h ngram-cache.cpp ngram-cache.h + peg-parser.cpp + peg-parser.h regex-partial.cpp regex-partial.h sampling.cpp diff --git a/common/chat-peg-parser-helper.cpp b/common/chat-peg-parser-helper.cpp index 9561619c2a96c..1edcdb1a5dd50 100644 --- a/common/chat-peg-parser-helper.cpp +++ b/common/chat-peg-parser-helper.cpp @@ -1,5 +1,5 @@ #include "chat-peg-parser-helper.h" -#include "chat-peg-parser.h" +#include "peg-parser.h" #include common_peg_parser common_peg_parser_builder_helper::reasoning(const std::string & tag) { diff --git a/common/chat-peg-parser-helper.h b/common/chat-peg-parser-helper.h index 1879834414b00..51381479fcb95 100644 --- a/common/chat-peg-parser-helper.h +++ b/common/chat-peg-parser-helper.h @@ -1,4 +1,4 @@ -#include "chat-peg-parser.h" +#include "peg-parser.h" #include "log.h" class common_peg_parser_builder_helper : public common_peg_parser_builder { diff --git a/common/chat-peg-parser.cpp b/common/peg-parser.cpp similarity index 99% rename from common/chat-peg-parser.cpp rename to common/peg-parser.cpp index cd98ad89ded57..922c1ad4317e3 100644 --- a/common/chat-peg-parser.cpp +++ b/common/peg-parser.cpp @@ -1,6 +1,6 @@ #include "log.h" #include "common.h" -#include "chat-peg-parser.h" +#include "peg-parser.h" #include "json-schema-to-grammar.h" #include "unicode.h" diff --git a/common/chat-peg-parser.h b/common/peg-parser.h similarity index 100% rename from common/chat-peg-parser.h rename to common/peg-parser.h diff --git a/tests/chat-peg-parser/test-example-minimax-m2.cpp b/tests/chat-peg-parser/test-example-minimax-m2.cpp index 52c39e2267632..2299ccfd1d80b 100644 --- a/tests/chat-peg-parser/test-example-minimax-m2.cpp +++ b/tests/chat-peg-parser/test-example-minimax-m2.cpp @@ -1,5 +1,5 @@ #include "common.h" -#include "chat-peg-parser.h" +#include "peg-parser.h" #include "nlohmann/json.hpp" #include "tests.h" diff --git a/tests/chat-peg-parser/test-unicode.cpp b/tests/chat-peg-parser/test-unicode.cpp index 1b777810c8b54..e3ec34e5152f4 100644 --- a/tests/chat-peg-parser/test-unicode.cpp +++ b/tests/chat-peg-parser/test-unicode.cpp @@ -1,7 +1,7 @@ #include "tests.h" #include "test_harness.h" -#include "chat-peg-parser.h" +#include "peg-parser.h" #include #include diff --git a/tests/chat-peg-parser/tests.h b/tests/chat-peg-parser/tests.h index 156ca0b37f3c2..793d64c149981 100644 --- a/tests/chat-peg-parser/tests.h +++ b/tests/chat-peg-parser/tests.h @@ -3,7 +3,7 @@ // Common includes for all test files #include "test_harness.h" #include -#include "chat-peg-parser.h" +#include "peg-parser.h" #include "chat-peg-parser-helper.h" #include #include From c4ce858560c76fd9e7ba83af6a08accf4418a4fa Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Wed, 19 Nov 2025 20:35:21 -0600 Subject: [PATCH 137/183] promote chat-peg-parser-helper to chat-peg-parser --- common/CMakeLists.txt | 4 +-- ...-parser-helper.cpp => chat-peg-parser.cpp} | 12 +++---- ...-peg-parser-helper.h => chat-peg-parser.h} | 15 ++++----- tests/.gitignore | 2 +- tests/CMakeLists.txt | 32 +++++++++---------- .../convo.json | 0 .../simple_tokenizer.cpp | 0 .../test-command7-parser-compare.cpp | 2 +- .../test-example-minimax-m2.cpp | 5 +-- .../test-example-qwen3-coder.cpp | 8 ++--- .../test-example-seed-oss.cpp | 5 +-- .../test-gbnf-generation.cpp | 0 .../test-json-parser.cpp | 0 .../test-json-serialization.cpp | 0 .../test-one.cpp | 0 .../test-optional.cpp | 0 .../test-partial-parsing.cpp | 0 .../test-recursive-references.cpp | 0 .../test-unicode.cpp | 0 .../test_harness.h | 0 tests/{chat-peg-parser => peg-parser}/tests.h | 2 +- ...hat-peg-parser.cpp => test-peg-parser.cpp} | 2 +- 22 files changed, 45 insertions(+), 44 deletions(-) rename common/{chat-peg-parser-helper.cpp => chat-peg-parser.cpp} (94%) rename common/{chat-peg-parser-helper.h => chat-peg-parser.h} (85%) rename tests/{chat-peg-parser => peg-parser}/convo.json (100%) rename tests/{chat-peg-parser => peg-parser}/simple_tokenizer.cpp (100%) rename tests/{chat-peg-parser => peg-parser}/test-command7-parser-compare.cpp (99%) rename tests/{chat-peg-parser => peg-parser}/test-example-minimax-m2.cpp (93%) rename tests/{chat-peg-parser => peg-parser}/test-example-qwen3-coder.cpp (95%) rename tests/{chat-peg-parser => peg-parser}/test-example-seed-oss.cpp (91%) rename tests/{chat-peg-parser => peg-parser}/test-gbnf-generation.cpp (100%) rename tests/{chat-peg-parser => peg-parser}/test-json-parser.cpp (100%) rename tests/{chat-peg-parser => peg-parser}/test-json-serialization.cpp (100%) rename tests/{chat-peg-parser => peg-parser}/test-one.cpp (100%) rename tests/{chat-peg-parser => peg-parser}/test-optional.cpp (100%) rename tests/{chat-peg-parser => peg-parser}/test-partial-parsing.cpp (100%) rename tests/{chat-peg-parser => peg-parser}/test-recursive-references.cpp (100%) rename tests/{chat-peg-parser => peg-parser}/test-unicode.cpp (100%) rename tests/{chat-peg-parser => peg-parser}/test_harness.h (100%) rename tests/{chat-peg-parser => peg-parser}/tests.h (96%) rename tests/{test-chat-peg-parser.cpp => test-peg-parser.cpp} (98%) diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 7f14b83ac839c..58c1c86409007 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -50,8 +50,8 @@ add_library(${TARGET} STATIC base64.hpp chat-parser.cpp chat-parser.h - chat-peg-parser-helper.cpp - chat-peg-parser-helper.h + chat-peg-parser.cpp + chat-peg-parser.h chat.cpp chat.h common.cpp diff --git a/common/chat-peg-parser-helper.cpp b/common/chat-peg-parser.cpp similarity index 94% rename from common/chat-peg-parser-helper.cpp rename to common/chat-peg-parser.cpp index 1edcdb1a5dd50..cf37dd314f90e 100644 --- a/common/chat-peg-parser-helper.cpp +++ b/common/chat-peg-parser.cpp @@ -1,8 +1,8 @@ -#include "chat-peg-parser-helper.h" +#include "chat-peg-parser.h" #include "peg-parser.h" #include -common_peg_parser common_peg_parser_builder_helper::reasoning(const std::string & tag) { +common_peg_parser common_chat_peg_parser_builder::reasoning(const std::string & tag) { std::string open_tag; open_tag.append("<").append(tag).append(">"); std::string close_tag; @@ -10,11 +10,11 @@ common_peg_parser common_peg_parser_builder_helper::reasoning(const std::string return rule("raw-reasoning", literal(open_tag) << rule("reasoning-content", until(close_tag)) << literal(close_tag)); } -common_peg_parser common_peg_parser_builder_helper::content_before_tools(const std::string & tag) { +common_peg_parser common_chat_peg_parser_builder::content_before_tools(const std::string & tag) { return rule("content", until(tag)); } -common_peg_parser common_peg_parser_builder_helper::quasi_xml_no_attr( +common_peg_parser common_chat_peg_parser_builder::quasi_xml_no_attr( const std::string & function_name, const std::vector & parameters, const std::string & function_tag, @@ -82,7 +82,7 @@ common_peg_parser common_peg_parser_builder_helper::quasi_xml_no_attr( return function; } -common_peg_parser common_peg_parser_builder_helper::quasi_xml_attr( +common_peg_parser common_chat_peg_parser_builder::quasi_xml_attr( const std::string & function_name, const std::vector & parameters, const std::string & function_tag, @@ -152,7 +152,7 @@ common_peg_parser common_peg_parser_builder_helper::quasi_xml_attr( return function; } -void common_peg_parse_simple_handler::operator()(const common_peg_parse_event & ev, common_peg_parse_semantics & semantics) const { +void common_chat_peg_simple_handler::operator()(const common_peg_parse_event & ev, common_peg_parse_semantics & semantics) const { if (log) { std::stringstream ss; ss << "Event: type=" << (ev.type == COMMON_PEG_PARSE_EVENT_NODE_START ? "start" : "end "); diff --git a/common/chat-peg-parser-helper.h b/common/chat-peg-parser.h similarity index 85% rename from common/chat-peg-parser-helper.h rename to common/chat-peg-parser.h index 51381479fcb95..ba35a8ae08f3b 100644 --- a/common/chat-peg-parser-helper.h +++ b/common/chat-peg-parser.h @@ -1,11 +1,10 @@ +#pragma once + #include "peg-parser.h" #include "log.h" -class common_peg_parser_builder_helper : public common_peg_parser_builder { - -public: - // Helper methods for common patterns - +class common_chat_peg_parser_builder : public common_peg_parser_builder { + public: // Adds raw-reasoning for the entire reasoning block plus reasoning-content for the contents, by default thinking tag is "think" common_peg_parser reasoning(const std::string & tag = "think"); @@ -25,14 +24,14 @@ class common_peg_parser_builder_helper : public common_peg_parser_builder { }; template -common_peg_arena build_peg_parser_helper(F && fn) { - common_peg_parser_builder_helper builder; +common_peg_arena build_chat_peg_parser(F && fn) { + common_chat_peg_parser_builder builder; auto root = fn(builder); builder.set_root(root); return builder.build(); } -struct common_peg_parse_simple_handler { +struct common_chat_peg_simple_handler { std::function log; void operator()(const common_peg_parse_event & ev, common_peg_parse_semantics & semantics) const; }; diff --git a/tests/.gitignore b/tests/.gitignore index a3b089be53e37..ba2b164fac5f1 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1,6 +1,6 @@ * -!chat-peg-parser !*.* *.o ggml-common.h **/*.swp +!peg-parser diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 033dcacee560f..77eeb02a5875b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -185,22 +185,22 @@ endif() llama_build_and_test(test-chat-parser.cpp) llama_build_and_test( - test-chat-peg-parser.cpp - chat-peg-parser/simple_tokenizer.cpp - chat-peg-parser/test-command7-parser-compare.cpp - chat-peg-parser/test-example-qwen3-coder.cpp - chat-peg-parser/test-example-minimax-m2.cpp - chat-peg-parser/test-example-seed-oss.cpp - chat-peg-parser/test-gbnf-generation.cpp - chat-peg-parser/test-json-parser.cpp - chat-peg-parser/test-json-serialization.cpp - chat-peg-parser/test-one.cpp - chat-peg-parser/test-optional.cpp - chat-peg-parser/test-partial-parsing.cpp - chat-peg-parser/test-recursive-references.cpp - chat-peg-parser/test-unicode.cpp - chat-peg-parser/test_harness.h - chat-peg-parser/tests.h + test-peg-parser.cpp + peg-parser/simple_tokenizer.cpp + peg-parser/test-command7-parser-compare.cpp + peg-parser/test-example-qwen3-coder.cpp + peg-parser/test-example-minimax-m2.cpp + peg-parser/test-example-seed-oss.cpp + peg-parser/test-gbnf-generation.cpp + peg-parser/test-json-parser.cpp + peg-parser/test-json-serialization.cpp + peg-parser/test-one.cpp + peg-parser/test-optional.cpp + peg-parser/test-partial-parsing.cpp + peg-parser/test-recursive-references.cpp + peg-parser/test-unicode.cpp + peg-parser/test_harness.h + peg-parser/tests.h ) llama_build_and_test(test-chat-template.cpp) llama_build_and_test(test-json-partial.cpp) diff --git a/tests/chat-peg-parser/convo.json b/tests/peg-parser/convo.json similarity index 100% rename from tests/chat-peg-parser/convo.json rename to tests/peg-parser/convo.json diff --git a/tests/chat-peg-parser/simple_tokenizer.cpp b/tests/peg-parser/simple_tokenizer.cpp similarity index 100% rename from tests/chat-peg-parser/simple_tokenizer.cpp rename to tests/peg-parser/simple_tokenizer.cpp diff --git a/tests/chat-peg-parser/test-command7-parser-compare.cpp b/tests/peg-parser/test-command7-parser-compare.cpp similarity index 99% rename from tests/chat-peg-parser/test-command7-parser-compare.cpp rename to tests/peg-parser/test-command7-parser-compare.cpp index ccfcb9ea6a336..95a0df53dc9ba 100644 --- a/tests/chat-peg-parser/test-command7-parser-compare.cpp +++ b/tests/peg-parser/test-command7-parser-compare.cpp @@ -8,7 +8,7 @@ #include static common_peg_arena create_command_r7b_parser() { - auto parser = build_peg_parser([](common_peg_parser_builder & p) { + auto parser = build_chat_peg_parser([](common_chat_peg_parser_builder & p) { auto thinking = p.rule("thinking", "<|START_THINKING|>" << p.rule("reasoning-content", p.until("<|END_THINKING|>")) << "<|END_THINKING|>"); diff --git a/tests/chat-peg-parser/test-example-minimax-m2.cpp b/tests/peg-parser/test-example-minimax-m2.cpp similarity index 93% rename from tests/chat-peg-parser/test-example-minimax-m2.cpp rename to tests/peg-parser/test-example-minimax-m2.cpp index 2299ccfd1d80b..752149425e9c2 100644 --- a/tests/chat-peg-parser/test-example-minimax-m2.cpp +++ b/tests/peg-parser/test-example-minimax-m2.cpp @@ -1,3 +1,4 @@ +#include "chat-peg-parser.h" #include "common.h" #include "peg-parser.h" #include "nlohmann/json.hpp" @@ -7,7 +8,7 @@ #include void test_example_minimax_m2(testing &t) { - auto helper_parser = build_peg_parser_helper([](common_peg_parser_builder_helper & p) { + auto helper_parser = build_chat_peg_parser([](common_chat_peg_parser_builder & p) { auto thinking = p.reasoning(); auto content = p.content_before_tools(""); auto function = p.quasi_xml_attr("generate_joke", @@ -44,7 +45,7 @@ void test_example_minimax_m2(testing &t) { common_peg_parse_semantics semantics; common_peg_parse_context ctx(in, &semantics, it + 1 == tokens.end()); - common_peg_parse_simple_handler handler; + common_chat_peg_simple_handler handler; ctx.set_event_handler(handler); auto result = helper_parser.parse(ctx); diff --git a/tests/chat-peg-parser/test-example-qwen3-coder.cpp b/tests/peg-parser/test-example-qwen3-coder.cpp similarity index 95% rename from tests/chat-peg-parser/test-example-qwen3-coder.cpp rename to tests/peg-parser/test-example-qwen3-coder.cpp index 9b078f8ff2268..36d31f276c1e4 100644 --- a/tests/chat-peg-parser/test-example-qwen3-coder.cpp +++ b/tests/peg-parser/test-example-qwen3-coder.cpp @@ -5,7 +5,7 @@ #include void test_example_qwen3_coder(testing &t) { - auto explicit_parser = build_peg_parser([](common_peg_parser_builder & p) { + auto explicit_parser = build_chat_peg_parser([](common_chat_peg_parser_builder & p) { auto thinking = p.rule("raw-reasoning", "" << p.rule("reasoning-content", p.until("")) << ""); @@ -35,7 +35,7 @@ void test_example_qwen3_coder(testing &t) { }); - auto helper_parser = build_peg_parser_helper([](common_peg_parser_builder_helper & p) { + auto helper_parser = build_chat_peg_parser([](common_chat_peg_parser_builder & p) { auto thinking = p.reasoning(); auto content = p.content_before_tools(""); auto function = p.quasi_xml_no_attr("search_files", @@ -81,7 +81,7 @@ void test_example_qwen3_coder(testing &t) { common_peg_parse_semantics semantics; common_peg_parse_context ctx(in, &semantics, it == tokens.end() - 1); - common_peg_parse_simple_handler handler; + common_chat_peg_simple_handler handler; // handler.log = [&](const std::string & msg) { // t.log(msg); // }; @@ -115,7 +115,7 @@ void test_example_qwen3_coder(testing &t) { common_peg_parse_semantics semantics; common_peg_parse_context ctx(in, &semantics, it + 1 == tokens.end()); - common_peg_parse_simple_handler handler; + common_chat_peg_simple_handler handler; ctx.set_event_handler(handler); auto result = helper_parser.parse(ctx); diff --git a/tests/chat-peg-parser/test-example-seed-oss.cpp b/tests/peg-parser/test-example-seed-oss.cpp similarity index 91% rename from tests/chat-peg-parser/test-example-seed-oss.cpp rename to tests/peg-parser/test-example-seed-oss.cpp index d789fc312c086..7cb3f49c4248e 100644 --- a/tests/chat-peg-parser/test-example-seed-oss.cpp +++ b/tests/peg-parser/test-example-seed-oss.cpp @@ -1,10 +1,11 @@ +#include "chat-peg-parser.h" #include "tests.h" #include #include void test_example_seed_oss(testing &t) { - auto helper_parser = build_peg_parser_helper([](common_peg_parser_builder_helper & p) { + auto helper_parser = build_chat_peg_parser([](common_chat_peg_parser_builder & p) { auto thinking = p.reasoning("seed:think"); auto content = p.content_before_tools(""); auto function = p.quasi_xml_no_attr("get_weather", @@ -37,7 +38,7 @@ void test_example_seed_oss(testing &t) { common_peg_parse_semantics semantics; common_peg_parse_context ctx(in, &semantics, it == tokens.end()); - common_peg_parse_simple_handler handler; + common_chat_peg_simple_handler handler; ctx.set_event_handler(handler); auto result = helper_parser.parse(ctx); diff --git a/tests/chat-peg-parser/test-gbnf-generation.cpp b/tests/peg-parser/test-gbnf-generation.cpp similarity index 100% rename from tests/chat-peg-parser/test-gbnf-generation.cpp rename to tests/peg-parser/test-gbnf-generation.cpp diff --git a/tests/chat-peg-parser/test-json-parser.cpp b/tests/peg-parser/test-json-parser.cpp similarity index 100% rename from tests/chat-peg-parser/test-json-parser.cpp rename to tests/peg-parser/test-json-parser.cpp diff --git a/tests/chat-peg-parser/test-json-serialization.cpp b/tests/peg-parser/test-json-serialization.cpp similarity index 100% rename from tests/chat-peg-parser/test-json-serialization.cpp rename to tests/peg-parser/test-json-serialization.cpp diff --git a/tests/chat-peg-parser/test-one.cpp b/tests/peg-parser/test-one.cpp similarity index 100% rename from tests/chat-peg-parser/test-one.cpp rename to tests/peg-parser/test-one.cpp diff --git a/tests/chat-peg-parser/test-optional.cpp b/tests/peg-parser/test-optional.cpp similarity index 100% rename from tests/chat-peg-parser/test-optional.cpp rename to tests/peg-parser/test-optional.cpp diff --git a/tests/chat-peg-parser/test-partial-parsing.cpp b/tests/peg-parser/test-partial-parsing.cpp similarity index 100% rename from tests/chat-peg-parser/test-partial-parsing.cpp rename to tests/peg-parser/test-partial-parsing.cpp diff --git a/tests/chat-peg-parser/test-recursive-references.cpp b/tests/peg-parser/test-recursive-references.cpp similarity index 100% rename from tests/chat-peg-parser/test-recursive-references.cpp rename to tests/peg-parser/test-recursive-references.cpp diff --git a/tests/chat-peg-parser/test-unicode.cpp b/tests/peg-parser/test-unicode.cpp similarity index 100% rename from tests/chat-peg-parser/test-unicode.cpp rename to tests/peg-parser/test-unicode.cpp diff --git a/tests/chat-peg-parser/test_harness.h b/tests/peg-parser/test_harness.h similarity index 100% rename from tests/chat-peg-parser/test_harness.h rename to tests/peg-parser/test_harness.h diff --git a/tests/chat-peg-parser/tests.h b/tests/peg-parser/tests.h similarity index 96% rename from tests/chat-peg-parser/tests.h rename to tests/peg-parser/tests.h index 793d64c149981..5481785ccf408 100644 --- a/tests/chat-peg-parser/tests.h +++ b/tests/peg-parser/tests.h @@ -4,7 +4,7 @@ #include "test_harness.h" #include #include "peg-parser.h" -#include "chat-peg-parser-helper.h" +#include "chat-peg-parser.h" #include #include #include diff --git a/tests/test-chat-peg-parser.cpp b/tests/test-peg-parser.cpp similarity index 98% rename from tests/test-chat-peg-parser.cpp rename to tests/test-peg-parser.cpp index 27d880d7a6fd9..2745520ea2015 100644 --- a/tests/test-chat-peg-parser.cpp +++ b/tests/test-peg-parser.cpp @@ -1,4 +1,4 @@ -#include "chat-peg-parser/tests.h" +#include "peg-parser/tests.h" #include "log.h" #include #include From b834f0094646603239146fe5a854c09bc0db0ed8 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Fri, 21 Nov 2025 16:22:14 -0600 Subject: [PATCH 138/183] checkpoint --- common/chat-peg-parser.cpp | 102 ++-- common/chat-peg-parser.h | 54 ++- common/peg-parser.cpp | 201 +++++--- common/peg-parser.h | 151 +++--- tests/CMakeLists.txt | 10 +- tests/peg-parser/playground.cpp | 436 ++++++++++++++++++ tests/peg-parser/simple_tokenizer.cpp | 2 +- tests/peg-parser/simple_tokenizer.h | 6 + tests/peg-parser/test-example-qwen3-coder.cpp | 6 - tests/peg-parser/test_harness.h | 35 +- tests/peg-parser/tests.h | 11 +- tests/test-chat-peg-parser.cpp | 265 +++++++++++ tests/test-peg-parser.cpp | 90 +--- 13 files changed, 1091 insertions(+), 278 deletions(-) create mode 100644 tests/peg-parser/playground.cpp create mode 100644 tests/peg-parser/simple_tokenizer.h create mode 100644 tests/test-chat-peg-parser.cpp diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index cf37dd314f90e..9d1a83ad9c876 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -2,6 +2,7 @@ #include "peg-parser.h" #include +/* common_peg_parser common_chat_peg_parser_builder::reasoning(const std::string & tag) { std::string open_tag; open_tag.append("<").append(tag).append(">"); @@ -151,70 +152,75 @@ common_peg_parser common_chat_peg_parser_builder::quasi_xml_attr( return function; } +*/ -void common_chat_peg_simple_handler::operator()(const common_peg_parse_event & ev, common_peg_parse_semantics & semantics) const { - if (log) { - std::stringstream ss; - ss << "Event: type=" << (ev.type == COMMON_PEG_PARSE_EVENT_NODE_START ? "start" : "end "); - ss << " rule=" << ev.rule; - ss << " result=" << common_peg_parse_result_type_name(ev.status); - ss << " text=" << ev.text; - log(ss.str()); +common_peg_ast_visitor common_chat_peg_constructed_builder::extractor::visitor() { + return [this](const common_peg_ast_node & node) { + extract(node); + }; +} + +void common_chat_peg_constructed_builder::extractor::extract(const common_peg_ast_node & node) { + bool is_reasoning_block = node.tag == REASONING_BLOCK; + bool is_reasoning = node.tag == REASONING; + bool is_content = node.tag == CONTENT; + bool is_tool_name = node.tag == TOOL_NAME; + bool is_tool_close = node.tag == TOOL_CLOSE; + bool is_arg_open = node.tag == TOOL_ARG_OPEN; + bool is_arg_close = node.tag == TOOL_ARG_CLOSE; + bool is_arg_name = node.tag == TOOL_ARG_NAME; + bool is_arg_string = node.tag == TOOL_ARG_STRING_VALUE; + bool is_arg_json = node.tag == TOOL_ARG_JSON_VALUE; + + if (is_reasoning_block) { + result.reasoning_content = std::string(node.text); } - if (ev.rule == "reasoning-content" && ev.ending()) { - semantics.reasoning_content = ev.text; - if (log) { - log(" reasoning_content=" + semantics.reasoning_content); - } + if (is_reasoning) { + result.reasoning_content = std::string(node.text); } - if (ev.rule == "content" && ev.ending()) { - semantics.content = ev.text; - if (log) { - log(" content=" + semantics.content); - } + if (is_content) { + result.content = std::string(node.text); } - if (ev.rule.find("function-start") != std::string::npos && ev.ending() && ev.success()) { - semantics.tool_calls.emplace_back(); - auto & tc = semantics.tool_calls.back(); - tc.name = semantics.captures["tool-name"]; - if (log) { - log(" tool call added"); - log(" name=" + tc.name); - } + if (is_tool_name) { + result.tool_calls.emplace_back(); + current_tool = &result.tool_calls.back(); + arg_count = 0; + + current_tool->name = std::string(node.text); + current_tool->arguments = "{"; } - if (ev.rule.find("arg-start") != std::string::npos && ev.ending() && ev.success()) { - auto & tc = semantics.tool_calls.back(); - auto name = semantics.captures["arg-name"]; - if (tc.arguments.empty()) { - tc.arguments += "{"; - } else { - tc.arguments += ", "; + if (is_arg_open) { + needs_closing_quote = false; + } + + if (is_arg_name) { + if (arg_count > 0) { + current_tool->arguments += ","; } - tc.arguments += "\"" + name + "\": "; + current_tool->arguments += "\"" + std::string(node.text) + "\":"; + ++arg_count; } - if (ev.rule == "arg-string-content" && ev.ending() && ev.success()) { - auto & tc = semantics.tool_calls.back(); - tc.arguments += "\"" + std::string(ev.text); + if (is_arg_string) { + current_tool->arguments += "\"" + std::string(node.text); + needs_closing_quote = true; } - if (ev.annotation == "arg-string" && ev.ending() && ev.success()) { - auto & tc = semantics.tool_calls.back(); - tc.arguments += "\""; - if (log) { - log(" args=" + tc.arguments); + if (is_arg_close) { + if (needs_closing_quote) { + current_tool->arguments += "\""; } } - if (ev.rule == "arg-json-content" && ev.ending() && (ev.success() || ev.need_more_input())) { - auto & tc = semantics.tool_calls.back(); - tc.arguments += std::string(ev.text); - if (log) { - log(" args=" + tc.arguments); - } + if (is_arg_json) { + current_tool->arguments += std::string(node.text); + } + + if (is_tool_close) { + current_tool->arguments += "}"; } } diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index ba35a8ae08f3b..47a85ed59a47f 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -1,8 +1,9 @@ #pragma once +#include "chat.h" #include "peg-parser.h" -#include "log.h" +/* class common_chat_peg_parser_builder : public common_peg_parser_builder { public: // Adds raw-reasoning for the entire reasoning block plus reasoning-content for the contents, by default thinking tag is "think" @@ -30,8 +31,53 @@ common_peg_arena build_chat_peg_parser(F && fn) { builder.set_root(root); return builder.build(); } +*/ -struct common_chat_peg_simple_handler { - std::function log; - void operator()(const common_peg_parse_event & ev, common_peg_parse_semantics & semantics) const; +class common_chat_peg_constructed_builder : public common_peg_parser_builder { + public: + static constexpr const char * REASONING_BLOCK = "reasoning-block"; + static constexpr const char * REASONING = "reasoning"; + static constexpr const char * CONTENT = "content"; + static constexpr const char * TOOL = "tool"; + static constexpr const char * TOOL_OPEN = "tool-open"; + static constexpr const char * TOOL_CLOSE = "tool-close"; + static constexpr const char * TOOL_NAME = "tool-name"; + static constexpr const char * TOOL_ARG = "tool-arg"; + static constexpr const char * TOOL_ARG_OPEN = "tool-arg-open"; + static constexpr const char * TOOL_ARG_CLOSE = "tool-arg-close"; + static constexpr const char * TOOL_ARG_NAME = "tool-arg-name"; + static constexpr const char * TOOL_ARG_STRING_VALUE = "tool-arg-string-value"; + static constexpr const char * TOOL_ARG_JSON_VALUE = "tool-arg-json-value"; + + struct extractor { + common_chat_msg & result; + common_chat_tool_call * current_tool; + int arg_count = 0; + bool needs_closing_quote = false; + + extractor(common_chat_msg & msg) : result(msg) { } + + void extract(const common_peg_ast_node & node); + common_peg_ast_visitor visitor(); + }; + + common_peg_parser reasoning_block(const common_peg_parser & p) { return tag(REASONING_BLOCK, p); } + common_peg_parser reasoning(const common_peg_parser & p) { return tag(REASONING, p); } + common_peg_parser content(const common_peg_parser & p) { return tag(CONTENT, p); } + common_peg_parser tool(const common_peg_parser & p) { return tag(TOOL, p); } + common_peg_parser tool_open(const common_peg_parser & p) { return atomic(tag(TOOL_OPEN, p)); } + common_peg_parser tool_close(const common_peg_parser & p) { return atomic(tag(TOOL_CLOSE, p)); } + common_peg_parser tool_name(const common_peg_parser & p) { return atomic(tag(TOOL_NAME, p)); } + common_peg_parser tool_arg(const common_peg_parser & p) { return tag(TOOL_ARG, p); } + common_peg_parser tool_arg_open(const common_peg_parser & p) { return atomic(tag(TOOL_ARG_OPEN, p)); } + common_peg_parser tool_arg_close(const common_peg_parser & p) { return atomic(tag(TOOL_ARG_CLOSE, p)); } + common_peg_parser tool_arg_name(const common_peg_parser & p) { return atomic(tag(TOOL_ARG_NAME, p)); } + common_peg_parser tool_arg_string_value(const common_peg_parser & p) { return tag(TOOL_ARG_STRING_VALUE, p); } + common_peg_parser tool_arg_json_value(const common_peg_parser & p) { return tag(TOOL_ARG_JSON_VALUE, p); } }; + +inline common_peg_arena build_chat_peg_constructed_parser(const std::function & fn) { + common_chat_peg_constructed_builder builder; + builder.set_root(fn(builder)); + return builder.build(); +} diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp index 922c1ad4317e3..c342c495095f6 100644 --- a/common/peg-parser.cpp +++ b/common/peg-parser.cpp @@ -20,7 +20,7 @@ const char * common_peg_parse_result_type_name(common_peg_parse_result_type type case COMMON_PEG_PARSE_RESULT_FAIL: return "fail"; case COMMON_PEG_PARSE_RESULT_SUCCESS: return "success"; case COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT: return "need_more_input"; - default: return "unknown"; + default: return "unknown"; } } @@ -247,23 +247,42 @@ static std::pair, bool> parse_c } // Parse cache implementation -common_peg_parse_result common_peg_parse_cache::set(common_peg_parser_id id, size_t start, common_peg_parse_result result) { - results[common_peg_parse_cache_key{id, start}] = result; - return result; +const common_peg_parse_result & common_peg_parse_cache::set(common_peg_parser_id id, size_t start, common_peg_parse_result result) { + auto & stored = results[common_peg_parse_cache_key{id, start}]; + stored = std::move(result); + return stored; } -std::optional common_peg_parse_cache::get(common_peg_parser_id id, size_t start) { +common_peg_parse_result * common_peg_parse_cache::get(common_peg_parser_id id, size_t start) { auto it = results.find(common_peg_parse_cache_key{id, start}); if (it != results.end()) { - return it->second; + return &it->second; } - return std::nullopt; + return nullptr; } void common_peg_parse_cache::clear() { results.clear(); } + +void common_peg_ast_arena::visit(common_peg_ast_id id, std::function visitor) { + if (id == COMMON_PEG_INVALID_AST_ID) { + return; + } + const auto & node = get(id); + visitor(node); + for (const auto & child : node.children) { + visit(child, visitor); + } +} + +void common_peg_ast_arena::visit(const common_peg_parse_result & result, std::function visitor) { + for (const auto & node : result.nodes) { + visit(node, visitor); + } +} + // Forward declaration of parser struct parser_executor; @@ -331,16 +350,26 @@ struct parser_executor { common_peg_parse_result operator()(const common_peg_sequence_parser & p) { auto pos = start_pos; + std::vector nodes; + for (const auto & child_id : p.children) { auto result = arena.parse(child_id, ctx, pos); - if (!result.success()) { - return common_peg_parse_result(result.type, start_pos, result.end); + if (result.fail()) { + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos, result.end); + } + + if (!result.nodes.empty()) { + nodes.insert(nodes.end(), result.nodes.begin(), result.nodes.end()); + } + + if (result.need_more_input()) { + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, result.end, std::move(nodes)); } pos = result.end; } - return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, std::move(nodes)); } common_peg_parse_result operator()(const common_peg_choice_parser & p) { @@ -358,6 +387,7 @@ struct parser_executor { common_peg_parse_result operator()(const common_peg_repetition_parser & p) { auto pos = start_pos; int match_count = 0; + std::vector nodes; // Try to match up to max_count times (or unlimited if max_count is -1) while (p.max_count == -1 || match_count < p.max_count) { @@ -372,13 +402,22 @@ struct parser_executor { if (result.end == pos) { break; } + + if (!result.nodes.empty()) { + nodes.insert(nodes.end(), result.nodes.begin(), result.nodes.end()); + } + pos = result.end; match_count++; continue; } if (result.need_more_input()) { - return common_peg_parse_result(result.type, start_pos, result.end); + if (!result.nodes.empty()) { + nodes.insert(nodes.end(), result.nodes.begin(), result.nodes.end()); + } + + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, result.end, std::move(nodes)); } // Child failed - stop trying @@ -388,12 +427,12 @@ struct parser_executor { // Check if we got enough matches if (p.min_count > 0 && match_count < p.min_count) { if (pos >= ctx.input.size() && !ctx.input_is_complete) { - return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos, std::move(nodes)); } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos, pos); } - return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos); + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, std::move(nodes)); } common_peg_parse_result operator()(const common_peg_and_parser & p) { @@ -650,43 +689,52 @@ struct parser_executor { } common_peg_parse_result operator()(const common_peg_rule_parser & p) { - // Fire NODE_START event - if (ctx.event_handler && ctx.semantics) { - ctx.event_handler(common_peg_parse_event{ - COMMON_PEG_PARSE_EVENT_NODE_START, + // Parse the child + auto result = arena.parse(p.child, ctx, start_pos); + + if (!result.fail()) { + std::string_view text; + if (result.start < ctx.input.size()) { + text = std::string_view(ctx.input).substr(result.start, result.end - result.start); + } + + auto node_id = ctx.ast_arena.add_node( p.name, - p.annotation, - start_pos, - start_pos, "", - COMMON_PEG_PARSE_RESULT_FAIL, - ctx.current_depth - }, *ctx.semantics); - ctx.current_depth++; + result.start, + result.end, + text, + std::move(result.nodes), + result.need_more_input() + ); + + return common_peg_parse_result(result.type, result.start, result.end, { node_id }); } + return result; + } + + common_peg_parse_result operator()(const common_peg_tag_parser & p) { // Parse the child auto result = arena.parse(p.child, ctx, start_pos); - // Fire NODE_END event - if (ctx.event_handler && ctx.semantics) { - ctx.current_depth--; - std::string_view text = ctx.input; + if (!result.fail()) { + std::string_view text; if (result.start < ctx.input.size()) { - text = text.substr(result.start, result.end - result.start); - } else { - text = ""; + text = std::string_view(ctx.input).substr(result.start, result.end - result.start); } - ctx.event_handler(common_peg_parse_event{ - COMMON_PEG_PARSE_EVENT_NODE_END, - p.name, - p.annotation, + + auto node_id = ctx.ast_arena.add_node( + "", + p.tag, result.start, result.end, text, - result.type, - ctx.current_depth - }, *ctx.semantics); + std::move(result.nodes), + result.need_more_input() + ); + + return common_peg_parse_result(result.type, result.start, result.end, { node_id }); } return result; @@ -699,14 +747,15 @@ struct parser_executor { common_peg_parse_result operator()(const common_peg_capture_parser & p) { auto result = arena.parse(p.child, ctx, start_pos); + return result; + } - if (!result.fail() && ctx.semantics) { - std::string_view matched = ctx.input; - matched = matched.substr(result.start, result.end - result.start); - std::string value = std::string(matched); - ctx.semantics->captures[p.key] = std::move(value); + common_peg_parse_result operator()(const common_peg_atomic_parser & p) { + auto result = arena.parse(p.child, ctx, start_pos); + if (result.need_more_input()) { + // Clear nodes so they don't propagate up. + result.nodes.clear(); } - return result; } }; @@ -720,7 +769,7 @@ common_peg_parse_result common_peg_arena::parse(common_peg_parse_context & ctx, common_peg_parse_result common_peg_arena::parse(common_peg_parser_id id, common_peg_parse_context & ctx, size_t start) const { // Check cache - auto cached = ctx.cache.get(id, start); + common_peg_parse_result * cached = ctx.cache.get(id, start); if (cached) { return *cached; } @@ -731,7 +780,7 @@ common_peg_parse_result common_peg_arena::parse(common_peg_parser_id id, common_ auto result = std::visit(exec, parser); // Cache result - return ctx.cache.set(id, start, result); + return ctx.cache.set(id, start, std::move(result)); } // Dump implementation (for debugging) @@ -1004,22 +1053,18 @@ common_peg_parser common_peg_parser_builder::capture(const std::string & key, co return wrap(arena_.add_parser(common_peg_capture_parser{p.id(), key})); } -common_peg_parser common_peg_parser_builder::rule(const std::string & name, common_peg_parser p, bool trigger) { - return rule(name, "", p, trigger); +common_peg_parser common_peg_parser_builder::atomic(common_peg_parser p) { + return wrap(arena_.add_parser(common_peg_atomic_parser{p.id()})); } -common_peg_parser common_peg_parser_builder::rule(const std::string & name, const std::string & annotation, common_peg_parser p, bool trigger) { +common_peg_parser common_peg_parser_builder::rule(const std::string & name, common_peg_parser p, bool trigger) { auto clean_name = rule_name(name); - auto rule_id = arena_.add_parser(common_peg_rule_parser{clean_name, annotation, p.id(), trigger}); + auto rule_id = arena_.add_parser(common_peg_rule_parser{clean_name, p.id(), trigger}); arena_.add_rule(clean_name, rule_id); return ref(clean_name); } common_peg_parser common_peg_parser_builder::rule(const std::string & name, const std::function & builder_fn, bool trigger) { - return rule(name, "", builder_fn, trigger); -} - -common_peg_parser common_peg_parser_builder::rule(const std::string & name, const std::string & annotation, const std::function & builder_fn, bool trigger) { auto clean_name = rule_name(name); if (arena_.has_rule(clean_name)) { return ref(clean_name); @@ -1027,19 +1072,23 @@ common_peg_parser common_peg_parser_builder::rule(const std::string & name, cons // Create placeholder rule to allow recursive references auto placeholder = any(); // Temporary placeholder - auto placeholder_rule_id = arena_.add_parser(common_peg_rule_parser{clean_name, annotation, placeholder.id(), trigger}); + auto placeholder_rule_id = arena_.add_parser(common_peg_rule_parser{clean_name, placeholder.id(), trigger}); arena_.add_rule(clean_name, placeholder_rule_id); // Build the actual parser auto parser = builder_fn(); // Replace placeholder with actual rule - auto rule_id = arena_.add_parser(common_peg_rule_parser{clean_name, annotation, parser.id(), trigger}); + auto rule_id = arena_.add_parser(common_peg_rule_parser{clean_name, parser.id(), trigger}); arena_.rules_[clean_name] = rule_id; return ref(clean_name); } +common_peg_parser common_peg_parser_builder::tag(const std::string & tag, common_peg_parser p) { + return wrap(arena_.add_parser(common_peg_tag_parser{p.id(), tag})); +} + void common_peg_parser_builder::set_root(common_peg_parser p) { arena_.set_root(p.id()); } @@ -1204,8 +1253,9 @@ static std::unordered_set collect_reachable_rules( } else if constexpr (std::is_same_v || std::is_same_v || std::is_same_v || - std::is_same_v || - std::is_same_v) { + std::is_same_v || + std::is_same_v || + std::is_same_v) { visit(p.child); } else if constexpr (std::is_same_v) { if (visited.find(p.name) == visited.end()) { @@ -1217,6 +1267,9 @@ static std::unordered_set collect_reachable_rules( // Traverse rules so we pick up everything auto referenced_rule = arena.get_rule(p.name); visit(referenced_rule); + } else if constexpr (std::is_same_v) { + // Schemas should not traverse children so we can prune their + // rules from the generated GBNF grammar. } }, parser); }; @@ -1328,6 +1381,10 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo return p.name; } else if constexpr (std::is_same_v) { return to_gbnf(p.child); + } else if constexpr (std::is_same_v) { + return to_gbnf(p.child); + } else if constexpr (std::is_same_v) { + return to_gbnf(p.child); } else { return ""; } @@ -1451,7 +1508,6 @@ static nlohmann::json serialize_parser_variant(const common_peg_parser_variant & } else if constexpr (std::is_same_v) { j["type"] = "rule"; j["name"] = p.name; - j["annotation"] = p.annotation; j["child"] = p.child; j["trigger"] = p.trigger; } else if constexpr (std::is_same_v) { @@ -1461,6 +1517,13 @@ static nlohmann::json serialize_parser_variant(const common_peg_parser_variant & j["type"] = "capture"; j["child"] = p.child; j["key"] = p.key; + } else if constexpr (std::is_same_v) { + j["type"] = "atomic"; + j["child"] = p.child; + } else if constexpr (std::is_same_v) { + j["type"] = "tag"; + j["child"] = p.child; + j["tag"] = p.tag; } return j; @@ -1584,12 +1647,11 @@ static common_peg_parser_variant deserialize_parser_variant(const nlohmann::json return parser; } if (type == "rule") { - if (!j.contains("name") || !j.contains("annotation") || !j.contains("child") || !j.contains("trigger")) { + if (!j.contains("name") || !j.contains("child") || !j.contains("trigger")) { throw std::runtime_error("rule parser missing required fields"); } return common_peg_rule_parser{ j["name"].get(), - j["annotation"].get(), j["child"].get(), j["trigger"].get() }; @@ -1609,6 +1671,23 @@ static common_peg_parser_variant deserialize_parser_variant(const nlohmann::json j["key"].get() }; } + if (type == "atomic") { + if (!j.contains("child")) { + throw std::runtime_error("tag parser missing required fields"); + } + return common_peg_atomic_parser{ + j["child"].get(), + }; + } + if (type == "tag") { + if (!j.contains("child") || !j.contains("tag")) { + throw std::runtime_error("tag parser missing required fields"); + } + return common_peg_tag_parser{ + j["child"].get(), + j["tag"].get(), + }; + } throw std::runtime_error("Unknown parser type: " + type); } diff --git a/common/peg-parser.h b/common/peg-parser.h index 792d08618c0e4..4603bf8e8b0a9 100644 --- a/common/peg-parser.h +++ b/common/peg-parser.h @@ -6,7 +6,6 @@ #include #include -#include #include #include #include @@ -19,6 +18,9 @@ struct common_grammar_builder; using common_peg_parser_id = size_t; constexpr common_peg_parser_id COMMON_PEG_INVALID_PARSER_ID = static_cast(-1); +using common_peg_ast_id = size_t; +constexpr common_peg_ast_id COMMON_PEG_INVALID_AST_ID = static_cast(-1); + // Forward declare builder for parser wrapper class common_peg_parser_builder; @@ -61,22 +63,6 @@ common_peg_parser operator+(const std::string & str, const common_peg_parser & p common_peg_parser operator<<(const char * str, const common_peg_parser & p); common_peg_parser operator<<(const std::string & str, const common_peg_parser & p); -struct common_peg_parse_semantics { - std::string content; - std::string reasoning_content; - std::vector tool_calls; - - std::unordered_map captures; - - common_chat_msg to_msg() const { - common_chat_msg msg; - msg.content = content; - msg.reasoning_content = reasoning_content; - msg.tool_calls = tool_calls; - return msg; - } -}; - enum common_peg_parse_result_type { COMMON_PEG_PARSE_RESULT_FAIL = 0, COMMON_PEG_PARSE_RESULT_SUCCESS = 1, @@ -85,6 +71,49 @@ enum common_peg_parse_result_type { const char * common_peg_parse_result_type_name(common_peg_parse_result_type type); +struct common_peg_ast_node { + common_peg_ast_id id; + std::string rule_name; + std::string tag; + size_t start; + size_t end; + std::string_view text; + std::vector children; + + bool is_partial = false; +}; + +struct common_peg_parse_result; + +using common_peg_ast_visitor = std::function; + +class common_peg_ast_arena { + std::vector nodes_; + public: + common_peg_ast_id add_node( + const std::string & rule_name, + const std::string & tag, + size_t start, + size_t end, + std::string_view text, + std::vector children, + bool is_partial = false + ) { + common_peg_ast_id id = nodes_.size(); + nodes_.push_back({id, rule_name, tag, start, end, text, std::move(children), is_partial}); + return id; + } + + const common_peg_ast_node & get(common_peg_ast_id id) const { return nodes_.at(id); } + + size_t size() const { return nodes_.size(); } + + void clear() { nodes_.clear(); } + + void visit(common_peg_ast_id id, common_peg_ast_visitor visitor); + void visit(const common_peg_parse_result & result, common_peg_ast_visitor visitor); +}; + struct common_peg_parse_cache_key { common_peg_parser_id id; size_t start; @@ -106,6 +135,8 @@ struct common_peg_parse_result { size_t start = 0; size_t end = 0; + std::vector nodes; + common_peg_parse_result() : type(COMMON_PEG_PARSE_RESULT_FAIL) {} common_peg_parse_result(common_peg_parse_result_type type, size_t start) @@ -114,42 +145,20 @@ struct common_peg_parse_result { common_peg_parse_result(common_peg_parse_result_type type, size_t start, size_t end) : type(type), start(start), end(end) {} + common_peg_parse_result(common_peg_parse_result_type type, size_t start, size_t end, std::vector nodes) + : type(type), start(start), end(end), nodes(std::move(nodes)) {} + bool fail() const { return type == COMMON_PEG_PARSE_RESULT_FAIL; } bool need_more_input() const { return type == COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT; } bool success() const { return type == COMMON_PEG_PARSE_RESULT_SUCCESS; } }; -enum common_peg_parse_event_type { - COMMON_PEG_PARSE_EVENT_NODE_START, - COMMON_PEG_PARSE_EVENT_NODE_END, -}; - -struct common_peg_parse_event { - common_peg_parse_event_type type; - std::string rule; - std::string annotation; - size_t start; - size_t end; - std::string_view text; - common_peg_parse_result_type status; - int depth; - - bool starting() const { return type == COMMON_PEG_PARSE_EVENT_NODE_START; } - bool ending() const { return type == COMMON_PEG_PARSE_EVENT_NODE_END; } - - bool success() const { return status == COMMON_PEG_PARSE_RESULT_SUCCESS; } - bool need_more_input() const { return status == COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT; } - bool fail() const { return status == COMMON_PEG_PARSE_RESULT_FAIL; } -}; - -using common_peg_parse_event_handler = std::function; - class common_peg_parse_cache { std::unordered_map results; public: - common_peg_parse_result set(common_peg_parser_id id, size_t start, common_peg_parse_result result); - std::optional get(common_peg_parser_id id, size_t start); + const common_peg_parse_result & set(common_peg_parser_id id, size_t start, common_peg_parse_result result); + common_peg_parse_result * get(common_peg_parser_id id, size_t start); void clear(); }; @@ -157,36 +166,18 @@ struct common_peg_parse_context { std::string input; bool input_is_complete; common_peg_parse_cache cache; - common_peg_parse_semantics * semantics; - common_peg_parse_event_handler event_handler; + common_peg_ast_arena ast_arena; - int current_depth; int parse_depth; common_peg_parse_context() - : input_is_complete(true), cache(), semantics(nullptr), event_handler(nullptr), current_depth(0), parse_depth(0) {} + : input_is_complete(true), cache(), parse_depth(0) {} common_peg_parse_context(const std::string & input) - : input(input), input_is_complete(true), cache(), semantics(nullptr), event_handler(nullptr), current_depth(0), parse_depth(0) {} + : input(input), input_is_complete(true), cache(), parse_depth(0) {} common_peg_parse_context(const std::string & input, bool complete) - : input(input), input_is_complete(complete), cache(), semantics(nullptr), event_handler(nullptr), current_depth(0), parse_depth(0) {} - - common_peg_parse_context(const std::string & input, common_peg_parse_semantics * semantics) - : input(input), input_is_complete(true), cache(), semantics(semantics), event_handler(nullptr), current_depth(0), parse_depth(0) {} - - common_peg_parse_context(const std::string & input, common_peg_parse_semantics * semantics, bool complete) - : input(input), input_is_complete(complete), cache(), semantics(semantics), event_handler(nullptr), current_depth(0), parse_depth(0) {} - - common_peg_parse_context(const std::string & input, common_peg_parse_semantics * semantics, common_peg_parse_event_handler handler, bool complete = true) - : input(input), input_is_complete(complete), cache(), semantics(semantics), event_handler(std::move(handler)), current_depth(0), parse_depth(0) {} - - template - void set_event_handler(const T & handler) { - event_handler = [&](const common_peg_parse_event & ev, common_peg_parse_semantics & semantics) { - handler(ev, semantics); - }; - } + : input(input), input_is_complete(complete), cache(), parse_depth(0) {} }; // Forward declaration @@ -255,7 +246,6 @@ struct common_peg_schema_parser { struct common_peg_rule_parser { std::string name; - std::string annotation; common_peg_parser_id child; bool trigger; }; @@ -269,6 +259,15 @@ struct common_peg_capture_parser { std::string key; }; +struct common_peg_atomic_parser { + common_peg_parser_id child; +}; + +struct common_peg_tag_parser { + common_peg_parser_id child; + std::string tag; +}; + // Variant holding all parser types using common_peg_parser_variant = std::variant< common_peg_start_parser, @@ -287,7 +286,9 @@ using common_peg_parser_variant = std::variant< common_peg_schema_parser, common_peg_rule_parser, common_peg_ref_parser, - common_peg_capture_parser + common_peg_capture_parser, + common_peg_atomic_parser, + common_peg_tag_parser >; // Arena owns all parsers @@ -452,7 +453,6 @@ class common_peg_parser_builder { // If trigger=true, marks this rule as an entry point for lazy grammar generation. // auto json = p.rule("json", json_obj | json_arr | ...) common_peg_parser rule(const std::string & name, common_peg_parser p, bool trigger = false); - common_peg_parser rule(const std::string & name, const std::string & annotation, common_peg_parser p, bool trigger = false); // Creates a named rule using a builder function. This handles recursive grammars by // inserting a placeholder rule before invoking the builder, allowing the @@ -461,7 +461,15 @@ class common_peg_parser_builder { // If trigger=true, marks this rule as an entry point for lazy grammar generation. // auto json = p.rule("json", [&]() { return json_object() | json_array() | ... }) common_peg_parser rule(const std::string & name, const std::function & builder, bool trigger = false); - common_peg_parser rule(const std::string & name, const std::string & annotation, const std::function & builder, bool trigger = false); + + // Creates an atomic parser. Atomic parsers do not create an AST node if + // the child results in a partial parse, i.e. NEEDS_MORE_INPUT. This is + // intended for situations where partial output is undesirable. + common_peg_parser atomic(common_peg_parser p); + + // Tags create nodes in the generated AST for semantic purposes. + // Unlike rules, you can tag multiple nodes with the same tag. + common_peg_parser tag(const std::string & tag, common_peg_parser p); void set_root(common_peg_parser p); @@ -472,7 +480,6 @@ class common_peg_parser_builder { template common_peg_arena build_peg_parser(F && fn) { common_peg_parser_builder builder; - auto root = fn(builder); - builder.set_root(root); + builder.set_root(fn(builder)); return builder.build(); } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 77eeb02a5875b..97f398c5bfc4a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -184,13 +184,15 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS) endif() llama_build_and_test(test-chat-parser.cpp) +llama_build_and_test(test-chat-peg-parser.cpp peg-parser/simple_tokenizer.cpp) +llama_build_and_test(peg-parser/playground.cpp NAME test-peg-parser-playground) llama_build_and_test( test-peg-parser.cpp peg-parser/simple_tokenizer.cpp - peg-parser/test-command7-parser-compare.cpp - peg-parser/test-example-qwen3-coder.cpp - peg-parser/test-example-minimax-m2.cpp - peg-parser/test-example-seed-oss.cpp + #peg-parser/test-command7-parser-compare.cpp + #peg-parser/test-example-qwen3-coder.cpp + #peg-parser/test-example-minimax-m2.cpp + #peg-parser/test-example-seed-oss.cpp peg-parser/test-gbnf-generation.cpp peg-parser/test-json-parser.cpp peg-parser/test-json-serialization.cpp diff --git a/tests/peg-parser/playground.cpp b/tests/peg-parser/playground.cpp new file mode 100644 index 0000000000000..0f2b6316a385e --- /dev/null +++ b/tests/peg-parser/playground.cpp @@ -0,0 +1,436 @@ +#include "test_harness.h" + +#include "peg-parser.h" +#include "chat-parser.h" +#include "json-schema-to-grammar.h" + +#include +#include +#include +#include +#include + +static common_peg_arena create_command_r7b_parser() { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { + auto thinking = p.rule("thinking", + "<|START_THINKING|>" << p.rule("reasoning-content", p.until("<|END_THINKING|>")) << "<|END_THINKING|>"); + + auto response = p.rule("response", + "<|START_RESPONSE|>" << p.rule("content", p.until("<|END_RESPONSE|>")) << "<|END_RESPONSE|>"); + + auto json = p.rule("json", p.json()); + + auto tool_call_id = p.rule("tool-call-id", + "\"tool_call_id\"" << (":" << p.rule("tool-call-id-value", "\"" + p.json_string_content() + "\""))); + + auto tool_call_name = p.rule("tool-name", + "\"tool_name\"" << (":" << p.rule("tool-name-value", "\"" + p.json_string_content() + "\""))); + + auto tool_call_args = p.rule("tool-args", + "\"parameters\"" << (":" << p.rule("tool-args-value", json))); + + auto tool_call_fields = p.rule("tool-call-fields", tool_call_id | tool_call_name | tool_call_args); + + auto tool_call = p.rule("tool-call", + "{" << tool_call_fields << p.zero_or_more(p.literal(",") << tool_call_fields) << "}"); + + auto tool_calls = p.rule("tool-calls", + "<|START_ACTION|>" + << ("[" << tool_call << p.zero_or_more(p.literal(",") << tool_call) << "]") + << "<|END_ACTION|>"); + + return p.optional(thinking) << (tool_calls | response); + }); + + // Check if + build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); + return parser; +} + +static void test_command_r7b_parser(const common_peg_arena & p, + const std::string & input, + bool need_more_input, + bool /* print_results */) { + common_peg_parse_context ctx(input, !need_more_input); + p.parse(ctx); +} + +static void test_command_r7b_legacy_parser(const std::string & input, + bool need_more_input, + bool print_results) { + // Original common_chat_combinator_parser taken from chat.cpp + common_chat_msg_parser builder(input, + /* .is_partial = */ need_more_input, + { + /* .format = */ COMMON_CHAT_FORMAT_GENERIC, + /* .reasoning_format = */ COMMON_REASONING_FORMAT_AUTO, + /* .reasoning_in_content = */ false, + /* .thinking_forced_open = */ false, + }); + + builder.try_parse_reasoning("<|START_THINKING|>", "<|END_THINKING|>"); + + static const common_regex start_action_regex("<\\|START_ACTION\\|>"); + static const common_regex end_action_regex("<\\|END_ACTION\\|>"); + static const common_regex start_response_regex("<\\|START_RESPONSE\\|>"); + static const common_regex end_response_regex("<\\|END_RESPONSE\\|>"); + + if (auto res = builder.try_find_regex(start_action_regex)) { + // If we didn't extract thoughts, prelude includes them. + auto tool_calls = builder.consume_json_with_dumped_args({ { "parameters" } }); + for (const auto & tool_call : tool_calls.value) { + std::string name = tool_call.contains("tool_name") ? tool_call.at("tool_name") : ""; + std::string id = tool_call.contains("tool_call_id") ? tool_call.at("tool_call_id") : ""; + std::string arguments = tool_call.contains("parameters") ? tool_call.at("parameters") : ""; + if (!builder.add_tool_call(name, id, arguments) || tool_calls.is_partial) { + throw common_chat_msg_partial_exception("incomplete tool call"); + } + } + if (tool_calls.is_partial) { + throw common_chat_msg_partial_exception("incomplete tool call"); + } + builder.consume_regex(end_action_regex); + } else if (auto res = builder.try_find_regex(start_response_regex)) { + if (!builder.try_find_regex(end_response_regex)) { + builder.add_content(builder.consume_rest()); + throw common_chat_msg_partial_exception(end_response_regex.str()); + } + } else { + builder.add_content(builder.consume_rest()); + } + + if (print_results) { + std::cout << "== Parsed (legacy) ==\n"; + std::cout << "=== Reasoning ===\n"; + std::cout << builder.result().reasoning_content << "\n"; + std::cout << "\n\n=== Content ===\n"; + std::cout << builder.result().content << "\n"; + std::cout << "\n\n=== Tool Calls ===\n"; + for (const auto & tc : builder.result().tool_calls) { + std::cout << "id: " << tc.id << "\n"; + std::cout << "name: " << tc.name << "\n"; + std::cout << "args: " << tc.arguments << "\n"; + } + } +} + +static std::vector simple_tokenize(const std::string & input) { + std::vector result; + std::string current; + + for (size_t i = 0; i < input.size(); i++) { + switch (input[i]) { + case ' ': + case '\n': + case '\t': + case '{': + case '}': + case ',': + case '[': + case '"': + case ']': + case '.': + case '<': + case '>': + case '=': + case '/': + if (!current.empty()) { + result.push_back(current); + current.clear(); + } + default:; + } + current += input[i]; + } + + if (!current.empty()) { + result.push_back(current); + } + + return result; +} + +static void print_ast(const common_peg_ast_arena & arena, common_peg_ast_id id, int indent = 0) { + const auto & node = arena.get(id); + + // Indentation + std::string space(indent * 2, ' '); + + // Print Node details + std::cout << space << "Node [" << id << "] " + << (node.rule_name.empty() ? "" : node.rule_name); + + if (node.is_partial) { + std::cout << "*"; + } + + if (!node.tag.empty()) { + std::cout << " (" << node.tag << ")"; + } + + // Print text content (truncated if too long for readability) + std::string_view text = node.text; + if (text.length() > 20) { + std::cout << " : \"" << text.substr(0, 17) << "...\""; + } else { + std::cout << " : \"" << text << "\""; + } + std::cout << "\n"; + + // Recursively print children + for (auto child_id : node.children) { + print_ast(arena, child_id, indent + 1); + } +} + +int main() { + testing t; + + auto explicit_parser = build_peg_parser([](common_peg_parser_builder & p) { + auto thinking = p.tag("raw-reasoning", + "" << p.tag("reasoning-content", p.until("")) << ""); + + auto content = p.tag("content", p.until("")); + + auto arg_open = p.rule("arg-open", + p.atomic("") + ); + + auto arg_close = p.rule( + "arg-close", + "" + + p.peek(p.literal("")) + ); + + auto string_arg = + arg_open + + p.tag("arg-string", p.until_one_of({""})) + + p.atomic(p.tag("arg-string-close", arg_close)); + + auto json = p.json(); + + auto json_arg = arg_open + p.tag("arg-json",json) + arg_close; + + auto function = p.rule("tool", + p.tag("tool-open", p.atomic("")) + + p.one_or_more(json_arg | string_arg) + + p.tag("tool-close", p.literal("")) + ); + + auto tool_call = p.rule("tool-call", + "" + + p.one_or_more(function) + + "", + true); + + return thinking + p.optional(p.space() + content) + p.zero_or_more(p.space() + tool_call) + p.end(); + }); + + std::string input = + "The user wants to find large log files that haven't been accessed recently. " + "I should search for files with .log extension, filter by size (over 100MB), " + "and check access time within the last 30 days. I'll need to use the search_files function." + "Based on your requirements, I'll search for log files over 100MB that haven't been " + "accessed in the last month. This will help identify candidates for cleanup or archival.\n\n" + "" + "" + "/var/log" + "*.log" + "100" + "5" + "false" + "30" + "true" + "size" + "{\"exclude_patterns\": [\"*temp*\", \"*cache*\"], \"file_types\": " + "[\"regular\"]}" + "" + ""; + + std::vector tokens = simple_tokenize(input); + + t.test("parse succeeds", [&](testing &t) { + common_chat_msg prev; + for (auto it = tokens.begin(); it != tokens.end(); it++) { + std::string in = std::accumulate(tokens.begin(), it + 1, std::string()); + + common_peg_parse_context ctx(in, it == tokens.end() - 1); + + auto result = explicit_parser.parse(ctx); + if (!t.assert_equal("not fail", false, result.fail())) { + t.log(in.substr(0, result.end) + "[failed->]" + in.substr(result.end)); + } + + common_chat_msg msg; + common_chat_tool_call *current; + int arg_count = 0; + + ctx.ast_arena.visit(result, [&](const common_peg_ast_node & node) { + bool is_reasoning = node.tag == "reasoning-content"; + bool is_content = node.tag == "content"; + bool is_tool_name = node.tag == "tool-name"; + bool is_tool_close = node.tag == "tool-close"; + bool is_arg_name = node.tag == "arg-name"; + bool is_arg_string = node.tag == "arg-string"; + bool is_arg_string_close = node.tag == "arg-string-close"; + bool is_arg_json = node.tag == "arg-json"; + + if (is_reasoning) { + msg.reasoning_content = std::string(node.text); + } + + if (is_content) { + msg.content = std::string(node.text); + } + + if (is_tool_name) { + msg.tool_calls.emplace_back(); + current = &msg.tool_calls.back(); + arg_count = 0; + + current->name = std::string(node.text); + current->arguments = "{"; + } + + if (is_arg_name) { + if (arg_count > 0) { + current->arguments += ","; + } + current->arguments += "\"" + std::string(node.text) + "\":"; + ++arg_count; + } + + if (is_arg_string) { + current->arguments += "\"" + std::string(node.text); + } + + if (is_arg_string_close) { + current->arguments += "\""; + } + + if (is_arg_json) { + current->arguments += std::string(node.text); + } + + if (is_tool_close) { + current->arguments += "}"; + } + }); + + //t.log("Input: " + input); + t.log("Reasoning: " + msg.reasoning_content); + t.log("Content : " + msg.content); + for (const auto & tc : msg.tool_calls) { + t.log("Tool name: " + tc.name); + t.log("Tool args: " + tc.arguments); + } + + try { + // This shouldn't emit any runtime errors + auto diffs = common_chat_msg_diff::compute_diffs(prev, msg); + } catch(const std::exception & e) { + t.log(in.substr(0, result.end) + "[failed->]" + in.substr(result.end)); + t.assert_true(std::string("failed with ") + e.what(), false); + } + + prev = msg; + } + }); + + // Setup data + auto parser = create_command_r7b_parser(); + + std::string reasoning = "To plan an effective trip to Japan that includes both historical sites and modern attractions within a " + "budget of $4000 for a two-week stay, we need to:\n\n" + "1. Identify key historical sites and modern attractions in Japan.\n" + "2. Find affordable accommodation options that provide a balance between comfort and cost.\n" + "3. Determine the best modes of transportation for getting around Japan.\n" + "4. Create a day-by-day itinerary that ensures the user gets to see a variety of attractions without " + "overspending.\n" + "5. Provide a detailed cost breakdown that includes accommodation, transportation, meals, and entry fees " + "to attractions."; + + std::string content = "For a two-week trip to Japan with a $4,000 budget, I recommend planning an itinerary that balances " + "historical sites with modern attractions. The destination will be Japan, with a duration of 14 days.\n\n" + "Given your interests in both historical sites and modern attractions, you'll want to focus on cities like " + "Kyoto for its temples and traditional culture, Tokyo for its cutting-edge technology and entertainment " + "districts, and possibly Hiroshima or Nara for additional historical significance.\n\n" + "For accommodation, I suggest looking for affordable options such as budget hotels, hostels, or " + "guesthouses that offer good value without sacrificing too much comfort. Japan has excellent mid-range " + "accommodation options that can keep your lodging costs manageable.\n\n" + "Transportation should prioritize efficiencyβ€”consider getting a JR Rail Pass for intercity travel, which " + "allows unlimited rides on most JR trains including the Shinkansen (bullet train). Within cities, use " + "local trains and subways, which are both affordable and highly reliable.\n\n" + "For meals, embrace local cuisine by eating at neighborhood restaurants, ramen shops, and izakayas rather " + "than touristy establishments. This will give you an authentic experience while keeping costs " + "reasonableβ€”you can enjoy excellent meals for $10-20 per person at local spots.\n\n"; + + std::vector> tool_calls = { + { "call_0", "plan_trip", nlohmann::json::parse(R"({ + "destination": "Japan", + "duration": 14, + "budget": 4000, + "interests": ["historical sites", "modern attractions"], + "accommodation_preferences": "affordable", + "transportation_preferences": "efficient", + "meal_preferences": "local cuisine" + })") } + }; + + tokens.clear(); + + // Build tokens + if (!reasoning.empty()) { + auto tokenized = simple_tokenize(reasoning); + tokens.emplace_back("<|START_THINKING|>"); + tokens.insert(tokens.end(), tokenized.begin(), tokenized.end()); + tokens.emplace_back("<|END_THINKING|>"); + } + + if (!content.empty()) { + auto tokenized = simple_tokenize(content); + tokens.emplace_back("<|START_RESPONSE|>"); + tokens.insert(tokens.end(), tokenized.begin(), tokenized.end()); + tokens.emplace_back("<|END_RESPONSE|>"); + } + + if (!tool_calls.empty()) { + tokens.emplace_back("<|START_ACTION|>"); + + auto json = nlohmann::json::array(); + for (const auto & tc : tool_calls) { + auto tc_json = nlohmann::json::object(); + tc_json["tool_call_id"] = std::get<0>(tc); + tc_json["tool_name"] = std::get<1>(tc); + tc_json["parameters"] = std::get<2>(tc); + json.push_back(tc_json); + } + + auto tokenized = simple_tokenize(json.dump(-1, ' ', true)); + tokens.insert(tokens.end(), tokenized.begin(), tokenized.end()); + + tokens.emplace_back("<|END_ACTION|>"); + } + + input = std::accumulate(tokens.begin(), tokens.end(), std::string()); + + // Run tests + t.test("legacy_parse", [&](testing & t) { + test_command_r7b_legacy_parser(input, false, false); + }); + + t.test("current_parse", [&](testing & t) { + test_command_r7b_parser(parser, input, false, false); + }); + + // Run benchmarks + t.bench("legacy_parse_benchmark", [&]() { + test_command_r7b_legacy_parser(input, false, false); + }, 100); + + t.bench("current_parse_benchmark", [&]() { + test_command_r7b_parser(parser, input, false, false); + }, 100); + + return t.summary(); +} diff --git a/tests/peg-parser/simple_tokenizer.cpp b/tests/peg-parser/simple_tokenizer.cpp index 7fb54f7525388..7fcca9390259f 100644 --- a/tests/peg-parser/simple_tokenizer.cpp +++ b/tests/peg-parser/simple_tokenizer.cpp @@ -1,4 +1,4 @@ -#include "tests.h" +#include "simple_tokenizer.h" std::vector simple_tokenize(const std::string & input) { std::vector result; diff --git a/tests/peg-parser/simple_tokenizer.h b/tests/peg-parser/simple_tokenizer.h new file mode 100644 index 0000000000000..1772432c5aa4e --- /dev/null +++ b/tests/peg-parser/simple_tokenizer.h @@ -0,0 +1,6 @@ +#pragma once + +#include +#include + +std::vector simple_tokenize(const std::string &); diff --git a/tests/peg-parser/test-example-qwen3-coder.cpp b/tests/peg-parser/test-example-qwen3-coder.cpp index 36d31f276c1e4..087d638140fdb 100644 --- a/tests/peg-parser/test-example-qwen3-coder.cpp +++ b/tests/peg-parser/test-example-qwen3-coder.cpp @@ -6,9 +6,6 @@ void test_example_qwen3_coder(testing &t) { auto explicit_parser = build_chat_peg_parser([](common_chat_peg_parser_builder & p) { - auto thinking = p.rule("raw-reasoning", - "" << p.rule("reasoning-content", p.until("")) << ""); - auto content = p.rule("content", p.until("")); auto arg_name = p.rule("arg-start", ""); @@ -51,9 +48,6 @@ void test_example_qwen3_coder(testing &t) { t.test("qwen3_accumulation_test", [&](testing &t) { std::string input = - "The user wants to find large log files that haven't been accessed recently. " - "I should search for files with .log extension, filter by size (over 100MB), " - "and check access time within the last 30 days. I'll need to use the search_files function." "Based on your requirements, I'll search for log files over 100MB that haven't been " "accessed in the last month. This will help identify candidates for cleanup or archival.\n\n" "" diff --git a/tests/peg-parser/test_harness.h b/tests/peg-parser/test_harness.h index e65819b2edea3..0562dfb1f69d8 100644 --- a/tests/peg-parser/test_harness.h +++ b/tests/peg-parser/test_harness.h @@ -1,14 +1,19 @@ #pragma once +#include "common.h" + #include #include #include #include +#include #include struct testing { std::ostream &out; std::vector stack; + std::regex filter; + bool filter_tests = false; bool throw_exception = false; int tests = 0; int assertions = 0; @@ -27,10 +32,28 @@ struct testing { return std::string((stack.size() - 1) * 2, ' '); } + std::string full_name() const { + return string_join(stack, "."); + } + void log(const std::string & msg) { out << indent() << " " << msg << "\n"; } + void set_filter(const std::string & re) { + filter = std::regex(re); + filter_tests = true; + } + + bool should_run() const { + if (filter_tests) { + if (!std::regex_match(full_name(), filter)) { + return false; + } + } + return true; + } + template void run_with_exceptions(F &&f, const char *ctx) { try { @@ -88,9 +111,13 @@ struct testing { template void test(const std::string &name, F f) { - ++tests; stack.push_back(name); + if (!should_run()) { + stack.pop_back(); + return; + } + ++tests; out << indent() << name << "\n"; int before_failures = failures; @@ -113,9 +140,13 @@ struct testing { template void bench(const std::string &name, F f, int iterations = 100) { - ++tests; stack.push_back(name); + if (!should_run()) { + stack.pop_back(); + return; + } + ++tests; out << indent() << "[bench] " << name << "\n"; int before_failures = failures; diff --git a/tests/peg-parser/tests.h b/tests/peg-parser/tests.h index 5481785ccf408..f5cbd3f1dd906 100644 --- a/tests/peg-parser/tests.h +++ b/tests/peg-parser/tests.h @@ -1,15 +1,14 @@ #pragma once // Common includes for all test files -#include "test_harness.h" #include -#include "peg-parser.h" -#include "chat-peg-parser.h" -#include -#include #include +#include -std::vector simple_tokenize(const std::string &); +#include "test_harness.h" +#include "peg-parser.h" +#include "chat-peg-parser.h" +#include "simple_tokenizer.h" struct bench_tool_call { std::string id; diff --git a/tests/test-chat-peg-parser.cpp b/tests/test-chat-peg-parser.cpp new file mode 100644 index 0000000000000..754ca497987a3 --- /dev/null +++ b/tests/test-chat-peg-parser.cpp @@ -0,0 +1,265 @@ +#include +#include +#include + +#include "chat-peg-parser.h" +#include "json-schema-to-grammar.h" +#include "peg-parser/test_harness.h" +#include "peg-parser/simple_tokenizer.h" +#include "nlohmann/json.hpp" + +using json = nlohmann::ordered_json; + +static json create_tools(); +static void test_example_qwen3_coder(testing & t); + +int main(int argc, char *argv[]) { + testing t(std::cout); + if (argc >= 2) { + t.set_filter(argv[1]); + } + + t.test("qwen3 coder", test_example_qwen3_coder); + + //t.test("seed_oss", test_example_seed_oss); + //t.test("minimax_m2", test_example_minimax_m2); + //t.test("command7_parser_compare", test_command7_parser_compare); + + return t.summary(); +} + +static json create_tools() { + json tools = json::array(); + + json tool_weather = { + {"type", "function"}, + {"function", { + {"name", "get_current_weather"}, + {"description", "Get the current weather in a given location"}, + {"parameters", { + {"type", "object"}, + {"properties", { + {"location", { + {"type", "string"}, + {"description", "The city and state, e.g. San Francisco, CA"} + }}, + {"unit", { + {"type", "string"}, + {"enum", {"celsius", "fahrenheit"}}, + {"description", "The temperature unit to use. Infer this from the users location."} + }} + }}, + {"required", {"location", "unit"}}, + }}, + }} + }; + tools.push_back(tool_weather); + + json tool_mortgage = { + {"type", "function"}, + {"function", { + {"name", "calculate_mortgage"}, + {"description", "Calculate the monthly mortgage payment based on principal, rate, and term."}, + {"parameters", { + {"type", "object"}, + {"properties", { + {"principal", { + {"type", "number"}, + {"description", "The loan amount in dollars."} + }}, + {"interest_rate", { + {"type", "number"}, + {"description", "Annual interest rate in percentage (e.g., 5.5 for 5.5%)."} + }}, + {"years", { + {"type", "integer"}, + {"description", "The loan term in years."} + }} + }}, + {"required", {"principal", "interest_rate", "years"}}, + {"additionalProperties", false} + }}, + {"strict", true} + }} + }; + tools.push_back(tool_mortgage); + + json tool_search = { + {"type", "function"}, + {"function", { + {"name", "search_knowledge_base"}, + {"description", "Search the internal technical documentation knowledge base."}, + {"parameters", { + {"type", "object"}, + {"properties", { + {"query", { + {"type", "string"}, + {"description", "The search query string."} + }}, + {"max_results", { + {"type", "integer"}, + {"description", "The maximum number of results to return."}, + {"default", 5} + }}, + {"category", { + {"type", "string"}, + {"enum", {"api", "troubleshooting", "billing", "general"}}, + {"description", "Filter search by specific category."} + }} + }}, + {"required", {"query", "category"}}, + {"additionalProperties", false} + }}, + {"strict", true} + }} + }; + tools.push_back(tool_search); + + return tools; +} + +struct tool_argument { + std::string name; + std::string type; + bool is_required; + json schema; +}; + +struct tool_definition { + std::string name; + std::vector arguments; +}; + +void foreach_tool(const json & json_tools, const std::function & fn) { + if (!json_tools.is_array()) { + return; + } + + for (const auto & item : json_tools) { + if (!item.contains("function") || !item["function"].is_object()) { + continue; + } + + const auto & func_node = item["function"]; + + tool_definition tool; + tool.name = func_node.value("name", "unknown_tool"); + + if (func_node.contains("parameters") && func_node["parameters"].is_object()) { + const auto& params_node = func_node["parameters"]; + + std::vector required_list; + if (params_node.contains("required") && params_node["required"].is_array()) { + required_list = params_node["required"].get>(); + } + + if (params_node.contains("properties") && params_node["properties"].is_object()) { + for (const auto & [key, value] : params_node["properties"].items()) { + tool_argument arg; + + arg.name = key; + arg.type = value.value("type", "string"); + arg.schema = value; + + auto it = std::find(required_list.begin(), required_list.end(), arg.name); + arg.is_required = (it != required_list.end()); + + tool.arguments.push_back(arg); + } + } + } + + fn(tool); + } +} + +static void test_example_qwen3_coder(testing & t) { + auto tools = create_tools(); + auto parser = build_chat_peg_constructed_parser([&](common_chat_peg_constructed_builder & p) { + auto content = p.rule("content", p.content(p.until(""))); + + std::vector tool_parsers; + foreach_tool(tools, [&](const tool_definition & def) { + t.log(def.name); + + std::vector arg_parsers; + for (const auto & arg_def : def.arguments) { + auto arg = p.tool_arg( + p.tool_arg_open("") + + (arg_def.type == "string" ? + p.tool_arg_string_value(p.until_one_of({""})) : + p.tool_arg_json_value(p.schema(p.json(), "tool-" + def.name + "-arg-" + def.name + "-schema", arg_def.schema))) + + p.tool_arg_close("" + p.peek(p.literal(""))) + ); + + arg_parsers.push_back(arg_def.is_required ? + p.rule("tool-" + def.name + "-arg-" + arg_def.name, arg) : + p.optional(p.rule("tool-" + def.name + "-arg-" + arg_def.name, arg))); + } + + tool_parsers.push_back(p.rule("tool-" + def.name, + p.tool_open("") + + p.sequence(arg_parsers) + + p.tool_close(p.literal("")) + )); + }); + + auto tool_call = p.rule("tool-call", "" + p.choice(tool_parsers) + "", true); + return content + p.zero_or_more(p.space() + tool_call) + p.end(); + }); + + auto grammar = build_grammar([&](const common_grammar_builder & builder) { + parser.build_grammar(builder); + }); + + t.log("Grammar:\n" + grammar); + + t.test("incremental parsing", [&](testing &t) { + std::string input = + "Let me search the knowledge base for cat pictures." + "" + "" + "cat pictures" + "general" + "" + ""; + + std::vector tokens = simple_tokenize(input); + + common_chat_msg prev; + for (auto it = tokens.begin(); it != tokens.end(); it++) { + std::string in = std::accumulate(tokens.begin(), it + 1, std::string()); + + common_peg_parse_context ctx(in, it == tokens.end() - 1); + + auto result = parser.parse(ctx); + if (!t.assert_equal("not fail", false, result.fail())) { + t.log(in.substr(0, result.end) + "[failed->]" + in.substr(result.end)); + } + + common_chat_msg msg; + auto extractor = common_chat_peg_constructed_builder::extractor(msg); + ctx.ast_arena.visit(result, extractor.visitor()); + + //t.log("Input: " + input); + t.log("==========================================="); + t.log("Iteration " + std::to_string(input.size())); + t.log("Reasoning: " + msg.reasoning_content); + t.log("Content : " + msg.content); + for (const auto & tc : msg.tool_calls) { + t.log("Tool name: " + tc.name); + t.log("Tool args: " + tc.arguments); + } + + try { + // This shouldn't emit any runtime errors + auto diffs = common_chat_msg_diff::compute_diffs(prev, msg); + } catch(const std::exception & e) { + t.log(in.substr(0, result.end) + "[failed->]" + in.substr(result.end)); + t.assert_true(std::string("failed with ") + e.what(), false); + } + + prev = msg; + } + }); +} diff --git a/tests/test-peg-parser.cpp b/tests/test-peg-parser.cpp index 2745520ea2015..954e97ed5b4fc 100644 --- a/tests/test-peg-parser.cpp +++ b/tests/test-peg-parser.cpp @@ -1,84 +1,26 @@ -#include "peg-parser/tests.h" -#include "log.h" -#include #include -#include #include -// Struct to hold test information -struct TestEntry { - std::string codename; - std::string function_name; - void (*test_func)(testing&); -}; - -// Dynamic list of all available tests -static const std::vector all_tests = { - {"partial", "test_partial_parsing", test_partial_parsing}, - {"one", "test_one", test_one}, - {"optional", "test_optional", test_optional}, - {"unicode", "test_unicode", test_unicode}, - {"recursive", "test_recursive_references", test_recursive_references}, - {"json", "test_json_parser", test_json_parser}, - {"gbnf", "test_gbnf_generation", test_gbnf_generation}, - {"qwen3_coder", "test_example_qwen3_coder", test_example_qwen3_coder}, - {"seed_oss", "test_example_seed_oss", test_example_seed_oss}, - {"minimax_m2", "test_example_minimax_m2", test_example_minimax_m2}, - {"command7_parser_compare", "test_command7_parser_compare", test_command7_parser_compare}, - {"serialization", "test_json_serialization", test_json_serialization} -}; - -// Function to list all available tests -static void list_available_tests() { - std::cout << "Available tests:\n"; - for (const auto& test : all_tests) { - std::cout << std::left << std::setw(25) << test.codename << "- " << test.function_name << "\n"; - } - std::cout << "\nUsage:\n"; - std::cout << " test-chat-peg-parser # Run all tests\n"; - std::cout << " test-chat-peg-parser test1 test2 # Run specific tests\n"; - std::cout << " test-chat-peg-parser --tests # List available tests\n"; -} - -// Function to check if a codename matches the provided arguments -static bool should_run_test(const std::vector& args, const std::string& codename) { - // If no arguments provided, run all tests - if (args.size() <= 1) { - return true; - } - - // Check if codename matches any of the provided arguments - return std::find(args.begin() + 1, args.end(), codename) != args.end(); -} - -// Helper to run a test conditionally -static void run_test_conditionally(testing& t, const std::vector& args, - const std::string& codename, void (*test_func)(testing&)) { - if (should_run_test(args, codename)) { - test_func(t); - } -} +#include "peg-parser/tests.h" int main(int argc, char *argv[]) { - // Convert argv to vector of strings for easier handling - std::vector args; - args.reserve(argc); - for (int i = 0; i < argc; ++i) { - args.push_back(argv[i]); - } - - // Special case: list available tests and exit - if (argc == 2 && args[1] == "--tests") { - list_available_tests(); - return 0; - } - testing t(std::cout); - - // Dynamically process all tests from the data structure - for (const auto& test : all_tests) { - run_test_conditionally(t, args, test.codename, test.test_func); + if (argc >= 2) { + t.set_filter(argv[1]); } + t.test("partial", test_partial_parsing); + t.test("one", test_one); + t.test("optional", test_optional); + t.test("unicode", test_unicode); + t.test("recursive", test_recursive_references); + t.test("json", test_json_parser); + t.test("gbnf", test_gbnf_generation); + t.test("serialization", test_json_serialization); + //t.test("qwen3_coder", test_example_qwen3_coder); + //t.test("seed_oss", test_example_seed_oss); + //t.test("minimax_m2", test_example_minimax_m2); + //t.test("command7_parser_compare", test_command7_parser_compare); + return t.summary(); } From 9b3bbd30617321e90e86b610811768c6f908e5c9 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Fri, 21 Nov 2025 19:42:04 -0600 Subject: [PATCH 139/183] use a static_assert to ensure we handle every branch --- common/peg-parser.cpp | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp index c342c495095f6..dc270a263940b 100644 --- a/common/peg-parser.cpp +++ b/common/peg-parser.cpp @@ -15,6 +15,10 @@ #include #include +// Trick to catch missing branches +template +inline constexpr bool is_always_false_v = false; + const char * common_peg_parse_result_type_name(common_peg_parse_result_type type) { switch (type) { case COMMON_PEG_PARSE_RESULT_FAIL: return "fail"; @@ -1242,7 +1246,17 @@ static std::unordered_set collect_reachable_rules( std::visit([&](const auto & p) { using T = std::decay_t; - if constexpr (std::is_same_v) { + if constexpr (std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v) { + // These parsers do not have any children + } else if constexpr (std::is_same_v) { for (auto child : p.children) { visit(child); } @@ -1267,9 +1281,8 @@ static std::unordered_set collect_reachable_rules( // Traverse rules so we pick up everything auto referenced_rule = arena.get_rule(p.name); visit(referenced_rule); - } else if constexpr (std::is_same_v) { - // Schemas should not traverse children so we can prune their - // rules from the generated GBNF grammar. + } else { + static_assert(is_always_false_v); } }, parser); }; @@ -1386,7 +1399,7 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo } else if constexpr (std::is_same_v) { return to_gbnf(p.child); } else { - return ""; + static_assert(is_always_false_v); } }, parser); }; From d38b74109f0a21d952491bf93115a14b2ac3d4dc Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Fri, 21 Nov 2025 20:10:11 -0600 Subject: [PATCH 140/183] inline trivial peg parser builders --- common/peg-parser.cpp | 72 ------------------------------------------- common/peg-parser.h | 38 +++++++++++------------ 2 files changed, 19 insertions(+), 91 deletions(-) diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp index dc270a263940b..37877866658d1 100644 --- a/common/peg-parser.cpp +++ b/common/peg-parser.cpp @@ -912,18 +912,6 @@ static std::string rule_name(const std::string & name) { // Builder implementation common_peg_parser_builder::common_peg_parser_builder() {} -common_peg_parser common_peg_parser_builder::start() { - return wrap(arena_.add_parser(common_peg_start_parser{})); -} - -common_peg_parser common_peg_parser_builder::end() { - return wrap(arena_.add_parser(common_peg_end_parser{})); -} - -common_peg_parser common_peg_parser_builder::literal(const std::string & literal) { - return wrap(arena_.add_parser(common_peg_literal_parser{literal})); -} - common_peg_parser common_peg_parser_builder::sequence(const std::vector & parsers) { // Flatten nested sequences std::vector flattened; @@ -988,63 +976,11 @@ common_peg_parser common_peg_parser_builder::choice(std::initializer_list{delimiter}})); -} - -common_peg_parser common_peg_parser_builder::until_one_of(const std::vector & delimiters) { - return wrap(arena_.add_parser(common_peg_until_parser{delimiters})); -} - -common_peg_parser common_peg_parser_builder::repeat(common_peg_parser p, int min, int max) { - return wrap(arena_.add_parser(common_peg_repetition_parser{p.id(), min, max})); -} - -common_peg_parser common_peg_parser_builder::repeat(common_peg_parser p, int n) { - return wrap(arena_.add_parser(common_peg_repetition_parser{p.id(), n, n})); -} - -common_peg_parser common_peg_parser_builder::optional(common_peg_parser p) { - return repeat(p, 0, 1); -} - -common_peg_parser common_peg_parser_builder::zero_or_more(common_peg_parser p) { - return repeat(p, 0, -1); -} - -common_peg_parser common_peg_parser_builder::one_or_more(common_peg_parser p) { - return repeat(p, 1, -1); -} - common_peg_parser common_peg_parser_builder::json_string_content() { return wrap(arena_.add_parser(common_peg_json_string_parser{})); } @@ -1057,10 +993,6 @@ common_peg_parser common_peg_parser_builder::capture(const std::string & key, co return wrap(arena_.add_parser(common_peg_capture_parser{p.id(), key})); } -common_peg_parser common_peg_parser_builder::atomic(common_peg_parser p) { - return wrap(arena_.add_parser(common_peg_atomic_parser{p.id()})); -} - common_peg_parser common_peg_parser_builder::rule(const std::string & name, common_peg_parser p, bool trigger) { auto clean_name = rule_name(name); auto rule_id = arena_.add_parser(common_peg_rule_parser{clean_name, p.id(), trigger}); @@ -1089,10 +1021,6 @@ common_peg_parser common_peg_parser_builder::rule(const std::string & name, cons return ref(clean_name); } -common_peg_parser common_peg_parser_builder::tag(const std::string & tag, common_peg_parser p) { - return wrap(arena_.add_parser(common_peg_tag_parser{p.id(), tag})); -} - void common_peg_parser_builder::set_root(common_peg_parser p) { arena_.set_root(p.id()); } diff --git a/common/peg-parser.h b/common/peg-parser.h index 4603bf8e8b0a9..d10257f53d303 100644 --- a/common/peg-parser.h +++ b/common/peg-parser.h @@ -340,23 +340,23 @@ class common_peg_arena { class common_peg_parser_builder { common_peg_arena arena_; - // Helper to wrap common_peg_parser_id with this builder common_peg_parser wrap(common_peg_parser_id id) { return common_peg_parser(id, this); } + common_peg_parser add(const common_peg_parser_variant & p) { return wrap(arena_.add_parser(p)); } public: common_peg_parser_builder(); // Matches the start of the input. // S -> ^ - common_peg_parser start(); + common_peg_parser start() { return add(common_peg_start_parser{}); } // Matches the end of the input. // S -> $ - common_peg_parser end(); + common_peg_parser end() { return add(common_peg_end_parser{}); } // Matches an exact literal string. // S -> "hello" - common_peg_parser literal(const std::string & literal); + common_peg_parser literal(const std::string & literal) { return add(common_peg_literal_parser{literal}); } // Matches a sequence of parsers in order, all must succeed. // S -> A B C @@ -372,27 +372,27 @@ class common_peg_parser_builder { // Matches one or more repetitions of a parser. // S -> A+ - common_peg_parser one_or_more(common_peg_parser p); + common_peg_parser one_or_more(common_peg_parser p) { return repeat(p, 1, -1); } // Matches zero or more repetitions of a parser, always succeeds. // S -> A* - common_peg_parser zero_or_more(common_peg_parser p); + common_peg_parser zero_or_more(common_peg_parser p) { return repeat(p, 0, -1); } // Matches zero or one occurrence of a parser, always succeeds. // S -> A? - common_peg_parser optional(common_peg_parser p); + common_peg_parser optional(common_peg_parser p) { return repeat(p, 0, 1); } // Positive lookahead: succeeds if child parser succeeds, consumes no input. // S -> &A - common_peg_parser peek(common_peg_parser p); + common_peg_parser peek(common_peg_parser p) { return add(common_peg_and_parser{p}); } // Negative lookahead: succeeds if child parser fails, consumes no input. // S -> !A - common_peg_parser negate(common_peg_parser p); + common_peg_parser negate(common_peg_parser p) { return add(common_peg_not_parser{p}); } // Matches any single character. // S -> . - common_peg_parser any(); + common_peg_parser any() { return add(common_peg_any_parser{}); } // Matches between min and max repetitions of characters from a character class. // S -> [a-z]{m,n} @@ -404,30 +404,30 @@ class common_peg_parser_builder { // S -> [a-z] or S -> [^0-9] // // Equivalent to chars(classes, 1, 1) - common_peg_parser one(const std::string & classes); + common_peg_parser one(const std::string & classes) { return chars(classes, 1, 1); } // Creates a lightweight reference to a named rule (resolved during build()). // Use this for forward references in recursive grammars. // expr_ref -> expr - common_peg_parser ref(const std::string & name); + common_peg_parser ref(const std::string & name) { return add(common_peg_ref_parser{name}); } // Matches zero or more whitespace characters (space, tab, newline). // S -> [ \t\n]* - common_peg_parser space(); + common_peg_parser space() { return add(common_peg_space_parser{}); } // Matches all characters until a delimiter is found (delimiter not consumed). // S -> (!delim .)* - common_peg_parser until(const std::string & delimiter); - common_peg_parser until_one_of(const std::vector & delimiters); + common_peg_parser until(const std::string & delimiter) { return add(common_peg_until_parser{{delimiter}}); } + common_peg_parser until_one_of(const std::vector & delimiters) { return add(common_peg_until_parser{delimiters}); } // Matches between min and max repetitions of a parser (inclusive). // S -> A{m,n} // Use -1 for max to represent unbounded repetition (equivalent to {m,}) - common_peg_parser repeat(common_peg_parser p, int min, int max); + common_peg_parser repeat(common_peg_parser p, int min, int max) { return add(common_peg_repetition_parser{p, min,max}); } // Matches exactly n repetitions of a parser. // S -> A{n} - common_peg_parser repeat(common_peg_parser p, int n); + common_peg_parser repeat(common_peg_parser p, int n) { return repeat(p, n, n); } // Creates a complete JSON parser supporting objects, arrays, strings, numbers, booleans, and null. // value -> object | array | string | number | true | false | null @@ -465,11 +465,11 @@ class common_peg_parser_builder { // Creates an atomic parser. Atomic parsers do not create an AST node if // the child results in a partial parse, i.e. NEEDS_MORE_INPUT. This is // intended for situations where partial output is undesirable. - common_peg_parser atomic(common_peg_parser p); + common_peg_parser atomic(common_peg_parser p) { return add(common_peg_atomic_parser{p}); } // Tags create nodes in the generated AST for semantic purposes. // Unlike rules, you can tag multiple nodes with the same tag. - common_peg_parser tag(const std::string & tag, common_peg_parser p); + common_peg_parser tag(const std::string & tag, common_peg_parser p) { return add(common_peg_tag_parser{p.id(), tag}); } void set_root(common_peg_parser p); From 54de91f29435dfdb91abb417edbfc8d2ab9858af Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Fri, 21 Nov 2025 20:25:30 -0600 Subject: [PATCH 141/183] use json strings for now --- common/peg-parser.h | 5 +++++ tests/test-chat-peg-parser.cpp | 17 ++++++++++------- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/common/peg-parser.h b/common/peg-parser.h index d10257f53d303..56ffd603da496 100644 --- a/common/peg-parser.h +++ b/common/peg-parser.h @@ -462,6 +462,11 @@ class common_peg_parser_builder { // auto json = p.rule("json", [&]() { return json_object() | json_array() | ... }) common_peg_parser rule(const std::string & name, const std::function & builder, bool trigger = false); + // Creates a trigger rule. When generating a lazy grammar from the parser, + // only trigger rules and descendents are emitted. + common_peg_parser trigger_rule(const std::string & name, common_peg_parser p) { return rule(name, p, true); } + common_peg_parser trigger_rule(const std::string & name, const std::function & builder) { return rule(name, builder, true); } + // Creates an atomic parser. Atomic parsers do not create an AST node if // the child results in a partial parse, i.e. NEEDS_MORE_INPUT. This is // intended for situations where partial output is undesirable. diff --git a/tests/test-chat-peg-parser.cpp b/tests/test-chat-peg-parser.cpp index 754ca497987a3..229091dfb97c4 100644 --- a/tests/test-chat-peg-parser.cpp +++ b/tests/test-chat-peg-parser.cpp @@ -128,9 +128,10 @@ struct tool_argument { struct tool_definition { std::string name; std::vector arguments; + json schema; }; -void foreach_tool(const json & json_tools, const std::function & fn) { +void foreach_tool(const json & json_tools, const std::function & fn) { if (!json_tools.is_array()) { return; } @@ -144,6 +145,7 @@ void foreach_tool(const json & json_tools, const std::function") + - (arg_def.type == "string" ? - p.tool_arg_string_value(p.until_one_of({""})) : - p.tool_arg_json_value(p.schema(p.json(), "tool-" + def.name + "-arg-" + def.name + "-schema", arg_def.schema))) + + p.tool_arg_json_value(p.schema(p.json(), "tool-" + def.name + "-arg-" + def.name + "-schema", arg_def.schema)) + p.tool_arg_close("" + p.peek(p.literal(""))) ); @@ -204,11 +204,14 @@ static void test_example_qwen3_coder(testing & t) { )); }); - auto tool_call = p.rule("tool-call", "" + p.choice(tool_parsers) + "", true); + auto tool_call = p.trigger_rule("tool-call", "" + p.choice(tool_parsers) + ""); return content + p.zero_or_more(p.space() + tool_call) + p.end(); }); auto grammar = build_grammar([&](const common_grammar_builder & builder) { + foreach_tool(tools, [&](tool_definition & def) { + builder.resolve_refs(def.schema); + }); parser.build_grammar(builder); }); @@ -219,8 +222,8 @@ static void test_example_qwen3_coder(testing & t) { "Let me search the knowledge base for cat pictures." "" "" - "cat pictures" - "general" + "\"cat pictures\"" + "\"general\"" "" ""; From 27fb129a08cce4b5024e1e9c4d64c9786248387f Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Fri, 21 Nov 2025 21:24:04 -0600 Subject: [PATCH 142/183] implement basic and native chat peg parser builders/extractors --- common/chat-peg-parser.cpp | 219 ++++++++------------------------- common/chat-peg-parser.h | 108 +++++++++------- tests/test-chat-peg-parser.cpp | 219 +++++++++++++++++++++++++++++++-- 3 files changed, 324 insertions(+), 222 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 9d1a83ad9c876..0232f56190cca 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -1,176 +1,15 @@ #include "chat-peg-parser.h" -#include "peg-parser.h" -#include - -/* -common_peg_parser common_chat_peg_parser_builder::reasoning(const std::string & tag) { - std::string open_tag; - open_tag.append("<").append(tag).append(">"); - std::string close_tag; - close_tag.append(""); - return rule("raw-reasoning", literal(open_tag) << rule("reasoning-content", until(close_tag)) << literal(close_tag)); -} - -common_peg_parser common_chat_peg_parser_builder::content_before_tools(const std::string & tag) { - return rule("content", until(tag)); -} - -common_peg_parser common_chat_peg_parser_builder::quasi_xml_no_attr( - const std::string & function_name, - const std::vector & parameters, - const std::string & function_tag, - const std::string & param_tag) { - std::vector args; - - for (auto it = parameters.begin(); it != parameters.end(); it++) { - std::string arg_start_name; - arg_start_name.append("arg-start-").append(*it); - - std::string param_open; - param_open.append("<").append(param_tag).append("="); - - std::string param_open_after_name = ">"; - - auto arg_name = rule(arg_start_name, literal(param_open) + capture("arg-name", literal(*it)) + literal(param_open_after_name)); - - std::string param_close_end; - param_close_end.append(""); - - std::string param_close_peek; - param_close_peek.append(""); - - std::string param_peek_open; - param_peek_open.append("<").append(param_tag).append("="); - auto arg_end = rule("arg-end", literal(param_close_end) + peek(literal(param_peek_open) | literal(param_close_peek))); - - std::string string_content_1; - string_content_1.append("<").append(param_tag).append("="); - - std::string string_content_2; - string_content_2.append(""); - - auto string_arg_content = rule("arg-string-content", until_one_of({ string_content_1, string_content_2 })); - - std::string arg_string_name; - arg_string_name.append("arg-string-").append(*it); - auto string_arg = rule(arg_string_name, "arg-string", arg_name + string_arg_content + arg_end); - auto json_sec = json(); - - std::string arg_json_name; - arg_json_name.append("arg-json-").append(*it); - auto json_arg = rule(arg_json_name, arg_name + rule("arg-json-content", json_sec) + arg_end); - auto arg_json_or_string = one_or_more(json_arg | string_arg); - args.push_back(arg_json_or_string); - } - auto args_sequence = sequence(args); - - std::string function_start_name; - function_start_name.append("function-start-").append(function_name); - - std::string function_open; - function_open.append("<").append(function_tag).append("="); - - std::string function_open_after_name; - function_open_after_name = ">"; - - std::string function_close; - function_close.append(""); - - std::string function_rule_name; - function_rule_name.append("function-").append(function_name); - auto function = rule(function_rule_name, rule(function_start_name, literal(function_open) + capture("tool-name", literal(function_name)) + literal(function_open_after_name)) + args_sequence + literal(function_close)); - - return function; -} - -common_peg_parser common_chat_peg_parser_builder::quasi_xml_attr( - const std::string & function_name, - const std::vector & parameters, - const std::string & function_tag, - const std::string & param_tag, - const std::string & name_attr) { - std::vector args; - - for (auto it = parameters.begin(); it != parameters.end(); it++) { - std::string arg_start_name; - arg_start_name.append("arg-start-").append(*it); - - std::string param_open; - param_open.append("<").append(param_tag).append(" ").append(name_attr).append("=\""); - - std::string param_open_after_name ="\">"; - - auto arg_name = rule(arg_start_name, literal(param_open) + capture("arg-name", literal(*it)) + literal(param_open_after_name)); - - std::string param_close_end; - param_close_end.append(""); - - std::string param_close_peek; - param_close_peek.append(""); - std::string param_peek_open; - param_peek_open.append("<").append(param_tag).append(" ").append(name_attr).append("=\""); - auto arg_end = rule("arg-end", literal(param_close_end) + peek(literal(param_peek_open) | literal(param_close_peek))); - - std::string string_content_1; - string_content_1.append("<").append(param_tag).append("="); - - std::string string_content_2; - string_content_2.append(""); - - auto string_arg_content = rule("arg-string-content", until_one_of({ string_content_1, string_content_2 })); - - std::string arg_string_name; - arg_string_name.append("arg-string-").append(*it); - auto string_arg = rule(arg_string_name, "arg-string", arg_name + string_arg_content + arg_end); - auto json_sec = json(); - - std::string arg_json_name; - arg_json_name.append("arg-json-").append(*it); - auto json_arg = rule(arg_json_name, arg_name + rule("arg-json-content", json_sec) + arg_end); - auto arg_json_or_string = one_or_more(json_arg | string_arg); - args.push_back(arg_json_or_string); - } - auto args_sequence = sequence(args); - - std::string function_start_name; - function_start_name.append("function-start-").append(function_name); - - std::string function_open; - function_open.append("<").append(function_tag).append(" ").append(name_attr).append("=\""); - - std::string function_open_after_name = "\">"; - - std::string function_close; - function_close.append(""); - - std::string function_rule_name; - function_rule_name.append("function-").append(function_name); - auto function = rule(function_rule_name, - rule(function_start_name, literal(function_open) + capture("tool-name", literal(function_name)) + - literal(function_open_after_name)) + args_sequence + literal(function_close)); - - return function; -} -*/ - -common_peg_ast_visitor common_chat_peg_constructed_builder::extractor::visitor() { +common_peg_ast_visitor common_chat_peg_extractor::visitor() { return [this](const common_peg_ast_node & node) { extract(node); }; } -void common_chat_peg_constructed_builder::extractor::extract(const common_peg_ast_node & node) { - bool is_reasoning_block = node.tag == REASONING_BLOCK; - bool is_reasoning = node.tag == REASONING; - bool is_content = node.tag == CONTENT; - bool is_tool_name = node.tag == TOOL_NAME; - bool is_tool_close = node.tag == TOOL_CLOSE; - bool is_arg_open = node.tag == TOOL_ARG_OPEN; - bool is_arg_close = node.tag == TOOL_ARG_CLOSE; - bool is_arg_name = node.tag == TOOL_ARG_NAME; - bool is_arg_string = node.tag == TOOL_ARG_STRING_VALUE; - bool is_arg_json = node.tag == TOOL_ARG_JSON_VALUE; +void common_chat_peg_extractor::extract(const common_peg_ast_node & node) { + bool is_reasoning_block = node.tag == common_chat_peg_builder::REASONING_BLOCK; + bool is_reasoning = node.tag == common_chat_peg_builder::REASONING; + bool is_content = node.tag == common_chat_peg_builder::CONTENT; if (is_reasoning_block) { result.reasoning_content = std::string(node.text); @@ -183,6 +22,44 @@ void common_chat_peg_constructed_builder::extractor::extract(const common_peg_as if (is_content) { result.content = std::string(node.text); } +} + +void common_chat_peg_native_extractor::extract(const common_peg_ast_node & node) { + common_chat_peg_extractor::extract(node); + + bool is_tool_open = node.tag == common_chat_peg_native_builder::TOOL_OPEN; + bool is_tool_name = node.tag == common_chat_peg_native_builder::TOOL_NAME; + bool is_tool_id = node.tag == common_chat_peg_native_builder::TOOL_ID; + bool is_tool_args = node.tag == common_chat_peg_native_builder::TOOL_ARGS; + + if (is_tool_open) { + result.tool_calls.emplace_back(); + current_tool = &result.tool_calls.back(); + } + + if (is_tool_id && current_tool) { + current_tool->id = std::string(node.text); + } + + if (is_tool_name && current_tool) { + current_tool->name = std::string(node.text); + } + + if (is_tool_args && current_tool) { + current_tool->arguments = std::string(node.text); + } +} + +void common_chat_peg_constructed_extractor::extract(const common_peg_ast_node & node) { + common_chat_peg_extractor::extract(node); + + bool is_tool_name = node.tag == common_chat_peg_constructed_builder::TOOL_NAME; + bool is_tool_close = node.tag == common_chat_peg_constructed_builder::TOOL_CLOSE; + bool is_arg_open = node.tag == common_chat_peg_constructed_builder::TOOL_ARG_OPEN; + bool is_arg_close = node.tag == common_chat_peg_constructed_builder::TOOL_ARG_CLOSE; + bool is_arg_name = node.tag == common_chat_peg_constructed_builder::TOOL_ARG_NAME; + bool is_arg_string = node.tag == common_chat_peg_constructed_builder::TOOL_ARG_STRING_VALUE; + bool is_arg_json = node.tag == common_chat_peg_constructed_builder::TOOL_ARG_JSON_VALUE; if (is_tool_name) { result.tool_calls.emplace_back(); @@ -197,7 +74,7 @@ void common_chat_peg_constructed_builder::extractor::extract(const common_peg_as needs_closing_quote = false; } - if (is_arg_name) { + if (is_arg_name && current_tool) { if (arg_count > 0) { current_tool->arguments += ","; } @@ -205,22 +82,22 @@ void common_chat_peg_constructed_builder::extractor::extract(const common_peg_as ++arg_count; } - if (is_arg_string) { + if (is_arg_string && current_tool) { current_tool->arguments += "\"" + std::string(node.text); needs_closing_quote = true; } - if (is_arg_close) { + if (is_arg_close && current_tool) { if (needs_closing_quote) { current_tool->arguments += "\""; } } - if (is_arg_json) { + if (is_arg_json && current_tool) { current_tool->arguments += std::string(node.text); } - if (is_tool_close) { + if (is_tool_close && current_tool) { current_tool->arguments += "}"; } } diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index 47a85ed59a47f..2567ccf5451d8 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -3,41 +3,67 @@ #include "chat.h" #include "peg-parser.h" -/* -class common_chat_peg_parser_builder : public common_peg_parser_builder { +class common_chat_peg_builder : public common_peg_parser_builder { public: - // Adds raw-reasoning for the entire reasoning block plus reasoning-content for the contents, by default thinking tag is "think" - common_peg_parser reasoning(const std::string & tag = "think"); - - // Adds main content block before tool call block, due to the varied nature of tool call openers (not always XML-like) full tag is required - common_peg_parser content_before_tools(const std::string &tag); - - // Adds a quasi-XML tool call spec without a separate name attribute (Qwen3 style); - // TODO: accept parameter schemas (required, value types etc.) - common_peg_parser quasi_xml_no_attr(const std::string &function_name, const std::vector ¶meters, - const std::string &function_tag = "function", const std::string ¶m_tag = "parameter"); - - // Adds a quasi-XML tool call spec with a separate name attribute (Minimax-M2 style) - // TODO: accept parameter schemas (required, value types etc.) - common_peg_parser quasi_xml_attr(const std::string &function_name, const std::vector ¶meters, - const std::string &function_tag = "invoke", const std::string ¶m_tag = "parameter", - const std::string &name_attr = "name"); + static constexpr const char * REASONING_BLOCK = "reasoning-block"; + static constexpr const char * REASONING = "reasoning"; + static constexpr const char * CONTENT = "content"; + + common_peg_parser reasoning_block(const common_peg_parser & p) { return tag(REASONING_BLOCK, p); } + common_peg_parser reasoning(const common_peg_parser & p) { return tag(REASONING, p); } + common_peg_parser content(const common_peg_parser & p) { return tag(CONTENT, p); } }; -template -common_peg_arena build_chat_peg_parser(F && fn) { - common_chat_peg_parser_builder builder; - auto root = fn(builder); - builder.set_root(root); +inline common_peg_arena build_chat_peg_parser(const std::function & fn) { + common_chat_peg_builder builder; + builder.set_root(fn(builder)); return builder.build(); } -*/ -class common_chat_peg_constructed_builder : public common_peg_parser_builder { +class common_chat_peg_extractor { + public: + common_chat_msg & result; + + common_chat_peg_extractor(common_chat_msg & msg) : result(msg) {} + + virtual void extract(const common_peg_ast_node & node); + common_peg_ast_visitor visitor(); +}; + +class common_chat_peg_native_builder : public common_chat_peg_builder { + public: + static constexpr const char * TOOL = "tool"; + static constexpr const char * TOOL_OPEN = "tool-open"; + static constexpr const char * TOOL_CLOSE = "tool-close"; + static constexpr const char * TOOL_ID = "tool-id"; + static constexpr const char * TOOL_NAME = "tool-name"; + static constexpr const char * TOOL_ARGS = "tool-args"; + + common_peg_parser tool(const common_peg_parser & p) { return tag(TOOL, p); } + common_peg_parser tool_open(const common_peg_parser & p) { return atomic(tag(TOOL_OPEN, p)); } + common_peg_parser tool_close(const common_peg_parser & p) { return atomic(tag(TOOL_CLOSE, p)); } + common_peg_parser tool_id(const common_peg_parser & p) { return atomic(tag(TOOL_ID, p)); } + common_peg_parser tool_name(const common_peg_parser & p) { return atomic(tag(TOOL_NAME, p)); } + common_peg_parser tool_args(const common_peg_parser & p) { return tag(TOOL_ARGS, p); } +}; + +class common_chat_peg_native_extractor : public common_chat_peg_extractor { + common_chat_tool_call * current_tool; + + public: + common_chat_peg_native_extractor(common_chat_msg & msg) : common_chat_peg_extractor(msg) {} + + void extract(const common_peg_ast_node & node) override; +}; + +inline common_peg_arena build_chat_peg_native_parser(const std::function & fn) { + common_chat_peg_native_builder builder; + builder.set_root(fn(builder)); + return builder.build(); +} + +class common_chat_peg_constructed_builder : public common_chat_peg_builder { public: - static constexpr const char * REASONING_BLOCK = "reasoning-block"; - static constexpr const char * REASONING = "reasoning"; - static constexpr const char * CONTENT = "content"; static constexpr const char * TOOL = "tool"; static constexpr const char * TOOL_OPEN = "tool-open"; static constexpr const char * TOOL_CLOSE = "tool-close"; @@ -49,21 +75,6 @@ class common_chat_peg_constructed_builder : public common_peg_parser_builder { static constexpr const char * TOOL_ARG_STRING_VALUE = "tool-arg-string-value"; static constexpr const char * TOOL_ARG_JSON_VALUE = "tool-arg-json-value"; - struct extractor { - common_chat_msg & result; - common_chat_tool_call * current_tool; - int arg_count = 0; - bool needs_closing_quote = false; - - extractor(common_chat_msg & msg) : result(msg) { } - - void extract(const common_peg_ast_node & node); - common_peg_ast_visitor visitor(); - }; - - common_peg_parser reasoning_block(const common_peg_parser & p) { return tag(REASONING_BLOCK, p); } - common_peg_parser reasoning(const common_peg_parser & p) { return tag(REASONING, p); } - common_peg_parser content(const common_peg_parser & p) { return tag(CONTENT, p); } common_peg_parser tool(const common_peg_parser & p) { return tag(TOOL, p); } common_peg_parser tool_open(const common_peg_parser & p) { return atomic(tag(TOOL_OPEN, p)); } common_peg_parser tool_close(const common_peg_parser & p) { return atomic(tag(TOOL_CLOSE, p)); } @@ -76,6 +87,17 @@ class common_chat_peg_constructed_builder : public common_peg_parser_builder { common_peg_parser tool_arg_json_value(const common_peg_parser & p) { return tag(TOOL_ARG_JSON_VALUE, p); } }; +class common_chat_peg_constructed_extractor : public common_chat_peg_extractor { + common_chat_tool_call * current_tool; + int arg_count = 0; + bool needs_closing_quote = false; + + public: + common_chat_peg_constructed_extractor(common_chat_msg & msg) : common_chat_peg_extractor(msg) {} + + void extract(const common_peg_ast_node & node) override; +}; + inline common_peg_arena build_chat_peg_constructed_parser(const std::function & fn) { common_chat_peg_constructed_builder builder; builder.set_root(fn(builder)); diff --git a/tests/test-chat-peg-parser.cpp b/tests/test-chat-peg-parser.cpp index 229091dfb97c4..60c5ea5069bb5 100644 --- a/tests/test-chat-peg-parser.cpp +++ b/tests/test-chat-peg-parser.cpp @@ -2,6 +2,7 @@ #include #include +#include "chat-parser.h" #include "chat-peg-parser.h" #include "json-schema-to-grammar.h" #include "peg-parser/test_harness.h" @@ -12,6 +13,7 @@ using json = nlohmann::ordered_json; static json create_tools(); static void test_example_qwen3_coder(testing & t); +static void test_command7_parser_compare(testing & t); int main(int argc, char *argv[]) { testing t(std::cout); @@ -20,10 +22,7 @@ int main(int argc, char *argv[]) { } t.test("qwen3 coder", test_example_qwen3_coder); - - //t.test("seed_oss", test_example_seed_oss); - //t.test("minimax_m2", test_example_minimax_m2); - //t.test("command7_parser_compare", test_command7_parser_compare); + t.test("comparison", test_command7_parser_compare); return t.summary(); } @@ -131,7 +130,7 @@ struct tool_definition { json schema; }; -void foreach_tool(const json & json_tools, const std::function & fn) { +static void foreach_tool(const json & json_tools, const std::function & fn) { if (!json_tools.is_array()) { return; } @@ -189,7 +188,7 @@ static void test_example_qwen3_coder(testing & t) { auto arg = p.tool_arg( p.tool_arg_open("") + p.tool_arg_json_value(p.schema(p.json(), "tool-" + def.name + "-arg-" + def.name + "-schema", arg_def.schema)) + - p.tool_arg_close("" + p.peek(p.literal(""))) + p.tool_arg_close(p.literal("")) ); arg_parsers.push_back(arg_def.is_required ? @@ -241,12 +240,12 @@ static void test_example_qwen3_coder(testing & t) { } common_chat_msg msg; - auto extractor = common_chat_peg_constructed_builder::extractor(msg); + auto extractor = common_chat_peg_constructed_extractor(msg); ctx.ast_arena.visit(result, extractor.visitor()); //t.log("Input: " + input); t.log("==========================================="); - t.log("Iteration " + std::to_string(input.size())); + t.log("Iteration " + std::to_string(in.size())); t.log("Reasoning: " + msg.reasoning_content); t.log("Content : " + msg.content); for (const auto & tc : msg.tool_calls) { @@ -266,3 +265,207 @@ static void test_example_qwen3_coder(testing & t) { } }); } + +void test_command7_parser_compare(testing & t) { + auto parser = build_chat_peg_native_parser([](common_chat_peg_native_builder & p) { + auto thinking = p.reasoning_block( + "<|START_THINKING|>" << p.reasoning(p.until("<|END_THINKING|>")) << "<|END_THINKING|>"); + + auto response = "<|START_RESPONSE|>" << p.content(p.until("<|END_RESPONSE|>")) << "<|END_RESPONSE|>"; + + auto tool_call_id = p.atomic("\"tool_call_id\"" << (":" << "\"" + p.tool_id(p.json_string_content()) + "\"")); + auto tool_call_name = p.atomic("\"tool_name\"" << (":" << "\"" + p.tool_name(p.json_string_content()) + "\"")); + auto tool_call_args = "\"parameters\"" << (":" << p.tool_args(p.json())); + + auto tool_call_fields = p.rule("tool-call-fields", tool_call_id | tool_call_name | tool_call_args); + auto tool_call = p.rule("tool-call", p.tool( + p.tool_open(p.literal("{")) + << tool_call_fields + << p.zero_or_more( p.literal(",") << tool_call_fields) + << p.tool_close(p.literal("}")) + )); + + auto tool_calls = p.rule("tool-calls", + "<|START_ACTION|>" + << ("[" << tool_call << p.zero_or_more(p.literal(",") << tool_call) << "]") + << "<|END_ACTION|>"); + + return p.optional(thinking) + p.optional(p.space() + response) + p.optional(p.space() + tool_calls) + p.end(); + }); + + auto test_current = [&](const common_peg_arena & p, const std::string & input, bool need_more_input, bool print_results) { + common_peg_parse_context ctx(input, !need_more_input); + auto result = p.parse(ctx); + + common_chat_msg msg; + auto extractor = common_chat_peg_native_extractor(msg); + ctx.ast_arena.visit(result, extractor.visitor()); + + if (print_results) { + std::cout << "== Parsed (new) ==\n"; + std::cout << "=== Reasoning ===\n"; + std::cout << msg.reasoning_content << "\n"; + std::cout << "\n\n=== Content ===\n"; + std::cout << msg.content << "\n"; + std::cout << "\n\n=== Tool Calls ===\n"; + for (const auto & tc : msg.tool_calls) { + std::cout << "id: " << tc.id << "\n"; + std::cout << "name: " << tc.name << "\n"; + std::cout << "args: " << tc.arguments << "\n"; + } + } + }; + + auto test_legacy = [&](const std::string & input, bool need_more_input, bool print_results) { + // Original common_chat_combinator_parser taken from chat.cpp + common_chat_msg_parser builder( + input, + /* .is_partial = */ need_more_input, + { + /* .format = */ COMMON_CHAT_FORMAT_GENERIC, + /* .reasoning_format = */ COMMON_REASONING_FORMAT_AUTO, + /* .reasoning_in_content = */ false, + /* .thinking_forced_open = */ false, + } + ); + + builder.try_parse_reasoning("<|START_THINKING|>", "<|END_THINKING|>"); + + static const common_regex start_action_regex("<\\|START_ACTION\\|>"); + static const common_regex end_action_regex("<\\|END_ACTION\\|>"); + static const common_regex start_response_regex("<\\|START_RESPONSE\\|>"); + static const common_regex end_response_regex("<\\|END_RESPONSE\\|>"); + + if (auto res = builder.try_find_regex(start_action_regex)) { + // If we didn't extract thoughts, prelude includes them. + auto tool_calls = builder.consume_json_with_dumped_args({ { "parameters" } }); + for (const auto & tool_call : tool_calls.value) { + std::string name = tool_call.contains("tool_name") ? tool_call.at("tool_name") : ""; + std::string id = tool_call.contains("tool_call_id") ? tool_call.at("tool_call_id") : ""; + std::string arguments = tool_call.contains("parameters") ? tool_call.at("parameters") : ""; + if (!builder.add_tool_call(name, id, arguments) || tool_calls.is_partial) { + throw common_chat_msg_partial_exception("incomplete tool call"); + } + } + if (tool_calls.is_partial) { + throw common_chat_msg_partial_exception("incomplete tool call"); + } + builder.consume_regex(end_action_regex); + } else if (auto res = builder.try_find_regex(start_response_regex)) { + if (!builder.try_find_regex(end_response_regex)) { + builder.add_content(builder.consume_rest()); + throw common_chat_msg_partial_exception(end_response_regex.str()); + } + } else { + builder.add_content(builder.consume_rest()); + } + + if (print_results) { + std::cout << "== Parsed (legacy) ==\n"; + std::cout << "=== Reasoning ===\n"; + std::cout << builder.result().reasoning_content << "\n"; + std::cout << "\n\n=== Content ===\n"; + std::cout << builder.result().content << "\n"; + std::cout << "\n\n=== Tool Calls ===\n"; + for (const auto & tc : builder.result().tool_calls) { + std::cout << "id: " << tc.id << "\n"; + std::cout << "name: " << tc.name << "\n"; + std::cout << "args: " << tc.arguments << "\n"; + } + } + }; + + std::string reasoning = "To plan an effective trip to Japan that includes both historical sites and modern attractions within a " + "budget of $4000 for a two-week stay, we need to:\n\n" + "1. Identify key historical sites and modern attractions in Japan.\n" + "2. Find affordable accommodation options that provide a balance between comfort and cost.\n" + "3. Determine the best modes of transportation for getting around Japan.\n" + "4. Create a day-by-day itinerary that ensures the user gets to see a variety of attractions without " + "overspending.\n" + "5. Provide a detailed cost breakdown that includes accommodation, transportation, meals, and entry fees " + "to attractions."; + + std::string content = "For a two-week trip to Japan with a $4,000 budget, I recommend planning an itinerary that balances " + "historical sites with modern attractions. The destination will be Japan, with a duration of 14 days.\n\n" + "Given your interests in both historical sites and modern attractions, you'll want to focus on cities like " + "Kyoto for its temples and traditional culture, Tokyo for its cutting-edge technology and entertainment " + "districts, and possibly Hiroshima or Nara for additional historical significance.\n\n" + "For accommodation, I suggest looking for affordable options such as budget hotels, hostels, or " + "guesthouses that offer good value without sacrificing too much comfort. Japan has excellent mid-range " + "accommodation options that can keep your lodging costs manageable.\n\n" + "Transportation should prioritize efficiencyβ€”consider getting a JR Rail Pass for intercity travel, which " + "allows unlimited rides on most JR trains including the Shinkansen (bullet train). Within cities, use " + "local trains and subways, which are both affordable and highly reliable.\n\n" + "For meals, embrace local cuisine by eating at neighborhood restaurants, ramen shops, and izakayas rather " + "than touristy establishments. This will give you an authentic experience while keeping costs " + "reasonableβ€”you can enjoy excellent meals for $10-20 per person at local spots.\n\n"; + + std::vector> tool_calls = {{ + "call_0", + "plan_trip", + nlohmann::json::parse(R"({ + "destination": "Japan", + "duration": 14, + "budget": 4000, + "interests": ["historical sites", "modern attractions"], + "accommodation_preferences": "affordable", + "transportation_preferences": "efficient", + "meal_preferences": "local cuisine" + })") + }}; + + std::vector tokens; + + // Build tokens + if (!reasoning.empty()) { + auto tokenized = simple_tokenize(reasoning); + tokens.emplace_back("<|START_THINKING|>"); + tokens.insert(tokens.end(), tokenized.begin(), tokenized.end()); + tokens.emplace_back("<|END_THINKING|>"); + } + + if (!content.empty()) { + auto tokenized = simple_tokenize(content); + tokens.emplace_back("<|START_RESPONSE|>"); + tokens.insert(tokens.end(), tokenized.begin(), tokenized.end()); + tokens.emplace_back("<|END_RESPONSE|>"); + } + + if (!tool_calls.empty()) { + tokens.emplace_back("<|START_ACTION|>"); + + auto json = nlohmann::json::array(); + for (const auto & tc : tool_calls) { + auto tc_json = nlohmann::json::object(); + tc_json["tool_call_id"] = std::get<0>(tc); + tc_json["tool_name"] = std::get<1>(tc); + tc_json["parameters"] = std::get<2>(tc); + json.push_back(tc_json); + } + + auto tokenized = simple_tokenize(json.dump(-1, ' ', true)); + tokens.insert(tokens.end(), tokenized.begin(), tokenized.end()); + + tokens.emplace_back("<|END_ACTION|>"); + } + + std::string input = std::accumulate(tokens.begin(), tokens.end(), std::string()); + + // Run tests + t.test("legacy_parse", [&](testing & /* t */) { + test_legacy(input, false, false); + }); + + t.test("current_parse", [&](testing & /* t */) { + test_current(parser, input, false, false); + }); + + // Run benchmarks + t.bench("legacy_parse_benchmark", [&]() { + test_legacy(input, false, false); + }, 1000); + + t.bench("current_parse_benchmark", [&]() { + test_current(parser, input, false, false); + }, 1000); +} From 4c61d93dc15709524605142be8c2a40422fac910 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 22 Nov 2025 00:39:07 -0600 Subject: [PATCH 143/183] resolve refs to their rules --- common/peg-parser.cpp | 59 ++++++++++++++++++++++++++++++++++++++++++- common/peg-parser.h | 5 ++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp index 37877866658d1..0f4584c6a239f 100644 --- a/common/peg-parser.cpp +++ b/common/peg-parser.cpp @@ -787,6 +787,61 @@ common_peg_parse_result common_peg_arena::parse(common_peg_parser_id id, common_ return ctx.cache.set(id, start, std::move(result)); } +common_peg_parser_id common_peg_arena::resolve_ref(common_peg_parser_id id) { + const auto & parser = parsers_.at(id); + if (auto ref = std::get_if(&parser)) { + return get_rule(ref->name); + } + return id; +} + +void common_peg_arena::resolve_refs() { + // Walk through all parsers and replace refs with their corresponding rule IDs + for (auto & parser : parsers_) { + std::visit([this](auto & p) { + using T = std::decay_t; + + if constexpr (std::is_same_v) { + for (auto & child : p.children) { + child = resolve_ref(child); + } + } else if constexpr (std::is_same_v) { + for (auto & child : p.children) { + child = resolve_ref(child); + } + } else if constexpr (std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v) { + p.child = resolve_ref(p.child); + } else if constexpr (std::is_same_v) { + p.child = resolve_ref(p.child); + } else if constexpr (std::is_same_v) { + p.child = resolve_ref(p.child); + } else if constexpr (std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v) { + // These rules do not have children + } else { + static_assert(is_always_false_v); + } + }, parser); + } + + // Also flatten root if it's a ref + if (root_ != COMMON_PEG_INVALID_PARSER_ID) { + root_ = resolve_ref(root_); + } +} + // Dump implementation (for debugging) std::string common_peg_arena::dump(common_peg_parser_id id) const { const auto & parser = parsers_.at(id); @@ -1026,6 +1081,7 @@ void common_peg_parser_builder::set_root(common_peg_parser p) { } common_peg_arena common_peg_parser_builder::build() { + arena_.resolve_refs(); return std::move(arena_); } @@ -1317,8 +1373,9 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo } return to_gbnf(p.child); } else if constexpr (std::is_same_v) { - return to_gbnf(p.child); + return p.name; } else if constexpr (std::is_same_v) { + // Refs should not exist after flattening, but kept just in case return p.name; } else if constexpr (std::is_same_v) { return to_gbnf(p.child); diff --git a/common/peg-parser.h b/common/peg-parser.h index 56ffd603da496..ea2e6e4231989 100644 --- a/common/peg-parser.h +++ b/common/peg-parser.h @@ -318,6 +318,9 @@ class common_peg_arena { common_peg_parse_result parse(common_peg_parse_context & ctx, size_t start = 0) const; common_peg_parse_result parse(common_peg_parser_id id, common_peg_parse_context & ctx, size_t start) const; + // Resolve all ref parsers to point directly to their corresponding rule parsers + void resolve_refs(); + // Grammar generation void build_grammar(const common_grammar_builder & builder, bool lazy = false) const; @@ -334,6 +337,8 @@ class common_peg_arena { private: common_peg_parser_id add_parser(common_peg_parser_variant parser); void add_rule(const std::string & name, common_peg_parser_id id); + + common_peg_parser_id resolve_ref(common_peg_parser_id id); }; // Builder for constructing parsers From e57c2015e483c518695413792aa84088626c7bbc Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 22 Nov 2025 00:41:10 -0600 Subject: [PATCH 144/183] remove packrat caching (for now) --- common/peg-parser.cpp | 33 +-------------------------------- common/peg-parser.h | 32 +++----------------------------- 2 files changed, 4 insertions(+), 61 deletions(-) diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp index 0f4584c6a239f..ff6d9bcd281b2 100644 --- a/common/peg-parser.cpp +++ b/common/peg-parser.cpp @@ -6,10 +6,8 @@ #include -#include #include #include -#include #include #include #include @@ -250,26 +248,6 @@ static std::pair, bool> parse_c return {ranges, negated}; } -// Parse cache implementation -const common_peg_parse_result & common_peg_parse_cache::set(common_peg_parser_id id, size_t start, common_peg_parse_result result) { - auto & stored = results[common_peg_parse_cache_key{id, start}]; - stored = std::move(result); - return stored; -} - -common_peg_parse_result * common_peg_parse_cache::get(common_peg_parser_id id, size_t start) { - auto it = results.find(common_peg_parse_cache_key{id, start}); - if (it != results.end()) { - return &it->second; - } - return nullptr; -} - -void common_peg_parse_cache::clear() { - results.clear(); -} - - void common_peg_ast_arena::visit(common_peg_ast_id id, std::function visitor) { if (id == COMMON_PEG_INVALID_AST_ID) { return; @@ -772,19 +750,10 @@ common_peg_parse_result common_peg_arena::parse(common_peg_parse_context & ctx, } common_peg_parse_result common_peg_arena::parse(common_peg_parser_id id, common_peg_parse_context & ctx, size_t start) const { - // Check cache - common_peg_parse_result * cached = ctx.cache.get(id, start); - if (cached) { - return *cached; - } - // Execute parser const auto & parser = parsers_.at(id); parser_executor exec(*this, ctx, start); - auto result = std::visit(exec, parser); - - // Cache result - return ctx.cache.set(id, start, std::move(result)); + return std::visit(exec, parser); } common_peg_parser_id common_peg_arena::resolve_ref(common_peg_parser_id id) { diff --git a/common/peg-parser.h b/common/peg-parser.h index ea2e6e4231989..649ff05370e65 100644 --- a/common/peg-parser.h +++ b/common/peg-parser.h @@ -114,22 +114,6 @@ class common_peg_ast_arena { void visit(const common_peg_parse_result & result, common_peg_ast_visitor visitor); }; -struct common_peg_parse_cache_key { - common_peg_parser_id id; - size_t start; - - bool operator==(const common_peg_parse_cache_key & other) const { - return id == other.id && start == other.start; - } -}; - -template <> -struct std::hash { - std::size_t operator()(const common_peg_parse_cache_key & k) const { - return std::hash{}((k.id << 32) | k.start); - } -}; - struct common_peg_parse_result { common_peg_parse_result_type type = COMMON_PEG_PARSE_RESULT_FAIL; size_t start = 0; @@ -153,31 +137,21 @@ struct common_peg_parse_result { bool success() const { return type == COMMON_PEG_PARSE_RESULT_SUCCESS; } }; -class common_peg_parse_cache { - std::unordered_map results; - - public: - const common_peg_parse_result & set(common_peg_parser_id id, size_t start, common_peg_parse_result result); - common_peg_parse_result * get(common_peg_parser_id id, size_t start); - void clear(); -}; - struct common_peg_parse_context { std::string input; bool input_is_complete; - common_peg_parse_cache cache; common_peg_ast_arena ast_arena; int parse_depth; common_peg_parse_context() - : input_is_complete(true), cache(), parse_depth(0) {} + : input_is_complete(true), parse_depth(0) {} common_peg_parse_context(const std::string & input) - : input(input), input_is_complete(true), cache(), parse_depth(0) {} + : input(input), input_is_complete(true), parse_depth(0) {} common_peg_parse_context(const std::string & input, bool complete) - : input(input), input_is_complete(complete), cache(), parse_depth(0) {} + : input(input), input_is_complete(complete), parse_depth(0) {} }; // Forward declaration From 221f4fedd3bb9859402621002ba93f82ca202f9d Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 22 Nov 2025 00:41:31 -0600 Subject: [PATCH 145/183] update tests --- tests/test-chat-peg-parser.cpp | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/tests/test-chat-peg-parser.cpp b/tests/test-chat-peg-parser.cpp index 60c5ea5069bb5..cfcd69b3578b9 100644 --- a/tests/test-chat-peg-parser.cpp +++ b/tests/test-chat-peg-parser.cpp @@ -290,7 +290,7 @@ void test_command7_parser_compare(testing & t) { << ("[" << tool_call << p.zero_or_more(p.literal(",") << tool_call) << "]") << "<|END_ACTION|>"); - return p.optional(thinking) + p.optional(p.space() + response) + p.optional(p.space() + tool_calls) + p.end(); + return p.optional(thinking) << (tool_calls | response) + p.end(); }); auto test_current = [&](const common_peg_arena & p, const std::string & input, bool need_more_input, bool print_results) { @@ -385,21 +385,6 @@ void test_command7_parser_compare(testing & t) { "5. Provide a detailed cost breakdown that includes accommodation, transportation, meals, and entry fees " "to attractions."; - std::string content = "For a two-week trip to Japan with a $4,000 budget, I recommend planning an itinerary that balances " - "historical sites with modern attractions. The destination will be Japan, with a duration of 14 days.\n\n" - "Given your interests in both historical sites and modern attractions, you'll want to focus on cities like " - "Kyoto for its temples and traditional culture, Tokyo for its cutting-edge technology and entertainment " - "districts, and possibly Hiroshima or Nara for additional historical significance.\n\n" - "For accommodation, I suggest looking for affordable options such as budget hotels, hostels, or " - "guesthouses that offer good value without sacrificing too much comfort. Japan has excellent mid-range " - "accommodation options that can keep your lodging costs manageable.\n\n" - "Transportation should prioritize efficiencyβ€”consider getting a JR Rail Pass for intercity travel, which " - "allows unlimited rides on most JR trains including the Shinkansen (bullet train). Within cities, use " - "local trains and subways, which are both affordable and highly reliable.\n\n" - "For meals, embrace local cuisine by eating at neighborhood restaurants, ramen shops, and izakayas rather " - "than touristy establishments. This will give you an authentic experience while keeping costs " - "reasonableβ€”you can enjoy excellent meals for $10-20 per person at local spots.\n\n"; - std::vector> tool_calls = {{ "call_0", "plan_trip", @@ -424,13 +409,6 @@ void test_command7_parser_compare(testing & t) { tokens.emplace_back("<|END_THINKING|>"); } - if (!content.empty()) { - auto tokenized = simple_tokenize(content); - tokens.emplace_back("<|START_RESPONSE|>"); - tokens.insert(tokens.end(), tokenized.begin(), tokenized.end()); - tokens.emplace_back("<|END_RESPONSE|>"); - } - if (!tool_calls.empty()) { tokens.emplace_back("<|START_ACTION|>"); @@ -463,9 +441,9 @@ void test_command7_parser_compare(testing & t) { // Run benchmarks t.bench("legacy_parse_benchmark", [&]() { test_legacy(input, false, false); - }, 1000); + }, 100); t.bench("current_parse_benchmark", [&]() { test_current(parser, input, false, false); - }, 1000); + }, 100); } From d14edcbf95fb6da582204ea4e8e3dbb35c10e2d1 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 22 Nov 2025 00:55:33 -0600 Subject: [PATCH 146/183] compare parsers with incremental input --- tests/test-chat-peg-parser.cpp | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/tests/test-chat-peg-parser.cpp b/tests/test-chat-peg-parser.cpp index cfcd69b3578b9..d088e778d9fc1 100644 --- a/tests/test-chat-peg-parser.cpp +++ b/tests/test-chat-peg-parser.cpp @@ -440,10 +440,23 @@ void test_command7_parser_compare(testing & t) { // Run benchmarks t.bench("legacy_parse_benchmark", [&]() { - test_legacy(input, false, false); - }, 100); + std::string in; + for (auto i = 0u; i < tokens.size(); i++) { + in += tokens[i]; + + try { + test_legacy(in, i + 1 < tokens.size(), false); + } catch (common_chat_msg_partial_exception & e) { + // Do nothing, this is expected + } + } + }, 20); t.bench("current_parse_benchmark", [&]() { - test_current(parser, input, false, false); - }, 100); + std::string in; + for (auto i = 0u; i < tokens.size(); i++) { + in += tokens[i]; + test_current(parser, input, i + 1 < tokens.size(), false); + } + }, 20); } From 90c7fb07e48a46ee95e69dd4a5b9498e54e55f0f Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 22 Nov 2025 12:56:54 -0600 Subject: [PATCH 147/183] benchmark both complete and incremental parsing --- tests/test-chat-peg-parser.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/test-chat-peg-parser.cpp b/tests/test-chat-peg-parser.cpp index d088e778d9fc1..a18432810693f 100644 --- a/tests/test-chat-peg-parser.cpp +++ b/tests/test-chat-peg-parser.cpp @@ -439,7 +439,11 @@ void test_command7_parser_compare(testing & t) { }); // Run benchmarks - t.bench("legacy_parse_benchmark", [&]() { + t.bench("legacy_parse_benchmark complete", [&]() { + test_legacy(input, false, false); + }); + + t.bench("legacy_parse_benchmark incremental", [&]() { std::string in; for (auto i = 0u; i < tokens.size(); i++) { in += tokens[i]; @@ -452,11 +456,15 @@ void test_command7_parser_compare(testing & t) { } }, 20); - t.bench("current_parse_benchmark", [&]() { + t.bench("current_parse_benchmark complete", [&]() { + test_current(parser, input, false, false); + }, 100); + + t.bench("current_parse_benchmark incremental", [&]() { std::string in; for (auto i = 0u; i < tokens.size(); i++) { in += tokens[i]; - test_current(parser, input, i + 1 < tokens.size(), false); + test_current(parser, in, i + 1 < tokens.size(), false); } }, 20); } From 60c93ea84fdf4fb48490f1efc3c317f794d44759 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 22 Nov 2025 13:36:48 -0600 Subject: [PATCH 148/183] add raw string generation from json schema --- common/json-schema-to-grammar.cpp | 110 ++++++++++++++++++++++-------- common/json-schema-to-grammar.h | 1 + 2 files changed, 82 insertions(+), 29 deletions(-) diff --git a/common/json-schema-to-grammar.cpp b/common/json-schema-to-grammar.cpp index e64dc059f31f7..26f2f85acf3e4 100644 --- a/common/json-schema-to-grammar.cpp +++ b/common/json-schema-to-grammar.cpp @@ -247,6 +247,12 @@ std::unordered_map PRIMITIVE_RULES = { {"null", {"\"null\" space", {}}}, }; +std::unordered_map PRIMITIVE_RAW_RULES = { + {"uuid-raw", {"[0-9a-fA-F]{8} \"-\" [0-9a-fA-F]{4} \"-\" [0-9a-fA-F]{4} \"-\" [0-9a-fA-F]{4} \"-\" [0-9a-fA-F]{12}", {}}}, + {"char-raw", {"[^\\x7F\\x00-\\x1F] | [\\x0A\\x0D]", {}}}, + {"string-raw", {"char-raw*", {"char-raw"}}}, +}; + std::unordered_map STRING_FORMAT_RULES = { {"date", {"[0-9]{4} \"-\" ( \"0\" [1-9] | \"1\" [0-2] ) \"-\" ( \"0\" [1-9] | [1-2] [0-9] | \"3\" [0-1] )", {}}}, {"time", {"([01] [0-9] | \"2\" [0-3]) \":\" [0-5] [0-9] \":\" [0-5] [0-9] ( \".\" [0-9]{3} )? ( \"Z\" | ( \"+\" | \"-\" ) ( [01] [0-9] | \"2\" [0-3] ) \":\" [0-5] [0-9] )", {}}}, @@ -332,15 +338,15 @@ class SchemaConverter { } } - std::string _generate_union_rule(const std::string & name, const std::vector & alt_schemas) { + std::string _generate_union_rule(const std::string & name, const std::vector & alt_schemas, bool is_raw) { std::vector rules; for (size_t i = 0; i < alt_schemas.size(); i++) { - rules.push_back(visit(alt_schemas[i], name + (name.empty() ? "alternative-" : "-") + std::to_string(i))); + rules.push_back(visit(alt_schemas[i], name + (name.empty() ? "alternative-" : "-") + std::to_string(i), is_raw)); } return string_join(rules, " | "); } - std::string _visit_pattern(const std::string & pattern, const std::string & name) { + std::string _visit_pattern(const std::string & pattern, const std::string & name, bool is_raw) { if (!(pattern.front() == '^' && pattern.back() == '$')) { _errors.push_back("Pattern must start with '^' and end with '$'"); return ""; @@ -513,7 +519,7 @@ class SchemaConverter { i += 2; } } else if (sub_pattern[i] == '"') { - literal += "\\\""; + literal += is_raw ? "\"" : "\\\""; i++; } else if (!is_non_literal(sub_pattern[i]) && (i == length - 1 || literal.empty() || sub_pattern[i + 1] == '.' || !is_non_literal(sub_pattern[i + 1]))) { @@ -530,6 +536,11 @@ class SchemaConverter { } return join_seq(); }; + + if (is_raw) { + // Do not wrap in quotes if raw + return _add_rule(name, to_rule(transform())); + } return _add_rule(name, "\"\\\"\" (" + to_rule(transform()) + ") \"\\\"\" space"); } @@ -602,16 +613,18 @@ class SchemaConverter { return out.str(); } - std::string _resolve_ref(const std::string & ref) { + std::string _resolve_ref(const std::string & ref, bool is_raw) { + auto ref_check_key = ref + (is_raw ? "-raw" : ""); + auto it = ref.find('#'); std::string ref_fragment = it != std::string::npos ? ref.substr(it + 1) : ref; static const std::regex nonalphanumeric_regex(R"([^a-zA-Z0-9-]+)"); - std::string ref_name = "ref" + std::regex_replace(ref_fragment, nonalphanumeric_regex, "-"); - if (_rules.find(ref_name) == _rules.end() && _refs_being_resolved.find(ref) == _refs_being_resolved.end()) { - _refs_being_resolved.insert(ref); - json resolved = _refs[ref]; - ref_name = visit(resolved, ref_name); - _refs_being_resolved.erase(ref); + std::string ref_name = "ref" + std::regex_replace(ref_fragment, nonalphanumeric_regex, "-") + (is_raw ? "-raw" : ""); + if (_rules.find(ref_name) == _rules.end() && _refs_being_resolved.find(ref_check_key) == _refs_being_resolved.end()) { + _refs_being_resolved.insert(ref_check_key); + json resolved = _refs[ref_check_key]; + ref_name = visit(resolved, ref_name, is_raw); + _refs_being_resolved.erase(ref_check_key); } return ref_name; } @@ -630,7 +643,7 @@ class SchemaConverter { const auto &prop_name = kv.first; const auto &prop_schema = kv.second; - std::string prop_rule_name = visit(prop_schema, name + (name.empty() ? "" : "-") + prop_name); + std::string prop_rule_name = visit(prop_schema, name + (name.empty() ? "" : "-") + prop_name, false); prop_kv_rule_names[prop_name] = _add_rule( name + (name.empty() ? "" : "-") + prop_name + "-kv", format_literal(json(prop_name).dump()) + " space \":\" space " + prop_rule_name @@ -645,7 +658,7 @@ class SchemaConverter { if ((additional_properties.is_boolean() && additional_properties.get()) || additional_properties.is_object()) { std::string sub_name = name + (name.empty() ? "" : "-") + "additional"; std::string value_rule = - additional_properties.is_object() ? visit(additional_properties, sub_name + "-value") + additional_properties.is_object() ? visit(additional_properties, sub_name + "-value", false) : _add_primitive("value", PRIMITIVE_RULES.at("value")); auto key_rule = @@ -715,10 +728,13 @@ class SchemaConverter { BuiltinRule dep_rule; auto it = PRIMITIVE_RULES.find(dep); if (it == PRIMITIVE_RULES.end()) { - it = STRING_FORMAT_RULES.find(dep); - if (it == STRING_FORMAT_RULES.end()) { - _errors.push_back("Rule " + dep + " not known"); - continue; + it = PRIMITIVE_RAW_RULES.find(dep); + if (it == PRIMITIVE_RAW_RULES.end()) { + it = STRING_FORMAT_RULES.find(dep); + if (it == STRING_FORMAT_RULES.end()) { + _errors.push_back("Rule " + dep + " not known"); + continue; + } } } if (_rules.find(dep) == _rules.end()) { @@ -815,16 +831,23 @@ class SchemaConverter { return format_literal(value.dump()); } - std::string visit(const json & schema, const std::string & name) { + std::string _generate_raw_constant_rule(const json & value) { + return format_literal(value.get()); + } + + std::string visit(const json & schema, const std::string & name, bool is_raw) { json schema_type = schema.contains("type") ? schema["type"] : json(); std::string schema_format = schema.contains("format") ? schema["format"].get() : ""; std::string rule_name = is_reserved_name(name) ? name + "-" : name.empty() ? "root" : name; + if (is_raw) { + rule_name += "-raw"; + } if (schema.contains("$ref")) { - return _add_rule(rule_name, _resolve_ref(schema["$ref"])); + return _add_rule(rule_name, _resolve_ref(schema["$ref"], is_raw)); } else if (schema.contains("oneOf") || schema.contains("anyOf")) { std::vector alt_schemas = schema.contains("oneOf") ? schema["oneOf"].get>() : schema["anyOf"].get>(); - return _add_rule(rule_name, _generate_union_rule(name, alt_schemas)); + return _add_rule(rule_name, _generate_union_rule(name, alt_schemas, is_raw)); } else if (schema_type.is_array()) { std::vector schema_types; for (const auto & t : schema_type) { @@ -832,13 +855,19 @@ class SchemaConverter { schema_copy["type"] = t; schema_types.push_back(schema_copy); } - return _add_rule(rule_name, _generate_union_rule(name, schema_types)); + return _add_rule(rule_name, _generate_union_rule(name, schema_types, is_raw)); } else if (schema.contains("const")) { + if (is_raw) { + return _add_rule(rule_name, _generate_raw_constant_rule(schema["const"])); + } return _add_rule(rule_name, _generate_constant_rule(schema["const"]) + " space"); } else if (schema.contains("enum")) { std::vector enum_values; for (const auto & v : schema["enum"]) { - enum_values.push_back(_generate_constant_rule(v)); + enum_values.push_back(is_raw ? _generate_raw_constant_rule(v) : _generate_constant_rule(v)); + } + if (is_raw) { + return _add_rule(rule_name, "(" + string_join(enum_values, " | ") + ")"); } return _add_rule(rule_name, "(" + string_join(enum_values, " | ") + ") space"); } else if ((schema_type.is_null() || schema_type == "object") @@ -879,7 +908,7 @@ class SchemaConverter { } } else if (comp_schema.contains("enum")) { for (const auto & v : comp_schema["enum"]) { - const auto rule = _generate_constant_rule(v); + const auto rule = is_raw ? _generate_raw_constant_rule(v) : _generate_constant_rule(v); if (enum_values.find(rule) == enum_values.end()) { enum_values[rule] = 0; } @@ -906,6 +935,9 @@ class SchemaConverter { } } if (!enum_intersection.empty()) { + if (is_raw) { + return _add_rule(rule_name, "(" + string_join(enum_intersection, " | ") + ")"); + } return _add_rule(rule_name, "(" + string_join(enum_intersection, " | ") + ") space"); } } @@ -918,12 +950,12 @@ class SchemaConverter { if (i > 0) { rule += " \",\" space "; } - rule += visit(items[i], name + (name.empty() ? "" : "-") + "tuple-" + std::to_string(i)); + rule += visit(items[i], name + (name.empty() ? "" : "-") + "tuple-" + std::to_string(i), false); } rule += " \"]\" space"; return _add_rule(rule_name, rule); } else { - std::string item_rule_name = visit(items, name + (name.empty() ? "" : "-") + "item"); + std::string item_rule_name = visit(items, name + (name.empty() ? "" : "-") + "item", false); int min_items = schema.contains("minItems") ? schema["minItems"].get() : 0; json max_items_json = schema.contains("maxItems") ? schema["maxItems"] : json(); int max_items = max_items_json.is_number_integer() ? max_items_json.get() : std::numeric_limits::max(); @@ -931,16 +963,24 @@ class SchemaConverter { return _add_rule(rule_name, "\"[\" space " + build_repetition(item_rule_name, min_items, max_items, "\",\" space") + " \"]\" space"); } } else if ((schema_type.is_null() || schema_type == "string") && schema.contains("pattern")) { - return _visit_pattern(schema["pattern"], rule_name); + return _visit_pattern(schema["pattern"], rule_name, is_raw); } else if ((schema_type.is_null() || schema_type == "string") && std::regex_match(schema_format, std::regex("^uuid[1-5]?$"))) { + if (is_raw) { + return _add_primitive(rule_name == "root" ? "root" : schema_format + "-raw", PRIMITIVE_RAW_RULES.at("uuid-raw")); + } return _add_primitive(rule_name == "root" ? "root" : schema_format, PRIMITIVE_RULES.at("uuid")); } else if ((schema_type.is_null() || schema_type == "string") && STRING_FORMAT_RULES.find(schema_format + "-string") != STRING_FORMAT_RULES.end()) { auto prim_name = schema_format + "-string"; return _add_rule(rule_name, _add_primitive(prim_name, STRING_FORMAT_RULES.at(prim_name))); } else if (schema_type == "string" && (schema.contains("minLength") || schema.contains("maxLength"))) { - std::string char_rule = _add_primitive("char", PRIMITIVE_RULES.at("char")); + std::string char_rule = is_raw ? + _add_primitive("char-raw", PRIMITIVE_RAW_RULES.at("char-raw")) : + _add_primitive("char", PRIMITIVE_RULES.at("char")); int min_len = schema.contains("minLength") ? schema["minLength"].get() : 0; int max_len = schema.contains("maxLength") ? schema["maxLength"].get() : std::numeric_limits::max(); + if (is_raw) { + return _add_rule(rule_name, build_repetition(char_rule, min_len, max_len)); + } return _add_rule(rule_name, "\"\\\"\" " + build_repetition(char_rule, min_len, max_len) + " \"\\\"\" space"); } else if (schema_type == "integer" && (schema.contains("minimum") || schema.contains("exclusiveMinimum") || schema.contains("maximum") || schema.contains("exclusiveMaximum"))) { int64_t min_value = std::numeric_limits::min(); @@ -958,7 +998,10 @@ class SchemaConverter { std::stringstream out; out << "("; _build_min_max_int(min_value, max_value, out); - out << ") space"; + out << ")"; + if (!is_raw) { + out << " space"; + } return _add_rule(rule_name, out.str()); } else if (schema.empty() || schema_type == "object") { return _add_rule(rule_name, _add_primitive("object", PRIMITIVE_RULES.at("object"))); @@ -968,6 +1011,12 @@ class SchemaConverter { return ""; } // TODO: support minimum, maximum, exclusiveMinimum, exclusiveMaximum at least for zero + if (is_raw) { + auto it = PRIMITIVE_RAW_RULES.find(schema_type.get()); + if (it != PRIMITIVE_RAW_RULES.end()) { + return _add_primitive(rule_name == "root" ? "root" : schema_type.get(), it->second); + } + } return _add_primitive(rule_name == "root" ? "root" : schema_type.get(), PRIMITIVE_RULES.at(schema_type.get())); } } @@ -1012,7 +1061,10 @@ std::string build_grammar(const std::function add_rule; std::function add_schema; + std::function add_string_schema; std::function resolve_refs; }; From aa5043b9a639c7d54e0d1678ffea18ecc2f2aada Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 22 Nov 2025 13:37:36 -0600 Subject: [PATCH 149/183] add support for string schemas in gbnf generation --- common/peg-parser.cpp | 17 +++++++++++++++-- common/peg-parser.h | 3 ++- tests/test-chat-peg-parser.cpp | 19 +++++++++++++++---- 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp index ff6d9bcd281b2..90487d98f5518 100644 --- a/common/peg-parser.cpp +++ b/common/peg-parser.cpp @@ -1009,8 +1009,8 @@ common_peg_parser common_peg_parser_builder::json_string_content() { return wrap(arena_.add_parser(common_peg_json_string_parser{})); } -common_peg_parser common_peg_parser_builder::schema(common_peg_parser p, const std::string & name, const nlohmann::ordered_json & schema) { - return wrap(arena_.add_parser(common_peg_schema_parser{p.id(), name, std::make_shared(schema)})); +common_peg_parser common_peg_parser_builder::schema(common_peg_parser p, const std::string & name, const nlohmann::ordered_json & schema, bool raw) { + return wrap(arena_.add_parser(common_peg_schema_parser{p.id(), name, std::make_shared(schema), raw})); } common_peg_parser common_peg_parser_builder::capture(const std::string & key, common_peg_parser p) { @@ -1338,6 +1338,19 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo return gbnf_excluding_pattern(p.delimiters); } else if constexpr (std::is_same_v) { if (p.schema) { + auto type = p.schema->value("type","object"); + if (p.raw && type == "string") { + if (p.schema->contains("pattern")) { + // heuristic: + // if .* is in the user's provided pattern, use the child's GBNF grammar. + // This is because .* will greedily match everything, past any delimiters. + auto pattern = p.schema->value("pattern", "^.*$"); + if (pattern.find(".*") != std::string::npos) { + return to_gbnf(p.child); + } + } + return builder.add_string_schema(p.name, *p.schema); + } return builder.add_schema(p.name, *p.schema); } return to_gbnf(p.child); diff --git a/common/peg-parser.h b/common/peg-parser.h index 649ff05370e65..7b5803c1bd215 100644 --- a/common/peg-parser.h +++ b/common/peg-parser.h @@ -216,6 +216,7 @@ struct common_peg_schema_parser { common_peg_parser_id child; std::string name; std::shared_ptr schema; + bool raw; }; struct common_peg_rule_parser { @@ -423,7 +424,7 @@ class common_peg_parser_builder { // Wraps a parser with JSON schema metadata for grammar generation. // Used internally to convert JSON schemas to GBNF grammar rules. - common_peg_parser schema(common_peg_parser p, const std::string & name, const nlohmann::ordered_json & schema); + common_peg_parser schema(common_peg_parser p, const std::string & name, const nlohmann::ordered_json & schema, bool raw = false); // Captures matched text to semantics.captures[key] common_peg_parser capture(const std::string & key, common_peg_parser p); diff --git a/tests/test-chat-peg-parser.cpp b/tests/test-chat-peg-parser.cpp index a18432810693f..cac26d280794f 100644 --- a/tests/test-chat-peg-parser.cpp +++ b/tests/test-chat-peg-parser.cpp @@ -185,10 +185,21 @@ static void test_example_qwen3_coder(testing & t) { std::vector arg_parsers; for (const auto & arg_def : def.arguments) { + auto type = arg_def.schema.value("type", "object"); auto arg = p.tool_arg( p.tool_arg_open("") + - p.tool_arg_json_value(p.schema(p.json(), "tool-" + def.name + "-arg-" + def.name + "-schema", arg_def.schema)) + - p.tool_arg_close(p.literal("")) + (type == "string" ? + p.tool_arg_string_value(p.schema( + p.until_one_of({""}), + "tool-" + def.name + "-arg-" + arg_def.name + "-schema", + arg_def.schema, + true)) : + p.tool_arg_json_value(p.schema( + p.json(), + "tool-" + def.name + "-arg-" + arg_def.name + "-schema", + arg_def.schema)) + ) + + p.tool_arg_close(p.literal("") + p.peek(p.literal(""))) ); arg_parsers.push_back(arg_def.is_required ? @@ -221,8 +232,8 @@ static void test_example_qwen3_coder(testing & t) { "Let me search the knowledge base for cat pictures." "" "" - "\"cat pictures\"" - "\"general\"" + "cat pictures" + "general" "" ""; From 2b72bfe635f27d489bf8052ba099df1795cd4ba2 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 22 Nov 2025 14:28:30 -0600 Subject: [PATCH 150/183] fix qwen example to include \n --- tests/test-chat-peg-parser.cpp | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/tests/test-chat-peg-parser.cpp b/tests/test-chat-peg-parser.cpp index cac26d280794f..dfd92e35eba46 100644 --- a/tests/test-chat-peg-parser.cpp +++ b/tests/test-chat-peg-parser.cpp @@ -181,8 +181,6 @@ static void test_example_qwen3_coder(testing & t) { std::vector tool_parsers; foreach_tool(tools, [&](const tool_definition & def) { - t.log(def.name); - std::vector arg_parsers; for (const auto & arg_def : def.arguments) { auto type = arg_def.schema.value("type", "object"); @@ -190,7 +188,7 @@ static void test_example_qwen3_coder(testing & t) { p.tool_arg_open("") + (type == "string" ? p.tool_arg_string_value(p.schema( - p.until_one_of({""}), + p.until_one_of({"\n\n"}), "tool-" + def.name + "-arg-" + arg_def.name + "-schema", arg_def.schema, true)) : @@ -199,7 +197,7 @@ static void test_example_qwen3_coder(testing & t) { "tool-" + def.name + "-arg-" + arg_def.name + "-schema", arg_def.schema)) ) + - p.tool_arg_close(p.literal("") + p.peek(p.literal(""))) + p.tool_arg_close(p.literal("\n") + p.peek(p.literal(""))) ); arg_parsers.push_back(arg_def.is_required ? @@ -208,13 +206,13 @@ static void test_example_qwen3_coder(testing & t) { } tool_parsers.push_back(p.rule("tool-" + def.name, - p.tool_open("") + - p.sequence(arg_parsers) + + p.tool_open("") << + p.sequence(arg_parsers) << p.tool_close(p.literal("")) )); }); - auto tool_call = p.trigger_rule("tool-call", "" + p.choice(tool_parsers) + ""); + auto tool_call = p.trigger_rule("tool-call", "" << p.choice(tool_parsers) << ""); return content + p.zero_or_more(p.space() + tool_call) + p.end(); }); @@ -230,11 +228,11 @@ static void test_example_qwen3_coder(testing & t) { t.test("incremental parsing", [&](testing &t) { std::string input = "Let me search the knowledge base for cat pictures." - "" - "" - "cat pictures" - "general" - "" + "\n" + "\n" + "cat pictures\n" + "general\n" + "\n" ""; std::vector tokens = simple_tokenize(input); From 2323c0061ee4d82d3578ecd8ab413281c76c5e87 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 22 Nov 2025 14:41:42 -0600 Subject: [PATCH 151/183] tidy up example --- tests/test-chat-peg-parser.cpp | 49 +++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/tests/test-chat-peg-parser.cpp b/tests/test-chat-peg-parser.cpp index dfd92e35eba46..e81496f3fdd67 100644 --- a/tests/test-chat-peg-parser.cpp +++ b/tests/test-chat-peg-parser.cpp @@ -184,21 +184,33 @@ static void test_example_qwen3_coder(testing & t) { std::vector arg_parsers; for (const auto & arg_def : def.arguments) { auto type = arg_def.schema.value("type", "object"); - auto arg = p.tool_arg( - p.tool_arg_open("") + + + auto arg = p.tool_arg(p.sequence({ + p.tool_arg_open(""), (type == "string" ? - p.tool_arg_string_value(p.schema( - p.until_one_of({"\n\n"}), + p.tool_arg_string_value( + p.schema( + p.until_one_of({ + "\n\n" + }), "tool-" + def.name + "-arg-" + arg_def.name + "-schema", arg_def.schema, - true)) : - p.tool_arg_json_value(p.schema( - p.json(), - "tool-" + def.name + "-arg-" + arg_def.name + "-schema", - arg_def.schema)) - ) + - p.tool_arg_close(p.literal("\n") + p.peek(p.literal(""))) - ); + true + ) + ) : p.tool_arg_json_value( + p.schema( + p.json(), + "tool-" + def.name + "-arg-" + arg_def.name + "-schema", + arg_def.schema + ) + ) + ), + p.tool_arg_close( + "\n" + + p.peek(p.literal("")) + ) + })); arg_parsers.push_back(arg_def.is_required ? p.rule("tool-" + def.name + "-arg-" + arg_def.name, arg) : @@ -206,13 +218,18 @@ static void test_example_qwen3_coder(testing & t) { } tool_parsers.push_back(p.rule("tool-" + def.name, - p.tool_open("") << - p.sequence(arg_parsers) << - p.tool_close(p.literal("")) + p.tool_open("") + << p.sequence(arg_parsers) + << p.tool_close(p.literal("")) )); }); - auto tool_call = p.trigger_rule("tool-call", "" << p.choice(tool_parsers) << ""); + auto tool_call = p.trigger_rule("tool-call", + "" + << p.choice(tool_parsers) + << "" + ); + return content + p.zero_or_more(p.space() + tool_call) + p.end(); }); From 0243c56a9ccb925da1122ff984f30156e33fc491 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 22 Nov 2025 14:56:15 -0600 Subject: [PATCH 152/183] rename extractor to mapper --- common/chat-peg-parser.cpp | 18 +++++++++--------- common/chat-peg-parser.h | 20 ++++++++++---------- common/peg-parser.cpp | 4 ++-- common/peg-parser.h | 4 ++-- tests/test-chat-peg-parser.cpp | 8 ++++---- 5 files changed, 27 insertions(+), 27 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 0232f56190cca..955eba8f0b09b 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -1,12 +1,12 @@ #include "chat-peg-parser.h" -common_peg_ast_visitor common_chat_peg_extractor::visitor() { - return [this](const common_peg_ast_node & node) { - extract(node); - }; +void common_chat_peg_mapper::from_ast(const common_peg_ast_arena & arena, const common_peg_parse_result & result) { + arena.visit(result, [this](const common_peg_ast_node & node) { + map(node); + }); } -void common_chat_peg_extractor::extract(const common_peg_ast_node & node) { +void common_chat_peg_mapper::map(const common_peg_ast_node & node) { bool is_reasoning_block = node.tag == common_chat_peg_builder::REASONING_BLOCK; bool is_reasoning = node.tag == common_chat_peg_builder::REASONING; bool is_content = node.tag == common_chat_peg_builder::CONTENT; @@ -24,8 +24,8 @@ void common_chat_peg_extractor::extract(const common_peg_ast_node & node) { } } -void common_chat_peg_native_extractor::extract(const common_peg_ast_node & node) { - common_chat_peg_extractor::extract(node); +void common_chat_peg_native_mapper::map(const common_peg_ast_node & node) { + common_chat_peg_mapper::map(node); bool is_tool_open = node.tag == common_chat_peg_native_builder::TOOL_OPEN; bool is_tool_name = node.tag == common_chat_peg_native_builder::TOOL_NAME; @@ -50,8 +50,8 @@ void common_chat_peg_native_extractor::extract(const common_peg_ast_node & node) } } -void common_chat_peg_constructed_extractor::extract(const common_peg_ast_node & node) { - common_chat_peg_extractor::extract(node); +void common_chat_peg_constructed_mapper::map(const common_peg_ast_node & node) { + common_chat_peg_mapper::map(node); bool is_tool_name = node.tag == common_chat_peg_constructed_builder::TOOL_NAME; bool is_tool_close = node.tag == common_chat_peg_constructed_builder::TOOL_CLOSE; diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index 2567ccf5451d8..b84cbed206902 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -20,14 +20,14 @@ inline common_peg_arena build_chat_peg_parser(const std::function & fn) { @@ -87,15 +87,15 @@ class common_chat_peg_constructed_builder : public common_chat_peg_builder { common_peg_parser tool_arg_json_value(const common_peg_parser & p) { return tag(TOOL_ARG_JSON_VALUE, p); } }; -class common_chat_peg_constructed_extractor : public common_chat_peg_extractor { +class common_chat_peg_constructed_mapper : public common_chat_peg_mapper { common_chat_tool_call * current_tool; int arg_count = 0; bool needs_closing_quote = false; public: - common_chat_peg_constructed_extractor(common_chat_msg & msg) : common_chat_peg_extractor(msg) {} + common_chat_peg_constructed_mapper(common_chat_msg & msg) : common_chat_peg_mapper(msg) {} - void extract(const common_peg_ast_node & node) override; + void map(const common_peg_ast_node & node) override; }; inline common_peg_arena build_chat_peg_constructed_parser(const std::function & fn) { diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp index 90487d98f5518..3fe7919d8049c 100644 --- a/common/peg-parser.cpp +++ b/common/peg-parser.cpp @@ -248,7 +248,7 @@ static std::pair, bool> parse_c return {ranges, negated}; } -void common_peg_ast_arena::visit(common_peg_ast_id id, std::function visitor) { +void common_peg_ast_arena::visit(common_peg_ast_id id, const common_peg_ast_visitor & visitor) const { if (id == COMMON_PEG_INVALID_AST_ID) { return; } @@ -259,7 +259,7 @@ void common_peg_ast_arena::visit(common_peg_ast_id id, std::function visitor) { +void common_peg_ast_arena::visit(const common_peg_parse_result & result, const common_peg_ast_visitor & visitor) const { for (const auto & node : result.nodes) { visit(node, visitor); } diff --git a/common/peg-parser.h b/common/peg-parser.h index 7b5803c1bd215..730c41cd74dcd 100644 --- a/common/peg-parser.h +++ b/common/peg-parser.h @@ -110,8 +110,8 @@ class common_peg_ast_arena { void clear() { nodes_.clear(); } - void visit(common_peg_ast_id id, common_peg_ast_visitor visitor); - void visit(const common_peg_parse_result & result, common_peg_ast_visitor visitor); + void visit(common_peg_ast_id id, const common_peg_ast_visitor & visitor) const; + void visit(const common_peg_parse_result & result, const common_peg_ast_visitor & visitor) const; }; struct common_peg_parse_result { diff --git a/tests/test-chat-peg-parser.cpp b/tests/test-chat-peg-parser.cpp index e81496f3fdd67..395232f4ba2ee 100644 --- a/tests/test-chat-peg-parser.cpp +++ b/tests/test-chat-peg-parser.cpp @@ -266,8 +266,8 @@ static void test_example_qwen3_coder(testing & t) { } common_chat_msg msg; - auto extractor = common_chat_peg_constructed_extractor(msg); - ctx.ast_arena.visit(result, extractor.visitor()); + auto mapper = common_chat_peg_constructed_mapper(msg); + mapper.from_ast(ctx.ast_arena, result); //t.log("Input: " + input); t.log("==========================================="); @@ -324,8 +324,8 @@ void test_command7_parser_compare(testing & t) { auto result = p.parse(ctx); common_chat_msg msg; - auto extractor = common_chat_peg_native_extractor(msg); - ctx.ast_arena.visit(result, extractor.visitor()); + auto mapper = common_chat_peg_native_mapper(msg); + mapper.from_ast(ctx.ast_arena, result); if (print_results) { std::cout << "== Parsed (new) ==\n"; From 6c1a1a86a31e1fbb1eaa32138a5b1a05a4c62915 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 22 Nov 2025 14:57:26 -0600 Subject: [PATCH 153/183] rename ast_arena to ast --- common/peg-parser.cpp | 4 ++-- common/peg-parser.h | 2 +- tests/test-chat-peg-parser.cpp | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp index 3fe7919d8049c..a3f72d3854a95 100644 --- a/common/peg-parser.cpp +++ b/common/peg-parser.cpp @@ -680,7 +680,7 @@ struct parser_executor { text = std::string_view(ctx.input).substr(result.start, result.end - result.start); } - auto node_id = ctx.ast_arena.add_node( + auto node_id = ctx.ast.add_node( p.name, "", result.start, @@ -706,7 +706,7 @@ struct parser_executor { text = std::string_view(ctx.input).substr(result.start, result.end - result.start); } - auto node_id = ctx.ast_arena.add_node( + auto node_id = ctx.ast.add_node( "", p.tag, result.start, diff --git a/common/peg-parser.h b/common/peg-parser.h index 730c41cd74dcd..a8f88b87a6597 100644 --- a/common/peg-parser.h +++ b/common/peg-parser.h @@ -140,7 +140,7 @@ struct common_peg_parse_result { struct common_peg_parse_context { std::string input; bool input_is_complete; - common_peg_ast_arena ast_arena; + common_peg_ast_arena ast; int parse_depth; diff --git a/tests/test-chat-peg-parser.cpp b/tests/test-chat-peg-parser.cpp index 395232f4ba2ee..340a3fec61faf 100644 --- a/tests/test-chat-peg-parser.cpp +++ b/tests/test-chat-peg-parser.cpp @@ -267,7 +267,7 @@ static void test_example_qwen3_coder(testing & t) { common_chat_msg msg; auto mapper = common_chat_peg_constructed_mapper(msg); - mapper.from_ast(ctx.ast_arena, result); + mapper.from_ast(ctx.ast, result); //t.log("Input: " + input); t.log("==========================================="); @@ -325,7 +325,7 @@ void test_command7_parser_compare(testing & t) { common_chat_msg msg; auto mapper = common_chat_peg_native_mapper(msg); - mapper.from_ast(ctx.ast_arena, result); + mapper.from_ast(ctx.ast, result); if (print_results) { std::cout << "== Parsed (new) ==\n"; From 40e46b29ce410395a02174d4114009e88bd552d2 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 22 Nov 2025 15:14:49 -0600 Subject: [PATCH 154/183] place basic tests into one --- tests/CMakeLists.txt | 16 +- tests/peg-parser/playground.cpp | 436 ----------------- tests/peg-parser/test-basic.cpp | 454 ++++++++++++++++++ .../test-command7-parser-compare.cpp | 256 ---------- tests/peg-parser/test-example-minimax-m2.cpp | 64 --- tests/peg-parser/test-example-qwen3-coder.cpp | 134 ------ tests/peg-parser/test-example-seed-oss.cpp | 54 --- tests/peg-parser/test-one.cpp | 99 ---- tests/peg-parser/test-optional.cpp | 38 -- tests/peg-parser/test-partial-parsing.cpp | 230 --------- .../peg-parser/test-recursive-references.cpp | 87 ---- tests/peg-parser/tests.h | 9 +- tests/test-peg-parser.cpp | 5 +- 13 files changed, 460 insertions(+), 1422 deletions(-) delete mode 100644 tests/peg-parser/playground.cpp create mode 100644 tests/peg-parser/test-basic.cpp delete mode 100644 tests/peg-parser/test-command7-parser-compare.cpp delete mode 100644 tests/peg-parser/test-example-minimax-m2.cpp delete mode 100644 tests/peg-parser/test-example-qwen3-coder.cpp delete mode 100644 tests/peg-parser/test-example-seed-oss.cpp delete mode 100644 tests/peg-parser/test-one.cpp delete mode 100644 tests/peg-parser/test-optional.cpp delete mode 100644 tests/peg-parser/test-partial-parsing.cpp delete mode 100644 tests/peg-parser/test-recursive-references.cpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 97f398c5bfc4a..a01ae40dfdb36 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -185,28 +185,20 @@ endif() llama_build_and_test(test-chat-parser.cpp) llama_build_and_test(test-chat-peg-parser.cpp peg-parser/simple_tokenizer.cpp) -llama_build_and_test(peg-parser/playground.cpp NAME test-peg-parser-playground) +llama_build_and_test(test-chat-template.cpp) +llama_build_and_test(test-json-partial.cpp) +llama_build_and_test(test-log.cpp) llama_build_and_test( test-peg-parser.cpp peg-parser/simple_tokenizer.cpp - #peg-parser/test-command7-parser-compare.cpp - #peg-parser/test-example-qwen3-coder.cpp - #peg-parser/test-example-minimax-m2.cpp - #peg-parser/test-example-seed-oss.cpp + peg-parser/test-basic.cpp peg-parser/test-gbnf-generation.cpp peg-parser/test-json-parser.cpp peg-parser/test-json-serialization.cpp - peg-parser/test-one.cpp - peg-parser/test-optional.cpp - peg-parser/test-partial-parsing.cpp - peg-parser/test-recursive-references.cpp peg-parser/test-unicode.cpp peg-parser/test_harness.h peg-parser/tests.h ) -llama_build_and_test(test-chat-template.cpp) -llama_build_and_test(test-json-partial.cpp) -llama_build_and_test(test-log.cpp) llama_build_and_test(test-regex-partial.cpp) if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "s390x") diff --git a/tests/peg-parser/playground.cpp b/tests/peg-parser/playground.cpp deleted file mode 100644 index 0f2b6316a385e..0000000000000 --- a/tests/peg-parser/playground.cpp +++ /dev/null @@ -1,436 +0,0 @@ -#include "test_harness.h" - -#include "peg-parser.h" -#include "chat-parser.h" -#include "json-schema-to-grammar.h" - -#include -#include -#include -#include -#include - -static common_peg_arena create_command_r7b_parser() { - auto parser = build_peg_parser([](common_peg_parser_builder & p) { - auto thinking = p.rule("thinking", - "<|START_THINKING|>" << p.rule("reasoning-content", p.until("<|END_THINKING|>")) << "<|END_THINKING|>"); - - auto response = p.rule("response", - "<|START_RESPONSE|>" << p.rule("content", p.until("<|END_RESPONSE|>")) << "<|END_RESPONSE|>"); - - auto json = p.rule("json", p.json()); - - auto tool_call_id = p.rule("tool-call-id", - "\"tool_call_id\"" << (":" << p.rule("tool-call-id-value", "\"" + p.json_string_content() + "\""))); - - auto tool_call_name = p.rule("tool-name", - "\"tool_name\"" << (":" << p.rule("tool-name-value", "\"" + p.json_string_content() + "\""))); - - auto tool_call_args = p.rule("tool-args", - "\"parameters\"" << (":" << p.rule("tool-args-value", json))); - - auto tool_call_fields = p.rule("tool-call-fields", tool_call_id | tool_call_name | tool_call_args); - - auto tool_call = p.rule("tool-call", - "{" << tool_call_fields << p.zero_or_more(p.literal(",") << tool_call_fields) << "}"); - - auto tool_calls = p.rule("tool-calls", - "<|START_ACTION|>" - << ("[" << tool_call << p.zero_or_more(p.literal(",") << tool_call) << "]") - << "<|END_ACTION|>"); - - return p.optional(thinking) << (tool_calls | response); - }); - - // Check if - build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); - return parser; -} - -static void test_command_r7b_parser(const common_peg_arena & p, - const std::string & input, - bool need_more_input, - bool /* print_results */) { - common_peg_parse_context ctx(input, !need_more_input); - p.parse(ctx); -} - -static void test_command_r7b_legacy_parser(const std::string & input, - bool need_more_input, - bool print_results) { - // Original common_chat_combinator_parser taken from chat.cpp - common_chat_msg_parser builder(input, - /* .is_partial = */ need_more_input, - { - /* .format = */ COMMON_CHAT_FORMAT_GENERIC, - /* .reasoning_format = */ COMMON_REASONING_FORMAT_AUTO, - /* .reasoning_in_content = */ false, - /* .thinking_forced_open = */ false, - }); - - builder.try_parse_reasoning("<|START_THINKING|>", "<|END_THINKING|>"); - - static const common_regex start_action_regex("<\\|START_ACTION\\|>"); - static const common_regex end_action_regex("<\\|END_ACTION\\|>"); - static const common_regex start_response_regex("<\\|START_RESPONSE\\|>"); - static const common_regex end_response_regex("<\\|END_RESPONSE\\|>"); - - if (auto res = builder.try_find_regex(start_action_regex)) { - // If we didn't extract thoughts, prelude includes them. - auto tool_calls = builder.consume_json_with_dumped_args({ { "parameters" } }); - for (const auto & tool_call : tool_calls.value) { - std::string name = tool_call.contains("tool_name") ? tool_call.at("tool_name") : ""; - std::string id = tool_call.contains("tool_call_id") ? tool_call.at("tool_call_id") : ""; - std::string arguments = tool_call.contains("parameters") ? tool_call.at("parameters") : ""; - if (!builder.add_tool_call(name, id, arguments) || tool_calls.is_partial) { - throw common_chat_msg_partial_exception("incomplete tool call"); - } - } - if (tool_calls.is_partial) { - throw common_chat_msg_partial_exception("incomplete tool call"); - } - builder.consume_regex(end_action_regex); - } else if (auto res = builder.try_find_regex(start_response_regex)) { - if (!builder.try_find_regex(end_response_regex)) { - builder.add_content(builder.consume_rest()); - throw common_chat_msg_partial_exception(end_response_regex.str()); - } - } else { - builder.add_content(builder.consume_rest()); - } - - if (print_results) { - std::cout << "== Parsed (legacy) ==\n"; - std::cout << "=== Reasoning ===\n"; - std::cout << builder.result().reasoning_content << "\n"; - std::cout << "\n\n=== Content ===\n"; - std::cout << builder.result().content << "\n"; - std::cout << "\n\n=== Tool Calls ===\n"; - for (const auto & tc : builder.result().tool_calls) { - std::cout << "id: " << tc.id << "\n"; - std::cout << "name: " << tc.name << "\n"; - std::cout << "args: " << tc.arguments << "\n"; - } - } -} - -static std::vector simple_tokenize(const std::string & input) { - std::vector result; - std::string current; - - for (size_t i = 0; i < input.size(); i++) { - switch (input[i]) { - case ' ': - case '\n': - case '\t': - case '{': - case '}': - case ',': - case '[': - case '"': - case ']': - case '.': - case '<': - case '>': - case '=': - case '/': - if (!current.empty()) { - result.push_back(current); - current.clear(); - } - default:; - } - current += input[i]; - } - - if (!current.empty()) { - result.push_back(current); - } - - return result; -} - -static void print_ast(const common_peg_ast_arena & arena, common_peg_ast_id id, int indent = 0) { - const auto & node = arena.get(id); - - // Indentation - std::string space(indent * 2, ' '); - - // Print Node details - std::cout << space << "Node [" << id << "] " - << (node.rule_name.empty() ? "" : node.rule_name); - - if (node.is_partial) { - std::cout << "*"; - } - - if (!node.tag.empty()) { - std::cout << " (" << node.tag << ")"; - } - - // Print text content (truncated if too long for readability) - std::string_view text = node.text; - if (text.length() > 20) { - std::cout << " : \"" << text.substr(0, 17) << "...\""; - } else { - std::cout << " : \"" << text << "\""; - } - std::cout << "\n"; - - // Recursively print children - for (auto child_id : node.children) { - print_ast(arena, child_id, indent + 1); - } -} - -int main() { - testing t; - - auto explicit_parser = build_peg_parser([](common_peg_parser_builder & p) { - auto thinking = p.tag("raw-reasoning", - "" << p.tag("reasoning-content", p.until("")) << ""); - - auto content = p.tag("content", p.until("")); - - auto arg_open = p.rule("arg-open", - p.atomic("") - ); - - auto arg_close = p.rule( - "arg-close", - "" + - p.peek(p.literal("")) - ); - - auto string_arg = - arg_open + - p.tag("arg-string", p.until_one_of({""})) + - p.atomic(p.tag("arg-string-close", arg_close)); - - auto json = p.json(); - - auto json_arg = arg_open + p.tag("arg-json",json) + arg_close; - - auto function = p.rule("tool", - p.tag("tool-open", p.atomic("")) + - p.one_or_more(json_arg | string_arg) + - p.tag("tool-close", p.literal("")) - ); - - auto tool_call = p.rule("tool-call", - "" + - p.one_or_more(function) + - "", - true); - - return thinking + p.optional(p.space() + content) + p.zero_or_more(p.space() + tool_call) + p.end(); - }); - - std::string input = - "The user wants to find large log files that haven't been accessed recently. " - "I should search for files with .log extension, filter by size (over 100MB), " - "and check access time within the last 30 days. I'll need to use the search_files function." - "Based on your requirements, I'll search for log files over 100MB that haven't been " - "accessed in the last month. This will help identify candidates for cleanup or archival.\n\n" - "" - "" - "/var/log" - "*.log" - "100" - "5" - "false" - "30" - "true" - "size" - "{\"exclude_patterns\": [\"*temp*\", \"*cache*\"], \"file_types\": " - "[\"regular\"]}" - "" - ""; - - std::vector tokens = simple_tokenize(input); - - t.test("parse succeeds", [&](testing &t) { - common_chat_msg prev; - for (auto it = tokens.begin(); it != tokens.end(); it++) { - std::string in = std::accumulate(tokens.begin(), it + 1, std::string()); - - common_peg_parse_context ctx(in, it == tokens.end() - 1); - - auto result = explicit_parser.parse(ctx); - if (!t.assert_equal("not fail", false, result.fail())) { - t.log(in.substr(0, result.end) + "[failed->]" + in.substr(result.end)); - } - - common_chat_msg msg; - common_chat_tool_call *current; - int arg_count = 0; - - ctx.ast_arena.visit(result, [&](const common_peg_ast_node & node) { - bool is_reasoning = node.tag == "reasoning-content"; - bool is_content = node.tag == "content"; - bool is_tool_name = node.tag == "tool-name"; - bool is_tool_close = node.tag == "tool-close"; - bool is_arg_name = node.tag == "arg-name"; - bool is_arg_string = node.tag == "arg-string"; - bool is_arg_string_close = node.tag == "arg-string-close"; - bool is_arg_json = node.tag == "arg-json"; - - if (is_reasoning) { - msg.reasoning_content = std::string(node.text); - } - - if (is_content) { - msg.content = std::string(node.text); - } - - if (is_tool_name) { - msg.tool_calls.emplace_back(); - current = &msg.tool_calls.back(); - arg_count = 0; - - current->name = std::string(node.text); - current->arguments = "{"; - } - - if (is_arg_name) { - if (arg_count > 0) { - current->arguments += ","; - } - current->arguments += "\"" + std::string(node.text) + "\":"; - ++arg_count; - } - - if (is_arg_string) { - current->arguments += "\"" + std::string(node.text); - } - - if (is_arg_string_close) { - current->arguments += "\""; - } - - if (is_arg_json) { - current->arguments += std::string(node.text); - } - - if (is_tool_close) { - current->arguments += "}"; - } - }); - - //t.log("Input: " + input); - t.log("Reasoning: " + msg.reasoning_content); - t.log("Content : " + msg.content); - for (const auto & tc : msg.tool_calls) { - t.log("Tool name: " + tc.name); - t.log("Tool args: " + tc.arguments); - } - - try { - // This shouldn't emit any runtime errors - auto diffs = common_chat_msg_diff::compute_diffs(prev, msg); - } catch(const std::exception & e) { - t.log(in.substr(0, result.end) + "[failed->]" + in.substr(result.end)); - t.assert_true(std::string("failed with ") + e.what(), false); - } - - prev = msg; - } - }); - - // Setup data - auto parser = create_command_r7b_parser(); - - std::string reasoning = "To plan an effective trip to Japan that includes both historical sites and modern attractions within a " - "budget of $4000 for a two-week stay, we need to:\n\n" - "1. Identify key historical sites and modern attractions in Japan.\n" - "2. Find affordable accommodation options that provide a balance between comfort and cost.\n" - "3. Determine the best modes of transportation for getting around Japan.\n" - "4. Create a day-by-day itinerary that ensures the user gets to see a variety of attractions without " - "overspending.\n" - "5. Provide a detailed cost breakdown that includes accommodation, transportation, meals, and entry fees " - "to attractions."; - - std::string content = "For a two-week trip to Japan with a $4,000 budget, I recommend planning an itinerary that balances " - "historical sites with modern attractions. The destination will be Japan, with a duration of 14 days.\n\n" - "Given your interests in both historical sites and modern attractions, you'll want to focus on cities like " - "Kyoto for its temples and traditional culture, Tokyo for its cutting-edge technology and entertainment " - "districts, and possibly Hiroshima or Nara for additional historical significance.\n\n" - "For accommodation, I suggest looking for affordable options such as budget hotels, hostels, or " - "guesthouses that offer good value without sacrificing too much comfort. Japan has excellent mid-range " - "accommodation options that can keep your lodging costs manageable.\n\n" - "Transportation should prioritize efficiencyβ€”consider getting a JR Rail Pass for intercity travel, which " - "allows unlimited rides on most JR trains including the Shinkansen (bullet train). Within cities, use " - "local trains and subways, which are both affordable and highly reliable.\n\n" - "For meals, embrace local cuisine by eating at neighborhood restaurants, ramen shops, and izakayas rather " - "than touristy establishments. This will give you an authentic experience while keeping costs " - "reasonableβ€”you can enjoy excellent meals for $10-20 per person at local spots.\n\n"; - - std::vector> tool_calls = { - { "call_0", "plan_trip", nlohmann::json::parse(R"({ - "destination": "Japan", - "duration": 14, - "budget": 4000, - "interests": ["historical sites", "modern attractions"], - "accommodation_preferences": "affordable", - "transportation_preferences": "efficient", - "meal_preferences": "local cuisine" - })") } - }; - - tokens.clear(); - - // Build tokens - if (!reasoning.empty()) { - auto tokenized = simple_tokenize(reasoning); - tokens.emplace_back("<|START_THINKING|>"); - tokens.insert(tokens.end(), tokenized.begin(), tokenized.end()); - tokens.emplace_back("<|END_THINKING|>"); - } - - if (!content.empty()) { - auto tokenized = simple_tokenize(content); - tokens.emplace_back("<|START_RESPONSE|>"); - tokens.insert(tokens.end(), tokenized.begin(), tokenized.end()); - tokens.emplace_back("<|END_RESPONSE|>"); - } - - if (!tool_calls.empty()) { - tokens.emplace_back("<|START_ACTION|>"); - - auto json = nlohmann::json::array(); - for (const auto & tc : tool_calls) { - auto tc_json = nlohmann::json::object(); - tc_json["tool_call_id"] = std::get<0>(tc); - tc_json["tool_name"] = std::get<1>(tc); - tc_json["parameters"] = std::get<2>(tc); - json.push_back(tc_json); - } - - auto tokenized = simple_tokenize(json.dump(-1, ' ', true)); - tokens.insert(tokens.end(), tokenized.begin(), tokenized.end()); - - tokens.emplace_back("<|END_ACTION|>"); - } - - input = std::accumulate(tokens.begin(), tokens.end(), std::string()); - - // Run tests - t.test("legacy_parse", [&](testing & t) { - test_command_r7b_legacy_parser(input, false, false); - }); - - t.test("current_parse", [&](testing & t) { - test_command_r7b_parser(parser, input, false, false); - }); - - // Run benchmarks - t.bench("legacy_parse_benchmark", [&]() { - test_command_r7b_legacy_parser(input, false, false); - }, 100); - - t.bench("current_parse_benchmark", [&]() { - test_command_r7b_parser(parser, input, false, false); - }, 100); - - return t.summary(); -} diff --git a/tests/peg-parser/test-basic.cpp b/tests/peg-parser/test-basic.cpp new file mode 100644 index 0000000000000..206cdc6f4f102 --- /dev/null +++ b/tests/peg-parser/test-basic.cpp @@ -0,0 +1,454 @@ +#include "tests.h" + +void test_basic(testing & t) { + t.test("one", [](testing & t) { + // Test common escape sequences - newline + t.test("escape_sequence_newline", [](testing &t) { + auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); + + common_peg_parse_context ctx; + common_peg_parse_result result; + + ctx = common_peg_parse_context("\n"); + result = common_chat_combinator_parser.parse(ctx); + t.assert_equal("escape_sequence_newline", true, result.success()); + }); + + // Test common escape sequences - tab + t.test("escape_sequence_tab", [](testing &t) { + auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); + + common_peg_parse_context ctx; + common_peg_parse_result result; + + ctx = common_peg_parse_context("\t"); + result = common_chat_combinator_parser.parse(ctx); + t.assert_equal("escape_sequence_tab", true, result.success()); + }); + + // Test common escape sequences - backslash + t.test("escape_sequence_backslash", [](testing &t) { + auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); + + common_peg_parse_context ctx; + common_peg_parse_result result; + + ctx = common_peg_parse_context("\\"); + result = common_chat_combinator_parser.parse(ctx); + t.assert_equal("escape_sequence_backslash", true, result.success()); + }); + + // Test common escape sequences - space (should ()) + t.test("escape_sequence_space_fail", [](testing &t) { + auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); + + common_peg_parse_context ctx; + common_peg_parse_result result; + + ctx = common_peg_parse_context(" "); + result = common_chat_combinator_parser.parse(ctx); + t.assert_equal("escape_sequence_space_fail", true, result.fail()); + }); + + // Test escaped dash - 'a' should succeed + t.test("escaped_dash_a", [](testing &t) { + auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[a\\-z]"); }); + + common_peg_parse_context ctx; + common_peg_parse_result result; + + ctx = common_peg_parse_context("a"); + result = common_chat_combinator_parser.parse(ctx); + t.assert_equal("escaped_dash_a", true, result.success()); + }); + + // Test escaped dash - '-' should succeed (literal dash) + t.test("escaped_dash_literal", [](testing &t) { + auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[a\\-z]"); }); + + common_peg_parse_context ctx; + common_peg_parse_result result; + + ctx = common_peg_parse_context("-"); + result = common_chat_combinator_parser.parse(ctx); + t.assert_equal("escaped_dash_literal", true, result.success()); + }); + + // Test escaped dash - 'z' should succeed + t.test("escaped_dash_z", [](testing &t) { + auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[a\\-z]"); }); + + common_peg_parse_context ctx; + common_peg_parse_result result; + + ctx = common_peg_parse_context("z"); + result = common_chat_combinator_parser.parse(ctx); + t.assert_equal("escaped_dash_z", true, result.success()); + }); + + // Test escaped dash - 'b' should NOT match (since \- is literal dash, not range) + t.test("escaped_dash_b_fail", [](testing &t) { + auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[a\\-z]"); }); + + common_peg_parse_context ctx; + common_peg_parse_result result; + + ctx = common_peg_parse_context("b"); + result = common_chat_combinator_parser.parse(ctx); + t.assert_equal("escaped_dash_b_fail", true, result.fail()); + }); + }); + + + t.test("optional", [](testing & t) { + // Full match with optional part present + t.test("optional_present", [](testing &t) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { + return p.literal("hello") + p.optional(p.literal(" world")); + }); + + auto ctx = common_peg_parse_context("hello world"); + auto result = parser.parse(ctx); + t.assert_equal("optional_present", true, result.success()); + t.assert_equal("optional_present_end", 11u, result.end); + }); + + // Full match with optional part absent + t.test("optional_absent", [](testing &t) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { + return p.literal("hello") + p.optional(p.literal(" world")); + }); + + auto ctx = common_peg_parse_context("hello", true); + auto result = parser.parse(ctx); + t.assert_equal("optional_absent", true, result.success()); + t.assert_equal("optional_absent_end", 5u, result.end); + }); + + // Partial match - waiting for more input to determine if optional matches + t.test("partial_match_need_more", [](testing &t) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { + return p.literal("hello") + p.optional(p.literal(" world")); + }); + + auto ctx = common_peg_parse_context("hello ", false); + auto result = parser.parse(ctx); + t.assert_equal("partial_match_need_more", true, result.need_more_input()); + }); + }); + + t.test("partial parsing", [](testing & t) { + // Literals - Basic Success + t.test("literal_success", [&](testing & t) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("hello"); }); + + common_peg_parse_context ctx; + common_peg_parse_result result; + + ctx = common_peg_parse_context("hello"); + result = parser.parse(ctx); + t.assert_equal("literal_success", true, result.success()); + }); + + // Char Classes - Basic Lowercase Success + t.test("char_class_lowercase_success", [&](testing & t) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("a-z"); }); + + common_peg_parse_context ctx; + common_peg_parse_result result; + + ctx = common_peg_parse_context("a"); + result = parser.parse(ctx); + t.assert_equal("char_class_lowercase_success", true, result.success()); + }); + + // Char Classes - Uppercase Fail + t.test("char_class_uppercase_fail", [&](testing & t) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("a-z"); }); + + common_peg_parse_context ctx; + common_peg_parse_result result; + + ctx = common_peg_parse_context("A"); + result = parser.parse(ctx); + t.assert_equal("char_class_uppercase_fail", true, result.fail()); + }); + + // Char Classes with Dash - Lowercase Success + t.test("char_class_with_dash_lowercase", [&](testing & t) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("a-z-"); }); + + common_peg_parse_context ctx; + common_peg_parse_result result; + + ctx = common_peg_parse_context("f"); + result = parser.parse(ctx); + t.assert_equal("char_class_with_dash_lowercase", true, result.success()); + }); + + // Char Classes with Dash - Literal Dash Success + t.test("char_class_with_dash_literal_dash", [&](testing & t) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("a-z-"); }); + + common_peg_parse_context ctx; + common_peg_parse_result result; + + ctx = common_peg_parse_context("-"); + result = parser.parse(ctx); + t.assert_equal("char_class_with_dash_literal_dash", true, result.success()); + }); + + // Char Classes with Dash - Uppercase Fail + t.test("char_class_with_dash_uppercase_fail", [&](testing & t) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("a-z-"); }); + + common_peg_parse_context ctx; + common_peg_parse_result result; + + ctx = common_peg_parse_context("A"); + result = parser.parse(ctx); + t.assert_equal("char_class_with_dash_uppercase_fail", true, result.fail()); + }); + + // Sequences - Partial Match 1 + t.test("sequence_partial_match_1", [&](testing & t) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("") + p.literal(""); }); + + auto ctx = common_peg_parse_context("") + p.literal(""); }); + + auto ctx = common_peg_parse_context("") + p.literal(""); }); + + auto ctx = common_peg_parse_context("I am common_chat_combinator_parser", false); + auto result = parser.parse(ctx); + t.assert_equal("sequence_no_match", true, result.fail()); + }); + + // Choices - Partial Match 1 + t.test("choices_partial_match_1", [&](testing & t) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("option1") | p.literal("option2"); }); + + auto ctx = common_peg_parse_context("opt", false); + auto result = parser.parse(ctx); + t.assert_equal("choices_partial_match_1", true, result.need_more_input()); + }); + + // Choices - Partial Match 2 + t.test("choices_partial_match_2", [&](testing & t) { + auto parser = + build_peg_parser([](common_peg_parser_builder & p) { return p.literal("choice_a") | p.literal("choice_b"); }); + + auto ctx = common_peg_parse_context("choice", false); + auto result = parser.parse(ctx); + t.assert_equal("choices_partial_match_2", true, result.need_more_input()); + }); + + // Choices - Full Match 1 + t.test("choices_full_match_1", [&](testing & t) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("first") | p.literal("second"); }); + + auto ctx = common_peg_parse_context("first", true); + auto result = parser.parse(ctx); + t.assert_equal("choices_full_match_1", true, result.success()); + }); + + // Choices - Full Match 2 + t.test("choices_full_match_2", [&](testing & t) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("alpha") | p.literal("beta"); }); + + auto ctx = common_peg_parse_context("beta", true); + auto result = parser.parse(ctx); + t.assert_equal("choices_full_match_2", true, result.success()); + }); + + // Choices - No Match + t.test("choices_no_match", [&](testing & t) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("good") | p.literal("better"); }); + + auto ctx = common_peg_parse_context("best", true); + auto result = parser.parse(ctx); + t.assert_equal("choices_no_match", true, result.fail()); + }); + + // Zero or More - Partial Match 1 + t.test("zero_or_more_partial_match_1", [&](testing & t) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.literal("ab")); }); + + auto ctx = common_peg_parse_context("a", false); + auto result = parser.parse(ctx); + t.assert_equal("zero_or_more_partial_match_1", true, result.need_more_input()); + }); + + // Zero or More - Partial Match 2 + t.test("zero_or_more_partial_match_2", [&](testing & t) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.literal("xy")); }); + + auto ctx = common_peg_parse_context("xyx", false); + auto result = parser.parse(ctx); + t.assert_equal("zero_or_more_partial_match_2", true, result.need_more_input()); + }); + + // Zero or More - Full Match + t.test("zero_or_more_full_match", [&](testing & t) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.literal("test")); }); + + auto ctx = common_peg_parse_context("test", true); + auto result = parser.parse(ctx); + t.assert_equal("zero_or_more_full_match", true, result.success()); + }); + + // One or More - Partial Match 1 + t.test("one_or_more_partial_match_1", [&](testing & t) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("repeat")); }); + + auto ctx = common_peg_parse_context("rep", false); + auto result = parser.parse(ctx); + t.assert_equal("one_or_more_partial_match_1", true, result.need_more_input()); + }); + + // One or More - Partial Match 2 + t.test("one_or_more_partial_match_2", [&](testing & t) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("ab")); }); + + auto ctx = common_peg_parse_context("aba", false); + auto result = parser.parse(ctx); + t.assert_equal("one_or_more_partial_match_2", true, result.need_more_input()); + }); + + // One or More - Full Match + t.test("one_or_more_full_match", [&](testing & t) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("single")); }); + + auto ctx = common_peg_parse_context("single", true); + auto result = parser.parse(ctx); + t.assert_equal("one_or_more_full_match", true, result.success()); + }); + + // One or More - No Match + t.test("one_or_more_no_match", [&](testing & t) { + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("()")); }); + + auto ctx = common_peg_parse_context("success", true); + auto result = parser.parse(ctx); + t.assert_equal("one_or_more_no_match", true, result.fail()); + }); + }); + + + t.test("recursive rules", [](testing &t) { + // Test simple number + t.test("simple_number", [](testing &t) { + auto value_parser = build_peg_parser([](common_peg_parser_builder & p) { + p.rule("number", p.one_or_more(p.one("0-9"))); + p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); + return p.rule("value", p.ref("number") | p.ref("list")); + }); + + common_peg_parse_context ctx("1", true); + auto result = value_parser.parse(ctx); + + t.assert_equal("result_is_success", true, result.success()); + }); + + // Test simple list + t.test("simple_list", [](testing &t) { + auto value_parser = build_peg_parser([](common_peg_parser_builder & p) { + p.rule("number", p.one_or_more(p.one("0-9"))); + p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); + return p.rule("value", p.ref("number") | p.ref("list")); + }); + + common_peg_parse_context ctx("[1]", true); + auto result = value_parser.parse(ctx); + + t.assert_equal("result_is_success", true, result.success()); + }); + + // Test nested list + t.test("nested_list", [](testing &t) { + auto value_parser = build_peg_parser([](common_peg_parser_builder & p) { + p.rule("number", p.one_or_more(p.one("0-9"))); + p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); + return p.rule("value", p.ref("number") | p.ref("list")); + }); + + common_peg_parse_context ctx("[[2]]", true); + auto result = value_parser.parse(ctx); + + t.assert_equal("result_is_success", true, result.success()); + }); + + // Test deeply nested list + t.test("deeply_nested_list", [](testing &t) { + auto value_parser = build_peg_parser([](common_peg_parser_builder & p) { + p.rule("number", p.one_or_more(p.one("0-9"))); + p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); + return p.rule("value", p.ref("number") | p.ref("list")); + }); + + common_peg_parse_context ctx("[[[3]]]", true); + auto result = value_parser.parse(ctx); + + t.assert_equal("result_is_success", true, result.success()); + }); + + // Test need_more_input match + t.test("need_more_input_match", [](testing &t) { + auto value_parser = build_peg_parser([](common_peg_parser_builder & p) { + p.rule("number", p.one_or_more(p.one("0-9"))); + p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); + return p.rule("value", p.ref("number") | p.ref("list")); + }); + + common_peg_parse_context ctx("[[", false); + auto result = value_parser.parse(ctx); + + t.assert_equal("result_is_need_more_input", true, result.need_more_input()); + }); + + // Test no match + t.test("no_match", [](testing &t) { + auto value_parser = build_peg_parser([](common_peg_parser_builder & p) { + p.rule("number", p.one_or_more(p.one("0-9"))); + p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); + return p.rule("value", p.ref("number") | p.ref("list")); + }); + + common_peg_parse_context ctx("[a]", true); + auto result = value_parser.parse(ctx); + + t.assert_equal("result_is_fail", true, result.fail()); + }); + }); +} diff --git a/tests/peg-parser/test-command7-parser-compare.cpp b/tests/peg-parser/test-command7-parser-compare.cpp deleted file mode 100644 index 95a0df53dc9ba..0000000000000 --- a/tests/peg-parser/test-command7-parser-compare.cpp +++ /dev/null @@ -1,256 +0,0 @@ -#include "../common/chat-parser.h" -#include "json-schema-to-grammar.h" -#include "tests.h" - -#include -#include -#include -#include - -static common_peg_arena create_command_r7b_parser() { - auto parser = build_chat_peg_parser([](common_chat_peg_parser_builder & p) { - auto thinking = p.rule("thinking", - "<|START_THINKING|>" << p.rule("reasoning-content", p.until("<|END_THINKING|>")) << "<|END_THINKING|>"); - - auto response = p.rule("response", - "<|START_RESPONSE|>" << p.rule("content", p.until("<|END_RESPONSE|>")) << "<|END_RESPONSE|>"); - - auto json = p.rule("json", p.json()); - - auto tool_call_id = p.rule("tool-call-id", - "\"tool_call_id\"" << (":" << p.rule("tool-call-id-value", "\"" + p.json_string_content() + "\""))); - - auto tool_call_name = p.rule("tool-name", - "\"tool_name\"" << (":" << p.rule("tool-name-value", "\"" + p.json_string_content() + "\""))); - - auto tool_call_args = p.rule("tool-args", - "\"parameters\"" << (":" << p.rule("tool-args-value", json))); - - auto tool_call_fields = p.rule("tool-call-fields", tool_call_id | tool_call_name | tool_call_args); - - auto tool_call = p.rule("tool-call", - "{" << tool_call_fields << p.zero_or_more(p.literal(",") << tool_call_fields) << "}"); - - auto tool_calls = p.rule("tool-calls", - "<|START_ACTION|>" - << ("[" << tool_call << p.zero_or_more(p.literal(",") << tool_call) << "]") - << "<|END_ACTION|>"); - - return p.optional(thinking) << (tool_calls | response); - }); - - // Check if - build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); - return parser; -} - -static common_peg_parse_event_handler create_command_r7b_event_handler() { - return [](const common_peg_parse_event & ev, common_peg_parse_semantics & semantics) { - if (ev.rule == "reasoning-content" && ev.ending()) { - semantics.reasoning_content = ev.text; - } - - if (ev.rule == "content" && ev.ending()) { - semantics.content = ev.text; - } - - if (ev.rule == "tool-call" && ev.starting()) { - semantics.tool_calls.emplace_back(); - } - - if (ev.rule == "tool-call-id-value" && ev.ending() && ev.success()) { - auto & tc = semantics.tool_calls.back(); - tc.id = nlohmann::json::parse(ev.text).get(); - } - - if (ev.rule == "tool-name-value" && ev.ending() && ev.success()) { - auto & tc = semantics.tool_calls.back(); - tc.name = nlohmann::json::parse(ev.text).get(); - } - - if (ev.rule == "tool-args-value" && ev.ending() && (ev.success() || ev.need_more_input())) { - auto & tc = semantics.tool_calls.back(); - tc.arguments = ev.text; - } - }; -} - -static void test_command_r7b_parser(const common_peg_arena & p, - const std::string & input, - bool need_more_input, - bool print_results) { - common_peg_parse_semantics semantics; - common_peg_parse_context ctx(input, &semantics, !need_more_input); - p.parse(ctx); - - if (print_results) { - std::cout << "== Parsed (new) ==\n"; - std::cout << "=== Reasoning ===\n"; - std::cout << semantics.reasoning_content << "\n"; - std::cout << "\n\n=== Content ===\n"; - std::cout << semantics.content << "\n"; - std::cout << "\n\n=== Tool Calls ===\n"; - for (const auto & tc : semantics.tool_calls) { - std::cout << "id: " << tc.id << "\n"; - std::cout << "name: " << tc.name << "\n"; - std::cout << "args: " << tc.arguments << "\n"; - } - } -} - -static void test_command_r7b_legacy_parser(const std::string & input, - bool need_more_input, - bool print_results) { - // Original common_chat_combinator_parser taken from chat.cpp - common_chat_msg_parser builder(input, - /* .is_partial = */ need_more_input, - { - /* .format = */ COMMON_CHAT_FORMAT_GENERIC, - /* .reasoning_format = */ COMMON_REASONING_FORMAT_AUTO, - /* .reasoning_in_content = */ false, - /* .thinking_forced_open = */ false, - }); - - builder.try_parse_reasoning("<|START_THINKING|>", "<|END_THINKING|>"); - - static const common_regex start_action_regex("<\\|START_ACTION\\|>"); - static const common_regex end_action_regex("<\\|END_ACTION\\|>"); - static const common_regex start_response_regex("<\\|START_RESPONSE\\|>"); - static const common_regex end_response_regex("<\\|END_RESPONSE\\|>"); - - if (auto res = builder.try_find_regex(start_action_regex)) { - // If we didn't extract thoughts, prelude includes them. - auto tool_calls = builder.consume_json_with_dumped_args({ { "parameters" } }); - for (const auto & tool_call : tool_calls.value) { - std::string name = tool_call.contains("tool_name") ? tool_call.at("tool_name") : ""; - std::string id = tool_call.contains("tool_call_id") ? tool_call.at("tool_call_id") : ""; - std::string arguments = tool_call.contains("parameters") ? tool_call.at("parameters") : ""; - if (!builder.add_tool_call(name, id, arguments) || tool_calls.is_partial) { - throw common_chat_msg_partial_exception("incomplete tool call"); - } - } - if (tool_calls.is_partial) { - throw common_chat_msg_partial_exception("incomplete tool call"); - } - builder.consume_regex(end_action_regex); - } else if (auto res = builder.try_find_regex(start_response_regex)) { - if (!builder.try_find_regex(end_response_regex)) { - builder.add_content(builder.consume_rest()); - throw common_chat_msg_partial_exception(end_response_regex.str()); - } - } else { - builder.add_content(builder.consume_rest()); - } - - if (print_results) { - std::cout << "== Parsed (legacy) ==\n"; - std::cout << "=== Reasoning ===\n"; - std::cout << builder.result().reasoning_content << "\n"; - std::cout << "\n\n=== Content ===\n"; - std::cout << builder.result().content << "\n"; - std::cout << "\n\n=== Tool Calls ===\n"; - for (const auto & tc : builder.result().tool_calls) { - std::cout << "id: " << tc.id << "\n"; - std::cout << "name: " << tc.name << "\n"; - std::cout << "args: " << tc.arguments << "\n"; - } - } -} - -void test_command7_parser_compare(testing &t) { - // Setup data - auto parser = create_command_r7b_parser(); - auto handler = create_command_r7b_event_handler(); - - std::string reasoning = "To plan an effective trip to Japan that includes both historical sites and modern attractions within a " - "budget of $4000 for a two-week stay, we need to:\n\n" - "1. Identify key historical sites and modern attractions in Japan.\n" - "2. Find affordable accommodation options that provide a balance between comfort and cost.\n" - "3. Determine the best modes of transportation for getting around Japan.\n" - "4. Create a day-by-day itinerary that ensures the user gets to see a variety of attractions without " - "overspending.\n" - "5. Provide a detailed cost breakdown that includes accommodation, transportation, meals, and entry fees " - "to attractions."; - - std::string content = "For a two-week trip to Japan with a $4,000 budget, I recommend planning an itinerary that balances " - "historical sites with modern attractions. The destination will be Japan, with a duration of 14 days.\n\n" - "Given your interests in both historical sites and modern attractions, you'll want to focus on cities like " - "Kyoto for its temples and traditional culture, Tokyo for its cutting-edge technology and entertainment " - "districts, and possibly Hiroshima or Nara for additional historical significance.\n\n" - "For accommodation, I suggest looking for affordable options such as budget hotels, hostels, or " - "guesthouses that offer good value without sacrificing too much comfort. Japan has excellent mid-range " - "accommodation options that can keep your lodging costs manageable.\n\n" - "Transportation should prioritize efficiencyβ€”consider getting a JR Rail Pass for intercity travel, which " - "allows unlimited rides on most JR trains including the Shinkansen (bullet train). Within cities, use " - "local trains and subways, which are both affordable and highly reliable.\n\n" - "For meals, embrace local cuisine by eating at neighborhood restaurants, ramen shops, and izakayas rather " - "than touristy establishments. This will give you an authentic experience while keeping costs " - "reasonableβ€”you can enjoy excellent meals for $10-20 per person at local spots.\n\n"; - - std::vector> tool_calls = { - { "call_0", "plan_trip", nlohmann::json::parse(R"({ - "destination": "Japan", - "duration": 14, - "budget": 4000, - "interests": ["historical sites", "modern attractions"], - "accommodation_preferences": "affordable", - "transportation_preferences": "efficient", - "meal_preferences": "local cuisine" - })") } - }; - - std::vector tokens; - - // Build tokens - if (!reasoning.empty()) { - auto tokenized = simple_tokenize(reasoning); - tokens.emplace_back("<|START_THINKING|>"); - tokens.insert(tokens.end(), tokenized.begin(), tokenized.end()); - tokens.emplace_back("<|END_THINKING|>"); - } - - if (!content.empty()) { - auto tokenized = simple_tokenize(content); - tokens.emplace_back("<|START_RESPONSE|>"); - tokens.insert(tokens.end(), tokenized.begin(), tokenized.end()); - tokens.emplace_back("<|END_RESPONSE|>"); - } - - if (!tool_calls.empty()) { - tokens.emplace_back("<|START_ACTION|>"); - - auto json = nlohmann::json::array(); - for (const auto & tc : tool_calls) { - auto tc_json = nlohmann::json::object(); - tc_json["tool_call_id"] = std::get<0>(tc); - tc_json["tool_name"] = std::get<1>(tc); - tc_json["parameters"] = std::get<2>(tc); - json.push_back(tc_json); - } - - auto tokenized = simple_tokenize(json.dump(-1, ' ', true)); - tokens.insert(tokens.end(), tokenized.begin(), tokenized.end()); - - tokens.emplace_back("<|END_ACTION|>"); - } - - std::string input = std::accumulate(tokens.begin(), tokens.end(), std::string()); - - // Run tests - t.test("legacy_parse", [&](testing & t) { - test_command_r7b_legacy_parser(input, false, false); - }); - - t.test("current_parse", [&](testing & t) { - test_command_r7b_parser(parser, input, false, false); - }); - - // Run benchmarks - t.bench("legacy_parse_benchmark", [&]() { - test_command_r7b_legacy_parser(input, false, false); - }, 100); - - t.bench("current_parse_benchmark", [&]() { - test_command_r7b_parser(parser, input, false, false); - }, 100); -} diff --git a/tests/peg-parser/test-example-minimax-m2.cpp b/tests/peg-parser/test-example-minimax-m2.cpp deleted file mode 100644 index 752149425e9c2..0000000000000 --- a/tests/peg-parser/test-example-minimax-m2.cpp +++ /dev/null @@ -1,64 +0,0 @@ -#include "chat-peg-parser.h" -#include "common.h" -#include "peg-parser.h" -#include "nlohmann/json.hpp" -#include "tests.h" - -#include -#include - -void test_example_minimax_m2(testing &t) { - auto helper_parser = build_chat_peg_parser([](common_chat_peg_parser_builder & p) { - auto thinking = p.reasoning(); - auto content = p.content_before_tools(""); - auto function = p.quasi_xml_attr("generate_joke", - std::vector({ - "category" - })); - auto tool_call = p.rule("tool-call", - "" + p.one_or_more(function) + "", true); - - return thinking + p.optional(p.space() + content) + p.zero_or_more(p.space() + tool_call); - }); - - - t.test("minimax_m2_accumulation_test", [&](testing &t) { - std::string input = - "" - "To keep the reply light I’ll fetch a random joke using the `generate_joke` tool." - "" - "" - "" - "funny" - ""; - - std::vector tokens = simple_tokenize(input); - // t.log("Tokens: " + string_join(tokens, ", ")); - - common_chat_msg prev; - common_peg_parse_result last_result; - t.test("helper_builder", [&](testing &t) { - for (auto it = tokens.begin(); it != tokens.end(); it++) { - std::string in = std::accumulate(tokens.begin(), it + 1, std::string()); - // t.log("Current input: " + in); - - common_peg_parse_semantics semantics; - common_peg_parse_context ctx(in, &semantics, it + 1 == tokens.end()); - - common_chat_peg_simple_handler handler; - ctx.set_event_handler(handler); - - auto result = helper_parser.parse(ctx); - last_result = result; - t.assert_equal("not fail", false, result.fail()); - - // This shouldn't emit any runtime errors - auto msg = semantics.to_msg(); - auto diffs = common_chat_msg_diff::compute_diffs(prev, msg); - prev = msg; - } - // t.log("Last message: " + prev.to_json_oaicompat().dump()); - t.assert_true("last_result_should_be_success", last_result.success()); - }); - }); -} diff --git a/tests/peg-parser/test-example-qwen3-coder.cpp b/tests/peg-parser/test-example-qwen3-coder.cpp deleted file mode 100644 index 087d638140fdb..0000000000000 --- a/tests/peg-parser/test-example-qwen3-coder.cpp +++ /dev/null @@ -1,134 +0,0 @@ -#include "log.h" -#include "tests.h" - -#include -#include - -void test_example_qwen3_coder(testing &t) { - auto explicit_parser = build_chat_peg_parser([](common_chat_peg_parser_builder & p) { - auto content = p.rule("content", p.until("")); - - auto arg_name = p.rule("arg-start", ""); - auto arg_end = p.rule("arg-end", "" + p.peek(p.literal("")); - - auto string_arg_content = p.rule("arg-string-content", - p.until_one_of({""})); - - auto string_arg = p.rule("arg-string", "arg-string", arg_name + string_arg_content + arg_end); - - auto json = p.json(); - - auto json_arg = p.rule("arg-json", arg_name + p.rule("arg-json-content", json) + arg_end); - - auto function = p.rule("function", - p.rule("function-start", "") - + p.one_or_more(json_arg | string_arg) - + ""); - - auto tool_call = p.rule("tool-call", - "" + p.one_or_more(function) + "", true); - - return thinking + p.optional(p.space() + content) + p.zero_or_more(p.space() + tool_call) + p.end(); - }); - - - auto helper_parser = build_chat_peg_parser([](common_chat_peg_parser_builder & p) { - auto thinking = p.reasoning(); - auto content = p.content_before_tools(""); - auto function = p.quasi_xml_no_attr("search_files", - std::vector({ - "path", "pattern", "min_size_mb", "max_depth", "include_hidden", "modified_days_ago", - "case_sensitive", "sort_by", "filters" - })); - auto tool_call = p.rule("tool-call", - "" + p.one_or_more(function) + "", true); - - return thinking + p.optional(p.space() + content) + p.zero_or_more(p.space() + tool_call) + p.end(); - }); - - t.test("qwen3_accumulation_test", [&](testing &t) { - std::string input = - "Based on your requirements, I'll search for log files over 100MB that haven't been " - "accessed in the last month. This will help identify candidates for cleanup or archival.\n\n" - "" - "" - "/var/log" - "*.log" - "100" - "5" - "false" - "30" - "true" - "size" - "{\"exclude_patterns\": [\"*temp*\", \"*cache*\"], \"file_types\": " - "[\"regular\"]}" - "" - ""; - - std::vector tokens = simple_tokenize(input); - - t.test("explicit_builder", [&](testing &t) { - common_chat_msg prev; - for (auto it = tokens.begin(); it != tokens.end(); it++) { - std::string in = std::accumulate(tokens.begin(), it + 1, std::string()); - - common_peg_parse_semantics semantics; - common_peg_parse_context ctx(in, &semantics, it == tokens.end() - 1); - - common_chat_peg_simple_handler handler; - // handler.log = [&](const std::string & msg) { - // t.log(msg); - // }; - - ctx.set_event_handler(handler); - - auto result = explicit_parser.parse(ctx); - if (!t.assert_equal("not fail", false, result.fail())) { - LOG_ERR("%s[failed-->]%s\n", in.substr(0, result.end).c_str(), in.substr(result.end).c_str()); - } - - auto msg = semantics.to_msg(); - - try { - // This shouldn't emit any runtime errors - auto diffs = common_chat_msg_diff::compute_diffs(prev, msg); - } catch(const std::exception & e) { - LOG_ERR("%s[failed-->]%s\n", in.substr(0, result.end).c_str(), in.substr(result.end).c_str()); - t.assert_true(std::string("failed with ") + e.what(), false); - } - - prev = msg; - } - }); - - t.test("helper_builder", [&](testing &t) { - common_chat_msg prev; - for (auto it = tokens.begin(); it != tokens.end(); it++) { - std::string in = std::accumulate(tokens.begin(), it + 1, std::string()); - - common_peg_parse_semantics semantics; - common_peg_parse_context ctx(in, &semantics, it + 1 == tokens.end()); - - common_chat_peg_simple_handler handler; - ctx.set_event_handler(handler); - - auto result = helper_parser.parse(ctx); - if (!t.assert_equal("not fail", false, result.fail())) { - LOG_ERR("%s[failed-->]%s\n", in.substr(0, result.end).c_str(), in.substr(result.end).c_str()); - } - - auto msg = semantics.to_msg(); - - try { - // This shouldn't emit any runtime errors - auto diffs = common_chat_msg_diff::compute_diffs(prev, msg); - } catch(const std::exception & e) { - LOG_ERR("%s[failed-->]%s\n", in.substr(0, result.end).c_str(), in.substr(result.end).c_str()); - t.assert_true(std::string("failed with ") + e.what(), false); - } - - prev = msg; - } - }); - }); -} diff --git a/tests/peg-parser/test-example-seed-oss.cpp b/tests/peg-parser/test-example-seed-oss.cpp deleted file mode 100644 index 7cb3f49c4248e..0000000000000 --- a/tests/peg-parser/test-example-seed-oss.cpp +++ /dev/null @@ -1,54 +0,0 @@ -#include "chat-peg-parser.h" -#include "tests.h" - -#include -#include - -void test_example_seed_oss(testing &t) { - auto helper_parser = build_chat_peg_parser([](common_chat_peg_parser_builder & p) { - auto thinking = p.reasoning("seed:think"); - auto content = p.content_before_tools(""); - auto function = p.quasi_xml_no_attr("get_weather", - std::vector({ - "location", "units" - })); - auto tool_call = p.rule("tool-call", - "" + p.one_or_more(function) + "", true); - - return thinking + p.optional(p.space() + content) + p.zero_or_more(p.space() + tool_call); - }); - - - t.test("seed_oss_accumulation_test", [&](testing &t) { - std::string input = - "Next I need the current weather for Berlin. I'll call the `get_weather` tool.assistant" - "" - "" - "Berlin" - "metric" - "" - ""; - std::vector tokens = simple_tokenize(input); - - common_chat_msg prev; - t.test("helper_builder", [&](testing &t) { - for (auto it = tokens.begin(); it != tokens.end(); it++) { - std::string in = std::accumulate(tokens.begin(), it + 1, std::string()); - - common_peg_parse_semantics semantics; - common_peg_parse_context ctx(in, &semantics, it == tokens.end()); - - common_chat_peg_simple_handler handler; - ctx.set_event_handler(handler); - - auto result = helper_parser.parse(ctx); - t.assert_equal("not fail", false, result.fail()); - - // This shouldn't emit any runtime errors - auto msg = semantics.to_msg(); - auto diffs = common_chat_msg_diff::compute_diffs(prev, msg); - prev = msg; - } - }); - }); -} diff --git a/tests/peg-parser/test-one.cpp b/tests/peg-parser/test-one.cpp deleted file mode 100644 index a2cdfe78eb2a6..0000000000000 --- a/tests/peg-parser/test-one.cpp +++ /dev/null @@ -1,99 +0,0 @@ -#include "tests.h" - -void test_one(testing &t) { - // Test common escape sequences - newline - t.test("escape_sequence_newline", [](testing &t) { - auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); - - common_peg_parse_context ctx; - common_peg_parse_result result; - - ctx = common_peg_parse_context("\n"); - result = common_chat_combinator_parser.parse(ctx); - t.assert_equal("escape_sequence_newline", true, result.success()); - }); - - // Test common escape sequences - tab - t.test("escape_sequence_tab", [](testing &t) { - auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); - - common_peg_parse_context ctx; - common_peg_parse_result result; - - ctx = common_peg_parse_context("\t"); - result = common_chat_combinator_parser.parse(ctx); - t.assert_equal("escape_sequence_tab", true, result.success()); - }); - - // Test common escape sequences - backslash - t.test("escape_sequence_backslash", [](testing &t) { - auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); - - common_peg_parse_context ctx; - common_peg_parse_result result; - - ctx = common_peg_parse_context("\\"); - result = common_chat_combinator_parser.parse(ctx); - t.assert_equal("escape_sequence_backslash", true, result.success()); - }); - - // Test common escape sequences - space (should ()) - t.test("escape_sequence_space_fail", [](testing &t) { - auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); - - common_peg_parse_context ctx; - common_peg_parse_result result; - - ctx = common_peg_parse_context(" "); - result = common_chat_combinator_parser.parse(ctx); - t.assert_equal("escape_sequence_space_fail", true, result.fail()); - }); - - // Test escaped dash - 'a' should succeed - t.test("escaped_dash_a", [](testing &t) { - auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[a\\-z]"); }); - - common_peg_parse_context ctx; - common_peg_parse_result result; - - ctx = common_peg_parse_context("a"); - result = common_chat_combinator_parser.parse(ctx); - t.assert_equal("escaped_dash_a", true, result.success()); - }); - - // Test escaped dash - '-' should succeed (literal dash) - t.test("escaped_dash_literal", [](testing &t) { - auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[a\\-z]"); }); - - common_peg_parse_context ctx; - common_peg_parse_result result; - - ctx = common_peg_parse_context("-"); - result = common_chat_combinator_parser.parse(ctx); - t.assert_equal("escaped_dash_literal", true, result.success()); - }); - - // Test escaped dash - 'z' should succeed - t.test("escaped_dash_z", [](testing &t) { - auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[a\\-z]"); }); - - common_peg_parse_context ctx; - common_peg_parse_result result; - - ctx = common_peg_parse_context("z"); - result = common_chat_combinator_parser.parse(ctx); - t.assert_equal("escaped_dash_z", true, result.success()); - }); - - // Test escaped dash - 'b' should NOT match (since \- is literal dash, not range) - t.test("escaped_dash_b_fail", [](testing &t) { - auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[a\\-z]"); }); - - common_peg_parse_context ctx; - common_peg_parse_result result; - - ctx = common_peg_parse_context("b"); - result = common_chat_combinator_parser.parse(ctx); - t.assert_equal("escaped_dash_b_fail", true, result.fail()); - }); -} diff --git a/tests/peg-parser/test-optional.cpp b/tests/peg-parser/test-optional.cpp deleted file mode 100644 index e47bb4b118e1a..0000000000000 --- a/tests/peg-parser/test-optional.cpp +++ /dev/null @@ -1,38 +0,0 @@ -#include "tests.h" - -void test_optional(testing &t) { - // Full match with optional part present - t.test("optional_present", [](testing &t) { - auto parser = build_peg_parser([](common_peg_parser_builder & p) { - return p.literal("hello") + p.optional(p.literal(" world")); - }); - - auto ctx = common_peg_parse_context("hello world"); - auto result = parser.parse(ctx); - t.assert_equal("optional_present", true, result.success()); - t.assert_equal("optional_present_end", 11u, result.end); - }); - - // Full match with optional part absent - t.test("optional_absent", [](testing &t) { - auto parser = build_peg_parser([](common_peg_parser_builder & p) { - return p.literal("hello") + p.optional(p.literal(" world")); - }); - - auto ctx = common_peg_parse_context("hello", true); - auto result = parser.parse(ctx); - t.assert_equal("optional_absent", true, result.success()); - t.assert_equal("optional_absent_end", 5u, result.end); - }); - - // Partial match - waiting for more input to determine if optional matches - t.test("partial_match_need_more", [](testing &t) { - auto parser = build_peg_parser([](common_peg_parser_builder & p) { - return p.literal("hello") + p.optional(p.literal(" world")); - }); - - auto ctx = common_peg_parse_context("hello ", false); - auto result = parser.parse(ctx); - t.assert_equal("partial_match_need_more", true, result.need_more_input()); - }); -} diff --git a/tests/peg-parser/test-partial-parsing.cpp b/tests/peg-parser/test-partial-parsing.cpp deleted file mode 100644 index 35d38e35b1cb4..0000000000000 --- a/tests/peg-parser/test-partial-parsing.cpp +++ /dev/null @@ -1,230 +0,0 @@ -#include "tests.h" -#include "test_harness.h" - -void test_partial_parsing(testing &t) { - // Literals - Basic Success - t.test("literal_success", [&](testing & t) { - auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("hello"); }); - - common_peg_parse_context ctx; - common_peg_parse_result result; - - ctx = common_peg_parse_context("hello"); - result = parser.parse(ctx); - t.assert_equal("literal_success", true, result.success()); - }); - - // Char Classes - Basic Lowercase Success - t.test("char_class_lowercase_success", [&](testing & t) { - auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("a-z"); }); - - common_peg_parse_context ctx; - common_peg_parse_result result; - - ctx = common_peg_parse_context("a"); - result = parser.parse(ctx); - t.assert_equal("char_class_lowercase_success", true, result.success()); - }); - - // Char Classes - Uppercase Fail - t.test("char_class_uppercase_fail", [&](testing & t) { - auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("a-z"); }); - - common_peg_parse_context ctx; - common_peg_parse_result result; - - ctx = common_peg_parse_context("A"); - result = parser.parse(ctx); - t.assert_equal("char_class_uppercase_fail", true, result.fail()); - }); - - // Char Classes with Dash - Lowercase Success - t.test("char_class_with_dash_lowercase", [&](testing & t) { - auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("a-z-"); }); - - common_peg_parse_context ctx; - common_peg_parse_result result; - - ctx = common_peg_parse_context("f"); - result = parser.parse(ctx); - t.assert_equal("char_class_with_dash_lowercase", true, result.success()); - }); - - // Char Classes with Dash - Literal Dash Success - t.test("char_class_with_dash_literal_dash", [&](testing & t) { - auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("a-z-"); }); - - common_peg_parse_context ctx; - common_peg_parse_result result; - - ctx = common_peg_parse_context("-"); - result = parser.parse(ctx); - t.assert_equal("char_class_with_dash_literal_dash", true, result.success()); - }); - - // Char Classes with Dash - Uppercase Fail - t.test("char_class_with_dash_uppercase_fail", [&](testing & t) { - auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("a-z-"); }); - - common_peg_parse_context ctx; - common_peg_parse_result result; - - ctx = common_peg_parse_context("A"); - result = parser.parse(ctx); - t.assert_equal("char_class_with_dash_uppercase_fail", true, result.fail()); - }); - - // Sequences - Partial Match 1 - t.test("sequence_partial_match_1", [&](testing & t) { - auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("") + p.literal(""); }); - - auto ctx = common_peg_parse_context("") + p.literal(""); }); - - auto ctx = common_peg_parse_context("") + p.literal(""); }); - - auto ctx = common_peg_parse_context("I am common_chat_combinator_parser", false); - auto result = parser.parse(ctx); - t.assert_equal("sequence_no_match", true, result.fail()); - }); - - // Choices - Partial Match 1 - t.test("choices_partial_match_1", [&](testing & t) { - auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("option1") | p.literal("option2"); }); - - auto ctx = common_peg_parse_context("opt", false); - auto result = parser.parse(ctx); - t.assert_equal("choices_partial_match_1", true, result.need_more_input()); - }); - - // Choices - Partial Match 2 - t.test("choices_partial_match_2", [&](testing & t) { - auto parser = - build_peg_parser([](common_peg_parser_builder & p) { return p.literal("choice_a") | p.literal("choice_b"); }); - - auto ctx = common_peg_parse_context("choice", false); - auto result = parser.parse(ctx); - t.assert_equal("choices_partial_match_2", true, result.need_more_input()); - }); - - // Choices - Full Match 1 - t.test("choices_full_match_1", [&](testing & t) { - auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("first") | p.literal("second"); }); - - auto ctx = common_peg_parse_context("first", true); - auto result = parser.parse(ctx); - t.assert_equal("choices_full_match_1", true, result.success()); - }); - - // Choices - Full Match 2 - t.test("choices_full_match_2", [&](testing & t) { - auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("alpha") | p.literal("beta"); }); - - auto ctx = common_peg_parse_context("beta", true); - auto result = parser.parse(ctx); - t.assert_equal("choices_full_match_2", true, result.success()); - }); - - // Choices - No Match - t.test("choices_no_match", [&](testing & t) { - auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("good") | p.literal("better"); }); - - auto ctx = common_peg_parse_context("best", true); - auto result = parser.parse(ctx); - t.assert_equal("choices_no_match", true, result.fail()); - }); - - // Zero or More - Partial Match 1 - t.test("zero_or_more_partial_match_1", [&](testing & t) { - auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.literal("ab")); }); - - auto ctx = common_peg_parse_context("a", false); - auto result = parser.parse(ctx); - t.assert_equal("zero_or_more_partial_match_1", true, result.need_more_input()); - }); - - // Zero or More - Partial Match 2 - t.test("zero_or_more_partial_match_2", [&](testing & t) { - auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.literal("xy")); }); - - auto ctx = common_peg_parse_context("xyx", false); - auto result = parser.parse(ctx); - t.assert_equal("zero_or_more_partial_match_2", true, result.need_more_input()); - }); - - // Zero or More - Full Match - t.test("zero_or_more_full_match", [&](testing & t) { - auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.literal("test")); }); - - auto ctx = common_peg_parse_context("test", true); - auto result = parser.parse(ctx); - t.assert_equal("zero_or_more_full_match", true, result.success()); - }); - - // One or More - Partial Match 1 - t.test("one_or_more_partial_match_1", [&](testing & t) { - auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("repeat")); }); - - auto ctx = common_peg_parse_context("rep", false); - auto result = parser.parse(ctx); - t.assert_equal("one_or_more_partial_match_1", true, result.need_more_input()); - }); - - // One or More - Partial Match 2 - t.test("one_or_more_partial_match_2", [&](testing & t) { - auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("ab")); }); - - auto ctx = common_peg_parse_context("aba", false); - auto result = parser.parse(ctx); - t.assert_equal("one_or_more_partial_match_2", true, result.need_more_input()); - }); - - // One or More - Full Match - t.test("one_or_more_full_match", [&](testing & t) { - auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("single")); }); - - auto ctx = common_peg_parse_context("single", true); - auto result = parser.parse(ctx); - t.assert_equal("one_or_more_full_match", true, result.success()); - }); - - // One or More - No Match - t.test("one_or_more_no_match", [&](testing & t) { - auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("()")); }); - - auto ctx = common_peg_parse_context("success", true); - auto result = parser.parse(ctx); - t.assert_equal("one_or_more_no_match", true, result.fail()); - }); -} diff --git a/tests/peg-parser/test-recursive-references.cpp b/tests/peg-parser/test-recursive-references.cpp deleted file mode 100644 index 8f7e8b0a191fd..0000000000000 --- a/tests/peg-parser/test-recursive-references.cpp +++ /dev/null @@ -1,87 +0,0 @@ -#include "tests.h" - -void test_recursive_references(testing &t) { - // Test simple number - t.test("simple_number", [](testing &t) { - auto value_parser = build_peg_parser([](common_peg_parser_builder & p) { - p.rule("number", p.one_or_more(p.one("0-9"))); - p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); - return p.rule("value", p.ref("number") | p.ref("list")); - }); - - common_peg_parse_context ctx("1", true); - auto result = value_parser.parse(ctx); - - t.assert_equal("result_is_success", true, result.success()); - }); - - // Test simple list - t.test("simple_list", [](testing &t) { - auto value_parser = build_peg_parser([](common_peg_parser_builder & p) { - p.rule("number", p.one_or_more(p.one("0-9"))); - p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); - return p.rule("value", p.ref("number") | p.ref("list")); - }); - - common_peg_parse_context ctx("[1]", true); - auto result = value_parser.parse(ctx); - - t.assert_equal("result_is_success", true, result.success()); - }); - - // Test nested list - t.test("nested_list", [](testing &t) { - auto value_parser = build_peg_parser([](common_peg_parser_builder & p) { - p.rule("number", p.one_or_more(p.one("0-9"))); - p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); - return p.rule("value", p.ref("number") | p.ref("list")); - }); - - common_peg_parse_context ctx("[[2]]", true); - auto result = value_parser.parse(ctx); - - t.assert_equal("result_is_success", true, result.success()); - }); - - // Test deeply nested list - t.test("deeply_nested_list", [](testing &t) { - auto value_parser = build_peg_parser([](common_peg_parser_builder & p) { - p.rule("number", p.one_or_more(p.one("0-9"))); - p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); - return p.rule("value", p.ref("number") | p.ref("list")); - }); - - common_peg_parse_context ctx("[[[3]]]", true); - auto result = value_parser.parse(ctx); - - t.assert_equal("result_is_success", true, result.success()); - }); - - // Test need_more_input match - t.test("need_more_input_match", [](testing &t) { - auto value_parser = build_peg_parser([](common_peg_parser_builder & p) { - p.rule("number", p.one_or_more(p.one("0-9"))); - p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); - return p.rule("value", p.ref("number") | p.ref("list")); - }); - - common_peg_parse_context ctx("[[", false); - auto result = value_parser.parse(ctx); - - t.assert_equal("result_is_need_more_input", true, result.need_more_input()); - }); - - // Test no match - t.test("no_match", [](testing &t) { - auto value_parser = build_peg_parser([](common_peg_parser_builder & p) { - p.rule("number", p.one_or_more(p.one("0-9"))); - p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); - return p.rule("value", p.ref("number") | p.ref("list")); - }); - - common_peg_parse_context ctx("[a]", true); - auto result = value_parser.parse(ctx); - - t.assert_equal("result_is_fail", true, result.fail()); - }); -} diff --git a/tests/peg-parser/tests.h b/tests/peg-parser/tests.h index f5cbd3f1dd906..0681f70358699 100644 --- a/tests/peg-parser/tests.h +++ b/tests/peg-parser/tests.h @@ -17,15 +17,8 @@ struct bench_tool_call { }; // Test function declarations -void test_partial_parsing(testing &t); -void test_one(testing &t); -void test_optional(testing &t); -void test_recursive_references(testing &t); +void test_basic(testing &t); void test_json_parser(testing &t); void test_gbnf_generation(testing &t); -void test_example_qwen3_coder(testing &t); -void test_example_seed_oss(testing &t); -void test_example_minimax_m2(testing &t); -void test_command7_parser_compare(testing &t); void test_unicode(testing &t); void test_json_serialization(testing &t); diff --git a/tests/test-peg-parser.cpp b/tests/test-peg-parser.cpp index 954e97ed5b4fc..20975eb25dfae 100644 --- a/tests/test-peg-parser.cpp +++ b/tests/test-peg-parser.cpp @@ -9,11 +9,8 @@ int main(int argc, char *argv[]) { t.set_filter(argv[1]); } - t.test("partial", test_partial_parsing); - t.test("one", test_one); - t.test("optional", test_optional); + t.test("basic", test_basic); t.test("unicode", test_unicode); - t.test("recursive", test_recursive_references); t.test("json", test_json_parser); t.test("gbnf", test_gbnf_generation); t.test("serialization", test_json_serialization); From d83db0bb52bd5a239420e0934e5cc8f87ec84afa Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 22 Nov 2025 15:25:19 -0600 Subject: [PATCH 155/183] use gbnf_format_literal from json-schema-to-grammar --- common/peg-parser.cpp | 19 ++----------------- tests/peg-parser/test-gbnf-generation.cpp | 4 ++-- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp index a3f72d3854a95..53ca86a7aa9a4 100644 --- a/common/peg-parser.cpp +++ b/common/peg-parser.cpp @@ -1129,21 +1129,6 @@ common_peg_parser common_peg_parser_builder::json() { // GBNF generation helper functions -static std::string gbnf_literal(const std::string & s) { - std::string escaped; - for (char c : s) { - switch (c) { - case '\n': escaped += "\\n"; break; - case '\t': escaped += "\\t"; break; - case '\r': escaped += "\\r"; break; - case '\\': escaped += "\\\\"; break; - case '"': escaped += "\\\""; break; - default: escaped += c; break; - } - } - return "\"" + escaped + "\""; -} - static std::string gbnf_escape_char_class(char c) { switch (c) { case '\n': return "\\n"; @@ -1176,7 +1161,7 @@ static std::string gbnf_excluding_pattern(const std::vector & strin } if (!pre.empty()) { - pattern += gbnf_literal(pre) + " [^" + cls + "]"; + pattern += gbnf_format_literal(pre) + " [^" + cls + "]"; } else { pattern += "[^" + cls + "]"; } @@ -1256,7 +1241,7 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo if constexpr (std::is_same_v || std::is_same_v) { return ""; } else if constexpr (std::is_same_v) { - return gbnf_literal(p.literal); + return gbnf_format_literal(p.literal); } else if constexpr (std::is_same_v) { std::string s; for (const auto & child : p.children) { diff --git a/tests/peg-parser/test-gbnf-generation.cpp b/tests/peg-parser/test-gbnf-generation.cpp index 7623d85f86168..8750d49919235 100644 --- a/tests/peg-parser/test-gbnf-generation.cpp +++ b/tests/peg-parser/test-gbnf-generation.cpp @@ -168,7 +168,7 @@ void test_gbnf_generation(testing &t) { t.test("escaping in literals", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { - return p.literal("hello\nworld\t!"); + return p.literal("hello\nworld\n!"); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { @@ -176,7 +176,7 @@ void test_gbnf_generation(testing &t) { }); assert_gbnf_equal(t, R"""( - root ::= "hello\nworld\t!" + root ::= "hello\nworld\n!" space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); From 89a80c791b4e7afe5824c42aeb551de5de2dd476 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 22 Nov 2025 22:06:25 -0600 Subject: [PATCH 156/183] integrate parser with common/chat and server --- common/chat-peg-parser.cpp | 10 +- common/chat.cpp | 189 +++++++++++++++++++++++------- common/chat.h | 10 +- common/json-schema-to-grammar.cpp | 4 +- common/peg-parser.cpp | 41 +++++-- common/peg-parser.h | 12 ++ tools/server/server.cpp | 22 +++- tools/server/utils.hpp | 3 + 8 files changed, 233 insertions(+), 58 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 955eba8f0b09b..13a96a9b220c9 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -1,5 +1,9 @@ #include "chat-peg-parser.h" +#include + +using json = nlohmann::json; + void common_chat_peg_mapper::from_ast(const common_peg_ast_arena & arena, const common_peg_parse_result & result) { arena.visit(result, [this](const common_peg_ast_node & node) { map(node); @@ -78,12 +82,14 @@ void common_chat_peg_constructed_mapper::map(const common_peg_ast_node & node) { if (arg_count > 0) { current_tool->arguments += ","; } - current_tool->arguments += "\"" + std::string(node.text) + "\":"; + current_tool->arguments += json(node.text).dump() + ":"; ++arg_count; } if (is_arg_string && current_tool) { - current_tool->arguments += "\"" + std::string(node.text); + // Serialize to JSON, but exclude the end quote + std::string dumped = json(node.text).dump(); + current_tool->arguments += dumped.substr(0, dumped.size() - 1); needs_closing_quote = true; } diff --git a/common/chat.cpp b/common/chat.cpp index 6fa05a60416d4..582275c6560c6 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1,9 +1,11 @@ #include "chat.h" #include "chat-parser.h" +#include "chat-peg-parser.h" #include "common.h" #include "json-partial.h" #include "json-schema-to-grammar.h" #include "log.h" +#include "peg-parser.h" #include "regex-partial.h" #include @@ -646,9 +648,11 @@ const char * common_chat_format_name(common_chat_format format) { case COMMON_CHAT_FORMAT_MINIMAX_M2: return "MiniMax-M2"; case COMMON_CHAT_FORMAT_GLM_4_5: return "GLM 4.5"; case COMMON_CHAT_FORMAT_KIMI_K2: return "Kimi K2"; - case COMMON_CHAT_FORMAT_QWEN3_CODER_XML: return "Qwen3 Coder"; case COMMON_CHAT_FORMAT_APRIEL_1_5: return "Apriel 1.5"; case COMMON_CHAT_FORMAT_XIAOMI_MIMO: return "Xiaomi MiMo"; + case COMMON_CHAT_FORMAT_PEG_SIMPLE: return "peg-simple"; + case COMMON_CHAT_FORMAT_PEG_NATIVE: return "peg-native"; + case COMMON_CHAT_FORMAT_PEG_CONSTRUCTED: return "peg-constructed"; default: throw std::runtime_error("Unknown chat format"); } @@ -796,6 +800,25 @@ static void foreach_function(const json & tools, const std::function & fn) { + if (!function.contains("parameters") || !function.at("parameters").is_object()) { + return; + } + const auto & params = function.at("parameters"); + if (!params.contains("properties") || !params.at("properties").is_object()) { + return; + } + const auto & props = params.at("properties"); + std::set required; + if (props.contains("required") && props.at("required").is_array()) { + props.at("required").get_to(required); + } + for (const auto & [name, prop] : props.items()) { + bool is_required = (required.find(name) != required.end()); + fn(name, prop, is_required); + } +} + static std::string apply( const common_chat_template & tmpl, const struct templates_params & inputs, @@ -1872,53 +1895,110 @@ static void common_chat_parse_minimax_m2(common_chat_msg_parser & builder) { static common_chat_params common_chat_params_init_qwen3_coder_xml(const common_chat_template & tmpl, const struct templates_params & params) { common_chat_params data; - data.grammar_lazy = params.tools.is_array() && !params.tools.empty() && params.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED; + + bool use_tools = params.tools.is_array() && !params.tools.empty() && params.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; + bool tool_required = params.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED; + + data.grammar_lazy = use_tools && !tool_required; data.prompt = apply(tmpl, params); - data.format = COMMON_CHAT_FORMAT_QWEN3_CODER_XML; + data.format = COMMON_CHAT_FORMAT_PEG_CONSTRUCTED; - data.preserved_tokens = { - "", - "", - "", - "", - }; + data.preserved_tokens = {"", ""}; - // build grammar for tool call - static const xml_tool_call_format form { - /* form.scope_start = */ "\n", - /* form.tool_start = */ "\n", - /* form.key_start = */ "\n", - /* form.val_end = */ "\n\n", - /* form.tool_end = */ "\n", - /* form.scope_end = */ "", + auto parser = build_chat_peg_constructed_parser([&](common_chat_peg_constructed_builder & p) { + if (!use_tools) { + return p.rule("content", p.content(p.rest())); + } + + auto content = p.rule("content", p.content(p.until_one_of({"", " tool_parsers; + foreach_function(params.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + std::string fn_name = function.at("name"); + + std::vector argument_parsers; + foreach_parameter(function, [&](const std::string & name, const json & schema, bool is_required) { + auto arg_value = p.literal(""); + if (schema.contains("type") && schema.at("type") == "string") { + arg_value = p.tool_arg_string_value(p.schema( + p.until_one_of({ + "\n\n\n" + }), + "tool-" + fn_name + "-arg-" + name + "-schema", + schema, + true + )); + } else { + arg_value = p.tool_arg_json_value(p.schema( + p.json(), + "tool-" + fn_name + "-arg-" + name + "-schema", + schema + )); + } + + auto arg = p.tool_arg(p.sequence({ + p.tool_arg_open(""), + p.space(), + arg_value, + p.space(), + p.tool_arg_close( + "\n" + + p.peek(p.literal("")) + ) + })); + + argument_parsers.push_back(is_required ? + p.rule("tool-" + fn_name + "-arg-" + name, arg) : + p.optional(p.rule("tool-" + fn_name + "-arg-" + name, arg))); + }); + + tool_parsers.push_back(p.rule("tool-" + fn_name, + p.tool_open("") + << p.sequence(argument_parsers) + << p.tool_close(p.literal("")) + )); + }); + + auto tool_call = p.trigger_rule("tool-call", + p.optional("" + p.space()) + + p.choice(tool_parsers) + + p.space() + + "" + // We have to handle parallel tool calls here because it is a trigger rule + + (params.parallel_tool_calls ? + p.repeat(p.space() + "" << p.choice(tool_parsers) << "", 0, -1) : + p.eps()) + ); + + return p.sequence({ + content, + p.repeat(p.space() + tool_call, (tool_required ? 1 : 0), 1), + p.end() + }); + }); + + data.parser = parser.to_json().dump(); + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + foreach_function(params.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + auto parameters = function.at("parameters"); + builder.resolve_refs(parameters); + }); + parser.build_grammar(builder, data.grammar_lazy); + }); + + data.grammar_triggers = { + {COMMON_GRAMMAR_TRIGGER_TYPE_WORD, ""}, + {COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "({msg}).at(0).dump().c_str()); + } + return msg; +} diff --git a/common/chat.h b/common/chat.h index 754c411e23718..89fccde6960e7 100644 --- a/common/chat.h +++ b/common/chat.h @@ -9,6 +9,8 @@ #include #include +class common_peg_arena; + struct common_chat_templates; struct common_chat_tool_call { @@ -120,10 +122,14 @@ enum common_chat_format { COMMON_CHAT_FORMAT_GLM_4_5, COMMON_CHAT_FORMAT_MINIMAX_M2, COMMON_CHAT_FORMAT_KIMI_K2, - COMMON_CHAT_FORMAT_QWEN3_CODER_XML, COMMON_CHAT_FORMAT_APRIEL_1_5, COMMON_CHAT_FORMAT_XIAOMI_MIMO, + // These are intended to be parsed by the PEG parser + COMMON_CHAT_FORMAT_PEG_SIMPLE, + COMMON_CHAT_FORMAT_PEG_NATIVE, + COMMON_CHAT_FORMAT_PEG_CONSTRUCTED, + COMMON_CHAT_FORMAT_COUNT, // Not a format, just the # formats }; @@ -154,6 +160,7 @@ struct common_chat_params { std::vector grammar_triggers; std::vector preserved_tokens; std::vector additional_stops; + std::string parser; }; struct common_chat_syntax { @@ -206,6 +213,7 @@ const char* common_chat_format_name(common_chat_format format); const char* common_reasoning_format_name(common_reasoning_format format); common_reasoning_format common_reasoning_format_from_name(const std::string & format); common_chat_msg common_chat_parse(const std::string & input, bool is_partial, const common_chat_syntax & syntax); +common_chat_msg common_chat_peg_parse(const std::string & input, bool is_partial, const common_peg_arena & parser, const common_chat_syntax & syntax); common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::string & tool_choice); diff --git a/common/json-schema-to-grammar.cpp b/common/json-schema-to-grammar.cpp index 26f2f85acf3e4..32c73f34e0d34 100644 --- a/common/json-schema-to-grammar.cpp +++ b/common/json-schema-to-grammar.cpp @@ -1012,9 +1012,9 @@ class SchemaConverter { } // TODO: support minimum, maximum, exclusiveMinimum, exclusiveMaximum at least for zero if (is_raw) { - auto it = PRIMITIVE_RAW_RULES.find(schema_type.get()); + auto it = PRIMITIVE_RAW_RULES.find(schema_type.get() + "-raw"); if (it != PRIMITIVE_RAW_RULES.end()) { - return _add_primitive(rule_name == "root" ? "root" : schema_type.get(), it->second); + return _add_primitive(rule_name == "root" ? "root" : schema_type.get() + "-raw", it->second); } } return _add_primitive(rule_name == "root" ? "root" : schema_type.get(), PRIMITIVE_RULES.at(schema_type.get())); diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp index 53ca86a7aa9a4..3e7b10dfbed75 100644 --- a/common/peg-parser.cpp +++ b/common/peg-parser.cpp @@ -298,6 +298,10 @@ struct parser_executor { parser_executor(const common_peg_arena & arena, common_peg_parse_context & ctx, size_t start) : arena(arena), ctx(ctx), start_pos(start) {} + common_peg_parse_result operator()(const common_peg_epsilon_parser & /* p */) const { + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos); + } + common_peg_parse_result operator()(const common_peg_start_parser & /* p */) const { return common_peg_parse_result( start_pos == 0 ? COMMON_PEG_PARSE_RESULT_SUCCESS : COMMON_PEG_PARSE_RESULT_FAIL, @@ -789,7 +793,8 @@ void common_peg_arena::resolve_refs() { p.child = resolve_ref(p.child); } else if constexpr (std::is_same_v) { p.child = resolve_ref(p.child); - } else if constexpr (std::is_same_v || + } else if constexpr (std::is_same_v || + std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || @@ -818,7 +823,9 @@ std::string common_peg_arena::dump(common_peg_parser_id id) const { return std::visit([this](const auto & p) -> std::string { using T = std::decay_t; - if constexpr (std::is_same_v) { + if constexpr (std::is_same_v) { + return "Epsilon"; + } else if constexpr (std::is_same_v) { return "Start"; } else if constexpr (std::is_same_v) { return "End"; @@ -1184,7 +1191,8 @@ static std::unordered_set collect_reachable_rules( std::visit([&](const auto & p) { using T = std::decay_t; - if constexpr (std::is_same_v || + if constexpr (std::is_same_v || + std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || @@ -1238,7 +1246,9 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo return std::visit([&](const auto & p) -> std::string { using T = std::decay_t; - if constexpr (std::is_same_v || std::is_same_v) { + if constexpr (std::is_same_v || + std::is_same_v || + std::is_same_v) { return ""; } else if constexpr (std::is_same_v) { return gbnf_format_literal(p.literal); @@ -1320,6 +1330,9 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo } else if constexpr (std::is_same_v) { return R"(( [^"\\] | "\\" ( ["\\/ bfnrt] | "u" [0-9a-fA-F]{4} ) )*)"; } else if constexpr (std::is_same_v) { + if (p.delimiters.empty()) { + return ".*"; + } return gbnf_excluding_pattern(p.delimiters); } else if constexpr (std::is_same_v) { if (p.schema) { @@ -1333,8 +1346,15 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo if (pattern.find(".*") != std::string::npos) { return to_gbnf(p.child); } + } else if (p.schema->contains("enum") || + p.schema->contains("const") || + p.schema->contains("minLength") || + p.schema->contains("maxLength") || + p.schema->contains("allOf") || + p.schema->contains("anyOf")) { + return builder.add_string_schema(p.name, *p.schema); } - return builder.add_string_schema(p.name, *p.schema); + return to_gbnf(p.child); } return builder.add_schema(p.name, *p.schema); } @@ -1414,7 +1434,9 @@ static nlohmann::json serialize_parser_variant(const common_peg_parser_variant & nlohmann::json j; - if constexpr (std::is_same_v) { + if constexpr (std::is_same_v) { + j["type"] = "epsilon"; + } else if constexpr (std::is_same_v) { j["type"] = "start"; } else if constexpr (std::is_same_v) { j["type"] = "end"; @@ -1470,6 +1492,7 @@ static nlohmann::json serialize_parser_variant(const common_peg_parser_variant & } else { j["schema"] = nullptr; } + j["raw"] = p.raw; } else if constexpr (std::is_same_v) { j["type"] = "rule"; j["name"] = p.name; @@ -1517,6 +1540,9 @@ static common_peg_parser_variant deserialize_parser_variant(const nlohmann::json std::string type = j["type"]; + if (type == "epsilon") { + return common_peg_epsilon_parser{}; + } if (type == "start") { return common_peg_start_parser{}; } @@ -1600,7 +1626,7 @@ static common_peg_parser_variant deserialize_parser_variant(const nlohmann::json return common_peg_until_parser{j["delimiters"].get>()}; } if (type == "schema") { - if (!j.contains("child") || !j.contains("name") || !j.contains("schema")) { + if (!j.contains("child") || !j.contains("name") || !j.contains("schema") || !j.contains("raw")) { throw std::runtime_error("schema parser missing required fields"); } common_peg_schema_parser parser; @@ -1609,6 +1635,7 @@ static common_peg_parser_variant deserialize_parser_variant(const nlohmann::json if (!j["schema"].is_null()) { parser.schema = std::make_shared(j["schema"]); } + parser.raw = j["raw"].get(); return parser; } if (type == "rule") { diff --git a/common/peg-parser.h b/common/peg-parser.h index a8f88b87a6597..e13b4972edda0 100644 --- a/common/peg-parser.h +++ b/common/peg-parser.h @@ -158,6 +158,8 @@ struct common_peg_parse_context { class common_peg_arena; // Parser variant structs (value-based, no inheritance) +struct common_peg_epsilon_parser {}; + struct common_peg_start_parser {}; struct common_peg_end_parser {}; @@ -245,6 +247,7 @@ struct common_peg_tag_parser { // Variant holding all parser types using common_peg_parser_variant = std::variant< + common_peg_epsilon_parser, common_peg_start_parser, common_peg_end_parser, common_peg_literal_parser, @@ -280,6 +283,7 @@ class common_peg_arena { common_peg_parser_variant & get(common_peg_parser_id id) { return parsers_.at(id); } size_t size() const { return parsers_.size(); } + bool empty() const { return parsers_.empty(); } // Rule lookup common_peg_parser_id get_rule(const std::string & name) const; @@ -326,6 +330,10 @@ class common_peg_parser_builder { public: common_peg_parser_builder(); + // Match nothing, always succeed. + // S -> Ξ΅ + common_peg_parser eps() { return add(common_peg_epsilon_parser{}); } + // Matches the start of the input. // S -> ^ common_peg_parser start() { return add(common_peg_start_parser{}); } @@ -400,6 +408,10 @@ class common_peg_parser_builder { common_peg_parser until(const std::string & delimiter) { return add(common_peg_until_parser{{delimiter}}); } common_peg_parser until_one_of(const std::vector & delimiters) { return add(common_peg_until_parser{delimiters}); } + // Matches everything + // S -> .* + common_peg_parser rest() { return until_one_of({}); } + // Matches between min and max repetitions of a parser (inclusive). // S -> A{m,n} // Use -1 for max to represent unbounded repetition (equivalent to {m,}) diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 3750c8fdb6065..7fee293de9b69 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -7,6 +7,7 @@ #include "json-schema-to-grammar.h" #include "llama.h" #include "log.h" +#include "peg-parser.h" #include "sampling.h" #include "speculative.h" #include "mtmd.h" @@ -142,6 +143,7 @@ struct slot_params { std::string oaicompat_model; std::string oaicompat_cmpl_id; common_chat_syntax oaicompat_chat_syntax; + common_peg_arena oaicompat_chat_parser; // Embeddings int32_t embd_normalize = 2; // (-1=none, 0=max absolute int16, 1=taxicab, 2=Euclidean/L2, >2=p-norm) @@ -454,6 +456,11 @@ struct server_task { params.oaicompat_chat_syntax.parse_tool_calls = json_value(data, "parse_tool_calls", false); } + if (data.contains("chat_parser")) { + auto parser = data.at("chat_parser").get(); + params.oaicompat_chat_parser = common_peg_arena::from_json(json::parse(parser)); + } + { const auto preserved_tokens = data.find("preserved_tokens"); if (preserved_tokens != data.end()) { @@ -1862,10 +1869,17 @@ struct server_slot { auto previous_msg = chat_msg; SRV_DBG("Parsing chat message: %s\n", generated_text.c_str()); - auto new_msg = common_chat_parse( - generated_text, - /* is_partial= */ stop != STOP_TYPE_EOS, - task->params.oaicompat_chat_syntax); + auto new_msg = task->params.oaicompat_chat_parser.empty() ? + common_chat_parse( + generated_text, + /* is_partial= */ stop != STOP_TYPE_EOS, + task->params.oaicompat_chat_syntax) : + common_chat_peg_parse( + generated_text, + /* is_partial= */ stop != STOP_TYPE_EOS, + task->params.oaicompat_chat_parser, + task->params.oaicompat_chat_syntax); + if (!new_msg.empty()) { new_msg.set_tool_call_ids(generated_tool_call_ids, gen_tool_call_id); chat_msg = new_msg; diff --git a/tools/server/utils.hpp b/tools/server/utils.hpp index bf21726051e55..974de0685dc06 100644 --- a/tools/server/utils.hpp +++ b/tools/server/utils.hpp @@ -778,6 +778,9 @@ static json oaicompat_chat_params_parse( for (const auto & stop : chat_params.additional_stops) { llama_params["stop"].push_back(stop); } + if (!chat_params.parser.empty()) { + llama_params["chat_parser"] = chat_params.parser; + } // Handle "n" field int n_choices = json_value(body, "n", 1); From eee2c8be33b0e64fa635eba96ad4ed12d35b276d Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 22 Nov 2025 22:39:36 -0600 Subject: [PATCH 157/183] clean up schema and serialization --- common/chat.cpp | 56 +++++++++++++++++++---------------------- common/peg-parser.cpp | 23 ++++++++++++++--- common/peg-parser.h | 3 +++ tools/server/server.cpp | 2 +- 4 files changed, 50 insertions(+), 34 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 582275c6560c6..fa26b66ae3178 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -810,8 +810,8 @@ static void foreach_parameter(const json & function, const std::function required; - if (props.contains("required") && props.at("required").is_array()) { - props.at("required").get_to(required); + if (params.contains("required") && params.at("required").is_array()) { + params.at("required").get_to(required); } for (const auto & [name, prop] : props.items()) { bool is_required = (required.find(name) != required.end()); @@ -1899,11 +1899,10 @@ static common_chat_params common_chat_params_init_qwen3_coder_xml(const common_c bool use_tools = params.tools.is_array() && !params.tools.empty() && params.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; bool tool_required = params.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED; - data.grammar_lazy = use_tools && !tool_required; - data.prompt = apply(tmpl, params); data.format = COMMON_CHAT_FORMAT_PEG_CONSTRUCTED; + data.grammar_lazy = use_tools && !tool_required; data.preserved_tokens = {"", ""}; auto parser = build_chat_peg_constructed_parser([&](common_chat_peg_constructed_builder & p) { @@ -1913,6 +1912,11 @@ static common_chat_params common_chat_params_init_qwen3_coder_xml(const common_c auto content = p.rule("content", p.content(p.until_one_of({"", "\n\n" + })); + std::vector tool_parsers; foreach_function(params.tools, [&](const json & tool) { const auto & function = tool.at("function"); @@ -1920,39 +1924,33 @@ static common_chat_params common_chat_params_init_qwen3_coder_xml(const common_c std::vector argument_parsers; foreach_parameter(function, [&](const std::string & name, const json & schema, bool is_required) { - auto arg_value = p.literal(""); + auto arg_value = p.eps(); if (schema.contains("type") && schema.at("type") == "string") { arg_value = p.tool_arg_string_value(p.schema( - p.until_one_of({ - "\n\n\n" - }), - "tool-" + fn_name + "-arg-" + name + "-schema", - schema, - true + until_end_of_param, + /* name = */ "tool-" + fn_name + "-arg-" + name + "-schema", + /* schema = */ schema, + /* raw = */ true )); } else { arg_value = p.tool_arg_json_value(p.schema( p.json(), - "tool-" + fn_name + "-arg-" + name + "-schema", - schema + /* name = */ "tool-" + fn_name + "-arg-" + name + "-schema", + /* schema = */ schema )); } - auto arg = p.tool_arg(p.sequence({ - p.tool_arg_open(""), - p.space(), - arg_value, - p.space(), - p.tool_arg_close( + auto arg = p.tool_arg( + p.tool_arg_open("") + << arg_value + << p.tool_arg_close( "\n" + p.peek(p.literal("")) ) - })); + ); - argument_parsers.push_back(is_required ? - p.rule("tool-" + fn_name + "-arg-" + name, arg) : - p.optional(p.rule("tool-" + fn_name + "-arg-" + name, arg))); + auto arg_rule = p.rule("tool-" + fn_name + "-arg-" + name, arg); + argument_parsers.push_back(p.repeat(arg_rule, (is_required ? 1 : 0), 1)); }); tool_parsers.push_back(p.rule("tool-" + fn_name, @@ -1973,14 +1971,12 @@ static common_chat_params common_chat_params_init_qwen3_coder_xml(const common_c p.eps()) ); - return p.sequence({ - content, - p.repeat(p.space() + tool_call, (tool_required ? 1 : 0), 1), - p.end() - }); + return content + + p.repeat(p.space() + tool_call, (tool_required ? 1 : 0), 1) + + p.end(); }); - data.parser = parser.to_json().dump(); + data.parser = parser.serialize(); data.grammar = build_grammar([&](const common_grammar_builder & builder) { foreach_function(params.tools, [&](const json & tool) { const auto & function = tool.at("function"); diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp index 3e7b10dfbed75..f426a2f15a9a6 100644 --- a/common/peg-parser.cpp +++ b/common/peg-parser.cpp @@ -1199,8 +1199,7 @@ static std::unordered_set collect_reachable_rules( std::is_same_v || std::is_same_v || std::is_same_v || - std::is_same_v || - std::is_same_v) { + std::is_same_v) { // These parsers do not have any children } else if constexpr (std::is_same_v) { for (auto child : p.children) { @@ -1215,7 +1214,8 @@ static std::unordered_set collect_reachable_rules( std::is_same_v || std::is_same_v || std::is_same_v || - std::is_same_v) { + std::is_same_v || + std::is_same_v) { visit(p.child); } else if constexpr (std::is_same_v) { if (visited.find(p.name) == visited.end()) { @@ -1302,6 +1302,12 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo if (p.max_count == -1) { return child_gbnf + "{" + std::to_string(p.min_count) + ",}"; } + if (p.min_count == p.max_count) { + if (p.min_count == 1) { + return child_gbnf; + } + return child_gbnf + "{" + std::to_string(p.min_count) + "}"; + } return child_gbnf + "{" + std::to_string(p.min_count) + "," + std::to_string(p.max_count) + "}"; } else if constexpr (std::is_same_v || std::is_same_v) { return ""; // Lookahead not supported in GBNF @@ -1311,6 +1317,9 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo return "space"; } else if constexpr (std::is_same_v) { std::string result = p.pattern; + if (p.min_count == 0 && p.max_count == 1) { + return result + "?"; + } if (p.min_count == 0 && p.max_count == -1) { return result + "*"; } @@ -1718,3 +1727,11 @@ common_peg_arena common_peg_arena::from_json(const nlohmann::json & j) { return arena; } + +std::string common_peg_arena::serialize() const { + return to_json().dump(); +} + +common_peg_arena common_peg_arena::deserialize(const std::string & data) { + return from_json(nlohmann::json::parse(data)); +} diff --git a/common/peg-parser.h b/common/peg-parser.h index e13b4972edda0..ad6890a4a8c26 100644 --- a/common/peg-parser.h +++ b/common/peg-parser.h @@ -310,6 +310,9 @@ class common_peg_arena { nlohmann::json to_json() const; static common_peg_arena from_json(const nlohmann::json & j); + std::string serialize() const; + static common_peg_arena deserialize(const std::string & data); + // Builder access (for adding parsers) friend class common_peg_parser_builder; diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 7fee293de9b69..7d4a2e4ce24ea 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -458,7 +458,7 @@ struct server_task { if (data.contains("chat_parser")) { auto parser = data.at("chat_parser").get(); - params.oaicompat_chat_parser = common_peg_arena::from_json(json::parse(parser)); + params.oaicompat_chat_parser = common_peg_arena::deserialize(parser); } { From 57e8cd8e88d9912c3c4720b1c08e875f74e9f563 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 22 Nov 2025 23:22:16 -0600 Subject: [PATCH 158/183] add json-schema raw string tests --- common/json-schema-to-grammar.cpp | 14 ++- common/json-schema-to-grammar.h | 3 +- tests/test-json-schema-to-grammar.cpp | 161 ++++++++++++++++++++++++++ 3 files changed, 171 insertions(+), 7 deletions(-) diff --git a/common/json-schema-to-grammar.cpp b/common/json-schema-to-grammar.cpp index 32c73f34e0d34..8631d398424cb 100644 --- a/common/json-schema-to-grammar.cpp +++ b/common/json-schema-to-grammar.cpp @@ -267,6 +267,7 @@ static bool is_reserved_name(const std::string & name) { std::unordered_set s; s.insert("root"); for (const auto & p : PRIMITIVE_RULES) s.insert(p.first); + for (const auto & p : PRIMITIVE_RAW_RULES) s.insert(p.first); for (const auto & p : STRING_FORMAT_RULES) s.insert(p.first); return s; }(); @@ -838,10 +839,7 @@ class SchemaConverter { std::string visit(const json & schema, const std::string & name, bool is_raw) { json schema_type = schema.contains("type") ? schema["type"] : json(); std::string schema_format = schema.contains("format") ? schema["format"].get() : ""; - std::string rule_name = is_reserved_name(name) ? name + "-" : name.empty() ? "root" : name; - if (is_raw) { - rule_name += "-raw"; - } + std::string rule_name = is_reserved_name(name) ? name + "-" : name.empty() ? "root" : is_raw ? name + "-raw" : name; if (schema.contains("$ref")) { return _add_rule(rule_name, _resolve_ref(schema["$ref"], is_raw)); @@ -1039,7 +1037,7 @@ class SchemaConverter { } }; -std::string json_schema_to_grammar(const json & schema, bool force_gbnf) { +std::string json_schema_to_grammar(const json & schema, bool force_gbnf, bool raw) { #ifdef LLAMA_USE_LLGUIDANCE if (!force_gbnf) { return "%llguidance {}\nstart: %json " + schema.dump(); @@ -1050,7 +1048,11 @@ std::string json_schema_to_grammar(const json & schema, bool force_gbnf) { return build_grammar([&](const common_grammar_builder & callbacks) { auto copy = schema; callbacks.resolve_refs(copy); - callbacks.add_schema("", copy); + if (raw) { + callbacks.add_string_schema("", copy); + } else { + callbacks.add_schema("", copy); + } }); } diff --git a/common/json-schema-to-grammar.h b/common/json-schema-to-grammar.h index b51930d47c3ea..138ad0a842244 100644 --- a/common/json-schema-to-grammar.h +++ b/common/json-schema-to-grammar.h @@ -6,7 +6,8 @@ #include std::string json_schema_to_grammar(const nlohmann::ordered_json & schema, - bool force_gbnf = false); + bool force_gbnf = false, + bool raw = false); struct common_grammar_builder { std::function add_rule; diff --git a/tests/test-json-schema-to-grammar.cpp b/tests/test-json-schema-to-grammar.cpp index 8a55bc54ae466..ad6374c03341a 100755 --- a/tests/test-json-schema-to-grammar.cpp +++ b/tests/test-json-schema-to-grammar.cpp @@ -1341,6 +1341,157 @@ static void test_all(const std::string & lang, std::function runner) { + fprintf(stderr, "#\n# Testing JSON schema conversion to raw string (%s)\n#\n", lang.c_str()); + auto test = [&](const TestCase & tc) { + fprintf(stderr, "- %s%s\n", tc.name.c_str(), tc.expected_status == FAILURE ? " (failure expected)" : ""); + runner(tc); + }; + + test({ + SUCCESS, + "string", + R"""({ + "type": "string" + })""", + R"""( + char-raw ::= [^\x7F\x00-\x1F] | [\x0A\x0D] + root ::= char-raw* + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""" + }); + + test({ + SUCCESS, + "string w/ min length 1", + R"""({ + "type": "string", + "minLength": 1 + })""", + R"""( + char-raw ::= [^\x7F\x00-\x1F] | [\x0A\x0D] + root ::= char-raw+ + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""" + }); + + test({ + SUCCESS, + "string w/ min length 3", + R"""({ + "type": "string", + "minLength": 3 + })""", + R"""( + char-raw ::= [^\x7F\x00-\x1F] | [\x0A\x0D] + root ::= char-raw{3,} + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""" + }); + + test({ + SUCCESS, + "string w/ max length", + R"""({ + "type": "string", + "maxLength": 3 + })""", + R"""( + char-raw ::= [^\x7F\x00-\x1F] | [\x0A\x0D] + root ::= char-raw{0,3} + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""" + }); + + test({ + SUCCESS, + "string w/ min & max length", + R"""({ + "type": "string", + "minLength": 1, + "maxLength": 4 + })""", + R"""( + char-raw ::= [^\x7F\x00-\x1F] | [\x0A\x0D] + root ::= char-raw{1,4} + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""" + }); + + test({ + SUCCESS, + "string w/ enum", + R"""({ + "type": "string", + "enum": ["a", "b", "c"] + })""", + R"""( + root ::= ("a" | "b" | "c") + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""" + }); + + test({ + SUCCESS, + "allOf w/ enum", + R"""({ + "type": "string", + "allOf": [ + {"$ref": "#/definitions/foo"} + ], + "definitions": { + "foo": { + "type": "string", + "enum": ["a", "b", "c"] + } + } + })""", + R"""( + root ::= ("a" | "b" | "c") + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""" + }); + + test({ + SUCCESS, + "allOf w/ multiple enum", + R"""({ + "type": "string", + "allOf": [ + {"$ref": "#/definitions/foo"}, + {"$ref": "#/definitions/bar"} + ], + "definitions": { + "foo": { + "type": "string", + "enum": ["a", "b", "c"] + }, + "bar": { + "type": "string", + "enum": ["b", "c", "d"] + } + } + })""", + R"""( + root ::= ("b" | "c") + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""" + }); + + test({ + SUCCESS, + "string w/ const", + R"""({ + "type": "string", + "const": "abc" + })""", + R"""( + root ::= "abc" + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""" + }); +} + int main() { fprintf(stderr, "LLAMA_NODE_AVAILABLE = %s\n", getenv("LLAMA_NODE_AVAILABLE") ? "true" : "false"); fprintf(stderr, "LLAMA_PYTHON_AVAILABLE = %s\n", getenv("LLAMA_PYTHON_AVAILABLE") ? "true" : "false"); @@ -1355,6 +1506,16 @@ int main() { } }); + test_all_raw("C++", [](const TestCase & tc) { + try { + tc.verify(json_schema_to_grammar(nlohmann::ordered_json::parse(tc.schema), true, true)); + tc.verify_status(SUCCESS); + } catch (const std::runtime_error & ex) { + fprintf(stderr, "Error: %s\n", ex.what()); + tc.verify_status(FAILURE); + } + }); + if (getenv("LLAMA_SKIP_TESTS_SLOW_ON_EMULATOR")) { fprintf(stderr, "\033[33mWARNING: Skipping slow tests on emulator.\n\033[0m"); } else { From 3aa4d0abaa1ab67f003ae1cc6d10a6512381995e Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 22 Nov 2025 23:39:44 -0600 Subject: [PATCH 159/183] clean up json creation and remove capture parser --- common/peg-parser.cpp | 149 ++++++++++++++++-------------------------- common/peg-parser.h | 9 --- 2 files changed, 56 insertions(+), 102 deletions(-) diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp index f426a2f15a9a6..603f5a209be42 100644 --- a/common/peg-parser.cpp +++ b/common/peg-parser.cpp @@ -731,11 +731,6 @@ struct parser_executor { return arena.parse(rule_id, ctx, start_pos); } - common_peg_parse_result operator()(const common_peg_capture_parser & p) { - auto result = arena.parse(p.child, ctx, start_pos); - return result; - } - common_peg_parse_result operator()(const common_peg_atomic_parser & p) { auto result = arena.parse(p.child, ctx, start_pos); if (result.need_more_input()) { @@ -785,7 +780,6 @@ void common_peg_arena::resolve_refs() { } else if constexpr (std::is_same_v || std::is_same_v || std::is_same_v || - std::is_same_v || std::is_same_v || std::is_same_v) { p.child = resolve_ref(p.child); @@ -871,8 +865,6 @@ std::string common_peg_arena::dump(common_peg_parser_id id) const { return "Rule(" + p.name + ", " + dump(p.child) + ")"; } else if constexpr (std::is_same_v) { return "Ref(" + p.name + ")"; - } else if constexpr (std::is_same_v) { - return "Capture(" + p.key + ", " + dump(p.child) + ")"; } else { return "Unknown"; } @@ -1020,10 +1012,6 @@ common_peg_parser common_peg_parser_builder::schema(common_peg_parser p, const s return wrap(arena_.add_parser(common_peg_schema_parser{p.id(), name, std::make_shared(schema), raw})); } -common_peg_parser common_peg_parser_builder::capture(const std::string & key, common_peg_parser p) { - return wrap(arena_.add_parser(common_peg_capture_parser{p.id(), key})); -} - common_peg_parser common_peg_parser_builder::rule(const std::string & name, common_peg_parser p, bool trigger) { auto clean_name = rule_name(name); auto rule_id = arena_.add_parser(common_peg_rule_parser{clean_name, p.id(), trigger}); @@ -1212,7 +1200,6 @@ static std::unordered_set collect_reachable_rules( } else if constexpr (std::is_same_v || std::is_same_v || std::is_same_v || - std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v) { @@ -1373,8 +1360,6 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo } else if constexpr (std::is_same_v) { // Refs should not exist after flattening, but kept just in case return p.name; - } else if constexpr (std::is_same_v) { - return to_gbnf(p.child); } else if constexpr (std::is_same_v) { return to_gbnf(p.child); } else if constexpr (std::is_same_v) { @@ -1438,107 +1423,94 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo // Serialization helper: convert parser variant to JSON static nlohmann::json serialize_parser_variant(const common_peg_parser_variant & variant) { - return std::visit([](const auto & p) -> nlohmann::json { - using T = std::decay_t; + using json = nlohmann::json; - nlohmann::json j; + return std::visit([](const auto & p) -> json { + using T = std::decay_t; if constexpr (std::is_same_v) { - j["type"] = "epsilon"; + return json{{"type", "epsilon"}}; } else if constexpr (std::is_same_v) { - j["type"] = "start"; + return json{{"type", "start"}}; } else if constexpr (std::is_same_v) { - j["type"] = "end"; + return json{{"type", "end"}}; } else if constexpr (std::is_same_v) { - j["type"] = "literal"; - j["literal"] = p.literal; + return json{{"type", "literal"}, {"literal", p.literal}}; } else if constexpr (std::is_same_v) { - j["type"] = "sequence"; - j["children"] = p.children; + return json{{"type", "sequence"}, {"children", p.children}}; } else if constexpr (std::is_same_v) { - j["type"] = "choice"; - j["children"] = p.children; + return json{{"type", "choice"}, {"children", p.children}}; } else if constexpr (std::is_same_v) { - j["type"] = "repetition"; - j["child"] = p.child; - j["min_count"] = p.min_count; - j["max_count"] = p.max_count; + return json{ + {"type", "repetition"}, + {"child", p.child}, + {"min_count", p.min_count}, + {"max_count", p.max_count} + }; } else if constexpr (std::is_same_v) { - j["type"] = "and"; - j["child"] = p.child; + return json{{"type", "and"}, {"child", p.child}}; } else if constexpr (std::is_same_v) { - j["type"] = "not"; - j["child"] = p.child; + return json{{"type", "not"}, {"child", p.child}}; } else if constexpr (std::is_same_v) { - j["type"] = "any"; + return json{{"type", "any"}}; } else if constexpr (std::is_same_v) { - j["type"] = "space"; + return json{{"type", "space"}}; } else if constexpr (std::is_same_v) { - j["type"] = "chars"; - j["pattern"] = p.pattern; - nlohmann::json ranges = nlohmann::json::array(); + json ranges = json::array(); for (const auto & range : p.ranges) { - ranges.push_back({ - {"start", range.start}, - {"end", range.end} - }); + ranges.push_back({{"start", range.start}, {"end", range.end}}); } - j["ranges"] = ranges; - j["negated"] = p.negated; - j["min_count"] = p.min_count; - j["max_count"] = p.max_count; + return json{ + {"type", "chars"}, + {"pattern", p.pattern}, + {"ranges", ranges}, + {"negated", p.negated}, + {"min_count", p.min_count}, + {"max_count", p.max_count} + }; } else if constexpr (std::is_same_v) { - j["type"] = "json_string"; + return json{{"type", "json_string"}}; } else if constexpr (std::is_same_v) { - j["type"] = "until"; - j["delimiters"] = p.delimiters; + return json{{"type", "until"}, {"delimiters", p.delimiters}}; } else if constexpr (std::is_same_v) { - j["type"] = "schema"; - j["child"] = p.child; - j["name"] = p.name; - if (p.schema) { - j["schema"] = *p.schema; - } else { - j["schema"] = nullptr; - } - j["raw"] = p.raw; + return json{ + {"type", "schema"}, + {"child", p.child}, + {"name", p.name}, + {"schema", p.schema ? *p.schema : nullptr}, + {"raw", p.raw} + }; } else if constexpr (std::is_same_v) { - j["type"] = "rule"; - j["name"] = p.name; - j["child"] = p.child; - j["trigger"] = p.trigger; + return json{ + {"type", "rule"}, + {"name", p.name}, + {"child", p.child}, + {"trigger", p.trigger} + }; } else if constexpr (std::is_same_v) { - j["type"] = "ref"; - j["name"] = p.name; - } else if constexpr (std::is_same_v) { - j["type"] = "capture"; - j["child"] = p.child; - j["key"] = p.key; + return json{{"type", "ref"}, {"name", p.name}}; } else if constexpr (std::is_same_v) { - j["type"] = "atomic"; - j["child"] = p.child; + return json{{"type", "atomic"}, {"child", p.child}}; } else if constexpr (std::is_same_v) { - j["type"] = "tag"; - j["child"] = p.child; - j["tag"] = p.tag; + return json{ + {"type", "tag"}, + {"child", p.child}, + {"tag", p.tag} + }; } - - return j; }, variant); } nlohmann::json common_peg_arena::to_json() const { - nlohmann::json j; - auto parsers = nlohmann::json::array(); for (const auto & parser : parsers_) { parsers.push_back(serialize_parser_variant(parser)); } - - j["parsers"] = parsers; - j["rules"] = rules_; - j["root"] = root_; - return j; + return nlohmann::json{ + {"parsers", parsers}, + {"rules", rules_}, + {"root", root_} + }; } // Deserialization helper: convert JSON to parser variant @@ -1663,15 +1635,6 @@ static common_peg_parser_variant deserialize_parser_variant(const nlohmann::json } return common_peg_ref_parser{j["name"]}; } - if (type == "capture") { - if (!j.contains("child") || !j.contains("key")) { - throw std::runtime_error("capture parser missing required fields"); - } - return common_peg_capture_parser{ - j["child"].get(), - j["key"].get() - }; - } if (type == "atomic") { if (!j.contains("child")) { throw std::runtime_error("tag parser missing required fields"); diff --git a/common/peg-parser.h b/common/peg-parser.h index ad6890a4a8c26..51fc993acfc2e 100644 --- a/common/peg-parser.h +++ b/common/peg-parser.h @@ -231,11 +231,6 @@ struct common_peg_ref_parser { std::string name; }; -struct common_peg_capture_parser { - common_peg_parser_id child; - std::string key; -}; - struct common_peg_atomic_parser { common_peg_parser_id child; }; @@ -264,7 +259,6 @@ using common_peg_parser_variant = std::variant< common_peg_schema_parser, common_peg_rule_parser, common_peg_ref_parser, - common_peg_capture_parser, common_peg_atomic_parser, common_peg_tag_parser >; @@ -441,9 +435,6 @@ class common_peg_parser_builder { // Used internally to convert JSON schemas to GBNF grammar rules. common_peg_parser schema(common_peg_parser p, const std::string & name, const nlohmann::ordered_json & schema, bool raw = false); - // Captures matched text to semantics.captures[key] - common_peg_parser capture(const std::string & key, common_peg_parser p); - // Creates a named rule, stores it in the grammar, and returns a reference to it. // If trigger=true, marks this rule as an entry point for lazy grammar generation. // auto json = p.rule("json", json_obj | json_arr | ...) From 94de8b80f474fcd77b6b8db3167ccef5be7299f8 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 23 Nov 2025 04:03:24 -0600 Subject: [PATCH 160/183] trim spaces from reasoning and content --- common/chat-peg-parser.cpp | 15 ++++++++++----- common/peg-parser.cpp | 10 ++-------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 13a96a9b220c9..53926e1052e8d 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -4,6 +4,13 @@ using json = nlohmann::json; +static std::string_view trim_trailing_space(std::string_view sv) { + while (!sv.empty() && std::isspace(static_cast(sv.back()))) { + sv.remove_suffix(1); + } + return sv; +} + void common_chat_peg_mapper::from_ast(const common_peg_ast_arena & arena, const common_peg_parse_result & result) { arena.visit(result, [this](const common_peg_ast_node & node) { map(node); @@ -15,16 +22,14 @@ void common_chat_peg_mapper::map(const common_peg_ast_node & node) { bool is_reasoning = node.tag == common_chat_peg_builder::REASONING; bool is_content = node.tag == common_chat_peg_builder::CONTENT; - if (is_reasoning_block) { - result.reasoning_content = std::string(node.text); - } + // TODO: Handle reasoning_format if (is_reasoning) { - result.reasoning_content = std::string(node.text); + result.reasoning_content = std::string(trim_trailing_space(node.text)); } if (is_content) { - result.content = std::string(node.text); + result.content = std::string(trim_trailing_space(node.text)); } } diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp index 603f5a209be42..55706014085c9 100644 --- a/common/peg-parser.cpp +++ b/common/peg-parser.cpp @@ -26,12 +26,6 @@ const char * common_peg_parse_result_type_name(common_peg_parse_result_type type } } -// We define our own space function because MSVC's std::isspace() -// crashes for non-printable characters in Debug builds. -static bool is_space(const char c) { - return (c == ' ' || c == '\t' || c == '\n'); -} - static bool is_hex_digit(const char c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); } @@ -463,8 +457,8 @@ struct parser_executor { common_peg_parse_result operator()(const common_peg_space_parser & /* p */) { auto pos = start_pos; while (pos < ctx.input.size()) { - char c = ctx.input[pos]; - if (is_space(c)) { + auto c = static_cast(ctx.input[pos]); + if (std::isspace(c)) { ++pos; } else { break; From 4b1bc1683aec39fce33f4ef3590c848b390b95d1 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 23 Nov 2025 04:58:44 -0600 Subject: [PATCH 161/183] clean up redundant rules and comments --- common/peg-parser.cpp | 43 ++++++++---- common/peg-parser.h | 83 +++++++++-------------- tests/peg-parser/test-basic.cpp | 40 +++++------ tests/peg-parser/test-gbnf-generation.cpp | 12 ++-- 4 files changed, 87 insertions(+), 91 deletions(-) diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp index 55706014085c9..4dc94ff393540 100644 --- a/common/peg-parser.cpp +++ b/common/peg-parser.cpp @@ -866,46 +866,51 @@ std::string common_peg_arena::dump(common_peg_parser_id id) const { } // Parser wrapper operator implementations +common_peg_parser & common_peg_parser::operator=(common_peg_parser const & other) { + id_ = other.id_; + return *this; +} + common_peg_parser common_peg_parser::operator+(const common_peg_parser & other) const { - return builder_->sequence({id_, other.id_}); + return builder_.sequence({id_, other.id_}); } common_peg_parser common_peg_parser::operator|(const common_peg_parser & other) const { - return builder_->choice({id_, other.id_}); + return builder_.choice({id_, other.id_}); } common_peg_parser common_peg_parser::operator<<(const common_peg_parser & other) const { - return builder_->sequence({id_, builder_->space(), other.id_}); + return builder_.sequence({id_, builder_.space(), other.id_}); } // String literal overloads common_peg_parser common_peg_parser::operator+(const char * str) const { - return *this + builder_->literal(str); + return *this + builder_.literal(str); } common_peg_parser common_peg_parser::operator+(const std::string & str) const { - return *this + builder_->literal(str); + return *this + builder_.literal(str); } -common_peg_parser common_peg_parser::operator|(const char * str) const { - return *this | builder_->literal(str); +common_peg_parser common_peg_parser::operator<<(const char * str) const { + return *this << builder_.literal(str); } -common_peg_parser common_peg_parser::operator|(const std::string & str) const { - return *this | builder_->literal(str); +common_peg_parser common_peg_parser::operator<<(const std::string & str) const { + return *this << builder_.literal(str); } -common_peg_parser common_peg_parser::operator<<(const char * str) const { - return *this << builder_->literal(str); +common_peg_parser common_peg_parser::operator|(const char * str) const { + return *this | builder_.literal(str); } -common_peg_parser common_peg_parser::operator<<(const std::string & str) const { - return *this << builder_->literal(str); +common_peg_parser common_peg_parser::operator|(const std::string & str) const { + return *this | builder_.literal(str); } // Free function operators for string + parser common_peg_parser operator+(const char * str, const common_peg_parser & p) { - return p.builder()->literal(str) + p; + return p.builder().literal(str) + p; } common_peg_parser operator+(const std::string & str, const common_peg_parser & p) { @@ -913,13 +918,21 @@ common_peg_parser operator+(const std::string & str, const common_peg_parser & p } common_peg_parser operator<<(const char * str, const common_peg_parser & p) { - return p.builder()->literal(str) << p; + return p.builder().literal(str) << p; } common_peg_parser operator<<(const std::string & str, const common_peg_parser & p) { return operator<<(str.c_str(), p); } +common_peg_parser operator|(const char * str, const common_peg_parser & p) { + return p.builder().literal(str) | p; +} + +common_peg_parser operator|(const std::string & str, const common_peg_parser & p) { + return operator|(str.c_str(), p); +} + // Rule name helper, intended to produce valid GBNF rule names static std::string rule_name(const std::string & name) { static const std::regex invalid_rule_chars_re("[^a-zA-Z0-9-]+"); diff --git a/common/peg-parser.h b/common/peg-parser.h index 51fc993acfc2e..43778aa4f12c9 100644 --- a/common/peg-parser.h +++ b/common/peg-parser.h @@ -14,54 +14,52 @@ struct common_grammar_builder; -// Forward declarations +class common_peg_parser_builder; + using common_peg_parser_id = size_t; constexpr common_peg_parser_id COMMON_PEG_INVALID_PARSER_ID = static_cast(-1); using common_peg_ast_id = size_t; constexpr common_peg_ast_id COMMON_PEG_INVALID_AST_ID = static_cast(-1); -// Forward declare builder for parser wrapper -class common_peg_parser_builder; - -// Lightweight wrapper around common_peg_parser_id that enables operator overloading -// and implicit conversions from strings/literals +// Lightweight wrapper around common_peg_parser_id for convenience class common_peg_parser { common_peg_parser_id id_; - common_peg_parser_builder * builder_; + common_peg_parser_builder & builder_; public: - // Construct from common_peg_parser_id - common_peg_parser(common_peg_parser_id id, common_peg_parser_builder * builder) : id_(id), builder_(builder) {} + common_peg_parser(common_peg_parser_id id, common_peg_parser_builder & builder) : id_(id), builder_(builder) {} - // Implicit conversion to common_peg_parser_id - operator common_peg_parser_id() const { return id_; } + common_peg_parser & operator=(common_peg_parser const & other); - // Get the underlying ID + operator common_peg_parser_id() const { return id_; } common_peg_parser_id id() const { return id_; } - // Get builder (for free function operators) - common_peg_parser_builder * builder() const { return builder_; } + common_peg_parser_builder & builder() const { return builder_; } - // Operator overloads + // Creates a sequence common_peg_parser operator+(const common_peg_parser & other) const; + + // Creates a sequence separated by spaces. + common_peg_parser operator<<(const common_peg_parser & other) const; + + // Creates a choice common_peg_parser operator|(const common_peg_parser & other) const; - common_peg_parser operator<<(const common_peg_parser & other) const; // sequence with space - // Overloads for string literals common_peg_parser operator+(const char * str) const; common_peg_parser operator+(const std::string & str) const; - common_peg_parser operator|(const char * str) const; - common_peg_parser operator|(const std::string & str) const; common_peg_parser operator<<(const char * str) const; common_peg_parser operator<<(const std::string & str) const; + common_peg_parser operator|(const char * str) const; + common_peg_parser operator|(const std::string & str) const; }; -// Free function operators for string + parser common_peg_parser operator+(const char * str, const common_peg_parser & p); common_peg_parser operator+(const std::string & str, const common_peg_parser & p); common_peg_parser operator<<(const char * str, const common_peg_parser & p); common_peg_parser operator<<(const std::string & str, const common_peg_parser & p); +common_peg_parser operator|(const char * str, const common_peg_parser & p); +common_peg_parser operator|(const std::string & str, const common_peg_parser & p); enum common_peg_parse_result_type { COMMON_PEG_PARSE_RESULT_FAIL = 0, @@ -73,7 +71,7 @@ const char * common_peg_parse_result_type_name(common_peg_parse_result_type type struct common_peg_ast_node { common_peg_ast_id id; - std::string rule_name; + std::string rule; std::string tag; size_t start; size_t end; @@ -91,7 +89,7 @@ class common_peg_ast_arena { std::vector nodes_; public: common_peg_ast_id add_node( - const std::string & rule_name, + const std::string & rule, const std::string & tag, size_t start, size_t end, @@ -100,7 +98,7 @@ class common_peg_ast_arena { bool is_partial = false ) { common_peg_ast_id id = nodes_.size(); - nodes_.push_back({id, rule_name, tag, start, end, text, std::move(children), is_partial}); + nodes_.push_back({id, rule, tag, start, end, text, std::move(children), is_partial}); return id; } @@ -121,7 +119,7 @@ struct common_peg_parse_result { std::vector nodes; - common_peg_parse_result() : type(COMMON_PEG_PARSE_RESULT_FAIL) {} + common_peg_parse_result() = default; common_peg_parse_result(common_peg_parse_result_type type, size_t start) : type(type), start(start), end(start) {} @@ -154,10 +152,9 @@ struct common_peg_parse_context { : input(input), input_is_complete(complete), parse_depth(0) {} }; -// Forward declaration class common_peg_arena; -// Parser variant structs (value-based, no inheritance) +// Parser variants struct common_peg_epsilon_parser {}; struct common_peg_start_parser {}; @@ -218,6 +215,8 @@ struct common_peg_schema_parser { common_peg_parser_id child; std::string name; std::shared_ptr schema; + + // Indicates if the GBNF should accept a raw string that matches the schema. bool raw; }; @@ -263,7 +262,6 @@ using common_peg_parser_variant = std::variant< common_peg_tag_parser >; -// Arena owns all parsers class common_peg_arena { std::vector parsers_; std::unordered_map rules_; @@ -272,42 +270,33 @@ class common_peg_arena { public: common_peg_arena(); - // Access const common_peg_parser_variant & get(common_peg_parser_id id) const { return parsers_.at(id); } common_peg_parser_variant & get(common_peg_parser_id id) { return parsers_.at(id); } size_t size() const { return parsers_.size(); } bool empty() const { return parsers_.empty(); } - // Rule lookup common_peg_parser_id get_rule(const std::string & name) const; bool has_rule(const std::string & name) const { return rules_.find(name) != rules_.end(); } - // Root common_peg_parser_id root() const { return root_; } void set_root(common_peg_parser_id id) { root_ = id; } - // Parse common_peg_parse_result parse(common_peg_parse_context & ctx, size_t start = 0) const; common_peg_parse_result parse(common_peg_parser_id id, common_peg_parse_context & ctx, size_t start) const; - // Resolve all ref parsers to point directly to their corresponding rule parsers void resolve_refs(); - // Grammar generation void build_grammar(const common_grammar_builder & builder, bool lazy = false) const; - // Dump for debugging std::string dump(common_peg_parser_id id) const; - // Serialization nlohmann::json to_json() const; static common_peg_arena from_json(const nlohmann::json & j); std::string serialize() const; static common_peg_arena deserialize(const std::string & data); - // Builder access (for adding parsers) friend class common_peg_parser_builder; private: @@ -317,11 +306,10 @@ class common_peg_arena { common_peg_parser_id resolve_ref(common_peg_parser_id id); }; -// Builder for constructing parsers class common_peg_parser_builder { common_peg_arena arena_; - common_peg_parser wrap(common_peg_parser_id id) { return common_peg_parser(id, this); } + common_peg_parser wrap(common_peg_parser_id id) { return common_peg_parser(id, *this); } common_peg_parser add(const common_peg_parser_variant & p) { return wrap(arena_.add_parser(p)); } public: @@ -385,12 +373,6 @@ class common_peg_parser_builder { // Use -1 for max to represent unbounded repetition (equivalent to {m,}) common_peg_parser chars(const std::string & classes, int min = 1, int max = -1); - // Matches a single character from a character class or range. - // S -> [a-z] or S -> [^0-9] - // - // Equivalent to chars(classes, 1, 1) - common_peg_parser one(const std::string & classes) { return chars(classes, 1, 1); } - // Creates a lightweight reference to a named rule (resolved during build()). // Use this for forward references in recursive grammars. // expr_ref -> expr @@ -403,6 +385,9 @@ class common_peg_parser_builder { // Matches all characters until a delimiter is found (delimiter not consumed). // S -> (!delim .)* common_peg_parser until(const std::string & delimiter) { return add(common_peg_until_parser{{delimiter}}); } + + // Matches all characters until one of the delimiters in the list is found (delimiter not consumed). + // S -> (!delim .)* common_peg_parser until_one_of(const std::vector & delimiters) { return add(common_peg_until_parser{delimiters}); } // Matches everything @@ -428,22 +413,20 @@ class common_peg_parser_builder { common_peg_parser json_bool(); common_peg_parser json_null(); - // Specialized single-pass JSON string parser with escape sequence handling + // Matches JSON string content without the surrounding quotes. + // Useful for extracting content within a JSON string. common_peg_parser json_string_content(); // Wraps a parser with JSON schema metadata for grammar generation. // Used internally to convert JSON schemas to GBNF grammar rules. common_peg_parser schema(common_peg_parser p, const std::string & name, const nlohmann::ordered_json & schema, bool raw = false); - // Creates a named rule, stores it in the grammar, and returns a reference to it. + // Creates a named rule, stores it in the grammar, and returns a ref. // If trigger=true, marks this rule as an entry point for lazy grammar generation. // auto json = p.rule("json", json_obj | json_arr | ...) common_peg_parser rule(const std::string & name, common_peg_parser p, bool trigger = false); - // Creates a named rule using a builder function. This handles recursive grammars by - // inserting a placeholder rule before invoking the builder, allowing the - // builder to reference the rule being defined via ref(). Use this when the rule - // definition needs to call back to itself (directly or indirectly). + // Creates a named rule using a builder function, and returns a ref. // If trigger=true, marks this rule as an entry point for lazy grammar generation. // auto json = p.rule("json", [&]() { return json_object() | json_array() | ... }) common_peg_parser rule(const std::string & name, const std::function & builder, bool trigger = false); diff --git a/tests/peg-parser/test-basic.cpp b/tests/peg-parser/test-basic.cpp index 206cdc6f4f102..4195081f3c80e 100644 --- a/tests/peg-parser/test-basic.cpp +++ b/tests/peg-parser/test-basic.cpp @@ -1,10 +1,10 @@ #include "tests.h" void test_basic(testing & t) { - t.test("one", [](testing & t) { + t.test("chars", [](testing & t) { // Test common escape sequences - newline t.test("escape_sequence_newline", [](testing &t) { - auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); + auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[\\n\\t\\\\]"); }); common_peg_parse_context ctx; common_peg_parse_result result; @@ -16,7 +16,7 @@ void test_basic(testing & t) { // Test common escape sequences - tab t.test("escape_sequence_tab", [](testing &t) { - auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); + auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[\\n\\t\\\\]"); }); common_peg_parse_context ctx; common_peg_parse_result result; @@ -28,7 +28,7 @@ void test_basic(testing & t) { // Test common escape sequences - backslash t.test("escape_sequence_backslash", [](testing &t) { - auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); + auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[\\n\\t\\\\]"); }); common_peg_parse_context ctx; common_peg_parse_result result; @@ -40,7 +40,7 @@ void test_basic(testing & t) { // Test common escape sequences - space (should ()) t.test("escape_sequence_space_fail", [](testing &t) { - auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[\\n\\t\\\\]"); }); + auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[\\n\\t\\\\]"); }); common_peg_parse_context ctx; common_peg_parse_result result; @@ -52,7 +52,7 @@ void test_basic(testing & t) { // Test escaped dash - 'a' should succeed t.test("escaped_dash_a", [](testing &t) { - auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[a\\-z]"); }); + auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[a\\-z]"); }); common_peg_parse_context ctx; common_peg_parse_result result; @@ -64,7 +64,7 @@ void test_basic(testing & t) { // Test escaped dash - '-' should succeed (literal dash) t.test("escaped_dash_literal", [](testing &t) { - auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[a\\-z]"); }); + auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[a\\-z]"); }); common_peg_parse_context ctx; common_peg_parse_result result; @@ -76,7 +76,7 @@ void test_basic(testing & t) { // Test escaped dash - 'z' should succeed t.test("escaped_dash_z", [](testing &t) { - auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[a\\-z]"); }); + auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[a\\-z]"); }); common_peg_parse_context ctx; common_peg_parse_result result; @@ -88,7 +88,7 @@ void test_basic(testing & t) { // Test escaped dash - 'b' should NOT match (since \- is literal dash, not range) t.test("escaped_dash_b_fail", [](testing &t) { - auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("[a\\-z]"); }); + auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[a\\-z]"); }); common_peg_parse_context ctx; common_peg_parse_result result; @@ -152,7 +152,7 @@ void test_basic(testing & t) { // Char Classes - Basic Lowercase Success t.test("char_class_lowercase_success", [&](testing & t) { - auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("a-z"); }); + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("a-z"); }); common_peg_parse_context ctx; common_peg_parse_result result; @@ -164,7 +164,7 @@ void test_basic(testing & t) { // Char Classes - Uppercase Fail t.test("char_class_uppercase_fail", [&](testing & t) { - auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("a-z"); }); + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("a-z"); }); common_peg_parse_context ctx; common_peg_parse_result result; @@ -176,7 +176,7 @@ void test_basic(testing & t) { // Char Classes with Dash - Lowercase Success t.test("char_class_with_dash_lowercase", [&](testing & t) { - auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("a-z-"); }); + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("a-z-"); }); common_peg_parse_context ctx; common_peg_parse_result result; @@ -188,7 +188,7 @@ void test_basic(testing & t) { // Char Classes with Dash - Literal Dash Success t.test("char_class_with_dash_literal_dash", [&](testing & t) { - auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("a-z-"); }); + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("a-z-"); }); common_peg_parse_context ctx; common_peg_parse_result result; @@ -200,7 +200,7 @@ void test_basic(testing & t) { // Char Classes with Dash - Uppercase Fail t.test("char_class_with_dash_uppercase_fail", [&](testing & t) { - auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one("a-z-"); }); + auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("a-z-"); }); common_peg_parse_context ctx; common_peg_parse_result result; @@ -370,7 +370,7 @@ void test_basic(testing & t) { // Test simple number t.test("simple_number", [](testing &t) { auto value_parser = build_peg_parser([](common_peg_parser_builder & p) { - p.rule("number", p.one_or_more(p.one("0-9"))); + p.rule("number", p.chars("0-9")); p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); return p.rule("value", p.ref("number") | p.ref("list")); }); @@ -384,7 +384,7 @@ void test_basic(testing & t) { // Test simple list t.test("simple_list", [](testing &t) { auto value_parser = build_peg_parser([](common_peg_parser_builder & p) { - p.rule("number", p.one_or_more(p.one("0-9"))); + p.rule("number", p.chars("0-9")); p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); return p.rule("value", p.ref("number") | p.ref("list")); }); @@ -398,7 +398,7 @@ void test_basic(testing & t) { // Test nested list t.test("nested_list", [](testing &t) { auto value_parser = build_peg_parser([](common_peg_parser_builder & p) { - p.rule("number", p.one_or_more(p.one("0-9"))); + p.rule("number", p.chars("0-9")); p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); return p.rule("value", p.ref("number") | p.ref("list")); }); @@ -412,7 +412,7 @@ void test_basic(testing & t) { // Test deeply nested list t.test("deeply_nested_list", [](testing &t) { auto value_parser = build_peg_parser([](common_peg_parser_builder & p) { - p.rule("number", p.one_or_more(p.one("0-9"))); + p.rule("number", p.chars("0-9")); p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); return p.rule("value", p.ref("number") | p.ref("list")); }); @@ -426,7 +426,7 @@ void test_basic(testing & t) { // Test need_more_input match t.test("need_more_input_match", [](testing &t) { auto value_parser = build_peg_parser([](common_peg_parser_builder & p) { - p.rule("number", p.one_or_more(p.one("0-9"))); + p.rule("number", p.chars("0-9")); p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); return p.rule("value", p.ref("number") | p.ref("list")); }); @@ -440,7 +440,7 @@ void test_basic(testing & t) { // Test no match t.test("no_match", [](testing &t) { auto value_parser = build_peg_parser([](common_peg_parser_builder & p) { - p.rule("number", p.one_or_more(p.one("0-9"))); + p.rule("number", p.chars("0-9")); p.rule("list", p.literal("[") + p.ref("value") + p.literal("]")); return p.rule("value", p.ref("number") | p.ref("list")); }); diff --git a/tests/peg-parser/test-gbnf-generation.cpp b/tests/peg-parser/test-gbnf-generation.cpp index 8750d49919235..68857a5e88742 100644 --- a/tests/peg-parser/test-gbnf-generation.cpp +++ b/tests/peg-parser/test-gbnf-generation.cpp @@ -31,7 +31,7 @@ void test_gbnf_generation(testing &t) { t.test("char class grammar", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { - return p.one("[a-z]"); + return p.chars("[a-z]", 1, 1); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { @@ -76,7 +76,7 @@ void test_gbnf_generation(testing &t) { t.test("one_or_more grammar", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { - return p.one_or_more(p.one("[0-9]")); + return p.one_or_more(p.literal("a")); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { @@ -84,14 +84,14 @@ void test_gbnf_generation(testing &t) { }); assert_gbnf_equal(t, R"""( - root ::= [0-9]+ + root ::= "a"+ space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); t.test("zero_or_more grammar", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { - return p.zero_or_more(p.one("[a-z]")); + return p.zero_or_more(p.literal("a")); }); auto gbnf = build_grammar([&](const common_grammar_builder & builder) { @@ -99,7 +99,7 @@ void test_gbnf_generation(testing &t) { }); assert_gbnf_equal(t, R"""( - root ::= [a-z]* + root ::= "a"* space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); @@ -151,7 +151,7 @@ void test_gbnf_generation(testing &t) { t.test("rule references", [](testing &t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { - auto digit = p.rule("digit", p.one("[0-9]")); + auto digit = p.rule("digit", p.chars("[0-9]", 1, 1)); return p.one_or_more(digit); }); From 1e61ffb0ef86dccd7f1675a5561230a26e516509 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 23 Nov 2025 05:11:12 -0600 Subject: [PATCH 162/183] rename input_is_complete to is_partial to match rest of project --- common/chat.cpp | 2 +- common/peg-parser.cpp | 20 +++++------ common/peg-parser.h | 10 +++--- tests/peg-parser/test-basic.cpp | 50 +++++++++++++-------------- tests/peg-parser/test-json-parser.cpp | 6 ++-- tests/peg-parser/test-unicode.cpp | 22 ++++++------ tests/test-chat-peg-parser.cpp | 6 ++-- 7 files changed, 58 insertions(+), 58 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index fa26b66ae3178..56cd7d7854650 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -3614,7 +3614,7 @@ common_chat_msg common_chat_parse(const std::string & input, bool is_partial, co common_chat_msg common_chat_peg_parse(const std::string & input, bool is_partial, const common_peg_arena & parser, const common_chat_syntax & syntax) { LOG_DBG("Parsing input with format %s: %s\n", common_chat_format_name(syntax.format), input.c_str()); - common_peg_parse_context ctx(input, !is_partial); + common_peg_parse_context ctx(input, is_partial); auto result = parser.parse(ctx); if (result.fail()) { // TODO: Add commit/expect parsers to formulate descriptive errors diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp index 4dc94ff393540..6330d022e63c4 100644 --- a/common/peg-parser.cpp +++ b/common/peg-parser.cpp @@ -314,7 +314,7 @@ struct parser_executor { auto pos = start_pos; for (auto i = 0u; i < p.literal.size(); ++i) { if (pos >= ctx.input.size()) { - if (ctx.input_is_complete) { + if (!ctx.is_partial) { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos); @@ -406,7 +406,7 @@ struct parser_executor { // Check if we got enough matches if (p.min_count > 0 && match_count < p.min_count) { - if (pos >= ctx.input.size() && !ctx.input_is_complete) { + if (pos >= ctx.input.size() && ctx.is_partial) { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos, std::move(nodes)); } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos, pos); @@ -443,7 +443,7 @@ struct parser_executor { auto result = parse_utf8_codepoint(ctx.input, start_pos); if (result.status == utf8_parse_result::INCOMPLETE) { - if (ctx.input_is_complete) { + if (!ctx.is_partial) { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos); @@ -482,7 +482,7 @@ struct parser_executor { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos); } // Not enough matches yet - if (ctx.input_is_complete) { + if (!ctx.is_partial) { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos); @@ -523,7 +523,7 @@ struct parser_executor { // Check if we got enough matches if (match_count < p.min_count) { - if (pos >= ctx.input.size() && !ctx.input_is_complete) { + if (pos >= ctx.input.size() && ctx.is_partial) { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos); } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos, pos); @@ -535,7 +535,7 @@ struct parser_executor { static common_peg_parse_result handle_escape_sequence(common_peg_parse_context & ctx, size_t start, size_t & pos) { ++pos; // consume '\' if (pos >= ctx.input.size()) { - if (ctx.input_is_complete) { + if (!ctx.is_partial) { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start); } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start, pos); @@ -564,7 +564,7 @@ struct parser_executor { ++pos; // consume 'u' for (int i = 0; i < 4; ++i) { if (pos >= ctx.input.size()) { - if (ctx.input_is_complete) { + if (!ctx.is_partial) { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start); } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start, pos); @@ -598,7 +598,7 @@ struct parser_executor { auto utf8_result = parse_utf8_codepoint(ctx.input, pos); if (utf8_result.status == utf8_parse_result::INCOMPLETE) { - if (ctx.input_is_complete) { + if (!ctx.is_partial) { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos); @@ -613,7 +613,7 @@ struct parser_executor { } // Reached end without finding closing quote - if (ctx.input_is_complete) { + if (!ctx.is_partial) { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos, pos); } return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos); @@ -631,7 +631,7 @@ struct parser_executor { if (utf8_result.status == utf8_parse_result::INCOMPLETE) { // Incomplete UTF-8 sequence - if (ctx.input_is_complete) { + if (!ctx.is_partial) { // Input is complete but UTF-8 is incomplete = malformed return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos); } diff --git a/common/peg-parser.h b/common/peg-parser.h index 43778aa4f12c9..3eaaed10b2663 100644 --- a/common/peg-parser.h +++ b/common/peg-parser.h @@ -137,19 +137,19 @@ struct common_peg_parse_result { struct common_peg_parse_context { std::string input; - bool input_is_complete; + bool is_partial; common_peg_ast_arena ast; int parse_depth; common_peg_parse_context() - : input_is_complete(true), parse_depth(0) {} + : is_partial(false), parse_depth(0) {} common_peg_parse_context(const std::string & input) - : input(input), input_is_complete(true), parse_depth(0) {} + : input(input), is_partial(false), parse_depth(0) {} - common_peg_parse_context(const std::string & input, bool complete) - : input(input), input_is_complete(complete), parse_depth(0) {} + common_peg_parse_context(const std::string & input, bool is_partial) + : input(input), is_partial(is_partial), parse_depth(0) {} }; class common_peg_arena; diff --git a/tests/peg-parser/test-basic.cpp b/tests/peg-parser/test-basic.cpp index 4195081f3c80e..1bda6f2e6906d 100644 --- a/tests/peg-parser/test-basic.cpp +++ b/tests/peg-parser/test-basic.cpp @@ -119,7 +119,7 @@ void test_basic(testing & t) { return p.literal("hello") + p.optional(p.literal(" world")); }); - auto ctx = common_peg_parse_context("hello", true); + auto ctx = common_peg_parse_context("hello", false); auto result = parser.parse(ctx); t.assert_equal("optional_absent", true, result.success()); t.assert_equal("optional_absent_end", 5u, result.end); @@ -131,7 +131,7 @@ void test_basic(testing & t) { return p.literal("hello") + p.optional(p.literal(" world")); }); - auto ctx = common_peg_parse_context("hello ", false); + auto ctx = common_peg_parse_context("hello ", true); auto result = parser.parse(ctx); t.assert_equal("partial_match_need_more", true, result.need_more_input()); }); @@ -214,7 +214,7 @@ void test_basic(testing & t) { t.test("sequence_partial_match_1", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("") + p.literal(""); }); - auto ctx = common_peg_parse_context("") + p.literal(""); }); - auto ctx = common_peg_parse_context("") + p.literal(""); }); - auto ctx = common_peg_parse_context("I am common_chat_combinator_parser", false); + auto ctx = common_peg_parse_context("I am common_chat_combinator_parser", true); auto result = parser.parse(ctx); t.assert_equal("sequence_no_match", true, result.fail()); }); @@ -259,7 +259,7 @@ void test_basic(testing & t) { t.test("choices_partial_match_1", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("option1") | p.literal("option2"); }); - auto ctx = common_peg_parse_context("opt", false); + auto ctx = common_peg_parse_context("opt", true); auto result = parser.parse(ctx); t.assert_equal("choices_partial_match_1", true, result.need_more_input()); }); @@ -269,7 +269,7 @@ void test_basic(testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("choice_a") | p.literal("choice_b"); }); - auto ctx = common_peg_parse_context("choice", false); + auto ctx = common_peg_parse_context("choice", true); auto result = parser.parse(ctx); t.assert_equal("choices_partial_match_2", true, result.need_more_input()); }); @@ -278,7 +278,7 @@ void test_basic(testing & t) { t.test("choices_full_match_1", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("first") | p.literal("second"); }); - auto ctx = common_peg_parse_context("first", true); + auto ctx = common_peg_parse_context("first", false); auto result = parser.parse(ctx); t.assert_equal("choices_full_match_1", true, result.success()); }); @@ -287,7 +287,7 @@ void test_basic(testing & t) { t.test("choices_full_match_2", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("alpha") | p.literal("beta"); }); - auto ctx = common_peg_parse_context("beta", true); + auto ctx = common_peg_parse_context("beta", false); auto result = parser.parse(ctx); t.assert_equal("choices_full_match_2", true, result.success()); }); @@ -296,7 +296,7 @@ void test_basic(testing & t) { t.test("choices_no_match", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("good") | p.literal("better"); }); - auto ctx = common_peg_parse_context("best", true); + auto ctx = common_peg_parse_context("best", false); auto result = parser.parse(ctx); t.assert_equal("choices_no_match", true, result.fail()); }); @@ -305,7 +305,7 @@ void test_basic(testing & t) { t.test("zero_or_more_partial_match_1", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.literal("ab")); }); - auto ctx = common_peg_parse_context("a", false); + auto ctx = common_peg_parse_context("a", true); auto result = parser.parse(ctx); t.assert_equal("zero_or_more_partial_match_1", true, result.need_more_input()); }); @@ -314,7 +314,7 @@ void test_basic(testing & t) { t.test("zero_or_more_partial_match_2", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.literal("xy")); }); - auto ctx = common_peg_parse_context("xyx", false); + auto ctx = common_peg_parse_context("xyx", true); auto result = parser.parse(ctx); t.assert_equal("zero_or_more_partial_match_2", true, result.need_more_input()); }); @@ -323,7 +323,7 @@ void test_basic(testing & t) { t.test("zero_or_more_full_match", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.literal("test")); }); - auto ctx = common_peg_parse_context("test", true); + auto ctx = common_peg_parse_context("test", false); auto result = parser.parse(ctx); t.assert_equal("zero_or_more_full_match", true, result.success()); }); @@ -332,7 +332,7 @@ void test_basic(testing & t) { t.test("one_or_more_partial_match_1", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("repeat")); }); - auto ctx = common_peg_parse_context("rep", false); + auto ctx = common_peg_parse_context("rep", true); auto result = parser.parse(ctx); t.assert_equal("one_or_more_partial_match_1", true, result.need_more_input()); }); @@ -341,7 +341,7 @@ void test_basic(testing & t) { t.test("one_or_more_partial_match_2", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("ab")); }); - auto ctx = common_peg_parse_context("aba", false); + auto ctx = common_peg_parse_context("aba", true); auto result = parser.parse(ctx); t.assert_equal("one_or_more_partial_match_2", true, result.need_more_input()); }); @@ -350,7 +350,7 @@ void test_basic(testing & t) { t.test("one_or_more_full_match", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("single")); }); - auto ctx = common_peg_parse_context("single", true); + auto ctx = common_peg_parse_context("single", false); auto result = parser.parse(ctx); t.assert_equal("one_or_more_full_match", true, result.success()); }); @@ -359,7 +359,7 @@ void test_basic(testing & t) { t.test("one_or_more_no_match", [&](testing & t) { auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("()")); }); - auto ctx = common_peg_parse_context("success", true); + auto ctx = common_peg_parse_context("success", false); auto result = parser.parse(ctx); t.assert_equal("one_or_more_no_match", true, result.fail()); }); @@ -375,7 +375,7 @@ void test_basic(testing & t) { return p.rule("value", p.ref("number") | p.ref("list")); }); - common_peg_parse_context ctx("1", true); + common_peg_parse_context ctx("1", false); auto result = value_parser.parse(ctx); t.assert_equal("result_is_success", true, result.success()); @@ -389,7 +389,7 @@ void test_basic(testing & t) { return p.rule("value", p.ref("number") | p.ref("list")); }); - common_peg_parse_context ctx("[1]", true); + common_peg_parse_context ctx("[1]", false); auto result = value_parser.parse(ctx); t.assert_equal("result_is_success", true, result.success()); @@ -403,7 +403,7 @@ void test_basic(testing & t) { return p.rule("value", p.ref("number") | p.ref("list")); }); - common_peg_parse_context ctx("[[2]]", true); + common_peg_parse_context ctx("[[2]]", false); auto result = value_parser.parse(ctx); t.assert_equal("result_is_success", true, result.success()); @@ -417,7 +417,7 @@ void test_basic(testing & t) { return p.rule("value", p.ref("number") | p.ref("list")); }); - common_peg_parse_context ctx("[[[3]]]", true); + common_peg_parse_context ctx("[[[3]]]", false); auto result = value_parser.parse(ctx); t.assert_equal("result_is_success", true, result.success()); @@ -431,7 +431,7 @@ void test_basic(testing & t) { return p.rule("value", p.ref("number") | p.ref("list")); }); - common_peg_parse_context ctx("[[", false); + common_peg_parse_context ctx("[[", true); auto result = value_parser.parse(ctx); t.assert_equal("result_is_need_more_input", true, result.need_more_input()); @@ -445,7 +445,7 @@ void test_basic(testing & t) { return p.rule("value", p.ref("number") | p.ref("list")); }); - common_peg_parse_context ctx("[a]", true); + common_peg_parse_context ctx("[a]", false); auto result = value_parser.parse(ctx); t.assert_equal("result_is_fail", true, result.fail()); diff --git a/tests/peg-parser/test-json-parser.cpp b/tests/peg-parser/test-json-parser.cpp index 096bcde90ddc1..61ec4d5c27488 100644 --- a/tests/peg-parser/test-json-parser.cpp +++ b/tests/peg-parser/test-json-parser.cpp @@ -46,7 +46,7 @@ void test_json_parser(testing &t) { auto json = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); }); std::string input = R"({"name": "test", "value": )"; - common_peg_parse_context ctx(input, false); + common_peg_parse_context ctx(input, true); auto result = json.parse(ctx); @@ -58,7 +58,7 @@ void test_json_parser(testing &t) { auto json = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); }); std::string input = R"([1, 2, 3, )"; - common_peg_parse_context ctx(input, false); + common_peg_parse_context ctx(input, true); auto result = json.parse(ctx); @@ -70,7 +70,7 @@ void test_json_parser(testing &t) { auto json = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); }); std::string input = R"({"data": {"nested": )"; - common_peg_parse_context ctx(input, false); + common_peg_parse_context ctx(input, true); auto result = json.parse(ctx); diff --git a/tests/peg-parser/test-unicode.cpp b/tests/peg-parser/test-unicode.cpp index e3ec34e5152f4..83591951eed5f 100644 --- a/tests/peg-parser/test-unicode.cpp +++ b/tests/peg-parser/test-unicode.cpp @@ -59,7 +59,7 @@ void test_unicode(testing &t) { std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input); t.test(test_name, [&](testing &t) { - common_peg_parse_context ctx(tc.input, false); + common_peg_parse_context ctx(tc.input, true); auto result = parser.parse(ctx); // Assert result type matches @@ -102,7 +102,7 @@ void test_unicode(testing &t) { std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input); t.test(test_name, [&](testing &t) { - common_peg_parse_context ctx(tc.input, false); + common_peg_parse_context ctx(tc.input, true); auto result = parser.parse(ctx); // Assert result type matches @@ -143,7 +143,7 @@ void test_unicode(testing &t) { std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input); t.test(test_name, [&](testing &t) { - common_peg_parse_context ctx(tc.input, false); + common_peg_parse_context ctx(tc.input, true); auto result = parser.parse(ctx); // Assert result type matches @@ -188,7 +188,7 @@ void test_unicode(testing &t) { std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input); t.test(test_name, [&](testing &t) { - common_peg_parse_context ctx(tc.input, false); + common_peg_parse_context ctx(tc.input, true); auto result = parser.parse(ctx); // Assert result type matches @@ -226,7 +226,7 @@ void test_unicode(testing &t) { std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input); t.test(test_name, [&](testing &t) { - common_peg_parse_context ctx(tc.input, true); + common_peg_parse_context ctx(tc.input, false); auto result = parser.parse(ctx); assert_result_equal(t, tc.expected_result, result.type); @@ -260,7 +260,7 @@ void test_unicode(testing &t) { std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input); t.test(test_name, [&](testing &t) { - common_peg_parse_context ctx(tc.input, false); // input_is_complete = false + common_peg_parse_context ctx(tc.input, true); auto result = parser.parse(ctx); assert_result_equal(t, tc.expected_result, result.type); @@ -294,7 +294,7 @@ void test_unicode(testing &t) { std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input); t.test(test_name, [&](testing &t) { - common_peg_parse_context ctx(tc.input, true); + common_peg_parse_context ctx(tc.input, false); auto result = parser.parse(ctx); assert_result_equal(t, tc.expected_result, result.type); @@ -331,7 +331,7 @@ void test_unicode(testing &t) { return p.sequence({p.json_string_content(), p.literal("\"")}); }); - common_peg_parse_context ctx(tc.input, true); + common_peg_parse_context ctx(tc.input, false); auto result = parser.parse(ctx); assert_result_equal(t, tc.expected_result, result.type); @@ -368,7 +368,7 @@ void test_unicode(testing &t) { return p.json_string_content(); }); - common_peg_parse_context ctx(tc.input, false); // input_is_complete = false + common_peg_parse_context ctx(tc.input, true); auto result = parser.parse(ctx); assert_result_equal(t, tc.expected_result, result.type); @@ -405,7 +405,7 @@ void test_unicode(testing &t) { return p.json_string_content(); }); - common_peg_parse_context ctx(tc.input, true); + common_peg_parse_context ctx(tc.input, false); auto result = parser.parse(ctx); assert_result_equal(t, tc.expected_result, result.type); @@ -434,7 +434,7 @@ void test_unicode(testing &t) { return p.sequence({p.json_string_content(), p.literal("\"")}); }); - common_peg_parse_context ctx(tc.input, true); + common_peg_parse_context ctx(tc.input, false); auto result = parser.parse(ctx); assert_result_equal(t, tc.expected_result, result.type); diff --git a/tests/test-chat-peg-parser.cpp b/tests/test-chat-peg-parser.cpp index 340a3fec61faf..99f2b71d27866 100644 --- a/tests/test-chat-peg-parser.cpp +++ b/tests/test-chat-peg-parser.cpp @@ -258,7 +258,7 @@ static void test_example_qwen3_coder(testing & t) { for (auto it = tokens.begin(); it != tokens.end(); it++) { std::string in = std::accumulate(tokens.begin(), it + 1, std::string()); - common_peg_parse_context ctx(in, it == tokens.end() - 1); + common_peg_parse_context ctx(in, it + 1 < tokens.end()); auto result = parser.parse(ctx); if (!t.assert_equal("not fail", false, result.fail())) { @@ -319,8 +319,8 @@ void test_command7_parser_compare(testing & t) { return p.optional(thinking) << (tool_calls | response) + p.end(); }); - auto test_current = [&](const common_peg_arena & p, const std::string & input, bool need_more_input, bool print_results) { - common_peg_parse_context ctx(input, !need_more_input); + auto test_current = [&](const common_peg_arena & p, const std::string & input, bool is_partial, bool print_results) { + common_peg_parse_context ctx(input, is_partial); auto result = p.parse(ctx); common_chat_msg msg; From 3c57fab0473a131196587c018cc9e02f026a2bcc Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 23 Nov 2025 05:16:29 -0600 Subject: [PATCH 163/183] simplify json rules --- common/peg-parser.cpp | 35 ++++++++++++++--------------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp index 6330d022e63c4..68921dc817ead 100644 --- a/common/peg-parser.cpp +++ b/common/peg-parser.cpp @@ -1058,40 +1058,36 @@ common_peg_arena common_peg_parser_builder::build() { // JSON parsers common_peg_parser common_peg_parser_builder::json_number() { - std::function builder = [this]() { + return rule("json-number", [this]() { auto digit1_9 = chars("[1-9]", 1, 1); auto digits = chars("[0-9]"); auto int_part = choice({literal("0"), sequence({digit1_9, chars("[0-9]", 0, -1)})}); auto frac = sequence({literal("."), digits}); auto exp = sequence({choice({literal("e"), literal("E")}), optional(chars("[+-]", 1, 1)), digits}); return sequence({optional(literal("-")), int_part, optional(frac), optional(exp)}); - }; - return rule("json-number", builder); + }); } common_peg_parser common_peg_parser_builder::json_string() { - std::function builder = [this]() { + return rule("json-string", [this]() { return sequence({literal("\""), json_string_content(), literal("\"")}); - }; - return rule("json-string", builder); + }); } common_peg_parser common_peg_parser_builder::json_bool() { - std::function builder = [this]() { + return rule("json-bool", [this]() { return choice({literal("true"), literal("false")}); - }; - return rule("json-bool", builder); + }); } common_peg_parser common_peg_parser_builder::json_null() { - std::function builder = [this]() { + return rule("json-null", [this]() { return literal("null"); - }; - return rule("json-null", builder); + }); } common_peg_parser common_peg_parser_builder::json_object() { - std::function builder = [this]() { + return rule("json-object", [this]() { auto ws = space(); auto member = sequence({json_string(), ws, literal(":"), ws, json()}); auto members = sequence({member, zero_or_more(sequence({ws, literal(","), ws, member}))}); @@ -1099,24 +1095,22 @@ common_peg_parser common_peg_parser_builder::json_object() { sequence({literal("{"), ws, literal("}")}), sequence({literal("{"), ws, members, ws, literal("}")}) }); - }; - return rule("json-object", builder); + }); } common_peg_parser common_peg_parser_builder::json_array() { - std::function builder = [this]() { + return rule("json-array", [this]() { auto ws = space(); auto elements = sequence({json(), zero_or_more(sequence({ws, literal(","), ws, json()}))}); return choice({ sequence({literal("["), ws, literal("]")}), sequence({literal("["), ws, elements, ws, literal("]")}) }); - }; - return rule("json-array", builder); + }); } common_peg_parser common_peg_parser_builder::json() { - std::function builder = [this]() { + return rule("json-value", [this]() { return choice({ json_object(), json_array(), @@ -1125,8 +1119,7 @@ common_peg_parser common_peg_parser_builder::json() { json_bool(), json_null() }); - }; - return rule("json-value", builder); + }); } From 1ff18945af303015861fc6fa64745eaecec47ca9 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 23 Nov 2025 05:17:59 -0600 Subject: [PATCH 164/183] remove extraneous file --- tests/peg-parser/convo.json | 159 ------------------------------------ 1 file changed, 159 deletions(-) delete mode 100644 tests/peg-parser/convo.json diff --git a/tests/peg-parser/convo.json b/tests/peg-parser/convo.json deleted file mode 100644 index d165fdb1a9ebd..0000000000000 --- a/tests/peg-parser/convo.json +++ /dev/null @@ -1,159 +0,0 @@ -{ - "model": "testing", - "messages": [ - { - "role": "system", - "content": "You are an assistant that can retrieve information using tools and provide concise, friendly summaries." - }, - { - "role": "user", - "content": "Please give me a summary of the latest news about renewable energy, the current weather in Berlin, and a random joke." - }, - - { - "role": "assistant", - "reasoning_content": "I need to fetch recent news about renewable energy. I'll call the `fetch_news` tool first." - }, - - { - "role": "assistant", - "content": null, - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "fetch_news", - "arguments": {"query":"renewable energy","language":"en","region":"global","max_results":5} - } - } - ] - }, - - { - "role": "assistant", - "reasoning_content": "Next I need the current weather for Berlin. I'll call the `get_weather` tool." - }, - - { - "role": "assistant", - "content": null, - "tool_calls": [ - { - "id": "call_2", - "type": "function", - "function": { - "name": "get_weather", - "arguments":{"location":"Berlin","units":"metric"} - } - } - ] - }, - - { - "role": "user", - "content": "Thanks! Also, can you give me a fun fact about cats?" - }, - - { - "role": "assistant", - "reasoning_content": "To keep the reply light I’ll fetch a random joke using the `generate_joke` tool." - }, - - { - "role": "assistant", - "content": null, - "tool_calls": [ - { - "id": "call_3", - "type": "function", - "function": { - "name": "generate_joke", - "arguments": {"category":"funny"} - } - } - ] - }, - - { - "role": "assistant", - "content": "**Renewable Energy News Summary**\n- *[Placeholder for news summary from `fetch_news`]*\n\n**Current Weather in Berlin**\n- *[Placeholder for weather data from `get_weather`]*\n\n**Joke**\n- *[Placeholder for joke from `generate_joke`]*\n\nI used three tool calls (β€―`fetch_news`,β€―`get_weather`,β€―`generate_joke`β€―) to gather this information." - } - ], - - "tools": [ - { - "type": "function", - "function": { - "name": "fetch_news", - "description": "Retrieve recent news articles based on a query.", - "parameters": { - "type": "object", - "properties": { - "query": { - "type": "string", - "description": "Search term (e.g., \"renewable energy\")." - }, - "language": { - "type": "string", - "description": "Two‑letter language code, e.g., \"en\"." - }, - "region": { - "type": "string", - "description": "Region or country code, e.g., \"global\"." - }, - "max_results": { - "type": "integer", - "description": "Maximum number of articles to return." - }, - "date": { - "type": "string", - "format": "date", - "description": "Specific date for news in ISO format (optional)." - } - }, - "required": ["query", "language", "region", "max_results"] - } - } - }, - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City name or geographic coordinates." - }, - "units": { - "type": "string", - "enum": ["metric", "imperial"], - "description": "Units for temperature." - } - }, - "required": ["location", "units"] - } - } - }, - { - "type": "function", - "function": { - "name": "generate_joke", - "description": "Generate a random joke.", - "parameters": { - "type": "object", - "properties": { - "category": { - "type": "string", - "description": "Optional joke category (e.g., \"funny\", \"dad\")." - } - }, - "required": [] - } - } - } - ] -} From a5e7a1c62864cb9bca825853d55e0f91d5ed26d1 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 23 Nov 2025 05:32:58 -0600 Subject: [PATCH 165/183] remove comment --- common/chat-peg-parser.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 53926e1052e8d..b79e7969c8dda 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -18,12 +18,9 @@ void common_chat_peg_mapper::from_ast(const common_peg_ast_arena & arena, const } void common_chat_peg_mapper::map(const common_peg_ast_node & node) { - bool is_reasoning_block = node.tag == common_chat_peg_builder::REASONING_BLOCK; bool is_reasoning = node.tag == common_chat_peg_builder::REASONING; bool is_content = node.tag == common_chat_peg_builder::CONTENT; - // TODO: Handle reasoning_format - if (is_reasoning) { result.reasoning_content = std::string(trim_trailing_space(node.text)); } From 449d9544b5e85ff4e7f865d6f5cac6e62d92d073 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 23 Nov 2025 05:43:27 -0600 Subject: [PATCH 166/183] implement += and |= operators --- common/chat.cpp | 17 ++++++++--------- common/peg-parser.cpp | 10 ++++++++++ common/peg-parser.h | 4 ++++ 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 56cd7d7854650..1a03b516c1f79 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1917,12 +1917,12 @@ static common_chat_params common_chat_params_init_qwen3_coder_xml(const common_c "\n\n" })); - std::vector tool_parsers; + auto tools = p.choice(); foreach_function(params.tools, [&](const json & tool) { const auto & function = tool.at("function"); std::string fn_name = function.at("name"); - std::vector argument_parsers; + auto args = p.sequence(); foreach_parameter(function, [&](const std::string & name, const json & schema, bool is_required) { auto arg_value = p.eps(); if (schema.contains("type") && schema.at("type") == "string") { @@ -1950,24 +1950,23 @@ static common_chat_params common_chat_params_init_qwen3_coder_xml(const common_c ); auto arg_rule = p.rule("tool-" + fn_name + "-arg-" + name, arg); - argument_parsers.push_back(p.repeat(arg_rule, (is_required ? 1 : 0), 1)); + args += p.repeat(arg_rule, (is_required ? 1 : 0), 1); }); - tool_parsers.push_back(p.rule("tool-" + fn_name, + tools |= p.rule("tool-" + fn_name, p.tool_open("") - << p.sequence(argument_parsers) - << p.tool_close(p.literal("")) - )); + << args + << p.tool_close(p.literal(""))); }); auto tool_call = p.trigger_rule("tool-call", p.optional("" + p.space()) - + p.choice(tool_parsers) + + tools + p.space() + "" // We have to handle parallel tool calls here because it is a trigger rule + (params.parallel_tool_calls ? - p.repeat(p.space() + "" << p.choice(tool_parsers) << "", 0, -1) : + p.repeat(p.space() + "" << tools << "", 0, -1) : p.eps()) ); diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp index 68921dc817ead..2d8378a8981f2 100644 --- a/common/peg-parser.cpp +++ b/common/peg-parser.cpp @@ -871,6 +871,16 @@ common_peg_parser & common_peg_parser::operator=(common_peg_parser const & other return *this; } +common_peg_parser & common_peg_parser::operator+=(common_peg_parser const & other) { + id_ = builder_.sequence({id_, other.id_}); + return *this; +} + +common_peg_parser & common_peg_parser::operator|=(common_peg_parser const & other) { + id_ = builder_.choice({id_, other.id_}); + return *this; +} + common_peg_parser common_peg_parser::operator+(const common_peg_parser & other) const { return builder_.sequence({id_, other.id_}); } diff --git a/common/peg-parser.h b/common/peg-parser.h index 3eaaed10b2663..ecfd87bd2b157 100644 --- a/common/peg-parser.h +++ b/common/peg-parser.h @@ -31,6 +31,8 @@ class common_peg_parser { common_peg_parser(common_peg_parser_id id, common_peg_parser_builder & builder) : id_(id), builder_(builder) {} common_peg_parser & operator=(common_peg_parser const & other); + common_peg_parser & operator+=(common_peg_parser const & other); + common_peg_parser & operator|=(common_peg_parser const & other); operator common_peg_parser_id() const { return id_; } common_peg_parser_id id() const { return id_; } @@ -333,12 +335,14 @@ class common_peg_parser_builder { // Matches a sequence of parsers in order, all must succeed. // S -> A B C + common_peg_parser sequence() { return add(common_peg_sequence_parser{}); } common_peg_parser sequence(const std::vector & parsers); common_peg_parser sequence(const std::vector & parsers); common_peg_parser sequence(std::initializer_list parsers); // Matches the first parser that succeeds from a list of alternatives. // S -> A | B | C + common_peg_parser choice() { return add(common_peg_choice_parser{}); } common_peg_parser choice(const std::vector & parsers); common_peg_parser choice(const std::vector & parsers); common_peg_parser choice(std::initializer_list parsers); From 39b8213a012fca71fa38a1b1e4fa88a89aa85483 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 23 Nov 2025 05:49:10 -0600 Subject: [PATCH 167/183] add comments to qwen3 implementation --- common/chat.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 1a03b516c1f79..ae30df484be80 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1960,13 +1960,16 @@ static common_chat_params common_chat_params_init_qwen3_coder_xml(const common_c }); auto tool_call = p.trigger_rule("tool-call", + // Qwen3-Coder may emit or optional but required in parallel calls p.optional("" + p.space()) + tools + p.space() + "" - // We have to handle parallel tool calls here because it is a trigger rule + // It seems more intuitive to place parallel calls as a repetition in the root rule, but + // it is here because it needs to be wrapped in a trigger rule. + (params.parallel_tool_calls ? - p.repeat(p.space() + "" << tools << "", 0, -1) : + p.zero_or_more(p.space() + "" << tools << "") : p.eps()) ); From 3fdfa7cb3f8da45c8ad7e1ab04c43d1c9220d85a Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 23 Nov 2025 05:56:29 -0600 Subject: [PATCH 168/183] reorder arguments to common_chat_peg_parse --- common/chat.cpp | 2 +- common/chat.h | 2 +- tools/server/server.cpp | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index ae30df484be80..c90d757e28722 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -3613,7 +3613,7 @@ common_chat_msg common_chat_parse(const std::string & input, bool is_partial, co return msg; } -common_chat_msg common_chat_peg_parse(const std::string & input, bool is_partial, const common_peg_arena & parser, const common_chat_syntax & syntax) { +common_chat_msg common_chat_peg_parse(const common_peg_arena & parser, const std::string & input, bool is_partial, const common_chat_syntax & syntax) { LOG_DBG("Parsing input with format %s: %s\n", common_chat_format_name(syntax.format), input.c_str()); common_peg_parse_context ctx(input, is_partial); diff --git a/common/chat.h b/common/chat.h index 89fccde6960e7..714122ce78229 100644 --- a/common/chat.h +++ b/common/chat.h @@ -213,7 +213,7 @@ const char* common_chat_format_name(common_chat_format format); const char* common_reasoning_format_name(common_reasoning_format format); common_reasoning_format common_reasoning_format_from_name(const std::string & format); common_chat_msg common_chat_parse(const std::string & input, bool is_partial, const common_chat_syntax & syntax); -common_chat_msg common_chat_peg_parse(const std::string & input, bool is_partial, const common_peg_arena & parser, const common_chat_syntax & syntax); +common_chat_msg common_chat_peg_parse(const common_peg_arena & parser, const std::string & input, bool is_partial, const common_chat_syntax & syntax); common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::string & tool_choice); diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 7d4a2e4ce24ea..708b5847285d5 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -1869,15 +1869,15 @@ struct server_slot { auto previous_msg = chat_msg; SRV_DBG("Parsing chat message: %s\n", generated_text.c_str()); - auto new_msg = task->params.oaicompat_chat_parser.empty() ? - common_chat_parse( + auto new_msg = !task->params.oaicompat_chat_parser.empty() ? + common_chat_peg_parse( + task->params.oaicompat_chat_parser, generated_text, /* is_partial= */ stop != STOP_TYPE_EOS, task->params.oaicompat_chat_syntax) : - common_chat_peg_parse( + common_chat_parse( generated_text, /* is_partial= */ stop != STOP_TYPE_EOS, - task->params.oaicompat_chat_parser, task->params.oaicompat_chat_syntax); if (!new_msg.empty()) { From 2ab9959eab2eed3452a82fb2458dbadf321c7a23 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 23 Nov 2025 06:02:17 -0600 Subject: [PATCH 169/183] remove commented outdated tests --- tests/test-peg-parser.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test-peg-parser.cpp b/tests/test-peg-parser.cpp index 20975eb25dfae..53d324234227a 100644 --- a/tests/test-peg-parser.cpp +++ b/tests/test-peg-parser.cpp @@ -14,10 +14,6 @@ int main(int argc, char *argv[]) { t.test("json", test_json_parser); t.test("gbnf", test_gbnf_generation); t.test("serialization", test_json_serialization); - //t.test("qwen3_coder", test_example_qwen3_coder); - //t.test("seed_oss", test_example_seed_oss); - //t.test("minimax_m2", test_example_minimax_m2); - //t.test("command7_parser_compare", test_command7_parser_compare); return t.summary(); } From 2a8bda389cf405293412b893fbfe0620416f5296 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 23 Nov 2025 06:12:35 -0600 Subject: [PATCH 170/183] add explicit copy constructor --- common/peg-parser.h | 1 + 1 file changed, 1 insertion(+) diff --git a/common/peg-parser.h b/common/peg-parser.h index ecfd87bd2b157..527e907158c5a 100644 --- a/common/peg-parser.h +++ b/common/peg-parser.h @@ -28,6 +28,7 @@ class common_peg_parser { common_peg_parser_builder & builder_; public: + common_peg_parser(common_peg_parser & other) : id_(other.id_), builder_(other.builder_) {} common_peg_parser(common_peg_parser_id id, common_peg_parser_builder & builder) : id_(id), builder_(builder) {} common_peg_parser & operator=(common_peg_parser const & other); From 3ff149e7383da72286a19f4537a6ab6b001e588e Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 23 Nov 2025 06:15:10 -0600 Subject: [PATCH 171/183] fix operators and constness --- common/peg-parser.cpp | 6 +++--- common/peg-parser.h | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp index 2d8378a8981f2..7ec4ac0e099a9 100644 --- a/common/peg-parser.cpp +++ b/common/peg-parser.cpp @@ -866,17 +866,17 @@ std::string common_peg_arena::dump(common_peg_parser_id id) const { } // Parser wrapper operator implementations -common_peg_parser & common_peg_parser::operator=(common_peg_parser const & other) { +common_peg_parser & common_peg_parser::operator=(const common_peg_parser & other) { id_ = other.id_; return *this; } -common_peg_parser & common_peg_parser::operator+=(common_peg_parser const & other) { +common_peg_parser & common_peg_parser::operator+=(const common_peg_parser & other) { id_ = builder_.sequence({id_, other.id_}); return *this; } -common_peg_parser & common_peg_parser::operator|=(common_peg_parser const & other) { +common_peg_parser & common_peg_parser::operator|=(const common_peg_parser & other) { id_ = builder_.choice({id_, other.id_}); return *this; } diff --git a/common/peg-parser.h b/common/peg-parser.h index 527e907158c5a..fcba45dfec719 100644 --- a/common/peg-parser.h +++ b/common/peg-parser.h @@ -28,12 +28,12 @@ class common_peg_parser { common_peg_parser_builder & builder_; public: - common_peg_parser(common_peg_parser & other) : id_(other.id_), builder_(other.builder_) {} + common_peg_parser(const common_peg_parser & other) : id_(other.id_), builder_(other.builder_) {} common_peg_parser(common_peg_parser_id id, common_peg_parser_builder & builder) : id_(id), builder_(builder) {} - common_peg_parser & operator=(common_peg_parser const & other); - common_peg_parser & operator+=(common_peg_parser const & other); - common_peg_parser & operator|=(common_peg_parser const & other); + common_peg_parser & operator=(const common_peg_parser & other); + common_peg_parser & operator+=(const common_peg_parser & other); + common_peg_parser & operator|=(const common_peg_parser & other); operator common_peg_parser_id() const { return id_; } common_peg_parser_id id() const { return id_; } From 5dc07cef338cdbdaf84ac387cbc45cf24c5b25b8 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 23 Nov 2025 17:24:52 -0600 Subject: [PATCH 172/183] wip: update test-chat for qwen3-coder --- common/chat.cpp | 4 +- common/peg-parser.cpp | 14 ++-- tests/test-chat.cpp | 155 +++++++++++++++++++++++++++--------------- 3 files changed, 112 insertions(+), 61 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index c90d757e28722..75ae65b94779f 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1925,7 +1925,7 @@ static common_chat_params common_chat_params_init_qwen3_coder_xml(const common_c auto args = p.sequence(); foreach_parameter(function, [&](const std::string & name, const json & schema, bool is_required) { auto arg_value = p.eps(); - if (schema.contains("type") && schema.at("type") == "string") { + if (schema.contains("type") && schema.at("type").is_string() && schema.at("type") == "string") { arg_value = p.tool_arg_string_value(p.schema( until_end_of_param, /* name = */ "tool-" + fn_name + "-arg-" + name + "-schema", @@ -3624,6 +3624,8 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & parser, const std } common_chat_msg msg; + msg.role = "assistant"; + if (syntax.format == COMMON_CHAT_FORMAT_PEG_NATIVE) { auto mapper = common_chat_peg_native_mapper(msg); mapper.from_ast(ctx.ast, result); diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp index 7ec4ac0e099a9..dba928eb7c050 100644 --- a/common/peg-parser.cpp +++ b/common/peg-parser.cpp @@ -1342,22 +1342,22 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo return gbnf_excluding_pattern(p.delimiters); } else if constexpr (std::is_same_v) { if (p.schema) { - auto type = p.schema->value("type","object"); - if (p.raw && type == "string") { - if (p.schema->contains("pattern")) { + // TODO: Handle anyOf that could contain raw strings and JSON. For now we only + // handle raw strings to cover most cases and require JSON for the more + // sophisticated schemas. + if (p.raw && p.schema->contains("type") && p.schema->at("type").is_string() && p.schema->at("type") == "string") { + if (p.schema->contains("pattern") && p.schema->at("pattern").is_string()) { // heuristic: // if .* is in the user's provided pattern, use the child's GBNF grammar. // This is because .* will greedily match everything, past any delimiters. - auto pattern = p.schema->value("pattern", "^.*$"); + std::string pattern = p.schema->at("pattern"); if (pattern.find(".*") != std::string::npos) { return to_gbnf(p.child); } } else if (p.schema->contains("enum") || p.schema->contains("const") || p.schema->contains("minLength") || - p.schema->contains("maxLength") || - p.schema->contains("allOf") || - p.schema->contains("anyOf")) { + p.schema->contains("maxLength")) { return builder.add_string_schema(p.name, *p.schema); } return to_gbnf(p.child); diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 62dd1583fa1a5..f2c91b7af2574 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -6,6 +6,7 @@ // cmake -B build && cmake --build build --parallel && ./build/bin/test-chat ../minja/build/tests/*.jinja 2>/dev/null // #include "chat.h" +#include "peg-parser.h" #include "log.h" @@ -647,6 +648,7 @@ static void test_template_output_parsers() { inputs_tools_builtin.messages = {message_user}; inputs_tools_builtin.tools = {python_tool}; + goto qwen; { // Not supported yet auto tmpls = read_templates("models/templates/CohereForAI-c4ai-command-r-plus-tool_use.jinja"); @@ -2766,21 +2768,53 @@ Hey there!<|im_end|> ); } +qwen: // Test Qwen3-Coder XML format { + auto tmpls = read_templates("models/templates/Qwen3-Coder.jinja"); + + // We need to construct a parser with specific tools + struct make_parser { + common_chat_params params_; + common_peg_arena arena_; + bool is_partial_; + + make_parser(common_chat_templates * tmpls, bool is_partial, const std::vector & tools) { + common_chat_templates_inputs inputs; + inputs.tools = tools; + params_ = common_chat_templates_apply(tmpls, inputs); + assert_equals(COMMON_CHAT_FORMAT_PEG_CONSTRUCTED, params_.format); + arena_ = common_peg_arena::deserialize(params_.parser); + is_partial_ = is_partial; + } + + common_chat_msg operator()(const std::string & msg) { + return common_chat_peg_parse(arena_, msg, /* is_partial = */ is_partial_, /* syntax = */ {params_.format}); + } + }; + // Basic XML tool call parsing assert_msg_equals( message_assist_call, - common_chat_parse( + make_parser(tmpls.get(), /* is_partial = */ false, /* tools */ {{ + /* .name = */ "special_function", + /* .description = */ "special function", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "arg1": { "type": "number" } + }, + "required": [] + })", + }})( "\n" - " \n" - " \n" - " 1\n" - " \n" - " \n" - "", - /* is_partial= */ false, - {COMMON_CHAT_FORMAT_QWEN3_CODER_XML})); + "\n" + "\n" + "1\n" + "\n" + "\n" + "" + )); // Multiple parameters with different types common_chat_msg expected_multi_param; @@ -2791,23 +2825,37 @@ Hey there!<|im_end|> test_parser_with_streaming(expected_multi_param, "\n" - " \n" - " \n" - " John Doe\n" - " \n" - " \n" - " 30\n" - " \n" - " \n" - " true\n" - " \n" - " \n" - " 95.5\n" - " \n" - " \n" + "\n" + "\n" + "John Doe\n" + "\n" + "\n" + "30\n" + "\n" + "\n" + "true\n" + "\n" + "\n" + "95.5\n" + "\n" + "\n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_QWEN3_CODER_XML}); }); + make_parser(tmpls.get(), /* is_partial = */ true, /* tools */ {{ + /* .name = */ "complex_function", + /* .description = */ "complex function", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "number"}, + "active": {"type": "boolean"}, + "score": {"type": "number"} + } + })" + }}) + ); + goto done; // Special characters and Unicode common_chat_msg expected_special_chars; expected_special_chars.role = "assistant"; @@ -2823,7 +2871,7 @@ Hey there!<|im_end|> " \n" " \n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_QWEN3_CODER_XML}); }); + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); // Multiline content with newlines and indentation common_chat_msg expected_multiline; @@ -2842,7 +2890,7 @@ Hey there!<|im_end|> " \n" " \n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_QWEN3_CODER_XML}); }); + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); // JSON object as parameter value common_chat_msg expected_json_param; @@ -2860,7 +2908,7 @@ Hey there!<|im_end|> " \n" " \n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_QWEN3_CODER_XML}); }); + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); // Array as parameter value common_chat_msg expected_array_param; @@ -2878,7 +2926,7 @@ Hey there!<|im_end|> " \n" " \n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_QWEN3_CODER_XML}); }); + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); // Empty parameter common_chat_msg expected_empty_param; @@ -2895,7 +2943,7 @@ Hey there!<|im_end|> " \n" " \n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_QWEN3_CODER_XML}); }); + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); // Boolean values (true/false) common_chat_msg expected_boolean; @@ -2916,7 +2964,7 @@ Hey there!<|im_end|> " \n" " \n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_QWEN3_CODER_XML}); }); + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); // Null value common_chat_msg expected_null; @@ -2934,7 +2982,7 @@ Hey there!<|im_end|> " \n" " \n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_QWEN3_CODER_XML}); }); + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); // Negative numbers and scientific notation common_chat_msg expected_numbers; @@ -2958,7 +3006,7 @@ Hey there!<|im_end|> " \n" " \n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_QWEN3_CODER_XML}); }); + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); // XML-like content in parameters (should be escaped) common_chat_msg expected_xml_content; @@ -2976,7 +3024,7 @@ Hey there!<|im_end|> " \n" " \n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_QWEN3_CODER_XML}); }); + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); // Quotes and escape characters common_chat_msg expected_quotes; @@ -2994,7 +3042,7 @@ Hey there!<|im_end|> " \n" " \n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_QWEN3_CODER_XML}); }); + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); // Long parameter value (simplified) std::string long_text = "This is a long text parameter that should test the parser's ability to handle larger amounts of text data."; @@ -3014,7 +3062,7 @@ Hey there!<|im_end|> " \n" " \n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_QWEN3_CODER_XML}); }); + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); // Mixed content with text before and after tool call common_chat_msg expected_mixed_content; @@ -3033,7 +3081,7 @@ Hey there!<|im_end|> " \n" " \n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_QWEN3_CODER_XML}); }); + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); // Compact format (no extra whitespace) common_chat_msg expected_compact; @@ -3045,7 +3093,7 @@ Hey there!<|im_end|> test_parser_with_streaming( expected_compact, "value", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_QWEN3_CODER_XML}); }); + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); // Function name with underscores and numbers common_chat_msg expected_complex_name; @@ -3063,7 +3111,7 @@ Hey there!<|im_end|> " \n" " \n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_QWEN3_CODER_XML}); }); + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); // Parameter names with underscores and numbers common_chat_msg expected_complex_params; @@ -3087,7 +3135,7 @@ Hey there!<|im_end|> " \n" " \n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_QWEN3_CODER_XML}); }); + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); // Very deeply nested XML content in parameter common_chat_msg expected_deep_xml; @@ -3105,7 +3153,7 @@ Hey there!<|im_end|> " \n" " \n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_QWEN3_CODER_XML}); }); + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); // Parameter with only whitespace common_chat_msg expected_whitespace_param; @@ -3123,7 +3171,7 @@ Hey there!<|im_end|> " \n" " \n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_QWEN3_CODER_XML}); }); + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); // Parameter with tabs and mixed whitespace common_chat_msg expected_mixed_whitespace; @@ -3143,7 +3191,7 @@ Hey there!<|im_end|> " \n" " \n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_QWEN3_CODER_XML}); }); + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); // Control characters and special Unicode common_chat_msg expected_control_chars; @@ -3161,7 +3209,7 @@ Hey there!<|im_end|> " \n" " \n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_QWEN3_CODER_XML}); }); + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); // Emoji and extended Unicode characters common_chat_msg expected_emoji; @@ -3179,7 +3227,7 @@ Hey there!<|im_end|> " \n" " \n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_QWEN3_CODER_XML}); }); + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); // Mathematical expressions and formulas common_chat_msg expected_math; @@ -3197,7 +3245,7 @@ Hey there!<|im_end|> " \n" " \n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_QWEN3_CODER_XML}); }); + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); // SQL injection-like content (should be safely escaped) common_chat_msg expected_sql; @@ -3215,7 +3263,7 @@ Hey there!<|im_end|> " \n" " \n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_QWEN3_CODER_XML}); }); + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); // HTML/XML injection content common_chat_msg expected_html; @@ -3233,7 +3281,7 @@ Hey there!<|im_end|> " \n" " \n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_QWEN3_CODER_XML}); }); + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); // Binary-like content (base64) common_chat_msg expected_binary; @@ -3251,7 +3299,7 @@ Hey there!<|im_end|> " \n" " \n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_QWEN3_CODER_XML}); }); + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); // Very large numbers (should be parsed as scientific notation) common_chat_msg expected_large_numbers; @@ -3269,9 +3317,10 @@ Hey there!<|im_end|> " \n" " \n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_QWEN3_CODER_XML}); }); + [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); } +done: { // Qwen3-Coder template auto tmpls = read_templates("models/templates/Qwen3-Coder.jinja"); @@ -3294,7 +3343,7 @@ Hey there!<|im_end|> inputs.tools = { qwen_union_tool }; auto params = common_chat_templates_apply(tmpls.get(), inputs); - assert_equals(COMMON_CHAT_FORMAT_QWEN3_CODER_XML, params.format); + assert_equals(COMMON_CHAT_FORMAT_PEG_CONSTRUCTED, params.format); assert_equals(false, params.grammar.empty()); // Grammar should compile successfully @@ -3423,9 +3472,9 @@ int main(int argc, char ** argv) { } else #endif { - test_msg_diffs_compute(); - test_msgs_oaicompat_json_conversion(); - test_tools_oaicompat_json_conversion(); + //test_msg_diffs_compute(); + //test_msgs_oaicompat_json_conversion(); + //test_tools_oaicompat_json_conversion(); test_template_output_parsers(); std::cout << "\n[chat] All tests passed!" << '\n'; } From 8e7fd16b950cee8eda07f474ed450141f1795b19 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 23 Nov 2025 18:07:30 -0600 Subject: [PATCH 173/183] bring json parser closer to json-schema-to-grammar rules --- common/peg-parser.cpp | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp index dba928eb7c050..55f5450510da9 100644 --- a/common/peg-parser.cpp +++ b/common/peg-parser.cpp @@ -1074,25 +1074,25 @@ common_peg_parser common_peg_parser_builder::json_number() { auto int_part = choice({literal("0"), sequence({digit1_9, chars("[0-9]", 0, -1)})}); auto frac = sequence({literal("."), digits}); auto exp = sequence({choice({literal("e"), literal("E")}), optional(chars("[+-]", 1, 1)), digits}); - return sequence({optional(literal("-")), int_part, optional(frac), optional(exp)}); + return sequence({optional(literal("-")), int_part, optional(frac), optional(exp), space()}); }); } common_peg_parser common_peg_parser_builder::json_string() { return rule("json-string", [this]() { - return sequence({literal("\""), json_string_content(), literal("\"")}); + return sequence({literal("\""), json_string_content(), literal("\""), space()}); }); } common_peg_parser common_peg_parser_builder::json_bool() { return rule("json-bool", [this]() { - return choice({literal("true"), literal("false")}); + return sequence({choice({literal("true"), literal("false")}), space()}); }); } common_peg_parser common_peg_parser_builder::json_null() { return rule("json-null", [this]() { - return literal("null"); + return sequence({literal("null"), space()}); }); } @@ -1101,9 +1101,14 @@ common_peg_parser common_peg_parser_builder::json_object() { auto ws = space(); auto member = sequence({json_string(), ws, literal(":"), ws, json()}); auto members = sequence({member, zero_or_more(sequence({ws, literal(","), ws, member}))}); - return choice({ - sequence({literal("{"), ws, literal("}")}), - sequence({literal("{"), ws, members, ws, literal("}")}) + return sequence({ + literal("{"), + ws, + choice({ + literal("}"), + sequence({members, ws, literal("}")}) + }), + ws }); }); } @@ -1111,10 +1116,15 @@ common_peg_parser common_peg_parser_builder::json_object() { common_peg_parser common_peg_parser_builder::json_array() { return rule("json-array", [this]() { auto ws = space(); - auto elements = sequence({json(), zero_or_more(sequence({ws, literal(","), ws, json()}))}); - return choice({ - sequence({literal("["), ws, literal("]")}), - sequence({literal("["), ws, elements, ws, literal("]")}) + auto elements = sequence({json(), zero_or_more(sequence({literal(","), ws, json()}))}); + return sequence({ + literal("["), + ws, + choice({ + literal("]"), + sequence({elements, ws, literal("]")}) + }), + ws }); }); } From 74997a5c420cd5bc2d6a871fa8f43731968a3def Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 23 Nov 2025 18:07:50 -0600 Subject: [PATCH 174/183] trim trailing space for most things --- common/chat-peg-parser.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index b79e7969c8dda..3dc0d1a04b09e 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -44,15 +44,15 @@ void common_chat_peg_native_mapper::map(const common_peg_ast_node & node) { } if (is_tool_id && current_tool) { - current_tool->id = std::string(node.text); + current_tool->id = std::string(trim_trailing_space(node.text)); } if (is_tool_name && current_tool) { - current_tool->name = std::string(node.text); + current_tool->name = std::string(trim_trailing_space(node.text)); } if (is_tool_args && current_tool) { - current_tool->arguments = std::string(node.text); + current_tool->arguments = std::string(trim_trailing_space(node.text)); } } @@ -84,13 +84,13 @@ void common_chat_peg_constructed_mapper::map(const common_peg_ast_node & node) { if (arg_count > 0) { current_tool->arguments += ","; } - current_tool->arguments += json(node.text).dump() + ":"; + current_tool->arguments += json(trim_trailing_space(node.text)).dump() + ":"; ++arg_count; } if (is_arg_string && current_tool) { // Serialize to JSON, but exclude the end quote - std::string dumped = json(node.text).dump(); + std::string dumped = json(trim_trailing_space(node.text)).dump(); current_tool->arguments += dumped.substr(0, dumped.size() - 1); needs_closing_quote = true; } @@ -102,7 +102,7 @@ void common_chat_peg_constructed_mapper::map(const common_peg_ast_node & node) { } if (is_arg_json && current_tool) { - current_tool->arguments += std::string(node.text); + current_tool->arguments += std::string(trim_trailing_space(node.text)); } if (is_tool_close && current_tool) { From 17ebea5faced1036a71bd180b58114760175ea4d Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 23 Nov 2025 18:08:12 -0600 Subject: [PATCH 175/183] fix qwen3 coder rules w.r.t. trailing spaces --- common/chat.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 75ae65b94779f..362402c5ffbb0 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1913,8 +1913,8 @@ static common_chat_params common_chat_params_init_qwen3_coder_xml(const common_c auto content = p.rule("content", p.content(p.until_one_of({"", "\n\n" + "\n\n" })); auto tools = p.choice(); @@ -1941,9 +1941,9 @@ static common_chat_params common_chat_params_init_qwen3_coder_xml(const common_c } auto arg = p.tool_arg( - p.tool_arg_open("") - << arg_value - << p.tool_arg_close( + p.tool_arg_open("\n") + + arg_value + + p.tool_arg_close( "\n" + p.peek(p.literal("")) ) From 668eee31adefcb2acb9d7fadc98ab2164d76c5ef Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 23 Nov 2025 20:26:19 -0600 Subject: [PATCH 176/183] group rules --- tests/test-chat-peg-parser.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test-chat-peg-parser.cpp b/tests/test-chat-peg-parser.cpp index 99f2b71d27866..00c819cb2a4be 100644 --- a/tests/test-chat-peg-parser.cpp +++ b/tests/test-chat-peg-parser.cpp @@ -299,8 +299,8 @@ void test_command7_parser_compare(testing & t) { auto response = "<|START_RESPONSE|>" << p.content(p.until("<|END_RESPONSE|>")) << "<|END_RESPONSE|>"; - auto tool_call_id = p.atomic("\"tool_call_id\"" << (":" << "\"" + p.tool_id(p.json_string_content()) + "\"")); - auto tool_call_name = p.atomic("\"tool_name\"" << (":" << "\"" + p.tool_name(p.json_string_content()) + "\"")); + auto tool_call_id = p.atomic("\"tool_call_id\"" << (":" << ("\"" + p.tool_id(p.json_string_content()) + "\""))); + auto tool_call_name = p.atomic("\"tool_name\"" << (":" << ("\"" + p.tool_name(p.json_string_content()) + "\""))); auto tool_call_args = "\"parameters\"" << (":" << p.tool_args(p.json())); auto tool_call_fields = p.rule("tool-call-fields", tool_call_id | tool_call_name | tool_call_args); @@ -476,7 +476,7 @@ void test_command7_parser_compare(testing & t) { try { test_legacy(in, i + 1 < tokens.size(), false); - } catch (common_chat_msg_partial_exception & e) { + } catch (common_chat_msg_partial_exception & /* e */) { // Do nothing, this is expected } } From 68d71c1b9c9e76054659189c86b218408c112601 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 23 Nov 2025 20:26:53 -0600 Subject: [PATCH 177/183] do not trim trailing space from string args --- common/chat-peg-parser.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 3dc0d1a04b09e..6ce66a9a869af 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -90,7 +90,7 @@ void common_chat_peg_constructed_mapper::map(const common_peg_ast_node & node) { if (is_arg_string && current_tool) { // Serialize to JSON, but exclude the end quote - std::string dumped = json(trim_trailing_space(node.text)).dump(); + std::string dumped = json(node.text).dump(); current_tool->arguments += dumped.substr(0, dumped.size() - 1); needs_closing_quote = true; } From 1b14331115ff7197bc5bf940fde3f8a36b5200c0 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 23 Nov 2025 20:42:09 -0600 Subject: [PATCH 178/183] tweak spacing of qwen3 grammar --- common/chat.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 362402c5ffbb0..b057ad4ec4aec 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1913,8 +1913,8 @@ static common_chat_params common_chat_params_init_qwen3_coder_xml(const common_c auto content = p.rule("content", p.content(p.until_one_of({"", "\n\n" + "\n\n\n" "\n\n" })); auto tools = p.choice(); @@ -1943,7 +1943,7 @@ static common_chat_params common_chat_params_init_qwen3_coder_xml(const common_c auto arg = p.tool_arg( p.tool_arg_open("\n") + arg_value - + p.tool_arg_close( + << p.tool_arg_close( "\n" + p.peek(p.literal("")) ) From 572a913f38eeb9770104fb0ab40b4f234e364428 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 23 Nov 2025 20:42:29 -0600 Subject: [PATCH 179/183] update qwen3-coder tests --- tests/test-chat.cpp | 549 ++++++++++++++++++++++++++++++-------------- 1 file changed, 378 insertions(+), 171 deletions(-) diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index f2c91b7af2574..39740382d23f7 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -648,7 +648,6 @@ static void test_template_output_parsers() { inputs_tools_builtin.messages = {message_user}; inputs_tools_builtin.tools = {python_tool}; - goto qwen; { // Not supported yet auto tmpls = read_templates("models/templates/CohereForAI-c4ai-command-r-plus-tool_use.jinja"); @@ -2768,7 +2767,6 @@ Hey there!<|im_end|> ); } -qwen: // Test Qwen3-Coder XML format { auto tmpls = read_templates("models/templates/Qwen3-Coder.jinja"); @@ -2855,7 +2853,6 @@ Hey there!<|im_end|> }}) ); - goto done; // Special characters and Unicode common_chat_msg expected_special_chars; expected_special_chars.role = "assistant"; @@ -2865,13 +2862,23 @@ Hey there!<|im_end|> test_parser_with_streaming(expected_special_chars, "\n" - " \n" - " \n" - " Hello δΈ–η•Œ! 🌍 Special chars: @#$%^&*()\n" - " \n" - " \n" + "\n" + "\n" + "Hello δΈ–η•Œ! 🌍 Special chars: @#$%^&*()\n" + "\n" + "\n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); + make_parser(tmpls.get(), /* is_partial = */ true, /* tools */ {{ + /* .name = */ "unicode_function", + /* .description = */ "unicode function", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "message": {"type": "string"} + } + })" + }}) + ); // Multiline content with newlines and indentation common_chat_msg expected_multiline; @@ -2882,15 +2889,24 @@ Hey there!<|im_end|> test_parser_with_streaming(expected_multiline, "\n" - " \n" - " \n" + "\n" + "\n" "def hello():\n" " print(\"Hello, World!\")\n" " return True\n" - " \n" - " \n" + "\n" + "\n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); + make_parser(tmpls.get(), /* is_partial = */ true, /* tools */ {{ + /* .name = */ "code_function", + /* .description = */ "code function", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "code": {"type": "string"} + } + })" + }})); // JSON object as parameter value common_chat_msg expected_json_param; @@ -2902,13 +2918,22 @@ Hey there!<|im_end|> test_parser_with_streaming( expected_json_param, "\n" - " \n" - " \n" - " {\"host\": \"localhost\", \"port\": 8080, \"ssl\": false}\n" - " \n" - " \n" + "\n" + "\n" + "{\"host\": \"localhost\", \"port\": 8080, \"ssl\": false}\n" + "\n" + "\n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); + make_parser(tmpls.get(), /* is_partial = */ true, /* tools */ {{ + /* .name = */ "json_function", + /* .description = */ "json function", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "config": {"type": "object"} + } + })" + }})); // Array as parameter value common_chat_msg expected_array_param; @@ -2920,13 +2945,22 @@ Hey there!<|im_end|> test_parser_with_streaming( expected_array_param, "\n" - " \n" - " \n" - " [\"apple\", \"banana\", \"cherry\"]\n" - " \n" - " \n" + "\n" + "\n" + "[\"apple\", \"banana\", \"cherry\"]\n" + "\n" + "\n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); + make_parser(tmpls.get(), /* is_partial = */ true, /* tools */ {{ + /* .name = */ "array_function", + /* .description = */ "array function", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "items": {"type": "array"} + } + })" + }})); // Empty parameter common_chat_msg expected_empty_param; @@ -2938,12 +2972,22 @@ Hey there!<|im_end|> test_parser_with_streaming( expected_empty_param, "\n" - " \n" - " \n" - " \n" - " \n" + "\n" + "\n" + "\n" // Qwen3 will always produce \n\n + "\n" + "\n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); + make_parser(tmpls.get(), /* is_partial = */ true, /* tools */ {{ + /* .name = */ "empty_function", + /* .description = */ "empty function", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "empty_param": {"type": "string"} + } + })" + }})); // Boolean values (true/false) common_chat_msg expected_boolean; @@ -2955,16 +2999,26 @@ Hey there!<|im_end|> test_parser_with_streaming( expected_boolean, "\n" - " \n" - " \n" - " true\n" - " \n" - " \n" - " false\n" - " \n" - " \n" + "\n" + "\n" + "true\n" + "\n" + "\n" + "false\n" + "\n" + "\n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); + make_parser(tmpls.get(), /* is_partial = */ true, /* tools */ {{ + /* .name = */ "boolean_function", + /* .description = */ "boolean function", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "enabled": {"type": "boolean"}, + "debug": {"type": "boolean"} + } + })" + }})); // Null value common_chat_msg expected_null; @@ -2976,13 +3030,22 @@ Hey there!<|im_end|> test_parser_with_streaming( expected_null, "\n" - " \n" - " \n" - " null\n" - " \n" - " \n" + "\n" + "\n" + "null\n" + "\n" + "\n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); + make_parser(tmpls.get(), /* is_partial = */ true, /* tools */ {{ + /* .name = */ "null_function", + /* .description = */ "null function", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "optional_param": {"type": "null"} + } + })" + }})); // Negative numbers and scientific notation common_chat_msg expected_numbers; @@ -2994,19 +3057,30 @@ Hey there!<|im_end|> test_parser_with_streaming( expected_numbers, "\n" - " \n" - " \n" - " -42\n" - " \n" - " \n" - " -3.14\n" - " \n" - " \n" - " 1.23e-4\n" - " \n" - " \n" + "\n" + "\n" + "-42\n" + "\n" + "\n" + "-3.14\n" + "\n" + "\n" + "1.23e-4\n" + "\n" + "\n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); + make_parser(tmpls.get(), /* is_partial = */ true, /* tools */ {{ + /* .name = */ "math_function", + /* .description = */ "math function", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "negative": {"type": "number"}, + "decimal": {"type": "number"}, + "scientific": {"type": "number"} + } + })" + }})); // XML-like content in parameters (should be escaped) common_chat_msg expected_xml_content; @@ -3018,13 +3092,22 @@ Hey there!<|im_end|> test_parser_with_streaming( expected_xml_content, "\n" - " \n" - " \n" - " value\n" - " \n" - " \n" + "\n" + "\n" + "value\n" + "\n" + "\n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); + make_parser(tmpls.get(), /* is_partial = */ true, /* tools */ {{ + /* .name = */ "xml_function", + /* .description = */ "xml function", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "xml_content": {"type": "string"} + } + })" + }})); // Quotes and escape characters common_chat_msg expected_quotes; @@ -3036,13 +3119,22 @@ Hey there!<|im_end|> test_parser_with_streaming( expected_quotes, "\n" - " \n" - " \n" - " She said \"Hello!\" and left.\n" - " \n" - " \n" + "\n" + "\n" + "She said \"Hello!\" and left.\n" + "\n" + "\n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); + make_parser(tmpls.get(), /* is_partial = */ true, /* tools */ {{ + /* .name = */ "quote_function", + /* .description = */ "quote function", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "message": {"type": "string"} + } + })" + }})); // Long parameter value (simplified) std::string long_text = "This is a long text parameter that should test the parser's ability to handle larger amounts of text data."; @@ -3056,13 +3148,22 @@ Hey there!<|im_end|> test_parser_with_streaming( expected_long_text, "\n" - " \n" - " \n" - " " + long_text + "\n" - " \n" - " \n" + "\n" + "\n" + + long_text + "\n" + "\n" + "\n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); + make_parser(tmpls.get(), /* is_partial = */ true, /* tools */ {{ + /* .name = */ "long_function", + /* .description = */ "long function", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "long_text": {"type": "string"} + } + })" + }})); // Mixed content with text before and after tool call common_chat_msg expected_mixed_content; @@ -3075,25 +3176,22 @@ Hey there!<|im_end|> test_parser_with_streaming( expected_mixed_content, "I'll help you search for products. \n" - " \n" - " \n" - " laptops\n" - " \n" - " \n" + "\n" + "\n" + "laptops\n" + "\n" + "\n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); - - // Compact format (no extra whitespace) - common_chat_msg expected_compact; - expected_compact.role = "assistant"; - expected_compact.tool_calls = { - { "compact_function", "{\"param\":\"value\"}", "" } - }; - - test_parser_with_streaming( - expected_compact, - "value", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); + make_parser(tmpls.get(), /* is_partial = */ true, /* tools */ {{ + /* .name = */ "search_function", + /* .description = */ "search function", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "query": {"type": "string"} + } + })" + }})); // Function name with underscores and numbers common_chat_msg expected_complex_name; @@ -3105,13 +3203,22 @@ Hey there!<|im_end|> test_parser_with_streaming( expected_complex_name, "\n" - " \n" - " \n" - " 12345\n" - " \n" - " \n" + "\n" + "\n" + "12345\n" + "\n" + "\n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); + make_parser(tmpls.get(), /* is_partial = */ true, /* tools */ {{ + /* .name = */ "get_user_data_v2", + /* .description = */ "get user data v2", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "user_id": {"type": "number"} + } + })" + }})); // Parameter names with underscores and numbers common_chat_msg expected_complex_params; @@ -3123,19 +3230,30 @@ Hey there!<|im_end|> test_parser_with_streaming( expected_complex_params, "\n" - " \n" - " \n" - " value1\n" - " \n" - " \n" - " value2\n" - " \n" - " \n" - " 123\n" - " \n" - " \n" + "\n" + "\n" + "value1\n" + "\n" + "\n" + "value2\n" + "\n" + "\n" + "123\n" + "\n" + "\n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); + make_parser(tmpls.get(), /* is_partial = */ true, /* tools */ {{ + /* .name = */ "test_function", + /* .description = */ "test function", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "param_1": {"type": "string"}, + "param_2_name": {"type": "string"}, + "param3": {"type": "number"} + } + })" + }})); // Very deeply nested XML content in parameter common_chat_msg expected_deep_xml; @@ -3147,13 +3265,22 @@ Hey there!<|im_end|> test_parser_with_streaming( expected_deep_xml, "\n" - " \n" - " \n" - " deep content\n" - " \n" - " \n" + "\n" + "\n" + "deep content\n" + "\n" + "\n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); + make_parser(tmpls.get(), /* is_partial = */ true, /* tools */ {{ + /* .name = */ "xml_parser", + /* .description = */ "xml parser", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "xml": {"type": "string"} + } + })" + }})); // Parameter with only whitespace common_chat_msg expected_whitespace_param; @@ -3165,13 +3292,22 @@ Hey there!<|im_end|> test_parser_with_streaming( expected_whitespace_param, "\n" - " \n" - " \n" - " \n" - " \n" - " \n" + "\n" + "\n" + "\n" + "\n" + "\n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); + make_parser(tmpls.get(), /* is_partial = */ true, /* tools */ {{ + /* .name = */ "whitespace_function", + /* .description = */ "whitespace function", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "spaces": {"type": "string"} + } + })" + }})); // Parameter with tabs and mixed whitespace common_chat_msg expected_mixed_whitespace; @@ -3183,15 +3319,24 @@ Hey there!<|im_end|> test_parser_with_streaming( expected_mixed_whitespace, "\n" - " \n" - " \n" + "\n" + "\n" "line1\n" "\tindented line\n" " spaces\n" - " \n" - " \n" + "\n" + "\n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); + make_parser(tmpls.get(), /* is_partial = */ true, /* tools */ {{ + /* .name = */ "tab_function", + /* .description = */ "tab function", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "content": {"type": "string"} + } + })" + }})); // Control characters and special Unicode common_chat_msg expected_control_chars; @@ -3203,13 +3348,22 @@ Hey there!<|im_end|> test_parser_with_streaming( expected_control_chars, "\n" - " \n" - " \n" + "\n" + "\n" "Line1\nLine2\tTabbed\rCarriage return\n" - " \n" - " \n" + "\n" + "\n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); + make_parser(tmpls.get(), /* is_partial = */ true, /* tools */ {{ + /* .name = */ "control_function", + /* .description = */ "control function", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "text": {"type": "string"} + } + })" + }})); // Emoji and extended Unicode characters common_chat_msg expected_emoji; @@ -3221,13 +3375,22 @@ Hey there!<|im_end|> test_parser_with_streaming( expected_emoji, "\n" - " \n" - " \n" - " Hello! πŸ‘‹ 🌟 πŸš€ Testing emojis: πŸ˜€πŸ˜ƒπŸ˜„πŸ˜ and symbols: βˆ‘βˆβˆ†βˆ‡\n" - " \n" - " \n" + "\n" + "\n" + "Hello! πŸ‘‹ 🌟 πŸš€ Testing emojis: πŸ˜€πŸ˜ƒπŸ˜„πŸ˜ and symbols: βˆ‘βˆβˆ†βˆ‡\n" + "\n" + "\n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); + make_parser(tmpls.get(), /* is_partial = */ true, /* tools */ {{ + /* .name = */ "emoji_function", + /* .description = */ "emoji function", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "message": {"type": "string"} + } + })" + }})); // Mathematical expressions and formulas common_chat_msg expected_math; @@ -3239,13 +3402,22 @@ Hey there!<|im_end|> test_parser_with_streaming( expected_math, "\n" - " \n" - " \n" - " E = mcΒ² and ∫f(x)dx = F(x) + C\n" - " \n" - " \n" + "\n" + "\n" + "E = mcΒ² and ∫f(x)dx = F(x) + C\n" + "\n" + "\n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); + make_parser(tmpls.get(), /* is_partial = */ true, /* tools */ {{ + /* .name = */ "math_function", + /* .description = */ "math function", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "formula": {"type": "string"} + } + })" + }})); // SQL injection-like content (should be safely escaped) common_chat_msg expected_sql; @@ -3257,13 +3429,22 @@ Hey there!<|im_end|> test_parser_with_streaming( expected_sql, "\n" - " \n" - " \n" - " SELECT * FROM users WHERE id = 1; DROP TABLE users; --\n" - " \n" - " \n" + "\n" + "\n" + "SELECT * FROM users WHERE id = 1; DROP TABLE users; --\n" + "\n" + "\n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); + make_parser(tmpls.get(), /* is_partial = */ true, /* tools */ {{ + /* .name = */ "sql_function", + /* .description = */ "sql function", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "query": {"type": "string"} + } + })" + }})); // HTML/XML injection content common_chat_msg expected_html; @@ -3275,13 +3456,22 @@ Hey there!<|im_end|> test_parser_with_streaming( expected_html, "\n" - " \n" - " \n" - " \n" - " \n" - " \n" + "\n" + "\n" + "\n" + "\n" + "\n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); + make_parser(tmpls.get(), /* is_partial = */ true, /* tools */ {{ + /* .name = */ "html_function", + /* .description = */ "html function", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "content": {"type": "string"} + } + })" + }})); // Binary-like content (base64) common_chat_msg expected_binary; @@ -3293,13 +3483,22 @@ Hey there!<|im_end|> test_parser_with_streaming( expected_binary, "\n" - " \n" - " \n" - " SGVsbG8gV29ybGQhIFRoaXMgaXMgYmFzZTY0IGVuY29kZWQgdGV4dC4=\n" - " \n" - " \n" + "\n" + "\n" + "SGVsbG8gV29ybGQhIFRoaXMgaXMgYmFzZTY0IGVuY29kZWQgdGV4dC4=\n" + "\n" + "\n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); + make_parser(tmpls.get(), /* is_partial = */ true, /* tools */ {{ + /* .name = */ "binary_function", + /* .description = */ "binary function", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "data": {"type": "string"} + } + })" + }})); // Very large numbers (should be parsed as scientific notation) common_chat_msg expected_large_numbers; @@ -3311,16 +3510,24 @@ Hey there!<|im_end|> test_parser_with_streaming( expected_large_numbers, "\n" - " \n" - " \n" - " 999999999999999999999999999999999999999999999999999999999999\n" - " \n" - " \n" + "\n" + "\n" + "999999999999999999999999999999999999999999999999999999999999\n" + "\n" + "\n" "", - [&](const std::string &msg) { return common_chat_parse(msg, /* is_partial= */ true, {COMMON_CHAT_FORMAT_PEG_CONSTRUCTED}); }); + make_parser(tmpls.get(), /* is_partial = */ true, /* tools */ {{ + /* .name = */ "number_function", + /* .description = */ "number function", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "big_int": {"type": "number"} + } + })" + }})); } -done: { // Qwen3-Coder template auto tmpls = read_templates("models/templates/Qwen3-Coder.jinja"); From bd26d8376e52c84b32d9d474ad06be505b685220 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 23 Nov 2025 20:49:49 -0600 Subject: [PATCH 180/183] qwen3-coder small fixes --- common/chat.cpp | 2 +- tests/test-chat.cpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/common/chat.cpp b/common/chat.cpp index b057ad4ec4aec..17b8c441bc51d 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1914,7 +1914,7 @@ static common_chat_params common_chat_params_init_qwen3_coder_xml(const common_c auto until_end_of_param = p.rule("string-arg-value", p.until_one_of({ "\n\n\n" "\n\n" + "\n", "\n\n" })); auto tools = p.choice(); diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 39740382d23f7..4f4efb57d0645 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -2779,6 +2779,7 @@ Hey there!<|im_end|> make_parser(common_chat_templates * tmpls, bool is_partial, const std::vector & tools) { common_chat_templates_inputs inputs; + inputs.messages = { message_user }; inputs.tools = tools; params_ = common_chat_templates_apply(tmpls, inputs); assert_equals(COMMON_CHAT_FORMAT_PEG_CONSTRUCTED, params_.format); From b0ec94f7ee4c913ab878d204e0025fbc5a0d84b2 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 23 Nov 2025 21:04:15 -0600 Subject: [PATCH 181/183] place parser in common_chat_syntax to simplify invocation --- common/chat.cpp | 3 +++ common/chat.h | 2 ++ common/peg-parser.h | 2 -- tools/server/server.cpp | 23 +++++++---------------- 4 files changed, 12 insertions(+), 18 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 17b8c441bc51d..1498096e75ad4 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -3595,6 +3595,9 @@ static void common_chat_parse(common_chat_msg_parser & builder) { } common_chat_msg common_chat_parse(const std::string & input, bool is_partial, const common_chat_syntax & syntax) { + if (!syntax.parser.empty()) { + return common_chat_peg_parse(syntax.parser, input, is_partial, syntax); + } common_chat_msg_parser builder(input, is_partial, syntax); try { common_chat_parse(builder); diff --git a/common/chat.h b/common/chat.h index 714122ce78229..bd53a4dba8835 100644 --- a/common/chat.h +++ b/common/chat.h @@ -3,6 +3,7 @@ #pragma once #include "common.h" +#include "peg-parser.h" #include #include #include @@ -170,6 +171,7 @@ struct common_chat_syntax { bool reasoning_in_content = false; bool thinking_forced_open = false; bool parse_tool_calls = true; + common_peg_arena parser; }; // Check if the template supplied via "--chat-template" is supported or not. Returns true if it's valid diff --git a/common/peg-parser.h b/common/peg-parser.h index fcba45dfec719..ec8b01bcdbf02 100644 --- a/common/peg-parser.h +++ b/common/peg-parser.h @@ -1,7 +1,5 @@ #pragma once -#include "chat.h" - #include #include diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 708b5847285d5..baa0a6b603260 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -143,7 +143,6 @@ struct slot_params { std::string oaicompat_model; std::string oaicompat_cmpl_id; common_chat_syntax oaicompat_chat_syntax; - common_peg_arena oaicompat_chat_parser; // Embeddings int32_t embd_normalize = 2; // (-1=none, 0=max absolute int16, 1=taxicab, 2=Euclidean/L2, >2=p-norm) @@ -454,12 +453,11 @@ struct server_task { params.oaicompat_chat_syntax.reasoning_in_content = params.stream && (reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY); params.oaicompat_chat_syntax.thinking_forced_open = json_value(data, "thinking_forced_open", false); params.oaicompat_chat_syntax.parse_tool_calls = json_value(data, "parse_tool_calls", false); + if (data.contains("chat_parser")) { + params.oaicompat_chat_syntax.parser = common_peg_arena::deserialize(data.at("chat_parser").get()); + } } - if (data.contains("chat_parser")) { - auto parser = data.at("chat_parser").get(); - params.oaicompat_chat_parser = common_peg_arena::deserialize(parser); - } { const auto preserved_tokens = data.find("preserved_tokens"); @@ -1869,17 +1867,10 @@ struct server_slot { auto previous_msg = chat_msg; SRV_DBG("Parsing chat message: %s\n", generated_text.c_str()); - auto new_msg = !task->params.oaicompat_chat_parser.empty() ? - common_chat_peg_parse( - task->params.oaicompat_chat_parser, - generated_text, - /* is_partial= */ stop != STOP_TYPE_EOS, - task->params.oaicompat_chat_syntax) : - common_chat_parse( - generated_text, - /* is_partial= */ stop != STOP_TYPE_EOS, - task->params.oaicompat_chat_syntax); - + auto new_msg = common_chat_parse( + generated_text, + /* is_partial= */ stop != STOP_TYPE_EOS, + task->params.oaicompat_chat_syntax); if (!new_msg.empty()) { new_msg.set_tool_call_ids(generated_tool_call_ids, gen_tool_call_id); chat_msg = new_msg; From c3b001bae66bcf873b982522a69d87bf843b8b69 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 23 Nov 2025 21:07:29 -0600 Subject: [PATCH 182/183] use std::set to collect rules to keep order predictable for tests --- common/peg-parser.cpp | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp index 55f5450510da9..54fbd34561c79 100644 --- a/common/peg-parser.cpp +++ b/common/peg-parser.cpp @@ -1,4 +1,3 @@ -#include "log.h" #include "common.h" #include "peg-parser.h" #include "json-schema-to-grammar.h" @@ -7,11 +6,11 @@ #include #include +#include #include -#include #include -#include -#include +#include +#include // Trick to catch missing branches template @@ -1186,12 +1185,12 @@ static std::string gbnf_excluding_pattern(const std::vector & strin } // Collect reachable rules from a given rule -static std::unordered_set collect_reachable_rules( +static std::set collect_reachable_rules( const common_peg_arena & arena, const common_peg_parser_id & rule ) { - std::unordered_set reachable; - std::unordered_set visited; + std::set reachable; + std::set visited; std::function visit = [&](common_peg_parser_id id) { const auto & parser = arena.get(id); @@ -1391,7 +1390,7 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo }; // Collect reachable rules - std::unordered_set reachable_rules; + std::set reachable_rules; if (lazy) { // Collect rules reachable from trigger rules From 9bdd3a3654b5ac94a2eff382e43913972338593e Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 23 Nov 2025 21:30:49 -0600 Subject: [PATCH 183/183] initialize parser to make certain platforms happy --- common/chat.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/chat.h b/common/chat.h index bd53a4dba8835..dd994fef63553 100644 --- a/common/chat.h +++ b/common/chat.h @@ -171,7 +171,7 @@ struct common_chat_syntax { bool reasoning_in_content = false; bool thinking_forced_open = false; bool parse_tool_calls = true; - common_peg_arena parser; + common_peg_arena parser = common_peg_arena(); // NOLINT }; // Check if the template supplied via "--chat-template" is supported or not. Returns true if it's valid