add free-form operators for const char * to simplify defining literals

Author: aldehir

common/chat-parser-combinator.cpp

@@ -1222,6 +1222,8 @@ parser::parser(std::shared_ptr<parser_base> parser) : ptr_(std::move(parser)) {}
 
 parser::parser(const std::string & literal) : ptr_(std::make_shared<literal_parser>(literal, -1)) {}
 
+parser::parser(const char * literal) : ptr_(std::make_shared<literal_parser>(literal, -1)) {}
+
 parser parser::operator~() const {
     return parser(std::make_shared<not_parser>(*this, -1));
 }
@@ -1230,30 +1232,18 @@ parser parser::operator+(const parser & other) const {
     return parser(std::make_shared<sequence_parser>(std::initializer_list<parser>{*this, other}, -1));
 }
 
-parser parser::operator+(const std::string & literal) const {
-    auto lit = parser(std::make_shared<literal_parser>(literal, -1));
-    return parser(std::make_shared<sequence_parser>(std::initializer_list<parser>{*this, lit}, -1));
-}
-
 parser parser::operator|(const parser & other) const {
     return parser(std::make_shared<choice_parser>(std::initializer_list<parser>{*this, other}, -1));
 }
 
-parser parser::operator|(const std::string & literal) const {
-    auto lit = parser(std::make_shared<literal_parser>(literal, -1));
-    return parser(std::make_shared<choice_parser>(std::initializer_list<parser>{*this, lit}, -1));
-}
-
 parser parser::operator<<(const parser & other) const {
     auto ws = parser(std::make_shared<space_parser>(-1));
     return parser(std::make_shared<sequence_parser>(std::initializer_list<parser>{*this, ws, other}, -1));
 }
 
-parser parser::operator<<(const std::string & literal) const {
-    auto ws = parser(std::make_shared<space_parser>(-1));
-    auto lit = parser(std::make_shared<literal_parser>(literal, -1));
-    return parser(std::make_shared<sequence_parser>(std::initializer_list<parser>{*this, ws, lit}, -1));
-}
+parser operator+(const char * lhs, const parser & rhs) { return parser(lhs) + rhs; }
+parser operator|(const char * lhs, const parser & rhs) { return parser(lhs) | rhs; }
+parser operator<<(const char * lhs, const parser & rhs) { return parser(lhs) << rhs; }
 
 parser_base & parser::operator*() const {
     return *ptr_;
@@ -1373,13 +1363,6 @@ parser parser_builder::json_string(const parser & p) {
     return quote + p + quote;
 }
 
-parser parser_builder::between(const std::string & left, const parser & p, const std::string & right, bool allow_spaces) {
-    if (allow_spaces) {
-        return literal(left) << p << literal(right);
-    }
-    return literal(left) + p + literal(right);
-}
-
 parser parser_builder::add_rule(const std::string & name, const parser & p) {
     (*rules_)[name] = p;
     return rule(name);

common/chat-parser-combinator.h

@@ -106,6 +106,8 @@ class parser {
     parser(std::shared_ptr<parser_base> parser);
     parser(const parser & other) = default;
     parser(const std::string & literal);
+    parser(const char * literal);
+
     parser & operator=(const parser & other) {
         if (this != &other) {
             ptr_ = other.ptr_;
@@ -114,15 +116,9 @@ class parser {
     }
 
     parser operator~() const;
-
     parser operator+(const parser & other) const;
-    parser operator+(const std::string & literal) const;
-
     parser operator|(const parser & other) const;
-    parser operator|(const std::string & literal) const;
-
     parser operator<<(const parser & other) const;
-    parser operator<<(const std::string & literal) const;
 
     parser_base & operator*() const;
     parser_base * operator->() const;
@@ -136,6 +132,10 @@ class parser {
     void build_grammar(const common_grammar_builder & builder) const;
 };
 
+parser operator+(const char * lhs, const parser & rhs);
+parser operator|(const char * lhs, const parser & rhs);
+parser operator<<(const char * lhs, const parser & rhs);
+
 class parser_id_counter {
     int next_id_;
   public:
@@ -231,8 +231,6 @@ class parser_builder {
     parser json_key(const std::string & name, const parser & p);
     parser json_string(const parser & p);
 
-    parser between(const std::string & left, const parser & p, const std::string & right, bool allow_spaces = true);
-
     // Wraps a parser with JSON schema metadata for grammar generation.
     // Used internally to convert JSON schemas to GBNF grammar rules.
     parser schema(const parser & p, const std::string & name, const nlohmann::ordered_json & schema);

tests/test-chat-parser-combinator.cpp

@@ -711,10 +711,10 @@ static void test_gbnf_generation() {
 static parser create_command_r7b_parser() {
     auto parser = build_parser([](parser_builder & p) {
         auto thinking = p.add_rule("thinking",
-            p.literal("<|START_THINKING|>") << p.until("<|END_THINKING|>") << "<|END_THINKING|>");
+            "<|START_THINKING|>" << p.until("<|END_THINKING|>") << "<|END_THINKING|>");
 
         auto response = p.add_rule("response",
-            p.literal("<|START_RESPONSE|>") << p.until("<|END_RESPONSE|>") << "<|END_RESPONSE|>");
+            "<|START_RESPONSE|>" << p.until("<|END_RESPONSE|>") << "<|END_RESPONSE|>");
 
         auto json = p.add_rule("json", p.json());
         auto tool_call_id = p.add_rule("tool-call-id", p.json_key("tool_call_id", p.json_string(p.until("\""))));
@@ -723,14 +723,14 @@ static parser create_command_r7b_parser() {
         auto tool_call_fields = p.add_rule("tool-call-fields", tool_call_id | tool_call_name | tool_call_args);
 
         auto tool_call = p.add_rule("tool-call",
-            p.between("{", tool_call_fields << p.zero_or_more(p.literal(",") << tool_call_fields), "}"));
+            "{" << tool_call_fields << p.zero_or_more(p.literal(",") << tool_call_fields) << "}");
 
         auto tool_calls = p.add_rule("tool-calls",
-            p.literal("<|START_ACTION|>")
-            << "[" << tool_call << p.zero_or_more(p.literal(",") << tool_call) << "]"
+            "<|START_ACTION|>"
+            << ("[" << tool_call << p.zero_or_more(p.literal(",") << tool_call) << "]")
             << "<|END_ACTION|>");
 
-        return p.optional(thinking) << p.add_rule("content", (tool_calls | response));
+        return p.optional(thinking) << p.add_rule("content", tool_calls | response);
     });
 
     auto grammar = build_grammar([&](const common_grammar_builder & builder) {