Small improvements of the SparqlLexer classes' interface

src/parser/SparqlLexer.cpp

@@ -98,6 +98,13 @@ SparqlLexer::SparqlLexer(const std::string& sparql)
   readNext();
 }
 
+// ____________________________________________________________________________
+void SparqlLexer::reset(std::string sparql) {
+  _sparql = std::move(sparql);
+  _re_string = re2::StringPiece{_sparql};
+  readNext();
+}
+
 bool SparqlLexer::empty() const { return _re_string.empty(); }
 
 void SparqlLexer::readNext() {
@@ -133,7 +140,14 @@ void SparqlLexer::readNext() {
       }
     }
     if (!regexMatched) {
-      throw ParseException("Unexpected input: " + _re_string.as_string());
+      if (_re_string[0] == '#') {
+        // Start of a comment. Consume everything up to the next newline.
+        while (!_re_string.empty() && _re_string[0] != '\n') {
+          _re_string.remove_prefix(1);
+        }
+      } else {
+        throw ParseException("Unexpected input: " + _re_string.as_string());
+      }
     }
   }
   _next.raw = raw;

src/parser/SparqlLexer.h

@@ -37,7 +37,6 @@ struct SparqlToken {
 
 class SparqlLexer {
  public:
- private:
   using RegexTokenMap =
       std::vector<std::pair<std::unique_ptr<re2::RE2>, SparqlToken::Type>>;
 
@@ -50,6 +49,16 @@ class SparqlLexer {
  public:
   SparqlLexer(const std::string& sparql);
 
+  // Copying and moving is disallowed, the default behavior is wrong,
+  // and we don't need it.
+  SparqlLexer(const SparqlLexer&) = delete;
+  SparqlLexer& operator=(const SparqlLexer&) = delete;
+  SparqlLexer(SparqlLexer&&) noexcept = delete;
+  SparqlLexer& operator=(SparqlLexer&&) = delete;
+
+  // Explicitly reset this Lexer to a new input
+  void reset(std::string sparql);
+
   // True if the entire input stream was consumed
   bool empty() const;
 
@@ -68,10 +77,18 @@ class SparqlLexer {
   const SparqlToken& current();
   const std::string& input() const;
 
+  // Get the part of the input that has not yet been consumed by calls to
+  // `accept` or `expect`
+  std::string getUnconsumedInput() {
+    auto delimiter =
+        _next.raw.empty() || _re_string.ToString().empty() ? "" : " ";
+    return _next.raw + delimiter + _re_string.ToString();
+  }
+
  private:
   void readNext();
 
-  const std::string _sparql;
+  std::string _sparql;
   re2::StringPiece _re_string;
   SparqlToken _current;
   SparqlToken _next;

test/SparqlLexerTest.cpp

@@ -7,6 +7,13 @@
 
 #include "../src/parser/SparqlLexer.h"
 
+TEST(SparqlLexerTest, unescapeLiteral) {
+  std::string input = R"("^\"biff")";
+  SparqlLexer lexer(input);
+  lexer.expect(SparqlToken::Type::RDFLITERAL);
+  ASSERT_EQ(R"("^"biff")", lexer.current().raw);
+}
+
 TEST(SparqlLexerTest, basicTest) {
   std::string query =
       ""
@@ -159,3 +166,19 @@ TEST(SparqlLexerTest, basicTest) {
   lexer2.expect(SparqlToken::Type::VARIABLE);  // ?a
   lexer2.expect(SparqlToken::Type::SYMBOL);    // )
 }
+
+TEST(SparqlLexer, reset) {
+  std::string query = "PREFIX wd: <http://www.wikidata.org/entity/>";
+
+  SparqlLexer lexer(query);
+  lexer.expect("prefix");
+  lexer.expect("wd:");
+  ASSERT_EQ(std::string("<http://www.wikidata.org/entity/>"),
+            lexer.getUnconsumedInput());
+
+  lexer.reset("PREFIX ql: <cs.uni-freiburg.de/>");
+  lexer.expect("prefix");
+  lexer.expect("ql:");
+  lexer.expect("<cs.uni-freiburg.de/>");
+  ASSERT_TRUE(lexer.getUnconsumedInput().empty());
+}