Merge pull request #343 from joka921/f.batchReadingError

Fix a tricky error

src/parser/TurtleParser.cpp

@@ -432,7 +432,8 @@ TurtleStreamParser<T>::backupState() const {
 // _______________________________________________________________
 template <class T>
 bool TurtleStreamParser<T>::resetStateAndRead(
-    const TurtleStreamParser::TurtleParserBackupState b) {
+    TurtleStreamParser::TurtleParserBackupState* bPtr) {
+  auto& b = *bPtr;
   auto nextBytesOpt = _fileBuffer->getNextBlock();
   if (!nextBytesOpt || nextBytesOpt.value().empty()) {
     // there are no more decompressed bytes, just continue with what we've got
@@ -457,6 +458,9 @@ bool TurtleStreamParser<T>::resetStateAndRead(
 
   LOG(TRACE) << "Succesfully decompressed next batch of " << nextBytes.size()
              << " << bytes to parser\n";
+
+  // important: our tokenizer may have a new position
+  b = backupState();
   return true;
 }
 
@@ -561,7 +565,7 @@ bool TurtleStreamParser<T>::getLine(std::array<string, 3>* triple) {
         // we read chunks of memories in a buffered way
         // try to parse with a larger buffer and repeat the reading process
         // (maybe the failure was due to statements crossing our block).
-        if (resetStateAndRead(b)) {
+        if (resetStateAndRead(&b)) {
           // we have succesfully extended our buffer
           if (_byteVec.size() > BZIP2_MAX_TOTAL_BUFFER_SIZE) {
             auto d = _tok.view();

src/parser/TurtleParser.h

@@ -31,6 +31,7 @@ class TurtleParser {
    public:
     ParseException() = default;
     explicit ParseException(string msg) : _msg(std::move(msg)) {}
+
     [[nodiscard]] const char* what() const noexcept override {
       return _msg.c_str();
     }
@@ -337,7 +338,7 @@ class TurtleStreamParser : public TurtleParser<Tokenizer_T> {
   // Reset the parser to the state indicated by the argument
   // Must be called on the same parser object that was used to create the backup
   // state (the actual triples are not backed up)
-  bool resetStateAndRead(TurtleParserBackupState state);
+  bool resetStateAndRead(TurtleParserBackupState* state);
 
   // stores the current batch of bytes we have to parse.
   // Might end in the middle of a statement or even a multibyte utf8 character,