Now the token-by-token parser and iterative parser are fully compatible

Adjust the position of SkipWhitespaceAndComments().
Tencent · May 13, 2020 · c44ab1a · c44ab1a
1 parent 4358049
commit c44ab1a
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 3 deletions.
diff --git a/include/rapidjson/reader.h b/include/rapidjson/reader.h
@@ -619,9 +619,9 @@ class GenericReader {
      */
     template <unsigned parseFlags, typename InputStream, typename Handler>
     bool IterativeParseNext(InputStream& is, Handler& handler) {
+        SkipWhitespaceAndComments<parseFlags>(is);
+        RAPIDJSON_PARSE_ERROR_EARLY_RETURN(false);
         while (RAPIDJSON_LIKELY(is.Peek() != '\0')) {
-            SkipWhitespaceAndComments<parseFlags>(is);
-
             Token t = Tokenize(is.Peek());
             IterativeParsingState n = Predict(state_, t);
             IterativeParsingState d = Transit<parseFlags>(state_, t, n, is, handler);
@@ -642,6 +642,7 @@ class GenericReader {
                 if (!(parseFlags & kParseStopWhenDoneFlag)) {
                     // ... and extra non-whitespace data is found...
                     SkipWhitespaceAndComments<parseFlags>(is);
+                    RAPIDJSON_PARSE_ERROR_EARLY_RETURN(false);
                     if (is.Peek() != '\0') {
                         // ... this is considered an error.
                         HandleError(state_, is);
@@ -659,6 +660,8 @@ class GenericReader {
             // If we parsed anything other than a delimiter, we invoked the handler, so we can return true now.
             if (!IsIterativeParsingDelimiterState(n))
                 return true;
+            SkipWhitespaceAndComments<parseFlags>(is);
+            RAPIDJSON_PARSE_ERROR_EARLY_RETURN(false);
         }
 
         // We reached the end of file.

diff --git a/test/unittest/readertest.cpp b/test/unittest/readertest.cpp
@@ -1607,7 +1607,7 @@ TEST(Reader, IterativePullParsing_General) {
             handler.LOG_ENDARRAY | 7
         };
 
-        StringStream is("[1, {\"k\": [1, 2]}, null, false, true, \"string\", 1.2]");
+        StringStream is("   [1,   {\"k\"  :   [1, 2]}, null, false, true, \"string\", 1.2]");
         Reader reader;
 
         reader.IterativeParseInit();
@@ -1629,6 +1629,18 @@ TEST(Reader, IterativePullParsing_General) {
         EXPECT_EQ(handler.LogCount, oldLogCount) << "parse-next past complete should not invoke handler";
         EXPECT_FALSE(reader.HasParseError()) << "parse-next past complete should not generate parse error";
     }
+
+    {
+        IterativeParsingReaderHandler<> handler;
+        StringStream is("   ");
+        Reader reader;
+        reader.IterativeParseInit();
+        while (!reader.IterativeParseComplete()) {
+            EXPECT_FALSE(reader.IterativeParseNext<kParseDefaultFlags>(is, handler));
+        }
+        EXPECT_TRUE(reader.HasParseError());
+        EXPECT_EQ(kParseErrorDocumentEmpty, reader.GetParseErrorCode());
+    }
 }
 
 // Test iterative parsing on kParseErrorTermination.