Refactor of wrong parse error fix when chunks are splitted on a white…

…space inside an object or array (#6)
discoveryjs · May 12, 2021 · 91ec781 · 91ec781
1 parent 293fbc2
commit 91ec781
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 23 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,7 @@
+## next
+
+- Fixed wrong parse error when chunks are splitted on a whitespace inside an object or array (#6, @alexei-vedder)
+
 ## 0.5.2 (2020-12-26)
 
 - Fixed `RangeError: Maximum call stack size exceeded` in `parseChunked()` on very long arrays (corner case)

diff --git a/src/parse-chunked.js b/src/parse-chunked.js
@@ -98,17 +98,20 @@ class ChunkParser {
         this.stateStringEscape = false;
         this.pendingByteSeq = null;
         this.pendingChunk = null;
-        this.pos = 0;
+        this.chunkOffset = 0;
         this.jsonParseOffset = 0;
     }
 
     flush(chunk, start, end) {
         let fragment = chunk.slice(start, end);
-        this.jsonParseOffset = this.pos; // using for position correction in JSON.parse() error if any
+
+        // Save position correction an error in JSON.parse() if any
+        this.jsonParseOffset = this.chunkOffset + start;
 
         // Prepend pending chunk if any
         if (this.pendingChunk !== null) {
             fragment = this.pendingChunk + fragment;
+            this.jsonParseOffset -= this.pendingChunk.length;
             this.pendingChunk = null;
         }
 
@@ -199,7 +202,6 @@ class ChunkParser {
             }
         }
 
-        this.pos += end - start;
         this.lastFlushDepth = this.flushDepth;
     }
 
@@ -285,20 +287,20 @@ class ChunkParser {
                     break;
 
                 case 0x7B: /* { */
-                    // begin object
+                    // Open an object
                     flushPoint = i + 1;
                     this.stack[this.flushDepth++] = STACK_OBJECT;
                     break;
 
                 case 0x5B: /* [ */
-                    // begin array
+                    // Open an array
                     flushPoint = i + 1;
                     this.stack[this.flushDepth++] = STACK_ARRAY;
                     break;
 
                 case 0x5D: /* ] */
                 case 0x7D: /* } */
-                    // end object or array
+                    // Close an object or array
                     flushPoint = i + 1;
                     this.flushDepth--;
 
@@ -308,14 +310,20 @@ class ChunkParser {
                     }
 
                     break;
+
                 case 0x09: /* \t */
                 case 0x0A: /* \n */
                 case 0x0D: /* \r */
                 case 0x20: /* space */
-                    // prevent passing trailing whitespaces to the next flush(..) call
+                    // Move points forward when they points on current position and it's a whitespace
                     if (lastFlushPoint === i) {
-                        ++lastFlushPoint;
+                        lastFlushPoint++;
                     }
+
+                    if (flushPoint === i) {
+                        flushPoint++;
+                    }
+
                     break;
             }
         }
@@ -324,20 +332,20 @@ class ChunkParser {
             this.flush(chunk, lastFlushPoint, flushPoint);
         }
 
-        // Produce pendingChunk if any
+        // Produce pendingChunk if something left
         if (flushPoint < chunkLength) {
-            let newPending = chunk.slice(flushPoint, chunkLength);
-
-            if (this.pendingChunk === null && !this.stateString) {
-                newPending = newPending.trimStart();
-            }
-
-            if (newPending.length) {
-                this.pendingChunk = this.pendingChunk !== null
-                    ? this.pendingChunk + newPending
-                    : newPending;
+            if (this.pendingChunk !== null) {
+                // When there is already a pending chunk then no flush happened,
+                // appending entire chunk to pending one
+                this.pendingChunk += chunk;
+            } else {
+                // Create a pending chunk, it will start with non-whitespace since
+                // flushPoint was moved forward away from whitespaces on scan
+                this.pendingChunk = chunk.slice(flushPoint, chunkLength);
             }
         }
+
+        this.chunkOffset += chunkLength;
     }
 
     finish() {

diff --git a/test/parse-chunked.js b/test/parse-chunked.js
@@ -103,10 +103,10 @@ describe('parseChunked()', () => {
         });
     }
 
-    describe('trailing whitespaces', () => {
-        describe('at the end', () => {
-            const expected = {};
-            const json = '{} \r\n\t';
+    describe('splitting on whitespaces', () => {
+        describe('inside an object and strings', () => {
+            const expected = { ' \r\n\t': ' \r\n\t', a: [1, 2] };
+            const json = ' \r\n\t{ \r\n\t" \\r\\n\\t" \r\n\t: \r\n\t" \\r\\n\\t" \r\n\t, \r\n\t"a": \r\n\t[ \r\n\t1 \r\n\t, \r\n\t2 \r\n\t] \r\n\t} \r\n\t';
 
             for (let len = 0; len <= json.length; len++) {
                 it(len ? len + ' char(s) length chunks' : 'parse full', async () =>
@@ -140,6 +140,12 @@ describe('parseChunked()', () => {
                 /Unexpected token , in JSON at position 18/
             )
         );
+        it('abs pos across chunks #3 (whitespaces)', () =>
+            assert.rejects(
+                async () => await parse(['[{"test" ', ' ', ' :"hello"} ', ' ', ',', ' ', ',}']),
+                /Unexpected token , in JSON at position 24/
+            )
+        );
     });
 
     describe('use with buffers', () => {