Recognise incomplete JSON content on python backend

As pointed out in #42, the python backend failed to properly recognise
valid but incomplete JSON content when the incomplete bits were the
beginning of the "true", "false" or "null" tokens.

This commit adds support for recognising these cases, and adds test
cases that will ensure these cases are properly recognised by all
backends, including the python one.

Signed-off-by: Rodrigo Tobar <rtobar@icrar.org>

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## Development version
+
+* Python backend raises errors on incomplete JSON content
+  that previously wasn't recognised as such,
+  aligning itself with the rest of the backends (#42).
+
 ## [3.1.3]
 
 * Python backed correctly raises errors

README.rst

@@ -542,6 +542,8 @@ FAQ
      values greater than 2^32 in 32-bit platforms or Windows.
      Numbers with leading zeros are not reported as invalid
      (although they are invalid JSON numbers).
+     Incomplete JSON tokens at the end of an incomplete document
+     (e.g., ``{"a": fals``) are not reported as ``IncompleteJSONError``.
 
    * The ``python`` backend doesn't support ``allow_comments=True``
      It also internally works with ``str`` objects, not ``bytes``,

ijson/backends/python.py

@@ -226,6 +226,8 @@ def parse_value(target, multivalue, use_float):
                     if number == inf:
                         raise common.JSONError("float overflow: %s" % (symbol,))
                 except:
+                    if 'true'.startswith(symbol) or 'false'.startswith(symbol) or 'null'.startswith(symbol):
+                        raise common.IncompleteJSONError('Incomplete JSON content')
                     raise UnexpectedSymbol(symbol, pos)
                 else:
                     send(('number', number))

test/test_base.py

@@ -237,6 +237,38 @@
     b'{"key": "value"',
     b'{"key": "value",',
 ]
+INCOMPLETE_JSON_TOKENS = [
+    b'n',
+    b'nu',
+    b'nul',
+    b't',
+    b'tr',
+    b'tru',
+    b'f',
+    b'fa',
+    b'fal',
+    b'fals',
+    b'[f',
+    b'[fa',
+    b'[fal',
+    b'[fals',
+    b'[t',
+    b'[tr',
+    b'[tru',
+    b'[n',
+    b'[nu',
+    b'[nul',
+    b'{"key": t',
+    b'{"key": tr',
+    b'{"key": tru',
+    b'{"key": f',
+    b'{"key": fa',
+    b'{"key": fal',
+    b'{"key": fals',
+    b'{"key": n',
+    b'{"key": nu',
+    b'{"key": nul',
+]
 STRINGS_JSON = br'''
 {
     "str1": "",
@@ -469,6 +501,13 @@ def test_incomplete(self):
             with self.assertRaises(common.IncompleteJSONError):
                 self.get_all(self.basic_parse, json)
 
+    def test_incomplete_tokens(self):
+        if not self.handles_incomplete_json_tokens:
+            return
+        for json in INCOMPLETE_JSON_TOKENS:
+            with self.assertRaises(common.IncompleteJSONError):
+                self.get_all(self.basic_parse, json)
+
     def test_invalid(self):
         for json in INVALID_JSONS:
             # Yajl1 doesn't complain about additional data after the end
@@ -574,6 +613,7 @@ def generate_test_cases(module, classname, method_suffix, *bases):
             'supports_multiple_values': name != 'yajl',
             'supports_comments': name != 'python',
             'detects_leading_zeros': name != 'yajl',
+            'handles_incomplete_json_tokens': name != 'yajl'
         }
         return generate_backend_specific_tests(module, classname, method_suffix,
                                                members=members, *_bases)