Initial implementation of kvitems functionality

This new functionality, suggested in #18, allows users to iterate over
(key, value) pairs representing object members for objects with a given
prefix rather than iterating over the objects themselves. This opens up
the possibility of iterating not only over big collections of objects,
but over big objects themselves as well, without exhausting system
memory.

This is a feature that users have required for a time now (see #18 and
isagalaev#62), so it makes sense to
offer it out of the box.

Signed-off-by: Rodrigo Tobar <rtobar@icrar.org>

ijson/backends/python.py

@@ -183,3 +183,10 @@ def items(file, prefix, map_type=None, **kwargs):
     Backend-specific wrapper for ijson.common.items.
     '''
     return common.items(parse(file, **kwargs), prefix, map_type=map_type)
+
+
+def kvitems(file, prefix, map_type=None, **kwargs):
+    '''
+    Backend-specific wrapper for ijson.common.kvitems.
+    '''
+    return common.kvitems(parse(file, **kwargs), prefix, map_type=map_type)

ijson/backends/yajl.py

@@ -113,3 +113,9 @@ def items(file, prefix, map_type=None, **kwargs):
     Backend-specific wrapper for ijson.common.items.
     '''
     return common.items(parse(compat.bytes_reader(file), **kwargs), prefix, map_type=map_type)
+
+def kvitems(file, prefix, map_type=None, **kwargs):
+    '''
+    Backend-specific wrapper for ijson.common.kvitems.
+    '''
+    return common.kvitems(parse(file, **kwargs), prefix, map_type=map_type)

ijson/backends/yajl2.py

@@ -117,3 +117,9 @@ def items(file, prefix, map_type=None, **kwargs):
     Backend-specific wrapper for ijson.common.items.
     '''
     return common.items(parse(compat.bytes_reader(file), **kwargs), prefix, map_type=map_type)
+
+def kvitems(file, prefix, map_type=None, **kwargs):
+    '''
+    Backend-specific wrapper for ijson.common.kvitems.
+    '''
+    return common.kvitems(parse(file, **kwargs), prefix, map_type=map_type)

ijson/backends/yajl2_c.py

@@ -24,3 +24,9 @@ def parse(file, **kwargs):
 def items(file, prefix, map_type=None, **kwargs):
     f = compat.bytes_reader(file)
     return _yajl2.items(prefix, f.read, decimal.Decimal, common.JSONError, common.IncompleteJSONError, map_type, **kwargs)
+
+def kvitems(file, prefix, map_type=None, **kwargs):
+    '''
+    Backend-specific wrapper for ijson.common.kvitems.
+    '''
+    return common.kvitems(parse(file, **kwargs), prefix, map_type=map_type)

ijson/backends/yajl2_cffi.py

@@ -242,3 +242,9 @@ def items(file, prefix, map_type=None, **kwargs):
     Backend-specific wrapper for ijson.common.items.
     '''
     return common.items(parse(compat.bytes_reader(file), **kwargs), prefix, map_type=map_type)
+
+def kvitems(file, prefix, map_type=None, **kwargs):
+    '''
+    Backend-specific wrapper for ijson.common.kvitems.
+    '''
+    return common.kvitems(parse(file, **kwargs), prefix, map_type=map_type)

ijson/common.py

@@ -152,6 +152,29 @@ def items(prefixed_events, prefix, map_type=None):
     except StopIteration:
         pass
 
+def kvitems(prefixed_events, prefix, map_type=None):
+    '''
+    An iterator returning (key, value) pairs constructed from the events
+    under a given prefix. The prefix should point to JSON objects
+    '''
+    builder = None
+    key = None
+    for current, event, value in prefixed_events:
+        if current == prefix and event == 'map_key':  # found new object at prefix
+            if builder:
+                yield key, builder.value
+            key = value
+            builder = ObjectBuilder(map_type=map_type)
+            if prefix:
+                object_prefix = '.'.join([prefix, key])
+            else:
+                object_prefix = key
+        elif builder and current == prefix and event == 'end_map':
+            yield key, builder.value
+            builder = None
+        elif builder and current.startswith(object_prefix):  # while at this key, build the object
+            builder.event(event, value)
+
 
 def number(str_value):
     '''

tests.py

@@ -64,6 +64,23 @@
         }
     ]
 }
+JSON_KVITEMS = [
+    ("null", None),
+    ("boolean", False),
+    ("true", True),
+    ("integer", 0),
+    ("double", Decimal("0.5")),
+    ("exponent", 1e+2),
+    ("long", 10000000000),
+    ("string", "строка - тест"),
+    ("ñandú", None),
+    ("meta", [[1], {}]),
+    ("meta", {"key": "value"}),
+    ("meta", None)
+]
+JSON_KVITEMS_META = [
+    ('key', 'value')
+]
 JSON_EVENTS = [
     ('start_map', None),
         ('map_key', 'docs'),
@@ -261,6 +278,33 @@ def test_items_twodictlevels(self):
         self.assertEqual(2, len(ids))
         self.assertListEqual([-2,-1], sorted(ids))
 
+    def test_kvitems(self):
+        kvitems = list(self.backend.kvitems(BytesIO(JSON), 'docs.item'))
+        self.assertEqual(JSON_KVITEMS, kvitems)
+
+    def test_kvitems_toplevel(self):
+        kvitems = list(self.backend.kvitems(BytesIO(JSON), ''))
+        self.assertEqual(1, len(kvitems))
+        key, value = kvitems[0]
+        self.assertEqual('docs', key)
+        self.assertEqual(JSON_OBJECT['docs'], value)
+
+    def test_kvitems_empty(self):
+        kvitems = list(self.backend.kvitems(BytesIO(JSON), 'docs'))
+        self.assertEqual([], kvitems)
+
+    def test_kvitems_twodictlevels(self):
+        f = BytesIO(b'{"meta":{"view":{"columns":[{"id": -1}, {"id": -2}]}}}')
+        view = list(self.backend.kvitems(f, 'meta.view'))
+        self.assertEqual(1, len(view))
+        key, value = view[0]
+        self.assertEqual('columns', key)
+        self.assertEqual([{'id': -1}, {'id': -2}], value)
+
+    def test_kvitems_different_underlying_types(self):
+        kvitems = list(self.backend.kvitems(BytesIO(JSON), 'docs.item.meta'))
+        self.assertEqual(JSON_KVITEMS_META, kvitems)
+
     def test_multiple_values(self):
         if not self.supports_multiple_values:
             return