Memory Footprint Resolved

petl/io/json.py

@@ -1,17 +1,15 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, print_function, division
 
-
 # standard library dependencies
 import io
 import json
 from json.encoder import JSONEncoder
-from petl.compat import PY2
-
 
+from petl.compat import PY2
+from petl.io.sources import read_source_from_arg, write_source_from_arg
 # internal dependencies
 from petl.util.base import data, Table, dicts as _dicts, iterpeek
-from petl.io.sources import read_source_from_arg, write_source_from_arg
 
 
 def fromjson(source, *args, **kwargs):
@@ -91,7 +89,6 @@ def fromjson(source, *args, **kwargs):
 
 
 class JsonView(Table):
-
     def __init__(self, source, *args, **kwargs):
         self.source = source
         self.missing = kwargs.pop('missing', None)
@@ -109,12 +106,13 @@ def __iter__(self):
                                      write_through=True)
             try:
                 if self.lines:
-                    dicts = [json.loads(jline) for jline in f.read().splitlines()]
+                    for row in iterjlines(f, self.header, self.missing):
+                        yield row
                 else:
                     dicts = json.load(f, *self.args, **self.kwargs)
-                for row in iterdicts(dicts, self.header, self.sample,
-                                     self.missing):
-                    yield row
+                    for row in iterdicts(dicts, self.header, self.sample,
+                                         self.missing):
+                        yield row
             finally:
                 if not PY2:
                     f.detach()
@@ -173,6 +171,22 @@ def __iter__(self):
         return iterdicts(self.dicts, self.header, self.sample, self.missing)
 
 
+def iterjlines(f, header, missing):
+    it = iter(f)
+
+    if header is None:
+        header = list()
+        peek, it = iterpeek(it, 1)
+        json_obj = json.loads(peek)
+        if hasattr(json_obj, 'keys'):
+            header += [k for k in json_obj.keys() if k not in header]
+    yield tuple(header)
+
+    for o in it:
+        json_obj = json.loads(o)
+        yield tuple(json_obj[f] if f in json_obj else missing for f in header)
+
+
 def iterdicts(dicts, header, sample, missing):
     it = iter(dicts)