Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Adding tests and bugfixes.

  • Loading branch information...
commit 3cc5dcdb43905f9b6bb3dce9589f19d4ccb999de 1 parent 40df7e7
@mikeal authored
Showing with 106 additions and 22 deletions.
  1. +72 −22 lazyjson.py
  2. +34 −0 test_lists.py
View
94 lazyjson.py
@@ -9,30 +9,44 @@
import json
parser_function = json.loads
-numbers = set(('0','1','2','3','4','5','6','7','8','9',))
+numbers = ['0','1','2','3','4','5','6','7','8','9']
-def scan_to_next(s, index, ignore=[' ']):
- while s[index] not in ignore:
+def scan_to_next(s, index):
+ while s[index] == ' ':
index += 1
return s[index], index
control_map = {'{':'}','"':'"','[':']'}
+control_map.update(dict([(x,',') for x in numbers]))
+
+start_control = set(('{','[',))
+end_control = set(('}',']',))
def scan_to_end(s, index, control=',', end_index=None):
- control_queue = []
+ current_control = None
+ in_string = False
index -= 1
x = 'fghjhvcvbnmmnbvcvbnm'
- while x != control and len(control_queue) is not 0:
+ while x != control or current_control is not None:
index += 1
if end_index is not None and index > end_index:
e = IndexError("Reached end_index without finding control character.")
e.index = index
raise e
x = s[index]
- if x in control_map:
- control_queue.append()
- elif len(control_queue) is not 0 and x == control_map[control_queue[-1]]:
- control_queue.pop()
+
+ if not in_string:
+ if current_control is None:
+ if x == '"':
+ in_string = True
+ elif x in start_control:
+ current_control = x
+ elif x in end_control and x == control_map[current_control]:
+ current_control = None
+ else:
+ if x == '"':
+ in_string = True
+
return index
def find_not_escaped(string, sub, index, escape='\\'):
@@ -71,19 +85,37 @@ def __init__(self, parser, start_index, end_index=None):
self.index_buffer_map = {} # {0:(56,190,)}
self.internal_index_map = {} # {0:'example'}
+ def to_json(self, stream):
+ encoded_blocks = sorted([(self.index_buffer_map[k], value) for k, value in
+ self.internal_index_map.items()])
+ if self.end_index is None:
+ self._set_end_index()
+ if len(encoded_blocks) is not 0:
+ i = self.start_index
+ for (s, e,), obj in encoded_blocks:
+ #self.parser.buffer[i:s+1]
+ stream.write(self.parser.buffer[i:s])
+ _dumps(obj, stream)
+ i = e
+ stream.write(self.parser.buffer[i:self.end_index+1])
+ else:
+ stream.write(self.parser.buffer[self.start_index:self.end_index+1])
+
def _find_item(self, index):
if self.end_index is None:
self._set_end_index()
i = self.start_index
for x in range(index):
- i = scan_to_end(self.parser.buffer, i, ',', self.end_index)
+ i = scan_to_end(self.parser.buffer, i+1, ',', self.end_index)
- s, start_index = scan_to_next(self.parser.buffer, i)
- end_index = scan_to_end(self.parser.buffer, start_index,
+ s, start_index = scan_to_next(self.parser.buffer, i+1)
+ end_index = scan_to_end(self.parser.buffer, start_index+1,
control_map[self.parser.buffer[start_index]])
-
+
obj = type_map[s](self.parser,start_index,end_index)
+ if type(obj) is str or type(obj) is unicode:
+ end_index += 1
self.index_buffer_map[index] = (start_index, end_index,)
return obj
@@ -103,9 +135,9 @@ def _set_end_index(self):
i = self.start_index
while self.end_index is None:
try:
- self.end_index = scan_to_end(self.parser.buffer, i, ']')
+ self.end_index = scan_to_end(self.parser.buffer, i+1, ']')
except IndexError, e:
- self.parser.buffer.read_chunk()
+ self.parser.read_chunk()
i = e.index
def __len__(self):
@@ -117,7 +149,7 @@ def __len__(self):
self.unparsed_length = 0
while working:
try:
- scan_to_end(self.parser.buffer, i, ',')
+ i = scan_to_end(self.parser.buffer, i+1, ',')
self.unparsed_length += 1
except IndexError, e:
working = False
@@ -125,16 +157,18 @@ def __len__(self):
def append(self, value):
self.additional_items.append(value)
-
def parse_string(p, s, e):
return json.loads(p.buffer[s:e+1])
-def parse_dict(b, s, e):
- return LazyDict(b, s, e)
-def parse_list(b, s, e):
- return LazyList(b, s, e)
+def parse_dict(p, s, e=None):
+ return LazyDict(p, s, e)
+def parse_list(p, s, e=None):
+ return LazyList(p, s, e)
+def parse_integer(p, s, e):
+ return json.loads(p.buffer[s:e].replace(' ',''))
-type_map = {'"':parse_string, '{':parse_dict, '[':parse_list}
+type_map = {'"':parse_string, '{':parse_dict, '[':parse_list}
+type_map.update(dict([(x, parse_integer) for x in numbers]))
def loads(stream, readsize=1024, parser_function=None, decoder_function=None):
if not hasattr(stream, 'read'):
@@ -142,4 +176,20 @@ def loads(stream, readsize=1024, parser_function=None, decoder_function=None):
readsize = stream.len
parser = Parser(stream, readsize)
parser.read_chunk()
+ return type_map[parser.buffer[0]](parser, 0, None)
+
+special_types = set((LazyList, LazyDict,))
+
+def _dumps(obj, stream):
+ if type(obj) in special_types:
+ obj.to_json(stream)
+ else:
+ stream.write(json.dumps(obj))
+
+def dumps(obj, stream=None):
+ if stream is None:
+ stream = StringIO()
+ _dumps(obj, stream)
+ if hasattr(stream, 'getvalue'):
+ return stream.getvalue()
View
34 test_lists.py
@@ -0,0 +1,34 @@
+import lazyjson
+import jsonlib2
+
+simple = '["asfdf",2343, 4.5, "asdfasdf","adf" ,"asdsdsa"]'
+
+def test_parse():
+ x = lazyjson.loads(simple)
+ assert x
+
+def test_access():
+ x = lazyjson.loads(simple)
+ assert x[0] == 'asfdf'
+ assert x[3] == 'asdfasdf'
+ assert x[1] == 2343
+ assert x[2] == 4.5
+ assert x[4] == 'adf'
+ assert x[5] == 'asdsdsa'
+
+def test_serialize_unchanged():
+ x = lazyjson.loads(simple)
+ assert lazyjson.dumps(x) == simple
+
+def test_serialize_changed():
+ x = lazyjson.loads(simple)
+ x[0] ; x[1] ; x[2] ; x[3]
+ y = lazyjson.dumps(x)
+ assert y == simple
+
+if __name__ == "__main__":
+ test_parse()
+ test_access()
+ test_serialize_unchanged()
+ test_serialize_changed()
+
Please sign in to comment.
Something went wrong with that request. Please try again.