Permalink
Browse files

Initial commit

  • Loading branch information...
0 parents commit 89196ca6cbe3bfa7f6117b5b5eb825266ae1a701 Ivan Sagalaev committed Aug 26, 2010
Showing with 169 additions and 0 deletions.
  1. +24 −0 LICENSE.txt
  2. +3 −0 MANIFEST.in
  3. +32 −0 README.txt
  4. +1 −0 ijson/__init__.py
  5. +6 −0 ijson/lib.py
  6. +89 −0 ijson/parse.py
  7. +14 −0 setup.py
24 LICENSE.txt
@@ -0,0 +1,24 @@
+Copyright (c) 2010, Ivan Sagalaev
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ * Neither the name "ijson" nor the names of its contributors
+ may be used to endorse or promote products derived from this software
+ without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY
+EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
3 MANIFEST.in
@@ -0,0 +1,3 @@
+include *.txt
+
+
32 README.txt
@@ -0,0 +1,32 @@
+=====
+ijson
+=====
+
+Ijson is a Python wrapper to `YAJL <http://lloyd.github.com/yajl/>`_ which is a
+streaming SAX-like JSON parser. Ijson provides a standard Python iterator
+interface for it.
+
+Usage
+=====
+
+Basic usage::
+
+ from ijson import parse
+
+ f = urlopen('http://.../') # some huge JSON
+ parser = parse(f)
+ while True:
+ event, value = parser.next()
+ if event == 'start_map':
+ while event != 'end_map':
+ event, value = parser.next()
+ if event == 'map_key' and value == 'title':
+ event, value = parser.next()
+ do_something_with(value)
+
+Acknowledgements
+================
+
+Ijson was inspired by `yajl-py <http://pykler.github.com/yajl-py/>`_ wrapper by
+Hatem Nassrat. Though ijson borrows almost nothing from the actual yajl-py code
+it was used as an example of integration with yajl using ctypes.
1 ijson/__init__.py
@@ -0,0 +1 @@
+from ijson.parse import parse
6 ijson/lib.py
@@ -0,0 +1,6 @@
+from ctypes import cdll, util
+
+name = util.find_library('yajl')
+if name is None:
+ raise Exception('YAJL shared object not found.')
+yajl = cdll.LoadLibrary(name)
89 ijson/parse.py
@@ -0,0 +1,89 @@
+from ctypes import Structure, c_uint, c_ubyte, c_int, c_long, c_double, \
+ c_void_p, c_char_p, CFUNCTYPE, POINTER, byref, string_at
+from decimal import Decimal
+
+from ijson.lib import yajl
+
+C_EMPTY = CFUNCTYPE(c_int, c_void_p)
+C_INT = CFUNCTYPE(c_int, c_void_p, c_int)
+C_LONG = CFUNCTYPE(c_int, c_void_p, c_long)
+C_DOUBLE = CFUNCTYPE(c_int, c_void_p, c_double)
+C_STR = CFUNCTYPE(c_int, c_void_p, POINTER(c_ubyte), c_uint)
+
+def number(value):
+ try:
+ return int(value)
+ except ValueError:
+ return Decimal(value)
+
+_callback_data = [
+ # Mapping of JSON parser events to callback C types and value converters.
+ # Used to define the Callbacks structure and actual callback functions
+ # inside the parse function.
+ ('null', C_EMPTY, lambda: None),
+ ('boolean', C_INT, lambda v: bool(v)),
+ ('integer', C_LONG, lambda v, l: int(string_at(v, l))),
+ ('double', C_DOUBLE, lambda v, l: float(string_at(v, l))),
+ ('number', C_STR, lambda v, l: number(string_at(v, l))),
+ ('string', C_STR, lambda v, l: string_at(v, l).decode('utf-8')),
+ ('start_map', C_EMPTY, lambda: None),
+ ('map_key', C_STR, lambda v, l: string_at(v, l)),
+ ('end_map', C_EMPTY, lambda: None),
+ ('start_array', C_EMPTY, lambda: None),
+ ('end_array', C_EMPTY, lambda: None),
+]
+
+class Callbacks(Structure):
+ _fields_ = [(name, type) for name, type, func in _callback_data]
+
+class Config(Structure):
+ _fields_ = [
+ ("allowComments", c_uint),
+ ("checkUTF8", c_uint)
+ ]
+
+YAJL_OK = 0
+YAJL_CANCELLED = 1
+YAJL_INSUFFICIENT_DATA = 2
+YAJL_ERROR = 3
+
+class JSONError(Exception):
+ pass
+
+class ParseCancelledError(JSONError):
+ def __init__(self):
+ super(ParseCancelledError, self).__init__('Parsing cancelled by a callback')
+
+def parse(f, allow_comments=False, check_utf8=False, buf_size=64 * 1024):
+ events = []
+
+ def callback(event, func_type, func):
+ def c_callback(context, *args):
+ events.append((event, func(*args)))
+ return 1
+ return func_type(c_callback)
+
+ yajl.yajl_get_error.restype = c_char_p
+ callbacks = Callbacks(*[callback(*data) for data in _callback_data])
+ config = Config(allow_comments, check_utf8)
+ handle = yajl.yajl_alloc(byref(callbacks), byref(config), None, None)
+ try:
+ while True:
+ buffer = f.read(buf_size)
+ if buffer:
+ result = yajl.yajl_parse(handle, buffer, len(buffer))
+ else:
+ result = yajl.yajl_parse_complete(handle)
+ if not buffer or (result in (YAJL_CANCELLED, YAJL_ERROR)):
+ break
+ for event, value in events:
+ yield event, value
+ events = []
+
+ if result == YAJL_CANCELLED:
+ raise ParseCancelledError()
+ elif result == YAJL_ERROR:
+ error = yajl.yajl_get_error(handle, 1, buffer, len(buffer))
+ raise JSONError(error)
+ finally:
+ yajl.yajl_free(handle)
14 setup.py
@@ -0,0 +1,14 @@
+# -*- coding:utf-8 -*-
+from distutils.core import setup
+
+setup(
+ name = 'ijson',
+ version = '0.1.0',
+ author = 'Ivan Sagalaev',
+ author_email = 'Maniac@SoftwareManiacs.Org',
+ packages = ['ijson'],
+ url = 'https://launchpad.net/ijson',
+ license = 'LICENSE.txt',
+ description = 'A Python wrapper to YAJL providing standard iterator interface to streaming JSON parsing',
+ long_description = open('README.txt').read(),
+)

0 comments on commit 89196ca

Please sign in to comment.