Permalink
Browse files

Upgraded included simplejson to 2.0.7.

Also changed importing logic to prefer a system-installed version of
simplejson (unless it's an earlier version that does not contian the C
speedups), then the json module from Python 2.6, then the version
shipped with Django.

Fixed #9266.

git-svn-id: http://code.djangoproject.com/svn/django/trunk@9707 bcc190cf-cafb-0310-a4f2-bffc1f526a37
  • Loading branch information...
1 parent 3b489b7 commit a9c2f033cdbef3e53f6d48bb5aadb89ed761e579 @malcolmt malcolmt committed Jan 6, 2009
@@ -1,67 +1,65 @@
-"""
-Iterator based sre token scanner
+"""JSON token scanner
"""
import re
-from re import VERBOSE, MULTILINE, DOTALL
-import sre_parse
-import sre_compile
-import sre_constants
-from sre_constants import BRANCH, SUBPATTERN
+try:
+ from simplejson._speedups import make_scanner as c_make_scanner
+except ImportError:
+ c_make_scanner = None
+
+__all__ = ['make_scanner']
-__all__ = ['Scanner', 'pattern']
+NUMBER_RE = re.compile(
+ r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?',
+ (re.VERBOSE | re.MULTILINE | re.DOTALL))
-FLAGS = (VERBOSE | MULTILINE | DOTALL)
+def py_make_scanner(context):
+ parse_object = context.parse_object
+ parse_array = context.parse_array
+ parse_string = context.parse_string
+ match_number = NUMBER_RE.match
+ encoding = context.encoding
+ strict = context.strict
+ parse_float = context.parse_float
+ parse_int = context.parse_int
+ parse_constant = context.parse_constant
+ object_hook = context.object_hook
-class Scanner(object):
- def __init__(self, lexicon, flags=FLAGS):
- self.actions = [None]
- # Combine phrases into a compound pattern
- s = sre_parse.Pattern()
- s.flags = flags
- p = []
- for idx, token in enumerate(lexicon):
- phrase = token.pattern
- try:
- subpattern = sre_parse.SubPattern(s,
- [(SUBPATTERN, (idx + 1, sre_parse.parse(phrase, flags)))])
- except sre_constants.error:
- raise
- p.append(subpattern)
- self.actions.append(token)
+ def _scan_once(string, idx):
+ try:
+ nextchar = string[idx]
+ except IndexError:
+ raise StopIteration
- s.groups = len(p) + 1 # NOTE(guido): Added to make SRE validation work
- p = sre_parse.SubPattern(s, [(BRANCH, (None, p))])
- self.scanner = sre_compile.compile(p)
+ if nextchar == '"':
+ return parse_string(string, idx + 1, encoding, strict)
+ elif nextchar == '{':
+ return parse_object((string, idx + 1), encoding, strict, _scan_once, object_hook)
+ elif nextchar == '[':
+ return parse_array((string, idx + 1), _scan_once)
+ elif nextchar == 'n' and string[idx:idx + 4] == 'null':
+ return None, idx + 4
+ elif nextchar == 't' and string[idx:idx + 4] == 'true':
+ return True, idx + 4
+ elif nextchar == 'f' and string[idx:idx + 5] == 'false':
+ return False, idx + 5
- def iterscan(self, string, idx=0, context=None):
- """
- Yield match, end_idx for each match
- """
- match = self.scanner.scanner(string, idx).match
- actions = self.actions
- lastend = idx
- end = len(string)
- while True:
- m = match()
- if m is None:
- break
- matchbegin, matchend = m.span()
- if lastend == matchend:
- break
- action = actions[m.lastindex]
- if action is not None:
- rval, next_pos = action(m, context)
- if next_pos is not None and next_pos != matchend:
- # "fast forward" the scanner
- matchend = next_pos
- match = self.scanner.scanner(string, matchend).match
- yield rval, matchend
- lastend = matchend
+ m = match_number(string, idx)
+ if m is not None:
+ integer, frac, exp = m.groups()
+ if frac or exp:
+ res = parse_float(integer + (frac or '') + (exp or ''))
+ else:
+ res = parse_int(integer)
+ return res, m.end()
+ elif nextchar == 'N' and string[idx:idx + 3] == 'NaN':
+ return parse_constant('NaN'), idx + 3
+ elif nextchar == 'I' and string[idx:idx + 8] == 'Infinity':
+ return parse_constant('Infinity'), idx + 8
+ elif nextchar == '-' and string[idx:idx + 9] == '-Infinity':
+ return parse_constant('-Infinity'), idx + 9
+ else:
+ raise StopIteration
+ return _scan_once
-def pattern(pattern, flags=FLAGS):
- def decorator(fn):
- fn.pattern = pattern
- fn.regex = re.compile(pattern, flags)
- return fn
- return decorator
+make_scanner = c_make_scanner or py_make_scanner
@@ -1,23 +1,14 @@
-r"""
-Using simplejson from the shell to validate and
+r"""Using simplejson from the shell to validate and
pretty-print::
-
- $ echo '{"json":"obj"}' | python -msimplejson
+
+ $ echo '{"json":"obj"}' | python -msimplejson.tool
{
"json": "obj"
}
- $ echo '{ 1.2:3.4}' | python -msimplejson
+ $ echo '{ 1.2:3.4}' | python -msimplejson.tool
Expecting property name: line 1 column 2 (char 2)
-
-Note that the JSON produced by this module's default settings
-is a subset of YAML, so it may be used as a serializer for that as well.
"""
-import django.utils.simplejson
-
-#
-# Pretty printer:
-# curl http://mochikit.com/examples/ajax_tables/domains.json | python -msimplejson.tool
-#
+from django.utils import simplejson
def main():
import sys
@@ -162,11 +162,17 @@ For example::
json_serializer = serializers.get_serializer("json")()
json_serializer.serialize(queryset, ensure_ascii=False, stream=response)
-The Django source code includes the simplejson_ module. Be aware that if you're
-serializing using that module directly, not all Django output can be passed
-unmodified to simplejson. In particular, :ref:`lazy translation objects
-<lazy-translations>` need a `special encoder`_ written for them. Something like
-this will work::
+The Django source code includes the simplejson_ module. However, if you're
+using Python 2.6 (which includes a builtin version of the module), Django will
+use the builtin ``json`` module automatically. If you have a system installed
+version that includes the C-based speedup extension, or your system version is
+more recent than the version shipped with Django (currently, 2.0.7), the
+system version will be used instead of the version included with Django.
+
+Be aware that if you're serializing using that module directly, not all Django
+output can be passed unmodified to simplejson. In particular, :ref:`lazy
+translation objects <lazy-translations>` need a `special encoder`_ written for
+them. Something like this will work::
from django.utils.functional import Promise
from django.utils.encoding import force_unicode
@@ -178,3 +184,4 @@ this will work::
return obj
.. _special encoder: http://svn.red-bean.com/bob/simplejson/tags/simplejson-1.7/docs/index.html
+

0 comments on commit a9c2f03

Please sign in to comment.