Permalink
Browse files

fix edge cases and pre-2.7 compatibility

  • Loading branch information...
1 parent 50d234a commit 4460550c22e5cd4f52fb1f0449c4b1c02ff38fd1 @etrepum etrepum committed Mar 6, 2012
Showing with 108 additions and 49 deletions.
  1. +7 −5 simplejson/__init__.py
  2. +39 −26 simplejson/_speedups.c
  3. +18 −6 simplejson/encoder.py
  4. +44 −12 simplejson/tests/test_bigint_as_string.py
@@ -138,14 +138,14 @@ def _import_c_make_encoder():
use_decimal=True,
namedtuple_as_object=True,
tuple_as_array=True,
- bigint_as_string=False
+ bigint_as_string=False,
)
def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True,
allow_nan=True, cls=None, indent=None, separators=None,
encoding='utf-8', default=None, use_decimal=True,
- namedtuple_as_object=True, tuple_as_array=True, bigint_as_string=False,
- **kw):
+ namedtuple_as_object=True, tuple_as_array=True,
+ bigint_as_string=False, **kw):
"""Serialize ``obj`` as a JSON formatted stream to ``fp`` (a
``.write()``-supporting file-like object).
@@ -194,9 +194,11 @@ def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True,
If *tuple_as_array* is true (default: ``True``),
:class:`tuple` (and subclasses) will be encoded as JSON arrays.
- If bigint_as_string is true (not the default), ints 2**53 and higher
+ If *bigint_as_string* is true (default: ``False``), ints 2**53 and higher
or lower than -2**53 will be encoded as strings. This is to avoid the
- rounding that happens in Javascript otherwise.
+ rounding that happens in Javascript otherwise. Note that this is still a
+ lossy operation that will not round-trip correctly and should be used
+ sparingly.
To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the
``.default()`` method to serialize additional types), specify it with
@@ -105,6 +105,9 @@ static PyMemberDef encoder_members[] = {
{NULL}
};
+static PyObject *
+maybe_quote_bigint(PyObject *encoded, PyObject *obj);
+
static Py_ssize_t
ascii_escape_char(Py_UNICODE c, char *output, Py_ssize_t chars);
static PyObject *
@@ -167,6 +170,35 @@ _is_namedtuple(PyObject *obj);
#define MAX_EXPANSION MIN_EXPANSION
#endif
+static PyObject *
+maybe_quote_bigint(PyObject *encoded, PyObject *obj)
+{
+ static PyObject *big_long = NULL;
+ static PyObject *small_long = NULL;
+ if (big_long == NULL) {
+ big_long = PyLong_FromLongLong(1 << 53);
+ if (big_long == NULL) {
+ Py_DECREF(encoded);
+ return NULL;
+ }
+ }
+ if (small_long == NULL) {
+ small_long = PyLong_FromLongLong(-1 << 53);
+ if (small_long == NULL) {
+ Py_DECREF(encoded);
+ return NULL;
+ }
+ }
+ if (PyObject_RichCompareBool(obj, big_long, Py_GE) ||
+ PyObject_RichCompareBool(obj, small_long, Py_LE)) {
+ PyObject* quoted = PyString_FromFormat("\"%s\"",
+ PyString_AsString(encoded));
+ Py_DECREF(encoded);
+ encoded = quoted;
+ }
+ return encoded;
+}
+
static int
_is_namedtuple(PyObject *obj)
{
@@ -1160,7 +1192,7 @@ _parse_object_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ss
if (rval == NULL)
return NULL;
}
-
+
/* skip whitespace after { */
while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
@@ -1901,7 +1933,7 @@ scanner_init(PyObject *self, PyObject *args, PyObject *kwds)
if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:make_scanner", kwlist, &ctx))
return -1;
-
+
if (s->memo == NULL) {
s->memo = PyDict_New();
if (s->memo == NULL)
@@ -2192,18 +2224,11 @@ encoder_listencode_obj(PyEncoderObject *s, PyObject *rval, PyObject *obj, Py_ssi
else if (PyInt_Check(obj) || PyLong_Check(obj)) {
PyObject *encoded = PyObject_Str(obj);
if (encoded != NULL) {
- if (s->bigint_as_string) {
- int overflow;
- PY_LONG_LONG value = PyLong_AsLongLongAndOverflow(obj, &overflow);
- if (value < 0) {
- value = ~value;
- }
- if (overflow || (value>>53)) {
- PyObject* quoted = PyString_FromFormat("\"%s\"", PyString_AsString(encoded));
- Py_DECREF(encoded);
- encoded = quoted;
- }
- }
+ if (s->bigint_as_string) {
+ encoded = maybe_quote_bigint(encoded, obj);
+ if (encoded == NULL)
+ break;
+ }
rv = _steal_list_append(rval, encoded);
}
}
@@ -2410,18 +2435,6 @@ encoder_listencode_dict(PyEncoderObject *s, PyObject *rval, PyObject *dct, Py_ss
kstr = PyObject_Str(key);
if (kstr == NULL)
goto bail;
- if (s->bigint_as_string) {
- int overflow;
- PY_LONG_LONG value = PyLong_AsLongLongAndOverflow(kstr, &overflow);
- if (value < 0) {
- value = ~value;
- }
- if (overflow || (value>>53)) {
- PyObject* quoted = PyString_FromFormat("\"%s\"", PyString_AsString(kstr));
- Py_DECREF(kstr);
- kstr = quoted;
- }
- }
}
else if (skipkeys) {
Py_DECREF(item);
View
@@ -290,13 +290,15 @@ def floatstr(o, allow_nan=self.allow_nan,
markers, self.default, _encoder, self.indent,
self.key_separator, self.item_separator, self.sort_keys,
self.skipkeys, self.allow_nan, key_memo, self.use_decimal,
- self.namedtuple_as_object, self.tuple_as_array, self.bigint_as_string)
+ self.namedtuple_as_object, self.tuple_as_array,
+ self.bigint_as_string)
else:
_iterencode = _make_iterencode(
markers, self.default, _encoder, self.indent, floatstr,
self.key_separator, self.item_separator, self.sort_keys,
self.skipkeys, _one_shot, self.use_decimal,
- self.namedtuple_as_object, self.tuple_as_array, self.bigint_as_string)
+ self.namedtuple_as_object, self.tuple_as_array,
+ self.bigint_as_string)
try:
return _iterencode(o, 0)
finally:
@@ -332,7 +334,8 @@ def iterencode(self, o, _one_shot=False):
def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
_key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot,
- _use_decimal, _namedtuple_as_object, _tuple_as_array, _bigint_as_string,
+ _use_decimal, _namedtuple_as_object, _tuple_as_array,
+ _bigint_as_string,
## HACK: hand-optimized bytecode; turn globals into locals
False=False,
True=True,
@@ -383,7 +386,10 @@ def _iterencode_list(lst, _current_indent_level):
elif value is False:
yield buf + 'false'
elif isinstance(value, (int, long)):
- yield buf + str(value) if not _bigint_as_string or -(1<<53) <= value < (1<<53) else buf + '"' + str(value) + '"'
+ yield ((buf + str(value))
+ if (not _bigint_as_string or
+ (-1 << 53) < value < (1 << 53))
+ else (buf + '"' + str(value) + '"'))
elif isinstance(value, float):
yield buf + _floatstr(value)
elif _use_decimal and isinstance(value, Decimal):
@@ -470,7 +476,10 @@ def _iterencode_dict(dct, _current_indent_level):
elif value is False:
yield 'false'
elif isinstance(value, (int, long)):
- yield str(value) if not _bigint_as_string or -(1<<53) <= value < (1<<53) else '"' + str(value) + '"'
+ yield (str(value)
+ if (not _bigint_as_string or
+ (-1 << 53) < value < (1 << 53))
+ else ('"' + str(value) + '"'))
elif isinstance(value, float):
yield _floatstr(value)
elif _use_decimal and isinstance(value, Decimal):
@@ -508,7 +517,10 @@ def _iterencode(o, _current_indent_level):
elif o is False:
yield 'false'
elif isinstance(o, (int, long)):
- yield str(o) if not _bigint_as_string or -(1<<53) <= o < (1<<53) else '"' + str(o) + '"'
+ yield (str(o)
+ if (not _bigint_as_string or
+ (-1 << 53) < o < (1 << 53))
+ else ('"' + str(o) + '"'))
elif isinstance(o, float):
yield _floatstr(o)
elif isinstance(o, list):
@@ -3,21 +3,53 @@
import simplejson as json
class TestBigintAsString(TestCase):
- values = [(200, 200), (2**53-1, 9007199254740991), (2**53, '"9007199254740992"'), (2**53+1, '"9007199254740993"'), (-100, -100), (-2**53, -9007199254740992), (-2**53-1, '"-9007199254740993"'), (-2**53+1, -9007199254740991)]
+ values = [(200, 200),
+ ((2 ** 53) - 1, 9007199254740991),
+ ((2 ** 53), '9007199254740992'),
+ ((2 ** 53) + 1, '9007199254740993'),
+ (-100, -100),
+ ((-2 ** 53), '-9007199254740992'),
+ ((-2 ** 53) - 1, '-9007199254740993'),
+ ((-2 ** 53) + 1, -9007199254740991)]
def test_ints(self):
- for value_pair in self.values:
- self.assertEquals('%s' % value_pair[0], json.dumps(value_pair[0]))
- self.assertEquals('%s' % value_pair[1], json.dumps(value_pair[0], bigint_as_string=True))
+ for val, expect in self.values:
+ self.assertEquals(
+ val,
+ json.loads(json.dumps(val)))
+ self.assertEquals(
+ expect,
+ json.loads(json.dumps(val, bigint_as_string=True)))
def test_lists(self):
- for value_pair in self.values:
- l = [value_pair[0], value_pair[0]]
- self.assertEquals('[%s, %s]' % (value_pair[0], value_pair[0]), json.dumps(l))
- self.assertEquals('[%s, %s]' % (value_pair[1], value_pair[1]), json.dumps(l, bigint_as_string=True))
+ for val, expect in self.values:
+ val = [val, val]
+ expect = [expect, expect]
+ self.assertEquals(
+ val,
+ json.loads(json.dumps(val)))
+ self.assertEquals(
+ expect,
+ json.loads(json.dumps(val, bigint_as_string=True)))
def test_dicts(self):
- for value_pair in self.values:
- d = {'value': value_pair[0]}
- self.assertEquals('{"value": %s}' % value_pair[0], json.dumps(d))
- self.assertEquals('{"value": %s}' % value_pair[1], json.dumps(d, bigint_as_string=True))
+ for val, expect in self.values:
+ val = {'k': val}
+ expect = {'k': expect}
+ self.assertEquals(
+ val,
+ json.loads(json.dumps(val)))
+ self.assertEquals(
+ expect,
+ json.loads(json.dumps(val, bigint_as_string=True)))
+
+ def test_dict_keys(self):
+ for val, _ in self.values:
+ expect = {str(val): 'value'}
+ val = {val: 'value'}
+ self.assertEquals(
+ expect,
+ json.loads(json.dumps(val)))
+ self.assertEquals(
+ expect,
+ json.loads(json.dumps(val, bigint_as_string=True)))

0 comments on commit 4460550

Please sign in to comment.