Skip to content

Commit

Permalink
Fix crash when dealing with surrogate pairs
Browse files Browse the repository at this point in the history
This addresses the "core dump" part of issue #81, but it's not yet clear what should
happen: simplejson handles them without problems, while ujson rejects them... Given that
AFAICT rapidjson cannot decode them (it raises `kParseErrorStringUnicodeSurrogateInvalid`)
it seems reasonable to reject them at encode time as well.
  • Loading branch information
lelit committed Sep 29, 2017
1 parent 945cd72 commit 6f12323
Show file tree
Hide file tree
Showing 2 changed files with 28 additions and 0 deletions.
6 changes: 6 additions & 0 deletions rapidjson.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1675,6 +1675,8 @@ dumps_internal(
else if (PyUnicode_Check(object)) {
Py_ssize_t l;
char* s = PyUnicode_AsUTF8AndSize(object, &l);
if (s == NULL)
return NULL;
writer->String(s, l);
}
else if (PyList_Check(object)) {
Expand Down Expand Up @@ -1712,6 +1714,8 @@ dumps_internal(
if (PyUnicode_Check(key)) {
Py_ssize_t l;
char* key_str = PyUnicode_AsUTF8AndSize(key, &l);
if (key_str == NULL)
goto error;
stack.push_back(WriterContext(NULL, 0, item, false, nextLevel));
stack.push_back(WriterContext(key_str, l, NULL, false, nextLevel));
}
Expand All @@ -1728,6 +1732,8 @@ dumps_internal(
if (PyUnicode_Check(key)) {
Py_ssize_t l;
char* key_str = PyUnicode_AsUTF8AndSize(key, &l);
if (key_str == NULL)
goto error;
items.push_back(DictItem(key_str, l, item));
}
else if (!skipKeys) {
Expand Down
22 changes: 22 additions & 0 deletions tests/test_unicode.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,25 @@ def test_unicode(u, dumps, loads):
assert ju == js
assert ju.lower() == json.dumps(u).lower()
assert dumps(u, ensure_ascii=False) == json.dumps(u, ensure_ascii=False)


@pytest.mark.unit
@pytest.mark.parametrize('o', [
"\ud80d",
{"foo": "\ud80d"},
{"\ud80d": "foo"},
])
def test_dump_surrogate(o, dumps):
with pytest.raises(UnicodeEncodeError, match="surrogates not allowed"):
dumps(o)


@pytest.mark.unit
@pytest.mark.parametrize('j', [
'"\\ud80d"',
'{"foo": "\\ud80d"}',
'{"\\ud80d": "foo"}',
])
def test_load_surrogate(j, loads):
with pytest.raises(ValueError, match="surrogate pair in string is invalid"):
loads(j)

0 comments on commit 6f12323

Please sign in to comment.