Skip to content

[3.9] bpo-45461: Fix IncrementalDecoder and StreamReader in the "unicode-escape" codec (GH-28939) #28945

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
serhiy-storchaka merged 2 commits into from
Oct 14, 2021
Merged

[3.9] bpo-45461: Fix IncrementalDecoder and StreamReader in the "unicode-escape" codec (GH-28939) #28945

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
110f9ae
[3.9] bpo-45461: Fix IncrementalDecoder and StreamReader in the "unic…
serhiy-storchaka Oct 14, 2021
3c2d151
Regenerate the ABI file.
serhiy-storchaka Oct 14, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23,654 changes: 14,253 additions & 9,401 deletions Doc/data/python3.9.abi
View file
Open in desktop

Large diffs are not rendered by default.

10 changes: 9 additions & 1 deletion Include/cpython/unicodeobject.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -857,12 +857,20 @@ PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF16(

/* --- Unicode-Escape Codecs ---------------------------------------------- */

/* Variant of PyUnicode_DecodeUnicodeEscape that supports partial decoding. */
PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscapeStateful(
const char *string, /* Unicode-Escape encoded string */
Py_ssize_t length, /* size of string */
const char *errors, /* error handling */
Py_ssize_t *consumed /* bytes consumed */
);
/* Helper for PyUnicode_DecodeUnicodeEscape that detects invalid escape
chars. */
PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscape(
PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscapeInternal(
const char *string, /* Unicode-Escape encoded string */
Py_ssize_t length, /* size of string */
const char *errors, /* error handling */
Py_ssize_t *consumed, /* bytes consumed */
const char **first_invalid_escape /* on return, points to first
invalid escaped char in
string. */
Expand Down
9 changes: 5 additions & 4 deletions Lib/encodings/unicode_escape.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,16 @@ class IncrementalEncoder(codecs.IncrementalEncoder):
def encode(self, input, final=False):
return codecs.unicode_escape_encode(input, self.errors)[0]

class IncrementalDecoder(codecs.IncrementalDecoder):
def decode(self, input, final=False):
return codecs.unicode_escape_decode(input, self.errors)[0]
class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
def _buffer_decode(self, input, errors, final):
return codecs.unicode_escape_decode(input, errors, final)

class StreamWriter(Codec,codecs.StreamWriter):
pass

class StreamReader(Codec,codecs.StreamReader):
pass
def decode(self, input, errors='strict'):
return codecs.unicode_escape_decode(input, errors, False)

### encodings module API

Expand Down
44 changes: 43 additions & 1 deletion Lib/test/test_codecs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2327,7 +2327,11 @@ def test_unicode_escape(self):
(r"\x5c\x55\x30\x30\x31\x31\x30\x30\x30\x30", 10))


class UnicodeEscapeTest(unittest.TestCase):
class UnicodeEscapeTest(ReadTest, unittest.TestCase):
encoding = "unicode-escape"

test_lone_surrogates = None

def test_empty(self):
self.assertEqual(codecs.unicode_escape_encode(""), (b"", 0))
self.assertEqual(codecs.unicode_escape_decode(b""), ("", 0))
Expand Down Expand Up @@ -2414,6 +2418,44 @@ def test_decode_errors(self):
self.assertEqual(decode(br"\U00110000", "ignore"), ("", 10))
self.assertEqual(decode(br"\U00110000", "replace"), ("\ufffd", 10))

def test_partial(self):
self.check_partial(
"\x00\t\n\r\\\xff\uffff\U00010000",
[
'',
'',
'',
'\x00',
'\x00',
'\x00\t',
'\x00\t',
'\x00\t\n',
'\x00\t\n',
'\x00\t\n\r',
'\x00\t\n\r',
'\x00\t\n\r\\',
'\x00\t\n\r\\',
'\x00\t\n\r\\',
'\x00\t\n\r\\',
'\x00\t\n\r\\\xff',
'\x00\t\n\r\\\xff',
'\x00\t\n\r\\\xff',
'\x00\t\n\r\\\xff',
'\x00\t\n\r\\\xff',
'\x00\t\n\r\\\xff',
'\x00\t\n\r\\\xff\uffff',
'\x00\t\n\r\\\xff\uffff',
'\x00\t\n\r\\\xff\uffff',
'\x00\t\n\r\\\xff\uffff',
'\x00\t\n\r\\\xff\uffff',
'\x00\t\n\r\\\xff\uffff',
'\x00\t\n\r\\\xff\uffff',
'\x00\t\n\r\\\xff\uffff',
'\x00\t\n\r\\\xff\uffff',
'\x00\t\n\r\\\xff\uffff',
'\x00\t\n\r\\\xff\uffff\U00010000',
]
)

class RawUnicodeEscapeTest(unittest.TestCase):
def test_empty(self):
Expand Down
2 changes: 2 additions & 0 deletions Misc/NEWS.d/next/Library/2021-10-14-00-19-02.bpo-45461.4LB_tJ.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Fix incremental decoder and stream reader in the "unicode-escape" codec.
Previously they failed if the escape sequence was split.
13 changes: 8 additions & 5 deletions Modules/_codecsmodule.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -487,17 +487,20 @@ _codecs_utf_32_ex_decode_impl(PyObject *module, Py_buffer *data,
_codecs.unicode_escape_decode
data: Py_buffer(accept={str, buffer})
errors: str(accept={str, NoneType}) = None
final: bool(accept={int}) = True
/
[clinic start generated code]*/

static PyObject *
_codecs_unicode_escape_decode_impl(PyObject *module, Py_buffer *data,
const char *errors)
/*[clinic end generated code: output=3ca3c917176b82ab input=8328081a3a569bd6]*/
const char *errors, int final)
/*[clinic end generated code: output=b284f97b12c635ee input=6154f039a9f7c639]*/
{
PyObject *decoded = PyUnicode_DecodeUnicodeEscape(data->buf, data->len,
errors);
return codec_tuple(decoded, data->len);
Py_ssize_t consumed = data->len;
PyObject *decoded = _PyUnicode_DecodeUnicodeEscapeStateful(data->buf, data->len,
errors,
final ? NULL : &consumed);
return codec_tuple(decoded, consumed);
}

/*[clinic input]
Expand Down
23 changes: 18 additions & 5 deletions Modules/clinic/_codecsmodule.c.h
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

49 changes: 37 additions & 12 deletions Objects/unicodeobject.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6271,9 +6271,10 @@ PyUnicode_AsUTF16String(PyObject *unicode)
static _PyUnicode_Name_CAPI *ucnhash_CAPI = NULL;

PyObject *
_PyUnicode_DecodeUnicodeEscape(const char *s,
_PyUnicode_DecodeUnicodeEscapeInternal(const char *s,
Py_ssize_t size,
const char *errors,
Py_ssize_t *consumed,
const char **first_invalid_escape)
{
const char *starts = s;
Expand All @@ -6286,6 +6287,9 @@ _PyUnicode_DecodeUnicodeEscape(const char *s,
*first_invalid_escape = NULL;

if (size == 0) {
if (consumed) {
*consumed = 0;
}
_Py_RETURN_UNICODE_EMPTY();
}
/* Escaped strings will always be longer than the resulting
Expand Down Expand Up @@ -6336,7 +6340,7 @@ _PyUnicode_DecodeUnicodeEscape(const char *s,
/* \ - Escapes */
if (s >= end) {
message = "\\ at end of string";
goto error;
goto incomplete;
}
c = (unsigned char) *s++;

Expand Down Expand Up @@ -6390,7 +6394,10 @@ _PyUnicode_DecodeUnicodeEscape(const char *s,
count = 8;
message = "truncated \\UXXXXXXXX escape";
hexescape:
for (ch = 0; count && s < end; ++s, --count) {
for (ch = 0; count; ++s, --count) {
if (s >= end) {
goto incomplete;
}
c = (unsigned char)*s;
ch <<= 4;
if (c >= '0' && c <= '9') {
Expand All @@ -6403,12 +6410,9 @@ _PyUnicode_DecodeUnicodeEscape(const char *s,
ch += c - ('A' - 10);
}
else {
break;
goto error;
}
}
if (count) {
goto error;
}

/* when we get here, ch is a 32-bit unicode character */
if (ch > MAX_UNICODE) {
Expand All @@ -6435,14 +6439,20 @@ _PyUnicode_DecodeUnicodeEscape(const char *s,
}

message = "malformed \\N character escape";
if (s < end && *s == '{') {
if (s >= end) {
goto incomplete;
}
if (*s == '{') {
const char *start = ++s;
size_t namelen;
/* look for the closing brace */
while (s < end && *s != '}')
s++;
if (s >= end) {
goto incomplete;
}
namelen = s - start;
if (namelen && s < end) {
if (namelen) {
/* found a name. look it up in the unicode database */
s++;
ch = 0xffffffff; /* in case 'getcode' messes up */
Expand All @@ -6468,6 +6478,11 @@ _PyUnicode_DecodeUnicodeEscape(const char *s,
continue;
}

incomplete:
if (consumed) {
*consumed = startinpos;
break;
}
error:
endinpos = s-starts;
writer.min_length = end - s + writer.pos;
Expand Down Expand Up @@ -6496,12 +6511,14 @@ _PyUnicode_DecodeUnicodeEscape(const char *s,
}

PyObject *
PyUnicode_DecodeUnicodeEscape(const char *s,
_PyUnicode_DecodeUnicodeEscapeStateful(const char *s,
Py_ssize_t size,
const char *errors)
const char *errors,
Py_ssize_t *consumed)
{
const char *first_invalid_escape;
PyObject *result = _PyUnicode_DecodeUnicodeEscape(s, size, errors,
PyObject *result = _PyUnicode_DecodeUnicodeEscapeInternal(s, size, errors,
consumed,
&first_invalid_escape);
if (result == NULL)
return NULL;
Expand All @@ -6516,6 +6533,14 @@ PyUnicode_DecodeUnicodeEscape(const char *s,
return result;
}

PyObject *
PyUnicode_DecodeUnicodeEscape(const char *s,
Py_ssize_t size,
const char *errors)
{
return _PyUnicode_DecodeUnicodeEscapeStateful(s, size, errors, NULL);
}

/* Return a Unicode-Escape string version of the Unicode object. */

PyObject *
Expand Down
2 changes: 1 addition & 1 deletion Parser/pegen/parse_string.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ decode_unicode_with_escapes(Parser *parser, const char *s, size_t len, Token *t)
s = buf;

const char *first_invalid_escape;
v = _PyUnicode_DecodeUnicodeEscape(s, len, NULL, &first_invalid_escape);
v = _PyUnicode_DecodeUnicodeEscapeInternal(s, len, NULL, NULL, &first_invalid_escape);

if (v != NULL && first_invalid_escape != NULL) {
if (warn_invalid_escape_sequence(parser, *first_invalid_escape, t) < 0) {
Expand Down
2 changes: 1 addition & 1 deletion Python/ast.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4640,7 +4640,7 @@ decode_unicode_with_escapes(struct compiling *c, const node *n, const char *s,
s = buf;

const char *first_invalid_escape;
v = _PyUnicode_DecodeUnicodeEscape(s, len, NULL, &first_invalid_escape);
v = _PyUnicode_DecodeUnicodeEscapeInternal(s, len, NULL, NULL, &first_invalid_escape);

if (v != NULL && first_invalid_escape != NULL) {
if (warn_invalid_escape_sequence(c, n, *first_invalid_escape) < 0) {
Expand Down