Skip to content

Commit 0afcdf1

Browse files
serhiy-storchakaserhiy-storchaka
committed
[3.13] pythongh-133767: Fix use-after-free in the unicode-escape decoder with an error handler (pythonGH-129648)
If the error handler is used, a new bytes object is created to set as the object attribute of UnicodeDecodeError, and that bytes object then replaces the original data. A pointer to the decoded data will became invalid after destroying that temporary bytes object. So we need other way to return the first invalid escape from _PyUnicode_DecodeUnicodeEscapeInternal(). _PyBytes_DecodeEscape() does not have such issue, because it does not use the error handlers registry, but it should be changed for compatibility with _PyUnicode_DecodeUnicodeEscapeInternal(). (cherry picked from commit 9f69a58) Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>
1 parent d9d8e9a commit 0afcdf1

File tree

8 files changed

+160
-63
lines changed

8 files changed

+160
-63
lines changed

‎Include/internal/pycore_bytesobject.h‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,9 @@ extern PyObject* _PyBytes_FromHex(
2020

2121
// Helper for PyBytes_DecodeEscape that detects invalid escape chars.
2222
// Export for test_peg_generator.
23-
PyAPI_FUNC(PyObject*) _PyBytes_DecodeEscape(const char *, Py_ssize_t,
24-
const char *, const char **);
23+
PyAPI_FUNC(PyObject*) _PyBytes_DecodeEscape2(const char *, Py_ssize_t,
24+
const char *,
25+
int *, const char **);
2526

2627

2728
// Substring Search.

‎Include/internal/pycore_unicodeobject.h‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -142,14 +142,18 @@ extern PyObject* _PyUnicode_DecodeUnicodeEscapeStateful(
142142
// Helper for PyUnicode_DecodeUnicodeEscape that detects invalid escape
143143
// chars.
144144
// Export for test_peg_generator.
145-
PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscapeInternal(
145+
PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscapeInternal2(
146146
const char *string, /* Unicode-Escape encoded string */
147147
Py_ssize_t length, /* size of string */
148148
const char *errors, /* error handling */
149149
Py_ssize_t *consumed, /* bytes consumed */
150-
const char **first_invalid_escape); /* on return, points to first
151-
invalid escaped char in
152-
string. */
150+
int *first_invalid_escape_char, /* on return, if not -1, contain the first
151+
invalid escaped char (<= 0xff) or invalid
152+
octal escape (> 0xff) in string. */
153+
const char **first_invalid_escape_ptr); /* on return, if not NULL, may
154+
point to the first invalid escaped
155+
char in string.
156+
May be NULL if errors is not NULL. */
153157

154158
/* --- Raw-Unicode-Escape Codecs ---------------------------------------------- */
155159

‎Lib/test/test_codeccallbacks.py‎

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import codecs
22
import html.entities
33
import itertools
4+
import re
45
import sys
56
import unicodedata
67
import unittest
@@ -1124,7 +1125,7 @@ def test_bug828737(self):
11241125
text = 'abc<def>ghi'*n
11251126
text.translate(charmap)
11261127

1127-
def test_mutatingdecodehandler(self):
1128+
def test_mutating_decode_handler(self):
11281129
baddata = [
11291130
("ascii", b"\xff"),
11301131
("utf-7", b"++"),
@@ -1159,6 +1160,42 @@ def mutating(exc):
11591160
for (encoding, data) in baddata:
11601161
self.assertEqual(data.decode(encoding, "test.mutating"), "\u4242")
11611162

1163+
def test_mutating_decode_handler_unicode_escape(self):
1164+
decode = codecs.unicode_escape_decode
1165+
def mutating(exc):
1166+
if isinstance(exc, UnicodeDecodeError):
1167+
r = data.get(exc.object[:exc.end])
1168+
if r is not None:
1169+
exc.object = r[0] + exc.object[exc.end:]
1170+
return ('\u0404', r[1])
1171+
raise AssertionError("don't know how to handle %r" % exc)
1172+
1173+
codecs.register_error('test.mutating2', mutating)
1174+
data = {
1175+
br'\x0': (b'\\', 0),
1176+
br'\x3': (b'xxx\\', 3),
1177+
br'\x5': (b'x\\', 1),
1178+
}
1179+
def check(input, expected, msg):
1180+
with self.assertWarns(DeprecationWarning) as cm:
1181+
self.assertEqual(decode(input, 'test.mutating2'), (expected, len(input)))
1182+
self.assertIn(msg, str(cm.warning))
1183+
1184+
check(br'\x0n\z', '\u0404\n\\z', r"invalid escape sequence '\z'")
1185+
check(br'\x0n\501', '\u0404\n\u0141', r"invalid octal escape sequence '\501'")
1186+
check(br'\x0z', '\u0404\\z', r"invalid escape sequence '\z'")
1187+
1188+
check(br'\x3n\zr', '\u0404\n\\zr', r"invalid escape sequence '\z'")
1189+
check(br'\x3zr', '\u0404\\zr', r"invalid escape sequence '\z'")
1190+
check(br'\x3z5', '\u0404\\z5', r"invalid escape sequence '\z'")
1191+
check(memoryview(br'\x3z5x')[:-1], '\u0404\\z5', r"invalid escape sequence '\z'")
1192+
check(memoryview(br'\x3z5xy')[:-2], '\u0404\\z5', r"invalid escape sequence '\z'")
1193+
1194+
check(br'\x5n\z', '\u0404\n\\z', r"invalid escape sequence '\z'")
1195+
check(br'\x5n\501', '\u0404\n\u0141', r"invalid octal escape sequence '\501'")
1196+
check(br'\x5z', '\u0404\\z', r"invalid escape sequence '\z'")
1197+
check(memoryview(br'\x5zy')[:-1], '\u0404\\z', r"invalid escape sequence '\z'")
1198+
11621199
# issue32583
11631200
def test_crashing_decode_handler(self):
11641201
# better generating one more character to fill the extra space slot

‎Lib/test/test_codecs.py‎

Lines changed: 42 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1196,23 +1196,39 @@ def test_escape(self):
11961196
check(br"[\1010]", b"[A0]")
11971197
check(br"[\x41]", b"[A]")
11981198
check(br"[\x410]", b"[A0]")
1199+
1200+
def test_warnings(self):
1201+
decode = codecs.escape_decode
1202+
check = coding_checker(self, decode)
11991203
for i in range(97, 123):
12001204
b = bytes([i])
12011205
if b not in b'abfnrtvx':
1202-
with self.assertWarns(DeprecationWarning):
1206+
with self.assertWarnsRegex(DeprecationWarning,
1207+
r"invalid escape sequence '\\%c'" % i):
12031208
check(b"\\" + b, b"\\" + b)
1204-
with self.assertWarns(DeprecationWarning):
1209+
with self.assertWarnsRegex(DeprecationWarning,
1210+
r"invalid escape sequence '\\%c'" % (i-32)):
12051211
check(b"\\" + b.upper(), b"\\" + b.upper())
1206-
with self.assertWarns(DeprecationWarning):
1212+
with self.assertWarnsRegex(DeprecationWarning,
1213+
r"invalid escape sequence '\\8'"):
12071214
check(br"\8", b"\\8")
12081215
with self.assertWarns(DeprecationWarning):
12091216
check(br"\9", b"\\9")
1210-
with self.assertWarns(DeprecationWarning):
1217+
with self.assertWarnsRegex(DeprecationWarning,
1218+
r"invalid escape sequence '\\\xfa'") as cm:
12111219
check(b"\\\xfa", b"\\\xfa")
12121220
for i in range(0o400, 0o1000):
1213-
with self.assertWarns(DeprecationWarning):
1221+
with self.assertWarnsRegex(DeprecationWarning,
1222+
r"invalid octal escape sequence '\\%o'" % i):
12141223
check(rb'\%o' % i, bytes([i & 0o377]))
12151224

1225+
with self.assertWarnsRegex(DeprecationWarning,
1226+
r"invalid escape sequence '\\z'"):
1227+
self.assertEqual(decode(br'\x\z', 'ignore'), (b'\\z', 4))
1228+
with self.assertWarnsRegex(DeprecationWarning,
1229+
r"invalid octal escape sequence '\\501'"):
1230+
self.assertEqual(decode(br'\x\501', 'ignore'), (b'A', 6))
1231+
12161232
def test_errors(self):
12171233
decode = codecs.escape_decode
12181234
self.assertRaises(ValueError, decode, br"\x")
@@ -2661,24 +2677,40 @@ def test_escape_decode(self):
26612677
check(br"[\x410]", "[A0]")
26622678
check(br"\u20ac", "\u20ac")
26632679
check(br"\U0001d120", "\U0001d120")
2680+
2681+
def test_decode_warnings(self):
2682+
decode = codecs.unicode_escape_decode
2683+
check = coding_checker(self, decode)
26642684
for i in range(97, 123):
26652685
b = bytes([i])
26662686
if b not in b'abfnrtuvx':
2667-
with self.assertWarns(DeprecationWarning):
2687+
with self.assertWarnsRegex(DeprecationWarning,
2688+
r"invalid escape sequence '\\%c'" % i):
26682689
check(b"\\" + b, "\\" + chr(i))
26692690
if b.upper() not in b'UN':
2670-
with self.assertWarns(DeprecationWarning):
2691+
with self.assertWarnsRegex(DeprecationWarning,
2692+
r"invalid escape sequence '\\%c'" % (i-32)):
26712693
check(b"\\" + b.upper(), "\\" + chr(i-32))
2672-
with self.assertWarns(DeprecationWarning):
2694+
with self.assertWarnsRegex(DeprecationWarning,
2695+
r"invalid escape sequence '\\8'"):
26732696
check(br"\8", "\\8")
26742697
with self.assertWarns(DeprecationWarning):
26752698
check(br"\9", "\\9")
2676-
with self.assertWarns(DeprecationWarning):
2699+
with self.assertWarnsRegex(DeprecationWarning,
2700+
r"invalid escape sequence '\\\xfa'") as cm:
26772701
check(b"\\\xfa", "\\\xfa")
26782702
for i in range(0o400, 0o1000):
2679-
with self.assertWarns(DeprecationWarning):
2703+
with self.assertWarnsRegex(DeprecationWarning,
2704+
r"invalid octal escape sequence '\\%o'" % i):
26802705
check(rb'\%o' % i, chr(i))
26812706

2707+
with self.assertWarnsRegex(DeprecationWarning,
2708+
r"invalid escape sequence '\\z'"):
2709+
self.assertEqual(decode(br'\x\z', 'ignore'), ('\\z', 4))
2710+
with self.assertWarnsRegex(DeprecationWarning,
2711+
r"invalid octal escape sequence '\\501'"):
2712+
self.assertEqual(decode(br'\x\501', 'ignore'), ('\u0141', 6))
2713+
26822714
def test_decode_errors(self):
26832715
decode = codecs.unicode_escape_decode
26842716
for c, d in (b'x', 2), (b'u', 4), (b'U', 4):

‎Misc/NEWS.d/next/Security/2025-05-09-20-22-54.gh-issue-133767.kN2i3Q.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Fix use-after-free in the "unicode-escape" decoder with a non-"strict" error
2+
handler.

‎Objects/bytesobject.c‎

Lines changed: 23 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1065,10 +1065,11 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len,
10651065
}
10661066

10671067
/* Unescape a backslash-escaped string. */
1068-
PyObject *_PyBytes_DecodeEscape(const char *s,
1068+
PyObject *_PyBytes_DecodeEscape2(const char *s,
10691069
Py_ssize_t len,
10701070
const char *errors,
1071-
const char **first_invalid_escape)
1071+
int *first_invalid_escape_char,
1072+
const char **first_invalid_escape_ptr)
10721073
{
10731074
int c;
10741075
char *p;
@@ -1082,7 +1083,8 @@ PyObject *_PyBytes_DecodeEscape(const char *s,
10821083
return NULL;
10831084
writer.overallocate = 1;
10841085

1085-
*first_invalid_escape = NULL;
1086+
*first_invalid_escape_char = -1;
1087+
*first_invalid_escape_ptr = NULL;
10861088

10871089
end = s + len;
10881090
while (s < end) {
@@ -1120,9 +1122,10 @@ PyObject *_PyBytes_DecodeEscape(const char *s,
11201122
c = (c<<3) + *s++ - '0';
11211123
}
11221124
if (c > 0377) {
1123-
if (*first_invalid_escape == NULL) {
1124-
*first_invalid_escape = s-3; /* Back up 3 chars, since we've
1125-
already incremented s. */
1125+
if (*first_invalid_escape_char == -1) {
1126+
*first_invalid_escape_char = c;
1127+
/* Back up 3 chars, since we've already incremented s. */
1128+
*first_invalid_escape_ptr = s - 3;
11261129
}
11271130
}
11281131
*p++ = c;
@@ -1163,9 +1166,10 @@ PyObject *_PyBytes_DecodeEscape(const char *s,
11631166
break;
11641167

11651168
default:
1166-
if (*first_invalid_escape == NULL) {
1167-
*first_invalid_escape = s-1; /* Back up one char, since we've
1168-
already incremented s. */
1169+
if (*first_invalid_escape_char == -1) {
1170+
*first_invalid_escape_char = (unsigned char)s[-1];
1171+
/* Back up one char, since we've already incremented s. */
1172+
*first_invalid_escape_ptr = s - 1;
11691173
}
11701174
*p++ = '\\';
11711175
s--;
@@ -1185,17 +1189,18 @@ PyObject *PyBytes_DecodeEscape(const char *s,
11851189
Py_ssize_t Py_UNUSED(unicode),
11861190
const char *Py_UNUSED(recode_encoding))
11871191
{
1188-
const char* first_invalid_escape;
1189-
PyObject *result = _PyBytes_DecodeEscape(s, len, errors,
1190-
&first_invalid_escape);
1192+
int first_invalid_escape_char;
1193+
const char *first_invalid_escape_ptr;
1194+
PyObject *result = _PyBytes_DecodeEscape2(s, len, errors,
1195+
&first_invalid_escape_char,
1196+
&first_invalid_escape_ptr);
11911197
if (result == NULL)
11921198
return NULL;
1193-
if (first_invalid_escape != NULL) {
1194-
unsigned char c = *first_invalid_escape;
1195-
if ('4' <= c && c <= '7') {
1199+
if (first_invalid_escape_char != -1) {
1200+
if (first_invalid_escape_char > 0xff) {
11961201
if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1,
1197-
"invalid octal escape sequence '\\%.3s'",
1198-
first_invalid_escape) < 0)
1202+
"invalid octal escape sequence '\\%o'",
1203+
first_invalid_escape_char) < 0)
11991204
{
12001205
Py_DECREF(result);
12011206
return NULL;
@@ -1204,7 +1209,7 @@ PyObject *PyBytes_DecodeEscape(const char *s,
12041209
else {
12051210
if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1,
12061211
"invalid escape sequence '\\%c'",
1207-
c) < 0)
1212+
first_invalid_escape_char) < 0)
12081213
{
12091214
Py_DECREF(result);
12101215
return NULL;

‎Objects/unicodeobject.c‎

Lines changed: 28 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -6184,21 +6184,24 @@ _PyUnicode_GetNameCAPI(void)
61846184
/* --- Unicode Escape Codec ----------------------------------------------- */
61856185

61866186
PyObject *
6187-
_PyUnicode_DecodeUnicodeEscapeInternal(const char *s,
6187+
_PyUnicode_DecodeUnicodeEscapeInternal2(const char *s,
61886188
Py_ssize_t size,
61896189
const char *errors,
61906190
Py_ssize_t *consumed,
6191-
const char **first_invalid_escape)
6191+
int *first_invalid_escape_char,
6192+
const char **first_invalid_escape_ptr)
61926193
{
61936194
const char *starts = s;
6195+
const char *initial_starts = starts;
61946196
_PyUnicodeWriter writer;
61956197
const char *end;
61966198
PyObject *errorHandler = NULL;
61976199
PyObject *exc = NULL;
61986200
_PyUnicode_Name_CAPI *ucnhash_capi;
61996201

62006202
// so we can remember if we've seen an invalid escape char or not
6201-
*first_invalid_escape = NULL;
6203+
*first_invalid_escape_char = -1;
6204+
*first_invalid_escape_ptr = NULL;
62026205

62036206
if (size == 0) {
62046207
if (consumed) {
@@ -6286,9 +6289,12 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s,
62866289
}
62876290
}
62886291
if (ch > 0377) {
6289-
if (*first_invalid_escape == NULL) {
6290-
*first_invalid_escape = s-3; /* Back up 3 chars, since we've
6291-
already incremented s. */
6292+
if (*first_invalid_escape_char == -1) {
6293+
*first_invalid_escape_char = ch;
6294+
if (starts == initial_starts) {
6295+
/* Back up 3 chars, since we've already incremented s. */
6296+
*first_invalid_escape_ptr = s - 3;
6297+
}
62926298
}
62936299
}
62946300
WRITE_CHAR(ch);
@@ -6383,9 +6389,12 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s,
63836389
goto error;
63846390

63856391
default:
6386-
if (*first_invalid_escape == NULL) {
6387-
*first_invalid_escape = s-1; /* Back up one char, since we've
6388-
already incremented s. */
6392+
if (*first_invalid_escape_char == -1) {
6393+
*first_invalid_escape_char = c;
6394+
if (starts == initial_starts) {
6395+
/* Back up one char, since we've already incremented s. */
6396+
*first_invalid_escape_ptr = s - 1;
6397+
}
63896398
}
63906399
WRITE_ASCII_CHAR('\\');
63916400
WRITE_CHAR(c);
@@ -6430,18 +6439,19 @@ _PyUnicode_DecodeUnicodeEscapeStateful(const char *s,
64306439
const char *errors,
64316440
Py_ssize_t *consumed)
64326441
{
6433-
const char *first_invalid_escape;
6434-
PyObject *result = _PyUnicode_DecodeUnicodeEscapeInternal(s, size, errors,
6442+
int first_invalid_escape_char;
6443+
const char *first_invalid_escape_ptr;
6444+
PyObject *result = _PyUnicode_DecodeUnicodeEscapeInternal2(s, size, errors,
64356445
consumed,
6436-
&first_invalid_escape);
6446+
&first_invalid_escape_char,
6447+
&first_invalid_escape_ptr);
64376448
if (result == NULL)
64386449
return NULL;
6439-
if (first_invalid_escape != NULL) {
6440-
unsigned char c = *first_invalid_escape;
6441-
if ('4' <= c && c <= '7') {
6450+
if (first_invalid_escape_char != -1) {
6451+
if (first_invalid_escape_char > 0xff) {
64426452
if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1,
6443-
"invalid octal escape sequence '\\%.3s'",
6444-
first_invalid_escape) < 0)
6453+
"invalid octal escape sequence '\\%o'",
6454+
first_invalid_escape_char) < 0)
64456455
{
64466456
Py_DECREF(result);
64476457
return NULL;
@@ -6450,7 +6460,7 @@ _PyUnicode_DecodeUnicodeEscapeStateful(const char *s,
64506460
else {
64516461
if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1,
64526462
"invalid escape sequence '\\%c'",
6453-
c) < 0)
6463+
first_invalid_escape_char) < 0)
64546464
{
64556465
Py_DECREF(result);
64566466
return NULL;

0 commit comments

Comments
 (0)