From b923398fbaf64394a6681fe96e9ac1dbc84e4b7c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 19 Dec 2019 21:16:32 +0900 Subject: [PATCH 1/9] bpo-39087: Add PyUnicode_GetUTF8Buffer(). --- Doc/c-api/unicode.rst | 11 +++++++ Include/cpython/unicodeobject.h | 15 ++++++++- Lib/test/test_unicode.py | 28 ++++++++++++++++ .../2019-12-19-21-19-53.bpo-39087.l4A11-.rst | 2 ++ Modules/_testcapimodule.c | 29 +++++++++++++++++ Objects/unicodeobject.c | 32 +++++++++++++++++++ PC/python3.def | 1 + 7 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/C API/2019-12-19-21-19-53.bpo-39087.l4A11-.rst diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 77f123cf1f2c08c..9281491462ccc5f 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -1061,6 +1061,17 @@ These are the UTF-8 codec APIs: raised by the codec. +.. c:function: int PyUnicode_GetUTF8Buffer(PyObject *unicode, const char errors, Py_buffer *view) + + Get a read-only buffer of the UTF-8 encoding of the Unicode object. + Return ``-1`` if an error happened. + + Successful calls to :c:func:`PyUnicode_GetUTF8Buffer` must be paired with + calls to :c:func:`PyBuffer_Release`, similar to :c:func:`PyObject_GetBuffer`. + + .. versionadded:: 3.9 + + .. c:function:: const char* PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *size) Return a pointer to the UTF-8 encoding of the Unicode object, and diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index 54a13e32ba22b55..c5223de19d6020d 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -734,6 +734,19 @@ PyAPI_FUNC(void*) _PyUnicode_AsKind(PyObject *s, unsigned int kind); /* --- Manage the default encoding ---------------------------------------- */ +/* Get a buffer to the UTF-8 encoding of the Unicode object unicode. + Returns -1 on error. + + Successful calls to PyUnicode_GetUTF8Buffer must be paired to + calls to PyBuffer_Release. +*/ + +PyAPI_FUNC(int) PyUnicode_GetUTF8Buffer( + PyObject *unicode, /* Unicode object */ + const char *errors, /* error handling */ + Py_buffer *view /* (out) buffer to the UTF-8 encoding */ + ); + /* Returns a pointer to the default encoding (UTF-8) of the Unicode object unicode and the size of the encoded representation in bytes stored in *size. @@ -751,7 +764,7 @@ PyAPI_FUNC(void*) _PyUnicode_AsKind(PyObject *s, unsigned int kind); *** be removed or changed in the future. *** If you need to access the Unicode object as UTF-8 bytes string, - *** please use PyUnicode_AsUTF8String() instead. + *** please use PyUnicode_GetUTF8Buffer() instead. */ PyAPI_FUNC(const char *) PyUnicode_AsUTF8AndSize( diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 8636f2b0bd50296..f8b81f0d9068d89 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -2830,6 +2830,34 @@ def test_asucs4(self): self.assertEqual(unicode_asucs4(s, len(s), True), s+'\0') self.assertEqual(unicode_asucs4(s, len(s), False), s+'\uffff') + # Test PyUnicode_GetUTF8Buffer() + @support.cpython_only + def test_getutf8buffer(self): + from _testcapi import unicode_getutf8buffer + + ascii = "foo" + c1 = sys.getrefcount(ascii) + mv = unicode_getutf8buffer(ascii) + self.assertEqual(mv.tobytes(), b"foo") + + c2 = sys.getrefcount(ascii) + self.assertEqual(c1 + 1, c2) + + mv.release() + del mv + + c3 = sys.getrefcount(ascii) + self.assertEqual(c1, c3) + + bmp = '\u0100' + bmp2 = '\uffff' + nonbmp = chr(0x10ffff) + + self.assertEqual(unicode_getutf8buffer(bmp).tobytes(), b'\xc4\x80') + self.assertEqual(unicode_getutf8buffer(bmp2).tobytes(), b'\xef\xbf\xbf') + self.assertEqual(unicode_getutf8buffer(nonbmp).tobytes(), b'\xf4\x8f\xbf\xbf') + self.assertRaises(UnicodeEncodeError, unicode_getutf8buffer, 'a\ud800b\udfffc') + # Test PyUnicode_AsUTF8() @support.cpython_only def test_asutf8(self): diff --git a/Misc/NEWS.d/next/C API/2019-12-19-21-19-53.bpo-39087.l4A11-.rst b/Misc/NEWS.d/next/C API/2019-12-19-21-19-53.bpo-39087.l4A11-.rst new file mode 100644 index 000000000000000..89e1a3f2f782435 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2019-12-19-21-19-53.bpo-39087.l4A11-.rst @@ -0,0 +1,2 @@ +Add new :c:func:`PyUnicode_GetUTF8Buffer` API to get UTF-8 encode of the +unicode object without cache or extra allocation. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 943bee6e21e197c..6d7a1b7f0c954bd 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1965,6 +1965,34 @@ unicode_asutf8andsize(PyObject *self, PyObject *args) return Py_BuildValue("(Nn)", result, utf8_len); } +static PyObject * +unicode_getutf8buffer(PyObject *self, PyObject *args) +{ + PyObject *unicode; + const char *errors = NULL; + if(!PyArg_ParseTuple(args, "U|s", &unicode, &errors)) { + return NULL; + } + + Py_buffer buffer; + if (PyUnicode_GetUTF8Buffer(unicode, errors, &buffer) < 0) { + return NULL; + } + + assert(buffer.obj != NULL); + assert(buffer.obj == unicode || PyBytes_CheckExact(buffer.obj)); + + PyObject *result = PyMemoryView_FromBuffer(&buffer); + if (result == NULL) { + PyBuffer_Release(&buffer); + return NULL; + } + // Instead of calling PyBuffer_Release(), move the ownership of the obj + // to the memoryview object. + ((PyMemoryViewObject*)result)->mbuf->master.obj = buffer.obj; + return result; +} + static PyObject * unicode_findchar(PyObject *self, PyObject *args) { @@ -5391,6 +5419,7 @@ static PyMethodDef TestMethods[] = { {"unicode_asucs4", unicode_asucs4, METH_VARARGS}, {"unicode_asutf8", unicode_asutf8, METH_VARARGS}, {"unicode_asutf8andsize", unicode_asutf8andsize, METH_VARARGS}, + {"unicode_getutf8buffer", unicode_getutf8buffer, METH_VARARGS}, {"unicode_findchar", unicode_findchar, METH_VARARGS}, {"unicode_copycharacters", unicode_copycharacters, METH_VARARGS}, {"unicode_encodedecimal", unicode_encodedecimal, METH_VARARGS}, diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 1ec2accdb09f202..a3912d6e7918cfb 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -3996,6 +3996,38 @@ PyUnicode_FSDecoder(PyObject* arg, void* addr) } +int +PyUnicode_GetUTF8Buffer(PyObject *unicode, const char *errors, + Py_buffer *view) +{ + if (!PyUnicode_Check(unicode)) { + PyErr_BadArgument(); + return -1; + } + if (PyUnicode_READY(unicode) == -1) { + return -1; + } + + if (PyUnicode_UTF8(unicode) != NULL) { + return PyBuffer_FillInfo(view, unicode, + PyUnicode_UTF8(unicode), + PyUnicode_UTF8_LENGTH(unicode), + 1, PyBUF_SIMPLE); + } + + PyObject *bytes = _PyUnicode_AsUTF8String(unicode, errors); + if (bytes == NULL) { + return -1; + } + assert(PyBytes_CheckExact(bytes)); + if (PyObject_GetBuffer(bytes, view, PyBUF_SIMPLE) < 0) { + Py_DECREF(bytes); + return -1; + } + return 0; +} + + const char * PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *psize) { diff --git a/PC/python3.def b/PC/python3.def index 4689b777a6933b1..f83b2d3bbc1ed71 100644 --- a/PC/python3.def +++ b/PC/python3.def @@ -694,6 +694,7 @@ EXPORTS PyUnicode_GetDefaultEncoding=python39.PyUnicode_GetDefaultEncoding PyUnicode_GetLength=python39.PyUnicode_GetLength PyUnicode_GetSize=python39.PyUnicode_GetSize + PyUnicode_GetUTF8Buffer=python39.PyUnicode_GetUTF8Buffer PyUnicode_InternFromString=python39.PyUnicode_InternFromString PyUnicode_InternImmortal=python39.PyUnicode_InternImmortal PyUnicode_InternInPlace=python39.PyUnicode_InternInPlace From 3fb62357124aefa76bbfe1e5b0537ba5e3166e1a Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 20 Dec 2019 19:45:34 +0900 Subject: [PATCH 2/9] Update doc --- Doc/c-api/unicode.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 9281491462ccc5f..234eb9706aed65b 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -1069,6 +1069,17 @@ These are the UTF-8 codec APIs: Successful calls to :c:func:`PyUnicode_GetUTF8Buffer` must be paired with calls to :c:func:`PyBuffer_Release`, similar to :c:func:`PyObject_GetBuffer`. + .. note:: + When compared to :c:function:`PyUnicode_AsUTF8String`, this function + does not create a bytes object when the *unicode* is ASCII string or + has UTF-8 cache. + + When compared to :c:function:`PyUnicode_AsUTF8` or + :c:function:`PyUnicode_AsUTF8AndSize`, this function does not cache the + UTF-8 representation of the string in the *unicode* object. + So this API is faster and more efficient when the *unicode* object is + not ASCII string and it is encoded into UTF-8 only once. + .. versionadded:: 3.9 From 3bac143d88e98520e9f629ecbc705027a199aa4a Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 20 Dec 2019 21:07:08 +0900 Subject: [PATCH 3/9] Write tests in C --- Lib/test/test_unicode.py | 32 +++--- Modules/_testcapimodule.c | 208 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 214 insertions(+), 26 deletions(-) diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index f8b81f0d9068d89..a08ed475f4848c5 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -2833,30 +2833,24 @@ def test_asucs4(self): # Test PyUnicode_GetUTF8Buffer() @support.cpython_only def test_getutf8buffer(self): - from _testcapi import unicode_getutf8buffer + from _testcapi import unicode_getutf8buffer, test_unicode_getutf8buffer - ascii = "foo" - c1 = sys.getrefcount(ascii) - mv = unicode_getutf8buffer(ascii) - self.assertEqual(mv.tobytes(), b"foo") - - c2 = sys.getrefcount(ascii) - self.assertEqual(c1 + 1, c2) - - mv.release() - del mv - - c3 = sys.getrefcount(ascii) - self.assertEqual(c1, c3) + # Run tests wrtten in C. Raise an error when test failed. + test_unicode_getutf8buffer() + ascii_ = "foo" bmp = '\u0100' bmp2 = '\uffff' nonbmp = chr(0x10ffff) - - self.assertEqual(unicode_getutf8buffer(bmp).tobytes(), b'\xc4\x80') - self.assertEqual(unicode_getutf8buffer(bmp2).tobytes(), b'\xef\xbf\xbf') - self.assertEqual(unicode_getutf8buffer(nonbmp).tobytes(), b'\xf4\x8f\xbf\xbf') - self.assertRaises(UnicodeEncodeError, unicode_getutf8buffer, 'a\ud800b\udfffc') + surrogates = 'a\ud800b\udfffc' + + self.assertEqual(unicode_getutf8buffer(ascii_), b'foo') + self.assertEqual(unicode_getutf8buffer(bmp), b'\xc4\x80') + self.assertEqual(unicode_getutf8buffer(bmp2), b'\xef\xbf\xbf') + self.assertEqual(unicode_getutf8buffer(nonbmp), b'\xf4\x8f\xbf\xbf') + self.assertRaises(UnicodeEncodeError, unicode_getutf8buffer, surrogates) + self.assertEqual(unicode_getutf8buffer(surrogates, "surrogatepass"), + b'a\xed\xa0\x80b\xed\xbf\xbfc') # Test PyUnicode_AsUTF8() @support.cpython_only diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 6d7a1b7f0c954bd..1f83761daf4111a 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1982,15 +1982,208 @@ unicode_getutf8buffer(PyObject *self, PyObject *args) assert(buffer.obj != NULL); assert(buffer.obj == unicode || PyBytes_CheckExact(buffer.obj)); - PyObject *result = PyMemoryView_FromBuffer(&buffer); - if (result == NULL) { - PyBuffer_Release(&buffer); + PyObject *result = PyBytes_FromStringAndSize(buffer.buf, buffer.len); + PyBuffer_Release(&buffer); + return result; +} + +static PyObject * +test_unicode_getutf8buffer(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + Py_buffer buf; + + // Test 1: ASCII string + PyObject *str = PyUnicode_FromString("hello"); + if (str == NULL) { return NULL; } - // Instead of calling PyBuffer_Release(), move the ownership of the obj - // to the memoryview object. - ((PyMemoryViewObject*)result)->mbuf->master.obj = buffer.obj; - return result; + Py_ssize_t refcnt = Py_REFCNT(str); + + if (PyUnicode_GetUTF8Buffer(str, NULL, &buf) < 0) { + Py_DECREF(str); + if (!PyErr_Occurred()) { + PyErr_Format(TestError, + "PyUnicode_GetUTF8Buffer() returned nonzero " + "without exception set. (%s:%d)", + __FILE__, __LINE__); + } + return NULL; + } + + if (buf.obj != str) { + PyErr_Format(TestError, + "buf.obj must be equal to str. (%s:%d)", + __FILE__, __LINE__); + PyBuffer_Release(&buf); + Py_DECREF(str); + return NULL; + } + + if (buf.len != PyUnicode_GET_LENGTH(str)) { + PyErr_Format(TestError, + "buf.len must be equal to len(str). (%s:%d)", + __FILE__, __LINE__); + PyBuffer_Release(&buf); + Py_DECREF(str); + return NULL; + } + assert(((const char*)buf.buf)[5] == '\0'); + + if ((Py_UCS1*)buf.buf != PyUnicode_1BYTE_DATA(str)) { + PyErr_Format(TestError, + "buf.buf must be equal to PyUnicode_1BYTE_DATA(str). (%s:%d)", + __FILE__, __LINE__); + PyBuffer_Release(&buf); + Py_DECREF(str); + return NULL; + } + + if (refcnt + 1 != Py_REFCNT(str)) { + PyErr_Format(TestError, + "Py_REFCNT(str); expected %zd, got %zd. (%s:%d)", + refcnt + 1, Py_REFCNT(str), + __FILE__, __LINE__); + PyBuffer_Release(&buf); + Py_DECREF(str); + return NULL; + } + + PyBuffer_Release(&buf); + + if (refcnt != Py_REFCNT(str)) { + PyErr_Format(TestError, + "Py_REFCNT(str); expected %zd, got %zd. (%s:%d)", + refcnt, Py_REFCNT(str), + __FILE__, __LINE__); + Py_DECREF(str); + return NULL; + } + + Py_DECREF(str); + + // Test 2: non-ASCII string + + // "hello" in Japanese. len(str)==5, len(str.encode()) == 15. + str = PyUnicode_FromString("\xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1\xe3\x81\xaf"); + if (str == NULL) { + return NULL; + } + refcnt = Py_REFCNT(str); + assert(PyUnicode_GET_LENGTH(str) == 5); + + if (PyUnicode_GetUTF8Buffer(str, NULL, &buf) < 0) { + Py_DECREF(str); + if (!PyErr_Occurred()) { + PyErr_Format(TestError, + "PyUnicode_GetUTF8Buffer() returned nonzero " + "without exception set. (%s:%d)", + __FILE__, __LINE__); + } + Py_DECREF(str); + return NULL; + } + + if (buf.obj == str || !PyBytes_CheckExact(buf.obj)) { + PyErr_Format(TestError, + "buf.obj must be a bytes object, got %R (%s:%d)", + buf.obj, __FILE__, __LINE__); + PyBuffer_Release(&buf); + Py_DECREF(str); + return NULL; + } + + if (buf.len != 15) { + PyErr_Format(TestError, + "Expected buf.len == 15, actual %zd (%s:%d)", + buf.len, __FILE__, __LINE__); + PyBuffer_Release(&buf); + Py_DECREF(str); + return NULL; + } + assert(((const char*)buf.buf)[15] == '\0'); + + if (refcnt != Py_REFCNT(str)) { + PyErr_Format(TestError, + "Py_REFCNT(str) must not be changed. (%s:%d)", + __FILE__, __LINE__); + PyBuffer_Release(&buf); + Py_DECREF(str); + return NULL; + } + + PyBuffer_Release(&buf); + + // Test 3: There is a UTF-8 cache + // Reuse str of the previoss test. + + const char *cache = PyUnicode_AsUTF8(str); + if (cache == NULL) { + return NULL; + } + + if (PyUnicode_GetUTF8Buffer(str, NULL, &buf) < 0) { + Py_DECREF(str); + if (!PyErr_Occurred()) { + PyErr_Format(TestError, + "PyUnicode_GetUTF8Buffer() returned nonzero " + "without exception set. (%s:%d)", + __FILE__, __LINE__); + } + Py_DECREF(str); + return NULL; + } + + if (buf.obj != str) { + PyErr_Format(TestError, + "buf.obj must be equal to str. (%s:%d)", + __FILE__, __LINE__); + PyBuffer_Release(&buf); + Py_DECREF(str); + return NULL; + } + + if (buf.buf != cache) { + PyErr_Format(TestError, + "buf.buf must be equal to the UTF-8 cache (%s:%d)", + __FILE__, __LINE__); + PyBuffer_Release(&buf); + Py_DECREF(str); + return NULL; + } + + if (buf.len != 15) { + PyErr_Format(TestError, + "Expected buf.len == 15, actual %zd (%s:%d)", + buf.len, __FILE__, __LINE__); + PyBuffer_Release(&buf); + Py_DECREF(str); + return NULL; + } + assert(((const char*)buf.buf)[15] == '\0'); + + if (refcnt + 1 != Py_REFCNT(str)) { + PyErr_Format(TestError, + "Py_REFCNT(str); expected %zd, got %zd. (%s:%d)", + refcnt + 1, Py_REFCNT(str), + __FILE__, __LINE__); + PyBuffer_Release(&buf); + Py_DECREF(str); + return NULL; + } + + PyBuffer_Release(&buf); + + if (refcnt != Py_REFCNT(str)) { + PyErr_Format(TestError, + "Py_REFCNT(str); expected %zd, got %zd. (%s:%d)", + refcnt, Py_REFCNT(str), + __FILE__, __LINE__); + Py_DECREF(str); + return NULL; + } + + Py_DECREF(str); + Py_RETURN_NONE; } static PyObject * @@ -5420,6 +5613,7 @@ static PyMethodDef TestMethods[] = { {"unicode_asutf8", unicode_asutf8, METH_VARARGS}, {"unicode_asutf8andsize", unicode_asutf8andsize, METH_VARARGS}, {"unicode_getutf8buffer", unicode_getutf8buffer, METH_VARARGS}, + {"test_unicode_getutf8buffer", test_unicode_getutf8buffer, METH_NOARGS}, {"unicode_findchar", unicode_findchar, METH_VARARGS}, {"unicode_copycharacters", unicode_copycharacters, METH_VARARGS}, {"unicode_encodedecimal", unicode_encodedecimal, METH_VARARGS}, From ec18baca8b873db2763da4c020cf9025345fd17a Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 20 Dec 2019 22:12:52 +0900 Subject: [PATCH 4/9] Add a comment --- Objects/unicodeobject.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index a3912d6e7918cfb..1795d32318acf08 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -4015,6 +4015,8 @@ PyUnicode_GetUTF8Buffer(PyObject *unicode, const char *errors, 1, PyBUF_SIMPLE); } + // Unlike PyUnicode_AsUTF8AndSize(), this function doesn't + // create a UTF-8 cache for speed and efficiency. PyObject *bytes = _PyUnicode_AsUTF8String(unicode, errors); if (bytes == NULL) { return -1; From 2f1f8ac4a0772eff58b7065a68fc79c571a17fcd Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 21 Dec 2019 05:30:47 +0900 Subject: [PATCH 5/9] Don't call test_unicode_getutf8buffer from test_unicode. --- Lib/test/test_unicode.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index a08ed475f4848c5..2f057cff6617d02 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -2833,10 +2833,7 @@ def test_asucs4(self): # Test PyUnicode_GetUTF8Buffer() @support.cpython_only def test_getutf8buffer(self): - from _testcapi import unicode_getutf8buffer, test_unicode_getutf8buffer - - # Run tests wrtten in C. Raise an error when test failed. - test_unicode_getutf8buffer() + from _testcapi import unicode_getutf8buffer ascii_ = "foo" bmp = '\u0100' From 3a27b8b44d8f16caa80ab3563a1f3dde67aa094c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 25 Dec 2019 17:14:38 +0900 Subject: [PATCH 6/9] fixup --- Lib/test/test_unicode.py | 5 ++++- Modules/_testcapimodule.c | 17 +++++------------ Objects/unicodeobject.c | 5 +++-- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 2f057cff6617d02..405e7b26d399c76 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -2833,7 +2833,10 @@ def test_asucs4(self): # Test PyUnicode_GetUTF8Buffer() @support.cpython_only def test_getutf8buffer(self): - from _testcapi import unicode_getutf8buffer + from _testcapi import unicode_getutf8buffer, unicode_test_getutf8buffer + + # Run tests wrtten in C. Raise an error when test failed. + unicode_test_getutf8buffer() ascii_ = "foo" bmp = '\u0100' diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 1f83761daf4111a..c1c2696936133ce 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1988,7 +1988,7 @@ unicode_getutf8buffer(PyObject *self, PyObject *args) } static PyObject * -test_unicode_getutf8buffer(PyObject *self, PyObject *Py_UNUSED(ignored)) +unicode_test_getutf8buffer(PyObject *self, PyObject *Py_UNUSED(ignored)) { Py_buffer buf; @@ -1999,16 +1999,9 @@ test_unicode_getutf8buffer(PyObject *self, PyObject *Py_UNUSED(ignored)) } Py_ssize_t refcnt = Py_REFCNT(str); - if (PyUnicode_GetUTF8Buffer(str, NULL, &buf) < 0) { - Py_DECREF(str); - if (!PyErr_Occurred()) { - PyErr_Format(TestError, - "PyUnicode_GetUTF8Buffer() returned nonzero " - "without exception set. (%s:%d)", - __FILE__, __LINE__); - } - return NULL; - } + // PyUnicode_GetUTF8Buffer() must not fail for ASCII string. + int ret = PyUnicode_GetUTF8Buffer(str, NULL, &buf); + assert(ret == 0); if (buf.obj != str) { PyErr_Format(TestError, @@ -5613,7 +5606,7 @@ static PyMethodDef TestMethods[] = { {"unicode_asutf8", unicode_asutf8, METH_VARARGS}, {"unicode_asutf8andsize", unicode_asutf8andsize, METH_VARARGS}, {"unicode_getutf8buffer", unicode_getutf8buffer, METH_VARARGS}, - {"test_unicode_getutf8buffer", test_unicode_getutf8buffer, METH_NOARGS}, + {"unicode_test_getutf8buffer", unicode_test_getutf8buffer, METH_NOARGS}, {"unicode_findchar", unicode_findchar, METH_VARARGS}, {"unicode_copycharacters", unicode_copycharacters, METH_VARARGS}, {"unicode_encodedecimal", unicode_encodedecimal, METH_VARARGS}, diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 1795d32318acf08..811bb5e955e66c1 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -4008,11 +4008,12 @@ PyUnicode_GetUTF8Buffer(PyObject *unicode, const char *errors, return -1; } - if (PyUnicode_UTF8(unicode) != NULL) { + if (PyUnicode_UTF8(unicode) != NULL + && Py_TYPE(unicode)->tp_as_buffer == NULL) { return PyBuffer_FillInfo(view, unicode, PyUnicode_UTF8(unicode), PyUnicode_UTF8_LENGTH(unicode), - 1, PyBUF_SIMPLE); + /* readonly */ 1, PyBUF_SIMPLE); } // Unlike PyUnicode_AsUTF8AndSize(), this function doesn't From 7cef9a1d15c13052d2a169e409a22faf73d32a36 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 25 Dec 2019 19:45:27 +0900 Subject: [PATCH 7/9] fixup --- Modules/_testcapimodule.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index c1c2696936133ce..4f150cbb9c550b4 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1970,7 +1970,7 @@ unicode_getutf8buffer(PyObject *self, PyObject *args) { PyObject *unicode; const char *errors = NULL; - if(!PyArg_ParseTuple(args, "U|s", &unicode, &errors)) { + if(!PyArg_ParseTuple(args, "O|s", &unicode, &errors)) { return NULL; } @@ -2072,11 +2072,10 @@ unicode_test_getutf8buffer(PyObject *self, PyObject *Py_UNUSED(ignored)) "without exception set. (%s:%d)", __FILE__, __LINE__); } - Py_DECREF(str); return NULL; } - if (buf.obj == str || !PyBytes_CheckExact(buf.obj)) { + if (!PyBytes_CheckExact(buf.obj)) { PyErr_Format(TestError, "buf.obj must be a bytes object, got %R (%s:%d)", buf.obj, __FILE__, __LINE__); @@ -2099,8 +2098,7 @@ unicode_test_getutf8buffer(PyObject *self, PyObject *Py_UNUSED(ignored)) PyErr_Format(TestError, "Py_REFCNT(str) must not be changed. (%s:%d)", __FILE__, __LINE__); - PyBuffer_Release(&buf); - Py_DECREF(str); + // Do not DECREF here because refcnt is broken. return NULL; } @@ -2122,7 +2120,6 @@ unicode_test_getutf8buffer(PyObject *self, PyObject *Py_UNUSED(ignored)) "without exception set. (%s:%d)", __FILE__, __LINE__); } - Py_DECREF(str); return NULL; } @@ -2159,8 +2156,7 @@ unicode_test_getutf8buffer(PyObject *self, PyObject *Py_UNUSED(ignored)) "Py_REFCNT(str); expected %zd, got %zd. (%s:%d)", refcnt + 1, Py_REFCNT(str), __FILE__, __LINE__); - PyBuffer_Release(&buf); - Py_DECREF(str); + // Do not DECREF here because refcnt is broken. return NULL; } @@ -2171,7 +2167,7 @@ unicode_test_getutf8buffer(PyObject *self, PyObject *Py_UNUSED(ignored)) "Py_REFCNT(str); expected %zd, got %zd. (%s:%d)", refcnt, Py_REFCNT(str), __FILE__, __LINE__); - Py_DECREF(str); + // Do not DECREF here because refcnt is broken. return NULL; } From d92ed646c30840a4984b494828c26850ea99ea25 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 30 Jan 2020 19:08:03 +0900 Subject: [PATCH 8/9] Update Include/cpython/unicodeobject.h Co-Authored-By: Victor Stinner --- Include/cpython/unicodeobject.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index c5223de19d6020d..b44acf86a709947 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -737,7 +737,7 @@ PyAPI_FUNC(void*) _PyUnicode_AsKind(PyObject *s, unsigned int kind); /* Get a buffer to the UTF-8 encoding of the Unicode object unicode. Returns -1 on error. - Successful calls to PyUnicode_GetUTF8Buffer must be paired to + Successful calls must be paired to calls to PyBuffer_Release. */ From 98ec45f4903d80fee06e036c0e731745f113bbe4 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 12 Mar 2020 17:25:41 +0900 Subject: [PATCH 9/9] Make the API private. --- Doc/c-api/unicode.rst | 22 ------------------- Include/cpython/unicodeobject.h | 8 +------ Lib/test/test_unicode.py | 2 +- .../2019-12-19-21-19-53.bpo-39087.l4A11-.rst | 2 +- Modules/_testcapimodule.c | 14 ++++++------ Objects/unicodeobject.c | 4 ++-- PC/python3.def | 1 - 7 files changed, 12 insertions(+), 41 deletions(-) diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 940094060b7685c..b1787ed1ce89cf0 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -1071,28 +1071,6 @@ These are the UTF-8 codec APIs: raised by the codec. -.. c:function: int PyUnicode_GetUTF8Buffer(PyObject *unicode, const char errors, Py_buffer *view) - - Get a read-only buffer of the UTF-8 encoding of the Unicode object. - Return ``-1`` if an error happened. - - Successful calls to :c:func:`PyUnicode_GetUTF8Buffer` must be paired with - calls to :c:func:`PyBuffer_Release`, similar to :c:func:`PyObject_GetBuffer`. - - .. note:: - When compared to :c:function:`PyUnicode_AsUTF8String`, this function - does not create a bytes object when the *unicode* is ASCII string or - has UTF-8 cache. - - When compared to :c:function:`PyUnicode_AsUTF8` or - :c:function:`PyUnicode_AsUTF8AndSize`, this function does not cache the - UTF-8 representation of the string in the *unicode* object. - So this API is faster and more efficient when the *unicode* object is - not ASCII string and it is encoded into UTF-8 only once. - - .. versionadded:: 3.9 - - .. c:function:: const char* PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *size) Return a pointer to the UTF-8 encoding of the Unicode object, and diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index b44acf86a709947..be91d2d9fc675f2 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -741,7 +741,7 @@ PyAPI_FUNC(void*) _PyUnicode_AsKind(PyObject *s, unsigned int kind); calls to PyBuffer_Release. */ -PyAPI_FUNC(int) PyUnicode_GetUTF8Buffer( +PyAPI_FUNC(int) _PyUnicode_GetUTF8Buffer( PyObject *unicode, /* Unicode object */ const char *errors, /* error handling */ Py_buffer *view /* (out) buffer to the UTF-8 encoding */ @@ -759,12 +759,6 @@ PyAPI_FUNC(int) PyUnicode_GetUTF8Buffer( _PyUnicode_AsStringAndSize is a #define for PyUnicode_AsUTF8AndSize to support the previous internal function with the same behaviour. - - *** This API is for interpreter INTERNAL USE ONLY and will likely - *** be removed or changed in the future. - - *** If you need to access the Unicode object as UTF-8 bytes string, - *** please use PyUnicode_GetUTF8Buffer() instead. */ PyAPI_FUNC(const char *) PyUnicode_AsUTF8AndSize( diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 99593d2321526f1..0522513777f60bc 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -2830,7 +2830,7 @@ def test_asucs4(self): self.assertEqual(unicode_asucs4(s, len(s), True), s+'\0') self.assertEqual(unicode_asucs4(s, len(s), False), s+'\uffff') - # Test PyUnicode_GetUTF8Buffer() + # Test _PyUnicode_GetUTF8Buffer() @support.cpython_only def test_getutf8buffer(self): from _testcapi import unicode_getutf8buffer, unicode_test_getutf8buffer diff --git a/Misc/NEWS.d/next/C API/2019-12-19-21-19-53.bpo-39087.l4A11-.rst b/Misc/NEWS.d/next/C API/2019-12-19-21-19-53.bpo-39087.l4A11-.rst index 89e1a3f2f782435..2c2c85d93b211f6 100644 --- a/Misc/NEWS.d/next/C API/2019-12-19-21-19-53.bpo-39087.l4A11-.rst +++ b/Misc/NEWS.d/next/C API/2019-12-19-21-19-53.bpo-39087.l4A11-.rst @@ -1,2 +1,2 @@ -Add new :c:func:`PyUnicode_GetUTF8Buffer` API to get UTF-8 encode of the +Add new ``_PyUnicode_GetUTF8Buffer`` private API to get UTF-8 encode of the unicode object without cache or extra allocation. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index eb90fb3729dc129..89d04c094a4b198 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1975,7 +1975,7 @@ unicode_getutf8buffer(PyObject *self, PyObject *args) } Py_buffer buffer; - if (PyUnicode_GetUTF8Buffer(unicode, errors, &buffer) < 0) { + if (_PyUnicode_GetUTF8Buffer(unicode, errors, &buffer) < 0) { return NULL; } @@ -1999,8 +1999,8 @@ unicode_test_getutf8buffer(PyObject *self, PyObject *Py_UNUSED(ignored)) } Py_ssize_t refcnt = Py_REFCNT(str); - // PyUnicode_GetUTF8Buffer() must not fail for ASCII string. - int ret = PyUnicode_GetUTF8Buffer(str, NULL, &buf); + // _PyUnicode_GetUTF8Buffer() must not fail for ASCII string. + int ret = _PyUnicode_GetUTF8Buffer(str, NULL, &buf); assert(ret == 0); if (buf.obj != str) { @@ -2064,11 +2064,11 @@ unicode_test_getutf8buffer(PyObject *self, PyObject *Py_UNUSED(ignored)) refcnt = Py_REFCNT(str); assert(PyUnicode_GET_LENGTH(str) == 5); - if (PyUnicode_GetUTF8Buffer(str, NULL, &buf) < 0) { + if (_PyUnicode_GetUTF8Buffer(str, NULL, &buf) < 0) { Py_DECREF(str); if (!PyErr_Occurred()) { PyErr_Format(TestError, - "PyUnicode_GetUTF8Buffer() returned nonzero " + "_PyUnicode_GetUTF8Buffer() returned nonzero " "without exception set. (%s:%d)", __FILE__, __LINE__); } @@ -2112,11 +2112,11 @@ unicode_test_getutf8buffer(PyObject *self, PyObject *Py_UNUSED(ignored)) return NULL; } - if (PyUnicode_GetUTF8Buffer(str, NULL, &buf) < 0) { + if (_PyUnicode_GetUTF8Buffer(str, NULL, &buf) < 0) { Py_DECREF(str); if (!PyErr_Occurred()) { PyErr_Format(TestError, - "PyUnicode_GetUTF8Buffer() returned nonzero " + "_PyUnicode_GetUTF8Buffer() returned nonzero " "without exception set. (%s:%d)", __FILE__, __LINE__); } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 5bde279422894ca..0fea435599be854 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -3992,8 +3992,8 @@ PyUnicode_FSDecoder(PyObject* arg, void* addr) int -PyUnicode_GetUTF8Buffer(PyObject *unicode, const char *errors, - Py_buffer *view) +_PyUnicode_GetUTF8Buffer(PyObject *unicode, const char *errors, + Py_buffer *view) { if (!PyUnicode_Check(unicode)) { PyErr_BadArgument(); diff --git a/PC/python3.def b/PC/python3.def index de8061f6b7b1cca..c7aed8789cfaaa9 100644 --- a/PC/python3.def +++ b/PC/python3.def @@ -694,7 +694,6 @@ EXPORTS PyUnicode_GetDefaultEncoding=python39.PyUnicode_GetDefaultEncoding PyUnicode_GetLength=python39.PyUnicode_GetLength PyUnicode_GetSize=python39.PyUnicode_GetSize - PyUnicode_GetUTF8Buffer=python39.PyUnicode_GetUTF8Buffer PyUnicode_InternFromString=python39.PyUnicode_InternFromString PyUnicode_InternImmortal=python39.PyUnicode_InternImmortal PyUnicode_InternInPlace=python39.PyUnicode_InternInPlace