From 542c4c46a08a503cee747c280c7d85f61fef9c22 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 7 Oct 2025 17:00:03 +0200 Subject: [PATCH 01/13] gh-139888: Add PyTupleWriter C API * Add _PyTuple_NewNoTrack() and _PyTuple_ResizeNoTrack() helper functions. * Modify PySequence_Tuple() to use PyTupleWriter API. * Soft deprecate _PyTuple_Resize(). --- Doc/c-api/tuple.rst | 65 ++++++ Doc/whatsnew/3.15.rst | 9 + Include/cpython/tupleobject.h | 11 + Include/internal/pycore_freelist_state.h | 2 + Lib/test/test_capi/test_tuple.py | 25 +++ ...-10-10-09-54-28.gh-issue-139888.Z4Px61.rst | 8 + ...-10-10-09-58-57.gh-issue-139888.xCfJ3L.rst | 2 + Modules/_testcapi/tuple.c | 131 +++++++++++ Objects/abstract.c | 53 ++--- Objects/object.c | 1 + Objects/tupleobject.c | 206 +++++++++++++++++- 11 files changed, 466 insertions(+), 47 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-10-10-09-54-28.gh-issue-139888.Z4Px61.rst create mode 100644 Misc/NEWS.d/next/C_API/2025-10-10-09-58-57.gh-issue-139888.xCfJ3L.rst diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index 65f8334c437974..d295130eed2a92 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -142,6 +142,71 @@ Tuple Objects ``*p`` is destroyed. On failure, returns ``-1`` and sets ``*p`` to ``NULL``, and raises :exc:`MemoryError` or :exc:`SystemError`. + .. deprecated:: 3.15 + The function is :term:`soft deprecated`, + use the :ref:`PyTupleWriter API ` instead. + + +.. _pytuplewriter: + +PyTupleWriter +------------- + +The :c:type:`PyTupleWriter` API can be used to create a Python :class:`tuple` +object. + +.. versionadded:: next + +.. c:type:: PyTupleWriter + + A tuple writer instance. + + The API is **not thread safe**: a writer should only be used by a single + thread at the same time. + + The instance must be destroyed by :c:func:`PyTupleWriter_Finish` on + success, or :c:func:`PyTupleWriter_Discard` on error. + + +.. c:function:: PyTupleWriter* PyTupleWriter_Create(Py_ssize_t size) + + Create a :c:type:`PyTupleWriter` to add *size* items. + + If *size* is greater than zero, preallocate *size* items. The caller is + responsible to add *size* items using :c:func:`PyTupleWriter_Add`. + + On error, set an exception and return ``NULL``. + + *size* must be positive or zero. + + +.. c:function:: PyObject* PyTupleWriter_Finish(PyTupleWriter *writer) + + Finish a :c:type:`PyTupleWriter` created by + :c:func:`PyTupleWriter_Create`. + + On success, return a Python :class:`tuple` object. + On error, set an exception and return ``NULL``. + + The writer instance is invalid after the call in any case. + No API can be called on the writer after :c:func:`PyTupleWriter_Finish`. + +.. c:function:: void PyTupleWriter_Discard(PyTupleWriter *writer) + + Discard a :c:type:`PyTupleWriter` created by :c:func:`PyTupleWriter_Create`. + + Do nothing if *writer* is ``NULL``. + + The writer instance is invalid after the call. + No API can be called on the writer after :c:func:`PyTupleWriter_Discard`. + +.. c:function:: int PyTupleWriter_Add(PyTupleWriter *writer, PyObject *item) + + Add an item to *writer*. + + On success, return ``0``. + On error, set an exception and return ``-1``. + .. _struct-sequence-objects: diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 40286d4fe857e8..ff9c3e42c6968c 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -855,6 +855,15 @@ New features * Add :c:func:`PyTuple_FromArray` to create a :class:`tuple` from an array. (Contributed by Victor Stinner in :gh:`111489`.) +* Add a :ref:`PyTupleWriter API ` to create :class:`tuple`: + + * :c:func:`PyTupleWriter_Create` + * :c:func:`PyTupleWriter_Add` + * :c:func:`PyTupleWriter_Finish` + * :c:func:`PyTupleWriter_Discard` + + (Contributed by Victor Stinner in :gh:`139888`.) + Porting to Python 3.15 ---------------------- diff --git a/Include/cpython/tupleobject.h b/Include/cpython/tupleobject.h index 888baaf3358267..a6f8f377e9145f 100644 --- a/Include/cpython/tupleobject.h +++ b/Include/cpython/tupleobject.h @@ -42,3 +42,14 @@ PyTuple_SET_ITEM(PyObject *op, Py_ssize_t index, PyObject *value) { PyAPI_FUNC(PyObject*) PyTuple_FromArray( PyObject *const *array, Py_ssize_t size); + +// --- Public PyUnicodeWriter API -------------------------------------------- + +typedef struct PyTupleWriter PyTupleWriter; + +PyAPI_FUNC(PyTupleWriter*) PyTupleWriter_Create(Py_ssize_t size); +PyAPI_FUNC(int) PyTupleWriter_Add( + PyTupleWriter *writer, + PyObject *item); +PyAPI_FUNC(PyObject*) PyTupleWriter_Finish(PyTupleWriter *writer); +PyAPI_FUNC(void) PyTupleWriter_Discard(PyTupleWriter *writer); diff --git a/Include/internal/pycore_freelist_state.h b/Include/internal/pycore_freelist_state.h index 46e2a82ea03456..51e6273b75df79 100644 --- a/Include/internal/pycore_freelist_state.h +++ b/Include/internal/pycore_freelist_state.h @@ -28,6 +28,7 @@ extern "C" { # define Py_object_stack_chunks_MAXFREELIST 4 # define Py_unicode_writers_MAXFREELIST 1 # define Py_bytes_writers_MAXFREELIST 1 +# define Py_tuple_writers_MAXFREELIST 1 # define Py_pycfunctionobject_MAXFREELIST 16 # define Py_pycmethodobject_MAXFREELIST 16 # define Py_pymethodobjects_MAXFREELIST 20 @@ -63,6 +64,7 @@ struct _Py_freelists { struct _Py_freelist object_stack_chunks; struct _Py_freelist unicode_writers; struct _Py_freelist bytes_writers; + struct _Py_freelist tuple_writers; struct _Py_freelist pycfunctionobject; struct _Py_freelist pycmethodobject; struct _Py_freelist pymethodobjects; diff --git a/Lib/test/test_capi/test_tuple.py b/Lib/test/test_capi/test_tuple.py index b6d6da008d0b7b..396bce99c33c2c 100644 --- a/Lib/test/test_capi/test_tuple.py +++ b/Lib/test/test_capi/test_tuple.py @@ -302,5 +302,30 @@ def my_iter(): self.assertEqual(tuples, []) +class TupleWriterTest(unittest.TestCase): + def create_writer(self, size): + return _testcapi.PyTupleWriter(size) + + def test_create(self): + # Test PyTupleWriter_Create() + writer = self.create_writer(0) + self.assertIs(writer.finish(), ()) + + writer = self.create_writer(123) + self.assertIs(writer.finish(), ()) + + def test_add(self): + # Test PyTupleWriter_Add() + writer = self.create_writer(3) + for ch in 'abc': + writer.add(ch) + self.assertEqual(writer.finish(), ('a', 'b', 'c')) + + writer = self.create_writer(0) + for i in range(1024): + writer.add(i) + self.assertEqual(writer.finish(), tuple(range(1024))) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/C_API/2025-10-10-09-54-28.gh-issue-139888.Z4Px61.rst b/Misc/NEWS.d/next/C_API/2025-10-10-09-54-28.gh-issue-139888.Z4Px61.rst new file mode 100644 index 00000000000000..39e644bc059fcf --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-10-10-09-54-28.gh-issue-139888.Z4Px61.rst @@ -0,0 +1,8 @@ +Add a :ref:`PyTupleWriter API ` to create :class:`tuple`: + +* :c:func:`PyTupleWriter_Create` +* :c:func:`PyTupleWriter_Add` +* :c:func:`PyTupleWriter_Finish` +* :c:func:`PyTupleWriter_Discard` + +Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2025-10-10-09-58-57.gh-issue-139888.xCfJ3L.rst b/Misc/NEWS.d/next/C_API/2025-10-10-09-58-57.gh-issue-139888.xCfJ3L.rst new file mode 100644 index 00000000000000..bf67f81102f6e3 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-10-10-09-58-57.gh-issue-139888.xCfJ3L.rst @@ -0,0 +1,2 @@ +The :c:func:`_PyTuple_Resize` function is now :term:`soft deprecated`, use +the :ref:`PyTupleWriter API ` instead. diff --git a/Modules/_testcapi/tuple.c b/Modules/_testcapi/tuple.c index 5de1c494c0a8c0..99e7595e418f4c 100644 --- a/Modules/_testcapi/tuple.c +++ b/Modules/_testcapi/tuple.c @@ -131,6 +131,127 @@ tuple_fromarray(PyObject* Py_UNUSED(module), PyObject *args) } +// --- PyTupleWriter type --------------------------------------------------- + +typedef struct { + PyObject_HEAD + PyTupleWriter *writer; +} WriterObject; + + +static PyObject * +writer_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + WriterObject *self = (WriterObject *)type->tp_alloc(type, 0); + if (!self) { + return NULL; + } + self->writer = NULL; + return (PyObject*)self; +} + + +static int +writer_init(PyObject *self_raw, PyObject *args, PyObject *kwargs) +{ + if (kwargs && PyDict_GET_SIZE(kwargs)) { + PyErr_Format(PyExc_TypeError, + "PyTupleWriter() takes exactly no keyword arguments"); + return -1; + } + + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "n", &size)) { + return -1; + } + + WriterObject *self = (WriterObject *)self_raw; + if (self->writer) { + PyTupleWriter_Discard(self->writer); + } + self->writer = PyTupleWriter_Create(size); + if (self->writer == NULL) { + return -1; + } + return 0; +} + + +static void +writer_dealloc(PyObject *self_raw) +{ + WriterObject *self = (WriterObject *)self_raw; + PyTypeObject *tp = Py_TYPE(self); + if (self->writer) { + PyTupleWriter_Discard(self->writer); + } + tp->tp_free(self); + Py_DECREF(tp); +} + + +static inline int +writer_check(WriterObject *self) +{ + if (self->writer == NULL) { + PyErr_SetString(PyExc_ValueError, "operation on finished writer"); + return -1; + } + return 0; +} + + +static PyObject* +writer_add(PyObject *self_raw, PyObject *item) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + if (PyTupleWriter_Add(self->writer, item) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + + +static PyObject* +writer_finish(PyObject *self_raw, PyObject *Py_UNUSED(args)) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + PyObject *tuple = PyTupleWriter_Finish(self->writer); + self->writer = NULL; + return tuple; +} + + +static PyMethodDef writer_methods[] = { + {"add", _PyCFunction_CAST(writer_add), METH_O}, + {"finish", _PyCFunction_CAST(writer_finish), METH_NOARGS}, + {NULL, NULL} /* sentinel */ +}; + +static PyType_Slot Writer_Type_slots[] = { + {Py_tp_new, writer_new}, + {Py_tp_init, writer_init}, + {Py_tp_dealloc, writer_dealloc}, + {Py_tp_methods, writer_methods}, + {0, 0}, /* sentinel */ +}; + +static PyType_Spec Writer_spec = { + .name = "_testcapi.PyTupleWriter", + .basicsize = sizeof(WriterObject), + .flags = Py_TPFLAGS_DEFAULT, + .slots = Writer_Type_slots, +}; + + static PyMethodDef test_methods[] = { {"tuple_get_size", tuple_get_size, METH_O}, {"tuple_get_item", tuple_get_item, METH_VARARGS}, @@ -148,5 +269,15 @@ _PyTestCapi_Init_Tuple(PyObject *m) return -1; } + PyTypeObject *writer_type = (PyTypeObject *)PyType_FromSpec(&Writer_spec); + if (writer_type == NULL) { + return -1; + } + if (PyModule_AddType(m, writer_type) < 0) { + Py_DECREF(writer_type); + return -1; + } + Py_DECREF(writer_type); + return 0; } diff --git a/Objects/abstract.c b/Objects/abstract.c index 8adad8407d04d4..7f0f9b4b3d2869 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -1996,8 +1996,6 @@ PySequence_DelSlice(PyObject *s, Py_ssize_t i1, Py_ssize_t i2) PyObject * PySequence_Tuple(PyObject *v) { - PyObject *it; /* iter(v) */ - if (v == NULL) { return null_error(); } @@ -2010,62 +2008,39 @@ PySequence_Tuple(PyObject *v) a copy, so there's no need for exactness below. */ return Py_NewRef(v); } - if (PyList_CheckExact(v)) + if (PyList_CheckExact(v)) { return PyList_AsTuple(v); + } /* Get iterator. */ - it = PyObject_GetIter(v); - if (it == NULL) + PyObject *it = PyObject_GetIter(v); + if (it == NULL) { return NULL; - - Py_ssize_t n; - PyObject *buffer[8]; - for (n = 0; n < 8; n++) { - PyObject *item = PyIter_Next(it); - if (item == NULL) { - if (PyErr_Occurred()) { - goto fail; - } - Py_DECREF(it); - return _PyTuple_FromArraySteal(buffer, n); - } - buffer[n] = item; } - PyListObject *list = (PyListObject *)PyList_New(16); - if (list == NULL) { + + PyTupleWriter *writer = PyTupleWriter_Create(0); + if (writer == NULL) { goto fail; } - assert(n == 8); - Py_SET_SIZE(list, n); - for (Py_ssize_t j = 0; j < n; j++) { - PyList_SET_ITEM(list, j, buffer[j]); - } + for (;;) { PyObject *item = PyIter_Next(it); if (item == NULL) { if (PyErr_Occurred()) { - Py_DECREF(list); - Py_DECREF(it); - return NULL; + goto fail; } break; } - if (_PyList_AppendTakeRef(list, item) < 0) { - Py_DECREF(list); - Py_DECREF(it); - return NULL; + if (PyTupleWriter_Add(writer, item) < 0) { + goto fail; } } Py_DECREF(it); - PyObject *res = _PyList_AsTupleAndClear(list); - Py_DECREF(list); - return res; + return PyTupleWriter_Finish(writer); + fail: Py_DECREF(it); - while (n > 0) { - n--; - Py_DECREF(buffer[n]); - } + PyTupleWriter_Discard(writer); return NULL; } diff --git a/Objects/object.c b/Objects/object.c index 0540112d7d2acf..a51a0a53b0728b 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -946,6 +946,7 @@ _PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization) } clear_freelist(&freelists->unicode_writers, is_finalization, PyMem_Free); clear_freelist(&freelists->bytes_writers, is_finalization, PyMem_Free); + clear_freelist(&freelists->tuple_writers, is_finalization, PyMem_Free); clear_freelist(&freelists->ints, is_finalization, free_object); clear_freelist(&freelists->pycfunctionobject, is_finalization, PyObject_GC_Del); clear_freelist(&freelists->pycmethodobject, is_finalization, PyObject_GC_Del); diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 1fa4bae638a1fe..26ba85622c47e3 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -71,8 +71,8 @@ tuple_get_empty(void) return (PyObject *)&_Py_SINGLETON(tuple_empty); } -PyObject * -PyTuple_New(Py_ssize_t size) +static PyObject * +_PyTuple_NewNoTrack(Py_ssize_t size) { PyTupleObject *op; if (size == 0) { @@ -85,10 +85,25 @@ PyTuple_New(Py_ssize_t size) for (Py_ssize_t i = 0; i < size; i++) { op->ob_item[i] = NULL; } - _PyObject_GC_TRACK(op); return (PyObject *) op; } + +PyObject * +PyTuple_New(Py_ssize_t size) +{ + if (size == 0) { + return tuple_get_empty(); + } + PyObject *op = _PyTuple_NewNoTrack(size); + if (op == NULL) { + return NULL; + } + _PyObject_GC_TRACK(op); + return op; +} + + Py_ssize_t PyTuple_Size(PyObject *op) { @@ -913,8 +928,8 @@ PyTypeObject PyTuple_Type = { efficiently. In any case, don't use this if the tuple may already be known to some other part of the code. */ -int -_PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) +static int +tuple_resize(PyObject **pv, Py_ssize_t newsize, int track) { PyTupleObject *v; PyTupleObject *sv; @@ -946,7 +961,12 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) /* The empty tuple is statically allocated so we never resize it in-place. */ Py_DECREF(v); - *pv = PyTuple_New(newsize); + if (track) { + *pv = PyTuple_New(newsize); + } + else { + *pv = _PyTuple_NewNoTrack(newsize); + } return *pv == NULL ? -1 : 0; } @@ -972,14 +992,32 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) } _Py_NewReferenceNoTotal((PyObject *) sv); /* Zero out items added by growing */ - if (newsize > oldsize) + if (newsize > oldsize) { memset(&sv->ob_item[oldsize], 0, sizeof(*sv->ob_item) * (newsize - oldsize)); + } *pv = (PyObject *) sv; - _PyObject_GC_TRACK(sv); + if (track) { + _PyObject_GC_TRACK(sv); + } return 0; } + +static int +_PyTuple_ResizeNoTrack(PyObject **pv, Py_ssize_t newsize) +{ + return tuple_resize(pv, newsize, 0); +} + + +int +_PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) +{ + return tuple_resize(pv, newsize, 1); +} + + /*********************** Tuple Iterator **************************/ #define _PyTupleIterObject_CAST(op) ((_PyTupleIterObject *)(op)) @@ -1181,3 +1219,155 @@ _PyTuple_DebugMallocStats(FILE *out) _PyObject_VAR_SIZE(&PyTuple_Type, len)); } } + + +// --- Public PyTupleWriter API ---------------------------------------------- + +struct PyTupleWriter { + PyObject* small_tuple[16]; + PyObject *tuple; + PyObject **items; + size_t size; + size_t allocated; +}; + + +static int +_PyTupleWriter_SetSize(PyTupleWriter *writer, size_t size, int overallocate) +{ + if (size > (size_t)PY_SSIZE_T_MAX) { + /* Check for overflow */ + PyErr_NoMemory(); + return -1; + } + if (size <= writer->allocated) { + return 0; + } + + if (size <= Py_ARRAY_LENGTH(writer->small_tuple)) { + assert(writer->tuple == NULL); + writer->items = writer->small_tuple; + writer->allocated = Py_ARRAY_LENGTH(writer->small_tuple); + return 0; + } + + assert(size >= 1); + if (overallocate) { + size += (size >> 2); // Over-allocate by 25% + } + + if (writer->tuple != NULL) { + if (_PyTuple_ResizeNoTrack(&writer->tuple, (Py_ssize_t)size) < 0) { + return -1; + } + } + else { + writer->tuple = _PyTuple_NewNoTrack((Py_ssize_t)size); + if (writer->tuple == NULL) { + return -1; + } + + if (writer->size > 0) { + memcpy(_PyTuple_ITEMS(writer->tuple), + writer->small_tuple, + writer->size * sizeof(writer->small_tuple[0])); + } + } + writer->items = _PyTuple_ITEMS(writer->tuple); + writer->allocated = size; + return 0; +} + + +PyTupleWriter* +PyTupleWriter_Create(Py_ssize_t size) +{ + if (size < 0) { + PyErr_SetString(PyExc_ValueError, "size must be >= 0"); + return NULL; + } + + PyTupleWriter *writer = _Py_FREELIST_POP_MEM(tuple_writers); + if (writer == NULL) { + writer = (PyTupleWriter *)PyMem_Malloc(sizeof(PyTupleWriter)); + if (writer == NULL) { + PyErr_NoMemory(); + return NULL; + } + } +#ifdef Py_DEBUG + memset(writer->small_tuple, 0xff, sizeof(writer->small_tuple)); +#endif + writer->tuple = NULL; + writer->items = NULL; + writer->size = 0; + writer->allocated = 0; + + if (size > 0) { + if (_PyTupleWriter_SetSize(writer, size, 0) < 0) { + PyTupleWriter_Discard(writer); + return NULL; + } + } + + return writer; +} + + +int PyTupleWriter_Add(PyTupleWriter *writer, PyObject *item) +{ + if (writer->size >= writer->allocated) { + if (_PyTupleWriter_SetSize(writer, writer->size + 1, 1) < 0) { + return -1; + } + assert(writer->size < writer->allocated); + } + assert(writer->items != NULL); + + writer->items[writer->size] = item; + writer->size++; + return 0; +} + + +PyObject* PyTupleWriter_Finish(PyTupleWriter *writer) +{ + if (writer->size == 0) { + PyTupleWriter_Discard(writer); + // return the empty tuple singleton + return PyTuple_New(0); + } + + PyObject *result; + if (writer->tuple != NULL) { + if (writer->size != writer->allocated) { + if (_PyTuple_ResizeNoTrack(&writer->tuple, + (Py_ssize_t)writer->size) < 0) { + PyTupleWriter_Discard(writer); + return NULL; + } + } + + result = writer->tuple; + writer->tuple = NULL; + _PyObject_GC_TRACK(result); + } + else { + result = _PyTuple_FromArraySteal(writer->items, + (Py_ssize_t)writer->size); + } + + PyTupleWriter_Discard(writer); + return result; +} + + +void PyTupleWriter_Discard(PyTupleWriter *writer) +{ + if (writer == NULL) { + return; + } + + Py_XDECREF(writer->tuple); + _Py_FREELIST_FREE(tuple_writers, writer, PyMem_Free); +} From 86d529b57d9d545af2a20192e070a73bcd90bb2d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 10 Oct 2025 11:38:14 +0200 Subject: [PATCH 02/13] Add AddSteal() and AddArray() functions --- Doc/c-api/tuple.rst | 11 +++++++ Include/cpython/tupleobject.h | 7 +++++ Lib/test/test_capi/test_tuple.py | 27 ++++++++++++++--- Modules/_testcapi/tuple.c | 39 ++++++++++++++++++++++++ Objects/tupleobject.c | 52 +++++++++++++++++++++++++++++--- 5 files changed, 127 insertions(+), 9 deletions(-) diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index d295130eed2a92..de3dbfa0fa2a8c 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -207,6 +207,17 @@ object. On success, return ``0``. On error, set an exception and return ``-1``. +.. c:function:: int PyTupleWriter_AddSteal(PyTupleWriter *writer, PyObject *item) + + Similar to :c:func:`PyTupleWriter_Add`, but take the ownership of *item*. + +.. c:function:: int PyTupleWriter_AddArray(PyTupleWriter *writer, PyObject *const *array, Py_ssize_t size) + + Add items from an array to *writer*. + + On success, return ``0``. + On error, set an exception and return ``-1``. + .. _struct-sequence-objects: diff --git a/Include/cpython/tupleobject.h b/Include/cpython/tupleobject.h index a6f8f377e9145f..70571b4f86d9d0 100644 --- a/Include/cpython/tupleobject.h +++ b/Include/cpython/tupleobject.h @@ -51,5 +51,12 @@ PyAPI_FUNC(PyTupleWriter*) PyTupleWriter_Create(Py_ssize_t size); PyAPI_FUNC(int) PyTupleWriter_Add( PyTupleWriter *writer, PyObject *item); +PyAPI_FUNC(int) PyTupleWriter_AddSteal( + PyTupleWriter *writer, + PyObject *item); +PyAPI_FUNC(int) PyTupleWriter_AddArray( + PyTupleWriter *writer, + PyObject *const *array, + Py_ssize_t size); PyAPI_FUNC(PyObject*) PyTupleWriter_Finish(PyTupleWriter *writer); PyAPI_FUNC(void) PyTupleWriter_Discard(PyTupleWriter *writer); diff --git a/Lib/test/test_capi/test_tuple.py b/Lib/test/test_capi/test_tuple.py index 396bce99c33c2c..f212b9dfc75d39 100644 --- a/Lib/test/test_capi/test_tuple.py +++ b/Lib/test/test_capi/test_tuple.py @@ -314,16 +314,35 @@ def test_create(self): writer = self.create_writer(123) self.assertIs(writer.finish(), ()) - def test_add(self): - # Test PyTupleWriter_Add() + def check_add(self, name): writer = self.create_writer(3) + add = getattr(writer, name) for ch in 'abc': - writer.add(ch) + add(ch) self.assertEqual(writer.finish(), ('a', 'b', 'c')) writer = self.create_writer(0) + add = getattr(writer, name) for i in range(1024): - writer.add(i) + add(i) + self.assertEqual(writer.finish(), tuple(range(1024))) + + def test_add(self): + # Test PyTupleWriter_Add() + self.check_add('add') + + def test_add_steal(self): + # Test PyTupleWriter_AddSteal() + self.check_add('add_steal') + + def test_add_array(self): + writer = self.create_writer(0) + writer.add_array(('a', 'b', 'c')) + writer.add_array(('d', 'e')) + self.assertEqual(writer.finish(), ('a', 'b', 'c', 'd', 'e')) + + writer = self.create_writer(0) + writer.add_array(tuple(range(1024))) self.assertEqual(writer.finish(), tuple(range(1024))) diff --git a/Modules/_testcapi/tuple.c b/Modules/_testcapi/tuple.c index 99e7595e418f4c..d0a264b11776d3 100644 --- a/Modules/_testcapi/tuple.c +++ b/Modules/_testcapi/tuple.c @@ -216,6 +216,43 @@ writer_add(PyObject *self_raw, PyObject *item) } +static PyObject* +writer_add_steal(PyObject *self_raw, PyObject *item) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + if (PyTupleWriter_AddSteal(self->writer, Py_NewRef(item)) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + + +static PyObject* +writer_add_array(PyObject *self_raw, PyObject *args) +{ + PyObject *tuple; + if (!PyArg_ParseTuple(args, "O!", &PyTuple_Type, &tuple)) { + return NULL; + } + + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + PyObject **array = &PyTuple_GET_ITEM(tuple, 0); + Py_ssize_t size = PyTuple_GET_SIZE(tuple); + if (PyTupleWriter_AddArray(self->writer, array, size) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + + static PyObject* writer_finish(PyObject *self_raw, PyObject *Py_UNUSED(args)) { @@ -232,6 +269,8 @@ writer_finish(PyObject *self_raw, PyObject *Py_UNUSED(args)) static PyMethodDef writer_methods[] = { {"add", _PyCFunction_CAST(writer_add), METH_O}, + {"add_steal", _PyCFunction_CAST(writer_add_steal), METH_O}, + {"add_array", _PyCFunction_CAST(writer_add_array), METH_VARARGS}, {"finish", _PyCFunction_CAST(writer_finish), METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 26ba85622c47e3..2f31fb9ed4f1fe 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -1314,15 +1314,30 @@ PyTupleWriter_Create(Py_ssize_t size) } -int PyTupleWriter_Add(PyTupleWriter *writer, PyObject *item) +int +PyTupleWriter_Add(PyTupleWriter *writer, PyObject *item) +{ + if (writer->size >= writer->allocated) { + if (_PyTupleWriter_SetSize(writer, writer->size + 1, 1) < 0) { + return -1; + } + } + + writer->items[writer->size] = Py_NewRef(item); + writer->size++; + return 0; +} + + +int +PyTupleWriter_AddSteal(PyTupleWriter *writer, PyObject *item) { if (writer->size >= writer->allocated) { if (_PyTupleWriter_SetSize(writer, writer->size + 1, 1) < 0) { + Py_DECREF(item); return -1; } - assert(writer->size < writer->allocated); } - assert(writer->items != NULL); writer->items[writer->size] = item; writer->size++; @@ -1330,7 +1345,33 @@ int PyTupleWriter_Add(PyTupleWriter *writer, PyObject *item) } -PyObject* PyTupleWriter_Finish(PyTupleWriter *writer) +int +PyTupleWriter_AddArray(PyTupleWriter *writer, + PyObject *const *array, Py_ssize_t size) +{ + if (size < 0) { + PyErr_BadInternalCall(); + return -1; + } + + if ((writer->size + (size_t)size) > writer->allocated) { + if (_PyTupleWriter_SetSize(writer, + writer->size + (size_t)size, 1) < 0) { + return -1; + } + } + + PyObject **items = &writer->items[writer->size]; + for (Py_ssize_t i=0; i < size; i++) { + *items++ = Py_NewRef(*array++); + } + writer->size += size; + return 0; +} + + +PyObject* +PyTupleWriter_Finish(PyTupleWriter *writer) { if (writer->size == 0) { PyTupleWriter_Discard(writer); @@ -1362,7 +1403,8 @@ PyObject* PyTupleWriter_Finish(PyTupleWriter *writer) } -void PyTupleWriter_Discard(PyTupleWriter *writer) +void +PyTupleWriter_Discard(PyTupleWriter *writer) { if (writer == NULL) { return; From afbe7bf10a5fb9374d62f07f4e6de1c425dc2f88 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 10 Oct 2025 11:42:32 +0200 Subject: [PATCH 03/13] Fix refleak in PySequence_Tuple() --- Objects/abstract.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/abstract.c b/Objects/abstract.c index 7f0f9b4b3d2869..fb666748b0797d 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2031,7 +2031,7 @@ PySequence_Tuple(PyObject *v) } break; } - if (PyTupleWriter_Add(writer, item) < 0) { + if (PyTupleWriter_AddSteal(writer, item) < 0) { goto fail; } } From 7787017073691053af34e2cd42a2f2ba909743db Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 10 Oct 2025 11:46:00 +0200 Subject: [PATCH 04/13] Hard deprecate _PyTuple_Resize() --- Doc/c-api/tuple.rst | 3 +-- Doc/whatsnew/3.15.rst | 3 +++ Include/cpython/tupleobject.h | 2 +- .../next/C_API/2025-10-10-09-58-57.gh-issue-139888.xCfJ3L.rst | 4 ++-- Modules/_testcapi/tuple.c | 3 +++ 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index de3dbfa0fa2a8c..b1b97ededa4449 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -143,8 +143,7 @@ Tuple Objects raises :exc:`MemoryError` or :exc:`SystemError`. .. deprecated:: 3.15 - The function is :term:`soft deprecated`, - use the :ref:`PyTupleWriter API ` instead. + Use the :ref:`PyTupleWriter API ` instead. .. _pytuplewriter: diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index ff9c3e42c6968c..db196419d51c18 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -924,6 +924,9 @@ Deprecated C APIs since 3.15 and will be removed in 3.17. (Contributed by Nikita Sobolev in :gh:`136355`.) +* Deprecate the :c:func:`_PyTuple_Resize` function: + use the new :ref:`PyTupleWriter API ` instead. + (Contributed by Victor Stinner in :gh:`139888`.) .. Add C API deprecations above alphabetically, not here at the end. diff --git a/Include/cpython/tupleobject.h b/Include/cpython/tupleobject.h index 70571b4f86d9d0..f6bfeb2acd4043 100644 --- a/Include/cpython/tupleobject.h +++ b/Include/cpython/tupleobject.h @@ -12,7 +12,7 @@ typedef struct { PyObject *ob_item[1]; } PyTupleObject; -PyAPI_FUNC(int) _PyTuple_Resize(PyObject **, Py_ssize_t); +_Py_DEPRECATED_EXTERNALLY(3.15) PyAPI_FUNC(int) _PyTuple_Resize(PyObject **, Py_ssize_t); /* Cast argument to PyTupleObject* type. */ #define _PyTuple_CAST(op) \ diff --git a/Misc/NEWS.d/next/C_API/2025-10-10-09-58-57.gh-issue-139888.xCfJ3L.rst b/Misc/NEWS.d/next/C_API/2025-10-10-09-58-57.gh-issue-139888.xCfJ3L.rst index bf67f81102f6e3..77bbda98441488 100644 --- a/Misc/NEWS.d/next/C_API/2025-10-10-09-58-57.gh-issue-139888.xCfJ3L.rst +++ b/Misc/NEWS.d/next/C_API/2025-10-10-09-58-57.gh-issue-139888.xCfJ3L.rst @@ -1,2 +1,2 @@ -The :c:func:`_PyTuple_Resize` function is now :term:`soft deprecated`, use -the :ref:`PyTupleWriter API ` instead. +Deprecate the :c:func:`_PyTuple_Resize` function: use the :ref:`PyTupleWriter +API ` instead. diff --git a/Modules/_testcapi/tuple.c b/Modules/_testcapi/tuple.c index d0a264b11776d3..ad2737e29f4d20 100644 --- a/Modules/_testcapi/tuple.c +++ b/Modules/_testcapi/tuple.c @@ -84,7 +84,10 @@ _tuple_resize(PyObject *Py_UNUSED(module), PyObject *args) NULLABLE(tup); Py_XINCREF(tup); } + _Py_COMP_DIAG_PUSH + _Py_COMP_DIAG_IGNORE_DEPR_DECLS int r = _PyTuple_Resize(&tup, newsize); + _Py_COMP_DIAG_POP if (r == -1) { assert(tup == NULL); return NULL; From a9a6aabedf3006f9d810ac7ef0afea17500aa6fe Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 10 Oct 2025 12:16:06 +0200 Subject: [PATCH 05/13] Optimize _PyTupleWriter_SetSize() --- Objects/tupleobject.c | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 2f31fb9ed4f1fe..96e9ca0ab1c55a 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -1235,23 +1235,14 @@ struct PyTupleWriter { static int _PyTupleWriter_SetSize(PyTupleWriter *writer, size_t size, int overallocate) { + assert(size >= 1); + if (size > (size_t)PY_SSIZE_T_MAX) { /* Check for overflow */ PyErr_NoMemory(); return -1; } - if (size <= writer->allocated) { - return 0; - } - if (size <= Py_ARRAY_LENGTH(writer->small_tuple)) { - assert(writer->tuple == NULL); - writer->items = writer->small_tuple; - writer->allocated = Py_ARRAY_LENGTH(writer->small_tuple); - return 0; - } - - assert(size >= 1); if (overallocate) { size += (size >> 2); // Over-allocate by 25% } @@ -1299,11 +1290,11 @@ PyTupleWriter_Create(Py_ssize_t size) memset(writer->small_tuple, 0xff, sizeof(writer->small_tuple)); #endif writer->tuple = NULL; - writer->items = NULL; + writer->items = writer->small_tuple; writer->size = 0; - writer->allocated = 0; + writer->allocated = Py_ARRAY_LENGTH(writer->small_tuple); - if (size > 0) { + if (size > writer->allocated) { if (_PyTupleWriter_SetSize(writer, size, 0) < 0) { PyTupleWriter_Discard(writer); return NULL; From 8719f11a3dcecb264ab29c6f4729bd2e4d3200e4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 10 Oct 2025 12:19:12 +0200 Subject: [PATCH 06/13] Add test on PyTupleWriter_Create(-2) Change also the exception to SystemError for this error. --- Lib/test/test_capi/test_tuple.py | 3 +++ Objects/tupleobject.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_capi/test_tuple.py b/Lib/test/test_capi/test_tuple.py index f212b9dfc75d39..71925164cdba24 100644 --- a/Lib/test/test_capi/test_tuple.py +++ b/Lib/test/test_capi/test_tuple.py @@ -314,6 +314,9 @@ def test_create(self): writer = self.create_writer(123) self.assertIs(writer.finish(), ()) + with self.assertRaises(SystemError): + self.create_writer(-2) + def check_add(self, name): writer = self.create_writer(3) add = getattr(writer, name) diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 96e9ca0ab1c55a..4c960c091bf6af 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -1274,7 +1274,7 @@ PyTupleWriter* PyTupleWriter_Create(Py_ssize_t size) { if (size < 0) { - PyErr_SetString(PyExc_ValueError, "size must be >= 0"); + PyErr_BadInternalCall(); return NULL; } From 9e42598538d42c1556c65d263666a532ec3ecee0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 10 Oct 2025 12:23:21 +0200 Subject: [PATCH 07/13] Fix compiler warning --- Objects/tupleobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 4c960c091bf6af..ab8cd8dc22bfe3 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -1294,7 +1294,7 @@ PyTupleWriter_Create(Py_ssize_t size) writer->size = 0; writer->allocated = Py_ARRAY_LENGTH(writer->small_tuple); - if (size > writer->allocated) { + if ((size_t)size > writer->allocated) { if (_PyTupleWriter_SetSize(writer, size, 0) < 0) { PyTupleWriter_Discard(writer); return NULL; From eef02ff3cb58d91238986fcb9f42b08c976e7620 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 10 Oct 2025 12:39:57 +0200 Subject: [PATCH 08/13] _PyTupleWriter_Create() doesn't need to copy small_tuple --- Objects/tupleobject.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index ab8cd8dc22bfe3..934ec7dfd44992 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -1233,7 +1233,7 @@ struct PyTupleWriter { static int -_PyTupleWriter_SetSize(PyTupleWriter *writer, size_t size, int overallocate) +_PyTupleWriter_SetSize(PyTupleWriter *writer, size_t size, int resize) { assert(size >= 1); @@ -1243,7 +1243,7 @@ _PyTupleWriter_SetSize(PyTupleWriter *writer, size_t size, int overallocate) return -1; } - if (overallocate) { + if (resize) { size += (size >> 2); // Over-allocate by 25% } @@ -1258,7 +1258,8 @@ _PyTupleWriter_SetSize(PyTupleWriter *writer, size_t size, int overallocate) return -1; } - if (writer->size > 0) { + if (resize) { + assert(writer->size > 0); memcpy(_PyTuple_ITEMS(writer->tuple), writer->small_tuple, writer->size * sizeof(writer->small_tuple[0])); From 5cf5915acd246f50a75d083366ec2d0bb4e65172 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 10 Oct 2025 13:03:03 +0200 Subject: [PATCH 09/13] Fix assertion --- Objects/tupleobject.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 934ec7dfd44992..f9294381211c16 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -1258,8 +1258,7 @@ _PyTupleWriter_SetSize(PyTupleWriter *writer, size_t size, int resize) return -1; } - if (resize) { - assert(writer->size > 0); + if (resize && writer->size > 0) { memcpy(_PyTuple_ITEMS(writer->tuple), writer->small_tuple, writer->size * sizeof(writer->small_tuple[0])); From 871ee857b0af2b9612c1d462f8a16905bfb2c7fe Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 10 Oct 2025 16:17:01 +0200 Subject: [PATCH 10/13] Fix leak in PyTupleWriter_Discard() --- Lib/test/test_capi/test_tuple.py | 14 +++++++++++ Modules/_testcapi/tuple.c | 15 ++++++++++++ Objects/tupleobject.c | 42 ++++++++++++++++++++++---------- 3 files changed, 58 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_capi/test_tuple.py b/Lib/test/test_capi/test_tuple.py index 71925164cdba24..536413f4c68a3b 100644 --- a/Lib/test/test_capi/test_tuple.py +++ b/Lib/test/test_capi/test_tuple.py @@ -348,6 +348,20 @@ def test_add_array(self): writer.add_array(tuple(range(1024))) self.assertEqual(writer.finish(), tuple(range(1024))) + def test_discard(self): + # test the small_tuple buffer (16 items) + writer = self.create_writer(3) + writer.add(object()) + writer.add(object()) + # must not leak references + writer.discard() + + # test the tuple code path (17 items or more) + writer = self.create_writer(1024) + writer.add_array(tuple(range(1024))) + # must not leak references + writer.discard() + if __name__ == "__main__": unittest.main() diff --git a/Modules/_testcapi/tuple.c b/Modules/_testcapi/tuple.c index ad2737e29f4d20..66379d5646ffc2 100644 --- a/Modules/_testcapi/tuple.c +++ b/Modules/_testcapi/tuple.c @@ -270,11 +270,26 @@ writer_finish(PyObject *self_raw, PyObject *Py_UNUSED(args)) } +static PyObject* +writer_discard(PyObject *self_raw, PyObject *Py_UNUSED(args)) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + PyTupleWriter_Discard(self->writer); + self->writer = NULL; + Py_RETURN_NONE; +} + + static PyMethodDef writer_methods[] = { {"add", _PyCFunction_CAST(writer_add), METH_O}, {"add_steal", _PyCFunction_CAST(writer_add_steal), METH_O}, {"add_array", _PyCFunction_CAST(writer_add_array), METH_VARARGS}, {"finish", _PyCFunction_CAST(writer_finish), METH_NOARGS}, + {"discard", _PyCFunction_CAST(writer_discard), METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index f9294381211c16..52756016a276d3 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -1361,6 +1361,34 @@ PyTupleWriter_AddArray(PyTupleWriter *writer, } +static inline void +tuplewriter_free(PyTupleWriter *writer) +{ + _Py_FREELIST_FREE(tuple_writers, writer, PyMem_Free); +} + + +void +PyTupleWriter_Discard(PyTupleWriter *writer) +{ + if (writer == NULL) { + return; + } + + if (writer->tuple != NULL) { + Py_DECREF(writer->tuple); + } + else { + PyObject **items = writer->items; + for (size_t i=0; i < writer->size; i++) { + Py_DECREF(items[i]); + } + } + + tuplewriter_free(writer); +} + + PyObject* PyTupleWriter_Finish(PyTupleWriter *writer) { @@ -1389,18 +1417,6 @@ PyTupleWriter_Finish(PyTupleWriter *writer) (Py_ssize_t)writer->size); } - PyTupleWriter_Discard(writer); + tuplewriter_free(writer); return result; } - - -void -PyTupleWriter_Discard(PyTupleWriter *writer) -{ - if (writer == NULL) { - return; - } - - Py_XDECREF(writer->tuple); - _Py_FREELIST_FREE(tuple_writers, writer, PyMem_Free); -} From 8e7025fd6650605763ef6f0d33541238b173b6d7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 11 Oct 2025 22:28:51 +0200 Subject: [PATCH 11/13] Accept NULL in Add() and AddSteal() --- Doc/c-api/tuple.rst | 5 +++-- Lib/test/test_capi/test_tuple.py | 5 +++++ Modules/_testcapi/tuple.c | 6 +++++- Objects/abstract.c | 7 ++----- Objects/tupleobject.c | 14 ++++++++++++++ 5 files changed, 29 insertions(+), 8 deletions(-) diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index b1b97ededa4449..cc6fbd24ced155 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -203,8 +203,9 @@ object. Add an item to *writer*. - On success, return ``0``. - On error, set an exception and return ``-1``. + * On success, return ``0``. + * If *item* is ``NULL``, return ``-1`` with an exception set. + * On error, set an exception and return ``-1``. .. c:function:: int PyTupleWriter_AddSteal(PyTupleWriter *writer, PyObject *item) diff --git a/Lib/test/test_capi/test_tuple.py b/Lib/test/test_capi/test_tuple.py index 536413f4c68a3b..5ed612db6b25dd 100644 --- a/Lib/test/test_capi/test_tuple.py +++ b/Lib/test/test_capi/test_tuple.py @@ -330,6 +330,11 @@ def check_add(self, name): add(i) self.assertEqual(writer.finish(), tuple(range(1024))) + writer = self.create_writer(1) + add = getattr(writer, name) + with self.assertRaises(SystemError): + add(NULL) + def test_add(self): # Test PyTupleWriter_Add() self.check_add('add') diff --git a/Modules/_testcapi/tuple.c b/Modules/_testcapi/tuple.c index 66379d5646ffc2..27f464a5ff1baf 100644 --- a/Modules/_testcapi/tuple.c +++ b/Modules/_testcapi/tuple.c @@ -207,6 +207,8 @@ writer_check(WriterObject *self) static PyObject* writer_add(PyObject *self_raw, PyObject *item) { + NULLABLE(item); + WriterObject *self = (WriterObject *)self_raw; if (writer_check(self) < 0) { return NULL; @@ -222,12 +224,14 @@ writer_add(PyObject *self_raw, PyObject *item) static PyObject* writer_add_steal(PyObject *self_raw, PyObject *item) { + NULLABLE(item); + WriterObject *self = (WriterObject *)self_raw; if (writer_check(self) < 0) { return NULL; } - if (PyTupleWriter_AddSteal(self->writer, Py_NewRef(item)) < 0) { + if (PyTupleWriter_AddSteal(self->writer, Py_XNewRef(item)) < 0) { return NULL; } Py_RETURN_NONE; diff --git a/Objects/abstract.c b/Objects/abstract.c index fb666748b0797d..60353547a3c959 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2024,11 +2024,8 @@ PySequence_Tuple(PyObject *v) } for (;;) { - PyObject *item = PyIter_Next(it); - if (item == NULL) { - if (PyErr_Occurred()) { - goto fail; - } + PyObject *item; + if (PyIter_NextItem(it, &item) == 0) { break; } if (PyTupleWriter_AddSteal(writer, item) < 0) { diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 52756016a276d3..3be7135036c9b8 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -1308,6 +1308,13 @@ PyTupleWriter_Create(Py_ssize_t size) int PyTupleWriter_Add(PyTupleWriter *writer, PyObject *item) { + if (item == NULL) { + if (!PyErr_Occurred()) { + PyErr_BadInternalCall(); + } + return -1; + } + if (writer->size >= writer->allocated) { if (_PyTupleWriter_SetSize(writer, writer->size + 1, 1) < 0) { return -1; @@ -1323,6 +1330,13 @@ PyTupleWriter_Add(PyTupleWriter *writer, PyObject *item) int PyTupleWriter_AddSteal(PyTupleWriter *writer, PyObject *item) { + if (item == NULL) { + if (!PyErr_Occurred()) { + PyErr_BadInternalCall(); + } + return -1; + } + if (writer->size >= writer->allocated) { if (_PyTupleWriter_SetSize(writer, writer->size + 1, 1) < 0) { Py_DECREF(item); From d27df40c8ef9c239b691628f7833d6c24ab04fc6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 11 Oct 2025 22:37:40 +0200 Subject: [PATCH 12/13] Use PyTupleWriter in multiple functions --- Modules/_tkinter.c | 75 ++++++------------ Objects/genericaliasobject.c | 143 ++++++++++++++++------------------- Objects/structseq.c | 31 ++++---- 3 files changed, 99 insertions(+), 150 deletions(-) diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index f0882191d3c3e8..c1a91373c5fd27 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -3080,60 +3080,33 @@ Tkapp_Dealloc(PyObject *op) /**** Tkinter Module ****/ -typedef struct { - PyObject* tuple; - Py_ssize_t size; /* current size */ - Py_ssize_t maxsize; /* allocated size */ -} FlattenContext; - static int -_bump(FlattenContext* context, Py_ssize_t size) -{ - /* expand tuple to hold (at least) size new items. - return true if successful, false if an exception was raised */ - - Py_ssize_t maxsize = context->maxsize * 2; /* never overflows */ - - if (maxsize < context->size + size) - maxsize = context->size + size; /* never overflows */ - - context->maxsize = maxsize; - - return _PyTuple_Resize(&context->tuple, maxsize) >= 0; -} - -static int -_flatten1(FlattenContext* context, PyObject* item, int depth) +_flatten1(PyTupleWriter *writer, PyObject *item, int depth) { /* add tuple or list to argument tuple (recursively) */ - Py_ssize_t i, size; - if (depth > 1000) { PyErr_SetString(PyExc_ValueError, "nesting too deep in _flatten"); return 0; - } else if (PyTuple_Check(item) || PyList_Check(item)) { - size = PySequence_Fast_GET_SIZE(item); - /* preallocate (assume no nesting) */ - if (context->size + size > context->maxsize && - !_bump(context, size)) - return 0; + } + + if (PyTuple_Check(item) || PyList_Check(item)) { + Py_ssize_t size = PySequence_Fast_GET_SIZE(item); /* copy items to output tuple */ - for (i = 0; i < size; i++) { + for (Py_ssize_t i = 0; i < size; i++) { PyObject *o = PySequence_Fast_GET_ITEM(item, i); if (PyList_Check(o) || PyTuple_Check(o)) { - if (!_flatten1(context, o, depth + 1)) + if (!_flatten1(writer, o, depth + 1)) return 0; } else if (o != Py_None) { - if (context->size + 1 > context->maxsize && - !_bump(context, 1)) + if (PyTupleWriter_Add(writer, o) < 0) { return 0; - PyTuple_SET_ITEM(context->tuple, - context->size++, Py_NewRef(o)); + } } } - } else { + } + else { PyErr_SetString(PyExc_TypeError, "argument must be sequence"); return 0; } @@ -3152,29 +3125,25 @@ static PyObject * _tkinter__flatten(PyObject *module, PyObject *item) /*[clinic end generated code: output=cad02a3f97f29862 input=6b9c12260aa1157f]*/ { - FlattenContext context; - - context.maxsize = PySequence_Size(item); - if (context.maxsize < 0) + Py_ssize_t maxsize = PySequence_Size(item); + if (maxsize < 0) { return NULL; - if (context.maxsize == 0) + } + if (maxsize == 0) { return PyTuple_New(0); + } - context.tuple = PyTuple_New(context.maxsize); - if (!context.tuple) - return NULL; - - context.size = 0; - - if (!_flatten1(&context, item, 0)) { - Py_XDECREF(context.tuple); + PyTupleWriter *writer = PyTupleWriter_Create(maxsize); + if (writer == NULL) { return NULL; } - if (_PyTuple_Resize(&context.tuple, context.size)) + if (!_flatten1(writer, item, 0)) { + PyTupleWriter_Discard(writer); return NULL; + } - return context.tuple; + return PyTupleWriter_Finish(writer); } /*[clinic input] diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 8b526f43f1e053..08a84555c34ee6 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -4,6 +4,7 @@ #include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_object.h" +#include "pycore_tuple.h" // _PyTuple_ITEMS() #include "pycore_typevarobject.h" // _Py_typing_type_repr #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString() #include "pycore_unionobject.h" // _Py_union_type_or, _PyGenericAlias_Check @@ -161,22 +162,6 @@ tuple_add(PyObject *self, Py_ssize_t len, PyObject *item) return 0; } -static Py_ssize_t -tuple_extend(PyObject **dst, Py_ssize_t dstindex, - PyObject **src, Py_ssize_t count) -{ - assert(count >= 0); - if (_PyTuple_Resize(dst, PyTuple_GET_SIZE(*dst) + count - 1) != 0) { - return -1; - } - assert(dstindex + count <= PyTuple_GET_SIZE(*dst)); - for (Py_ssize_t i = 0; i < count; ++i) { - PyObject *item = src[i]; - PyTuple_SET_ITEM(*dst, dstindex + i, Py_NewRef(item)); - } - return dstindex + count; -} - PyObject * _Py_make_parameters(PyObject *args) { @@ -274,15 +259,14 @@ subs_tvars(PyObject *obj, PyObject *params, if (PyObject_GetOptionalAttr(obj, &_Py_ID(__parameters__), &subparams) < 0) { return NULL; } + if (subparams && PyTuple_Check(subparams) && PyTuple_GET_SIZE(subparams)) { Py_ssize_t nparams = PyTuple_GET_SIZE(params); Py_ssize_t nsubargs = PyTuple_GET_SIZE(subparams); - PyObject *subargs = PyTuple_New(nsubargs); - if (subargs == NULL) { - Py_DECREF(subparams); - return NULL; + PyTupleWriter *writer = PyTupleWriter_Create(nsubargs); + if (writer == NULL) { + goto error; } - Py_ssize_t j = 0; for (Py_ssize_t i = 0; i < nsubargs; ++i) { PyObject *arg = PyTuple_GET_ITEM(subparams, i); Py_ssize_t iparam = tuple_index(params, nparams, arg); @@ -290,27 +274,39 @@ subs_tvars(PyObject *obj, PyObject *params, PyObject *param = PyTuple_GET_ITEM(params, iparam); arg = argitems[iparam]; if (Py_TYPE(param)->tp_iter && PyTuple_Check(arg)) { // TypeVarTuple - j = tuple_extend(&subargs, j, - &PyTuple_GET_ITEM(arg, 0), - PyTuple_GET_SIZE(arg)); - if (j < 0) { - return NULL; + if (PyTupleWriter_AddArray(writer, + _PyTuple_ITEMS(arg), + PyTuple_GET_SIZE(arg)) < 0) { + goto error; } continue; } } - PyTuple_SET_ITEM(subargs, j, Py_NewRef(arg)); - j++; + if (PyTupleWriter_Add(writer, arg) < 0) { + goto error; + } } - assert(j == PyTuple_GET_SIZE(subargs)); - obj = PyObject_GetItem(obj, subargs); + PyObject *subargs = PyTupleWriter_Finish(writer); + if (subargs != NULL) { + obj = PyObject_GetItem(obj, subargs); + Py_DECREF(subargs); + } + else { + obj = NULL; + } + goto done; - Py_DECREF(subargs); +error: + PyTupleWriter_Discard(writer); + Py_DECREF(subparams); + return NULL; } else { Py_INCREF(obj); } + +done: Py_XDECREF(subparams); return obj; } @@ -454,60 +450,50 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje } } Py_ssize_t nargs = PyTuple_GET_SIZE(args); - PyObject *newargs = PyTuple_New(nargs); + PyTupleWriter *newargs = PyTupleWriter_Create(nargs); if (newargs == NULL) { - Py_DECREF(item); - Py_XDECREF(tuple_args); - return NULL; + goto error; } - for (Py_ssize_t iarg = 0, jarg = 0; iarg < nargs; iarg++) { + + for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { PyObject *arg = PyTuple_GET_ITEM(args, iarg); if (PyType_Check(arg)) { - PyTuple_SET_ITEM(newargs, jarg, Py_NewRef(arg)); - jarg++; + if (PyTupleWriter_Add(newargs, arg) < 0) { + goto error; + } continue; } + // Recursively substitute params in lists/tuples. if (PyTuple_Check(arg) || PyList_Check(arg)) { PyObject *subargs = _Py_subs_parameters(self, arg, parameters, item); if (subargs == NULL) { - Py_DECREF(newargs); - Py_DECREF(item); - Py_XDECREF(tuple_args); - return NULL; + goto error; } if (PyTuple_Check(arg)) { - PyTuple_SET_ITEM(newargs, jarg, subargs); + if (PyTupleWriter_AddSteal(newargs, subargs) < 0) { + goto error; + } } else { // _Py_subs_parameters returns a tuple. If the original arg was a list, // convert subargs to a list as well. PyObject *subargs_list = PySequence_List(subargs); Py_DECREF(subargs); - if (subargs_list == NULL) { - Py_DECREF(newargs); - Py_DECREF(item); - Py_XDECREF(tuple_args); - return NULL; + if (PyTupleWriter_AddSteal(newargs, subargs_list) < 0) { + goto error; } - PyTuple_SET_ITEM(newargs, jarg, subargs_list); } - jarg++; continue; } + int unpack = _is_unpacked_typevartuple(arg); if (unpack < 0) { - Py_DECREF(newargs); - Py_DECREF(item); - Py_XDECREF(tuple_args); - return NULL; + goto error; } PyObject *subst; if (PyObject_GetOptionalAttr(arg, &_Py_ID(__typing_subst__), &subst) < 0) { - Py_DECREF(newargs); - Py_DECREF(item); - Py_XDECREF(tuple_args); - return NULL; + goto error; } if (subst) { Py_ssize_t iparam = tuple_index(parameters, nparams, arg); @@ -518,43 +504,42 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje else { arg = subs_tvars(arg, parameters, argitems, nitems); } - if (arg == NULL) { - Py_DECREF(newargs); - Py_DECREF(item); - Py_XDECREF(tuple_args); - return NULL; - } if (unpack) { + if (arg == NULL) { + goto error; + } if (!PyTuple_Check(arg)) { - Py_DECREF(newargs); - Py_DECREF(item); - Py_XDECREF(tuple_args); PyObject *original = PyTuple_GET_ITEM(args, iarg); PyErr_Format(PyExc_TypeError, "expected __typing_subst__ of %T objects to return a tuple, not %T", original, arg); Py_DECREF(arg); - return NULL; + goto error; } - jarg = tuple_extend(&newargs, jarg, - &PyTuple_GET_ITEM(arg, 0), PyTuple_GET_SIZE(arg)); - Py_DECREF(arg); - if (jarg < 0) { - Py_DECREF(item); - Py_XDECREF(tuple_args); - assert(newargs == NULL); - return NULL; + if (PyTupleWriter_AddArray(newargs, + _PyTuple_ITEMS(arg), + PyTuple_GET_SIZE(arg)) < 0) { + Py_DECREF(arg); + goto error; } + Py_DECREF(arg); } else { - PyTuple_SET_ITEM(newargs, jarg, arg); - jarg++; + if (PyTupleWriter_AddSteal(newargs, arg) < 0) { + goto error; + } } } Py_DECREF(item); Py_XDECREF(tuple_args); - return newargs; + return PyTupleWriter_Finish(newargs); + +error: + PyTupleWriter_Discard(newargs); + Py_DECREF(item); + Py_XDECREF(tuple_args); + return NULL; } PyDoc_STRVAR(genericalias__doc__, diff --git a/Objects/structseq.c b/Objects/structseq.c index c05bcde24c441b..eed4dcc26fccb2 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -496,38 +496,33 @@ initialize_structseq_dict(PyStructSequence_Desc *desc, PyObject* dict, SET_DICT_FROM_SIZE(unnamed_fields_key, n_unnamed_members); // Prepare and set __match_args__ - Py_ssize_t i, k; - PyObject* keys = PyTuple_New(desc->n_in_sequence); - if (keys == NULL) { + PyTupleWriter *writer = PyTupleWriter_Create(desc->n_in_sequence); + if (writer == NULL) { return -1; } - for (i = k = 0; i < desc->n_in_sequence; ++i) { + for (Py_ssize_t i = 0; i < desc->n_in_sequence; ++i) { if (desc->fields[i].name == PyStructSequence_UnnamedField) { continue; } - PyObject* new_member = PyUnicode_FromString(desc->fields[i].name); - if (new_member == NULL) { - goto error; + + PyObject *new_member = PyUnicode_FromString(desc->fields[i].name); + if (PyTupleWriter_AddSteal(writer, new_member) < 0) { + PyTupleWriter_Discard(writer); + return -1; } - PyTuple_SET_ITEM(keys, k, new_member); - k++; } - if (_PyTuple_Resize(&keys, k) == -1) { - goto error; + PyObject *keys = PyTupleWriter_Finish(writer); + if (keys == NULL) { + return -1; } - if (PyDict_SetItemString(dict, match_args_key, keys) < 0) { - goto error; + Py_DECREF(keys); + return -1; } - Py_DECREF(keys); return 0; - -error: - Py_DECREF(keys); - return -1; } static PyMemberDef * From 537989d3d5d91d82fa0fcce2214a2dfbd743ae1c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 11 Oct 2025 23:37:21 +0200 Subject: [PATCH 13/13] Update What's New --- Doc/c-api/tuple.rst | 2 +- Doc/whatsnew/3.15.rst | 2 ++ .../next/C_API/2025-10-10-09-54-28.gh-issue-139888.Z4Px61.rst | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index cc6fbd24ced155..05fdf8d1f16b2d 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -204,7 +204,7 @@ object. Add an item to *writer*. * On success, return ``0``. - * If *item* is ``NULL``, return ``-1`` with an exception set. + * If *item* is ``NULL`` with an exception set, return ``-1``. * On error, set an exception and return ``-1``. .. c:function:: int PyTupleWriter_AddSteal(PyTupleWriter *writer, PyObject *item) diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index db196419d51c18..cd02d32d554dda 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -859,6 +859,8 @@ New features * :c:func:`PyTupleWriter_Create` * :c:func:`PyTupleWriter_Add` + * :c:func:`PyTupleWriter_AddSteal` + * :c:func:`PyTupleWriter_AddArray` * :c:func:`PyTupleWriter_Finish` * :c:func:`PyTupleWriter_Discard` diff --git a/Misc/NEWS.d/next/C_API/2025-10-10-09-54-28.gh-issue-139888.Z4Px61.rst b/Misc/NEWS.d/next/C_API/2025-10-10-09-54-28.gh-issue-139888.Z4Px61.rst index 39e644bc059fcf..d4c692578b04ce 100644 --- a/Misc/NEWS.d/next/C_API/2025-10-10-09-54-28.gh-issue-139888.Z4Px61.rst +++ b/Misc/NEWS.d/next/C_API/2025-10-10-09-54-28.gh-issue-139888.Z4Px61.rst @@ -2,6 +2,8 @@ Add a :ref:`PyTupleWriter API ` to create :class:`tuple`: * :c:func:`PyTupleWriter_Create` * :c:func:`PyTupleWriter_Add` +* :c:func:`PyTupleWriter_AddSteal` +* :c:func:`PyTupleWriter_AddArray` * :c:func:`PyTupleWriter_Finish` * :c:func:`PyTupleWriter_Discard`