diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index 65f8334c437974..05fdf8d1f16b2d 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -142,6 +142,82 @@ Tuple Objects ``*p`` is destroyed. On failure, returns ``-1`` and sets ``*p`` to ``NULL``, and raises :exc:`MemoryError` or :exc:`SystemError`. + .. deprecated:: 3.15 + Use the :ref:`PyTupleWriter API ` instead. + + +.. _pytuplewriter: + +PyTupleWriter +------------- + +The :c:type:`PyTupleWriter` API can be used to create a Python :class:`tuple` +object. + +.. versionadded:: next + +.. c:type:: PyTupleWriter + + A tuple writer instance. + + The API is **not thread safe**: a writer should only be used by a single + thread at the same time. + + The instance must be destroyed by :c:func:`PyTupleWriter_Finish` on + success, or :c:func:`PyTupleWriter_Discard` on error. + + +.. c:function:: PyTupleWriter* PyTupleWriter_Create(Py_ssize_t size) + + Create a :c:type:`PyTupleWriter` to add *size* items. + + If *size* is greater than zero, preallocate *size* items. The caller is + responsible to add *size* items using :c:func:`PyTupleWriter_Add`. + + On error, set an exception and return ``NULL``. + + *size* must be positive or zero. + + +.. c:function:: PyObject* PyTupleWriter_Finish(PyTupleWriter *writer) + + Finish a :c:type:`PyTupleWriter` created by + :c:func:`PyTupleWriter_Create`. + + On success, return a Python :class:`tuple` object. + On error, set an exception and return ``NULL``. + + The writer instance is invalid after the call in any case. + No API can be called on the writer after :c:func:`PyTupleWriter_Finish`. + +.. c:function:: void PyTupleWriter_Discard(PyTupleWriter *writer) + + Discard a :c:type:`PyTupleWriter` created by :c:func:`PyTupleWriter_Create`. + + Do nothing if *writer* is ``NULL``. + + The writer instance is invalid after the call. + No API can be called on the writer after :c:func:`PyTupleWriter_Discard`. + +.. c:function:: int PyTupleWriter_Add(PyTupleWriter *writer, PyObject *item) + + Add an item to *writer*. + + * On success, return ``0``. + * If *item* is ``NULL`` with an exception set, return ``-1``. + * On error, set an exception and return ``-1``. + +.. c:function:: int PyTupleWriter_AddSteal(PyTupleWriter *writer, PyObject *item) + + Similar to :c:func:`PyTupleWriter_Add`, but take the ownership of *item*. + +.. c:function:: int PyTupleWriter_AddArray(PyTupleWriter *writer, PyObject *const *array, Py_ssize_t size) + + Add items from an array to *writer*. + + On success, return ``0``. + On error, set an exception and return ``-1``. + .. _struct-sequence-objects: diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 40286d4fe857e8..cd02d32d554dda 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -855,6 +855,17 @@ New features * Add :c:func:`PyTuple_FromArray` to create a :class:`tuple` from an array. (Contributed by Victor Stinner in :gh:`111489`.) +* Add a :ref:`PyTupleWriter API ` to create :class:`tuple`: + + * :c:func:`PyTupleWriter_Create` + * :c:func:`PyTupleWriter_Add` + * :c:func:`PyTupleWriter_AddSteal` + * :c:func:`PyTupleWriter_AddArray` + * :c:func:`PyTupleWriter_Finish` + * :c:func:`PyTupleWriter_Discard` + + (Contributed by Victor Stinner in :gh:`139888`.) + Porting to Python 3.15 ---------------------- @@ -915,6 +926,9 @@ Deprecated C APIs since 3.15 and will be removed in 3.17. (Contributed by Nikita Sobolev in :gh:`136355`.) +* Deprecate the :c:func:`_PyTuple_Resize` function: + use the new :ref:`PyTupleWriter API ` instead. + (Contributed by Victor Stinner in :gh:`139888`.) .. Add C API deprecations above alphabetically, not here at the end. diff --git a/Include/cpython/tupleobject.h b/Include/cpython/tupleobject.h index 888baaf3358267..f6bfeb2acd4043 100644 --- a/Include/cpython/tupleobject.h +++ b/Include/cpython/tupleobject.h @@ -12,7 +12,7 @@ typedef struct { PyObject *ob_item[1]; } PyTupleObject; -PyAPI_FUNC(int) _PyTuple_Resize(PyObject **, Py_ssize_t); +_Py_DEPRECATED_EXTERNALLY(3.15) PyAPI_FUNC(int) _PyTuple_Resize(PyObject **, Py_ssize_t); /* Cast argument to PyTupleObject* type. */ #define _PyTuple_CAST(op) \ @@ -42,3 +42,21 @@ PyTuple_SET_ITEM(PyObject *op, Py_ssize_t index, PyObject *value) { PyAPI_FUNC(PyObject*) PyTuple_FromArray( PyObject *const *array, Py_ssize_t size); + +// --- Public PyUnicodeWriter API -------------------------------------------- + +typedef struct PyTupleWriter PyTupleWriter; + +PyAPI_FUNC(PyTupleWriter*) PyTupleWriter_Create(Py_ssize_t size); +PyAPI_FUNC(int) PyTupleWriter_Add( + PyTupleWriter *writer, + PyObject *item); +PyAPI_FUNC(int) PyTupleWriter_AddSteal( + PyTupleWriter *writer, + PyObject *item); +PyAPI_FUNC(int) PyTupleWriter_AddArray( + PyTupleWriter *writer, + PyObject *const *array, + Py_ssize_t size); +PyAPI_FUNC(PyObject*) PyTupleWriter_Finish(PyTupleWriter *writer); +PyAPI_FUNC(void) PyTupleWriter_Discard(PyTupleWriter *writer); diff --git a/Include/internal/pycore_freelist_state.h b/Include/internal/pycore_freelist_state.h index 46e2a82ea03456..51e6273b75df79 100644 --- a/Include/internal/pycore_freelist_state.h +++ b/Include/internal/pycore_freelist_state.h @@ -28,6 +28,7 @@ extern "C" { # define Py_object_stack_chunks_MAXFREELIST 4 # define Py_unicode_writers_MAXFREELIST 1 # define Py_bytes_writers_MAXFREELIST 1 +# define Py_tuple_writers_MAXFREELIST 1 # define Py_pycfunctionobject_MAXFREELIST 16 # define Py_pycmethodobject_MAXFREELIST 16 # define Py_pymethodobjects_MAXFREELIST 20 @@ -63,6 +64,7 @@ struct _Py_freelists { struct _Py_freelist object_stack_chunks; struct _Py_freelist unicode_writers; struct _Py_freelist bytes_writers; + struct _Py_freelist tuple_writers; struct _Py_freelist pycfunctionobject; struct _Py_freelist pycmethodobject; struct _Py_freelist pymethodobjects; diff --git a/Include/internal/pycore_tuple.h b/Include/internal/pycore_tuple.h index be1961cbf77a2d..3556feadb28ad6 100644 --- a/Include/internal/pycore_tuple.h +++ b/Include/internal/pycore_tuple.h @@ -71,6 +71,10 @@ _PyTuple_Recycle(PyObject *op) #endif #define _PyTuple_HASH_EMPTY (_PyTuple_HASH_XXPRIME_5 + (_PyTuple_HASH_XXPRIME_5 ^ 3527539UL)) +extern PyObject** _PyTupleWriter_GetItems( + PyTupleWriter *writer, + Py_ssize_t *size); + #ifdef __cplusplus } #endif diff --git a/Lib/test/test_capi/test_tuple.py b/Lib/test/test_capi/test_tuple.py index b6d6da008d0b7b..5ed612db6b25dd 100644 --- a/Lib/test/test_capi/test_tuple.py +++ b/Lib/test/test_capi/test_tuple.py @@ -302,5 +302,71 @@ def my_iter(): self.assertEqual(tuples, []) +class TupleWriterTest(unittest.TestCase): + def create_writer(self, size): + return _testcapi.PyTupleWriter(size) + + def test_create(self): + # Test PyTupleWriter_Create() + writer = self.create_writer(0) + self.assertIs(writer.finish(), ()) + + writer = self.create_writer(123) + self.assertIs(writer.finish(), ()) + + with self.assertRaises(SystemError): + self.create_writer(-2) + + def check_add(self, name): + writer = self.create_writer(3) + add = getattr(writer, name) + for ch in 'abc': + add(ch) + self.assertEqual(writer.finish(), ('a', 'b', 'c')) + + writer = self.create_writer(0) + add = getattr(writer, name) + for i in range(1024): + add(i) + self.assertEqual(writer.finish(), tuple(range(1024))) + + writer = self.create_writer(1) + add = getattr(writer, name) + with self.assertRaises(SystemError): + add(NULL) + + def test_add(self): + # Test PyTupleWriter_Add() + self.check_add('add') + + def test_add_steal(self): + # Test PyTupleWriter_AddSteal() + self.check_add('add_steal') + + def test_add_array(self): + writer = self.create_writer(0) + writer.add_array(('a', 'b', 'c')) + writer.add_array(('d', 'e')) + self.assertEqual(writer.finish(), ('a', 'b', 'c', 'd', 'e')) + + writer = self.create_writer(0) + writer.add_array(tuple(range(1024))) + self.assertEqual(writer.finish(), tuple(range(1024))) + + def test_discard(self): + # test the small_tuple buffer (16 items) + writer = self.create_writer(3) + writer.add(object()) + writer.add(object()) + # must not leak references + writer.discard() + + # test the tuple code path (17 items or more) + writer = self.create_writer(1024) + writer.add_array(tuple(range(1024))) + # must not leak references + writer.discard() + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/C_API/2025-10-10-09-54-28.gh-issue-139888.Z4Px61.rst b/Misc/NEWS.d/next/C_API/2025-10-10-09-54-28.gh-issue-139888.Z4Px61.rst new file mode 100644 index 00000000000000..d4c692578b04ce --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-10-10-09-54-28.gh-issue-139888.Z4Px61.rst @@ -0,0 +1,10 @@ +Add a :ref:`PyTupleWriter API ` to create :class:`tuple`: + +* :c:func:`PyTupleWriter_Create` +* :c:func:`PyTupleWriter_Add` +* :c:func:`PyTupleWriter_AddSteal` +* :c:func:`PyTupleWriter_AddArray` +* :c:func:`PyTupleWriter_Finish` +* :c:func:`PyTupleWriter_Discard` + +Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2025-10-10-09-58-57.gh-issue-139888.xCfJ3L.rst b/Misc/NEWS.d/next/C_API/2025-10-10-09-58-57.gh-issue-139888.xCfJ3L.rst new file mode 100644 index 00000000000000..77bbda98441488 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-10-10-09-58-57.gh-issue-139888.xCfJ3L.rst @@ -0,0 +1,2 @@ +Deprecate the :c:func:`_PyTuple_Resize` function: use the :ref:`PyTupleWriter +API ` instead. diff --git a/Modules/_testcapi/tuple.c b/Modules/_testcapi/tuple.c index 5de1c494c0a8c0..27f464a5ff1baf 100644 --- a/Modules/_testcapi/tuple.c +++ b/Modules/_testcapi/tuple.c @@ -84,7 +84,10 @@ _tuple_resize(PyObject *Py_UNUSED(module), PyObject *args) NULLABLE(tup); Py_XINCREF(tup); } + _Py_COMP_DIAG_PUSH + _Py_COMP_DIAG_IGNORE_DEPR_DECLS int r = _PyTuple_Resize(&tup, newsize); + _Py_COMP_DIAG_POP if (r == -1) { assert(tup == NULL); return NULL; @@ -131,6 +134,185 @@ tuple_fromarray(PyObject* Py_UNUSED(module), PyObject *args) } +// --- PyTupleWriter type --------------------------------------------------- + +typedef struct { + PyObject_HEAD + PyTupleWriter *writer; +} WriterObject; + + +static PyObject * +writer_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + WriterObject *self = (WriterObject *)type->tp_alloc(type, 0); + if (!self) { + return NULL; + } + self->writer = NULL; + return (PyObject*)self; +} + + +static int +writer_init(PyObject *self_raw, PyObject *args, PyObject *kwargs) +{ + if (kwargs && PyDict_GET_SIZE(kwargs)) { + PyErr_Format(PyExc_TypeError, + "PyTupleWriter() takes exactly no keyword arguments"); + return -1; + } + + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "n", &size)) { + return -1; + } + + WriterObject *self = (WriterObject *)self_raw; + if (self->writer) { + PyTupleWriter_Discard(self->writer); + } + self->writer = PyTupleWriter_Create(size); + if (self->writer == NULL) { + return -1; + } + return 0; +} + + +static void +writer_dealloc(PyObject *self_raw) +{ + WriterObject *self = (WriterObject *)self_raw; + PyTypeObject *tp = Py_TYPE(self); + if (self->writer) { + PyTupleWriter_Discard(self->writer); + } + tp->tp_free(self); + Py_DECREF(tp); +} + + +static inline int +writer_check(WriterObject *self) +{ + if (self->writer == NULL) { + PyErr_SetString(PyExc_ValueError, "operation on finished writer"); + return -1; + } + return 0; +} + + +static PyObject* +writer_add(PyObject *self_raw, PyObject *item) +{ + NULLABLE(item); + + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + if (PyTupleWriter_Add(self->writer, item) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + + +static PyObject* +writer_add_steal(PyObject *self_raw, PyObject *item) +{ + NULLABLE(item); + + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + if (PyTupleWriter_AddSteal(self->writer, Py_XNewRef(item)) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + + +static PyObject* +writer_add_array(PyObject *self_raw, PyObject *args) +{ + PyObject *tuple; + if (!PyArg_ParseTuple(args, "O!", &PyTuple_Type, &tuple)) { + return NULL; + } + + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + PyObject **array = &PyTuple_GET_ITEM(tuple, 0); + Py_ssize_t size = PyTuple_GET_SIZE(tuple); + if (PyTupleWriter_AddArray(self->writer, array, size) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + + +static PyObject* +writer_finish(PyObject *self_raw, PyObject *Py_UNUSED(args)) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + PyObject *tuple = PyTupleWriter_Finish(self->writer); + self->writer = NULL; + return tuple; +} + + +static PyObject* +writer_discard(PyObject *self_raw, PyObject *Py_UNUSED(args)) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + PyTupleWriter_Discard(self->writer); + self->writer = NULL; + Py_RETURN_NONE; +} + + +static PyMethodDef writer_methods[] = { + {"add", _PyCFunction_CAST(writer_add), METH_O}, + {"add_steal", _PyCFunction_CAST(writer_add_steal), METH_O}, + {"add_array", _PyCFunction_CAST(writer_add_array), METH_VARARGS}, + {"finish", _PyCFunction_CAST(writer_finish), METH_NOARGS}, + {"discard", _PyCFunction_CAST(writer_discard), METH_NOARGS}, + {NULL, NULL} /* sentinel */ +}; + +static PyType_Slot Writer_Type_slots[] = { + {Py_tp_new, writer_new}, + {Py_tp_init, writer_init}, + {Py_tp_dealloc, writer_dealloc}, + {Py_tp_methods, writer_methods}, + {0, 0}, /* sentinel */ +}; + +static PyType_Spec Writer_spec = { + .name = "_testcapi.PyTupleWriter", + .basicsize = sizeof(WriterObject), + .flags = Py_TPFLAGS_DEFAULT, + .slots = Writer_Type_slots, +}; + + static PyMethodDef test_methods[] = { {"tuple_get_size", tuple_get_size, METH_O}, {"tuple_get_item", tuple_get_item, METH_VARARGS}, @@ -148,5 +330,15 @@ _PyTestCapi_Init_Tuple(PyObject *m) return -1; } + PyTypeObject *writer_type = (PyTypeObject *)PyType_FromSpec(&Writer_spec); + if (writer_type == NULL) { + return -1; + } + if (PyModule_AddType(m, writer_type) < 0) { + Py_DECREF(writer_type); + return -1; + } + Py_DECREF(writer_type); + return 0; } diff --git a/Objects/abstract.c b/Objects/abstract.c index 8adad8407d04d4..60353547a3c959 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -1996,8 +1996,6 @@ PySequence_DelSlice(PyObject *s, Py_ssize_t i1, Py_ssize_t i2) PyObject * PySequence_Tuple(PyObject *v) { - PyObject *it; /* iter(v) */ - if (v == NULL) { return null_error(); } @@ -2010,62 +2008,36 @@ PySequence_Tuple(PyObject *v) a copy, so there's no need for exactness below. */ return Py_NewRef(v); } - if (PyList_CheckExact(v)) + if (PyList_CheckExact(v)) { return PyList_AsTuple(v); + } /* Get iterator. */ - it = PyObject_GetIter(v); - if (it == NULL) + PyObject *it = PyObject_GetIter(v); + if (it == NULL) { return NULL; - - Py_ssize_t n; - PyObject *buffer[8]; - for (n = 0; n < 8; n++) { - PyObject *item = PyIter_Next(it); - if (item == NULL) { - if (PyErr_Occurred()) { - goto fail; - } - Py_DECREF(it); - return _PyTuple_FromArraySteal(buffer, n); - } - buffer[n] = item; } - PyListObject *list = (PyListObject *)PyList_New(16); - if (list == NULL) { + + PyTupleWriter *writer = PyTupleWriter_Create(0); + if (writer == NULL) { goto fail; } - assert(n == 8); - Py_SET_SIZE(list, n); - for (Py_ssize_t j = 0; j < n; j++) { - PyList_SET_ITEM(list, j, buffer[j]); - } + for (;;) { - PyObject *item = PyIter_Next(it); - if (item == NULL) { - if (PyErr_Occurred()) { - Py_DECREF(list); - Py_DECREF(it); - return NULL; - } + PyObject *item; + if (PyIter_NextItem(it, &item) == 0) { break; } - if (_PyList_AppendTakeRef(list, item) < 0) { - Py_DECREF(list); - Py_DECREF(it); - return NULL; + if (PyTupleWriter_AddSteal(writer, item) < 0) { + goto fail; } } Py_DECREF(it); - PyObject *res = _PyList_AsTupleAndClear(list); - Py_DECREF(list); - return res; + return PyTupleWriter_Finish(writer); + fail: Py_DECREF(it); - while (n > 0) { - n--; - Py_DECREF(buffer[n]); - } + PyTupleWriter_Discard(writer); return NULL; } diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 8b526f43f1e053..4eef4ca6150e86 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -4,6 +4,7 @@ #include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_object.h" +#include "pycore_tuple.h" // _PyTuple_ITEMS() #include "pycore_typevarobject.h" // _Py_typing_type_repr #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString() #include "pycore_unionobject.h" // _Py_union_type_or, _PyGenericAlias_Check @@ -152,29 +153,17 @@ tuple_index(PyObject *self, Py_ssize_t len, PyObject *item) } static int -tuple_add(PyObject *self, Py_ssize_t len, PyObject *item) +tuplewriter_add(PyTupleWriter *writer, PyObject *item) { - if (tuple_index(self, len, item) < 0) { - PyTuple_SET_ITEM(self, len, Py_NewRef(item)); - return 1; + Py_ssize_t len; + PyObject **items = _PyTupleWriter_GetItems(writer, &len); + for (Py_ssize_t i = 0; i < len; i++) { + if (items[i] == item) { + return 0; + } } - return 0; -} -static Py_ssize_t -tuple_extend(PyObject **dst, Py_ssize_t dstindex, - PyObject **src, Py_ssize_t count) -{ - assert(count >= 0); - if (_PyTuple_Resize(dst, PyTuple_GET_SIZE(*dst) + count - 1) != 0) { - return -1; - } - assert(dstindex + count <= PyTuple_GET_SIZE(*dst)); - for (Py_ssize_t i = 0; i < count; ++i) { - PyObject *item = src[i]; - PyTuple_SET_ITEM(*dst, dstindex + i, Py_NewRef(item)); - } - return dstindex + count; + return PyTupleWriter_Add(writer, item); } PyObject * @@ -190,13 +179,10 @@ _Py_make_parameters(PyObject *args) } } Py_ssize_t nargs = PyTuple_GET_SIZE(args); - Py_ssize_t len = nargs; - PyObject *parameters = PyTuple_New(len); + PyTupleWriter *parameters = PyTupleWriter_Create(nargs); if (parameters == NULL) { - Py_XDECREF(tuple_args); - return NULL; + goto error; } - Py_ssize_t iparam = 0; for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { PyObject *t = PyTuple_GET_ITEM(args, iarg); // We don't want __parameters__ descriptor of a bare Python class. @@ -205,60 +191,47 @@ _Py_make_parameters(PyObject *args) } int rc = PyObject_HasAttrWithError(t, &_Py_ID(__typing_subst__)); if (rc < 0) { - Py_DECREF(parameters); - Py_XDECREF(tuple_args); - return NULL; + goto error; } if (rc) { - iparam += tuple_add(parameters, iparam, t); + if (tuplewriter_add(parameters, t) < 0) { + goto error; + } } else { PyObject *subparams; if (PyObject_GetOptionalAttr(t, &_Py_ID(__parameters__), &subparams) < 0) { - Py_DECREF(parameters); - Py_XDECREF(tuple_args); - return NULL; + goto error; } if (!subparams && (PyTuple_Check(t) || PyList_Check(t))) { // Recursively call _Py_make_parameters for lists/tuples and // add the results to the current parameters. subparams = _Py_make_parameters(t); if (subparams == NULL) { - Py_DECREF(parameters); - Py_XDECREF(tuple_args); - return NULL; + goto error; } } if (subparams && PyTuple_Check(subparams)) { Py_ssize_t len2 = PyTuple_GET_SIZE(subparams); - Py_ssize_t needed = len2 - 1 - (iarg - iparam); - if (needed > 0) { - len += needed; - if (_PyTuple_Resize(¶meters, len) < 0) { - Py_DECREF(subparams); - Py_DECREF(parameters); - Py_XDECREF(tuple_args); - return NULL; - } - } for (Py_ssize_t j = 0; j < len2; j++) { PyObject *t2 = PyTuple_GET_ITEM(subparams, j); - iparam += tuple_add(parameters, iparam, t2); + if (tuplewriter_add(parameters, t2) < 0) { + Py_DECREF(subparams); + goto error; + } } } Py_XDECREF(subparams); } } - if (iparam < len) { - if (_PyTuple_Resize(¶meters, iparam) < 0) { - Py_XDECREF(parameters); - Py_XDECREF(tuple_args); - return NULL; - } - } Py_XDECREF(tuple_args); - return parameters; + return PyTupleWriter_Finish(parameters); + +error: + PyTupleWriter_Discard(parameters); + Py_XDECREF(tuple_args); + return NULL; } /* If obj is a generic alias, substitute type variables params @@ -274,15 +247,14 @@ subs_tvars(PyObject *obj, PyObject *params, if (PyObject_GetOptionalAttr(obj, &_Py_ID(__parameters__), &subparams) < 0) { return NULL; } + if (subparams && PyTuple_Check(subparams) && PyTuple_GET_SIZE(subparams)) { Py_ssize_t nparams = PyTuple_GET_SIZE(params); Py_ssize_t nsubargs = PyTuple_GET_SIZE(subparams); - PyObject *subargs = PyTuple_New(nsubargs); - if (subargs == NULL) { - Py_DECREF(subparams); - return NULL; + PyTupleWriter *writer = PyTupleWriter_Create(nsubargs); + if (writer == NULL) { + goto error; } - Py_ssize_t j = 0; for (Py_ssize_t i = 0; i < nsubargs; ++i) { PyObject *arg = PyTuple_GET_ITEM(subparams, i); Py_ssize_t iparam = tuple_index(params, nparams, arg); @@ -290,27 +262,39 @@ subs_tvars(PyObject *obj, PyObject *params, PyObject *param = PyTuple_GET_ITEM(params, iparam); arg = argitems[iparam]; if (Py_TYPE(param)->tp_iter && PyTuple_Check(arg)) { // TypeVarTuple - j = tuple_extend(&subargs, j, - &PyTuple_GET_ITEM(arg, 0), - PyTuple_GET_SIZE(arg)); - if (j < 0) { - return NULL; + if (PyTupleWriter_AddArray(writer, + _PyTuple_ITEMS(arg), + PyTuple_GET_SIZE(arg)) < 0) { + goto error; } continue; } } - PyTuple_SET_ITEM(subargs, j, Py_NewRef(arg)); - j++; + if (PyTupleWriter_Add(writer, arg) < 0) { + goto error; + } } - assert(j == PyTuple_GET_SIZE(subargs)); - obj = PyObject_GetItem(obj, subargs); + PyObject *subargs = PyTupleWriter_Finish(writer); + if (subargs != NULL) { + obj = PyObject_GetItem(obj, subargs); + Py_DECREF(subargs); + } + else { + obj = NULL; + } + goto done; - Py_DECREF(subargs); +error: + PyTupleWriter_Discard(writer); + Py_DECREF(subparams); + return NULL; } else { Py_INCREF(obj); } + +done: Py_XDECREF(subparams); return obj; } @@ -454,60 +438,50 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje } } Py_ssize_t nargs = PyTuple_GET_SIZE(args); - PyObject *newargs = PyTuple_New(nargs); + PyTupleWriter *newargs = PyTupleWriter_Create(nargs); if (newargs == NULL) { - Py_DECREF(item); - Py_XDECREF(tuple_args); - return NULL; + goto error; } - for (Py_ssize_t iarg = 0, jarg = 0; iarg < nargs; iarg++) { + + for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { PyObject *arg = PyTuple_GET_ITEM(args, iarg); if (PyType_Check(arg)) { - PyTuple_SET_ITEM(newargs, jarg, Py_NewRef(arg)); - jarg++; + if (PyTupleWriter_Add(newargs, arg) < 0) { + goto error; + } continue; } + // Recursively substitute params in lists/tuples. if (PyTuple_Check(arg) || PyList_Check(arg)) { PyObject *subargs = _Py_subs_parameters(self, arg, parameters, item); if (subargs == NULL) { - Py_DECREF(newargs); - Py_DECREF(item); - Py_XDECREF(tuple_args); - return NULL; + goto error; } if (PyTuple_Check(arg)) { - PyTuple_SET_ITEM(newargs, jarg, subargs); + if (PyTupleWriter_AddSteal(newargs, subargs) < 0) { + goto error; + } } else { // _Py_subs_parameters returns a tuple. If the original arg was a list, // convert subargs to a list as well. PyObject *subargs_list = PySequence_List(subargs); Py_DECREF(subargs); - if (subargs_list == NULL) { - Py_DECREF(newargs); - Py_DECREF(item); - Py_XDECREF(tuple_args); - return NULL; + if (PyTupleWriter_AddSteal(newargs, subargs_list) < 0) { + goto error; } - PyTuple_SET_ITEM(newargs, jarg, subargs_list); } - jarg++; continue; } + int unpack = _is_unpacked_typevartuple(arg); if (unpack < 0) { - Py_DECREF(newargs); - Py_DECREF(item); - Py_XDECREF(tuple_args); - return NULL; + goto error; } PyObject *subst; if (PyObject_GetOptionalAttr(arg, &_Py_ID(__typing_subst__), &subst) < 0) { - Py_DECREF(newargs); - Py_DECREF(item); - Py_XDECREF(tuple_args); - return NULL; + goto error; } if (subst) { Py_ssize_t iparam = tuple_index(parameters, nparams, arg); @@ -518,43 +492,42 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje else { arg = subs_tvars(arg, parameters, argitems, nitems); } - if (arg == NULL) { - Py_DECREF(newargs); - Py_DECREF(item); - Py_XDECREF(tuple_args); - return NULL; - } if (unpack) { + if (arg == NULL) { + goto error; + } if (!PyTuple_Check(arg)) { - Py_DECREF(newargs); - Py_DECREF(item); - Py_XDECREF(tuple_args); PyObject *original = PyTuple_GET_ITEM(args, iarg); PyErr_Format(PyExc_TypeError, "expected __typing_subst__ of %T objects to return a tuple, not %T", original, arg); Py_DECREF(arg); - return NULL; + goto error; } - jarg = tuple_extend(&newargs, jarg, - &PyTuple_GET_ITEM(arg, 0), PyTuple_GET_SIZE(arg)); - Py_DECREF(arg); - if (jarg < 0) { - Py_DECREF(item); - Py_XDECREF(tuple_args); - assert(newargs == NULL); - return NULL; + if (PyTupleWriter_AddArray(newargs, + _PyTuple_ITEMS(arg), + PyTuple_GET_SIZE(arg)) < 0) { + Py_DECREF(arg); + goto error; } + Py_DECREF(arg); } else { - PyTuple_SET_ITEM(newargs, jarg, arg); - jarg++; + if (PyTupleWriter_AddSteal(newargs, arg) < 0) { + goto error; + } } } Py_DECREF(item); Py_XDECREF(tuple_args); - return newargs; + return PyTupleWriter_Finish(newargs); + +error: + PyTupleWriter_Discard(newargs); + Py_DECREF(item); + Py_XDECREF(tuple_args); + return NULL; } PyDoc_STRVAR(genericalias__doc__, diff --git a/Objects/object.c b/Objects/object.c index 0540112d7d2acf..a51a0a53b0728b 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -946,6 +946,7 @@ _PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization) } clear_freelist(&freelists->unicode_writers, is_finalization, PyMem_Free); clear_freelist(&freelists->bytes_writers, is_finalization, PyMem_Free); + clear_freelist(&freelists->tuple_writers, is_finalization, PyMem_Free); clear_freelist(&freelists->ints, is_finalization, free_object); clear_freelist(&freelists->pycfunctionobject, is_finalization, PyObject_GC_Del); clear_freelist(&freelists->pycmethodobject, is_finalization, PyObject_GC_Del); diff --git a/Objects/structseq.c b/Objects/structseq.c index c05bcde24c441b..eed4dcc26fccb2 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -496,38 +496,33 @@ initialize_structseq_dict(PyStructSequence_Desc *desc, PyObject* dict, SET_DICT_FROM_SIZE(unnamed_fields_key, n_unnamed_members); // Prepare and set __match_args__ - Py_ssize_t i, k; - PyObject* keys = PyTuple_New(desc->n_in_sequence); - if (keys == NULL) { + PyTupleWriter *writer = PyTupleWriter_Create(desc->n_in_sequence); + if (writer == NULL) { return -1; } - for (i = k = 0; i < desc->n_in_sequence; ++i) { + for (Py_ssize_t i = 0; i < desc->n_in_sequence; ++i) { if (desc->fields[i].name == PyStructSequence_UnnamedField) { continue; } - PyObject* new_member = PyUnicode_FromString(desc->fields[i].name); - if (new_member == NULL) { - goto error; + + PyObject *new_member = PyUnicode_FromString(desc->fields[i].name); + if (PyTupleWriter_AddSteal(writer, new_member) < 0) { + PyTupleWriter_Discard(writer); + return -1; } - PyTuple_SET_ITEM(keys, k, new_member); - k++; } - if (_PyTuple_Resize(&keys, k) == -1) { - goto error; + PyObject *keys = PyTupleWriter_Finish(writer); + if (keys == NULL) { + return -1; } - if (PyDict_SetItemString(dict, match_args_key, keys) < 0) { - goto error; + Py_DECREF(keys); + return -1; } - Py_DECREF(keys); return 0; - -error: - Py_DECREF(keys); - return -1; } static PyMemberDef * diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 1fa4bae638a1fe..a126f54d9da1f6 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -71,8 +71,8 @@ tuple_get_empty(void) return (PyObject *)&_Py_SINGLETON(tuple_empty); } -PyObject * -PyTuple_New(Py_ssize_t size) +static PyObject * +_PyTuple_NewNoTrack(Py_ssize_t size) { PyTupleObject *op; if (size == 0) { @@ -85,10 +85,25 @@ PyTuple_New(Py_ssize_t size) for (Py_ssize_t i = 0; i < size; i++) { op->ob_item[i] = NULL; } - _PyObject_GC_TRACK(op); return (PyObject *) op; } + +PyObject * +PyTuple_New(Py_ssize_t size) +{ + if (size == 0) { + return tuple_get_empty(); + } + PyObject *op = _PyTuple_NewNoTrack(size); + if (op == NULL) { + return NULL; + } + _PyObject_GC_TRACK(op); + return op; +} + + Py_ssize_t PyTuple_Size(PyObject *op) { @@ -913,8 +928,8 @@ PyTypeObject PyTuple_Type = { efficiently. In any case, don't use this if the tuple may already be known to some other part of the code. */ -int -_PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) +static int +tuple_resize(PyObject **pv, Py_ssize_t newsize, int track) { PyTupleObject *v; PyTupleObject *sv; @@ -946,7 +961,12 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) /* The empty tuple is statically allocated so we never resize it in-place. */ Py_DECREF(v); - *pv = PyTuple_New(newsize); + if (track) { + *pv = PyTuple_New(newsize); + } + else { + *pv = _PyTuple_NewNoTrack(newsize); + } return *pv == NULL ? -1 : 0; } @@ -972,14 +992,32 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) } _Py_NewReferenceNoTotal((PyObject *) sv); /* Zero out items added by growing */ - if (newsize > oldsize) + if (newsize > oldsize) { memset(&sv->ob_item[oldsize], 0, sizeof(*sv->ob_item) * (newsize - oldsize)); + } *pv = (PyObject *) sv; - _PyObject_GC_TRACK(sv); + if (track) { + _PyObject_GC_TRACK(sv); + } return 0; } + +static int +_PyTuple_ResizeNoTrack(PyObject **pv, Py_ssize_t newsize) +{ + return tuple_resize(pv, newsize, 0); +} + + +int +_PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) +{ + return tuple_resize(pv, newsize, 1); +} + + /*********************** Tuple Iterator **************************/ #define _PyTupleIterObject_CAST(op) ((_PyTupleIterObject *)(op)) @@ -1181,3 +1219,228 @@ _PyTuple_DebugMallocStats(FILE *out) _PyObject_VAR_SIZE(&PyTuple_Type, len)); } } + + +// --- Public PyTupleWriter API ---------------------------------------------- + +struct PyTupleWriter { + PyObject* small_tuple[16]; + PyObject *tuple; + PyObject **items; + size_t size; + size_t allocated; +}; + + +static inline void +tuplewriter_free(PyTupleWriter *writer) +{ + _Py_FREELIST_FREE(tuple_writers, writer, PyMem_Free); +} + + +static int +_PyTupleWriter_SetSize(PyTupleWriter *writer, size_t size) +{ + assert(size >= 1); + + size += (size >> 1); // Over-allocate by 50% + + if (size > (size_t)PY_SSIZE_T_MAX) { + /* Check for overflow */ + PyErr_NoMemory(); + return -1; + } + + if (writer->tuple != NULL) { + if (_PyTuple_ResizeNoTrack(&writer->tuple, (Py_ssize_t)size) < 0) { + return -1; + } + } + else { + writer->tuple = _PyTuple_NewNoTrack((Py_ssize_t)size); + if (writer->tuple == NULL) { + return -1; + } + + if (writer->size > 0) { + memcpy(_PyTuple_ITEMS(writer->tuple), + writer->small_tuple, + writer->size * sizeof(writer->small_tuple[0])); + } + } + writer->items = _PyTuple_ITEMS(writer->tuple); + writer->allocated = size; + return 0; +} + + +PyTupleWriter* +PyTupleWriter_Create(Py_ssize_t size) +{ + if (size < 0) { + PyErr_BadInternalCall(); + return NULL; + } + + PyTupleWriter *writer = _Py_FREELIST_POP_MEM(tuple_writers); + if (writer == NULL) { + writer = (PyTupleWriter *)PyMem_Malloc(sizeof(PyTupleWriter)); + if (writer == NULL) { + PyErr_NoMemory(); + return NULL; + } + } + writer->size = 0; + + if (size != 0) { + writer->tuple = _PyTuple_NewNoTrack(size); + if (writer->tuple == NULL) { + tuplewriter_free(writer); + return NULL; + } + writer->items = _PyTuple_ITEMS(writer->tuple); + writer->allocated = size; + } + else { +#ifdef Py_DEBUG + memset(writer->small_tuple, 0xff, sizeof(writer->small_tuple)); +#endif + writer->tuple = NULL; + writer->items = writer->small_tuple; + writer->allocated = Py_ARRAY_LENGTH(writer->small_tuple); + } + return writer; +} + + +int +PyTupleWriter_Add(PyTupleWriter *writer, PyObject *item) +{ + if (item == NULL) { + if (!PyErr_Occurred()) { + PyErr_BadInternalCall(); + } + return -1; + } + + if (writer->size >= writer->allocated) { + if (_PyTupleWriter_SetSize(writer, writer->size + 1) < 0) { + return -1; + } + } + + writer->items[writer->size] = Py_NewRef(item); + writer->size++; + return 0; +} + + +int +PyTupleWriter_AddSteal(PyTupleWriter *writer, PyObject *item) +{ + if (item == NULL) { + if (!PyErr_Occurred()) { + PyErr_BadInternalCall(); + } + return -1; + } + + if (writer->size >= writer->allocated) { + if (_PyTupleWriter_SetSize(writer, writer->size + 1) < 0) { + Py_DECREF(item); + return -1; + } + } + + writer->items[writer->size] = item; + writer->size++; + return 0; +} + + +int +PyTupleWriter_AddArray(PyTupleWriter *writer, + PyObject *const *array, Py_ssize_t size) +{ + if (size < 0) { + PyErr_BadInternalCall(); + return -1; + } + + if ((writer->size + (size_t)size) > writer->allocated) { + if (_PyTupleWriter_SetSize(writer, + writer->size + (size_t)size) < 0) { + return -1; + } + } + + PyObject **items = &writer->items[writer->size]; + for (Py_ssize_t i=0; i < size; i++) { + *items++ = Py_NewRef(*array++); + } + writer->size += size; + return 0; +} + + +PyObject** +_PyTupleWriter_GetItems(PyTupleWriter *writer, Py_ssize_t *size) +{ + *size = writer->size; + return writer->items; +} + + +void +PyTupleWriter_Discard(PyTupleWriter *writer) +{ + if (writer == NULL) { + return; + } + + if (writer->tuple != NULL) { + Py_DECREF(writer->tuple); + } + else { + PyObject **items = writer->items; + for (size_t i=0; i < writer->size; i++) { + Py_DECREF(items[i]); + } + } + + tuplewriter_free(writer); +} + + +PyObject* +PyTupleWriter_Finish(PyTupleWriter *writer) +{ + if (writer->size == 0) { + PyTupleWriter_Discard(writer); + // return the empty tuple singleton + return PyTuple_New(0); + } + + PyObject *result; + if (writer->tuple != NULL) { + if (writer->size != writer->allocated) { + if (_PyTuple_ResizeNoTrack(&writer->tuple, + (Py_ssize_t)writer->size) < 0) { + PyTupleWriter_Discard(writer); + return NULL; + } + } + + result = writer->tuple; + writer->tuple = NULL; + _PyObject_GC_TRACK(result); + } + else { + result = _PyTuple_FromArraySteal(writer->items, + (Py_ssize_t)writer->size); + } + + tuplewriter_free(writer); + return result; +}