Skip to content

bpo-36030: Add an internal _PyTuple_NewUnsafe() function #11927

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
sir-sigurd wants to merge 1 commit into from
Closed

bpo-36030: Add an internal _PyTuple_NewUnsafe() function #11927

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Include/internal/pycore_tupleobject.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ extern "C" {
#include "tupleobject.h"

#define _PyTuple_ITEMS(op) (_PyTuple_CAST(op)->ob_item)
PyAPI_FUNC(PyObject *) _PyTuple_NewUnsafe(Py_ssize_t size);

#ifdef __cplusplus
}
Expand Down
8 changes: 4 additions & 4 deletions Objects/call.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -339,7 +339,7 @@ _PyFunction_FastCallDict(PyObject *func, PyObject *const *args, Py_ssize_t nargs

/* bpo-29318, bpo-27840: Caller and callee functions must not share
the dictionary: kwargs must be copied. */
kwtuple = PyTuple_New(2 * nk);
kwtuple = _PyTuple_NewUnsafe(2 * nk);
if (kwtuple == NULL) {
return NULL;
}
Expand Down Expand Up @@ -1279,7 +1279,7 @@ _PyStack_AsTuple(PyObject *const *stack, Py_ssize_t nargs)
PyObject *args;
Py_ssize_t i;

args = PyTuple_New(nargs);
args = _PyTuple_NewUnsafe(nargs);
if (args == NULL) {
return NULL;
}
Expand All @@ -1304,7 +1304,7 @@ _PyStack_AsTupleSlice(PyObject *const *stack, Py_ssize_t nargs,
assert(end <= nargs);
assert(start <= end);

args = PyTuple_New(end - start);
args = _PyTuple_NewUnsafe(end - start);
if (args == NULL) {
return NULL;
}
Expand Down Expand Up @@ -1375,7 +1375,7 @@ _PyStack_UnpackDict(PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs,
return -1;
}

kwnames = PyTuple_New(nkwargs);
kwnames = _PyTuple_NewUnsafe(nkwargs);
if (kwnames == NULL) {
PyMem_Free(stack);
return -1;
Expand Down
3 changes: 2 additions & 1 deletion Objects/listobject.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#include "pycore_object.h"
#include "pycore_pystate.h"
#include "pycore_accu.h"
#include "pycore_tupleobject.h"

#ifdef STDC_HEADERS
#include <stddef.h>
Expand Down Expand Up @@ -2502,7 +2503,7 @@ PyList_AsTuple(PyObject *v)
return NULL;
}
n = Py_SIZE(v);
w = PyTuple_New(n);
w = _PyTuple_NewUnsafe(n);
if (w == NULL)
return NULL;
p = ((PyTupleObject *)w)->ob_item;
Expand Down
32 changes: 22 additions & 10 deletions Objects/tupleobject.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,11 +76,10 @@ _PyTuple_DebugMallocStats(FILE *out)
#endif
}

PyObject *
PyTuple_New(Py_ssize_t size)
static PyObject *
tuple_new_internal(Py_ssize_t size, int initialize)

@vstinner vstinner Feb 19, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Uninitialized tuples must be initialized with Py_SIZE(op) = 0 since they are tracked by the GC (_PyObject_GC_TRACK call below). A GC collection can be triggered while the tuple is filled, whereas tupletraverse() uses Py_SIZE() to iterate on items. If Py_SIZE() isn't 0, tupletraverse() will likely crash.

I suggest the function name "tuple_new_impl".

@sir-sigurd sir-sigurd Feb 19, 2019

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Uninitialized tuples must be initialized with Py_SIZE(op) = 0 since they are tracked by the GC (_PyObject_GC_TRACK call below). A GC collection can be triggered while the tuple is filled, whereas tupletraverse() uses Py_SIZE() to iterate on items. If Py_SIZE() isn't 0, tupletraverse() will likely crash.

I don't like setting Py_SIZE(op) = 0 and I see these alternatives:

  1. if I understand correctly GC collection can be triggered only when some objects are Py_DECREFed, if that's true we can limit usage of proposed function only to the cases when it's quite obvious that there are no Py_DECREFs until tuple is filled (it's done this way in bpo-34151: Improve performance of some list operations #8332), in the current patch there are just 2 not so obvious cases when PyDict_Next() is used
  2. call _PyObject_GC_TRACK() in caller when tuple is filled as you proposed in bpo-34151: Improve performance of some list operations #8332 (comment)

@sir-sigurd sir-sigurd Feb 19, 2019

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I suggest the function name "tuple_new_impl".

There is already this one:

cpython/Objects/tupleobject.c

Lines 696 to 707 in 5105483

static PyObject *
tuple_new_impl(PyTypeObject *type, PyObject *iterable)
/*[clinic end generated code: output=4546d9f0d469bce7 input=86963bcde633b5a2]*/
{
if (type != &PyTuple_Type)
return tuple_subtype_new(type, iterable);
if (iterable == NULL)
return PyTuple_New(0);
else
return PySequence_Tuple(iterable);
}

{
PyTupleObject *op;
Py_ssize_t i;
if (size < 0) {
PyErr_BadInternalCall();
return NULL;
Expand Down Expand Up @@ -119,8 +118,9 @@ PyTuple_New(Py_ssize_t size)
if (op == NULL)
return NULL;
}
for (i=0; i < size; i++)
op->ob_item[i] = NULL;
if (initialize) {
memset(op->ob_item, 0, sizeof(PyObject *) * size);
}
#if PyTuple_MAXSAVESIZE > 0
if (size == 0) {
free_list[0] = op;
Expand All @@ -135,6 +135,18 @@ PyTuple_New(Py_ssize_t size)
return (PyObject *) op;
}

PyObject *
_PyTuple_NewUnsafe(Py_ssize_t size)

@sir-sigurd sir-sigurd Feb 19, 2019

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure if _PyTuple_NewNotInitialized is better.

@vstinner vstinner Feb 19, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I suggest the name _PyTuple_NewUninitialized.

{
return tuple_new_internal(size, 0);
}

PyObject *
PyTuple_New(Py_ssize_t size)
{
return tuple_new_internal(size, 1);
}

Py_ssize_t
PyTuple_Size(PyObject *op)
{
Expand Down Expand Up @@ -216,7 +228,7 @@ PyTuple_Pack(Py_ssize_t n, ...)
va_list vargs;

va_start(vargs, n);
result = PyTuple_New(n);
result = _PyTuple_NewUnsafe(n);
if (result == NULL) {
va_end(vargs);
return NULL;
Expand Down Expand Up @@ -438,7 +450,7 @@ tupleslice(PyTupleObject *a, Py_ssize_t ilow,
return (PyObject *)a;
}
len = ihigh - ilow;
np = (PyTupleObject *)PyTuple_New(len);
np = (PyTupleObject *)_PyTuple_NewUnsafe(len);
if (np == NULL)
return NULL;
src = a->ob_item + ilow;
Expand Down Expand Up @@ -486,7 +498,7 @@ tupleconcat(PyTupleObject *a, PyObject *bb)
if (Py_SIZE(a) > PY_SSIZE_T_MAX - Py_SIZE(b))
return PyErr_NoMemory();
size = Py_SIZE(a) + Py_SIZE(b);
np = (PyTupleObject *) PyTuple_New(size);
np = (PyTupleObject *) _PyTuple_NewUnsafe(size);
if (np == NULL) {
return NULL;
}
Expand Down Expand Up @@ -530,7 +542,7 @@ tuplerepeat(PyTupleObject *a, Py_ssize_t n)
if (n > PY_SSIZE_T_MAX / Py_SIZE(a))
return PyErr_NoMemory();
size = Py_SIZE(a) * n;
np = (PyTupleObject *) PyTuple_New(size);
np = (PyTupleObject *) _PyTuple_NewUnsafe(size);
if (np == NULL)
return NULL;
p = np->ob_item;
Expand Down Expand Up @@ -773,7 +785,7 @@ tuplesubscript(PyTupleObject* self, PyObject* item)
return (PyObject *)self;
}
else {
result = PyTuple_New(slicelength);
result = _PyTuple_NewUnsafe(slicelength);
if (!result) return NULL;

src = self->ob_item;
Expand Down
2 changes: 1 addition & 1 deletion Python/ceval.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2407,7 +2407,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)
}

case TARGET(BUILD_TUPLE): {
PyObject *tup = PyTuple_New(oparg);
PyObject *tup = _PyTuple_NewUnsafe(oparg);
if (tup == NULL)
goto error;
while (--oparg >= 0) {
Expand Down