Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

gh-111623 Serialize tuples with sub interpreters #111628

Merged
merged 34 commits into from Nov 7, 2023
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
6eb3562
Initial design for shareable tuples
tonybaloney Nov 2, 2023
d3a5cfc
Add memory error
tonybaloney Nov 2, 2023
439b652
Add a check for XID allocation failures
tonybaloney Nov 2, 2023
2d8602d
Add blurb
tonybaloney Nov 2, 2023
3711bb5
tweaks for pep7
tonybaloney Nov 2, 2023
7072d36
Add tests for non shareable tuples
tonybaloney Nov 2, 2023
448d017
Merge branch 'main' into serialize_tuples
tonybaloney Nov 2, 2023
d6f386f
fix patchcheck
tonybaloney Nov 2, 2023
fa958b3
Merge branch 'serialize_tuples' of github.com:tonybaloney/cpython int…
tonybaloney Nov 2, 2023
08cdfcb
Update Misc/NEWS.d/next/Core and Builtins/2023-11-02-15-00-57.gh-issu…
tonybaloney Nov 2, 2023
654689e
Update Lib/test/test_interpreters.py
tonybaloney Nov 2, 2023
994fab0
Add two fault cases
tonybaloney Nov 2, 2023
302dc8d
Update test__xxsubinterpreters.py
tonybaloney Nov 4, 2023
86c1494
Update test__xxsubinterpreters.py
tonybaloney Nov 4, 2023
3b30b77
Update test__xxsubinterpreters.py
tonybaloney Nov 4, 2023
9abda96
Update test__xxsubinterpreters.py
tonybaloney Nov 4, 2023
82f4836
Update test__xxsubinterpreters.py
tonybaloney Nov 4, 2023
0fc4eab
Update test_interpreters.py
tonybaloney Nov 4, 2023
e234be8
Update 2023-11-02-15-00-57.gh-issue-111623.BZxYc8.rst
tonybaloney Nov 4, 2023
1b3bb14
Update crossinterp.c
tonybaloney Nov 4, 2023
e70dba9
Update crossinterp.c
tonybaloney Nov 4, 2023
c883040
Update crossinterp.c
tonybaloney Nov 4, 2023
7acf36f
Update crossinterp.c
tonybaloney Nov 4, 2023
beae22a
Update crossinterp.c
tonybaloney Nov 4, 2023
063c8f7
Update crossinterp.c
tonybaloney Nov 4, 2023
0bd3b88
Update the previous two blurbs to match this one
tonybaloney Nov 4, 2023
e40381c
Merge branch 'main' into serialize_tuples
tonybaloney Nov 4, 2023
97f281c
Add a recursion check
tonybaloney Nov 5, 2023
3b96658
Merge branch 'serialize_tuples' of github.com:tonybaloney/cpython int…
tonybaloney Nov 5, 2023
f4ee75f
Update Python/crossinterp.c
tonybaloney Nov 6, 2023
1e1141e
Update Python/crossinterp.c
tonybaloney Nov 6, 2023
1ea81a3
Fix parens mismatch
tonybaloney Nov 6, 2023
a18939a
Refactor to avoid the failure case running PyMem_RawFree on already f…
tonybaloney Nov 6, 2023
b986e3d
Merge branch 'main' into serialize_tuples
tonybaloney Nov 6, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
24 changes: 24 additions & 0 deletions Lib/test/test__xxsubinterpreters.py
Expand Up @@ -105,6 +105,7 @@ def test_default_shareables(self):
True,
False,
100.0,
(),
tonybaloney marked this conversation as resolved.
Show resolved Hide resolved
]
for obj in shareables:
with self.subTest(obj):
Expand Down Expand Up @@ -195,6 +196,29 @@ def test_bool(self):
def test_float(self):
self._assert_values([0.0, 1.1, -1.0, 0.12345678, -0.12345678])

def test_tuple(self):
self._assert_values([(), (1,), ("hello", "world", ), (1, True, "hello")])
# Test nesting
self._assert_values([((1,),), ((1, 2), (3, 4)), ((1, 2), (3, 4), (5, 6))])
tonybaloney marked this conversation as resolved.
Show resolved Hide resolved

def test_tuples_containing_non_shareable_types(self):
non_shareables = [
Exception(),
object(),
]
for s in non_shareables:
value = tuple([0, 1., s])
tonybaloney marked this conversation as resolved.
Show resolved Hide resolved
with self.subTest(repr(value)):
# XXX Assert the NotShareableError when it is exported
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll work on that.

with self.assertRaises(Exception):
tonybaloney marked this conversation as resolved.
Show resolved Hide resolved
_testinternalcapi.get_crossinterp_data(value)
# Check nested as well
value = tuple([0, 1., (s,)])
with self.subTest("nested " + repr(value)):
# XXX Assert the NotShareableError when it is exported
with self.assertRaises(Exception):
tonybaloney marked this conversation as resolved.
Show resolved Hide resolved
_testinternalcapi.get_crossinterp_data(value)


class ModuleTests(TestBase):

Expand Down
1 change: 1 addition & 0 deletions Lib/test/test_interpreters.py
Expand Up @@ -781,6 +781,7 @@ def test_default_shareables(self):
True,
False,
100.0,
(),
tonybaloney marked this conversation as resolved.
Show resolved Hide resolved
]
for obj in shareables:
with self.subTest(obj):
Expand Down
@@ -0,0 +1,2 @@
Support for sharing tuples between interpreters using the cross-interpreter
tonybaloney marked this conversation as resolved.
Show resolved Hide resolved
API
68 changes: 68 additions & 0 deletions Python/crossinterp.c
Expand Up @@ -713,6 +713,69 @@ _bool_shared(PyThreadState *tstate, PyObject *obj,
return 0;
}

struct _shared_tuple_data {
Py_ssize_t len;
_PyCrossInterpreterData ** data;
tonybaloney marked this conversation as resolved.
Show resolved Hide resolved
};

static PyObject *
_new_tuple_object(_PyCrossInterpreterData *data)
{
struct _shared_tuple_data *shared = (struct _shared_tuple_data *)(data->data);
PyObject *tuple = PyTuple_New(shared->len);
tonybaloney marked this conversation as resolved.
Show resolved Hide resolved

for (Py_ssize_t i = 0; i < shared->len; i++) {
PyObject *item = _PyCrossInterpreterData_NewObject(shared->data[i]);
PyTuple_SET_ITEM(tuple, i, item);
tonybaloney marked this conversation as resolved.
Show resolved Hide resolved
}
return tuple;
}

static void
_tuple_shared_free(void* data)
{
struct _shared_tuple_data *shared = (struct _shared_tuple_data *)(data);
for (Py_ssize_t i = 0; i < shared->len; i++) {
_PyCrossInterpreterData_Release(shared->data[i]);
}
tonybaloney marked this conversation as resolved.
Show resolved Hide resolved
PyMem_Free(shared->data);
PyMem_RawFree(shared);
}

static int
_tuple_shared(PyThreadState *tstate, PyObject *obj,
_PyCrossInterpreterData *data)
{
if (_PyCrossInterpreterData_InitWithSize(
data, tstate->interp, sizeof(struct _shared_tuple_data), obj,
_new_tuple_object
) < 0)
{
return -1;
}
struct _shared_tuple_data *shared = (struct _shared_tuple_data *)data->data;
shared->len = PyTuple_GET_SIZE(obj);
tonybaloney marked this conversation as resolved.
Show resolved Hide resolved
shared->data = (_PyCrossInterpreterData **) PyMem_Calloc(shared->len, sizeof(_PyCrossInterpreterData *));
if (shared->data == NULL) {
PyErr_Format(PyExc_MemoryError, "not enough memory");
tonybaloney marked this conversation as resolved.
Show resolved Hide resolved
return -1;
}

for (Py_ssize_t i = 0; i < shared->len; i++) {
shared->data[i] = _PyCrossInterpreterData_New();
if (shared->data[i] == NULL){
PyMem_Free(shared->data);
tonybaloney marked this conversation as resolved.
Show resolved Hide resolved
return -1; // PyErr_NoMemory already set
}
PyObject *item = PyTuple_GET_ITEM(obj, i);
if (_PyObject_GetCrossInterpreterData(item, shared->data[i]) != 0) {
return -1;
}
tonybaloney marked this conversation as resolved.
Show resolved Hide resolved
}
data->free = _tuple_shared_free;
return 0;
}

static void
_register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry)
{
Expand Down Expand Up @@ -745,6 +808,11 @@ _register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry)
if (_xidregistry_add_type(xidregistry, &PyFloat_Type, _float_shared) != 0) {
Py_FatalError("could not register float for cross-interpreter sharing");
}

// tuple
if (_xidregistry_add_type(xidregistry, &PyTuple_Type, _tuple_shared) != 0) {
Py_FatalError("could not register tuple for cross-interpreter sharing");
}
}

/* registry lifecycle */
Expand Down