Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions Lib/test/test_struct.py
Original file line number Diff line number Diff line change
Expand Up @@ -800,6 +800,23 @@ def test_c_complex_round_trip(self):
round_trip = struct.unpack(f, struct.pack(f, z))[0]
self.assertComplexesAreIdentical(z, round_trip)

@unittest.skipIf(
support.is_android or support.is_apple_mobile,
"Subinterpreters are not supported on Android and iOS"
)
def test_endian_table_init_subinterpreters(self):
# Verify that the _struct extension module can be initialized
# concurrently in subinterpreters (gh-140260).
try:
from concurrent.futures import InterpreterPoolExecutor
except ImportError:
raise unittest.SkipTest("InterpreterPoolExecutor not available")

code = "import struct"
with InterpreterPoolExecutor(max_workers=5) as executor:
results = executor.map(exec, [code] * 5)
self.assertListEqual(list(results), [None] * 5)


class UnpackIteratorTest(unittest.TestCase):
"""
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Fix :mod:`struct` data race in endian table initialization with
subinterpreters. Patch by Shamil Abdulaev.
91 changes: 50 additions & 41 deletions Modules/_struct.c
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

#include "Python.h"
#include "pycore_bytesobject.h" // _PyBytesWriter
#include "pycore_lock.h" // _PyOnceFlag_CallOnce()
#include "pycore_long.h" // _PyLong_AsByteArray()
#include "pycore_moduleobject.h" // _PyModule_GetState()
#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS()
Expand Down Expand Up @@ -1505,6 +1506,53 @@ static formatdef lilendian_table[] = {
{0}
};

/* Ensure endian table optimization happens exactly once across all interpreters */
static _PyOnceFlag endian_tables_init_once = {0};

static int
init_endian_tables(void *Py_UNUSED(arg))
{
const formatdef *native = native_table;
formatdef *other, *ptr;
#if PY_LITTLE_ENDIAN
other = lilendian_table;
#else
other = bigendian_table;
#endif
/* Scan through the native table, find a matching
entry in the endian table and swap in the
native implementations whenever possible
(64-bit platforms may not have "standard" sizes) */
while (native->format != '\0' && other->format != '\0') {
ptr = other;
while (ptr->format != '\0') {
if (ptr->format == native->format) {
/* Match faster when formats are
listed in the same order */
if (ptr == other)
other++;
/* Only use the trick if the
size matches */
if (ptr->size != native->size)
break;
/* Skip float and double, could be
"unknown" float format */
if (ptr->format == 'd' || ptr->format == 'f')
break;
/* Skip _Bool, semantics are different for standard size */
if (ptr->format == '?')
break;
ptr->pack = native->pack;
ptr->unpack = native->unpack;
break;
}
ptr++;
}
native++;
}
return 0;
}


static const formatdef *
whichtable(const char **pfmt)
Expand Down Expand Up @@ -2710,47 +2758,8 @@ _structmodule_exec(PyObject *m)
return -1;
}

/* Check endian and swap in faster functions */
{
const formatdef *native = native_table;
formatdef *other, *ptr;
#if PY_LITTLE_ENDIAN
other = lilendian_table;
#else
other = bigendian_table;
#endif
/* Scan through the native table, find a matching
entry in the endian table and swap in the
native implementations whenever possible
(64-bit platforms may not have "standard" sizes) */
while (native->format != '\0' && other->format != '\0') {
ptr = other;
while (ptr->format != '\0') {
if (ptr->format == native->format) {
/* Match faster when formats are
listed in the same order */
if (ptr == other)
other++;
/* Only use the trick if the
size matches */
if (ptr->size != native->size)
break;
/* Skip float and double, could be
"unknown" float format */
if (ptr->format == 'd' || ptr->format == 'f')
break;
/* Skip _Bool, semantics are different for standard size */
if (ptr->format == '?')
break;
ptr->pack = native->pack;
ptr->unpack = native->unpack;
break;
}
ptr++;
}
native++;
}
}
/* init cannot fail */
(void)_PyOnceFlag_CallOnce(&endian_tables_init_once, init_endian_tables, NULL);

/* Add some symbolic constants to the module */
state->StructError = PyErr_NewException("struct.error", NULL, NULL);
Expand Down
1 change: 1 addition & 0 deletions Tools/c-analyzer/cpython/ignored.tsv
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ Modules/posixmodule.c os_dup2_impl dup3_works -

## guards around resource init
Python/thread_pthread.h PyThread__init_thread lib_initialized -
Modules/_struct.c - endian_tables_init_once -

##-----------------------
## other values (not Python-specific)
Expand Down
Loading