pythongh-114314: ctypes: remove stgdict and switch to heap types (pyt…

…honGH-116458)

Before this change, ctypes classes used a custom dict subclass, `StgDict`,
as their `tp_dict`. This acts like a regular dict but also includes extra information
about the type.

This replaces stgdict by `StgInfo`, a C struct on the type, accessed by
`PyObject_GetTypeData()` (PEP-697).
All usage of `StgDict` (mainly variables named `stgdict`, `dict`, `edict` etc.) is
converted to `StgInfo` (named `stginfo`, `info`, `einfo`, etc.).
Where the dict is actually used for class attributes (as a regular PyDict), it's now
called `attrdict`.

This change -- not overriding `tp_dict` -- is made to make me comfortable with
the next part of this PR: moving the initialization logic from `tp_new` to `tp_init`.

The `StgInfo` is set up in `__init__` of each class, with a guard that prevents
calling `__init__` more than once. Note that abstract classes (like `Array` or
`Structure`) are created using `PyType_FromMetaclass` and do not have
`__init__` called.
Previously, this was done in `__new__`, which also wasn't called for abstract
classes.
Since `__init__` can be called from Python code or skipped, there is a tested
guard to ensure `StgInfo` is initialized exactly once before it's used.

Co-authored-by: neonene <53406459+neonene@users.noreply.github.com>
Co-authored-by: Erlend E. Aasland <erlend.aasland@protonmail.com>

Lib/test/test_ctypes/test_arrays.py

@@ -37,6 +37,22 @@ def test_type_flags(self):
                 self.assertTrue(cls.__flags__ & Py_TPFLAGS_IMMUTABLETYPE)
                 self.assertFalse(cls.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION)
 
+    def test_metaclass_details(self):
+        # Abstract classes (whose metaclass __init__ was not called) can't be
+        # instantiated directly
+        NewArray = PyCArrayType.__new__(PyCArrayType, 'NewArray', (Array,), {})
+        for cls in Array, NewArray:
+            with self.subTest(cls=cls):
+                with self.assertRaisesRegex(TypeError, "abstract class"):
+                    obj = cls()
+
+        # Cannot call the metaclass __init__ more than once
+        class T(Array):
+            _type_ = c_int
+            _length_ = 13
+        with self.assertRaisesRegex(SystemError, "already initialized"):
+            PyCArrayType.__init__(T, 'ptr', (), {})
+
     def test_simple(self):
         # create classes holding simple numeric types, and check
         # various properties.

Lib/test/test_ctypes/test_callbacks.py

@@ -106,7 +106,7 @@ def test_pyobject(self):
 
     def test_unsupported_restype_1(self):
         # Only "fundamental" result types are supported for callback
-        # functions, the type must have a non-NULL stgdict->setfunc.
+        # functions, the type must have a non-NULL stginfo->setfunc.
         # POINTER(c_double), for example, is not supported.
 
         prototype = self.functype.__func__(POINTER(c_double))

Lib/test/test_ctypes/test_funcptr.py

@@ -29,6 +29,12 @@ def test_type_flags(self):
                 self.assertTrue(_CFuncPtr.__flags__ & Py_TPFLAGS_IMMUTABLETYPE)
                 self.assertFalse(_CFuncPtr.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION)
 
+    def test_metaclass_details(self):
+        # Cannot call the metaclass __init__ more than once
+        CdeclCallback = CFUNCTYPE(c_int, c_int, c_int)
+        with self.assertRaisesRegex(SystemError, "already initialized"):
+            PyCFuncPtrType.__init__(CdeclCallback, 'ptr', (), {})
+
     def test_basic(self):
         X = WINFUNCTYPE(c_int, c_int, c_int)
 

Lib/test/test_ctypes/test_pointers.py

@@ -33,6 +33,11 @@ def test_type_flags(self):
                 self.assertTrue(_Pointer.__flags__ & Py_TPFLAGS_IMMUTABLETYPE)
                 self.assertFalse(_Pointer.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION)
 
+    def test_metaclass_details(self):
+        # Cannot call the metaclass __init__ more than once
+        with self.assertRaisesRegex(SystemError, "already initialized"):
+            PyCPointerType.__init__(POINTER(c_byte), 'ptr', (), {})
+
     def test_pointer_crash(self):
 
         class A(POINTER(c_ulong)):

Lib/test/test_ctypes/test_simplesubclasses.py

@@ -26,6 +26,29 @@ def test_type_flags(self):
                 self.assertTrue(_SimpleCData.__flags__ & Py_TPFLAGS_IMMUTABLETYPE)
                 self.assertFalse(_SimpleCData.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION)
 
+    def test_metaclass_details(self):
+        # Abstract classes (whose metaclass __init__ was not called) can't be
+        # instantiated directly
+        NewT = PyCSimpleType.__new__(PyCSimpleType, 'NewT', (_SimpleCData,), {})
+        for cls in _SimpleCData, NewT:
+            with self.subTest(cls=cls):
+                with self.assertRaisesRegex(TypeError, "abstract class"):
+                    obj = cls()
+
+        # Cannot call the metaclass __init__ more than once
+        class T(_SimpleCData):
+            _type_ = "i"
+        with self.assertRaisesRegex(SystemError, "already initialized"):
+            PyCSimpleType.__init__(T, 'ptr', (), {})
+
+    def test_swapped_type_creation(self):
+        cls = PyCSimpleType.__new__(PyCSimpleType, '', (), {'_type_': 'i'})
+        with self.assertRaises(TypeError):
+            PyCSimpleType.__init__(cls)
+        PyCSimpleType.__init__(cls, '', (), {'_type_': 'i'})
+        self.assertEqual(cls.__ctype_le__.__dict__.get('_type_'), 'i')
+        self.assertEqual(cls.__ctype_be__.__dict__.get('_type_'), 'i')
+
     def test_compare(self):
         self.assertEqual(MyInt(3), MyInt(3))
         self.assertNotEqual(MyInt(42), MyInt(43))

Lib/test/test_ctypes/test_struct_fields.py

@@ -69,7 +69,7 @@ def test_cfield_inheritance_hierarchy(self):
     def test_gh99275(self):
         class BrokenStructure(Structure):
             def __init_subclass__(cls, **kwargs):
-                cls._fields_ = []  # This line will fail, `stgdict` is not ready
+                cls._fields_ = []  # This line will fail, `stginfo` is not ready
 
         with self.assertRaisesRegex(TypeError,
                                     'ctypes state is not initialized'):

Lib/test/test_ctypes/test_structures.py

@@ -85,6 +85,23 @@ def test_type_flags(self):
                 self.assertTrue(Structure.__flags__ & Py_TPFLAGS_IMMUTABLETYPE)
                 self.assertFalse(Structure.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION)
 
+    def test_metaclass_details(self):
+        # Abstract classes (whose metaclass __init__ was not called) can't be
+        # instantiated directly
+        NewStructure = PyCStructType.__new__(PyCStructType, 'NewStructure',
+                                             (Structure,), {})
+        for cls in Structure, NewStructure:
+            with self.subTest(cls=cls):
+                with self.assertRaisesRegex(TypeError, "abstract class"):
+                    obj = cls()
+
+        # Cannot call the metaclass __init__ more than once
+        class T(Structure):
+            _fields_ = [("x", c_char),
+                        ("y", c_char)]
+        with self.assertRaisesRegex(SystemError, "already initialized"):
+            PyCStructType.__init__(T, 'ptr', (), {})
+
     def test_simple_structs(self):
         for code, tp in self.formats.items():
             class X(Structure):
@@ -507,8 +524,8 @@ def _test_issue18060(self, Vector):
     @unittest.skipUnless(sys.byteorder == 'little', "can't test on this platform")
     def test_issue18060_a(self):
         # This test case calls
-        # PyCStructUnionType_update_stgdict() for each
-        # _fields_ assignment, and PyCStgDict_clone()
+        # PyCStructUnionType_update_stginfo() for each
+        # _fields_ assignment, and PyCStgInfo_clone()
         # for the Mid and Vector class definitions.
         class Base(Structure):
             _fields_ = [('y', c_double),
@@ -523,7 +540,7 @@ class Vector(Mid): pass
     @unittest.skipUnless(sys.byteorder == 'little', "can't test on this platform")
     def test_issue18060_b(self):
         # This test case calls
-        # PyCStructUnionType_update_stgdict() for each
+        # PyCStructUnionType_update_stginfo() for each
         # _fields_ assignment.
         class Base(Structure):
             _fields_ = [('y', c_double),
@@ -538,7 +555,7 @@ class Vector(Mid):
     @unittest.skipUnless(sys.byteorder == 'little', "can't test on this platform")
     def test_issue18060_c(self):
         # This test case calls
-        # PyCStructUnionType_update_stgdict() for each
+        # PyCStructUnionType_update_stginfo() for each
         # _fields_ assignment.
         class Base(Structure):
             _fields_ = [('y', c_double)]

Lib/test/test_ctypes/test_unions.py

@@ -1,5 +1,5 @@
 import unittest
-from ctypes import Union
+from ctypes import Union, c_char
 from ._support import (_CData, UnionType, Py_TPFLAGS_DISALLOW_INSTANTIATION,
                        Py_TPFLAGS_IMMUTABLETYPE)
 
@@ -16,3 +16,20 @@ def test_type_flags(self):
             with self.subTest(cls=Union):
                 self.assertTrue(Union.__flags__ & Py_TPFLAGS_IMMUTABLETYPE)
                 self.assertFalse(Union.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION)
+
+    def test_metaclass_details(self):
+        # Abstract classes (whose metaclass __init__ was not called) can't be
+        # instantiated directly
+        NewUnion = UnionType.__new__(UnionType, 'NewUnion',
+                                     (Union,), {})
+        for cls in Union, NewUnion:
+            with self.subTest(cls=cls):
+                with self.assertRaisesRegex(TypeError, "abstract class"):
+                    obj = cls()
+
+        # Cannot call the metaclass __init__ more than once
+        class T(Union):
+            _fields_ = [("x", c_char),
+                        ("y", c_char)]
+        with self.assertRaisesRegex(SystemError, "already initialized"):
+            UnionType.__init__(T, 'ptr', (), {})

Misc/NEWS.d/next/Library/2024-03-07-11-10-27.gh-issue-114314.iEhAMH.rst

@@ -0,0 +1,3 @@
+In :mod:`ctypes`, ctype data is now stored in type objects directly rather
+than in a dict subclass. This is an internal change that should not affect
+usage.