gh-109252: Remove leading whitespace email

Lib/email/_policybase.py

@@ -307,8 +307,9 @@ def header_source_parse(self, sourcelines):
 
         """
         name, value = sourcelines[0].split(':', 1)
-        value = value.lstrip(' \t') + ''.join(sourcelines[1:])
-        return (name, value.rstrip('\r\n'))
+        value = value + ''.join(sourcelines[1:])
+        return (name, value.lstrip(" \r\n\t").rstrip('\r\n'))
+
 
     def header_store_parse(self, name, value):
         """+

Lib/test/test_structseq.py

@@ -2,8 +2,10 @@
 import os
 import pickle
 import re
+import textwrap
 import time
 import unittest
+from test.support import script_helper
 
 
 class StructSeqTest(unittest.TestCase):
@@ -342,6 +344,17 @@ def test_copy_replace_with_unnamed_fields(self):
         with self.assertRaisesRegex(TypeError, error_message):
             copy.replace(r, st_mode=1, error=2)
 
+    def test_reference_cycle(self):
+        # gh-122527: Check that a structseq that's part of a reference cycle
+        # with its own type doesn't crash. Previously, if the type's dictionary
+        # was cleared first, the structseq instance would crash in the
+        # destructor.
+        script_helper.assert_python_ok("-c", textwrap.dedent(r"""
+            import time
+            t = time.gmtime()
+            type(t).refcyle = t
+        """))
+
 
 if __name__ == "__main__":
     unittest.main()

Lib/test/test_symtable.py

@@ -528,6 +528,27 @@ def test_symtable_entry_repr(self):
         self.assertEqual(repr(self.top._table), expected)
 
 
+class ComprehensionTests(unittest.TestCase):
+    def get_identifiers_recursive(self, st, res):
+        res.extend(st.get_identifiers())
+        for ch in st.get_children():
+            self.get_identifiers_recursive(ch, res)
+
+    def test_loopvar_in_only_one_scope(self):
+        # ensure that the loop variable appears only once in the symtable
+        comps = [
+            "[x for x in [1]]",
+            "{x for x in [1]}",
+            "{x:x*x for x in [1]}",
+        ]
+        for comp in comps:
+            with self.subTest(comp=comp):
+                st = symtable.symtable(comp, "?", "exec")
+                ids = []
+                self.get_identifiers_recursive(st, ids)
+                self.assertEqual(len([x for x in ids if x == 'x']), 1)
+
+
 class CommandLineTest(unittest.TestCase):
     maxDiff = None
 

Lib/test/test_sys.py

@@ -1823,7 +1823,8 @@ def test_pythontypes(self):
         # symtable entry
         # XXX
         # sys.flags
-        check(sys.flags, vsize('') + self.P * len(sys.flags))
+        # FIXME: The +1 will not be necessary once gh-122575 is fixed
+        check(sys.flags, vsize('') + self.P * (1 + len(sys.flags)))
 
     def test_asyncgen_hooks(self):
         old = sys.get_asyncgen_hooks()

Misc/NEWS.d/next/Core_and_Builtins/2024-08-01-19-13-58.gh-issue-122527.eztso6.rst

@@ -0,0 +1,4 @@
+Fix a crash that occurred when a ``PyStructSequence`` was deallocated after
+its type's dictionary was cleared by the GC.  The type's
+:c:member:`~PyTypeObject.tp_basicsize` now accounts for non-sequence fields
+that aren't included in the :c:macro:`Py_SIZE` of the sequence.

Misc/NEWS.d/next/Library/2024-08-02-14-37-04.gh-issue-109252._NyO6i.rst

@@ -0,0 +1 @@
+Fixed an existing issue with email header lib adding whitespace

Objects/structseq.c

@@ -41,12 +41,20 @@ get_type_attr_as_size(PyTypeObject *tp, PyObject *name)
     get_type_attr_as_size(tp, &_Py_ID(n_sequence_fields))
 #define REAL_SIZE_TP(tp) \
     get_type_attr_as_size(tp, &_Py_ID(n_fields))
-#define REAL_SIZE(op) REAL_SIZE_TP(Py_TYPE(op))
+#define REAL_SIZE(op) get_real_size((PyObject *)op)
 
 #define UNNAMED_FIELDS_TP(tp) \
     get_type_attr_as_size(tp, &_Py_ID(n_unnamed_fields))
 #define UNNAMED_FIELDS(op) UNNAMED_FIELDS_TP(Py_TYPE(op))
 
+static Py_ssize_t
+get_real_size(PyObject *op)
+{
+    // Compute the real size from the visible size (i.e., Py_SIZE()) and the
+    // number of non-sequence fields accounted for in tp_basicsize.
+    Py_ssize_t hidden = Py_TYPE(op)->tp_basicsize - offsetof(PyStructSequence, ob_item);
+    return Py_SIZE(op) + hidden / sizeof(PyObject *);
+}
 
 PyObject *
 PyStructSequence_New(PyTypeObject *type)
@@ -120,6 +128,9 @@ structseq_dealloc(PyStructSequence *obj)
     PyObject_GC_UnTrack(obj);
 
     PyTypeObject *tp = Py_TYPE(obj);
+    // gh-122527: We can't use REAL_SIZE_TP() or any macros that access the
+    // type's dictionary here, because the dictionary may have already been
+    // cleared by the garbage collector.
     size = REAL_SIZE(obj);
     for (i = 0; i < size; ++i) {
         Py_XDECREF(obj->ob_item[i]);
@@ -565,10 +576,14 @@ initialize_members(PyStructSequence_Desc *desc,
 
 static void
 initialize_static_fields(PyTypeObject *type, PyStructSequence_Desc *desc,
-                         PyMemberDef *tp_members, unsigned long tp_flags)
+                         PyMemberDef *tp_members, Py_ssize_t n_members,
+                         unsigned long tp_flags)
 {
     type->tp_name = desc->name;
-    type->tp_basicsize = sizeof(PyStructSequence) - sizeof(PyObject *);
+    // Account for hidden members in tp_basicsize because they are not
+    // included in the variable size.
+    Py_ssize_t n_hidden = n_members - desc->n_in_sequence;
+    type->tp_basicsize = sizeof(PyStructSequence) + (n_hidden - 1) * sizeof(PyObject *);
     type->tp_itemsize = sizeof(PyObject *);
     type->tp_dealloc = (destructor)structseq_dealloc;
     type->tp_repr = (reprfunc)structseq_repr;
@@ -621,7 +636,7 @@ _PyStructSequence_InitBuiltinWithFlags(PyInterpreterState *interp,
         if (members == NULL) {
             goto error;
         }
-        initialize_static_fields(type, desc, members, tp_flags);
+        initialize_static_fields(type, desc, members, n_members, tp_flags);
 
         _Py_SetImmortal((PyObject *)type);
     }
@@ -684,7 +699,7 @@ PyStructSequence_InitType2(PyTypeObject *type, PyStructSequence_Desc *desc)
     if (members == NULL) {
         return -1;
     }
-    initialize_static_fields(type, desc, members, 0);
+    initialize_static_fields(type, desc, members, n_members, 0);
     if (initialize_static_type(type, desc, n_members, n_unnamed_members) < 0) {
         PyMem_Free(members);
         return -1;
@@ -760,7 +775,8 @@ _PyStructSequence_NewType(PyStructSequence_Desc *desc, unsigned long tp_flags)
     /* The name in this PyType_Spec is statically allocated so it is */
     /* expected that it'll outlive the PyType_Spec */
     spec.name = desc->name;
-    spec.basicsize = sizeof(PyStructSequence) - sizeof(PyObject *);
+    Py_ssize_t hidden = n_members - desc->n_in_sequence;
+    spec.basicsize = (int)(sizeof(PyStructSequence) + (hidden - 1) * sizeof(PyObject *));
     spec.itemsize = sizeof(PyObject *);
     spec.flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | tp_flags;
     spec.slots = slots;