[pull] main from python:main

Doc/c-api/module.rst

@@ -230,6 +230,9 @@ Feature slots
    When creating a module, Python checks the value of this slot
    using :c:func:`PyABIInfo_Check`.
 
+   This slot is required, except for modules created from
+   :c:struct:`PyModuleDef`.
+
    .. versionadded:: 3.15
 
 .. c:macro:: Py_mod_multiple_interpreters
@@ -620,9 +623,9 @@ rather than from an extension's :ref:`export hook <extension-export-hook>`.
    and the :py:class:`~importlib.machinery.ModuleSpec` *spec*.
 
    The *slots* argument must point to an array of :c:type:`PyModuleDef_Slot`
-   structures, terminated by an entry slot with slot ID of 0
+   structures, terminated by an entry with slot ID of 0
    (typically written as ``{0}`` or ``{0, NULL}`` in C).
-   The *slots* argument may not be ``NULL``.
+   The array must include a :c:data:`Py_mod_abi` entry.
 
    The *spec* argument may be any ``ModuleSpec``-like object, as described
    in :c:macro:`Py_mod_create` documentation.

Doc/extending/first-extension-module.rst

@@ -265,12 +265,19 @@ Define this array just before your export hook:
 
 .. code-block:: c
 
+   PyABIInfo_VAR(abi_info);
+
    static PyModuleDef_Slot spam_slots[] = {
+      {Py_mod_abi, &abi_info},
       {Py_mod_name, "spam"},
       {Py_mod_doc, "A wonderful module with an example function"},
       {0, NULL}
    };
 
+The ``PyABIInfo_VAR(abi_info);`` macro and the :c:data:`Py_mod_abi` slot
+are a bit of boilerplate that helps prevent extensions compiled for
+a different version of Python from crashing the interpreter.
+
 For both :c:data:`Py_mod_name` and :c:data:`Py_mod_doc`, the values are C
 strings -- that is, NUL-terminated, UTF-8 encoded byte arrays.
 

Doc/includes/capi-extension/spammodule-01.c

@@ -35,7 +35,10 @@ static PyMethodDef spam_methods[] = {
 
 /// Module slot table
 
+PyABIInfo_VAR(abi_info);
+
 static PyModuleDef_Slot spam_slots[] = {
+    {Py_mod_abi, &abi_info},
     {Py_mod_name, "spam"},
     {Py_mod_doc, "A wonderful module with an example function"},
     {Py_mod_methods, spam_methods},

Include/cpython/longintrepr.h

@@ -133,6 +133,11 @@ _PyLong_CompactValue(const PyLongObject *op)
     assert(PyType_HasFeature(op->ob_base.ob_type, Py_TPFLAGS_LONG_SUBCLASS));
     assert(PyUnstable_Long_IsCompact(op));
     sign = 1 - (op->long_value.lv_tag & _PyLong_SIGN_MASK);
+    if (sign == 0) {
+        // gh-147988: Make sure that the digit is zero.
+        // It helps detecting the usage of uninitialized digits.
+        assert(op->long_value.ob_digit[0] == 0);
+    }
     return sign * (Py_ssize_t)op->long_value.ob_digit[0];
 }
 

Include/internal/pycore_ceval.h

@@ -211,16 +211,16 @@ extern void _PyEval_DeactivateOpCache(void);
 
 /* --- _Py_EnterRecursiveCall() ----------------------------------------- */
 
-static inline int _Py_ReachedRecursionLimit(PyThreadState *tstate)  {
+static inline int _Py_MakeRecCheck(PyThreadState *tstate)  {
     uintptr_t here_addr = _Py_get_machine_stack_pointer();
     _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate;
-    // Possible overflow if stack pointer is beyond the soft limit.
-    // _Py_CheckRecursiveCall will check for corner cases and
-    // report an error if there is an overflow.
+    // Overflow if stack pointer is between soft limit and the base of the hardware stack.
+    // If it is below the hardware stack base, assume that we have the wrong stack limits, and do nothing.
+    // We could have the wrong stack limits because of limited platform support, or user-space threads.
 #if _Py_STACK_GROWS_DOWN
-    return here_addr < _tstate->c_stack_soft_limit;
+    return here_addr < _tstate->c_stack_soft_limit && here_addr >= _tstate->c_stack_soft_limit - 2 * _PyOS_STACK_MARGIN_BYTES;
 #else
-    return here_addr > _tstate->c_stack_soft_limit;
+    return here_addr > _tstate->c_stack_soft_limit && here_addr <= _tstate->c_stack_soft_limit + 2 * _PyOS_STACK_MARGIN_BYTES;
 #endif
 }
 
@@ -235,7 +235,7 @@ PyAPI_FUNC(int) _Py_CheckRecursiveCallPy(
 
 static inline int _Py_EnterRecursiveCallTstate(PyThreadState *tstate,
                                                const char *where) {
-    return (_Py_ReachedRecursionLimit(tstate) && _Py_CheckRecursiveCall(tstate, where));
+    return (_Py_MakeRecCheck(tstate) && _Py_CheckRecursiveCall(tstate, where));
 }
 
 static inline int _Py_EnterRecursiveCall(const char *where) {
@@ -249,6 +249,8 @@ static inline void _Py_LeaveRecursiveCallTstate(PyThreadState *tstate) {
 
 PyAPI_FUNC(void) _Py_InitializeRecursionLimits(PyThreadState *tstate);
 
+PyAPI_FUNC(int) _Py_ReachedRecursionLimit(PyThreadState *tstate);
+
 // Export for test_peg_generator
 PyAPI_FUNC(int) _Py_ReachedRecursionLimitWithMargin(
     PyThreadState *tstate,

Include/internal/pycore_pystate.h

@@ -306,23 +306,23 @@ _Py_AssertHoldsTstateFunc(const char *func)
 #define _Py_AssertHoldsTstate()
 #endif
 
+#if !_Py__has_builtin(__builtin_frame_address) && !defined(__GNUC__) && !defined(_MSC_VER)
+static uintptr_t return_pointer_as_int(char* p) {
+    return (uintptr_t)p;
+}
+#endif
 
 static inline uintptr_t
 _Py_get_machine_stack_pointer(void) {
-    uintptr_t result;
-#if !defined(_MSC_VER) && defined(_M_ARM64)
-    result = __getReg(31);
-#elif defined(_MSC_VER) && defined(_M_X64)
-    result = (uintptr_t)_AddressOfReturnAddress();
-#elif defined(__aarch64__)
-    __asm__ ("mov %0, sp" : "=r" (result));
-#elif defined(__x86_64__)
-    __asm__("{movq %%rsp, %0" : "=r" (result));
+#if _Py__has_builtin(__builtin_frame_address) || defined(__GNUC__)
+    return (uintptr_t)__builtin_frame_address(0);
+#elif defined(_MSC_VER)
+    return (uintptr_t)_AddressOfReturnAddress();
 #else
     char here;
-    result = (uintptr_t)&here;
+    /* Avoid compiler warning about returning stack address */
+    return return_pointer_as_int(&here);
 #endif
-    return result;
 }
 
 static inline intptr_t

Include/internal/pycore_pythonrun.h

@@ -46,8 +46,7 @@ extern PyObject * _Py_CompileStringObjectWithModule(
  * stack consumption of PyEval_EvalDefault */
 #if (defined(Py_DEBUG) \
      || defined(_Py_ADDRESS_SANITIZER) \
-     || defined(_Py_THREAD_SANITIZER)) \
-     || defined(_Py_UNDEFINED_BEHAVIOR_SANITIZER)
+     || defined(_Py_THREAD_SANITIZER))
 #  define _PyOS_LOG2_STACK_MARGIN 12
 #else
 #  define _PyOS_LOG2_STACK_MARGIN 11

Include/pyport.h

@@ -598,11 +598,6 @@ extern "C" {
 #      define _Py_NO_SANITIZE_THREAD __attribute__((no_sanitize_thread))
 #    endif
 #  endif
-#  if __has_feature(undefined_behavior_sanitizer)
-#    if !defined(_Py_UNDEFINED_BEHAVIOR_SANITIZER)
-#      define _Py_UNDEFINED_BEHAVIOR_SANITIZER
-#    endif
-#  endif
 #elif defined(__GNUC__)
 #  if defined(__SANITIZE_ADDRESS__)
 #    define _Py_ADDRESS_SANITIZER

Lib/test/support/__init__.py

@@ -71,7 +71,8 @@
     "BrokenIter",
     "in_systemd_nspawn_sync_suppressed",
     "run_no_yield_async_fn", "run_yielding_async_fn", "async_yield",
-    "reset_code", "on_github_actions"
+    "reset_code", "on_github_actions",
+    "requires_root_user", "requires_non_root_user",
     ]
 
 
@@ -3317,3 +3318,8 @@ def control_characters_c0() -> list[str]:
     C0 control characters defined as the byte range 0x00-0x1F, and 0x7F.
     """
     return [chr(c) for c in range(0x00, 0x20)] + ["\x7F"]
+
+
+_ROOT_IN_POSIX = hasattr(os, 'geteuid') and os.geteuid() == 0
+requires_root_user = unittest.skipUnless(_ROOT_IN_POSIX, "test needs root privilege")
+requires_non_root_user = unittest.skipIf(_ROOT_IN_POSIX, "test needs non-root account")

Lib/test/test_capi/test_long.py

@@ -803,6 +803,16 @@ def to_digits(num):
                 self.assertEqual(pylongwriter_create(negative, digits), num,
                                  (negative, digits))
 
+    @unittest.skipUnless(support.Py_DEBUG, "need a debug build (Py_DEBUG)")
+    def test_longwriter_finish(self):
+        # Test PyLongWriter_Create(0, 3, &digits) with PyLongWriter_Finish()
+        # where the last digit is left uninitialized
+        pylongwriter_finish_bug = _testcapi.pylongwriter_finish_bug
+        with self.assertRaises(SystemError) as cm:
+            pylongwriter_finish_bug()
+        self.assertEqual(str(cm.exception),
+                         'PyLongWriter_Finish: digit 2 is uninitialized')
+
     def test_bug_143050(self):
         with support.adjust_int_max_str_digits(0):
             # Bug coming from using _pylong.int_from_string(), that

Lib/test/test_capi/test_module.py

@@ -25,9 +25,13 @@ def def_and_token(mod):
     )
 
 class TestModFromSlotsAndSpec(unittest.TestCase):
-    @requires_gil_enabled("empty slots re-enable GIL")
     def test_empty(self):
-        mod = _testcapi.module_from_slots_empty(FakeSpec())
+        with self.assertRaises(SystemError):
+            _testcapi.module_from_slots_empty(FakeSpec())
+
+    @requires_gil_enabled("minimal slots re-enable GIL")
+    def test_minimal(self):
+        mod = _testcapi.module_from_slots_minimal(FakeSpec())
         self.assertIsInstance(mod, types.ModuleType)
         self.assertEqual(def_and_token(mod), (0, 0))
         self.assertEqual(mod.__name__, 'testmod')
@@ -159,6 +163,16 @@ def test_null_def_slot(self):
                 self.assertIn(name, str(cm.exception))
                 self.assertIn("NULL", str(cm.exception))
 
+    def test_bad_abiinfo(self):
+        """Slots that incompatible ABI is rejected"""
+        with self.assertRaises(ImportError) as cm:
+            _testcapi.module_from_bad_abiinfo(FakeSpec())
+
+    def test_multiple_abiinfo(self):
+        """Slots that Py_mod_abiinfo can be repeated"""
+        mod = _testcapi.module_from_multiple_abiinfo(FakeSpec())
+        self.assertEqual(mod.__name__, 'testmod')
+
     def test_def_multiple_exec(self):
         """PyModule_Exec runs all exec slots of PyModuleDef-defined module"""
         mod = _testcapi.module_from_def_multiple_exec(FakeSpec())

Lib/test/test_cext/extension.c

@@ -119,8 +119,10 @@ _Py_COMP_DIAG_PUSH
 #endif
 
 PyDoc_STRVAR(_testcext_doc, "C test extension.");
+PyABIInfo_VAR(abi_info);
 
 static PyModuleDef_Slot _testcext_slots[] = {
+    {Py_mod_abi, &abi_info},
     {Py_mod_name, STR(MODULE_NAME)},
     {Py_mod_doc, (void*)(char*)_testcext_doc},
     {Py_mod_exec, (void*)_testcext_exec},

Lib/test/test_import/__init__.py

@@ -3499,12 +3499,20 @@ class Sub(tp):
             pass
         self.assertEqual(_testcapi.pytype_getmodulebytoken(Sub, token), module)
 
-    @requires_gil_enabled("empty slots re-enable GIL")
     def test_from_modexport_empty_slots(self):
+        # Module to test that Py_mod_abi is mandatory for PyModExport
+        modname = '_test_from_modexport_empty_slots'
+        filename = _testmultiphase.__file__
+        with self.assertRaises(SystemError):
+            import_extension_from_file(
+                modname, filename, put_in_sys_modules=False)
+
+    @requires_gil_enabled("this module re-enables GIL")
+    def test_from_modexport_minimal_slots(self):
         # Module to test that:
-        # - no slots are mandatory for PyModExport
+        # - no slots except Py_mod_abi is mandatory for PyModExport
         # - the slots array is used as the default token
-        modname = '_test_from_modexport_empty_slots'
+        modname = '_test_from_modexport_minimal_slots'
         filename = _testmultiphase.__file__
         module = import_extension_from_file(
             modname, filename, put_in_sys_modules=False)
@@ -3516,7 +3524,7 @@ def test_from_modexport_empty_slots(self):
         smoke_mod = import_extension_from_file(
             '_test_from_modexport_smoke', filename, put_in_sys_modules=False)
         self.assertEqual(_testcapi.pymodule_get_token(module),
-                         smoke_mod.get_modexport_empty_slots())
+                         smoke_mod.get_modexport_minimal_slots())
 
 @cpython_only
 class TestMagicNumber(unittest.TestCase):

Lib/test/test_itertools.py

@@ -754,6 +754,38 @@ def keys():
         next(g)
         next(g)  # must pass with address sanitizer
 
+    def test_grouper_reentrant_eq_does_not_crash(self):
+        # regression test for gh-146613
+        grouper_iter = None
+
+        class Key:
+            __hash__ = None
+
+            def __init__(self, do_advance):
+                self.do_advance = do_advance
+
+            def __eq__(self, other):
+                nonlocal grouper_iter
+                if self.do_advance:
+                    self.do_advance = False
+                    if grouper_iter is not None:
+                        try:
+                            next(grouper_iter)
+                        except StopIteration:
+                            pass
+                    return NotImplemented
+                return True
+
+        def keyfunc(element):
+            if element == 0:
+                return Key(do_advance=True)
+            return Key(do_advance=False)
+
+        g = itertools.groupby(range(4), keyfunc)
+        key, grouper_iter = next(g)
+        items = list(grouper_iter)
+        self.assertEqual(len(items), 1)
+
     def test_filter(self):
         self.assertEqual(list(filter(isEven, range(6))), [0,2,4])
         self.assertEqual(list(filter(None, [0,1,0,2,0])), [1,2])

Lib/test/test_mailbox.py

@@ -11,6 +11,7 @@
 from test.support import import_helper, warnings_helper
 from test.support import os_helper
 from test.support import refleak_helper
+from test.support import requires_root_user
 from test.support import socket_helper
 import unittest
 import textwrap
@@ -1086,6 +1087,7 @@ def test_permissions_after_flush(self):
 
         self.assertEqual(os.stat(self._path).st_mode, mode)
 
+    @requires_root_user
     @unittest.skipUnless(hasattr(os, 'chown'), 'requires os.chown')
     def test_ownership_after_flush(self):
         # See issue gh-117467
@@ -1108,10 +1110,7 @@ def test_ownership_after_flush(self):
         else:
             self.skipTest("test needs more than one group")
 
-        try:
-            os.chown(self._path, other_uid, other_gid)
-        except OSError:
-            self.skipTest('test needs root privilege')
+        os.chown(self._path, other_uid, other_gid)
         # Change permissions as in test_permissions_after_flush.
         mode = st.st_mode | 0o666
         os.chmod(self._path, mode)

Lib/test/test_os/test_os.py

@@ -33,6 +33,8 @@
 from test.support import os_helper
 from test.support import socket_helper
 from test.support import infinite_recursion
+from test.support import requires_root_user
+from test.support import requires_non_root_user
 from test.support import warnings_helper
 from platform import win32_is_iot
 from .utils import create_file
@@ -67,10 +69,6 @@
 from test.support.os_helper import FakePath
 
 
-root_in_posix = False
-if hasattr(os, 'geteuid'):
-    root_in_posix = (os.geteuid() == 0)
-
 # Detect whether we're on a Linux system that uses the (now outdated
 # and unmaintained) linuxthreads threading library.  There's an issue
 # when combining linuxthreads with a failed execv call: see
@@ -2257,8 +2255,8 @@ def test_chown_gid(self):
         gid = os.stat(os_helper.TESTFN).st_gid
         self.assertEqual(gid, gid_2)
 
-    @unittest.skipUnless(root_in_posix and len(all_users) > 1,
-                         "test needs root privilege and more than one user")
+    @requires_root_user
+    @unittest.skipUnless(len(all_users) > 1, "test needs more than one user")
     def test_chown_with_root(self):
         uid_1, uid_2 = all_users[:2]
         gid = os.stat(os_helper.TESTFN).st_gid
@@ -2269,8 +2267,8 @@ def test_chown_with_root(self):
         uid = os.stat(os_helper.TESTFN).st_uid
         self.assertEqual(uid, uid_2)
 
-    @unittest.skipUnless(not root_in_posix and len(all_users) > 1,
-                         "test needs non-root account and more than one user")
+    @requires_non_root_user
+    @unittest.skipUnless(len(all_users) > 1, "test needs and more than one user")
     def test_chown_without_permission(self):
         uid_1, uid_2 = all_users[:2]
         gid = os.stat(os_helper.TESTFN).st_gid