From 6af6165c21ddc6adc6208d69606dbdcdbdb06418 Mon Sep 17 00:00:00 2001 From: Vyron Vasileiadis Date: Wed, 29 Jul 2026 09:07:15 +0300 Subject: [PATCH 01/13] gh-154788: Make curses window.getparent() unconditional (GH-154789) getparent() calls no curses function, it returns the window's stored parent, but it was compiled only when the ncurses extension functions were present. Close the guard after getscrreg(), which does need it. The test moves out of test_state_getters, which is gated on is_scrollok(), so that getparent() is covered on backends without the extensions. --- Lib/test/test_curses.py | 12 +++++++++--- Modules/_cursesmodule.c | 4 +--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_curses.py b/Lib/test/test_curses.py index 9f7f8535b6d9a80..4e5d5f1da0be95e 100644 --- a/Lib/test/test_curses.py +++ b/Lib/test/test_curses.py @@ -278,6 +278,14 @@ def test_subwindows_references(self): del win2 gc_collect() + def test_getparent(self): + # getparent() calls no curses function, so it works with any backend + # and is not gated like the is_*() getters below. + stdscr = self.stdscr + self.assertIsNone(stdscr.getparent()) + sub = stdscr.subwin(3, 3, 0, 0) + self.assertIs(sub.getparent(), stdscr) + def test_dupwin(self): win = curses.newwin(5, 10, 2, 3) win.addstr(0, 0, 'ABCDE') @@ -1771,13 +1779,11 @@ def test_state_getters(self): stdscr.setscrreg(5, 10) self.assertEqual(stdscr.getscrreg(), (5, 10)) - # is_pad()/is_subwin()/getparent(). + # is_pad()/is_subwin(). self.assertIs(stdscr.is_pad(), False) self.assertIs(stdscr.is_subwin(), False) - self.assertIsNone(stdscr.getparent()) sub = stdscr.subwin(3, 3, 0, 0) self.assertIs(sub.is_subwin(), True) - self.assertIs(sub.getparent(), stdscr) pad = curses.newpad(5, 5) self.assertIs(pad.is_pad(), True) diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 01ea3c43cce2e6f..a913a7a0babe356 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -1936,6 +1936,7 @@ PyCursesWindow_getscrreg(PyObject *op, PyObject *Py_UNUSED(ignored)) } return Py_BuildValue("(ii)", top, bottom); } +#endif /* NCURSES_EXT_FUNCS >= 20110404 || PDCURSES */ static PyObject * PyCursesWindow_getparent(PyObject *op, PyObject *Py_UNUSED(ignored)) @@ -1948,7 +1949,6 @@ PyCursesWindow_getparent(PyObject *op, PyObject *Py_UNUSED(ignored)) } return Py_NewRef((PyObject *)self->orig); } -#endif /* NCURSES_EXT_FUNCS >= 20110404 || PDCURSES */ Window_NoArgNoReturnVoidFunction(wsyncup) Window_NoArgNoReturnVoidFunction(wsyncdown) @@ -5061,11 +5061,9 @@ static PyMethodDef PyCursesWindow_methods[] = { {"getmaxyx", PyCursesWindow_getmaxyx, METH_NOARGS, "getmaxyx($self, /)\n--\n\n" "Return a tuple (y, x) of the window height and width."}, -#if (defined(NCURSES_EXT_FUNCS) && NCURSES_EXT_FUNCS >= 20110404) || defined(PDCURSES) {"getparent", PyCursesWindow_getparent, METH_NOARGS, "getparent($self, /)\n--\n\n" "Return the parent window, or None if this is not a subwindow."}, -#endif {"getparyx", PyCursesWindow_getparyx, METH_NOARGS, "getparyx($self, /)\n--\n\n" "Return (y, x) relative to the parent window, or (-1, -1) if none."}, From 9c91fd958f3f16e2ea091013847cb024f9da5e48 Mon Sep 17 00:00:00 2001 From: Vyron Vasileiadis Date: Wed, 29 Jul 2026 09:30:37 +0300 Subject: [PATCH 02/13] gh-154781: Fix garbage from curses window.in_wstr() (GH-154782) in_wstr() searched the result for a terminating null, but winnwstr() writes one only if it stored at least one character. Use the number of characters it returns as the length. --- Lib/test/test_curses.py | 6 ++++++ Modules/_cursesmodule.c | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_curses.py b/Lib/test/test_curses.py index 4e5d5f1da0be95e..4ba210c162d0f6d 100644 --- a/Lib/test/test_curses.py +++ b/Lib/test/test_curses.py @@ -522,6 +522,12 @@ def test_in_wstr(self): self.assertEqual(stdscr.in_wstr(0, 0, len(s)), s) self.assertIsInstance(stdscr.instr(0, 0, len(s)), bytes) + # Reading no characters gives an empty string, like instr() and + # in_wchstr() do. curses does not terminate the buffer in this case. + stdscr.addstr(0, 0, 'abz') + self.assertEqual(stdscr.in_wstr(0, 0, 0), '') + self.assertEqual(stdscr.in_wstr(0), '') + def test_complexchar(self): # A complexchar is a styled wide-character cell: str() is its text, # and the attr and pair attributes are its rendition. diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index a913a7a0babe356..b8680edc6c0bed0 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -3956,7 +3956,7 @@ PyCursesWindow_in_wstr(PyObject *op, PyObject *args) PyMem_Free(buf); return Py_GetConstant(Py_CONSTANT_EMPTY_STR); } - PyObject *res = PyUnicode_FromWideChar(buf, -1); + PyObject *res = PyUnicode_FromWideChar(buf, rtn); PyMem_Free(buf); return res; #else From 9ccd5bb81edde823fb5fdbd51287f4a0ddfcd149 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Wed, 29 Jul 2026 12:05:11 +0530 Subject: [PATCH 03/13] gh-153531: fix thread safety of setting func.__doc__ and func.__module__ (#154851) --- .../test_free_threading/test_functions.py | 6 ++ Objects/funcobject.c | 90 ++++++++++++------- 2 files changed, 64 insertions(+), 32 deletions(-) diff --git a/Lib/test/test_free_threading/test_functions.py b/Lib/test/test_free_threading/test_functions.py index 65a90a17bd5b8db..7a59638d27dc9d4 100644 --- a/Lib/test/test_free_threading/test_functions.py +++ b/Lib/test/test_free_threading/test_functions.py @@ -58,6 +58,12 @@ def test_annotate(self): def test_type_params(self): self.stress_attribute("__type_params__", lambda: (random_string(),)) + def test_doc(self): + self.stress_attribute("__doc__", random_string) + + def test_module(self): + self.stress_attribute("__module__", random_string) + def test_annotations_and_annotate(self): # The __annotations__ and __annotate__ setters clear each other. def target(): pass diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 49a28e8ad667141..0c1fab7f6d33a8a 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -628,9 +628,7 @@ PyFunction_SetAnnotations(PyObject *op, PyObject *annotations) static PyMemberDef func_memberlist[] = { {"__closure__", _Py_T_OBJECT, OFF(func_closure), Py_READONLY}, - {"__doc__", _Py_T_OBJECT, OFF(func_doc), 0}, {"__globals__", _Py_T_OBJECT, OFF(func_globals), Py_READONLY}, - {"__module__", _Py_T_OBJECT, OFF(func_module), 0}, {"__builtins__", _Py_T_OBJECT, OFF(func_builtins), Py_READONLY}, {NULL} /* Sentinel */ }; @@ -762,6 +760,56 @@ func_set_qualname(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) return 0; } +static PyObject * +func_get_doc(PyObject *self, void *Py_UNUSED(ignored)) +{ + PyFunctionObject *op = _PyFunction_CAST(self); + PyObject *doc = op->func_doc; + if (doc == NULL) { + doc = Py_None; + } + return Py_NewRef(doc); +} + +static int +func_set_doc(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) +{ + /* Legal to del f.__doc__ or to set it to any object. */ + PyFunctionObject *op = _PyFunction_CAST(self); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); + PyObject *old_doc = op->func_doc; + op->func_doc = Py_XNewRef(value); + _PyEval_StartTheWorld(interp); + Py_XDECREF(old_doc); + return 0; +} + +static PyObject * +func_get_module(PyObject *self, void *Py_UNUSED(ignored)) +{ + PyFunctionObject *op = _PyFunction_CAST(self); + PyObject *module = op->func_module; + if (module == NULL) { + module = Py_None; + } + return Py_NewRef(module); +} + +static int +func_set_module(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) +{ + /* Legal to del f.__module__ or to set it to any object. */ + PyFunctionObject *op = _PyFunction_CAST(self); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); + PyObject *old_module = op->func_module; + op->func_module = Py_XNewRef(value); + _PyEval_StartTheWorld(interp); + Py_XDECREF(old_module); + return 0; +} + static PyObject * func_get_defaults(PyObject *self, void *Py_UNUSED(ignored)) { @@ -891,24 +939,12 @@ function___annotate___set_impl(PyFunctionObject *self, PyObject *value) return -1; } if (Py_IsNone(value)) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - _PyEval_StopTheWorld(interp); - PyObject *old_annotate = self->func_annotate; - self->func_annotate = Py_NewRef(value); - _PyEval_StartTheWorld(interp); - Py_XDECREF(old_annotate); + Py_XSETREF(self->func_annotate, Py_NewRef(value)); return 0; } else if (PyCallable_Check(value)) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - _PyEval_StopTheWorld(interp); - PyObject *old_annotate = self->func_annotate; - self->func_annotate = Py_NewRef(value); - PyObject *old_annotations = self->func_annotations; - self->func_annotations = NULL; - _PyEval_StartTheWorld(interp); - Py_XDECREF(old_annotate); - Py_XDECREF(old_annotations); + Py_XSETREF(self->func_annotate, Py_NewRef(value)); + Py_CLEAR(self->func_annotations); return 0; } else { @@ -961,15 +997,8 @@ function___annotations___set_impl(PyFunctionObject *self, PyObject *value) "__annotations__ must be set to a dict object"); return -1; } - PyInterpreterState *interp = _PyInterpreterState_GET(); - _PyEval_StopTheWorld(interp); - PyObject *old_annotations = self->func_annotations; - self->func_annotations = Py_XNewRef(value); - PyObject *old_annotate = self->func_annotate; - self->func_annotate = NULL; - _PyEval_StartTheWorld(interp); - Py_XDECREF(old_annotations); - Py_XDECREF(old_annotate); + Py_XSETREF(self->func_annotations, Py_XNewRef(value)); + Py_CLEAR(self->func_annotate); return 0; } @@ -1010,12 +1039,7 @@ function___type_params___set_impl(PyFunctionObject *self, PyObject *value) "__type_params__ must be set to a tuple"); return -1; } - PyInterpreterState *interp = _PyInterpreterState_GET(); - _PyEval_StopTheWorld(interp); - PyObject *old_typeparams = self->func_typeparams; - self->func_typeparams = Py_NewRef(value); - _PyEval_StartTheWorld(interp); - Py_XDECREF(old_typeparams); + Py_XSETREF(self->func_typeparams, Py_NewRef(value)); return 0; } @@ -1037,6 +1061,8 @@ static PyGetSetDef func_getsetlist[] = { FUNCTION___ANNOTATIONS___GETSETDEF FUNCTION___ANNOTATE___GETSETDEF {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict}, + {"__doc__", func_get_doc, func_set_doc}, + {"__module__", func_get_module, func_set_module}, {"__name__", func_get_name, func_set_name}, {"__qualname__", func_get_qualname, func_set_qualname}, FUNCTION___TYPE_PARAMS___GETSETDEF From ae18daedc75f40fdca7921bee3ce61dd1192c475 Mon Sep 17 00:00:00 2001 From: Oleksandr Baltian Date: Wed, 29 Jul 2026 15:56:07 +0200 Subject: [PATCH 04/13] gh-64192: Add *buffersize* to `imap()`/`imap_unordered()` in `multiprocessing.pool` (GH-136871) The new argument allows consuming the input iterator lazily, potentially saving memory, and allowing long/infinite iterators. It mirrors the *buffersize* argument to `concurrent.futures.Executor.map` that was added in 3.14. Co-authored-by: Oleksandr Baltian Co-authored-by: Petr Viktorin Co-authored-by: Sasha Baltian --- Doc/library/multiprocessing.rst | 19 +- Doc/whatsnew/3.16.rst | 18 +- Lib/multiprocessing/pool.py | 147 ++++++++------ Lib/test/_test_multiprocessing.py | 186 ++++++++++++++++-- ...5-07-28-11-00-49.gh-issue-64192.7htLtg.rst | 9 + 5 files changed, 305 insertions(+), 74 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-07-28-11-00-49.gh-issue-64192.7htLtg.rst diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 2d13053915830b0..bedea46cb16d60d 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -2536,7 +2536,7 @@ with the :class:`Pool` class. Callbacks should complete immediately since otherwise the thread which handles the results will get blocked. - .. method:: imap(func, iterable[, chunksize]) + .. method:: imap(func, iterable, chunksize=1, *, buffersize=None) A lazier version of :meth:`.map`. @@ -2550,12 +2550,27 @@ with the :class:`Pool` class. ``next(timeout)`` will raise :exc:`multiprocessing.TimeoutError` if the result cannot be returned within *timeout* seconds. - .. method:: imap_unordered(func, iterable[, chunksize]) + The *iterable* is collected immediately rather than lazily, unless a + *buffersize* is specified to limit the number of submitted tasks whose + results have not yet been yielded. If the buffer is full, iteration over + the *iterables* pauses until a result is yielded from the buffer. + To fully utilize pool's capacity when using this feature, + set *buffersize* at least to the number of processes in pool + (to consume *iterable* as you go), or even higher + (to prefetch the next ``N=buffersize-processes`` arguments). + + .. versionchanged:: next + Added the *buffersize* parameter. + + .. method:: imap_unordered(func, iterable, chunksize=1, *, buffersize=None) The same as :meth:`imap` except that the ordering of the results from the returned iterator should be considered arbitrary. (Only when there is only one worker process is the order guaranteed to be "correct".) + .. versionchanged:: next + Added the *buffersize* parameter. + .. method:: starmap(func, iterable[, chunksize]) Like :meth:`~multiprocessing.pool.Pool.map` except that the diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 06e831ea1e34631..6e69737768d5e15 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -376,6 +376,22 @@ math (Contributed by Jeff Epler in :gh:`150534`.) +multiprocessing +--------------- + +* Add the optional ``buffersize`` parameter to + :meth:`multiprocessing.pool.Pool.imap` and + :meth:`multiprocessing.pool.Pool.imap_unordered` to limit the number of + submitted tasks whose results have not yet been yielded. If the buffer is + full, iteration over the *iterables* pauses until a result is yielded from + the buffer. To fully utilize pool's capacity when using this feature, set + *buffersize* at least to the number of processes in pool (to consume + *iterable* as you go), or even higher (to prefetch the next + ``N=buffersize-processes`` arguments). + + (Contributed by Oleksandr Baltian in :gh:`136871`.) + + os -- @@ -606,7 +622,7 @@ module_name Removed -======= +======== annotationlib ------------- diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index f979890170b1a1f..8fd0f98a02dd3a6 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -190,6 +190,11 @@ def __init__(self, processes=None, initializer=None, initargs=(), self._ctx = context or get_context() self._setup_queues() self._taskqueue = queue.SimpleQueue() + # The _taskqueue_buffersize_semaphores exist to allow calling .release() + # on every active semaphore when the pool is terminating to let task_handler + # wake up to stop. It's a set so that each iterator object can efficiently + # deregister its semaphore when iterator finishes. + self._taskqueue_buffersize_semaphores = set() # The _change_notifier queue exist to wake up self._handle_workers() # when the cache (self._cache) is empty or when there is a change in # the _state variable of the thread that runs _handle_workers. @@ -256,7 +261,8 @@ def __init__(self, processes=None, initializer=None, initargs=(), self, self._terminate_pool, args=(self._taskqueue, self._inqueue, self._outqueue, self._pool, self._change_notifier, self._worker_handler, self._task_handler, - self._result_handler, self._cache), + self._result_handler, self._cache, + self._taskqueue_buffersize_semaphores), exitpriority=15 ) self._state = RUN @@ -382,73 +388,43 @@ def starmap_async(self, func, iterable, chunksize=None, callback=None, return self._map_async(func, iterable, starmapstar, chunksize, callback, error_callback) - def _guarded_task_generation(self, result_job, func, iterable): + def _guarded_task_generation(self, result_job, func, iterable, sema=None): '''Provides a generator of tasks for imap and imap_unordered with appropriate handling for iterables which throw exceptions during iteration.''' try: i = -1 - for i, x in enumerate(iterable): - yield (result_job, i, func, (x,), {}) + + if sema is None: + for i, x in enumerate(iterable): + yield (result_job, i, func, (x,), {}) + + else: + enumerated_iter = iter(enumerate(iterable)) + while True: + sema.acquire() + try: + i, x = next(enumerated_iter) + except StopIteration: + break + yield (result_job, i, func, (x,), {}) + except Exception as e: yield (result_job, i+1, _helper_reraises_exception, (e,), {}) - def imap(self, func, iterable, chunksize=1): + def imap(self, func, iterable, chunksize=1, *, buffersize=None): ''' Equivalent of `map()` -- can be MUCH slower than `Pool.map()`. ''' - self._check_running() - if chunksize == 1: - result = IMapIterator(self) - self._taskqueue.put( - ( - self._guarded_task_generation(result._job, func, iterable), - result._set_length - )) - return result - else: - if chunksize < 1: - raise ValueError( - "Chunksize must be 1+, not {0:n}".format( - chunksize)) - task_batches = Pool._get_tasks(func, iterable, chunksize) - result = IMapIterator(self) - self._taskqueue.put( - ( - self._guarded_task_generation(result._job, - mapstar, - task_batches), - result._set_length - )) - return (item for chunk in result for item in chunk) + return self._imap(IMapIterator, func, iterable, chunksize, + buffersize=buffersize) - def imap_unordered(self, func, iterable, chunksize=1): + def imap_unordered(self, func, iterable, chunksize=1, *, buffersize=None): ''' Like `imap()` method but ordering of results is arbitrary. ''' - self._check_running() - if chunksize == 1: - result = IMapUnorderedIterator(self) - self._taskqueue.put( - ( - self._guarded_task_generation(result._job, func, iterable), - result._set_length - )) - return result - else: - if chunksize < 1: - raise ValueError( - "Chunksize must be 1+, not {0!r}".format(chunksize)) - task_batches = Pool._get_tasks(func, iterable, chunksize) - result = IMapUnorderedIterator(self) - self._taskqueue.put( - ( - self._guarded_task_generation(result._job, - mapstar, - task_batches), - result._set_length - )) - return (item for chunk in result for item in chunk) + return self._imap(IMapUnorderedIterator, func, iterable, chunksize, + buffersize=buffersize) def apply_async(self, func, args=(), kwds={}, callback=None, error_callback=None): @@ -497,6 +473,41 @@ def _map_async(self, func, iterable, mapper, chunksize=None, callback=None, ) return result + def _imap(self, iterator_cls, func, iterable, chunksize=1, + *, buffersize=None): + self._check_running() + if chunksize < 1: + raise ValueError( + f"Chunksize must be 1+, not {chunksize}" + ) + if buffersize is not None: + if not isinstance(buffersize, int): + raise TypeError("buffersize must be an integer or None") + if buffersize < 1: + raise ValueError("buffersize must be None or > 0") + + result = iterator_cls(self, buffersize=buffersize) + if chunksize == 1: + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, func, iterable, + result._buffersize_sema), + result._set_length, + ) + ) + return result + else: + task_batches = Pool._get_tasks(func, iterable, chunksize) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, mapstar, + task_batches, + result._buffersize_sema), + result._set_length, + ) + ) + return (item for chunk in result for item in chunk) + @staticmethod def _wait_for_updates(sentinels, change_notifier, timeout=None): wait(sentinels, timeout=timeout) @@ -679,7 +690,8 @@ def _help_stuff_finish(inqueue, task_handler, size): @classmethod def _terminate_pool(cls, taskqueue, inqueue, outqueue, pool, change_notifier, - worker_handler, task_handler, result_handler, cache): + worker_handler, task_handler, result_handler, cache, + taskqueue_buffersize_semaphores): # this is guaranteed to only be called once util.debug('finalizing pool') @@ -690,6 +702,10 @@ def _terminate_pool(cls, taskqueue, inqueue, outqueue, pool, change_notifier, change_notifier.put(None) task_handler._state = TERMINATE + # Release all semaphores to wake up task_handler to stop. + for buffersize_sema in tuple(taskqueue_buffersize_semaphores): + buffersize_sema.release() + taskqueue_buffersize_semaphores.discard(buffersize_sema) util.debug('helping task handler/workers to finish') cls._help_stuff_finish(inqueue, task_handler, len(pool)) @@ -836,7 +852,7 @@ def _set(self, i, success_result): class IMapIterator(object): - def __init__(self, pool): + def __init__(self, pool, *, buffersize=None): self._pool = pool self._cond = threading.Condition(threading.Lock()) self._job = next(job_counter) @@ -846,6 +862,11 @@ def __init__(self, pool): self._length = None self._unsorted = {} self._cache[self._job] = self + if buffersize is None: + self._buffersize_sema = None + else: + self._buffersize_sema = threading.Semaphore(buffersize) + self._pool._taskqueue_buffersize_semaphores.add(self._buffersize_sema) def __iter__(self): return self @@ -856,22 +877,30 @@ def next(self, timeout=None): item = self._items.popleft() except IndexError: if self._index == self._length: - self._pool = None - raise StopIteration from None + self._stop_iterator() self._cond.wait(timeout) try: item = self._items.popleft() except IndexError: if self._index == self._length: - self._pool = None - raise StopIteration from None + self._stop_iterator() raise TimeoutError from None + if self._buffersize_sema is not None: + self._buffersize_sema.release() + success, value = item if success: return value raise value + def _stop_iterator(self): + if self._pool is not None: + # `self._pool` could be set to `None` in previous `.next()` calls + self._pool._taskqueue_buffersize_semaphores.discard(self._buffersize_sema) + self._pool = None + raise StopIteration from None + __next__ = next # XXX def _set(self, i, obj): diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 8f665b3a98a0371..36e0880bc088189 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -2904,11 +2904,13 @@ def exception_throwing_generator(total, when): class _TestPool(BaseTestCase): + _POOL_SIZE = 4 + @classmethod def setUpClass(cls): with warnings_helper.ignore_fork_in_thread_deprecation_warnings(): super().setUpClass() - cls.pool = cls.Pool(4) + cls.pool = cls.Pool(cls._POOL_SIZE) @classmethod def tearDownClass(cls): @@ -3022,18 +3024,36 @@ def test_async_timeout(self): p.terminate() p.join() - def test_imap(self): - it = self.pool.imap(sqr, list(range(10))) - self.assertEqual(list(it), list(map(sqr, list(range(10))))) - - it = self.pool.imap(sqr, list(range(10))) + @support.subTests('buffersize', ( + None, + 1, + _POOL_SIZE, + _POOL_SIZE * 2, + )) + def test_imap(self, buffersize): + iterable = range(10) + if self.TYPE != "threads": + iterable = list(iterable) + it = self.pool.imap(sqr, iterable, buffersize=buffersize) for i in range(10): - self.assertEqual(next(it), i*i) + self.assertEqual(next(it), i * i) + self.assertRaises(StopIteration, it.__next__) + # again, verify that it's truly exhausted self.assertRaises(StopIteration, it.__next__) - it = self.pool.imap(sqr, list(range(1000)), chunksize=100) + @support.subTests(('chunksize', 'buffersize'), ( + (100, None), + (100, _POOL_SIZE), + )) + def test_imap_with_chunksize(self, chunksize, buffersize): + iterable = range(1000) + if self.TYPE != "threads": + iterable = list(iterable) + it = self.pool.imap(sqr, iterable, chunksize=chunksize, buffersize=buffersize) for i in range(1000): - self.assertEqual(next(it), i*i) + self.assertEqual(next(it), i * i) + self.assertRaises(StopIteration, it.__next__) + # again, verify that it's truly exhausted self.assertRaises(StopIteration, it.__next__) def test_imap_handle_iterable_exception(self): @@ -3062,11 +3082,29 @@ def test_imap_handle_iterable_exception(self): self.assertEqual(next(it), i*i) self.assertRaises(SayWhenError, it.__next__) - def test_imap_unordered(self): - it = self.pool.imap_unordered(sqr, list(range(10))) + @support.subTests('buffersize', ( + None, + 1, + _POOL_SIZE, + _POOL_SIZE * 2, + )) + def test_imap_unordered(self, buffersize): + iterable = range(10) + if self.TYPE != "threads": + iterable = list(iterable) + it = self.pool.imap(sqr, iterable, buffersize=buffersize) self.assertEqual(sorted(it), list(map(sqr, list(range(10))))) - it = self.pool.imap_unordered(sqr, list(range(1000)), chunksize=100) + @support.subTests(('chunksize', 'buffersize'), ( + (100, None), + (100, _POOL_SIZE), + )) + def test_imap_unordered_with_chunksize(self, chunksize, buffersize): + iterable = range(1000) + if self.TYPE != "threads": + iterable = list(iterable) + it = self.pool.imap_unordered(sqr, iterable, chunksize=chunksize, + buffersize=buffersize) self.assertEqual(sorted(it), list(map(sqr, list(range(1000))))) def test_imap_unordered_handle_iterable_exception(self): @@ -3105,6 +3143,130 @@ def test_imap_unordered_handle_iterable_exception(self): self.assertIn(value, expected_values) expected_values.remove(value) + @support.subTests('method_name', ("imap", "imap_unordered")) + @support.subTests(('buffersize', 'expected_exception', 'expected_regex'), ( + ("foo", TypeError, "buffersize must be an integer or None"), + (2.0, TypeError, "buffersize must be an integer or None"), + (0, ValueError, "buffersize must be None or > 0"), + (-1, ValueError, "buffersize must be None or > 0"), + )) + def test_imap_and_imap_unordered_with_buffersize_type_validation( + self, method_name, buffersize, expected_exception, expected_regex + ): + method = getattr(self.pool, method_name) + with self.assertRaisesRegex(expected_exception, expected_regex): + method(str, range(4), buffersize=buffersize) + + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() + @support.subTests('method_name', ("imap", "imap_unordered")) + def test_imap_and_imap_unordered_when_buffer_is_full(self, method_name): + if self.TYPE != "threads": + self.skipTest("test not appropriate for {}".format(self.TYPE)) + + processes = 4 + p = self.Pool(processes) + last_produced_task_arg = Value("i") + + def produce_args(): + for arg in itertools.count(1): + last_produced_task_arg.value = arg + yield arg + + method = getattr(p, method_name) + it = method(functools.partial(sqr, wait=0.2), produce_args()) + + time.sleep(0.2) + # `iterable` could've been advanced only `processes` times, + # but in fact it advances further (`> processes`) because of + # not waiting for workers or user code to catch up. + self.assertGreater(last_produced_task_arg.value, processes) + + next(it) + time.sleep(0.2) + self.assertGreater(last_produced_task_arg.value, processes + 1) + + next(it) + time.sleep(0.2) + self.assertGreater(last_produced_task_arg.value, processes + 2) + + p.terminate() + p.join() + + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() + @support.subTests('method_name', ("imap", "imap_unordered")) + def test_imap_and_imap_unordered_with_buffersize_when_buffer_is_full( + self, method_name + ): + if self.TYPE != "threads": + self.skipTest("test not appropriate for {}".format(self.TYPE)) + + processes = 4 + p = self.Pool(processes) + last_produced_task_arg = Value("i") + + def produce_args(): + for arg in itertools.count(1): + last_produced_task_arg.value = arg + yield arg + + method = getattr(p, method_name) + it = method(functools.partial(sqr, wait=0.2), produce_args(), + buffersize=processes) + + time.sleep(0.2) + self.assertEqual(last_produced_task_arg.value, processes) + + next(it) + time.sleep(0.2) + self.assertEqual(last_produced_task_arg.value, processes + 1) + + next(it) + time.sleep(0.2) + self.assertEqual(last_produced_task_arg.value, processes + 2) + + p.terminate() + p.join() + + @support.subTests('method_name', ("imap", "imap_unordered")) + def test_imap_and_imap_unordered_with_buffersize_on_empty_iterable( + self, method_name + ): + method = getattr(self.pool, method_name) + res = method(str, [], buffersize=2) + self.assertIsNone(next(res, None)) + self.assertIsNone(next(res, None)) + + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() + def test_imap_with_buffersize_on_infinite_iterable(self): + if self.TYPE != "threads": + self.skipTest("test not appropriate for {}".format(self.TYPE)) + + p = self.Pool(4) + res = p.imap(str, itertools.count(), buffersize=2) + + self.assertEqual(next(res, None), "0") + self.assertEqual(next(res, None), "1") + self.assertEqual(next(res, None), "2") + + p.terminate() + p.join() + + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() + def test_imap_unordered_with_buffersize_on_infinite_iterable(self): + if self.TYPE != "threads": + self.skipTest("test not appropriate for {}".format(self.TYPE)) + + p = self.Pool(4) + res = p.imap_unordered(str, itertools.count(), buffersize=2) + + # (4, 5, ...) can also be submitted to the pool, so assert just 3 unique results + first_three_results = [next(res, None) for _ in range(3)] + self.assertEqual(len(first_three_results), 3) + self.assertEqual(len(set(first_three_results)), 3) + + p.terminate() + p.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_make_pool(self): expected_error = (RemoteError if self.TYPE == 'manager' diff --git a/Misc/NEWS.d/next/Library/2025-07-28-11-00-49.gh-issue-64192.7htLtg.rst b/Misc/NEWS.d/next/Library/2025-07-28-11-00-49.gh-issue-64192.7htLtg.rst new file mode 100644 index 000000000000000..ca40bc111176e07 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-28-11-00-49.gh-issue-64192.7htLtg.rst @@ -0,0 +1,9 @@ +Add the optional ``buffersize`` parameter to +:meth:`multiprocessing.pool.Pool.imap` and +:meth:`multiprocessing.pool.Pool.imap_unordered` to limit the number of +submitted tasks whose results have not yet been yielded. If the buffer is +full, iteration over the *iterables* pauses until a result is yielded from +the buffer. To fully utilize pool's capacity when using this feature, set +*buffersize* at least to the number of processes in pool (to consume +*iterable* as you go), or even higher (to prefetch the next +``N=buffersize-processes`` arguments). From bfc16a71cf18015c47c1b9cae9196e279d8f60d7 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 29 Jul 2026 17:47:52 +0300 Subject: [PATCH 05/13] gh-131565: Implement ctypes.util.dllist() in the _ctypes extension (GH-154255) On NetBSD dl_iterate_phdr() reports only the link-map group of the calling object. Called through ctypes, the caller is libffi's closure trampoline, which belongs to no object, so only the main executable was reported. Calling dl_iterate_phdr() from the _ctypes extension module makes _ctypes the caller and reports all loaded shared libraries. Co-Authored-By: Claude Fable 5 --- Lib/ctypes/util.py | 53 +++---------------- ...-07-20-17-15-00.gh-issue-131565.dllist.rst | 4 ++ Modules/_ctypes/callproc.c | 40 ++++++++++++++ configure | 9 ++++ configure.ac | 3 ++ pyconfig.h.in | 3 ++ 6 files changed, 65 insertions(+), 47 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-20-17-15-00.gh-issue-131565.dllist.rst diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index c0a578c86549ec9..141d3fbe9382472 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -412,53 +412,12 @@ def find_library(name): _get_soname(_findLib_gcc(name)) or _get_soname(_findLib_ld(name)) -# Listing loaded libraries on other systems will try to use -# functions common to Linux and a few other Unix-like systems. -# See the following for several platforms' documentation of the same API: -# https://man7.org/linux/man-pages/man3/dl_iterate_phdr.3.html -# https://man.freebsd.org/cgi/man.cgi?query=dl_iterate_phdr -# https://man.openbsd.org/dl_iterate_phdr -# https://docs.oracle.com/cd/E88353_01/html/E37843/dl-iterate-phdr-3c.html -if (os.name == "posix" and - sys.platform not in {"darwin", "ios", "tvos", "watchos"}): - import ctypes - if hasattr((_libc := ctypes.CDLL(None)), "dl_iterate_phdr"): - - class _dl_phdr_info(ctypes.Structure): - _fields_ = [ - ("dlpi_addr", ctypes.c_void_p), - ("dlpi_name", ctypes.c_char_p), - ("dlpi_phdr", ctypes.c_void_p), - ("dlpi_phnum", ctypes.c_ushort), - ] - - _dl_phdr_callback = ctypes.CFUNCTYPE( - ctypes.c_int, - ctypes.POINTER(_dl_phdr_info), - ctypes.c_size_t, - ctypes.POINTER(ctypes.py_object), - ) - - @_dl_phdr_callback - def _info_callback(info, _size, data): - libraries = data.contents.value - name = os.fsdecode(info.contents.dlpi_name) - libraries.append(name) - return 0 - - _dl_iterate_phdr = _libc["dl_iterate_phdr"] - _dl_iterate_phdr.argtypes = [ - _dl_phdr_callback, - ctypes.POINTER(ctypes.py_object), - ] - _dl_iterate_phdr.restype = ctypes.c_int - - def dllist(): - """Return a list of loaded shared libraries in the current process.""" - libraries = [] - _dl_iterate_phdr(_info_callback, - ctypes.byref(ctypes.py_object(libraries))) - return libraries +# On platforms which provide dl_iterate_phdr(), dllist() is implemented +# in _ctypes. +try: + from _ctypes import dllist +except ImportError: + pass @dataclass(slots=True, frozen=True) diff --git a/Misc/NEWS.d/next/Library/2026-07-20-17-15-00.gh-issue-131565.dllist.rst b/Misc/NEWS.d/next/Library/2026-07-20-17-15-00.gh-issue-131565.dllist.rst new file mode 100644 index 000000000000000..5fb2dd099bd419f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-20-17-15-00.gh-issue-131565.dllist.rst @@ -0,0 +1,4 @@ +:func:`ctypes.util.dllist` now works on NetBSD. It is implemented in the +:mod:`!_ctypes` extension module so that ``dl_iterate_phdr()`` reports all +loaded shared libraries: on NetBSD it only reports the link-map group of +the calling object, which excluded them when called through ctypes. diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index ccc57e347b07acf..746034c8004c5c4 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1660,6 +1660,42 @@ static PyObject *py_dl_sym(PyObject *self, PyObject *args) PyErr_Format(PyExc_OSError, "symbol '%s' not found", name); return NULL; } + +// Apple platforms use the dyld API in ctypes.util instead. +#if defined(HAVE_DL_ITERATE_PHDR) && !defined(__APPLE__) +#include + +static int +_dllist_callback(struct dl_phdr_info *info, size_t size, void *data) +{ + PyObject *list = (PyObject *)data; + PyObject *name = PyUnicode_DecodeFSDefault(info->dlpi_name); + if (name == NULL) { + return -1; + } + int res = PyList_Append(list, name); + Py_DECREF(name); + return res; +} + +static PyObject * +dllist(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + // On NetBSD dl_iterate_phdr() only reports the link-map group of the + // caller, so it cannot be called via a libffi trampoline. + PyObject *list = PyList_New(0); + if (list == NULL) { + return NULL; + } + // The return value only echoes the callback result. + dl_iterate_phdr(_dllist_callback, list); + if (PyErr_Occurred()) { + Py_DECREF(list); + return NULL; + } + return list; +} +#endif #endif /* @@ -2036,6 +2072,10 @@ PyMethodDef _ctypes_module_methods[] = { "dlopen(name, flag={RTLD_GLOBAL|RTLD_LOCAL}) open a shared library"}, {"dlclose", py_dl_close, METH_VARARGS, "dlclose a library"}, {"dlsym", py_dl_sym, METH_VARARGS, "find symbol in shared library"}, +#if defined(HAVE_DL_ITERATE_PHDR) && !defined(__APPLE__) + {"dllist", dllist, METH_NOARGS, + "dllist() return a list of loaded shared libraries"}, +#endif #endif #ifdef __APPLE__ {"_dyld_shared_cache_contains_path", py_dyld_shared_cache_contains_path, METH_VARARGS, "check if path is in the shared cache"}, diff --git a/configure b/configure index 8e7accb4bc793da..2c82da923f68d4a 100755 --- a/configure +++ b/configure @@ -20229,6 +20229,15 @@ then : fi +# Used by ctypes.util.dllist(). +ac_fn_c_check_func "$LINENO" "dl_iterate_phdr" "ac_cv_func_dl_iterate_phdr" +if test "x$ac_cv_func_dl_iterate_phdr" = xyes +then : + printf "%s\n" "#define HAVE_DL_ITERATE_PHDR 1" >>confdefs.h + +fi + + # DYNLOADFILE specifies which dynload_*.o file we will use for dynamic # loading of modules. diff --git a/configure.ac b/configure.ac index ce146cfa1cbc4cd..771260c76b7f094 100644 --- a/configure.ac +++ b/configure.ac @@ -5421,6 +5421,9 @@ DLINCLDIR=. # platforms have dlopen(), but don't want to use it. AC_CHECK_FUNCS([dlopen]) +# Used by ctypes.util.dllist(). +AC_CHECK_FUNCS([dl_iterate_phdr]) + # DYNLOADFILE specifies which dynload_*.o file we will use for dynamic # loading of modules. AC_SUBST([DYNLOADFILE]) diff --git a/pyconfig.h.in b/pyconfig.h.in index 2658fe8116781db..691c6c0d9feb6d0 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -398,6 +398,9 @@ /* Define to 1 if you have the 'dlopen' function. */ #undef HAVE_DLOPEN +/* Define to 1 if you have the 'dl_iterate_phdr' function. */ +#undef HAVE_DL_ITERATE_PHDR + /* Define to 1 if you have the 'dup' function. */ #undef HAVE_DUP From a528a24f9d739126d790d4430050eed85ba60946 Mon Sep 17 00:00:00 2001 From: coroma01 Date: Wed, 29 Jul 2026 18:06:03 +0100 Subject: [PATCH 06/13] gh-151321: Fix incorrect opcode metadata flags for LOAD_DEREF opcode (GH-154778) --- Include/internal/pycore_opcode_metadata.h | 2 +- Include/internal/pycore_uop_metadata.h | 2 +- Tools/cases_generator/analyzer.py | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 31d2e31573d8c53..457e5c5bf20d2b9 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1245,7 +1245,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_COMMON_CONSTANT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, - [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FAST_BORROW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 990706c0b329223..e52233b21277591 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -199,7 +199,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_MAKE_CELL] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_DELETE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_LOAD_FROM_DICT_OR_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_DEREF] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG, [_COPY_FREE_VARS] = HAS_ARG_FLAG, [_BUILD_STRING] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 4eed4d8d60e4856..78dfdc5d6a89d57 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -971,6 +971,7 @@ def compute_properties(op: parser.CodeDef) -> Properties: or variable_used(op, "PyCell_GetRef") or variable_used(op, "PyCell_SetTakeRef") or variable_used(op, "PyCell_SwapTakeRef") + or variable_used(op, "_PyCell_GetStackRef") ) deopts_if = variable_used(op, "DEOPT_IF") exits_if = variable_used(op, "EXIT_IF") From 3fd36fadbe8ad31213b90757d196870213bf07ba Mon Sep 17 00:00:00 2001 From: Bhuvansh Date: Wed, 29 Jul 2026 22:56:45 +0530 Subject: [PATCH 07/13] gh-154709: Fix out-of-bounds access in dict reverse iterator (GH-154721) --- Lib/test/test_dict.py | 21 +++++++++++++++++++ ...-07-26-09-07-41.gh-issue-154709.M2uZ76.rst | 2 ++ Objects/dictobject.c | 5 ++++- 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-07-26-09-07-41.gh-issue-154709.M2uZ76.rst diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index dc31d403b837adb..1e665c86303078c 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1403,6 +1403,27 @@ def __init__(self, x, y): self.assertEqual(list(reversed(A(1, 0).__dict__)), ['x']) self.assertEqual(list(reversed(A(0, 1).__dict__)), ['y']) + def test_reversed_dict_after_clear_and_restore(self): + d = {} + for i in range(1000): + d[f"k{i}"] = i + + for i in range(1, 1000): + del d[f"k{i}"] + + iterators = ( + reversed(d), + reversed(d.keys()), + reversed(d.values()), + reversed(d.items()), + ) + + d.clear() + d["k0"] = 0 + + for it in iterators: + self.assertEqual(list(it), []) + def test_dict_copy_order(self): # bpo-34320 od = collections.OrderedDict([('a', 1), ('b', 2)]) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-26-09-07-41.gh-issue-154709.M2uZ76.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-26-09-07-41.gh-issue-154709.M2uZ76.rst new file mode 100644 index 000000000000000..6eb1ce6a8b6970b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-26-09-07-41.gh-issue-154709.M2uZ76.rst @@ -0,0 +1,2 @@ +Fix an out-of-bounds access in reverse dictionary iterators when the +underlying dictionary is cleared and modified after the iterator is created. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index c650aa456d2cc9d..74b6d5d779a064c 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -6276,9 +6276,12 @@ dictreviter_iter_lock_held(PyDictObject *d, PyObject *self) int index = get_index_from_order(d, i); key = LOAD_SHARED_KEY(DK_UNICODE_ENTRIES(k)[index].me_key); value = d->ma_values->values[index]; - assert (value != NULL); + assert(value != NULL); } else { + if (i >= k->dk_nentries) { + goto fail; + } if (DK_IS_UNICODE(k)) { PyDictUnicodeEntry *entry_ptr = &DK_UNICODE_ENTRIES(k)[i]; while (entry_ptr->me_value == NULL) { From 11d0da5b54b1a162e8f2566675007cfb2db797b5 Mon Sep 17 00:00:00 2001 From: Calvin Prewitt Date: Wed, 29 Jul 2026 12:45:41 -0500 Subject: [PATCH 08/13] gh-154836: Fix Popen.wait() with very large timeouts on the pidfd/kqueue wait paths (GH-154837) The event-driven wait introduced by gh-83069 passes the caller's timeout unclamped to poll() / kqueue.control(), so values that do not fit the C timestamp conversion (float('inf'), sys.maxsize, 1e10, ...) raise OverflowError on Linux and, on macOS/BSD, a misleading "TypeError: timeout must be a real number or None" -- all of which worked on 3.14 and earlier. - Lib/subprocess.py: clamp each wait to _MAXIMUM_WAIT_TIMEOUT (24h, following asyncio's MAXIMUM_SELECT_TIMEOUT precedent) and loop until the real deadline in both _wait_pidfd() and _wait_kqueue(). - Modules/selectmodule.c: only rewrite the kqueue.control() timeout conversion failure into TypeError when the original exception IS a TypeError, exactly like the select()/poll()/devpoll()/epoll() sites, so OverflowError surfaces for out-of-range values. Co-authored-by: Claude Fable 5 --- Lib/subprocess.py | 49 ++++++++++++++----- Lib/test/test_kqueue.py | 14 ++++++ Lib/test/test_subprocess.py | 26 ++++++++++ ...-07-19-20-00-00.gh-issue-154836.kqWait.rst | 5 ++ Modules/selectmodule.c | 8 +-- 5 files changed, 88 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-19-20-00-00.gh-issue-154836.kqWait.rst diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 6fe2ec98fb40888..054860a19c74b6d 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -917,6 +917,12 @@ def _can_use_kqueue(): _CAN_USE_PIDFD_OPEN = not _mswindows and _can_use_pidfd_open() _CAN_USE_KQUEUE = not _mswindows and _can_use_kqueue() +# Maximum timeout passed to poll() / kqueue.control() by Popen._wait(): +# very large values (e.g. timeout=float('inf')) overflow the C timestamp +# conversion (gh-154836). Longer waits are performed in bounded slices +# until the deadline, like asyncio's MAXIMUM_SELECT_TIMEOUT. +_MAXIMUM_WAIT_TIMEOUT = 24 * 3600 + # These are primarily fail-safe knobs for negatives. A True value does not # guarantee the given libc/syscall API will be used. @@ -2228,10 +2234,19 @@ def _wait_pidfd(self, timeout): try: poller = select.poll() poller.register(pidfd, select.POLLIN) - events = poller.poll(timeout * 1000) - if not events: - raise TimeoutExpired(self.args, timeout) - return True + endtime = _time() + timeout + while True: + # Clamp very large timeouts (e.g. float('inf')): + # they overflow the C timestamp conversion in + # poll(). Wait in bounded slices until the real + # deadline (gh-154836). + delay = min(_deadline_remaining(endtime), + _MAXIMUM_WAIT_TIMEOUT) + events = poller.poll(max(delay, 0) * 1000) + if events: + return True + if _deadline_remaining(endtime) <= 0: + raise TimeoutExpired(self.args, timeout) finally: os.close(pidfd) @@ -2252,14 +2267,26 @@ def _wait_kqueue(self, timeout): flags=select.KQ_EV_ADD | select.KQ_EV_ONESHOT, fflags=select.KQ_NOTE_EXIT, ) - try: - events = kq.control([kev], 1, timeout) # wait - except OSError: - return False - else: - if not events: + changelist = [kev] + endtime = _time() + timeout + while True: + # Clamp very large timeouts (e.g. float('inf')): + # they overflow the C timestamp conversion in + # kqueue.control(). Wait in bounded slices until + # the real deadline (gh-154836). + delay = min(_deadline_remaining(endtime), + _MAXIMUM_WAIT_TIMEOUT) + try: + events = kq.control(changelist, 1, max(delay, 0)) + except OSError: + return False + if events: + return True + if _deadline_remaining(endtime) <= 0: raise TimeoutExpired(self.args, timeout) - return True + # The kevent was registered by the first control() + # call; don't re-add it on later slices. + changelist = None finally: kq.close() diff --git a/Lib/test/test_kqueue.py b/Lib/test/test_kqueue.py index 2cf99be9e2c3baa..2649f3a7aee9b96 100644 --- a/Lib/test/test_kqueue.py +++ b/Lib/test/test_kqueue.py @@ -23,6 +23,20 @@ def test_create_queue(self): self.assertTrue(kq.closed) self.assertRaises(ValueError, kq.fileno) + def test_control_overflowing_timeout(self): + # gh-154836: out-of-range timeouts must raise OverflowError, + # not a (misleading) TypeError, like select(), poll() and + # epoll() do. + kq = select.kqueue() + self.addCleanup(kq.close) + for timeout in (1e300, float('inf'), 2**200): + with self.subTest(timeout=timeout): + with self.assertRaises(OverflowError): + kq.control(None, 0, timeout) + # Non-numbers still raise TypeError. + with self.assertRaises(TypeError): + kq.control(None, 0, "0.1") + def test_create_event(self): from operator import lt, le, gt, ge diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index d066ae85dfc51a6..4fd14d98b0324c0 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -4300,5 +4300,31 @@ def test_fast_path_avoid_busy_loop(self): self.assertEqual(p.wait(timeout=support.LONG_TIMEOUT), 0) self.assertFalse(m.called) + @unittest.skipIf(mswindows, "requires the POSIX wait implementation") + def test_wait_huge_timeout(self): + # gh-154836: very large timeout values used to overflow the C + # timestamp conversion in poll() / kqueue.control() and raise + # OverflowError / TypeError. + for timeout in (10**10, sys.maxsize, float('inf')): + with self.subTest(timeout=timeout): + p = subprocess.Popen(ZERO_RETURN_CMD) + self.assertEqual(p.wait(timeout=timeout), 0) + + @unittest.skipIf(mswindows, "requires the POSIX wait implementation") + def test_run_huge_timeout(self): + # gh-154836: same as test_wait_huge_timeout, via the + # subprocess.run() / communicate() code path. + cp = subprocess.run(ZERO_RETURN_CMD, timeout=1e10) + self.assertEqual(cp.returncode, 0) + + @unittest.skipIf(mswindows, "requires the POSIX wait implementation") + def test_wait_slices_do_not_expire_early(self): + # A clamped wait slice must not raise TimeoutExpired before the + # real deadline: with a tiny slice limit, a process that + # outlives many slices must still be waited for successfully. + with mock.patch.object(subprocess, "_MAXIMUM_WAIT_TIMEOUT", 0.01): + p = subprocess.Popen(self.COMMAND) # sleeps 0.3s + self.assertEqual(p.wait(timeout=support.SHORT_TIMEOUT), 0) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2026-07-19-20-00-00.gh-issue-154836.kqWait.rst b/Misc/NEWS.d/next/Library/2026-07-19-20-00-00.gh-issue-154836.kqWait.rst new file mode 100644 index 000000000000000..3691d60413e6a1f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-19-20-00-00.gh-issue-154836.kqWait.rst @@ -0,0 +1,5 @@ +Fix :meth:`subprocess.Popen.wait` raising :exc:`TypeError` (macOS and other +BSDs) or :exc:`OverflowError` (Linux) for very large *timeout* values such +as ``float('inf')``, a 3.15 regression in the new event-driven wait. Also +fix :meth:`select.kqueue.control` masking :exc:`OverflowError` for +out-of-range timeouts as :exc:`TypeError`. diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 2c56dbc6a541f7a..b9fb7762e3dacdf 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -2365,9 +2365,11 @@ select_kqueue_control_impl(kqueue_queue_Object *self, PyObject *changelist, else { if (_PyTime_FromSecondsObject(&timeout, otimeout, _PyTime_ROUND_TIMEOUT) < 0) { - PyErr_Format(PyExc_TypeError, - "timeout must be a real number or None, not %T", - otimeout); + if (PyErr_ExceptionMatches(PyExc_TypeError)) { + PyErr_Format(PyExc_TypeError, + "timeout must be a real number or None, not %T", + otimeout); + } return NULL; } From 49f96670d98c50eca769dd3f6fb5b6de11010617 Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Wed, 29 Jul 2026 14:08:45 -0700 Subject: [PATCH 09/13] gh-135736: Fix interaction between TaskGroup and aclose() (#154649) Currently if you have a generator like: ``` async def test(): async with asyncio.TaskGroup() as tg: async for x in whatever(): yield x ``` If aclose() is called on the generator and GeneratorExit is raised, the TaskGroup's __aexit__ will raise a BaseExceptionGroup, and that will be raised from the aclose() call instead of it being swallowed. Fix this by raising GeneratorExit from __aexit__ when: 1. The body of task group raised it 2. No subtasks raised exceptions This makes GeneratorExit work without swallowing other exceptions (like it would if we just added it to _is_base_error). --- Doc/library/asyncio-task.rst | 8 +++ Lib/asyncio/taskgroups.py | 21 ++++-- Lib/test/test_asyncio/test_taskgroups.py | 66 +++++++++++++++++++ ...-07-24-12-29-15.gh-issue-135736.DyvA2m.rst | 3 + 4 files changed, 94 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-24-12-29-15.gh-issue-135736.DyvA2m.rst diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 86b8c5aa34d2fe9..b6f3662862eb38f 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -433,6 +433,10 @@ unless it is :exc:`asyncio.CancelledError`, is also included in the exception group. The same special case is made for :exc:`KeyboardInterrupt` and :exc:`SystemExit` as in the previous paragraph. +There is an additional special case made only for the body of the +``async with``: if it raises :exc:`GeneratorExit` and none of the +other tasks raise exceptions that would be reported, then the +:exc:`GeneratorExit` is reraised. Task groups are careful not to mix up the internal cancellation used to "wake up" their :meth:`~object.__aexit__` with cancellation requests @@ -456,6 +460,10 @@ reported by :meth:`asyncio.Task.cancelling`. Improved handling of simultaneous internal and external cancellations and correct preservation of cancellation counts. +.. versionchanged:: 3.15 + + Addition of the special case for :exc:`GeneratorExit`. + Sleeping ======== diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index e1ec025791a52e6..a431b45a19489d5 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -174,10 +174,23 @@ async def _aexit(self, et, exc): self._parent_task.uncancel() self._parent_task.cancel() try: - raise BaseExceptionGroup( - 'unhandled errors in a TaskGroup', - self._errors, - ) from None + # If the *only* error is a GeneratorExit from the body + # of the group, then instead of raising an + # ExceptionGroup we raise GeneratorExit. This ensures + # that async generators that use TaskGroup properly + # swallow the exception on `aclose()` while ensuring + # that no exceptions from subtasks are swallowed. + if ( + et is not None + and issubclass(et, GeneratorExit) + and len(self._errors) == 1 + ): + raise exc + else: + raise BaseExceptionGroup( + 'unhandled errors in a TaskGroup', + self._errors, + ) from None finally: exc = None diff --git a/Lib/test/test_asyncio/test_taskgroups.py b/Lib/test/test_asyncio/test_taskgroups.py index e1eaa60e4df85df..bc246400b83e9b2 100644 --- a/Lib/test/test_asyncio/test_taskgroups.py +++ b/Lib/test/test_asyncio/test_taskgroups.py @@ -1227,6 +1227,72 @@ async def fn_3(): self.assertEqual(await race(fn_1, fn_2, fn_3), 1) self.assertListEqual(record, ["1 started", "2 started", "3 started", "1 finished"]) + async def test_taskgroup_generator_exit_01(self): + # GeneratorExit in a TaskGroup should be fine + async def gen(): + yield 1 + + async def fn(): + async with asyncio.TaskGroup() as tg: + async for n in gen(): + yield n + + g = fn() + await g.asend(None) + await g.aclose() + + async def test_taskgroup_generator_exit_02(self): + # A lone GeneratorExit in a task should still give an ExceptionGroup + async def t(): + raise GeneratorExit + + async def fn(): + async with asyncio.TaskGroup() as tg: + tg.create_task(t()) + + with self.assertRaises(BaseExceptionGroup) as cm: + await fn() + self.assertEqual(get_error_types(cm.exception), {GeneratorExit}) + + async def test_taskgroup_generator_exit_03(self): + # A GeneratorExit in one task and an error in another should + # still give an ExceptionGroup + async def t1(): + raise GeneratorExit + + async def t2(): + raise AssertionError('t2 failed') + + async def fn(): + async with asyncio.TaskGroup() as tg: + tg.create_task(t1()) + tg.create_task(t2()) + + with self.assertRaises(BaseExceptionGroup) as cm: + await fn() + + self.assertEqual(get_error_types(cm.exception), {GeneratorExit, AssertionError}) + + async def test_taskgroup_generator_exit_04(self): + event = asyncio.Event() + async def t(): + event.set() + raise AssertionError('t failed') + + async def fn(): + async with asyncio.TaskGroup() as tg: + tg.create_task(t()) + yield 1 + + g = fn() + await g.asend(None) + await event.wait() # wait for t() to run + + with self.assertRaises(BaseExceptionGroup) as cm: + await g.aclose() + + self.assertEqual(get_error_types(cm.exception), {GeneratorExit, AssertionError}) + class TestTaskGroup(BaseTestTaskGroup, unittest.IsolatedAsyncioTestCase): loop_factory = asyncio.EventLoop diff --git a/Misc/NEWS.d/next/Library/2026-07-24-12-29-15.gh-issue-135736.DyvA2m.rst b/Misc/NEWS.d/next/Library/2026-07-24-12-29-15.gh-issue-135736.DyvA2m.rst new file mode 100644 index 000000000000000..5eaa596afba6095 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-24-12-29-15.gh-issue-135736.DyvA2m.rst @@ -0,0 +1,3 @@ +Fix ::class:`asyncio.TaskGroup` to not wrap a :exc:`GeneratorExit` into a +:exc:`BaseExceptionGroup` if it was raised by the body of the task group and +none of the tasks in the group raised exceptions. From 51ae3c0ee585e2bc56d0a58c619eefa1b4b4f929 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 30 Jul 2026 03:04:46 +0100 Subject: [PATCH 10/13] Make `os.get_terminal_size` check `isatty` before calling `ioctl` (#154885) Calling ioctl on stdout raises warnings on Android. Ensure we have a TTY before doing terminal size calls. --- Lib/test/test_os/test_os.py | 7 +------ .../Library/2026-07-29-16-53-50.gh-issue-154885.ptofmI.rst | 2 ++ Modules/posixmodule.c | 7 +++++++ 3 files changed, 10 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-29-16-53-50.gh-issue-154885.ptofmI.rst diff --git a/Lib/test/test_os/test_os.py b/Lib/test/test_os/test_os.py index 3e5ad52c4ab130d..328a0dbeb99f8fa 100644 --- a/Lib/test/test_os/test_os.py +++ b/Lib/test/test_os/test_os.py @@ -3970,12 +3970,7 @@ def test_does_not_crash(self): try: size = os.get_terminal_size() except OSError as e: - known_errnos = [errno.EINVAL, errno.ENOTTY] - if sys.platform == "android": - # The Android testbed redirects the native stdout to a pipe, - # which returns a different error code. - known_errnos.append(errno.EACCES) - if sys.platform == "win32" or e.errno in known_errnos: + if sys.platform == "win32" or e.errno in (errno.EINVAL, errno.ENOTTY): # Under win32 a generic OSError can be thrown if the # handle cannot be retrieved self.skipTest("failed to query terminal size") diff --git a/Misc/NEWS.d/next/Library/2026-07-29-16-53-50.gh-issue-154885.ptofmI.rst b/Misc/NEWS.d/next/Library/2026-07-29-16-53-50.gh-issue-154885.ptofmI.rst new file mode 100644 index 000000000000000..7c3839ad1a59e4f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-29-16-53-50.gh-issue-154885.ptofmI.rst @@ -0,0 +1,2 @@ +:func:`os.get_terminal_size` now checks ``isatty`` before calling ``ioctl``, +which reduces log noise on Android. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index f754d0e18b5fb09..c34e3fc5eb600df 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -15971,6 +15971,13 @@ os_get_terminal_size_impl(PyObject *module, int fd) #ifdef TERMSIZE_USE_IOCTL { + // On Android, stdout is probably not connected, and calling TIOCGWINSZ + // on an invalid file descriptor causes a log message "avc: denied { + // ioctl }". Some common tools such as pytest call get_terminal_size + // very often, so check it's a TTY first to avoid cluttering the log. + if (!isatty(fd)) + return PyErr_SetFromErrno(PyExc_OSError); + struct winsize w; if (ioctl(fd, TIOCGWINSZ, &w)) return PyErr_SetFromErrno(PyExc_OSError); From f4b1d3e891d0d2055e53df3ccb84030bcaa148b5 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 30 Jul 2026 03:31:40 +0100 Subject: [PATCH 11/13] Minor fixes for Android (#154895) A collection of small cleanups for Android support: * Clarifies the documentation around version number handling for iOS and Android in os.uname and platform.release * Ensures that automated NDK installs surface messages written to stderr * Makes the Android NDK check more robust for incomplete downloads * Corrects some linting errors in Android build scripts --- Doc/library/os.rst | 4 ++-- Doc/library/platform.rst | 5 ++--- Platforms/Android/__main__.py | 5 +++-- Platforms/Android/android-env.sh | 4 ++-- .../app/src/main/java/org/python/testbed/MainActivity.kt | 9 ++++----- 5 files changed, 13 insertions(+), 14 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index be7ea26356bebe3..7bdf415db655f32 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -802,9 +802,9 @@ process and user. Returns information identifying the current operating system. The return value is a :class:`uname_result`. - On macOS, iOS and Android, this returns the *kernel* name and version (i.e., + On macOS, iOS and Android, this returns the *kernel* name and release (i.e., ``'Darwin'`` on macOS and iOS; ``'Linux'`` on Android). :func:`platform.uname` - can be used to get the user-facing operating system name and version on iOS and + can be used to get the user-facing operating system name and release on iOS and Android. .. seealso:: diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst index 1d30966794fd1bf..f728a7b32e789f4 100644 --- a/Doc/library/platform.rst +++ b/Doc/library/platform.rst @@ -141,6 +141,8 @@ Cross platform Returns the system's release, e.g. ``'2.2.0'`` or ``'NT'``. An empty string is returned if the value cannot be determined. + On iOS and Android, this is the user-facing OS release. To obtain the + Darwin or Linux kernel release, use :func:`os.uname`. .. function:: system() @@ -163,9 +165,6 @@ Cross platform Returns the system's release version, e.g. ``'#3 on degas'``. An empty string is returned if the value cannot be determined. - On iOS and Android, this is the user-facing OS version. To obtain the - Darwin or Linux kernel version, use :func:`os.uname`. - .. function:: uname() Fairly portable uname interface. Returns a :func:`~collections.namedtuple` diff --git a/Platforms/Android/__main__.py b/Platforms/Android/__main__.py index 78f94b317ab0478..705ca6f26221fda 100755 --- a/Platforms/Android/__main__.py +++ b/Platforms/Android/__main__.py @@ -158,7 +158,7 @@ def android_env(host): f"PREFIX={prefix}; " f". {ENV_SCRIPT}; " f"export", - check=True, shell=True, capture_output=True, encoding='utf-8', + check=True, shell=True, stdout=subprocess.PIPE, encoding='utf-8', ).stdout env = {} @@ -625,7 +625,8 @@ async def read_int(size): except ValueError: priority = LogPriority.UNKNOWN - payload_fields = (await read_bytes(payload_len - 1)).split(b"\0") + payload = await read_bytes(payload_len - 1) + payload_fields = payload.split(b"\0") if len(payload_fields) < 2: raise ValueError( f"payload {payload!r} does not contain at least 2 " diff --git a/Platforms/Android/android-env.sh b/Platforms/Android/android-env.sh index 5859c0eac4a88fb..59ce2eeb7d62244 100644 --- a/Platforms/Android/android-env.sh +++ b/Platforms/Android/android-env.sh @@ -7,7 +7,7 @@ : "${PREFIX:-}" # Path in which to find required libraries -# Print all messages on stderr so they're visible when running within build-wheel. +# Print all messages on stderr so they're visible when stdout is captured. log() { echo "$1" >&2 } @@ -27,7 +27,7 @@ fail() { ndk_version=27.3.13750724 ndk=$ANDROID_HOME/ndk/$ndk_version -if ! [ -e "$ndk" ]; then +if ! [ -e "$ndk/package.xml" ]; then log "Installing NDK - this may take several minutes" yes | "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" "ndk;$ndk_version" fi diff --git a/Platforms/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt b/Platforms/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt index dc49cdb9a9f7395..c8fe3acd849ac7a 100644 --- a/Platforms/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt +++ b/Platforms/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt @@ -28,12 +28,11 @@ class PythonTestRunner(val context: Context) { * @param args Python command-line, encoded as JSON. * @return The Python exit status: zero on success, nonzero on failure. */ fun run(args: String) : Int { - // We leave argument 0 as an empty string, which is a placeholder for the - // executable name in embedded mode. + // Argument 0 is a placeholder for the executable name in embedded mode. val argsJsonArray = JSONArray(args) - val argsStringArray = Array(argsJsonArray.length() + 1) { it -> ""} - for (i in 0..(argsJsonArray.length() + 1) { i -> + if (i == 0) "" + else argsJsonArray.getString(i - 1) } // Python needs this variable to help it find the temporary directory, From 22a6c51c94a4fde986b8964f1d36d5ec3ac20dcc Mon Sep 17 00:00:00 2001 From: Wenzel Jakob Date: Thu, 30 Jul 2026 14:00:08 +0200 Subject: [PATCH 12/13] gh-151728: Clear the typing caches at interpreter shutdown (GH-154858) The typing module caches every subscripted type. When an extension module leaks a reference to typing, these caches also keep types owned by other (correct) extension modules alive past interpreter shutdown, where nothing can free them anymore. The cache_clear callables are already collected in typing._cleanups, so registering them with atexit is enough to avoid this. --- Lib/test/libregrtest/utils.py | 3 +-- Lib/typing.py | 11 +++++++++++ .../2026-07-29-11-05-00.gh-issue-151728.Kq3vTn.rst | 4 ++++ 3 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-29-11-05-00.gh-issue-151728.Kq3vTn.rst diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 32f02429ff33076..892d204700179f6 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -272,8 +272,7 @@ def clear_caches(): except KeyError: pass else: - for f in typing._cleanups: - f() + typing._clear_caches() import inspect abs_classes = filter(inspect.isabstract, typing.__dict__.values()) diff --git a/Lib/typing.py b/Lib/typing.py index 054420865d7fb50..809c0ff88607a59 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -19,6 +19,7 @@ """ from abc import abstractmethod, ABCMeta +import atexit import collections from collections import defaultdict import collections.abc @@ -392,6 +393,16 @@ def _flatten_literal_params(parameters): _caches = {} +def _clear_caches(): + for cleanup in _cleanups: + cleanup() + + +# Release the LRU caches at shutdown, they otherwise redistribute reference +# leaks of one extension to types of unrelated ones. See GH-151728. +atexit.register(_clear_caches) + + def _tp_cache(func=None, /, *, typed=False): """Internal wrapper caching __getitem__ of generic types. diff --git a/Misc/NEWS.d/next/Library/2026-07-29-11-05-00.gh-issue-151728.Kq3vTn.rst b/Misc/NEWS.d/next/Library/2026-07-29-11-05-00.gh-issue-151728.Kq3vTn.rst new file mode 100644 index 000000000000000..39c63a1aea3193a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-29-11-05-00.gh-issue-151728.Kq3vTn.rst @@ -0,0 +1,4 @@ +Clear the internal :mod:`typing` caches from an exit handler. Previously, an +extension module that leaked a reference to :mod:`typing` would also keep every +subscripted type alive past interpreter shutdown, including types owned by +unrelated extension modules. From b3be16db02e71368774aab62c8ce3f6fb8cc5452 Mon Sep 17 00:00:00 2001 From: Aniket <148300120+Aniketsy@users.noreply.github.com> Date: Thu, 30 Jul 2026 18:18:38 +0530 Subject: [PATCH 13/13] gh-151941: Fix Sphinx reference warnings in `Doc/c-api/` (GH-152044) --- Doc/c-api/exceptions.rst | 15 +++++++++++++++ Doc/c-api/init_config.rst | 9 ++++----- Doc/c-api/intro.rst | 2 +- Doc/tools/.nitignore | 3 --- Tools/check-c-api-docs/ignored_c_api.txt | 2 -- 5 files changed, 20 insertions(+), 11 deletions(-) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index d04074f6e6a7993..40522f8c7b13756 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -119,6 +119,21 @@ Printing and clearing .. versionadded:: 3.12 +.. c:function:: void PyErr_Display(PyObject *unused, PyObject *value, PyObject *tb) + + Legacy variant of :c:func:`PyErr_DisplayException`. + + Print the exception *value* with its traceback to :data:`sys.stderr`. + If *value* has no traceback set, *tb* is used as its traceback. + The first argument is ignored. + + If :data:`sys.stderr` is ``None``, nothing is printed. + If :data:`sys.stderr` is not set, the exception is dumped to the + C ``stderr`` stream instead. + + .. deprecated:: 3.12 + Use :c:func:`PyErr_DisplayException` instead. + Raising exceptions ================== diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index d6b9837987a3999..ef09639189a6c27 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -1235,9 +1235,9 @@ PyConfig .. c:member:: wchar_t* base_executable - Python base executable: :data:`sys._base_executable`. + Python base executable: ``sys._base_executable``. - Set by the :envvar:`__PYVENV_LAUNCHER__` environment variable. + Set by the ``__PYVENV_LAUNCHER__`` environment variable. Set from :c:member:`PyConfig.executable` if ``NULL``. @@ -1748,7 +1748,7 @@ PyConfig * On macOS, use :envvar:`PYTHONEXECUTABLE` environment variable if set. * If the ``WITH_NEXT_FRAMEWORK`` macro is defined, use - :envvar:`__PYVENV_LAUNCHER__` environment variable if set. + ``__PYVENV_LAUNCHER__`` environment variable if set. * Use ``argv[0]`` of :c:member:`~PyConfig.argv` if available and non-empty. * Otherwise, use ``L"python"`` on Windows, or ``L"python3"`` on other @@ -1984,8 +1984,7 @@ PyConfig The :mod:`warnings` module adds :data:`sys.warnoptions` in the reverse order: the last :c:member:`PyConfig.warnoptions` item becomes the first - item of :data:`warnings.filters` which is checked first (highest - priority). + item of ``warnings.filters`` which is checked first (highest priority). The :option:`-W` command line options adds its value to :c:member:`~PyConfig.warnoptions`, it can be used multiple times. diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 4c0c9af45e8360d..701d2a31b8c9059 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -1163,7 +1163,7 @@ when defined by the compiler, will also implicitly enable :c:macro:`!Py_DEBUG`. In addition to the reference count debugging described below, extra checks are performed. See :ref:`Python Debug Build ` for more details. -Defining :c:macro:`Py_TRACE_REFS` enables reference tracing +Defining ``Py_TRACE_REFS`` enables reference tracing (see the :option:`configure --with-trace-refs option <--with-trace-refs>`). When defined, a circular doubly linked list of active objects is maintained by adding two extra fields to every :c:type:`PyObject`. Total allocations are tracked as well. Upon diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index ab592cfa5a1bbbd..6a4ab9f5aa3bf65 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -2,9 +2,6 @@ # as tested on the CI via check-warnings.py in reusable-docs.yml. # Keep lines sorted lexicographically to help avoid merge conflicts. -Doc/c-api/init_config.rst -Doc/c-api/intro.rst -Doc/c-api/stable.rst Doc/library/ast.rst Doc/library/asyncio-extending.rst Doc/library/email.charset.rst diff --git a/Tools/check-c-api-docs/ignored_c_api.txt b/Tools/check-c-api-docs/ignored_c_api.txt index aeae9e6553a3aa6..af7b2772ef5e085 100644 --- a/Tools/check-c-api-docs/ignored_c_api.txt +++ b/Tools/check-c-api-docs/ignored_c_api.txt @@ -37,8 +37,6 @@ PyWrapperFlag_KEYWORDS Py_UniversalNewlineFgets # cpython/pylifecycle.h Py_FrozenMain -# pythonrun.h -PyErr_Display # cpython/objimpl.h PyObject_GET_WEAKREFS_LISTPTR # cpython/pythonrun.h