From 069fb0cffcee780267c6151838519883f3be084d Mon Sep 17 00:00:00 2001 From: Lucca Bertoncini Date: Thu, 20 Feb 2025 20:16:12 -0800 Subject: [PATCH 01/16] fix resource tracker leak --- Lib/multiprocessing/resource_tracker.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index 90e036ae905afa..23213f76004fce 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -212,6 +212,9 @@ def _send(self, cmd, name, rtype): assert nbytes == len(msg), "nbytes {0:n} but len(msg) {1:n}".format( nbytes, len(msg)) + def __del__(self): + self._stop() + _resource_tracker = ResourceTracker() ensure_running = _resource_tracker.ensure_running From 5e338289c119c719ab07fb9e8df54933e755d6e8 Mon Sep 17 00:00:00 2001 From: Lucca Bertoncini Date: Thu, 20 Feb 2025 22:20:27 -0800 Subject: [PATCH 02/16] try block --- Lib/multiprocessing/resource_tracker.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index 23213f76004fce..38879fbb9cd5ce 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -213,7 +213,10 @@ def _send(self, cmd, name, rtype): nbytes, len(msg)) def __del__(self): - self._stop() + try: + self._stop() + except: + pass _resource_tracker = ResourceTracker() From 0586696d17279c172fc28e07ae34f7b843469e25 Mon Sep 17 00:00:00 2001 From: Lucca Bertoncini Date: Fri, 21 Feb 2025 14:38:11 -0800 Subject: [PATCH 03/16] add comment to resource tracker __del__ --- Lib/multiprocessing/resource_tracker.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index 38879fbb9cd5ce..7316cc9bc135bd 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -213,6 +213,9 @@ def _send(self, cmd, name, rtype): nbytes, len(msg)) def __del__(self): + # making sure child processess are cleaned before ResourceTracker + # gets destructed. + # see https://github.com/python/cpython/issues/88887 try: self._stop() except: From eeef2e4a9ab8d31578fc3840128f6a7668ba5a63 Mon Sep 17 00:00:00 2001 From: Lucca Bertoncini Date: Fri, 21 Feb 2025 14:48:02 -0800 Subject: [PATCH 04/16] blurb add --- .../2025-02-21-14-47-46.gh-issue-88887.V3U0CV.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-02-21-14-47-46.gh-issue-88887.V3U0CV.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-02-21-14-47-46.gh-issue-88887.V3U0CV.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-02-21-14-47-46.gh-issue-88887.V3U0CV.rst new file mode 100644 index 00000000000000..a178d045e0ed28 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-02-21-14-47-46.gh-issue-88887.V3U0CV.rst @@ -0,0 +1 @@ +Fixing multiprocessing Resource Tracker process leaking From 933f70c8199b049196742ed0507d551f9608d69c Mon Sep 17 00:00:00 2001 From: Lucca Bertoncini Date: Mon, 24 Feb 2025 12:12:07 -0800 Subject: [PATCH 05/16] catch specific exceptions --- Lib/multiprocessing/resource_tracker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index 7316cc9bc135bd..a4a84d80992c48 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -218,7 +218,7 @@ def __del__(self): # see https://github.com/python/cpython/issues/88887 try: self._stop() - except: + except (OSError, TypeError, AttributeError) as e: pass From 799ee32d79eabcc9597480cc54edfadd0026fd8e Mon Sep 17 00:00:00 2001 From: luccabb <32229669+luccabb@users.noreply.github.com> Date: Mon, 24 Feb 2025 12:58:14 -0800 Subject: [PATCH 06/16] Update Lib/multiprocessing/resource_tracker.py Co-authored-by: Victor Stinner --- Lib/multiprocessing/resource_tracker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index a4a84d80992c48..1c7c15a73da566 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -218,7 +218,7 @@ def __del__(self): # see https://github.com/python/cpython/issues/88887 try: self._stop() - except (OSError, TypeError, AttributeError) as e: + except (OSError, TypeError, AttributeError): pass From 8e43a8393305b49915f150f1593273fa220c6e15 Mon Sep 17 00:00:00 2001 From: Lucca Bertoncini Date: Thu, 6 Mar 2025 11:53:15 -0800 Subject: [PATCH 07/16] fixing _stop for None pids --- Lib/multiprocessing/resource_tracker.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index a4a84d80992c48..946405f48ec555 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -75,8 +75,16 @@ def _reentrant_call_error(self): raise ReentrantCallError( "Reentrant call into the multiprocessing resource tracker") + def __del__(self): + # making sure child processess are cleaned before ResourceTracker + # gets destructed. + # see https://github.com/python/cpython/issues/88887 + self._stop() + def _stop(self): with self._lock: + if self._pid is None: + return # This should not happen (_stop() isn't called by a finalizer) # but we check for it anyway. if self._lock._recursion_count() > 1: @@ -191,7 +199,7 @@ def unregister(self, name, rtype): '''Unregister name of resource with resource tracker.''' self._send('UNREGISTER', name, rtype) - def _send(self, cmd, name, rtype): + def _send(self, cmd, name, rtype)::77 try: self.ensure_running() except ReentrantCallError: @@ -212,16 +220,6 @@ def _send(self, cmd, name, rtype): assert nbytes == len(msg), "nbytes {0:n} but len(msg) {1:n}".format( nbytes, len(msg)) - def __del__(self): - # making sure child processess are cleaned before ResourceTracker - # gets destructed. - # see https://github.com/python/cpython/issues/88887 - try: - self._stop() - except (OSError, TypeError, AttributeError) as e: - pass - - _resource_tracker = ResourceTracker() ensure_running = _resource_tracker.ensure_running register = _resource_tracker.register From 805ad1d76557cc24142caef94e3f9fbec38db67e Mon Sep 17 00:00:00 2001 From: Lucca Bertoncini Date: Thu, 6 Mar 2025 12:01:47 -0800 Subject: [PATCH 08/16] remove :77 typo --- Lib/multiprocessing/resource_tracker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index 946405f48ec555..91b30cfd998573 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -199,7 +199,7 @@ def unregister(self, name, rtype): '''Unregister name of resource with resource tracker.''' self._send('UNREGISTER', name, rtype) - def _send(self, cmd, name, rtype)::77 + def _send(self, cmd, name, rtype): try: self.ensure_running() except ReentrantCallError: From 5d6c9b3dabf74adfdcfd884ef5f2faa72dade207 Mon Sep 17 00:00:00 2001 From: Lucca Bertoncini Date: Thu, 6 Mar 2025 12:02:46 -0800 Subject: [PATCH 09/16] add extra line back --- Lib/multiprocessing/resource_tracker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index 91b30cfd998573..d6bf49527a6290 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -220,6 +220,7 @@ def _send(self, cmd, name, rtype): assert nbytes == len(msg), "nbytes {0:n} but len(msg) {1:n}".format( nbytes, len(msg)) + _resource_tracker = ResourceTracker() ensure_running = _resource_tracker.ensure_running register = _resource_tracker.register From c8e2704c2572f6e52cb04769b20b476321a9daa3 Mon Sep 17 00:00:00 2001 From: Lucca Bertoncini Date: Thu, 6 Mar 2025 15:42:01 -0800 Subject: [PATCH 10/16] catch AttributeError only and return early from _stop if _pid is None --- Lib/multiprocessing/resource_tracker.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index d6bf49527a6290..b4574bf897a9a2 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -79,12 +79,18 @@ def __del__(self): # making sure child processess are cleaned before ResourceTracker # gets destructed. # see https://github.com/python/cpython/issues/88887 - self._stop() + try: + self._stop() + except AttributeError: + # AttributeError is likely caused by module teardown + # > __del__() can be executed during interpreter shutdown. As a + # > consequence, the global variables it needs to access (including + # > other modules) may already have been deleted or set to None. + # see https://docs.python.org/3/reference/datamodel.html#object.__del__ + pass def _stop(self): with self._lock: - if self._pid is None: - return # This should not happen (_stop() isn't called by a finalizer) # but we check for it anyway. if self._lock._recursion_count() > 1: @@ -92,6 +98,8 @@ def _stop(self): if self._fd is None: # not running return + if self._pid is None: + return # closing the "alive" file descriptor stops main() os.close(self._fd) From 9fd5872c8f09718fe0c6171fd5d70b207982732e Mon Sep 17 00:00:00 2001 From: Lucca Bertoncini Date: Thu, 6 Mar 2025 15:47:50 -0800 Subject: [PATCH 11/16] fix lint --- Lib/multiprocessing/resource_tracker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index b4574bf897a9a2..eb5c2ea997b575 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -83,8 +83,8 @@ def __del__(self): self._stop() except AttributeError: # AttributeError is likely caused by module teardown - # > __del__() can be executed during interpreter shutdown. As a - # > consequence, the global variables it needs to access (including + # > __del__() can be executed during interpreter shutdown. As a + # > consequence, the global variables it needs to access (including # > other modules) may already have been deleted or set to None. # see https://docs.python.org/3/reference/datamodel.html#object.__del__ pass From aaa8378f4da2c5a6059e761e6097adec67e66c68 Mon Sep 17 00:00:00 2001 From: Lucca Bertoncini Date: Fri, 7 Mar 2025 10:55:03 -0800 Subject: [PATCH 12/16] injecting os methods on _stop --- Lib/multiprocessing/resource_tracker.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index eb5c2ea997b575..feacf7a329269a 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -79,17 +79,9 @@ def __del__(self): # making sure child processess are cleaned before ResourceTracker # gets destructed. # see https://github.com/python/cpython/issues/88887 - try: - self._stop() - except AttributeError: - # AttributeError is likely caused by module teardown - # > __del__() can be executed during interpreter shutdown. As a - # > consequence, the global variables it needs to access (including - # > other modules) may already have been deleted or set to None. - # see https://docs.python.org/3/reference/datamodel.html#object.__del__ - pass + self._stop() - def _stop(self): + def _stop(self, close=os.close, waitpid=os.waitpid, waitstatus_to_exitcode=os.waitstatus_to_exitcode): with self._lock: # This should not happen (_stop() isn't called by a finalizer) # but we check for it anyway. @@ -102,15 +94,15 @@ def _stop(self): return # closing the "alive" file descriptor stops main() - os.close(self._fd) + close(self._fd) self._fd = None - _, status = os.waitpid(self._pid, 0) + _, status = waitpid(self._pid, 0) self._pid = None try: - self._exitcode = os.waitstatus_to_exitcode(status) + self._exitcode = waitstatus_to_exitcode(status) except ValueError: # os.waitstatus_to_exitcode may raise an exception for invalid values self._exitcode = None From dc2d578508d320496c7c26e459e6f06e39cd51df Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Fri, 7 Mar 2025 13:29:02 -0800 Subject: [PATCH 13/16] Update Lib/multiprocessing/resource_tracker.py comment --- Lib/multiprocessing/resource_tracker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index feacf7a329269a..125a6190d2be16 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -83,8 +83,8 @@ def __del__(self): def _stop(self, close=os.close, waitpid=os.waitpid, waitstatus_to_exitcode=os.waitstatus_to_exitcode): with self._lock: - # This should not happen (_stop() isn't called by a finalizer) - # but we check for it anyway. + # This shouldn't happen (it might when called by a finalizer) + # so we check for it anyway. if self._lock._recursion_count() > 1: return self._reentrant_call_error() if self._fd is None: From 98473429b989c347ff7b7ee840d90b00274941dd Mon Sep 17 00:00:00 2001 From: Lucca Bertoncini Date: Sat, 8 Mar 2025 22:53:11 -0800 Subject: [PATCH 14/16] non blocking acquire for __del__ --- Lib/multiprocessing/resource_tracker.py | 54 ++++++++++++++----------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index feacf7a329269a..3b635d58ccb546 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -79,33 +79,41 @@ def __del__(self): # making sure child processess are cleaned before ResourceTracker # gets destructed. # see https://github.com/python/cpython/issues/88887 - self._stop() + self._stop(use_blocking_lock=False) - def _stop(self, close=os.close, waitpid=os.waitpid, waitstatus_to_exitcode=os.waitstatus_to_exitcode): - with self._lock: - # This should not happen (_stop() isn't called by a finalizer) - # but we check for it anyway. - if self._lock._recursion_count() > 1: - return self._reentrant_call_error() - if self._fd is None: - # not running - return - if self._pid is None: - return - - # closing the "alive" file descriptor stops main() - close(self._fd) - self._fd = None + def _stop(self, use_blocking_lock=True): + if use_blocking_lock: + with self._lock: + self._cleanup() + else: + self._lock.acquire(blocking=False) + self._cleanup() + self._lock.release() + + def _cleanup(self, close=os.close, waitpid=os.waitpid, waitstatus_to_exitcode=os.waitstatus_to_exitcode): + # This should not happen (_stop() isn't called by a finalizer) + # but we check for it anyway. + if self._lock._recursion_count() > 1: + return self._reentrant_call_error() + if self._fd is None: + # not running + return + if self._pid is None: + return + + # closing the "alive" file descriptor stops main() + close(self._fd) + self._fd = None - _, status = waitpid(self._pid, 0) + _, status = waitpid(self._pid, 0) - self._pid = None + self._pid = None - try: - self._exitcode = waitstatus_to_exitcode(status) - except ValueError: - # os.waitstatus_to_exitcode may raise an exception for invalid values - self._exitcode = None + try: + self._exitcode = waitstatus_to_exitcode(status) + except ValueError: + # os.waitstatus_to_exitcode may raise an exception for invalid values + self._exitcode = None def getfd(self): self.ensure_running() From d497deecf19f5c56ed629eec0bbf21534449674d Mon Sep 17 00:00:00 2001 From: Lucca Bertoncini Date: Mon, 10 Mar 2025 09:57:40 -0700 Subject: [PATCH 15/16] release only if acquired --- Lib/multiprocessing/resource_tracker.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index 8c6dd0052ad742..54f95f5cc0bbdd 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -84,13 +84,19 @@ def __del__(self): def _stop(self, use_blocking_lock=True): if use_blocking_lock: with self._lock: - self._cleanup() + self._stop_unlocked() else: - self._lock.acquire(blocking=False) - self._cleanup() - self._lock.release() - - def _cleanup(self, close=os.close, waitpid=os.waitpid, waitstatus_to_exitcode=os.waitstatus_to_exitcode): + acquired = self._lock.acquire(blocking=False) + self._stop_unlocked() + if acquired: + self._lock.release() + + def _stop_unlocked( + self, + close=os.close, + waitpid=os.waitpid, + waitstatus_to_exitcode=os.waitstatus_to_exitcode, + ): # This shouldn't happen (it might when called by a finalizer) # so we check for it anyway. if self._lock._recursion_count() > 1: From ba424061c93cb318964f1c4420a8fc720b5329de Mon Sep 17 00:00:00 2001 From: Lucca Bertoncini Date: Wed, 19 Mar 2025 11:29:44 -0700 Subject: [PATCH 16/16] always release + changelog --- Lib/multiprocessing/resource_tracker.py | 12 +++++++----- .../2025-02-21-14-47-46.gh-issue-88887.V3U0CV.rst | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index 54f95f5cc0bbdd..05633ac21a259c 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -84,14 +84,16 @@ def __del__(self): def _stop(self, use_blocking_lock=True): if use_blocking_lock: with self._lock: - self._stop_unlocked() + self._stop_locked() else: acquired = self._lock.acquire(blocking=False) - self._stop_unlocked() - if acquired: - self._lock.release() + try: + self._stop_locked() + finally: + if acquired: + self._lock.release() - def _stop_unlocked( + def _stop_locked( self, close=os.close, waitpid=os.waitpid, diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-02-21-14-47-46.gh-issue-88887.V3U0CV.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-02-21-14-47-46.gh-issue-88887.V3U0CV.rst index a178d045e0ed28..1a6c9483e2a35e 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-02-21-14-47-46.gh-issue-88887.V3U0CV.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-02-21-14-47-46.gh-issue-88887.V3U0CV.rst @@ -1 +1 @@ -Fixing multiprocessing Resource Tracker process leaking +Fixing multiprocessing Resource Tracker process leaking, usually observed when running Python as PID 1.