From f90cd4f01c5ff8ff9c3cd78246ac9f1945ccd1e6 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Fri, 29 Dec 2023 17:46:13 +0100 Subject: [PATCH 1/5] gh-101225: Fix hang when passing Pipe instances to child in multiprocessing This PR uses the same backlog value when creating a `.connection.Client` in `.resource_sharer` as is used in `.manager`. On macOS the default backlog (1) is small enough to cause the socket accept queue to fill up when starting a number of children. --- Lib/multiprocessing/resource_sharer.py | 2 +- .../next/Library/2023-12-29-17-46-06.gh-issue-101225.QaEyxF.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-12-29-17-46-06.gh-issue-101225.QaEyxF.rst diff --git a/Lib/multiprocessing/resource_sharer.py b/Lib/multiprocessing/resource_sharer.py index 66076509a1202e..6c81ce3c0ca86b 100644 --- a/Lib/multiprocessing/resource_sharer.py +++ b/Lib/multiprocessing/resource_sharer.py @@ -123,7 +123,7 @@ def _start(self): from .connection import Listener assert self._listener is None, "Already have Listener" util.debug('starting listener and thread for sending handles') - self._listener = Listener(authkey=process.current_process().authkey) + self._listener = Listener(authkey=process.current_process().authkey, backlog=16) self._address = self._listener.address t = threading.Thread(target=self._serve) t.daemon = True diff --git a/Misc/NEWS.d/next/Library/2023-12-29-17-46-06.gh-issue-101225.QaEyxF.rst b/Misc/NEWS.d/next/Library/2023-12-29-17-46-06.gh-issue-101225.QaEyxF.rst new file mode 100644 index 00000000000000..64329cd2ceac80 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-29-17-46-06.gh-issue-101225.QaEyxF.rst @@ -0,0 +1,2 @@ +Fix a hang in :mod:`multiprocessing` when passing ``Pipe`` instances to +workers on macOS. From 9a5df53c2d224cbe061bf3fb7d12a0b94f1e44a7 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Sat, 30 Dec 2023 11:45:27 +0100 Subject: [PATCH 2/5] Increase backlog to 128 --- Lib/multiprocessing/managers.py | 2 +- Lib/multiprocessing/resource_sharer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 96cebc6eabec89..76b915de74d94e 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -156,7 +156,7 @@ def __init__(self, registry, address, authkey, serializer): Listener, Client = listener_client[serializer] # do authentication later - self.listener = Listener(address=address, backlog=16) + self.listener = Listener(address=address, backlog=128) self.address = self.listener.address self.id_to_obj = {'0': (None, ())} diff --git a/Lib/multiprocessing/resource_sharer.py b/Lib/multiprocessing/resource_sharer.py index 6c81ce3c0ca86b..b8afb0fbed3a3c 100644 --- a/Lib/multiprocessing/resource_sharer.py +++ b/Lib/multiprocessing/resource_sharer.py @@ -123,7 +123,7 @@ def _start(self): from .connection import Listener assert self._listener is None, "Already have Listener" util.debug('starting listener and thread for sending handles') - self._listener = Listener(authkey=process.current_process().authkey, backlog=16) + self._listener = Listener(authkey=process.current_process().authkey, backlog=128) self._address = self._listener.address t = threading.Thread(target=self._serve) t.daemon = True From ced11b124c28835dbbc5fc7e6ad5cb1992d6ebd4 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Sun, 7 Jan 2024 20:02:26 +0100 Subject: [PATCH 3/5] More elaborate explaination of the change in the news entry. --- .../Library/2023-12-29-17-46-06.gh-issue-101225.QaEyxF.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2023-12-29-17-46-06.gh-issue-101225.QaEyxF.rst b/Misc/NEWS.d/next/Library/2023-12-29-17-46-06.gh-issue-101225.QaEyxF.rst index 64329cd2ceac80..a8f418807092d2 100644 --- a/Misc/NEWS.d/next/Library/2023-12-29-17-46-06.gh-issue-101225.QaEyxF.rst +++ b/Misc/NEWS.d/next/Library/2023-12-29-17-46-06.gh-issue-101225.QaEyxF.rst @@ -1,2 +1,4 @@ -Fix a hang in :mod:`multiprocessing` when passing ``Pipe`` instances to -workers on macOS. +Increase the backlog for :class:`multiprocessing.connection.Listener` objects created +by :mod:`multiprocessing.manaager` and :mod:`multiprocessing.resource_sharer` to +significantly reduce the risk of getting a connection refused error when creating +a :class:`multiprocessing.connection.Connection` to them. From 7ed4542403eded8f7ee8ebbce28f148f259ba4d3 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Sun, 7 Jan 2024 20:14:45 +0100 Subject: [PATCH 4/5] As usual, fix whitespace at end of line --- .../next/Library/2023-12-29-17-46-06.gh-issue-101225.QaEyxF.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2023-12-29-17-46-06.gh-issue-101225.QaEyxF.rst b/Misc/NEWS.d/next/Library/2023-12-29-17-46-06.gh-issue-101225.QaEyxF.rst index a8f418807092d2..01bfa1c1ec7d20 100644 --- a/Misc/NEWS.d/next/Library/2023-12-29-17-46-06.gh-issue-101225.QaEyxF.rst +++ b/Misc/NEWS.d/next/Library/2023-12-29-17-46-06.gh-issue-101225.QaEyxF.rst @@ -1,4 +1,4 @@ Increase the backlog for :class:`multiprocessing.connection.Listener` objects created -by :mod:`multiprocessing.manaager` and :mod:`multiprocessing.resource_sharer` to +by :mod:`multiprocessing.manaager` and :mod:`multiprocessing.resource_sharer` to significantly reduce the risk of getting a connection refused error when creating a :class:`multiprocessing.connection.Connection` to them. From 7b515735dd6e1b0495ddef5ed178a5ddca2aef31 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Sat, 13 Jan 2024 10:30:16 +0100 Subject: [PATCH 5/5] Update Misc/NEWS.d/next/Library/2023-12-29-17-46-06.gh-issue-101225.QaEyxF.rst --- .../next/Library/2023-12-29-17-46-06.gh-issue-101225.QaEyxF.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2023-12-29-17-46-06.gh-issue-101225.QaEyxF.rst b/Misc/NEWS.d/next/Library/2023-12-29-17-46-06.gh-issue-101225.QaEyxF.rst index 01bfa1c1ec7d20..ab3c3a5ef23f45 100644 --- a/Misc/NEWS.d/next/Library/2023-12-29-17-46-06.gh-issue-101225.QaEyxF.rst +++ b/Misc/NEWS.d/next/Library/2023-12-29-17-46-06.gh-issue-101225.QaEyxF.rst @@ -1,4 +1,4 @@ Increase the backlog for :class:`multiprocessing.connection.Listener` objects created -by :mod:`multiprocessing.manaager` and :mod:`multiprocessing.resource_sharer` to +by :mod:`multiprocessing.manager` and :mod:`multiprocessing.resource_sharer` to significantly reduce the risk of getting a connection refused error when creating a :class:`multiprocessing.connection.Connection` to them.