From b1b8962443e7d418601658a4b05347a5a9161910 Mon Sep 17 00:00:00 2001 From: "Yuichiro Tachibana (Tsuchiya)" Date: Wed, 21 May 2025 19:18:00 -0500 Subject: [PATCH 1/4] gh-127960 Fix the REPL to set the correct namespace by setting the correct `__main__` module (gh-134275) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `__main__` module imported in the `_pyrepl` module points to the `_pyrepl` module itself when the interpreter was launched without `-m` option and didn't execute a module, while it's an unexpected behavior that `__main__` can be `_pyrepl` and relative imports such as `from . import *` works based on the `_pyrepl` module. Co-authored-by: Łukasz Langa --- Lib/_pyrepl/_module_completer.py | 4 +- Lib/_pyrepl/main.py | 11 ++- Lib/_pyrepl/readline.py | 1 + Lib/test/support/__init__.py | 6 -- Lib/test/test_pyrepl/test_pyrepl.py | 78 +++++++++++++++---- ...-05-21-18-02-56.gh-issue-127960.W3J_2X.rst | 3 + Modules/main.c | 16 +++- 7 files changed, 85 insertions(+), 34 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-21-18-02-56.gh-issue-127960.W3J_2X.rst diff --git a/Lib/_pyrepl/_module_completer.py b/Lib/_pyrepl/_module_completer.py index 9aafb55090e2ce..494a501101a9b2 100644 --- a/Lib/_pyrepl/_module_completer.py +++ b/Lib/_pyrepl/_module_completer.py @@ -17,8 +17,8 @@ def make_default_module_completer() -> ModuleCompleter: - # Inside pyrepl, __package__ is set to '_pyrepl' - return ModuleCompleter(namespace={'__package__': '_pyrepl'}) + # Inside pyrepl, __package__ is set to None by default + return ModuleCompleter(namespace={'__package__': None}) class ModuleCompleter: diff --git a/Lib/_pyrepl/main.py b/Lib/_pyrepl/main.py index a6f824dcc4ad14..447eb1e551e774 100644 --- a/Lib/_pyrepl/main.py +++ b/Lib/_pyrepl/main.py @@ -1,6 +1,7 @@ import errno import os import sys +import types CAN_USE_PYREPL: bool @@ -29,12 +30,10 @@ def interactive_console(mainmodule=None, quiet=False, pythonstartup=False): print(FAIL_REASON, file=sys.stderr) return sys._baserepl() - if mainmodule: - namespace = mainmodule.__dict__ - else: - import __main__ - namespace = __main__.__dict__ - namespace.pop("__pyrepl_interactive_console", None) + if not mainmodule: + mainmodule = types.ModuleType("__main__") + + namespace = mainmodule.__dict__ # sys._baserepl() above does this internally, we do it here startup_path = os.getenv("PYTHONSTARTUP") diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 560a9db192169e..572eee520e53f3 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -606,6 +606,7 @@ def _setup(namespace: Mapping[str, Any]) -> None: # set up namespace in rlcompleter, which requires it to be a bona fide dict if not isinstance(namespace, dict): namespace = dict(namespace) + _wrapper.config.module_completer = ModuleCompleter(namespace) _wrapper.config.readline_completer = RLCompleter(namespace).complete # this is not really what readline.c does. Better than nothing I guess diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 9b6e80fdad9747..b7cd7940eb15b3 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2929,12 +2929,6 @@ def make_clean_env() -> dict[str, str]: return clean_env -def initialized_with_pyrepl(): - """Detect whether PyREPL was used during Python initialization.""" - # If the main module has a __file__ attribute it's a Python module, which means PyREPL. - return hasattr(sys.modules["__main__"], "__file__") - - WINDOWS_STATUS = { 0xC0000005: "STATUS_ACCESS_VIOLATION", 0xC00000FD: "STATUS_STACK_OVERFLOW", diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index 29762232d43b89..abb4bd1bc25fb1 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -926,6 +926,7 @@ def tearDown(self): def prepare_reader(self, events, namespace): console = FakeConsole(events) config = ReadlineConfig() + config.module_completer = ModuleCompleter(namespace) config.readline_completer = rlcompleter.Completer(namespace).complete reader = ReadlineAlikeReader(console=console, config=config) return reader @@ -1022,13 +1023,15 @@ def test_builtin_completion_top_level(self): def test_relative_import_completions(self): cases = ( - ("from .readl\t\n", "from .readline"), - ("from . import readl\t\n", "from . import readline"), + (None, "from .readl\t\n", "from .readl"), + (None, "from . import readl\t\n", "from . import readl"), + ("_pyrepl", "from .readl\t\n", "from .readline"), + ("_pyrepl", "from . import readl\t\n", "from . import readline"), ) - for code, expected in cases: + for package, code, expected in cases: with self.subTest(code=code): events = code_to_events(code) - reader = self.prepare_reader(events, namespace={}) + reader = self.prepare_reader(events, namespace={"__package__": package}) output = reader.readline() self.assertEqual(output, expected) @@ -1397,7 +1400,7 @@ def _assertMatchOK( ) @force_not_colorized - def _run_repl_globals_test(self, expectations, *, as_file=False, as_module=False): + def _run_repl_globals_test(self, expectations, *, as_file=False, as_module=False, pythonstartup=False): clean_env = make_clean_env() clean_env["NO_COLOR"] = "1" # force_not_colorized doesn't touch subprocesses @@ -1406,9 +1409,13 @@ def _run_repl_globals_test(self, expectations, *, as_file=False, as_module=False blue.mkdir() mod = blue / "calx.py" mod.write_text("FOO = 42", encoding="utf-8") + startup = blue / "startup.py" + startup.write_text("BAR = 64", encoding="utf-8") commands = [ "print(f'^{" + var + "=}')" for var in expectations ] + ["exit()"] + if pythonstartup: + clean_env["PYTHONSTARTUP"] = str(startup) if as_file and as_module: self.fail("as_file and as_module are mutually exclusive") elif as_file: @@ -1427,7 +1434,13 @@ def _run_repl_globals_test(self, expectations, *, as_file=False, as_module=False skip=True, ) else: - self.fail("Choose one of as_file or as_module") + output, exit_code = self.run_repl( + commands, + cmdline_args=[], + env=clean_env, + cwd=td, + skip=True, + ) self.assertEqual(exit_code, 0) for var, expected in expectations.items(): @@ -1440,6 +1453,23 @@ def _run_repl_globals_test(self, expectations, *, as_file=False, as_module=False self.assertNotIn("Exception", output) self.assertNotIn("Traceback", output) + def test_globals_initialized_as_default(self): + expectations = { + "__name__": "'__main__'", + "__package__": "None", + # "__file__" is missing in -i, like in the basic REPL + } + self._run_repl_globals_test(expectations) + + def test_globals_initialized_from_pythonstartup(self): + expectations = { + "BAR": "64", + "__name__": "'__main__'", + "__package__": "None", + # "__file__" is missing in -i, like in the basic REPL + } + self._run_repl_globals_test(expectations, pythonstartup=True) + def test_inspect_keeps_globals_from_inspected_file(self): expectations = { "FOO": "42", @@ -1449,6 +1479,16 @@ def test_inspect_keeps_globals_from_inspected_file(self): } self._run_repl_globals_test(expectations, as_file=True) + def test_inspect_keeps_globals_from_inspected_file_with_pythonstartup(self): + expectations = { + "FOO": "42", + "BAR": "64", + "__name__": "'__main__'", + "__package__": "None", + # "__file__" is missing in -i, like in the basic REPL + } + self._run_repl_globals_test(expectations, as_file=True, pythonstartup=True) + def test_inspect_keeps_globals_from_inspected_module(self): expectations = { "FOO": "42", @@ -1458,26 +1498,32 @@ def test_inspect_keeps_globals_from_inspected_module(self): } self._run_repl_globals_test(expectations, as_module=True) + def test_inspect_keeps_globals_from_inspected_module_with_pythonstartup(self): + expectations = { + "FOO": "42", + "BAR": "64", + "__name__": "'__main__'", + "__package__": "'blue'", + "__file__": re.compile(r"^'.*calx.py'$"), + } + self._run_repl_globals_test(expectations, as_module=True, pythonstartup=True) + @force_not_colorized def test_python_basic_repl(self): env = os.environ.copy() - commands = ("from test.support import initialized_with_pyrepl\n" - "initialized_with_pyrepl()\n" - "exit()\n") - + pyrepl_commands = "clear\nexit()\n" env.pop("PYTHON_BASIC_REPL", None) - output, exit_code = self.run_repl(commands, env=env, skip=True) + output, exit_code = self.run_repl(pyrepl_commands, env=env, skip=True) self.assertEqual(exit_code, 0) - self.assertIn("True", output) - self.assertNotIn("False", output) self.assertNotIn("Exception", output) + self.assertNotIn("NameError", output) self.assertNotIn("Traceback", output) + basic_commands = "help\nexit()\n" env["PYTHON_BASIC_REPL"] = "1" - output, exit_code = self.run_repl(commands, env=env) + output, exit_code = self.run_repl(basic_commands, env=env) self.assertEqual(exit_code, 0) - self.assertIn("False", output) - self.assertNotIn("True", output) + self.assertIn("Type help() for interactive help", output) self.assertNotIn("Exception", output) self.assertNotIn("Traceback", output) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-21-18-02-56.gh-issue-127960.W3J_2X.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-21-18-02-56.gh-issue-127960.W3J_2X.rst new file mode 100644 index 00000000000000..730d8a5af51f54 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-21-18-02-56.gh-issue-127960.W3J_2X.rst @@ -0,0 +1,3 @@ +PyREPL interactive shell no longer starts with ``__package__`` and +``__file__`` global names set to ``_pyrepl`` package internals. Contributed +by Yuichiro Tachibana. diff --git a/Modules/main.c b/Modules/main.c index 2be194bdadf7d0..2d7ed25f5f9790 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -269,13 +269,14 @@ pymain_run_command(wchar_t *command) static int -pymain_start_pyrepl_no_main(void) +pymain_start_pyrepl(int pythonstartup) { int res = 0; PyObject *console = NULL; PyObject *empty_tuple = NULL; PyObject *kwargs = NULL; PyObject *console_result = NULL; + PyObject *main_module = NULL; PyObject *pyrepl = PyImport_ImportModule("_pyrepl.main"); if (pyrepl == NULL) { @@ -299,7 +300,13 @@ pymain_start_pyrepl_no_main(void) res = pymain_exit_err_print(); goto done; } - if (!PyDict_SetItemString(kwargs, "pythonstartup", _PyLong_GetOne())) { + main_module = PyImport_AddModuleRef("__main__"); + if (main_module == NULL) { + res = pymain_exit_err_print(); + goto done; + } + if (!PyDict_SetItemString(kwargs, "mainmodule", main_module) + && !PyDict_SetItemString(kwargs, "pythonstartup", pythonstartup ? Py_True : Py_False)) { console_result = PyObject_Call(console, empty_tuple, kwargs); if (console_result == NULL) { res = pymain_exit_err_print(); @@ -311,6 +318,7 @@ pymain_start_pyrepl_no_main(void) Py_XDECREF(empty_tuple); Py_XDECREF(console); Py_XDECREF(pyrepl); + Py_XDECREF(main_module); return res; } @@ -562,7 +570,7 @@ pymain_run_stdin(PyConfig *config) int run = PyRun_AnyFileExFlags(stdin, "", 0, &cf); return (run != 0); } - return pymain_run_module(L"_pyrepl", 0); + return pymain_start_pyrepl(0); } @@ -595,7 +603,7 @@ pymain_repl(PyConfig *config, int *exitcode) *exitcode = (run != 0); return; } - int run = pymain_start_pyrepl_no_main(); + int run = pymain_start_pyrepl(1); *exitcode = (run != 0); return; } From 518c95b5529ed3379b5a3065b09f71411efe72fb Mon Sep 17 00:00:00 2001 From: Marcin Bachry Date: Thu, 22 May 2025 04:38:01 +0200 Subject: [PATCH 2/4] gh-127840: pass flags and address from send_fds (GH-127841) socket: pass flags and address from send_fds Co-authored-by: Peter Bierma --- Lib/socket.py | 2 +- Lib/test/test_socket.py | 24 +++++++++++++++++++ ...-12-11-20-15-00.gh-issue-127840.pt8fiQ.rst | 1 + 3 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-12-11-20-15-00.gh-issue-127840.pt8fiQ.rst diff --git a/Lib/socket.py b/Lib/socket.py index 727b0e75f03595..2cbaf221a59dea 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -563,7 +563,7 @@ def send_fds(sock, buffers, fds, flags=0, address=None): import array return sock.sendmsg(buffers, [(_socket.SOL_SOCKET, - _socket.SCM_RIGHTS, array.array("i", fds))]) + _socket.SCM_RIGHTS, array.array("i", fds))], flags, address) __all__.append("send_fds") if hasattr(_socket.socket, "recvmsg"): diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 03c54151a2218f..04dfb682ec6831 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -7366,6 +7366,30 @@ def close_fds(fds): data = os.read(rfd, 100) self.assertEqual(data, str(index).encode()) + def testSendAndRecvFdsByAddress(self): + rfd, wfd = os.pipe() + self.addCleanup(os.close, rfd) + self.addCleanup(os.close, wfd) + + sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + address = socket_helper.create_unix_domain_name() + self.addCleanup(os_helper.unlink, address) + socket_helper.bind_unix_socket(sock, address) + + socket.send_fds(sock, [MSG], [rfd], 0, address) + + # request more data and file descriptors than expected + msg, (rfd2,), flags, addr = socket.recv_fds(sock, len(MSG) * 2, 2) + self.addCleanup(os.close, rfd2) + self.assertEqual(msg, MSG) + self.assertEqual(flags, 0) + self.assertEqual(addr, address) + + # test that the file descriptor is connected + os.write(wfd, b'data') + data = os.read(rfd2, 100) + self.assertEqual(data, b'data') + class FreeThreadingTests(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2024-12-11-20-15-00.gh-issue-127840.pt8fiQ.rst b/Misc/NEWS.d/next/Library/2024-12-11-20-15-00.gh-issue-127840.pt8fiQ.rst new file mode 100644 index 00000000000000..51eaaa96ebaa07 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-11-20-15-00.gh-issue-127840.pt8fiQ.rst @@ -0,0 +1 @@ +Fix :func:`socket.send_fds` ignoring flags and address parameters. From f3fc0c16e08b317cb201cf1073e934e6909f1251 Mon Sep 17 00:00:00 2001 From: Mike Salvatore Date: Wed, 21 May 2025 22:48:10 -0400 Subject: [PATCH 3/4] gh-134062: Fix hash collisions in IPv4Network and IPv6Network (GH-134063) gh-134062: Fix hash collisions in IPv4Network and IPv6Network gh-134062: Add hash collision regression test --- Lib/ipaddress.py | 2 +- Lib/test/test_ipaddress.py | 28 +++++++++++++++++++ ...-05-15-14-27-01.gh-issue-134062.fRbJet.rst | 3 ++ 3 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-15-14-27-01.gh-issue-134062.fRbJet.rst diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index 703fa289dda1fb..d8a84f33264dc5 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -729,7 +729,7 @@ def __eq__(self, other): return NotImplemented def __hash__(self): - return hash(int(self.network_address) ^ int(self.netmask)) + return hash((int(self.network_address), int(self.netmask))) def __contains__(self, other): # always false if one is v4 and the other is v6. diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index d04012d1afd540..a06608d0016253 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -2762,6 +2762,34 @@ def testV6HashIsNotConstant(self): ipv6_address2 = ipaddress.IPv6Interface("2001:658:22a:cafe:200:0:0:2") self.assertNotEqual(ipv6_address1.__hash__(), ipv6_address2.__hash__()) + # issue 134062 Hash collisions in IPv4Network and IPv6Network + def testNetworkV4HashCollisions(self): + self.assertNotEqual( + ipaddress.IPv4Network("192.168.1.255/32").__hash__(), + ipaddress.IPv4Network("192.168.1.0/24").__hash__() + ) + self.assertNotEqual( + ipaddress.IPv4Network("172.24.255.0/24").__hash__(), + ipaddress.IPv4Network("172.24.0.0/16").__hash__() + ) + self.assertNotEqual( + ipaddress.IPv4Network("192.168.1.87/32").__hash__(), + ipaddress.IPv4Network("192.168.1.86/31").__hash__() + ) + + # issue 134062 Hash collisions in IPv4Network and IPv6Network + def testNetworkV6HashCollisions(self): + self.assertNotEqual( + ipaddress.IPv6Network("fe80::/64").__hash__(), + ipaddress.IPv6Network("fe80::ffff:ffff:ffff:0/112").__hash__() + ) + self.assertNotEqual( + ipaddress.IPv4Network("10.0.0.0/8").__hash__(), + ipaddress.IPv6Network( + "ffff:ffff:ffff:ffff:ffff:ffff:aff:0/112" + ).__hash__() + ) + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2025-05-15-14-27-01.gh-issue-134062.fRbJet.rst b/Misc/NEWS.d/next/Library/2025-05-15-14-27-01.gh-issue-134062.fRbJet.rst new file mode 100644 index 00000000000000..f62a3ec480193d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-15-14-27-01.gh-issue-134062.fRbJet.rst @@ -0,0 +1,3 @@ +:mod:`ipaddress`: fix collisions in :meth:`~object.__hash__` for +:class:`~ipaddress.IPv4Network` and :class:`~ipaddress.IPv6Network` +objects. From 296a66051ede5cc112ca38d17304e518ffb02e23 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Wed, 21 May 2025 23:38:44 -0400 Subject: [PATCH 4/4] gh-127840: Revert "gh-127840: pass flags and address from send_fds (GH-127841)" (#134482) Revert "gh-127840: pass flags and address from send_fds (GH-127841)" This reverts commit 518c95b5529ed3379b5a3065b09f71411efe72fb. --- Lib/socket.py | 2 +- Lib/test/test_socket.py | 24 ------------------- ...-12-11-20-15-00.gh-issue-127840.pt8fiQ.rst | 1 - 3 files changed, 1 insertion(+), 26 deletions(-) delete mode 100644 Misc/NEWS.d/next/Library/2024-12-11-20-15-00.gh-issue-127840.pt8fiQ.rst diff --git a/Lib/socket.py b/Lib/socket.py index 2cbaf221a59dea..727b0e75f03595 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -563,7 +563,7 @@ def send_fds(sock, buffers, fds, flags=0, address=None): import array return sock.sendmsg(buffers, [(_socket.SOL_SOCKET, - _socket.SCM_RIGHTS, array.array("i", fds))], flags, address) + _socket.SCM_RIGHTS, array.array("i", fds))]) __all__.append("send_fds") if hasattr(_socket.socket, "recvmsg"): diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 04dfb682ec6831..03c54151a2218f 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -7366,30 +7366,6 @@ def close_fds(fds): data = os.read(rfd, 100) self.assertEqual(data, str(index).encode()) - def testSendAndRecvFdsByAddress(self): - rfd, wfd = os.pipe() - self.addCleanup(os.close, rfd) - self.addCleanup(os.close, wfd) - - sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) - address = socket_helper.create_unix_domain_name() - self.addCleanup(os_helper.unlink, address) - socket_helper.bind_unix_socket(sock, address) - - socket.send_fds(sock, [MSG], [rfd], 0, address) - - # request more data and file descriptors than expected - msg, (rfd2,), flags, addr = socket.recv_fds(sock, len(MSG) * 2, 2) - self.addCleanup(os.close, rfd2) - self.assertEqual(msg, MSG) - self.assertEqual(flags, 0) - self.assertEqual(addr, address) - - # test that the file descriptor is connected - os.write(wfd, b'data') - data = os.read(rfd2, 100) - self.assertEqual(data, b'data') - class FreeThreadingTests(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2024-12-11-20-15-00.gh-issue-127840.pt8fiQ.rst b/Misc/NEWS.d/next/Library/2024-12-11-20-15-00.gh-issue-127840.pt8fiQ.rst deleted file mode 100644 index 51eaaa96ebaa07..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-11-20-15-00.gh-issue-127840.pt8fiQ.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :func:`socket.send_fds` ignoring flags and address parameters.