From 26ac7a025c199ad2b5687eb661ee3500d2eb636f Mon Sep 17 00:00:00 2001 From: zhaoyu Date: Sat, 30 Nov 2024 16:18:29 +0800 Subject: [PATCH 1/7] Try using `base_executable` to detect `prefix` and `exec_prefix` --- Lib/test/test_getpath.py | 37 +++++++++++++++++++++++++++++++++++++ Modules/getpath.py | 10 ++++++++++ 2 files changed, 47 insertions(+) diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index d5dcdad9614ecc..c92a05419e651f 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -386,6 +386,43 @@ def test_venv_changed_name_posix(self): actual = getpath(ns, expected) self.assertEqual(expected, actual) + def test_venv_with_invalid_home_posix(self): + ns = MockPosixNamespace( + argv0="/venv/bin/python", + PREFIX="/usr", + ENV_PATH="/venv/bin:/usr/bin", + ) + ns.add_known_xfile("/path/to/copy_dir/bin/python") + ns.add_known_xfile("/path/to/python-link") + ns.add_known_link("/path/to/python-link", + "/path/to/copy_dir/bin/python") + + ns.add_known_xfile("/venv/bin/python") + ns.add_known_link("/venv/bin/python", + "/path/to/python-link") + + ns.add_known_file("/path/to/copy_dir/lib/python9.8/os.py") + ns.add_known_dir("/path/to/copy_dir/lib/python9.8/lib-dynload") + ns.add_known_file("/venv/pyvenv.cfg", [ + r"home = /" + ]) + expected = dict( + executable="/venv/bin/python", + prefix="/path/to/copy_dir", + exec_prefix="/path/to/copy_dir", + base_executable="/path/to/copy_dir/bin/python", + base_prefix="/path/to/copy_dir", + base_exec_prefix="/path/to/copy_dir", + module_search_paths_set=1, + module_search_paths=[ + "/path/to/copy_dir/lib/python98.zip", + "/path/to/copy_dir/lib/python9.8", + "/path/to/copy_dir/lib/python9.8/lib-dynload", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + def test_venv_non_installed_zip_path_posix(self): "Test a venv created from non-installed python has correct zip path.""" ns = MockPosixNamespace( diff --git a/Modules/getpath.py b/Modules/getpath.py index 1f1bfcb4f64dd4..b3fded05352042 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -593,6 +593,13 @@ def search_up(prefix, *landmarks, test=isfile): if prefix and not stdlib_dir: stdlib_dir = joinpath(prefix, STDLIB_SUBDIR) + # Note: the `home` variable in pyvenv.cfg is not always accurate. + # Detect prefix by searching from *real* executable location for the stdlib_dir. + if STDLIB_SUBDIR and STDLIB_LANDMARKS and base_executable and not prefix: + prefix = search_up(base_executable, *STDLIB_LANDMARKS) + if prefix and not stdlib_dir: + stdlib_dir = joinpath(prefix, STDLIB_SUBDIR) + if PREFIX and not prefix: prefix = PREFIX if not any(isfile(joinpath(prefix, f)) for f in STDLIB_LANDMARKS): @@ -612,6 +619,9 @@ def search_up(prefix, *landmarks, test=isfile): exec_prefix = prefix if not exec_prefix and executable_dir: exec_prefix = search_up(executable_dir, PLATSTDLIB_LANDMARK, test=isdir) + if not exec_prefix and base_executable: + base_executable_dir = dirname(base_executable) + exec_prefix = search_up(base_executable_dir, PLATSTDLIB_LANDMARK, test=isdir) if not exec_prefix and EXEC_PREFIX: exec_prefix = EXEC_PREFIX if not exec_prefix or not isdir(joinpath(exec_prefix, PLATSTDLIB_LANDMARK)): From 10c2db0767a5d58f7284f9e2dbcea60dba57eb10 Mon Sep 17 00:00:00 2001 From: zhaoyu Date: Mon, 2 Dec 2024 09:39:09 +0800 Subject: [PATCH 2/7] Fix test case based on PR-126985 --- Lib/test/test_getpath.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index a2f0d3dd732c1a..4797b1585ea4d5 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -386,7 +386,7 @@ def test_venv_changed_name_posix(self): actual = getpath(ns, expected) self.assertEqual(expected, actual) - def test_venv_with_invalid_home_posix(self): + def test_venv_with_invalid_home_in_pyvenv_posix(self): ns = MockPosixNamespace( argv0="/venv/bin/python", PREFIX="/usr", @@ -408,8 +408,8 @@ def test_venv_with_invalid_home_posix(self): ]) expected = dict( executable="/venv/bin/python", - prefix="/path/to/copy_dir", - exec_prefix="/path/to/copy_dir", + prefix="/venv", + exec_prefix="/venv", base_executable="/path/to/copy_dir/bin/python", base_prefix="/path/to/copy_dir", base_exec_prefix="/path/to/copy_dir", From d3c6e8de84ce46c2ecff88eab87dbbb7a272b015 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 02:22:17 +0000 Subject: [PATCH 3/7] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2024-12-02-02-22-13.gh-issue-127440.vY5uvA.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-12-02-02-22-13.gh-issue-127440.vY5uvA.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-02-02-22-13.gh-issue-127440.vY5uvA.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-02-02-22-13.gh-issue-127440.vY5uvA.rst new file mode 100644 index 00000000000000..1c5643d9493c39 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-02-02-22-13.gh-issue-127440.vY5uvA.rst @@ -0,0 +1 @@ +Fix calculation of `base_prefix` and `base_exec_prefix` when the `home` in `pyvenv.cfg` is inaccurate and the installation directory (`PREFIX`) does not exist. Calculation will try to use `base_executable` now. Patch by Charlie Zhao. From 051f94acd3e09d3790fc1de512699303249d6cb1 Mon Sep 17 00:00:00 2001 From: zhaoyu Date: Mon, 2 Dec 2024 10:26:48 +0800 Subject: [PATCH 4/7] fix lint --- .../2024-12-02-02-22-13.gh-issue-127440.vY5uvA.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-02-02-22-13.gh-issue-127440.vY5uvA.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-02-02-22-13.gh-issue-127440.vY5uvA.rst index 1c5643d9493c39..77f718f5ef5cf1 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-02-02-22-13.gh-issue-127440.vY5uvA.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-02-02-22-13.gh-issue-127440.vY5uvA.rst @@ -1 +1 @@ -Fix calculation of `base_prefix` and `base_exec_prefix` when the `home` in `pyvenv.cfg` is inaccurate and the installation directory (`PREFIX`) does not exist. Calculation will try to use `base_executable` now. Patch by Charlie Zhao. +Fix calculation of ``base_prefix`` and ``base_exec_prefix`` when the ``home`` in ``pyvenv.cfg`` is inaccurate and the installation directory (``PREFIX``) does not exist. Calculation will try to use ``base_executable`` now. Patch by Charlie Zhao. From 5328f62cf8e029d6ee63ee2dbb3f99b70e668328 Mon Sep 17 00:00:00 2001 From: zhaoyu Date: Wed, 18 Dec 2024 19:35:41 +0800 Subject: [PATCH 5/7] Apply Filipe's comments. --- Modules/getpath.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/Modules/getpath.py b/Modules/getpath.py index 3b0bdeb683dd12..1dff5a28a1ee9c 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -236,6 +236,7 @@ def search_up(prefix, *landmarks, test=isfile): real_executable_dir = None platstdlib_dir = None +pyvenvcfg_home = None # ****************************************************************************** # CALCULATE program_name @@ -366,7 +367,7 @@ def search_up(prefix, *landmarks, test=isfile): for line in pyvenvcfg: key, had_equ, value = line.partition('=') if had_equ and key.strip().lower() == 'home': - executable_dir = real_executable_dir = value.strip() + pyvenvcfg_home = executable_dir = real_executable_dir = value.strip() if not base_executable: # First try to resolve symlinked executables, since that may be # more accurate than assuming the executable in 'home'. @@ -593,18 +594,17 @@ def search_up(prefix, *landmarks, test=isfile): if prefix and not stdlib_dir: stdlib_dir = joinpath(prefix, STDLIB_SUBDIR) + if PREFIX and STDLIB_LANDMARKS and not prefix and any(isfile(joinpath(PREFIX, f)) for f in STDLIB_LANDMARKS): + prefix = PREFIX + # Note: the `home` variable in pyvenv.cfg is not always accurate. # Detect prefix by searching from *real* executable location for the stdlib_dir. - if STDLIB_SUBDIR and STDLIB_LANDMARKS and base_executable and not prefix: + # See details: https://github.com/python/cpython/issues/127440 + if pyvenvcfg_home and STDLIB_SUBDIR and STDLIB_LANDMARKS and base_executable and not prefix: prefix = search_up(base_executable, *STDLIB_LANDMARKS) if prefix and not stdlib_dir: stdlib_dir = joinpath(prefix, STDLIB_SUBDIR) - if PREFIX and not prefix: - prefix = PREFIX - if not any(isfile(joinpath(prefix, f)) for f in STDLIB_LANDMARKS): - warn('Could not find platform independent libraries ') - if not prefix: prefix = abspath('') warn('Could not find platform independent libraries ') @@ -619,9 +619,12 @@ def search_up(prefix, *landmarks, test=isfile): exec_prefix = prefix if not exec_prefix and executable_dir: exec_prefix = search_up(executable_dir, PLATSTDLIB_LANDMARK, test=isdir) - if not exec_prefix and base_executable: + + # See details: https://github.com/python/cpython/issues/127440 + if pyvenvcfg_home and not exec_prefix and base_executable: base_executable_dir = dirname(base_executable) exec_prefix = search_up(base_executable_dir, PLATSTDLIB_LANDMARK, test=isdir) + if not exec_prefix and EXEC_PREFIX: exec_prefix = EXEC_PREFIX if not exec_prefix or not isdir(joinpath(exec_prefix, PLATSTDLIB_LANDMARK)): From 36f7726652121317f1a4731b1864d45a65c05326 Mon Sep 17 00:00:00 2001 From: zhaoyu Date: Wed, 18 Dec 2024 20:05:01 +0800 Subject: [PATCH 6/7] Trim whitespace for lint --- Modules/getpath.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/getpath.py b/Modules/getpath.py index 6df90dd5fe5afc..4a76dd9e3b0e9e 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -634,12 +634,12 @@ def search_up(prefix, *landmarks, test=isfile): exec_prefix = prefix if not exec_prefix and executable_dir: exec_prefix = search_up(executable_dir, PLATSTDLIB_LANDMARK, test=isdir) - + # See details: https://github.com/python/cpython/issues/127440 if pyvenvcfg_home and not exec_prefix and base_executable: base_executable_dir = dirname(base_executable) exec_prefix = search_up(base_executable_dir, PLATSTDLIB_LANDMARK, test=isdir) - + if not exec_prefix and EXEC_PREFIX: exec_prefix = EXEC_PREFIX if not exec_prefix or not isdir(joinpath(exec_prefix, PLATSTDLIB_LANDMARK)): From 2a25da2130bc64a6ba1de21e636b70f7f7d1766c Mon Sep 17 00:00:00 2001 From: zhaoyu Date: Wed, 25 Dec 2024 15:42:49 +0800 Subject: [PATCH 7/7] Fix failed tests --- Lib/test/test_embed.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 7110fb889f3c8e..bfd75e71c12e72 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1635,7 +1635,8 @@ def test_init_pyvenv_cfg(self): print("home = %s" % pyvenv_home, file=fp) print("include-system-site-packages = false", file=fp) - paths = self.module_search_paths() + paths = self.module_search_paths(prefix=tmpdir, + exec_prefix=tmpdir) if not MS_WINDOWS: paths[-1] = lib_dynload else: @@ -1649,7 +1650,7 @@ def test_init_pyvenv_cfg(self): base_executable = os.path.join(pyvenv_home, os.path.basename(executable)) exec_prefix = pyvenv_home config = { - 'base_prefix': sysconfig.get_config_var("prefix"), + 'base_prefix': tmpdir, 'base_exec_prefix': exec_prefix, 'exec_prefix': tmpdir, 'prefix': tmpdir,