diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index 0b3a25014..2a1b732f7 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -1,3 +1,11 @@ +--- +name: Bug report +about: Create a report to help us improve +title: "" +labels: bug +assignees: "" +--- + **Issue** Describe what's the expected behavior and what you're observing. @@ -7,6 +15,8 @@ Describe what's the expected behavior and what you're observing. Provide at least: - OS: +- Shell: +- Python version and path: - `pip list` of the host python where `virtualenv` is installed: ```console diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index 258ad844a..000000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -name: Bug report -about: Create a report to help us improve -title: "" -labels: bug -assignees: "" ---- - -**Issue** - -Describe what's the expected behavior and what you're observing. - -**Environment** - -Provide at least: - -- OS: -- `pip list` of the host python where `virtualenv` is installed: - - ```console - - ``` - -**Output of the virtual environment creation** - -Make sure to run the creation with `-vvv --with-traceback`: - -```console - -``` diff --git a/.github/release.yml b/.github/release.yml index 9d1e0987b..5f89818f9 100644 --- a/.github/release.yml +++ b/.github/release.yml @@ -1,5 +1,5 @@ changelog: exclude: authors: - - dependabot - - pre-commit-ci + - dependabot[bot] + - pre-commit-ci[bot] diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index a5b66730a..68120bde6 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -1,9 +1,8 @@ -name: check +name: ๐Ÿงช check on: workflow_dispatch: push: branches: ["main"] - tags-ignore: ["**"] pull_request: schedule: - cron: "0 8 * * *" @@ -14,7 +13,7 @@ concurrency: jobs: test: - name: test ${{ matrix.py }} - ${{ matrix.os }} + name: ๐Ÿงช test ${{ matrix.py }} - ${{ matrix.os }} if: github.event_name != 'schedule' || github.repository_owner == 'pypa' runs-on: ${{ matrix.os }} timeout-minutes: 40 @@ -22,102 +21,109 @@ jobs: fail-fast: false matrix: py: + - "3.14t" + - "3.14" + - "3.13t" - "3.13" - "3.12" - "3.11" - "3.10" - "3.9" - "3.8" + - pypy-3.11 - pypy-3.10 - pypy-3.9 - pypy-3.8 + - graalpy-24.1 os: - - ubuntu-latest - - macos-latest - - windows-latest + - ubuntu-24.04 + - macos-15 + - windows-2025 include: - - { os: macos-latest, py: "brew@3.11" } - - { os: macos-latest, py: "brew@3.10" } - - { os: macos-latest, py: "brew@3.9" } + - { os: macos-15, py: "brew@3.11" } + - { os: macos-15, py: "brew@3.10" } + - { os: macos-15, py: "brew@3.9" } exclude: - - { os: windows-latest, py: "pypy-3.10" } - - { os: windows-latest, py: "pypy-3.9" } - - { os: windows-latest, py: "pypy-3.8" } + - { os: windows-2025, py: "graalpy-24.1" } + - { os: windows-2025, py: "pypy-3.10" } + - { os: windows-2025, py: "pypy-3.9" } + - { os: windows-2025, py: "pypy-3.8" } steps: - - uses: taiki-e/install-action@cargo-binstall - - name: Install OS dependencies - run: | - set -x - for i in 1 2 3; do - echo "try $i" && \ - ${{ runner.os == 'Linux' && 'sudo apt-get update -y && sudo apt-get install snapd fish csh -y' || true }} && \ - ${{ runner.os == 'Linux' && 'cargo binstall -y nu' || true }} && \ - ${{ runner.os == 'macOS' && 'brew install fish tcsh nushell' || true }} && \ - ${{ runner.os == 'Windows' && 'choco install nushell' || true }} && \ - exit 0 || true; - sleep 1 - done - exit 1 - shell: bash - - uses: actions/checkout@v4 + - name: ๐Ÿš€ Install uv + uses: astral-sh/setup-uv@v4 + - name: ๐Ÿ“ฅ Checkout code + uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Install the latest version of uv - uses: astral-sh/setup-uv@v3 + - name: ๐Ÿ Setup Python for tox + uses: actions/setup-python@v5 with: - enable-cache: true - cache-dependency-glob: "pyproject.toml" - github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Add .local/bin to PATH Windows - if: runner.os == 'Windows' + python-version: "3.14" + - name: ๐Ÿ“ฆ Install tox with this virtualenv shell: bash - run: echo "$USERPROFILE/.local/bin" >> $GITHUB_PATH - - name: Add .local/bin to PATH macos-13 - if: matrix.os == 'macos-13' - shell: bash - run: echo ~/.local/bin >> $GITHUB_PATH - - name: Install tox - if: matrix.py == '3.13' - run: uv tool install --python-preference only-managed --python 3.12 tox --with tox-uv - - name: Install tox - if: matrix.py != '3.13' - run: uv tool install --python-preference only-managed --python 3.13 tox --with tox-uv - - name: Setup brew python for test ${{ matrix.py }} - if: startsWith(matrix.py,'brew@') run: | - set -e - PY=$(echo '${{ matrix.py }}' | cut -c 6-) - brew cleanup && brew upgrade python@$PY || brew install python@$PY - echo "/usr/local/opt/python@$PY/libexec/bin" >>"${GITHUB_PATH}" - shell: bash - - name: Setup python for test ${{ matrix.py }} - if: "!( startsWith(matrix.py,'brew@') || endsWith(matrix.py, '-dev') )" + if [[ "${{ matrix.py }}" == "3.13t" || "${{ matrix.py }}" == "3.14t" ]]; then + uv tool install --no-managed-python --python 3.14 tox --with . + else + uv tool install --no-managed-python --python 3.14 tox --with tox-uv --with . + fi + - name: ๐Ÿ Setup Python for test ${{ matrix.py }} uses: actions/setup-python@v5 + if: ${{ !startsWith(matrix.py, 'brew@') }} with: python-version: ${{ matrix.py }} - allow-prereleases: true - - name: Pick environment to run - run: python tasks/pick_tox_env.py ${{ matrix.py }} - - name: Setup test suite - run: tox run -vv --notest --skip-missing-interpreters false - - name: Run test suite + - name: ๐Ÿ› ๏ธ Install OS dependencies + shell: bash + run: | + if [ "${{ runner.os }}" = "Linux" ]; then + sudo apt-get install -y software-properties-common + sudo apt-add-repository ppa:fish-shell/release-4 -y + curl -fsSL https://apt.fury.io/nushell/gpg.key | sudo gpg --dearmor -o /etc/apt/trusted.gpg.d/fury-nushell.gpg + echo "deb https://apt.fury.io/nushell/ /" | sudo tee /etc/apt/sources.list.d/fury.list + sudo apt-get update -y + sudo apt-get install snapd fish csh nushell -y + elif [ "${{ runner.os }}" = "macOS" ]; then + brew update + if [[ "${{ matrix.py }}" == brew@* ]]; then + PY=$(echo '${{ matrix.py }}' | cut -c 6-) + brew install python@$PY || brew upgrade python@$PY + echo "/usr/local/opt/python@$PY/libexec/bin" >>"${GITHUB_PATH}" + fi + brew install fish tcsh nushell || brew upgrade fish tcsh nushell + elif [ "${{ runner.os }}" = "Windows" ]; then + choco install nushell + fi + - name: ๐Ÿงฌ Pick environment to run + shell: bash + run: | + py="${{ matrix.py }}" + if [[ "$py" == brew@* ]]; then + brew_version="${py#brew@}" + echo "TOX_DISCOVER=/opt/homebrew/bin/python${brew_version}" >> "$GITHUB_ENV" + py="$brew_version" + fi + [[ "$py" == graalpy-* ]] && py="graalpy" + echo "TOXENV=$py" >> "$GITHUB_ENV" + echo "Set TOXENV=$py" + - name: ๐Ÿ—๏ธ Setup test suite + run: tox run -vvvv --notest --skip-missing-interpreters false + - name: ๐Ÿƒ Run test suite run: tox run --skip-pkg-install timeout-minutes: 20 env: PYTEST_ADDOPTS: "-vv --durations=20" CI_RUN: "yes" DIFF_AGAINST: HEAD - check: - name: ${{ matrix.tox_env }} - ${{ matrix.os }} + name: ๐Ÿ”Ž check ${{ matrix.tox_env }} - ${{ matrix.os }} if: github.event_name != 'schedule' || github.repository_owner == 'pypa' runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: - - ubuntu-latest - - windows-latest + - ubuntu-24.04 + - windows-2025 tox_env: - dev - docs @@ -125,25 +131,18 @@ jobs: - upgrade - zipapp exclude: - - { os: windows-latest, tox_env: readme } - - { os: windows-latest, tox_env: docs } + - { os: windows-2025, tox_env: readme } + - { os: windows-2025, tox_env: docs } steps: - - uses: actions/checkout@v4 + - name: ๐Ÿš€ Install uv + uses: astral-sh/setup-uv@v4 + - name: ๐Ÿ“ฆ Install tox + run: uv tool install --python-preference only-managed --python 3.14 tox --with tox-uv + - name: ๐Ÿ“ฅ Checkout code + uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Install the latest version of uv - uses: astral-sh/setup-uv@v3 - with: - enable-cache: true - cache-dependency-glob: "pyproject.toml" - github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Add .local/bin to Windows PATH - if: runner.os == 'Windows' - shell: bash - run: echo "$USERPROFILE/.local/bin" >> $GITHUB_PATH - - name: Install tox - run: uv tool install --python-preference only-managed --python 3.13 tox --with tox-uv - - name: Setup check suite - run: tox r -vv --notest --skip-missing-interpreters false -e ${{ matrix.tox_env }} - - name: Run check for ${{ matrix.tox_env }} - run: tox r --skip-pkg-install -e ${{ matrix.tox_env }} + - name: ๐Ÿ—๏ธ Setup check suite + run: tox run -vv --notest --skip-missing-interpreters false -e ${{ matrix.tox_env }} + - name: ๐Ÿƒ Run check for ${{ matrix.tox_env }} + run: tox run --skip-pkg-install -e ${{ matrix.tox_env }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 0928ac66c..def30f5d9 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -8,20 +8,21 @@ env: jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4 + - name: ๐Ÿ“ฅ Checkout code + uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Install the latest version of uv - uses: astral-sh/setup-uv@v3 + - name: ๐Ÿš€ Install the latest version of uv + uses: astral-sh/setup-uv@v4 with: enable-cache: true cache-dependency-glob: "pyproject.toml" github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Build package - run: uv build --python 3.13 --python-preference only-managed --sdist --wheel . --out-dir dist - - name: Store the distribution packages + - name: ๐Ÿ“ฆ Build package + run: uv build --python 3.14 --python-preference only-managed --sdist --wheel . --out-dir dist + - name: ๐Ÿ“ฆ Store the distribution packages uses: actions/upload-artifact@v4 with: name: ${{ env.dists-artifact-name }} @@ -30,19 +31,19 @@ jobs: release: needs: - build - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 environment: name: release url: https://pypi.org/project/virtualenv/${{ github.ref_name }} permissions: id-token: write steps: - - name: Download all the dists + - name: ๐Ÿ“ฅ Download all the dists uses: actions/download-artifact@v4 with: name: ${{ env.dists-artifact-name }} path: dist/ - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@v1.12.2 + - name: ๐Ÿš€ Publish to PyPI + uses: pypa/gh-action-pypi-publish@v1.13.0 with: attestations: true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a19389600..f626eaea8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,36 +1,36 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.29.4 + rev: 0.34.1 hooks: - id: check-github-workflows args: ["--verbose"] - repo: https://github.com/codespell-project/codespell - rev: v2.3.0 + rev: v2.4.1 hooks: - id: codespell args: ["--write-changes"] - repo: https://github.com/tox-dev/tox-ini-fmt - rev: "1.4.1" + rev: "1.7.0" hooks: - id: tox-ini-fmt args: ["-p", "fix"] - repo: https://github.com/tox-dev/pyproject-fmt - rev: "v2.5.0" + rev: "v2.11.1" hooks: - id: pyproject-fmt - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.7.2" + rev: "v0.14.4" hooks: - id: ruff-format - id: ruff args: ["--fix", "--unsafe-fixes", "--exit-non-zero-on-fix"] - repo: https://github.com/rbubley/mirrors-prettier - rev: "v3.3.3" + rev: "v3.6.2" hooks: - id: prettier additional_dependencies: diff --git a/docs/changelog.rst b/docs/changelog.rst index 7c4c43198..d23e3e4ed 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,235 @@ Release History .. towncrier release notes start +v20.35.4 (2025-10-28) +--------------------- + +Bugfixes - 20.35.4 +~~~~~~~~~~~~~~~~~~ +- Fix race condition in ``_virtualenv.py`` when file is overwritten during import, preventing ``NameError`` when ``_DISTUTILS_PATCH`` is accessed - by :user:`gracetyy`. (:issue:`2969`) +- Upgrade embedded wheels: + + * pip to ``25.3`` from ``25.2`` (:issue:`2989`) + +v20.35.3 (2025-10-10) +--------------------- + +Bugfixes - 20.35.3 +~~~~~~~~~~~~~~~~~~ +- Accept RuntimeError in `test_too_many_open_files`, by :user:`esafak` (:issue:`2935`) + +v20.35.2 (2025-10-10) +--------------------- + +Bugfixes - 20.35.2 +~~~~~~~~~~~~~~~~~~ +- Revert out changes related to the extraction of the discovery module - by :user:`gaborbernat`. (:issue:`2978`) + +v20.35.1 (2025-10-09) +--------------------- + +Bugfixes - 20.35.1 +~~~~~~~~~~~~~~~~~~ +- Patch get_interpreter to handle missing cache and app_data - by :user:`esafak` (:issue:`2972`) +- Fix backwards incompatible changes to ``PythonInfo`` - by :user:`gaborbernat`. (:issue:`2975`) + +v20.35.0 (2025-10-08) +--------------------- + +Features - 20.35.0 +~~~~~~~~~~~~~~~~~~ +- Add AppData and Cache protocols to discovery for decoupling - by :user:`esafak`. (:issue:`2074`) +- Ensure python3.exe and python3 on Windows for Python 3 - by :user:`esafak`. (:issue:`2774`) + +Bugfixes - 20.35.0 +~~~~~~~~~~~~~~~~~~ +- Replaced direct references to tcl/tk library paths with getattr - by :user:`esafak` (:issue:`2944`) +- Restore absolute import of fs_is_case_sensitive - by :user:`esafak`. (:issue:`2955`) + +v20.34.0 (2025-08-13) +--------------------- + +Features - 20.34.0 +~~~~~~~~~~~~~~~~~~ +- Abstract out caching in discovery - by :user:`esafak`. + Decouple `FileCache` from `py_info` (discovery) - by :user:`esafak`. + Remove references to py_info in FileCache - by :user:`esafak`. + Decouple discovery from creator plugins - by :user:`esafak`. + Decouple discovery by duplicating info utils - by :user:`esafak`. (:issue:`2074`) +- Add PyPy 3.11 support. Contributed by :user:`esafak`. (:issue:`2932`) + +Bugfixes - 20.34.0 +~~~~~~~~~~~~~~~~~~ +- Upgrade embedded wheel pip to ``25.2`` from ``25.1.1`` - by :user:`gaborbernat`. (:issue:`2333`) +- Accept RuntimeError in `test_too_many_open_files`, by :user:`esafak` (:issue:`2935`) +- Python in PATH takes precedence over uv-managed python. Contributed by :user:`edgarrmondragon`. (:issue:`2952`) + +v20.33.1 (2025-08-05) +--------------------- + +Bugfixes - 20.33.1 +~~~~~~~~~~~~~~~~~~ +- Correctly unpack _get_tcl_tk_libs() response in PythonInfo. + Contributed by :user:`esafak`. (:issue:`2930`) +- Restore `py_info.py` timestamp in `test_py_info_cache_invalidation_on_py_info_change` + Contributed by :user:`esafak`. (:issue:`2933`) + +v20.33.0 (2025-08-03) +--------------------- + +Features - 20.33.0 +~~~~~~~~~~~~~~~~~~ +- Added support for Tcl and Tkinter. You're welcome. + Contributed by :user:`esafak`. (:issue:`425`) + +Bugfixes - 20.33.0 +~~~~~~~~~~~~~~~~~~ +- Prevent logging setup when --help is passed, fixing a flaky test. + Contributed by :user:`esafak`. (:issue:`u`) +- Fix cache invalidation for PythonInfo by hashing `py_info.py`. + Contributed by :user:`esafak`. (:issue:`2467`) +- When no discovery plugins are found, the application would crash with a StopIteration. + This change catches the StopIteration and raises a RuntimeError with a more informative message. + Contributed by :user:`esafak`. (:issue:`2493`) +- Stop `--try-first-with` overriding absolute `--python` paths. + Contributed by :user:`esafak`. (:issue:`2659`) +- Force UTF-8 encoding for pip download + Contributed by :user:`esafak`. (:issue:`2780`) +- Creating a virtual environment on a filesystem without symlink-support would fail even with `--copies` + Make `fs_supports_symlink` perform a real symlink creation check on all platforms. + Contributed by :user:`esafak`. (:issue:`2786`) +- Add a note to the user guide recommending the use of a specific Python version when creating virtual environments. + Contributed by :user:`esafak`. (:issue:`2808`) +- Fix 'Too many open files' error due to a file descriptor leak in virtualenv's locking mechanism. + Contributed by :user:`esafak`. (:issue:`2834`) +- Support renamed Windows venv redirector (`venvlauncher.exe` and `venvwlauncher.exe`) on Python 3.13 + Contributed by :user:`esafak`. (:issue:`2851`) +- Resolve Nushell activation script deprecation warnings by dynamically selecting the ``--optional`` flag for Nushell + ``get`` command on version 0.106.0 and newer, while retaining the deprecated ``-i`` flag for older versions to maintain + compatibility. Contributed by :user:`gaborbernat`. (:issue:`2910`) + +v20.32.0 (2025-07-20) +--------------------- + +Features - 20.32.0 +~~~~~~~~~~~~~~~~~~ +- Warn on incorrect invocation of Nushell activation script - by :user:`esafak`. (:issue:`nushell_activation`) +- Discover uv-managed Python installations (:issue:`2901`) + +Bugfixes - 20.32.0 +~~~~~~~~~~~~~~~~~~ +- Ignore missing absolute paths for python discovery - by :user:`esafak` (:issue:`2870`) +- Upgrade embedded setuptools to ``80.9.0`` from ``80.3.1`` - by :user:`gaborbernat`. (:issue:`2900`) + +v20.31.2 (2025-05-08) +--------------------- + +No significant changes. + + +v20.31.1 (2025-05-05) +--------------------- + +Bugfixes - 20.31.1 +~~~~~~~~~~~~~~~~~~ +- Upgrade embedded wheels: + + * pip to ``25.1.1`` from ``25.1`` + * setuptools to ``80.3.1`` from ``78.1.0`` (:issue:`2880`) + +v20.31.0 (2025-05-05) +--------------------- + +Features - 20.31.0 +~~~~~~~~~~~~~~~~~~ +- No longer bundle ``wheel`` wheels (except on Python 3.8), ``setuptools`` includes native ``bdist_wheel`` support. Update ``pip`` to ``25.1``. (:issue:`2868`) + +Bugfixes - 20.31.0 +~~~~~~~~~~~~~~~~~~ +- ``get_embed_wheel()`` no longer fails with a :exc:`TypeError` when it is + called with an unknown *distribution*. (:issue:`2877`) +- Fix ``HelpFormatter`` error with Python 3.14.0b1. (:issue:`2878`) + +v20.30.0 (2025-03-31) +--------------------- + +Features - 20.30.0 +~~~~~~~~~~~~~~~~~~ +- Add support for `GraalPy `_. (:issue:`2832`) + +Bugfixes - 20.30.0 +~~~~~~~~~~~~~~~~~~ +- Upgrade embedded wheels: + + * setuptools to ``78.1.0`` from ``75.3.2`` (:issue:`2863`) + +v20.29.3 (2025-03-06) +--------------------- + +Bugfixes - 20.29.3 +~~~~~~~~~~~~~~~~~~ +- Ignore unreadable directories in ``PATH``. (:issue:`2794`) + +v20.29.2 (2025-02-10) +--------------------- + +Bugfixes - 20.29.2 +~~~~~~~~~~~~~~~~~~ +- Remove old virtualenv wheel from the source distribution - by :user:`gaborbernat`. (:issue:`2841`) +- Upgrade embedded wheel pip to ``25.0.1`` from ``24.3.1`` - by :user:`gaborbernat`. (:issue:`2843`) + +v20.29.1 (2025-01-17) +--------------------- + +Bugfixes - 20.29.1 +~~~~~~~~~~~~~~~~~~ +- Fix PyInfo cache incompatibility warnings - by :user:`robsdedude`. (:issue:`2827`) + +v20.29.0 (2025-01-15) +--------------------- + +Features - 20.29.0 +~~~~~~~~~~~~~~~~~~ +- Add support for selecting free-threaded Python interpreters, e.g., `python3.13t`. (:issue:`2809`) + +Bugfixes - 20.29.0 +~~~~~~~~~~~~~~~~~~ +- Upgrade embedded wheels: + + * setuptools to ``75.8.0`` from ``75.6.0`` (:issue:`2823`) + +v20.28.1 (2025-01-02) +--------------------- + +Bugfixes - 20.28.1 +~~~~~~~~~~~~~~~~~~ +- Skip tcsh tests on broken tcsh versions - by :user:`gaborbernat`. (:issue:`2814`) + +v20.28.0 (2024-11-25) +--------------------- + +Features - 20.28.0 +~~~~~~~~~~~~~~~~~~ +- Write CACHEDIR.TAG file on creation - by "user:`neilramsay`. (:issue:`2803`) + +v20.27.2 (2024-11-25) +--------------------- + +Bugfixes - 20.27.2 +~~~~~~~~~~~~~~~~~~ +- Upgrade embedded wheels: + + * setuptools to ``75.3.0`` from ``75.2.0`` (:issue:`2798`) +- Upgrade embedded wheels: + + * wheel to ``0.45.0`` from ``0.44.0`` + * setuptools to ``75.5.0`` (:issue:`2800`) +- no longer forcibly echo off during windows batch activation (:issue:`2801`) +- Upgrade embedded wheels: + + * setuptools to ``75.6.0`` from ``75.5.0`` + * wheel to ``0.45.1`` from ``0.45.0`` (:issue:`2804`) + v20.27.1 (2024-10-28) --------------------- diff --git a/docs/changelog/2798.bugfix.rst b/docs/changelog/2798.bugfix.rst deleted file mode 100644 index 3c3356900..000000000 --- a/docs/changelog/2798.bugfix.rst +++ /dev/null @@ -1,3 +0,0 @@ -Upgrade embedded wheels: - -* setuptools to ``75.3.0`` from ``75.2.0`` diff --git a/docs/cli_interface.rst b/docs/cli_interface.rst index 1bbbe234b..27ea231ef 100644 --- a/docs/cli_interface.rst +++ b/docs/cli_interface.rst @@ -24,6 +24,51 @@ The options that can be passed to virtualenv, along with their default values an :module: virtualenv.run :func: build_parser_only +Discovery options +~~~~~~~~~~~~~~~~~ + +Understanding Interpreter Discovery: ``--python`` vs. ``--try-first-with`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You can control which Python interpreter ``virtualenv`` selects using the ``--python`` and ``--try-first-with`` flags. +To avoid confusion, it's best to think of them as the "rule" and the "hint". + +**``--python ``: The Rule** + +This flag sets the mandatory requirements for the interpreter. The ```` can be: + +- **A version string** (e.g., ``python3.8``, ``pypy3``). ``virtualenv`` will search for any interpreter that matches this version. +- **An absolute path** (e.g., ``/usr/bin/python3.8``). This is a *strict* requirement. Only the interpreter at this exact path will be used. If it does not exist or is not a valid interpreter, creation will fail. + +**``--try-first-with ``: The Hint** + +This flag provides a path to a Python executable to check *before* ``virtualenv`` performs its standard search. This can speed up discovery or help select a specific interpreter when multiple versions exist on your system. + +**How They Work Together** + +``virtualenv`` will only use an interpreter from ``--try-first-with`` if it **satisfies the rule** from the ``--python`` flag. The ``--python`` rule always wins. + +**Examples:** + +1. **Hint does not match the rule:** + + .. code-block:: bash + + virtualenv --python python3.8 --try-first-with /usr/bin/python3.10 my-env + + - **Result:** ``virtualenv`` first inspects ``/usr/bin/python3.10``. It sees this does not match the ``python3.8`` rule and **rejects it**. It then proceeds with its normal search to find a ``python3.8`` interpreter elsewhere. + +2. **Hint does not match a strict path rule:** + + .. code-block:: bash + + virtualenv --python /usr/bin/python3.8 --try-first-with /usr/bin/python3.10 my-env + + - **Result:** The rule is strictly ``/usr/bin/python3.8``. ``virtualenv`` checks the ``/usr/bin/python3.10`` hint, sees the path doesn't match, and **rejects it**. It then moves on to test ``/usr/bin/python3.8`` and successfully creates the environment. + +This approach ensures that the behavior is predictable and that ``--python`` remains the definitive source of truth for the user's intent. + + Defaults ~~~~~~~~ diff --git a/docs/development.rst b/docs/development.rst index ef5b75e61..941a8b5c9 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -106,7 +106,7 @@ that folder. Release ~~~~~~~ -virtualenv's release schedule is tied to ``pip``, ``setuptools`` and ``wheel``. We bundle the latest version of these +virtualenv's release schedule is tied to ``pip`` and ``setuptools``. We bundle the latest version of these libraries so each time there's a new version of any of these, there will be a new virtualenv release shortly afterwards (we usually wait just a few days to avoid pulling in any broken releases). diff --git a/docs/index.rst b/docs/index.rst index e31741978..52484a9ae 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -30,7 +30,12 @@ virtualenv :target: https://pypistats.org/packages/virtualenv :alt: Package popularity -``virtualenv`` is a tool to create isolated Python environments. Since Python ``3.3``, a subset of it has been +``virtualenv`` is a tool to create isolated Python environments. + +virtualenv vs venv +------------------ + +Since Python ``3.3``, a subset of it has been integrated into the standard library under the `venv module `_. The ``venv`` module does not offer all features of this library, to name just a few more prominent: @@ -40,6 +45,9 @@ integrated into the standard library under the `venv module `_, - does not have as rich programmatic API (describe virtual environments without creating them). +Concept and purpose of virtualenv +--------------------------------- + The basic problem being addressed is one of dependencies and versions, and indirectly permissions. Imagine you have an application that needs version ``1`` of ``LibFoo``, but another application requires version ``2``. How can you use both these libraries? If you install everything into your host python (e.g. ``python3.8``) @@ -53,6 +61,31 @@ In all these cases, ``virtualenv`` can help you. It creates an environment that that doesn't share libraries with other virtualenv environments (and optionally doesn't access the globally installed libraries either). + +Compatibility +------------- + +With the release of virtualenv 20.22, April 2023, (`release note `__) target interpreters are now limited to Python v. 3.7+. + +Trying to use an earlier version will normally result in the target interpreter raising a syntax error. This virtualenv tool will then print some details about the exception and abort, ie no explicit warning about trying to use an outdated/incompatible version. It may look like this: + +.. code-block:: console + + $ virtualenv --discovery pyenv -p python3.6 foo + RuntimeError: failed to query /home/velle/.pyenv/versions/3.6.15/bin/python3.6 with code 1 err: ' File "/home/velle/.virtualenvs/toxrunner/lib/python3.12/site-packages/virtualenv/discovery/py_info.py", line 7 + from __future__ import annotations + ^ + SyntaxError: future feature annotations is not defined + + +In tox, even if the interpreter is installed and available, the message is (somewhat misleading): + +.. code-block:: console + + py36: skipped because could not find python interpreter with spec(s): py36 + + + Useful links ------------ diff --git a/docs/installation.rst b/docs/installation.rst index c48db8f07..b82d1dd19 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -84,7 +84,7 @@ Python and OS Compatibility virtualenv works with the following Python interpreter implementations: -- `CPython `_: ``3.12 >= python_version >= 3.7`` +- `CPython `_: ``3.13 >= python_version >= 3.7`` - `PyPy `_: ``3.10 >= python_version >= 3.7`` This means virtualenv works on the latest patch version of each of these minor versions. Previous patch versions are diff --git a/docs/user_guide.rst b/docs/user_guide.rst index 54169e062..82cc235b1 100644 --- a/docs/user_guide.rst +++ b/docs/user_guide.rst @@ -30,6 +30,16 @@ Virtualenv has one basic command: virtualenv venv +.. note:: + + When creating a virtual environment, it's recommended to use a specific Python version, for example, by invoking + virtualenv with ``python3.10 -m virtualenv venv``. If you use a generic command like ``python3 -m virtualenv venv``, + the created environment will be linked to ``/usr/bin/python3``. This can be problematic because when a new Python + version is installed on the system, the ``/usr/bin/python3`` symlink will likely be updated to point to the new + version. This will cause the virtual environment to inadvertently use the new Python version, which is often not the + desired behavior. Using a specific version ensures that the virtual environment is tied to that exact version, + providing stability and predictability. + This will create a python virtual environment of the same version as virtualenv, installed into the subdirectory ``venv``. The command line tool has quite a few of flags that modify the tool's behavior, for a full list make sure to check out :ref:`cli_flags`. @@ -86,13 +96,14 @@ format is either: - the python implementation is all alphabetic characters (``python`` means any implementation, and if is missing it defaults to ``python``), - - the version is a dot separated version number, + - the version is a dot separated version number optionally followed by ``t`` for free-threading, - the architecture is either ``-64`` or ``-32`` (missing means ``any``). For example: - ``python3.8.1`` means any python implementation having the version ``3.8.1``, - ``3`` means any python implementation having the major version ``3``, + - ``3.13t`` means any python implementation having the version ``3.13`` with free threading, - ``cpython3`` means a ``CPython`` implementation having the version ``3``, - ``pypy2`` means a python interpreter with the ``PyPy`` implementation and major version ``2``. @@ -101,6 +112,8 @@ format is either: - If we're on Windows look into the Windows registry, and check if we see any registered Python implementations that match the specification. This is in line with expectation laid out inside `PEP-514 `_ + - If `uv-managed `_ Python installations are available, use the + first one that matches the specification. - Try to discover a matching python executable within the folders enumerated on the ``PATH`` environment variable. In this case we'll try to find an executable that has a name roughly similar to the specification (for exact logic, please see the implementation code). @@ -139,8 +152,7 @@ Seeders ------- These will install for you some seed packages (one or more of: :pypi:`pip`, :pypi:`setuptools`, :pypi:`wheel`) that enables you to install additional python packages into the created virtual environment (by invoking pip). Installing -:pypi:`setuptools` and :pypi:`wheel` is disabled by default on Python 3.12+ environments. There are two -main seed mechanisms available: +:pypi:`setuptools` is disabled by default on Python 3.12+ environments. :pypi:`wheel` is only installed on Python 3.8, by default. There are two main seed mechanisms available: - ``pip`` - this method uses the bundled pip with virtualenv to install the seed packages (note, a new child process needs to be created to do this, which can be expensive especially on Windows). diff --git a/pyproject.toml b/pyproject.toml index 851b90bc8..85ff87ae8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Software Development :: Libraries", @@ -47,6 +48,7 @@ dependencies = [ "filelock>=3.12.2,<4", "importlib-metadata>=6.6; python_version<'3.8'", "platformdirs>=3.9.1,<5", + "typing-extensions>=4.13.2; python_version<'3.11'", ] optional-dependencies.docs = [ "furo>=2023.7.26", @@ -64,7 +66,7 @@ optional-dependencies.test = [ "packaging>=23.1", "pytest>=7.4", "pytest-env>=0.8.2", - "pytest-freezer>=0.4.8; platform_python_implementation=='PyPy' or (platform_python_implementation=='CPython' and sys_platform=='win32' and python_version>='3.13')", + "pytest-freezer>=0.4.8; platform_python_implementation=='PyPy' or platform_python_implementation=='GraalVM' or (platform_python_implementation=='CPython' and sys_platform=='win32' and python_version>='3.13')", "pytest-mock>=3.11.1", "pytest-randomly>=3.12", "pytest-timeout>=2.1", @@ -87,6 +89,8 @@ entry-points."virtualenv.create".cpython3-mac-brew = "virtualenv.create.via_glob entry-points."virtualenv.create".cpython3-mac-framework = "virtualenv.create.via_global_ref.builtin.cpython.mac_os:CPython3macOsFramework" entry-points."virtualenv.create".cpython3-posix = "virtualenv.create.via_global_ref.builtin.cpython.cpython3:CPython3Posix" entry-points."virtualenv.create".cpython3-win = "virtualenv.create.via_global_ref.builtin.cpython.cpython3:CPython3Windows" +entry-points."virtualenv.create".graalpy-posix = "virtualenv.create.via_global_ref.builtin.graalpy:GraalPyPosix" +entry-points."virtualenv.create".graalpy-win = "virtualenv.create.via_global_ref.builtin.graalpy:GraalPyWindows" entry-points."virtualenv.create".pypy3-posix = "virtualenv.create.via_global_ref.builtin.pypy.pypy3:PyPy3Posix" entry-points."virtualenv.create".pypy3-win = "virtualenv.create.via_global_ref.builtin.pypy.pypy3:Pypy3Windows" entry-points."virtualenv.create".venv = "virtualenv.create.via_global_ref.venv:Venv" @@ -156,7 +160,7 @@ builtin = "clear,usage,en-GB_to_en-US" count = true [tool.pyproject-fmt] -max_supported_python = "3.13" +max_supported_python = "3.14" [tool.pytest.ini_options] markers = [ diff --git a/src/virtualenv/__main__.py b/src/virtualenv/__main__.py index d0979a665..49f59da38 100644 --- a/src/virtualenv/__main__.py +++ b/src/virtualenv/__main__.py @@ -1,10 +1,13 @@ from __future__ import annotations +import errno import logging import os import sys from timeit import default_timer +LOGGER = logging.getLogger(__name__) + def run(args=None, options=None, env=None): env = os.environ if env is None else env @@ -16,12 +19,21 @@ def run(args=None, options=None, env=None): args = sys.argv[1:] try: session = cli_run(args, options, env) - logging.warning(LogSession(session, start)) + LOGGER.warning(LogSession(session, start)) except ProcessCallFailedError as exception: print(f"subprocess call failed for {exception.cmd} with code {exception.code}") # noqa: T201 print(exception.out, file=sys.stdout, end="") # noqa: T201 print(exception.err, file=sys.stderr, end="") # noqa: T201 raise SystemExit(exception.code) # noqa: B904 + except OSError as exception: + if exception.errno == errno.EMFILE: + print( # noqa: T201 + "OSError: [Errno 24] Too many open files. You may need to increase your OS open files limit.\n" + " On macOS/Linux, try 'ulimit -n 2048'.\n" + " For Windows, this is not a common issue, but you can try to close some applications.", + file=sys.stderr, + ) + raise class LogSession: @@ -54,16 +66,17 @@ def run_with_catch(args=None, env=None): options = VirtualEnvOptions() try: run(args, options, env) - except (KeyboardInterrupt, SystemExit, Exception) as exception: + except (KeyboardInterrupt, SystemExit, Exception) as exception: # noqa: BLE001 try: if getattr(options, "with_traceback", False): raise if not (isinstance(exception, SystemExit) and exception.code == 0): - logging.error("%s: %s", type(exception).__name__, exception) # noqa: TRY400 + LOGGER.error("%s: %s", type(exception).__name__, exception) # noqa: TRY400 code = exception.code if isinstance(exception, SystemExit) else 1 sys.exit(code) finally: - logging.shutdown() # force flush of log messages before the trace is printed + for handler in LOGGER.handlers: # force flush of log messages before the trace is printed + handler.flush() if __name__ == "__main__": # pragma: no cov diff --git a/src/virtualenv/activation/bash/__init__.py b/src/virtualenv/activation/bash/__init__.py index 5e095ddf0..4f160744f 100644 --- a/src/virtualenv/activation/bash/__init__.py +++ b/src/virtualenv/activation/bash/__init__.py @@ -12,6 +12,14 @@ def templates(self): def as_name(self, template): return Path(template).stem + def replacements(self, creator, dest): + data = super().replacements(creator, dest) + data.update({ + "__TCL_LIBRARY__": getattr(creator.interpreter, "tcl_lib", None) or "", + "__TK_LIBRARY__": getattr(creator.interpreter, "tk_lib", None) or "", + }) + return data + __all__ = [ "BashActivator", diff --git a/src/virtualenv/activation/bash/activate.sh b/src/virtualenv/activation/bash/activate.sh index d3cf34784..b54358e9e 100644 --- a/src/virtualenv/activation/bash/activate.sh +++ b/src/virtualenv/activation/bash/activate.sh @@ -23,6 +23,17 @@ deactivate () { unset _OLD_VIRTUAL_PYTHONHOME fi + if ! [ -z "${_OLD_VIRTUAL_TCL_LIBRARY+_}" ]; then + TCL_LIBRARY="$_OLD_VIRTUAL_TCL_LIBRARY" + export TCL_LIBRARY + unset _OLD_VIRTUAL_TCL_LIBRARY + fi + if ! [ -z "${_OLD_VIRTUAL_TK_LIBRARY+_}" ]; then + TK_LIBRARY="$_OLD_VIRTUAL_TK_LIBRARY" + export TK_LIBRARY + unset _OLD_VIRTUAL_TK_LIBRARY + fi + # The hash command must be called to get it to forget past # commands. Without forgetting past commands the $PATH changes # we made may not be respected @@ -68,6 +79,22 @@ if ! [ -z "${PYTHONHOME+_}" ] ; then unset PYTHONHOME fi +if [ __TCL_LIBRARY__ != "" ]; then + if ! [ -z "${TCL_LIBRARY+_}" ] ; then + _OLD_VIRTUAL_TCL_LIBRARY="$TCL_LIBRARY" + fi + TCL_LIBRARY=__TCL_LIBRARY__ + export TCL_LIBRARY +fi + +if [ __TK_LIBRARY__ != "" ]; then + if ! [ -z "${TK_LIBRARY+_}" ] ; then + _OLD_VIRTUAL_TK_LIBRARY="$TK_LIBRARY" + fi + TK_LIBRARY=__TK_LIBRARY__ + export TK_LIBRARY +fi + if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then _OLD_VIRTUAL_PS1="${PS1-}" PS1="(${VIRTUAL_ENV_PROMPT}) ${PS1-}" diff --git a/src/virtualenv/activation/batch/activate.bat b/src/virtualenv/activation/batch/activate.bat index 3e34ee4db..62f393c80 100644 --- a/src/virtualenv/activation/batch/activate.bat +++ b/src/virtualenv/activation/batch/activate.bat @@ -1,8 +1,6 @@ @REM This file is UTF-8 encoded, so we need to update the current code page while executing it -@echo off -@for /f "tokens=2 delims=:." %%a in ('"%SystemRoot%\System32\chcp.com"') do ( - @set _OLD_CODEPAGE=%%a -) +@for /f "tokens=2 delims=:." %%a in ('"%SystemRoot%\System32\chcp.com"') do @set _OLD_CODEPAGE=%%a + @if defined _OLD_CODEPAGE ( "%SystemRoot%\System32\chcp.com" 65001 > nul ) @@ -35,6 +33,12 @@ @set PYTHONHOME= +@if defined TCL_LIBRARY @set "_OLD_VIRTUAL_TCL_LIBRARY=%TCL_LIBRARY%" +@if NOT "__TCL_LIBRARY__"=="" @set "TCL_LIBRARY=__TCL_LIBRARY__" + +@if defined TK_LIBRARY @set "_OLD_VIRTUAL_TK_LIBRARY=%TK_LIBRARY%" +@if NOT "__TK_LIBRARY__"=="" @set "TK_LIBRARY=__TK_LIBRARY__" + @REM if defined _OLD_VIRTUAL_PATH ( @if not defined _OLD_VIRTUAL_PATH @goto ENDIFVPATH1 @set "PATH=%_OLD_VIRTUAL_PATH%" diff --git a/src/virtualenv/activation/batch/deactivate.bat b/src/virtualenv/activation/batch/deactivate.bat index 8939c6c0d..7a12d47ed 100644 --- a/src/virtualenv/activation/batch/deactivate.bat +++ b/src/virtualenv/activation/batch/deactivate.bat @@ -12,6 +12,14 @@ @set _OLD_VIRTUAL_PYTHONHOME= :ENDIFVHOME +@if defined _OLD_VIRTUAL_TCL_LIBRARY @set "TCL_LIBRARY=%_OLD_VIRTUAL_TCL_LIBRARY%" +@if not defined _OLD_VIRTUAL_TCL_LIBRARY @set TCL_LIBRARY= +@set _OLD_VIRTUAL_TCL_LIBRARY= + +@if defined _OLD_VIRTUAL_TK_LIBRARY @set "TK_LIBRARY=%_OLD_VIRTUAL_TK_LIBRARY%" +@if not defined _OLD_VIRTUAL_TK_LIBRARY @set TK_LIBRARY= +@set _OLD_VIRTUAL_TK_LIBRARY= + @if not defined _OLD_VIRTUAL_PATH @goto ENDIFVPATH @set "PATH=%_OLD_VIRTUAL_PATH%" @set _OLD_VIRTUAL_PATH= diff --git a/src/virtualenv/activation/cshell/activate.csh b/src/virtualenv/activation/cshell/activate.csh index 24de5508b..5c02616d7 100644 --- a/src/virtualenv/activation/cshell/activate.csh +++ b/src/virtualenv/activation/cshell/activate.csh @@ -5,7 +5,7 @@ set newline='\ ' -alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH:q" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT:q" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; unsetenv VIRTUAL_ENV_PROMPT; test "\!:*" != "nondestructive" && unalias deactivate && unalias pydoc' +alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH:q" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_TCL_LIBRARY != 0 && setenv TCL_LIBRARY "$_OLD_VIRTUAL_TCL_LIBRARY:q" && unset _OLD_VIRTUAL_TCL_LIBRARY || unsetenv TCL_LIBRARY; test $?_OLD_VIRTUAL_TK_LIBRARY != 0 && setenv TK_LIBRARY "$_OLD_VIRTUAL_TK_LIBRARY:q" && unset _OLD_VIRTUAL_TK_LIBRARY || unsetenv TK_LIBRARY; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT:q" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; unsetenv VIRTUAL_ENV_PROMPT; test "\!:*" != "nondestructive" && unalias deactivate && unalias pydoc' # Unset irrelevant variables. deactivate nondestructive @@ -15,7 +15,19 @@ setenv VIRTUAL_ENV __VIRTUAL_ENV__ set _OLD_VIRTUAL_PATH="$PATH:q" setenv PATH "$VIRTUAL_ENV:q/"__BIN_NAME__":$PATH:q" +if (__TCL_LIBRARY__ != "") then + if ($?TCL_LIBRARY) then + set _OLD_VIRTUAL_TCL_LIBRARY="$TCL_LIBRARY" + endif + setenv TCL_LIBRARY __TCL_LIBRARY__ +endif +if (__TK_LIBRARY__ != "") then + if ($?TK_LIBRARY) then + set _OLD_VIRTUAL_TK_LIBRARY="$TK_LIBRARY" + endif + setenv TK_LIBRARY __TK_LIBRARY__ +endif if (__VIRTUAL_PROMPT__ != "") then setenv VIRTUAL_ENV_PROMPT __VIRTUAL_PROMPT__ diff --git a/src/virtualenv/activation/fish/__init__.py b/src/virtualenv/activation/fish/__init__.py index 57f790f47..26263566e 100644 --- a/src/virtualenv/activation/fish/__init__.py +++ b/src/virtualenv/activation/fish/__init__.py @@ -7,6 +7,14 @@ class FishActivator(ViaTemplateActivator): def templates(self): yield "activate.fish" + def replacements(self, creator, dest): + data = super().replacements(creator, dest) + data.update({ + "__TCL_LIBRARY__": getattr(creator.interpreter, "tcl_lib", None) or "", + "__TK_LIBRARY__": getattr(creator.interpreter, "tk_lib", None) or "", + }) + return data + __all__ = [ "FishActivator", diff --git a/src/virtualenv/activation/fish/activate.fish b/src/virtualenv/activation/fish/activate.fish index f3cd1f2ab..c9d174997 100644 --- a/src/virtualenv/activation/fish/activate.fish +++ b/src/virtualenv/activation/fish/activate.fish @@ -18,7 +18,7 @@ function deactivate -d 'Exit virtualenv mode and return to the normal environmen # reset old environment variables if test -n "$_OLD_VIRTUAL_PATH" # https://github.com/fish-shell/fish-shell/issues/436 altered PATH handling - if test (echo $FISH_VERSION | head -c 1) -lt 3 + if test (string sub -s 1 -l 1 $FISH_VERSION) -lt 3 set -gx PATH (_fishify_path "$_OLD_VIRTUAL_PATH") else set -gx PATH $_OLD_VIRTUAL_PATH @@ -26,6 +26,23 @@ function deactivate -d 'Exit virtualenv mode and return to the normal environmen set -e _OLD_VIRTUAL_PATH end + if test -n __TCL_LIBRARY__ + if test -n "$_OLD_VIRTUAL_TCL_LIBRARY"; + set -gx TCL_LIBRARY "$_OLD_VIRTUAL_TCL_LIBRARY"; + set -e _OLD_VIRTUAL_TCL_LIBRARY; + else; + set -e TCL_LIBRARY; + end + end + if test -n __TK_LIBRARY__ + if test -n "$_OLD_VIRTUAL_TK_LIBRARY"; + set -gx TK_LIBRARY "$_OLD_VIRTUAL_TK_LIBRARY"; + set -e _OLD_VIRTUAL_TK_LIBRARY; + else; + set -e TK_LIBRARY; + end + end + if test -n "$_OLD_VIRTUAL_PYTHONHOME" set -gx PYTHONHOME "$_OLD_VIRTUAL_PYTHONHOME" set -e _OLD_VIRTUAL_PYTHONHOME @@ -61,13 +78,26 @@ deactivate nondestructive set -gx VIRTUAL_ENV __VIRTUAL_ENV__ # https://github.com/fish-shell/fish-shell/issues/436 altered PATH handling -if test (echo $FISH_VERSION | head -c 1) -lt 3 +if test (string sub -s 1 -l 1 $FISH_VERSION) -lt 3 set -gx _OLD_VIRTUAL_PATH (_bashify_path $PATH) else set -gx _OLD_VIRTUAL_PATH $PATH end set -gx PATH "$VIRTUAL_ENV"'/'__BIN_NAME__ $PATH +if test -n __TCL_LIBRARY__ + if set -q TCL_LIBRARY; + set -gx _OLD_VIRTUAL_TCL_LIBRARY $TCL_LIBRARY; + end + set -gx TCL_LIBRARY '__TCL_LIBRARY__' +end +if test -n __TK_LIBRARY__ + if set -q TK_LIBRARY; + set -gx _OLD_VIRTUAL_TK_LIBRARY $TK_LIBRARY; + end + set -gx TK_LIBRARY '__TK_LIBRARY__' +end + # Prompt override provided? # If not, just use the environment name. if test -n __VIRTUAL_PROMPT__ diff --git a/src/virtualenv/activation/nushell/__init__.py b/src/virtualenv/activation/nushell/__init__.py index ef7a79a9c..d3b312497 100644 --- a/src/virtualenv/activation/nushell/__init__.py +++ b/src/virtualenv/activation/nushell/__init__.py @@ -12,6 +12,8 @@ def quote(string): """ Nushell supports raw strings like: r###'this is a string'###. + https://github.com/nushell/nushell.github.io/blob/main/book/working_with_strings.md + This method finds the maximum continuous sharps in the string and then quote it with an extra sharp. """ @@ -32,6 +34,8 @@ def replacements(self, creator, dest_folder): # noqa: ARG002 "__VIRTUAL_ENV__": str(creator.dest), "__VIRTUAL_NAME__": creator.env_name, "__BIN_NAME__": str(creator.bin_dir.relative_to(creator.dest)), + "__TCL_LIBRARY__": getattr(creator.interpreter, "tcl_lib", None) or "", + "__TK_LIBRARY__": getattr(creator.interpreter, "tk_lib", None) or "", } diff --git a/src/virtualenv/activation/nushell/activate.nu b/src/virtualenv/activation/nushell/activate.nu index 4da1c8c07..b48fdd03f 100644 --- a/src/virtualenv/activation/nushell/activate.nu +++ b/src/virtualenv/activation/nushell/activate.nu @@ -1,94 +1,85 @@ -# virtualenv activation module -# Activate with `overlay use activate.nu` -# Deactivate with `deactivate`, as usual +# virtualenv activation module: +# - Activate with `overlay use activate.nu` +# - Deactivate with `deactivate`, as usual # -# To customize the overlay name, you can call `overlay use activate.nu as foo`, -# but then simply `deactivate` won't work because it is just an alias to hide -# the "activate" overlay. You'd need to call `overlay hide foo` manually. +# To customize the overlay name, you can call `overlay use activate.nu as foo`, but then simply `deactivate` won't work +# because it is just an alias to hide the "activate" overlay. You'd need to call `overlay hide foo` manually. + +module warning { + export-env { + const file = path self + error make -u { + msg: $"`($file | path basename)` is meant to be used with `overlay use`, not `source`" + } + } + +} + +use warning export-env { + + let nu_ver = (version | get version | split row '.' | take 2 | each { into int }) + if $nu_ver.0 == 0 and $nu_ver.1 < 106 { + error make { + msg: 'virtualenv Nushell activation requires Nushell 0.106 or greater.' + } + } + def is-string [x] { ($x | describe) == 'string' } def has-env [...names] { - $names | each {|n| - $n in $env - } | all {|i| $i == true} + $names | each {|n| $n in $env } | all {|i| $i } } - # Emulates a `test -z`, but better as it handles e.g 'false' def is-env-true [name: string] { - if (has-env $name) { - # Try to parse 'true', '0', '1', and fail if not convertible - let parsed = (do -i { $env | get $name | into bool }) - if ($parsed | describe) == 'bool' { - $parsed + if (has-env $name) { + let val = ($env | get --optional $name) + if ($val | describe) == 'bool' { + $val + } else { + not ($val | is-empty) + } } else { - not ($env | get -i $name | is-empty) + false } - } else { - false - } } let virtual_env = __VIRTUAL_ENV__ let bin = __BIN_NAME__ - - let is_windows = ($nu.os-info.family) == 'windows' - let path_name = (if (has-env 'Path') { - 'Path' - } else { - 'PATH' - } - ) - + let path_name = if (has-env 'Path') { 'Path' } else { 'PATH' } let venv_path = ([$virtual_env $bin] | path join) let new_path = ($env | get $path_name | prepend $venv_path) - - # If there is no default prompt, then use the env name instead - let virtual_env_prompt = (if (__VIRTUAL_PROMPT__ | is-empty) { + let virtual_env_prompt = if (__VIRTUAL_PROMPT__ | is-empty) { ($virtual_env | path basename) } else { __VIRTUAL_PROMPT__ - }) - - let new_env = { - $path_name : $new_path - VIRTUAL_ENV : $virtual_env - VIRTUAL_ENV_PROMPT : $virtual_env_prompt } - - let new_env = (if (is-env-true 'VIRTUAL_ENV_DISABLE_PROMPT') { - $new_env + let new_env = { $path_name: $new_path VIRTUAL_ENV: $virtual_env VIRTUAL_ENV_PROMPT: $virtual_env_prompt } + if (has-env 'TCL_LIBRARY') { + let $new_env = $new_env | insert TCL_LIBRARY __TCL_LIBRARY__ + } + if (has-env 'TK_LIBRARY') { + let $new_env = $new_env | insert TK_LIBRARY __TK_LIBRARY__ + } + let old_prompt_command = if (has-env 'PROMPT_COMMAND') { $env.PROMPT_COMMAND } else { '' } + let new_env = if (is-env-true 'VIRTUAL_ENV_DISABLE_PROMPT') { + $new_env } else { - # Creating the new prompt for the session - let virtual_prefix = $'(char lparen)($virtual_env_prompt)(char rparen) ' - - # Back up the old prompt builder - let old_prompt_command = (if (has-env 'PROMPT_COMMAND') { - $env.PROMPT_COMMAND - } else { - '' - }) - - let new_prompt = (if (has-env 'PROMPT_COMMAND') { - if 'closure' in ($old_prompt_command | describe) { - {|| $'($virtual_prefix)(do $old_prompt_command)' } - } else { - {|| $'($virtual_prefix)($old_prompt_command)' } - } - } else { - {|| $'($virtual_prefix)' } - }) - - $new_env | merge { - PROMPT_COMMAND : $new_prompt - VIRTUAL_PREFIX : $virtual_prefix - } - }) - - # Environment variables that will be loaded as the virtual env + let virtual_prefix = $'(char lparen)($virtual_env_prompt)(char rparen) ' + let new_prompt = if (has-env 'PROMPT_COMMAND') { + if ('closure' in ($old_prompt_command | describe)) { + {|| $'($virtual_prefix)(do $old_prompt_command)' } + } else { + {|| $'($virtual_prefix)($old_prompt_command)' } + } + } else { + {|| $'($virtual_prefix)' } + } + $new_env | merge { PROMPT_COMMAND: $new_prompt VIRTUAL_PREFIX: $virtual_prefix } + } load-env $new_env } diff --git a/src/virtualenv/activation/powershell/activate.ps1 b/src/virtualenv/activation/powershell/activate.ps1 index bd30e2eed..9f95e4370 100644 --- a/src/virtualenv/activation/powershell/activate.ps1 +++ b/src/virtualenv/activation/powershell/activate.ps1 @@ -7,6 +7,24 @@ function global:deactivate([switch] $NonDestructive) { Remove-Variable "_OLD_VIRTUAL_PATH" -Scope global } + if (Test-Path variable:_OLD_VIRTUAL_TCL_LIBRARY) { + $env:TCL_LIBRARY = $variable:_OLD_VIRTUAL_TCL_LIBRARY + Remove-Variable "_OLD_VIRTUAL_TCL_LIBRARY" -Scope global + } else { + if (Test-Path env:TCL_LIBRARY) { + Remove-Item env:TCL_LIBRARY -ErrorAction SilentlyContinue + } + } + + if (Test-Path variable:_OLD_VIRTUAL_TK_LIBRARY) { + $env:TK_LIBRARY = $variable:_OLD_VIRTUAL_TK_LIBRARY + Remove-Variable "_OLD_VIRTUAL_TK_LIBRARY" -Scope global + } else { + if (Test-Path env:TK_LIBRARY) { + Remove-Item env:TK_LIBRARY -ErrorAction SilentlyContinue + } + } + if (Test-Path function:_old_virtual_prompt) { $function:prompt = $function:_old_virtual_prompt Remove-Item function:\_old_virtual_prompt @@ -44,6 +62,20 @@ else { $env:VIRTUAL_ENV_PROMPT = $( Split-Path $env:VIRTUAL_ENV -Leaf ) } +if (__TCL_LIBRARY__ -ne "") { + if (Test-Path env:TCL_LIBRARY) { + New-Variable -Scope global -Name _OLD_VIRTUAL_TCL_LIBRARY -Value $env:TCL_LIBRARY + } + $env:TCL_LIBRARY = __TCL_LIBRARY__ +} + +if (__TK_LIBRARY__ -ne "") { + if (Test-Path env:TK_LIBRARY) { + New-Variable -Scope global -Name _OLD_VIRTUAL_TK_LIBRARY -Value $env:TK_LIBRARY + } + $env:TK_LIBRARY = __TK_LIBRARY__ +} + New-Variable -Scope global -Name _OLD_VIRTUAL_PATH -Value $env:PATH $env:PATH = "$env:VIRTUAL_ENV/" + __BIN_NAME__ + __PATH_SEP__ + $env:PATH diff --git a/src/virtualenv/activation/via_template.py b/src/virtualenv/activation/via_template.py index 6fa4474d2..85f932605 100644 --- a/src/virtualenv/activation/via_template.py +++ b/src/virtualenv/activation/via_template.py @@ -47,6 +47,8 @@ def replacements(self, creator, dest_folder): # noqa: ARG002 "__VIRTUAL_NAME__": creator.env_name, "__BIN_NAME__": str(creator.bin_dir.relative_to(creator.dest)), "__PATH_SEP__": os.pathsep, + "__TCL_LIBRARY__": getattr(creator.interpreter, "tcl_lib", None) or "", + "__TK_LIBRARY__": getattr(creator.interpreter, "tk_lib", None) or "", } def _generate(self, replacements, templates, to_folder, creator): diff --git a/src/virtualenv/app_data/__init__.py b/src/virtualenv/app_data/__init__.py index 148c94183..d7f148023 100644 --- a/src/virtualenv/app_data/__init__.py +++ b/src/virtualenv/app_data/__init__.py @@ -12,6 +12,8 @@ from .via_disk_folder import AppDataDiskFolder from .via_tempdir import TempAppData +LOGGER = logging.getLogger(__name__) + def _default_app_data_dir(env): key = "VIRTUALENV_OVERRIDE_APP_DATA" @@ -37,13 +39,13 @@ def make_app_data(folder, **kwargs): if not os.path.isdir(folder): try: os.makedirs(folder) - logging.debug("created app data folder %s", folder) + LOGGER.debug("created app data folder %s", folder) except OSError as exception: - logging.info("could not create app data folder %s due to %r", folder, exception) + LOGGER.info("could not create app data folder %s due to %r", folder, exception) if os.access(folder, os.W_OK): return AppDataDiskFolder(folder) - logging.debug("app data folder %s has no write access", folder) + LOGGER.debug("app data folder %s has no write access", folder) return TempAppData() diff --git a/src/virtualenv/app_data/via_disk_folder.py b/src/virtualenv/app_data/via_disk_folder.py index 5228e49a8..9ebe91c2e 100644 --- a/src/virtualenv/app_data/via_disk_folder.py +++ b/src/virtualenv/app_data/via_disk_folder.py @@ -37,6 +37,8 @@ from .base import AppData, ContentStore +LOGGER = logging.getLogger(__name__) + class AppDataDiskFolder(AppData): """Store the application data on the disk within a folder layout.""" @@ -54,7 +56,7 @@ def __str__(self) -> str: return str(self.lock.path) def reset(self): - logging.debug("reset app data folder %s", self.lock.path) + LOGGER.debug("reset app data folder %s", self.lock.path) safe_delete(self.lock.path) def close(self): @@ -77,7 +79,7 @@ def extract(self, path, to_folder): @property def py_info_at(self): - return self.lock / "py_info" / "1" + return self.lock / "py_info" / "2" def py_info(self, path): return PyInfoStoreDisk(self.py_info_at, path) @@ -106,10 +108,9 @@ def wheel_image(self, for_py_version, name): class JSONStoreDisk(ContentStore, ABC): - def __init__(self, in_folder, key, msg, msg_args) -> None: + def __init__(self, in_folder, key, msg_args) -> None: self.in_folder = in_folder self.key = key - self.msg = msg self.msg_args = (*msg_args, self.file) @property @@ -128,7 +129,7 @@ def read(self): except Exception: # noqa: BLE001, S110 pass else: - logging.debug("got %s from %s", self.msg, self.msg_args) + LOGGER.debug("got %s %s from %s", *self.msg_args) return data if bad_format: with suppress(OSError): # reading and writing on the same file may cause race on multiple processes @@ -137,7 +138,7 @@ def read(self): def remove(self): self.file.unlink() - logging.debug("removed %s at %s", self.msg, self.msg_args) + LOGGER.debug("removed %s %s at %s", *self.msg_args) @contextmanager def locked(self): @@ -148,13 +149,13 @@ def write(self, content): folder = self.file.parent folder.mkdir(parents=True, exist_ok=True) self.file.write_text(json.dumps(content, sort_keys=True, indent=2), encoding="utf-8") - logging.debug("wrote %s at %s", self.msg, self.msg_args) + LOGGER.debug("wrote %s %s at %s", *self.msg_args) class PyInfoStoreDisk(JSONStoreDisk): def __init__(self, in_folder, path) -> None: key = sha256(str(path).encode("utf-8")).hexdigest() - super().__init__(in_folder, key, "python info of %s", (path,)) + super().__init__(in_folder, key, ("python info of", path)) class EmbedDistributionUpdateStoreDisk(JSONStoreDisk): @@ -162,8 +163,7 @@ def __init__(self, in_folder, distribution) -> None: super().__init__( in_folder, distribution, - "embed update of distribution %s", - (distribution,), + ("embed update of distribution", distribution), ) diff --git a/src/virtualenv/app_data/via_tempdir.py b/src/virtualenv/app_data/via_tempdir.py index 0a30dfe1c..884a570ce 100644 --- a/src/virtualenv/app_data/via_tempdir.py +++ b/src/virtualenv/app_data/via_tempdir.py @@ -7,6 +7,8 @@ from .via_disk_folder import AppDataDiskFolder +LOGGER = logging.getLogger(__name__) + class TempAppData(AppDataDiskFolder): transient = True @@ -14,13 +16,13 @@ class TempAppData(AppDataDiskFolder): def __init__(self) -> None: super().__init__(folder=mkdtemp()) - logging.debug("created temporary app data folder %s", self.lock.path) + LOGGER.debug("created temporary app data folder %s", self.lock.path) def reset(self): """This is a temporary folder, is already empty to start with.""" def close(self): - logging.debug("remove temporary app data folder %s", self.lock.path) + LOGGER.debug("remove temporary app data folder %s", self.lock.path) safe_delete(self.lock.path) def embed_update_log(self, distribution, for_py_version): diff --git a/src/virtualenv/config/cli/parser.py b/src/virtualenv/config/cli/parser.py index a1770f2ac..fd1dc0956 100644 --- a/src/virtualenv/config/cli/parser.py +++ b/src/virtualenv/config/cli/parser.py @@ -1,4 +1,4 @@ -from __future__ import annotations # noqa: A005 +from __future__ import annotations import os from argparse import SUPPRESS, ArgumentDefaultsHelpFormatter, ArgumentParser, Namespace @@ -107,8 +107,8 @@ def parse_known_args(self, args=None, namespace=None): class HelpFormatter(ArgumentDefaultsHelpFormatter): - def __init__(self, prog) -> None: - super().__init__(prog, max_help_position=32, width=240) + def __init__(self, prog, **kwargs) -> None: + super().__init__(prog, max_help_position=32, width=240, **kwargs) def _get_help_string(self, action): text = super()._get_help_string(action) diff --git a/src/virtualenv/config/convert.py b/src/virtualenv/config/convert.py index ecd9d2b5b..ef7581dbd 100644 --- a/src/virtualenv/config/convert.py +++ b/src/virtualenv/config/convert.py @@ -4,6 +4,8 @@ import os from typing import ClassVar +LOGGER = logging.getLogger(__name__) + class TypeData: def __init__(self, default_type, as_type) -> None: @@ -81,7 +83,7 @@ def convert(value, as_type, source): try: return as_type.convert(value) except Exception as exception: - logging.warning("%s failed to convert %r as %r because %r", source, value, as_type, exception) + LOGGER.warning("%s failed to convert %r as %r because %r", source, value, as_type, exception) raise diff --git a/src/virtualenv/config/ini.py b/src/virtualenv/config/ini.py index cd6ecf504..ed0a1b930 100644 --- a/src/virtualenv/config/ini.py +++ b/src/virtualenv/config/ini.py @@ -10,6 +10,8 @@ from .convert import convert +LOGGER = logging.getLogger(__name__) + class IniConfig: VIRTUALENV_CONFIG_FILE_ENV_VAR: ClassVar[str] = "VIRTUALENV_CONFIG_FILE" @@ -44,7 +46,7 @@ def __init__(self, env=None) -> None: except Exception as exc: # noqa: BLE001 exception = exc if exception is not None: - logging.error("failed to read config file %s because %r", config_file, exception) + LOGGER.error("failed to read config file %s because %r", config_file, exception) def _load(self): with self.config_file.open("rt", encoding="utf-8") as file_handler: diff --git a/src/virtualenv/create/creator.py b/src/virtualenv/create/creator.py index 7a98a5124..1e577ada5 100644 --- a/src/virtualenv/create/creator.py +++ b/src/virtualenv/create/creator.py @@ -4,6 +4,7 @@ import logging import os import sys +import textwrap from abc import ABC, abstractmethod from argparse import ArgumentTypeError from ast import literal_eval @@ -19,6 +20,7 @@ HERE = Path(os.path.abspath(__file__)).parent DEBUG_SCRIPT = HERE / "debug.py" +LOGGER = logging.getLogger(__name__) class CreatorMeta: @@ -153,13 +155,26 @@ def non_write_able(dest, value): def run(self): if self.dest.exists() and self.clear: - logging.debug("delete %s", self.dest) + LOGGER.debug("delete %s", self.dest) safe_delete(self.dest) self.create() + self.add_cachedir_tag() self.set_pyenv_cfg() if not self.no_vcs_ignore: self.setup_ignore_vcs() + def add_cachedir_tag(self): + """Generate a file indicating that this is not meant to be backed up.""" + cachedir_tag_file = self.dest / "CACHEDIR.TAG" + if not cachedir_tag_file.exists(): + cachedir_tag_text = textwrap.dedent(""" + Signature: 8a477f597d28d172789f06886806bc55 + # This file is a cache directory tag created by Python virtualenv. + # For information about cache directory tags, see: + # https://bford.info/cachedir/ + """).strip() + cachedir_tag_file.write_text(cachedir_tag_text, encoding="utf-8") + def set_pyenv_cfg(self): self.pyenv_cfg.content = OrderedDict() self.pyenv_cfg["home"] = os.path.dirname(os.path.abspath(self.interpreter.system_executable)) @@ -197,7 +212,7 @@ def get_env_debug_info(env_exe, debug_script, app_data, env): with app_data.ensure_extracted(debug_script) as debug_script_extracted: cmd = [str(env_exe), str(debug_script_extracted)] - logging.debug("debug via %r", LogCmd(cmd)) + LOGGER.debug("debug via %r", LogCmd(cmd)) code, out, err = run_cmd(cmd) try: diff --git a/src/virtualenv/create/pyenv_cfg.py b/src/virtualenv/create/pyenv_cfg.py index 04883dea2..1d1beacff 100644 --- a/src/virtualenv/create/pyenv_cfg.py +++ b/src/virtualenv/create/pyenv_cfg.py @@ -4,6 +4,8 @@ import os from collections import OrderedDict +LOGGER = logging.getLogger(__name__) + class PyEnvCfg: def __init__(self, content, path) -> None: @@ -30,12 +32,12 @@ def _read_values(path): return content def write(self): - logging.debug("write %s", self.path) + LOGGER.debug("write %s", self.path) text = "" for key, value in self.content.items(): normalized_value = os.path.realpath(value) if value and os.path.exists(value) else value line = f"{key} = {normalized_value}" - logging.debug("\t%s", line) + LOGGER.debug("\t%s", line) text += line text += "\n" self.path.write_text(text, encoding="utf-8") diff --git a/src/virtualenv/create/via_global_ref/_virtualenv.py b/src/virtualenv/create/via_global_ref/_virtualenv.py index b61db3079..0d95b28e0 100644 --- a/src/virtualenv/create/via_global_ref/_virtualenv.py +++ b/src/virtualenv/create/via_global_ref/_virtualenv.py @@ -2,10 +2,11 @@ from __future__ import annotations +import contextlib import os import sys -VIRTUALENV_PATCH_FILE = os.path.join(__file__) +VIRTUALENV_PATCH_FILE = os.path.abspath(__file__) def patch_dist(dist): @@ -50,7 +51,14 @@ class _Finder: lock = [] # noqa: RUF012 def find_spec(self, fullname, path, target=None): # noqa: ARG002 - if fullname in _DISTUTILS_PATCH and self.fullname is None: # noqa: PLR1702 + # Guard against race conditions during file rewrite by checking if _DISTUTILS_PATCH is defined. + # This can happen when the file is being overwritten while it's being imported by another process. + # See https://github.com/pypa/virtualenv/issues/2969 for details. + try: + distutils_patch = _DISTUTILS_PATCH + except NameError: + return None + if fullname in distutils_patch and self.fullname is None: # noqa: PLR1702 # initialize lock[0] lazily if len(self.lock) == 0: import threading # noqa: PLC0415 @@ -89,14 +97,26 @@ def find_spec(self, fullname, path, target=None): # noqa: ARG002 @staticmethod def exec_module(old, module): old(module) - if module.__name__ in _DISTUTILS_PATCH: - patch_dist(module) + try: + distutils_patch = _DISTUTILS_PATCH + except NameError: + return + if module.__name__ in distutils_patch: + # patch_dist or its dependencies may not be defined during file rewrite + with contextlib.suppress(NameError): + patch_dist(module) @staticmethod def load_module(old, name): module = old(name) - if module.__name__ in _DISTUTILS_PATCH: - patch_dist(module) + try: + distutils_patch = _DISTUTILS_PATCH + except NameError: + return module + if module.__name__ in distutils_patch: + # patch_dist or its dependencies may not be defined during file rewrite + with contextlib.suppress(NameError): + patch_dist(module) return module diff --git a/src/virtualenv/create/via_global_ref/api.py b/src/virtualenv/create/via_global_ref/api.py index d29067c8b..3f04f4653 100644 --- a/src/virtualenv/create/via_global_ref/api.py +++ b/src/virtualenv/create/via_global_ref/api.py @@ -8,6 +8,8 @@ from virtualenv.create.creator import Creator, CreatorMeta from virtualenv.info import fs_supports_symlink +LOGGER = logging.getLogger(__name__) + class ViaGlobalRefMeta(CreatorMeta): def __init__(self) -> None: @@ -60,7 +62,12 @@ def add_parser_arguments(cls, parser, interpreter, meta, app_data): help="give the virtual environment access to the system site-packages dir", ) if not meta.can_symlink and not meta.can_copy: - msg = "neither symlink or copy method supported" + errors = [] + if meta.symlink_error: + errors.append(f"symlink: {meta.symlink_error}") + if meta.copy_error: + errors.append(f"copy: {meta.copy_error}") + msg = f"neither symlink or copy method supported: {', '.join(errors)}" raise RuntimeError(msg) group = parser.add_mutually_exclusive_group() if meta.can_symlink: @@ -88,10 +95,10 @@ def install_patch(self): text = self.env_patch_text() if text: pth = self.purelib / "_virtualenv.pth" - logging.debug("create virtualenv import hook file %s", pth) + LOGGER.debug("create virtualenv import hook file %s", pth) pth.write_text("import _virtualenv", encoding="utf-8") dest_path = self.purelib / "_virtualenv.py" - logging.debug("create %s", dest_path) + LOGGER.debug("create %s", dest_path) dest_path.write_text(text, encoding="utf-8") def env_patch_text(self): diff --git a/src/virtualenv/create/via_global_ref/builtin/cpython/common.py b/src/virtualenv/create/via_global_ref/builtin/cpython/common.py index 7c2a04a32..5cc993bda 100644 --- a/src/virtualenv/create/via_global_ref/builtin/cpython/common.py +++ b/src/virtualenv/create/via_global_ref/builtin/cpython/common.py @@ -38,7 +38,10 @@ def _executables(cls, interpreter): # - https://bugs.python.org/issue42013 # - venv host = cls.host_python(interpreter) - for path in (host.parent / n for n in {"python.exe", host.name}): + names = {"python.exe", host.name} + if interpreter.version_info.major == 3: # noqa: PLR2004 + names.update({"python3.exe", "python3"}) + for path in (host.parent / n for n in names): yield host, [path.name], RefMust.COPY, RefWhen.ANY # for more info on pythonw.exe see https://stackoverflow.com/a/30313091 python_w = host.parent / "pythonw.exe" diff --git a/src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py b/src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py index daa474103..a46da45dd 100644 --- a/src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py +++ b/src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py @@ -8,7 +8,7 @@ from textwrap import dedent from virtualenv.create.describe import Python3Supports -from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest +from virtualenv.create.via_global_ref.builtin.ref import ExePathRefToDest, PathRefToDest from virtualenv.create.via_global_ref.store import is_store_python from .common import CPython, CPythonPosix, CPythonWindows, is_mac_os_framework, is_macos_brew @@ -69,7 +69,35 @@ def sources(cls, interpreter): @classmethod def executables(cls, interpreter): - return super().sources(interpreter) + sources = super().sources(interpreter) + if interpreter.version_info >= (3, 13): + # Create new refs with corrected launcher paths + updated_sources = [] + for ref in sources: + if ref.src.name == "python.exe": + launcher_path = ref.src.with_name("venvlauncher.exe") + if launcher_path.exists(): + new_ref = ExePathRefToDest( + launcher_path, dest=ref.dest, targets=[ref.base, *ref.aliases], must=ref.must, when=ref.when + ) + updated_sources.append(new_ref) + continue + elif ref.src.name == "pythonw.exe": + w_launcher_path = ref.src.with_name("venvwlauncher.exe") + if w_launcher_path.exists(): + new_ref = ExePathRefToDest( + w_launcher_path, + dest=ref.dest, + targets=[ref.base, *ref.aliases], + must=ref.must, + when=ref.when, + ) + updated_sources.append(new_ref) + continue + # Keep the original ref unchanged + updated_sources.append(ref) + return updated_sources + return sources @classmethod def has_shim(cls, interpreter): @@ -77,7 +105,11 @@ def has_shim(cls, interpreter): @classmethod def shim(cls, interpreter): - shim = Path(interpreter.system_stdlib) / "venv" / "scripts" / "nt" / "python.exe" + root = Path(interpreter.system_stdlib) / "venv" / "scripts" / "nt" + # Before 3.13 the launcher was called python.exe, after is venvlauncher.exe + # https://github.com/python/cpython/issues/112984 + exe_name = "venvlauncher.exe" if interpreter.version_info >= (3, 13) else "python.exe" + shim = root / exe_name if shim.exists(): return shim return None diff --git a/src/virtualenv/create/via_global_ref/builtin/cpython/mac_os.py b/src/virtualenv/create/via_global_ref/builtin/cpython/mac_os.py index d0ffa8f00..0ddbf9a33 100644 --- a/src/virtualenv/create/via_global_ref/builtin/cpython/mac_os.py +++ b/src/virtualenv/create/via_global_ref/builtin/cpython/mac_os.py @@ -20,6 +20,8 @@ from .common import CPython, CPythonPosix, is_mac_os_framework, is_macos_brew from .cpython3 import CPython3 +LOGGER = logging.getLogger(__name__) + class CPythonmacOsFramework(CPython, ABC): @classmethod @@ -115,10 +117,10 @@ def fix_mach_o(exe, current, new, max_size): unneeded bits of information, however Mac OS X 10.5 and earlier cannot read this new Link Edit table format. """ try: - logging.debug("change Mach-O for %s from %s to %s", exe, current, new) + LOGGER.debug("change Mach-O for %s from %s to %s", exe, current, new) _builtin_change_mach_o(max_size)(exe, current, new) except Exception as e: # noqa: BLE001 - logging.warning("Could not call _builtin_change_mac_o: %s. Trying to call install_name_tool instead.", e) + LOGGER.warning("Could not call _builtin_change_mac_o: %s. Trying to call install_name_tool instead.", e) try: cmd = ["install_name_tool", "-change", current, new, exe] subprocess.check_call(cmd) diff --git a/src/virtualenv/create/via_global_ref/builtin/graalpy/__init__.py b/src/virtualenv/create/via_global_ref/builtin/graalpy/__init__.py new file mode 100644 index 000000000..8bfe887e7 --- /dev/null +++ b/src/virtualenv/create/via_global_ref/builtin/graalpy/__init__.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +from abc import ABC +from pathlib import Path + +from virtualenv.create.describe import PosixSupports, WindowsSupports +from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest, RefMust, RefWhen +from virtualenv.create.via_global_ref.builtin.via_global_self_do import ViaGlobalRefVirtualenvBuiltin + + +class GraalPy(ViaGlobalRefVirtualenvBuiltin, ABC): + @classmethod + def can_describe(cls, interpreter): + return interpreter.implementation == "GraalVM" and super().can_describe(interpreter) + + @classmethod + def exe_stem(cls): + return "graalpy" + + @classmethod + def exe_names(cls, interpreter): + return { + cls.exe_stem(), + "python", + f"python{interpreter.version_info.major}", + f"python{interpreter.version_info.major}.{interpreter.version_info.minor}", + } + + @classmethod + def _executables(cls, interpreter): + host = Path(interpreter.system_executable) + targets = sorted(f"{name}{cls.suffix}" for name in cls.exe_names(interpreter)) + yield host, targets, RefMust.NA, RefWhen.ANY + + @classmethod + def sources(cls, interpreter): + yield from super().sources(interpreter) + python_dir = Path(interpreter.system_executable).resolve().parent + if python_dir.name in {"bin", "Scripts"}: + python_dir = python_dir.parent + + native_lib = cls._native_lib(python_dir / "lib", interpreter.platform) + if native_lib.exists(): + yield PathRefToDest(native_lib, dest=lambda self, s: self.bin_dir.parent / "lib" / s.name) + + for jvm_dir_name in ("jvm", "jvmlibs", "modules"): + jvm_dir = python_dir / jvm_dir_name + if jvm_dir.exists(): + yield PathRefToDest(jvm_dir, dest=lambda self, s: self.bin_dir.parent / s.name) + + @classmethod + def _shared_libs(cls, python_dir): + raise NotImplementedError + + def set_pyenv_cfg(self): + super().set_pyenv_cfg() + # GraalPy 24.0 and older had home without the bin + version = self.interpreter.version_info + if version.major == 3 and version.minor <= 10: # noqa: PLR2004 + home = Path(self.pyenv_cfg["home"]) + if home.name == "bin": + self.pyenv_cfg["home"] = str(home.parent) + + +class GraalPyPosix(GraalPy, PosixSupports): + @classmethod + def _native_lib(cls, lib_dir, platform): + if platform == "darwin": + return lib_dir / "libpythonvm.dylib" + return lib_dir / "libpythonvm.so" + + +class GraalPyWindows(GraalPy, WindowsSupports): + @classmethod + def _native_lib(cls, lib_dir, _platform): + return lib_dir / "pythonvm.dll" + + def set_pyenv_cfg(self): + # GraalPy needs an additional entry in pyvenv.cfg on Windows + super().set_pyenv_cfg() + self.pyenv_cfg["venvlauncher_command"] = self.interpreter.system_executable + + +__all__ = [ + "GraalPyPosix", + "GraalPyWindows", +] diff --git a/src/virtualenv/create/via_global_ref/venv.py b/src/virtualenv/create/via_global_ref/venv.py index e1c9f64d0..d5037bc3b 100644 --- a/src/virtualenv/create/via_global_ref/venv.py +++ b/src/virtualenv/create/via_global_ref/venv.py @@ -1,4 +1,4 @@ -from __future__ import annotations # noqa: A005 +from __future__ import annotations import logging from copy import copy @@ -13,6 +13,8 @@ from .builtin.cpython.mac_os import CPython3macOsBrew from .builtin.pypy.pypy3 import Pypy3Windows +LOGGER = logging.getLogger(__name__) + class Venv(ViaGlobalRefApi): def __init__(self, options, interpreter) -> None: @@ -70,7 +72,7 @@ def create_inline(self): def create_via_sub_process(self): cmd = self.get_host_create_cmd() - logging.info("using host built-in venv to create via %s", " ".join(cmd)) + LOGGER.info("using host built-in venv to create via %s", " ".join(cmd)) code, out, err = run_cmd(cmd) if code != 0: raise ProcessCallFailedError(code, out, err, cmd) diff --git a/src/virtualenv/discovery/builtin.py b/src/virtualenv/discovery/builtin.py index d2f1cf433..e2d193911 100644 --- a/src/virtualenv/discovery/builtin.py +++ b/src/virtualenv/discovery/builtin.py @@ -5,7 +5,9 @@ import sys from contextlib import suppress from pathlib import Path -from typing import TYPE_CHECKING, Callable +from typing import TYPE_CHECKING + +from platformdirs import user_data_path from virtualenv.info import IS_WIN, fs_path_id @@ -15,9 +17,10 @@ if TYPE_CHECKING: from argparse import ArgumentParser - from collections.abc import Generator, Iterable, Mapping, Sequence + from collections.abc import Callable, Generator, Iterable, Mapping, Sequence from virtualenv.app_data.base import AppData +LOGGER = logging.getLogger(__name__) class Builtin(Discover): @@ -28,6 +31,8 @@ class Builtin(Discover): def __init__(self, options) -> None: super().__init__(options) self.python_spec = options.python or [sys.executable] + if self._env.get("VIRTUALENV_PYTHON"): + self.python_spec = self.python_spec[1:] + self.python_spec[:1] # Rotate the list self.app_data = options.app_data self.try_first_with = options.try_first_with @@ -70,16 +75,16 @@ def get_interpreter( key, try_first_with: Iterable[str], app_data: AppData | None = None, env: Mapping[str, str] | None = None ) -> PythonInfo | None: spec = PythonSpec.from_string_spec(key) - logging.info("find interpreter for spec %r", spec) + LOGGER.info("find interpreter for spec %r", spec) proposed_paths = set() env = os.environ if env is None else env for interpreter, impl_must_match in propose_interpreters(spec, try_first_with, app_data, env): key = interpreter.system_executable, impl_must_match if key in proposed_paths: continue - logging.info("proposed %s", interpreter) + LOGGER.info("proposed %s", interpreter) if interpreter.satisfies(spec, impl_must_match): - logging.debug("accepted %s", interpreter) + LOGGER.debug("accepted %s", interpreter) return interpreter proposed_paths.add(key) return None @@ -91,9 +96,23 @@ def propose_interpreters( # noqa: C901, PLR0912, PLR0915 app_data: AppData | None = None, env: Mapping[str, str] | None = None, ) -> Generator[tuple[PythonInfo, bool], None, None]: - # 0. try with first + # 0. if it's a path and exists, and is absolute path, this is the only option we consider env = os.environ if env is None else env tested_exes: set[str] = set() + if spec.is_abs: + try: + os.lstat(spec.path) # Windows Store Python does not work with os.path.exists, but does for os.lstat + except OSError: + pass + else: + exe_raw = os.path.abspath(spec.path) + exe_id = fs_path_id(exe_raw) + if exe_id not in tested_exes: + tested_exes.add(exe_id) + yield PythonInfo.from_exe(exe_raw, app_data, env=env), True + return + + # 1. try with first for py_exe in try_first_with: path = os.path.abspath(py_exe) try: @@ -113,8 +132,7 @@ def propose_interpreters( # noqa: C901, PLR0912, PLR0915 try: os.lstat(spec.path) # Windows Store Python does not work with os.path.exists, but does for os.lstat except OSError: - if spec.is_abs: - raise + pass else: exe_raw = os.path.abspath(spec.path) exe_id = fs_path_id(exe_raw) @@ -143,10 +161,11 @@ def propose_interpreters( # noqa: C901, PLR0912, PLR0915 continue tested_exes.add(exe_id) yield interpreter, True - # finally just find on path, the path order matters (as the candidates are less easy to control by end user) + + # try to find on path, the path order matters (as the candidates are less easy to control by end user) find_candidates = path_exe_finder(spec) for pos, path in enumerate(get_paths(env)): - logging.debug(LazyPathDump(pos, path, env)) + LOGGER.debug(LazyPathDump(pos, path, env)) for exe, impl_must_match in find_candidates(path): exe_raw = str(exe) exe_id = fs_path_id(exe_raw) @@ -157,6 +176,19 @@ def propose_interpreters( # noqa: C901, PLR0912, PLR0915 if interpreter is not None: yield interpreter, impl_must_match + # otherwise try uv-managed python (~/.local/share/uv/python or platform equivalent) + if uv_python_dir := os.getenv("UV_PYTHON_INSTALL_DIR"): + uv_python_path = Path(uv_python_dir).expanduser() + elif xdg_data_home := os.getenv("XDG_DATA_HOME"): + uv_python_path = Path(xdg_data_home).expanduser() / "uv" / "python" + else: + uv_python_path = user_data_path("uv") / "python" + + for exe_path in uv_python_path.glob("*/bin/python"): + interpreter = PathPythonInfo.from_exe(str(exe_path), app_data, raise_on_error=False, env=env) + if interpreter is not None: + yield interpreter, True + def get_paths(env: Mapping[str, str]) -> Generator[Path, None, None]: path = env.get("PATH", None) @@ -168,7 +200,7 @@ def get_paths(env: Mapping[str, str]) -> Generator[Path, None, None]: if path: for p in map(Path, path.split(os.pathsep)): with suppress(OSError): - if p.exists(): + if p.is_dir() and next(p.iterdir(), None): yield p @@ -184,7 +216,13 @@ def __repr__(self) -> str: content += " with =>" for file_path in self.path.iterdir(): try: - if file_path.is_dir() or not (file_path.stat().st_mode & os.X_OK): + if file_path.is_dir(): + continue + if IS_WIN: + pathext = self.env.get("PATHEXT", ".COM;.EXE;.BAT;.CMD").split(";") + if not any(file_path.name.upper().endswith(ext) for ext in pathext): + continue + elif not (file_path.stat().st_mode & os.X_OK): continue except OSError: pass diff --git a/src/virtualenv/discovery/cached_py_info.py b/src/virtualenv/discovery/cached_py_info.py index 8f4c7291b..ee2034615 100644 --- a/src/virtualenv/discovery/cached_py_info.py +++ b/src/virtualenv/discovery/cached_py_info.py @@ -7,6 +7,7 @@ from __future__ import annotations +import hashlib import logging import os import random @@ -23,6 +24,7 @@ _CACHE = OrderedDict() _CACHE[Path(sys.executable)] = PythonInfo() +LOGGER = logging.getLogger(__name__) def from_exe(cls, app_data, exe, env=None, raise_on_error=True, ignore_cache=False): # noqa: FBT002, PLR0913 @@ -31,7 +33,7 @@ def from_exe(cls, app_data, exe, env=None, raise_on_error=True, ignore_cache=Fal if isinstance(result, Exception): if raise_on_error: raise result - logging.info("%s", result) + LOGGER.info("%s", result) result = None return result @@ -57,14 +59,23 @@ def _get_via_file_cache(cls, app_data, path, exe, env): path_modified = path.stat().st_mtime except OSError: path_modified = -1 + py_info_script = Path(os.path.abspath(__file__)).parent / "py_info.py" + try: + py_info_hash = hashlib.sha256(py_info_script.read_bytes()).hexdigest() + except OSError: + py_info_hash = None + if app_data is None: app_data = AppDataDisabled() py_info, py_info_store = None, app_data.py_info(path) with py_info_store.locked(): if py_info_store.exists(): # if exists and matches load data = py_info_store.read() - of_path, of_st_mtime, of_content = data["path"], data["st_mtime"], data["content"] - if of_path == path_text and of_st_mtime == path_modified: + of_path = data.get("path") + of_st_mtime = data.get("st_mtime") + of_content = data.get("content") + of_hash = data.get("hash") + if of_path == path_text and of_st_mtime == path_modified and of_hash == py_info_hash: py_info = cls._from_dict(of_content.copy()) sys_exe = py_info.system_executable if sys_exe is not None and not os.path.exists(sys_exe): @@ -75,7 +86,12 @@ def _get_via_file_cache(cls, app_data, path, exe, env): if py_info is None: # if not loaded run and save failure, py_info = _run_subprocess(cls, exe, app_data, env) if failure is None: - data = {"st_mtime": path_modified, "path": path_text, "content": py_info._to_dict()} # noqa: SLF001 + data = { + "st_mtime": path_modified, + "path": path_text, + "content": py_info._to_dict(), # noqa: SLF001 + "hash": py_info_hash, + } py_info_store.write(data) else: py_info = failure @@ -109,7 +125,7 @@ def _run_subprocess(cls, exe, app_data, env): # prevent sys.prefix from leaking into the child process - see https://bugs.python.org/issue22490 env = env.copy() env.pop("__PYVENV_LAUNCHER__", None) - logging.debug("get interpreter info via cmd: %s", LogCmd(cmd)) + LOGGER.debug("get interpreter info via cmd: %s", LogCmd(cmd)) try: process = Popen( cmd, diff --git a/src/virtualenv/discovery/py_info.py b/src/virtualenv/discovery/py_info.py index 882daa331..c2310cd7e 100644 --- a/src/virtualenv/discovery/py_info.py +++ b/src/virtualenv/discovery/py_info.py @@ -11,6 +11,7 @@ import os import platform import re +import struct import sys import sysconfig import warnings @@ -18,6 +19,7 @@ from string import digits VersionInfo = namedtuple("VersionInfo", ["major", "minor", "micro", "releaselevel", "serial"]) # noqa: PYI024 +LOGGER = logging.getLogger(__name__) def _get_path_extensions(): @@ -25,7 +27,7 @@ def _get_path_extensions(): EXTENSIONS = _get_path_extensions() -_CONF_VAR_RE = re.compile(r"\{\w+\}") +_CONF_VAR_RE = re.compile(r"\{\w+}") class PythonInfo: # noqa: PLR0904 @@ -43,7 +45,10 @@ def abs_path(v): # this is a tuple in earlier, struct later, unify to our own named tuple self.version_info = VersionInfo(*sys.version_info) - self.architecture = 64 if sys.maxsize > 2**32 else 32 + # Use the same implementation as found in stdlib platform.architecture + # to account for platforms where the maximum integer is not equal the + # pointer size. + self.architecture = 32 if struct.calcsize("P") == 4 else 64 # noqa: PLR2004 # Used to determine some file names. # See `CPython3Windows.python_zip()`. @@ -51,6 +56,7 @@ def abs_path(v): self.version = sys.version self.os = os.name + self.free_threaded = sysconfig.get_config_var("Py_GIL_DISABLED") == 1 # information about the prefix - determines python home self.prefix = abs_path(getattr(sys, "prefix", None)) # prefix we think @@ -117,6 +123,11 @@ def abs_path(v): self.sysconfig_vars = {i: sysconfig.get_config_var(i or "") for i in config_var_keys} + if "TCL_LIBRARY" in os.environ: + self.tcl_lib, self.tk_lib = self._get_tcl_tk_libs() + else: + self.tcl_lib, self.tk_lib = None, None + confs = { k: (self.system_prefix if v is not None and v.startswith(self.prefix) else v) for k, v in self.sysconfig_vars.items() @@ -126,36 +137,112 @@ def abs_path(v): self.max_size = getattr(sys, "maxsize", getattr(sys, "maxint", None)) self._creators = None + @staticmethod + def _get_tcl_tk_libs(): + """ + Detects the tcl and tk libraries using tkinter. + + This works reliably but spins up tkinter, which is heavy if you don't need it. + """ + tcl_lib, tk_lib = None, None + try: + import tkinter as tk # noqa: PLC0415 + except ImportError: + pass + else: + try: + tcl = tk.Tcl() + tcl_lib = tcl.eval("info library") + + # Try to get TK library path directly first + try: + tk_lib = tcl.eval("set tk_library") + if tk_lib and os.path.isdir(tk_lib): + pass # We found it directly + else: + tk_lib = None # Reset if invalid + except tk.TclError: + tk_lib = None + + # If direct query failed, try constructing the path + if tk_lib is None: + tk_version = tcl.eval("package require Tk") + tcl_parent = os.path.dirname(tcl_lib) + + # Try different version formats + version_variants = [ + tk_version, # Full version like "8.6.12" + ".".join(tk_version.split(".")[:2]), # Major.minor like "8.6" + tk_version.split(".")[0], # Just major like "8" + ] + + for version in version_variants: + tk_lib_path = os.path.join(tcl_parent, f"tk{version}") + if not os.path.isdir(tk_lib_path): + continue + # Validate it's actually a TK directory + if os.path.exists(os.path.join(tk_lib_path, "tk.tcl")): + tk_lib = tk_lib_path + break + + except tk.TclError: + pass + + return tcl_lib, tk_lib + def _fast_get_system_executable(self): """Try to get the system executable by just looking at properties.""" - if self.real_prefix or ( # noqa: PLR1702 - self.base_prefix is not None and self.base_prefix != self.prefix - ): # if this is a virtual environment - if self.real_prefix is None: - base_executable = getattr(sys, "_base_executable", None) # some platforms may set this to help us - if base_executable is not None: # noqa: SIM102 # use the saved system executable if present - if sys.executable != base_executable: # we know we're in a virtual environment, cannot be us - if os.path.exists(base_executable): - return base_executable - # Python may return "python" because it was invoked from the POSIX virtual environment - # however some installs/distributions do not provide a version-less "python" binary in - # the system install location (see PEP 394) so try to fallback to a versioned binary. - # - # Gate this to Python 3.11 as `sys._base_executable` path resolution is now relative to - # the 'home' key from pyvenv.cfg which often points to the system install location. - major, minor = self.version_info.major, self.version_info.minor - if self.os == "posix" and (major, minor) >= (3, 11): - # search relative to the directory of sys._base_executable - base_dir = os.path.dirname(base_executable) - for base_executable in [ - os.path.join(base_dir, exe) for exe in (f"python{major}", f"python{major}.{minor}") - ]: - if os.path.exists(base_executable): - return base_executable - return None # in this case we just can't tell easily without poking around FS and calling them, bail # if we're not in a virtual environment, this is already a system python, so return the original executable # note we must choose the original and not the pure executable as shim scripts might throw us off - return self.original_executable + if not (self.real_prefix or (self.base_prefix is not None and self.base_prefix != self.prefix)): + return self.original_executable + + # if this is NOT a virtual environment, can't determine easily, bail out + if self.real_prefix is not None: + return None + + base_executable = getattr(sys, "_base_executable", None) # some platforms may set this to help us + if base_executable is None: # use the saved system executable if present + return None + + # we know we're in a virtual environment, can not be us + if sys.executable == base_executable: + return None + + # We're not in a venv and base_executable exists; use it directly + if os.path.exists(base_executable): + return base_executable + + # Try fallback for POSIX virtual environments + return self._try_posix_fallback_executable(base_executable) + + def _try_posix_fallback_executable(self, base_executable): + """ + Try to find a versioned Python binary as fallback for POSIX virtual environments. + + Python may return "python" because it was invoked from the POSIX virtual environment + however some installs/distributions do not provide a version-less "python" binary in + the system install location (see PEP 394) so try to fallback to a versioned binary. + + Gate this to Python 3.11 as `sys._base_executable` path resolution is now relative to + the 'home' key from pyvenv.cfg which often points to the system install location. + """ + major, minor = self.version_info.major, self.version_info.minor + if self.os != "posix" or (major, minor) < (3, 11): + return None + + # search relative to the directory of sys._base_executable + base_dir = os.path.dirname(base_executable) + candidates = [f"python{major}", f"python{major}.{minor}"] + if self.implementation == "PyPy": + candidates.extend(["pypy", "pypy3", f"pypy{major}", f"pypy{major}.{minor}"]) + + for candidate in candidates: + full_path = os.path.join(base_dir, candidate) + if os.path.exists(full_path): + return full_path + + return None # in this case we just can't tell easily without poking around FS and calling them, bail def install_path(self, key): result = self.distutils_install.get(key) @@ -213,7 +300,9 @@ def is_venv(self): return self.base_prefix is not None def sysconfig_path(self, key, config_var=None, sep=os.sep): - pattern = self.sysconfig_paths[key] + pattern = self.sysconfig_paths.get(key) + if pattern is None: + return "" if config_var is None: config_var = self.sysconfig_vars else: @@ -289,7 +378,12 @@ def __str__(self) -> str: @property def spec(self): - return "{}{}-{}".format(self.implementation, ".".join(str(i) for i in self.version_info), self.architecture) + return "{}{}{}-{}".format( + self.implementation, + ".".join(str(i) for i in self.version_info), + "t" if self.free_threaded else "", + self.architecture, + ) @classmethod def clear_cache(cls, app_data): @@ -299,7 +393,7 @@ def clear_cache(cls, app_data): clear(app_data) cls._cache_exe_discovery.clear() - def satisfies(self, spec, impl_must_match): # noqa: C901 + def satisfies(self, spec, impl_must_match): # noqa: C901, PLR0911 """Check if a given specification can be satisfied by the this python interpreter instance.""" if spec.path: if self.executable == os.path.abspath(spec.path): @@ -325,6 +419,9 @@ def satisfies(self, spec, impl_must_match): # noqa: C901 if spec.architecture is not None and spec.architecture != self.architecture: return False + if spec.free_threaded is not None and spec.free_threaded != self.free_threaded: + return False + for our, req in zip(self.version_info[0:3], (spec.major, spec.minor, spec.micro)): if req is not None and our is not None and our != req: return False @@ -386,7 +483,7 @@ def from_exe( # noqa: PLR0913 except Exception as exception: if raise_on_error: raise - logging.info("ignore %s due cannot resolve system due to %r", proposed.original_executable, exception) + LOGGER.info("ignore %s due cannot resolve system due to %r", proposed.original_executable, exception) proposed = None return proposed @@ -412,12 +509,12 @@ def _resolve_to_system(cls, app_data, target): if prefix in prefixes: if len(prefixes) == 1: # if we're linking back to ourselves accept ourselves with a WARNING - logging.info("%r links back to itself via prefixes", target) + LOGGER.info("%r links back to itself via prefixes", target) target.system_executable = target.executable break for at, (p, t) in enumerate(prefixes.items(), start=1): - logging.error("%d: prefix=%s, info=%r", at, p, t) - logging.error("%d: prefix=%s, info=%r", len(prefixes) + 1, prefix, target) + LOGGER.error("%d: prefix=%s, info=%r", at, p, t) + LOGGER.error("%d: prefix=%s, info=%r", len(prefixes) + 1, prefix, target) msg = "prefixes are causing a circle {}".format("|".join(prefixes.keys())) raise RuntimeError(msg) prefixes[prefix] = target @@ -432,9 +529,9 @@ def _resolve_to_system(cls, app_data, target): def discover_exe(self, app_data, prefix, exact=True, env=None): # noqa: FBT002 key = prefix, exact if key in self._cache_exe_discovery and prefix: - logging.debug("discover exe from cache %s - exact %s: %r", prefix, exact, self._cache_exe_discovery[key]) + LOGGER.debug("discover exe from cache %s - exact %s: %r", prefix, exact, self._cache_exe_discovery[key]) return self._cache_exe_discovery[key] - logging.debug("discover exe for %s in %s", self, prefix) + LOGGER.debug("discover exe for %s in %s", self, prefix) # we don't know explicitly here, do some guess work - our executable name should tell possible_names = self._find_possible_exe_names() possible_folders = self._find_possible_folders(prefix) @@ -450,7 +547,7 @@ def discover_exe(self, app_data, prefix, exact=True, env=None): # noqa: FBT002 info = self._select_most_likely(discovered, self) folders = os.pathsep.join(possible_folders) self._cache_exe_discovery[key] = info - logging.debug("no exact match found, chosen most similar of %s within base folders %s", info, folders) + LOGGER.debug("no exact match found, chosen most similar of %s within base folders %s", info, folders) return info msg = "failed to detect {} in {}".format("|".join(possible_names), os.pathsep.join(possible_folders)) raise RuntimeError(msg) @@ -469,7 +566,7 @@ def _check_exe(self, app_data, folder, name, exact, discovered, env): # noqa: P if item == "version_info": found, searched = ".".join(str(i) for i in found), ".".join(str(i) for i in searched) executable = info.executable - logging.debug("refused interpreter %s because %s differs %s != %s", executable, item, found, searched) + LOGGER.debug("refused interpreter %s because %s differs %s != %s", executable, item, found, searched) if exact is False: discovered.append(info) break @@ -521,10 +618,14 @@ def _find_possible_exe_names(self): for name in self._possible_base(): for at in (3, 2, 1, 0): version = ".".join(str(i) for i in self.version_info[:at]) - for arch in [f"-{self.architecture}", ""]: - for ext in EXTENSIONS: - candidate = f"{name}{version}{arch}{ext}" - name_candidate[candidate] = None + mods = [""] + if self.free_threaded: + mods.append("t") + for mod in mods: + for arch in [f"-{self.architecture}", ""]: + for ext in EXTENSIONS: + candidate = f"{name}{version}{mod}{arch}{ext}" + name_candidate[candidate] = None return list(name_candidate.keys()) def _possible_base(self): diff --git a/src/virtualenv/discovery/py_spec.py b/src/virtualenv/discovery/py_spec.py index dcd84f423..d8519c23d 100644 --- a/src/virtualenv/discovery/py_spec.py +++ b/src/virtualenv/discovery/py_spec.py @@ -5,7 +5,7 @@ import os import re -PATTERN = re.compile(r"^(?P[a-zA-Z]+)?(?P[0-9.]+)?(?:-(?P32|64))?$") +PATTERN = re.compile(r"^(?P[a-zA-Z]+)?(?P[0-9.]+)?(?Pt)?(?:-(?P32|64))?$") class PythonSpec: @@ -20,18 +20,21 @@ def __init__( # noqa: PLR0913 micro: int | None, architecture: int | None, path: str | None, + *, + free_threaded: bool | None = None, ) -> None: self.str_spec = str_spec self.implementation = implementation self.major = major self.minor = minor self.micro = micro + self.free_threaded = free_threaded self.architecture = architecture self.path = path @classmethod def from_string_spec(cls, string_spec: str): # noqa: C901, PLR0912 - impl, major, minor, micro, arch, path = None, None, None, None, None, None + impl, major, minor, micro, threaded, arch, path = None, None, None, None, None, None, None if os.path.isabs(string_spec): # noqa: PLR1702 path = string_spec else: @@ -58,6 +61,7 @@ def _int_or_none(val): major = int(str(version_data)[0]) # first digit major if version_data > 9: # noqa: PLR2004 minor = int(str(version_data)[1:]) + threaded = bool(groups["threaded"]) ok = True except ValueError: pass @@ -70,7 +74,7 @@ def _int_or_none(val): if not ok: path = string_spec - return cls(string_spec, impl, major, minor, micro, arch, path) + return cls(string_spec, impl, major, minor, micro, arch, path, free_threaded=threaded) def generate_re(self, *, windows: bool) -> re.Pattern: """Generate a regular expression for matching against a filename.""" @@ -78,6 +82,7 @@ def generate_re(self, *, windows: bool) -> re.Pattern: *(r"\d+" if v is None else v for v in (self.major, self.minor, self.micro)) ) impl = "python" if self.implementation is None else f"python|{re.escape(self.implementation)}" + mod = "t?" if self.free_threaded else "" suffix = r"\.exe" if windows else "" version_conditional = ( "?" @@ -89,7 +94,7 @@ def generate_re(self, *, windows: bool) -> re.Pattern: ) # Try matching `direct` first, so the `direct` group is filled when possible. return re.compile( - rf"(?P{impl})(?P{version}){version_conditional}{suffix}$", + rf"(?P{impl})(?P{version}{mod}){version_conditional}{suffix}$", flags=re.IGNORECASE, ) @@ -105,6 +110,8 @@ def satisfies(self, spec): return False if spec.architecture is not None and spec.architecture != self.architecture: return False + if spec.free_threaded is not None and spec.free_threaded != self.free_threaded: + return False for our, req in zip((self.major, self.minor, self.micro), (spec.major, spec.minor, spec.micro)): if req is not None and our is not None and our != req: @@ -113,7 +120,7 @@ def satisfies(self, spec): def __repr__(self) -> str: name = type(self).__name__ - params = "implementation", "major", "minor", "micro", "architecture", "path" + params = "implementation", "major", "minor", "micro", "architecture", "path", "free_threaded" return f"{name}({', '.join(f'{k}={getattr(self, k)}' for k in params if getattr(self, k) is not None)})" diff --git a/src/virtualenv/discovery/windows/__init__.py b/src/virtualenv/discovery/windows/__init__.py index 9efd5b6ab..b7206406a 100644 --- a/src/virtualenv/discovery/windows/__init__.py +++ b/src/virtualenv/discovery/windows/__init__.py @@ -27,14 +27,14 @@ def propose_interpreters(spec, cache_dir, env): reverse=True, ) - for name, major, minor, arch, exe, _ in existing: + for name, major, minor, arch, threaded, exe, _ in existing: # Map well-known/most common organizations to a Python implementation, use the org name as a fallback for # backwards compatibility. implementation = _IMPLEMENTATION_BY_ORG.get(name, name) # Pre-filtering based on Windows Registry metadata, for CPython only skip_pre_filter = implementation.lower() != "cpython" - registry_spec = PythonSpec(None, implementation, major, minor, None, arch, exe) + registry_spec = PythonSpec(None, implementation, major, minor, None, arch, exe, free_threaded=threaded) if skip_pre_filter or registry_spec.satisfies(spec): interpreter = Pep514PythonInfo.from_exe(exe, cache_dir, env=env, raise_on_error=False) if interpreter is not None and interpreter.satisfies(spec, impl_must_match=True): diff --git a/src/virtualenv/discovery/windows/pep514.py b/src/virtualenv/discovery/windows/pep514.py index 8bc9e3060..a75dad36d 100644 --- a/src/virtualenv/discovery/windows/pep514.py +++ b/src/virtualenv/discovery/windows/pep514.py @@ -65,7 +65,8 @@ def process_tag(hive_name, company, company_key, tag, default_arch): exe_data = load_exe(hive_name, company, company_key, tag) if exe_data is not None: exe, args = exe_data - return company, major, minor, arch, exe, args + threaded = load_threaded(hive_name, company, tag, tag_key) + return company, major, minor, arch, threaded, exe, args return None return None return None @@ -138,6 +139,18 @@ def parse_version(version_str): raise ValueError(error) +def load_threaded(hive_name, company, tag, tag_key): + display_name = get_value(tag_key, "DisplayName") + if display_name is not None: + if isinstance(display_name, str): + if "freethreaded" in display_name.lower(): + return True + else: + key_path = f"{hive_name}/{company}/{tag}/DisplayName" + msg(key_path, f"display name is not string: {display_name!r}") + return bool(re.match(r"^\d+(\.\d+){0,2}t$", tag, flags=re.IGNORECASE)) + + def msg(path, what): LOGGER.warning("PEP-514 violation in Windows Registry at %s error: %s", path, what) diff --git a/src/virtualenv/info.py b/src/virtualenv/info.py index 8b217d0c3..e3542a7e2 100644 --- a/src/virtualenv/info.py +++ b/src/virtualenv/info.py @@ -8,12 +8,14 @@ IMPLEMENTATION = platform.python_implementation() IS_PYPY = IMPLEMENTATION == "PyPy" +IS_GRAALPY = IMPLEMENTATION == "GraalVM" IS_CPYTHON = IMPLEMENTATION == "CPython" IS_WIN = sys.platform == "win32" IS_MAC_ARM64 = sys.platform == "darwin" and platform.machine() == "arm64" ROOT = os.path.realpath(os.path.join(os.path.abspath(__file__), os.path.pardir, os.path.pardir)) IS_ZIPAPP = os.path.isfile(ROOT) _CAN_SYMLINK = _FS_CASE_SENSITIVE = _CFG_DIR = _DATA_DIR = None +LOGGER = logging.getLogger(__name__) def fs_is_case_sensitive(): @@ -22,7 +24,7 @@ def fs_is_case_sensitive(): if _FS_CASE_SENSITIVE is None: with tempfile.NamedTemporaryFile(prefix="TmP") as tmp_file: _FS_CASE_SENSITIVE = not os.path.exists(tmp_file.name.lower()) - logging.debug("filesystem is %scase-sensitive", "" if _FS_CASE_SENSITIVE else "not ") + LOGGER.debug("filesystem is %scase-sensitive", "" if _FS_CASE_SENSITIVE else "not ") return _FS_CASE_SENSITIVE @@ -32,18 +34,20 @@ def fs_supports_symlink(): if _CAN_SYMLINK is None: can = False if hasattr(os, "symlink"): - if IS_WIN: - with tempfile.NamedTemporaryFile(prefix="TmP") as tmp_file: - temp_dir = os.path.dirname(tmp_file.name) - dest = os.path.join(temp_dir, f"{tmp_file.name}-{'b'}") - try: - os.symlink(tmp_file.name, dest) - can = True - except (OSError, NotImplementedError): - pass - logging.debug("symlink on filesystem does%s work", "" if can else " not") - else: - can = True + # Creating a symlink can fail for a variety of reasons, indicating that the filesystem does not support it. + # E.g. on Linux with a VFAT partition mounted. + with tempfile.NamedTemporaryFile(prefix="TmP") as tmp_file: + temp_dir = os.path.dirname(tmp_file.name) + dest = os.path.join(temp_dir, f"{tmp_file.name}-{'b'}") + try: + os.symlink(tmp_file.name, dest) + can = True + except (OSError, NotImplementedError): + pass # symlink is not supported + finally: + if os.path.lexists(dest): + os.remove(dest) + LOGGER.debug("symlink on filesystem does%s work", "" if can else " not") _CAN_SYMLINK = can return _CAN_SYMLINK @@ -54,6 +58,7 @@ def fs_path_id(path: str) -> str: __all__ = ( "IS_CPYTHON", + "IS_GRAALPY", "IS_MAC_ARM64", "IS_PYPY", "IS_WIN", diff --git a/src/virtualenv/report.py b/src/virtualenv/report.py index 9ad52a12a..c9682a8f6 100644 --- a/src/virtualenv/report.py +++ b/src/virtualenv/report.py @@ -33,7 +33,7 @@ def setup_report(verbosity, show_pid=False): # noqa: FBT002 stream_handler.setFormatter(formatter) LOGGER.addHandler(stream_handler) level_name = logging.getLevelName(level) - logging.debug("setup logging to %s", level_name) + LOGGER.debug("setup logging to %s", level_name) logging.getLogger("distlib").setLevel(logging.ERROR) return verbosity diff --git a/src/virtualenv/run/__init__.py b/src/virtualenv/run/__init__.py index 48647ec7c..03190502b 100644 --- a/src/virtualenv/run/__init__.py +++ b/src/virtualenv/run/__init__.py @@ -48,6 +48,7 @@ def session_via_cli(args, options=None, setup_logging=True, env=None): # noqa: env = os.environ if env is None else env parser, elements = build_parser(args, options, setup_logging, env) options = parser.parse_args(args) + options.py_version = parser._interpreter.version_info # noqa: SLF001 creator, seeder, activators = tuple(e.create(options) for e in elements) # create types return Session( options.verbosity, @@ -153,6 +154,9 @@ def _do_report_setup(parser, args, setup_logging): verbosity = verbosity_group.add_mutually_exclusive_group() verbosity.add_argument("-v", "--verbose", action="count", dest="verbose", help="increase verbosity", default=2) verbosity.add_argument("-q", "--quiet", action="count", dest="quiet", help="decrease verbosity", default=0) + # do not configure logging if only help is requested, as no logging is required for this + if args and any(i in args for i in ("-h", "--help")): + return option, _ = parser.parse_known_args(args) if setup_logging: setup_report(option.verbosity) diff --git a/src/virtualenv/run/plugin/discovery.py b/src/virtualenv/run/plugin/discovery.py index a963042d0..5e8b2392f 100644 --- a/src/virtualenv/run/plugin/discovery.py +++ b/src/virtualenv/run/plugin/discovery.py @@ -16,10 +16,15 @@ def get_discover(parser, args): choices = _get_default_discovery(discover_types) # prefer the builtin if present, otherwise fallback to first defined type choices = sorted(choices, key=lambda a: 0 if a == "builtin" else 1) + try: + default_discovery = next(iter(choices)) + except StopIteration as e: + msg = "No discovery plugin found. Try reinstalling virtualenv to fix this issue." + raise RuntimeError(msg) from e discovery_parser.add_argument( "--discovery", choices=choices, - default=next(iter(choices)), + default=default_discovery, required=False, help="interpreter discovery method", ) diff --git a/src/virtualenv/run/session.py b/src/virtualenv/run/session.py index 9ffd89082..def795328 100644 --- a/src/virtualenv/run/session.py +++ b/src/virtualenv/run/session.py @@ -3,6 +3,8 @@ import json import logging +LOGGER = logging.getLogger(__name__) + class Session: """Represents a virtual environment creation session.""" @@ -47,20 +49,20 @@ def run(self): self.creator.pyenv_cfg.write() def _create(self): - logging.info("create virtual environment via %s", self.creator) + LOGGER.info("create virtual environment via %s", self.creator) self.creator.run() - logging.debug(_DEBUG_MARKER) - logging.debug("%s", _Debug(self.creator)) + LOGGER.debug(_DEBUG_MARKER) + LOGGER.debug("%s", _Debug(self.creator)) def _seed(self): if self.seeder is not None and self.seeder.enabled: - logging.info("add seed packages via %s", self.seeder) + LOGGER.info("add seed packages via %s", self.seeder) self.seeder.run(self.creator) def _activate(self): if self.activators: active = ", ".join(type(i).__name__.replace("Activator", "") for i in self.activators) - logging.info("add activators for %s", active) + LOGGER.info("add activators for %s", active) for activator in self.activators: activator.generate(self.creator) diff --git a/src/virtualenv/seed/embed/base_embed.py b/src/virtualenv/seed/embed/base_embed.py index 864cc49ca..72fc5a34a 100644 --- a/src/virtualenv/seed/embed/base_embed.py +++ b/src/virtualenv/seed/embed/base_embed.py @@ -1,11 +1,14 @@ from __future__ import annotations +import logging from abc import ABC +from argparse import SUPPRESS from pathlib import Path from virtualenv.seed.seeder import Seeder from virtualenv.seed.wheels import Version +LOGGER = logging.getLogger(__name__) PERIODIC_UPDATE_ON_BY_DEFAULT = True @@ -18,7 +21,11 @@ def __init__(self, options) -> None: self.pip_version = options.pip self.setuptools_version = options.setuptools - self.wheel_version = options.wheel + + # wheel version needs special handling + # on Python > 3.8, the default is None (as in not used) + # so we can differentiate between explicit and implicit none + self.wheel_version = options.wheel or "none" self.no_pip = options.no_pip self.no_setuptools = options.no_setuptools @@ -26,6 +33,15 @@ def __init__(self, options) -> None: self.app_data = options.app_data self.periodic_update = not options.no_periodic_update + if options.py_version[:2] >= (3, 9): + if options.wheel is not None or options.no_wheel: + LOGGER.warning( + "The --no-wheel and --wheel options are deprecated. " + "They have no effect for Python > 3.8 as wheel is no longer " + "bundled in virtualenv.", + ) + self.no_wheel = True + if not self.distribution_to_versions(): self.enabled = False @@ -41,7 +57,7 @@ def distribution_to_versions(self) -> dict[str, str]: return { distribution: getattr(self, f"{distribution}_version") for distribution in self.distributions() - if getattr(self, f"no_{distribution}") is False and getattr(self, f"{distribution}_version") != "none" + if getattr(self, f"no_{distribution}", None) is False and getattr(self, f"{distribution}_version") != "none" } @classmethod @@ -71,21 +87,28 @@ def add_parser_arguments(cls, parser, interpreter, app_data): # noqa: ARG003 default=[], ) for distribution, default in cls.distributions().items(): + help_ = f"version of {distribution} to install as seed: embed, bundle, none or exact version" if interpreter.version_info[:2] >= (3, 12) and distribution in {"wheel", "setuptools"}: default = "none" # noqa: PLW2901 + if interpreter.version_info[:2] >= (3, 9) and distribution == "wheel": + default = None # noqa: PLW2901 + help_ = SUPPRESS parser.add_argument( f"--{distribution}", dest=distribution, metavar="version", - help=f"version of {distribution} to install as seed: embed, bundle, none or exact version", + help=help_, default=default, ) for distribution in cls.distributions(): + help_ = f"do not install {distribution}" + if interpreter.version_info[:2] >= (3, 9) and distribution == "wheel": + help_ = SUPPRESS parser.add_argument( f"--no-{distribution}", dest=f"no_{distribution}", action="store_true", - help=f"do not install {distribution}", + help=help_, default=False, ) parser.add_argument( @@ -103,7 +126,7 @@ def __repr__(self) -> str: result += f"extra_search_dir={', '.join(str(i) for i in self.extra_search_dir)}," result += f"download={self.download}," for distribution in self.distributions(): - if getattr(self, f"no_{distribution}"): + if getattr(self, f"no_{distribution}", None): continue version = getattr(self, f"{distribution}_version", None) if version == "none": diff --git a/src/virtualenv/seed/embed/pip_invoke.py b/src/virtualenv/seed/embed/pip_invoke.py index 815753c46..b733c5148 100644 --- a/src/virtualenv/seed/embed/pip_invoke.py +++ b/src/virtualenv/seed/embed/pip_invoke.py @@ -8,6 +8,8 @@ from virtualenv.seed.embed.base_embed import BaseEmbed from virtualenv.seed.wheels import Version, get_wheel, pip_wheel_env_run +LOGGER = logging.getLogger(__name__) + class PipInvoke(BaseEmbed): def __init__(self, options) -> None: @@ -23,7 +25,7 @@ def run(self, creator): @staticmethod def _execute(cmd, env): - logging.debug("pip seed by running: %s", LogCmd(cmd, env)) + LOGGER.debug("pip seed by running: %s", LogCmd(cmd, env)) process = Popen(cmd, env=env) process.communicate() if process.returncode != 0: diff --git a/src/virtualenv/seed/embed/via_app_data/pip_install/base.py b/src/virtualenv/seed/embed/via_app_data/pip_install/base.py index 0ca829713..d5e87ad93 100644 --- a/src/virtualenv/seed/embed/via_app_data/pip_install/base.py +++ b/src/virtualenv/seed/embed/via_app_data/pip_install/base.py @@ -14,6 +14,8 @@ from virtualenv.util.path import safe_delete +LOGGER = logging.getLogger(__name__) + class PipInstall(ABC): def __init__(self, wheel, creator, image_folder) -> None: @@ -40,11 +42,11 @@ def install(self, version_info): script_dir = self._creator.script_dir for name, module in self._console_scripts.items(): consoles.update(self._create_console_entry_point(name, module, script_dir, version_info)) - logging.debug("generated console scripts %s", " ".join(i.name for i in consoles)) + LOGGER.debug("generated console scripts %s", " ".join(i.name for i in consoles)) def build_image(self): # 1. first extract the wheel - logging.debug("build install image for %s to %s", self._wheel.name, self._image_dir) + LOGGER.debug("build install image for %s to %s", self._wheel.name, self._image_dir) with zipfile.ZipFile(str(self._wheel)) as zip_ref: self._shorten_path_if_needed(zip_ref) zip_ref.extractall(str(self._image_dir)) @@ -151,7 +153,7 @@ def _uninstall_previous_version(self): @staticmethod def _uninstall_dist(dist): dist_base = dist.parent - logging.debug("uninstall existing distribution %s from %s", dist.stem, dist_base) + LOGGER.debug("uninstall existing distribution %s from %s", dist.stem, dist_base) top_txt = dist / "top_level.txt" # add top level packages at folder level paths = ( @@ -181,7 +183,7 @@ def clear(self): safe_delete(self._image_dir) def has_image(self): - return self._image_dir.exists() and next(self._image_dir.iterdir()) is not None + return self._image_dir.exists() and any(self._image_dir.iterdir()) class ScriptMakerCustom(ScriptMaker): diff --git a/src/virtualenv/seed/embed/via_app_data/pip_install/copy.py b/src/virtualenv/seed/embed/via_app_data/pip_install/copy.py index af50e8d12..b5f01aa91 100644 --- a/src/virtualenv/seed/embed/via_app_data/pip_install/copy.py +++ b/src/virtualenv/seed/embed/via_app_data/pip_install/copy.py @@ -1,4 +1,4 @@ -from __future__ import annotations # noqa: A005 +from __future__ import annotations import os from pathlib import Path diff --git a/src/virtualenv/seed/embed/via_app_data/via_app_data.py b/src/virtualenv/seed/embed/via_app_data/via_app_data.py index 7e58bfc6e..a2e9630c6 100644 --- a/src/virtualenv/seed/embed/via_app_data/via_app_data.py +++ b/src/virtualenv/seed/embed/via_app_data/via_app_data.py @@ -17,6 +17,8 @@ from .pip_install.copy import CopyPipInstall from .pip_install.symlink import SymlinkPipInstall +LOGGER = logging.getLogger(__name__) + class FromAppData(BaseEmbed): def __init__(self, options) -> None: @@ -46,7 +48,7 @@ def run(self, creator): def _install(name, wheel): try: - logging.debug("install %s from wheel %s via %s", name, wheel, installer_class.__name__) + LOGGER.debug("install %s from wheel %s via %s", name, wheel, installer_class.__name__) key = Path(installer_class.__name__) / wheel.path.stem wheel_img = self.app_data.wheel_image(creator.interpreter.version_release_str, key) installer = installer_class(wheel.path, creator, wheel_img) @@ -94,7 +96,7 @@ def _get(distribution, version): if result is not None: break except Exception as exception: - logging.exception("fail") + LOGGER.exception("fail") failure = exception if failure: if isinstance(failure, CalledProcessError): @@ -108,7 +110,7 @@ def _get(distribution, version): msg += output else: msg = repr(failure) - logging.error(msg) + LOGGER.error(msg) with lock: fail[distribution] = version else: diff --git a/src/virtualenv/seed/wheels/acquire.py b/src/virtualenv/seed/wheels/acquire.py index 8b3ddd8eb..eb2fb5b45 100644 --- a/src/virtualenv/seed/wheels/acquire.py +++ b/src/virtualenv/seed/wheels/acquire.py @@ -12,6 +12,8 @@ from .periodic_update import add_wheel_to_update_log from .util import Version, Wheel, discover_wheels +LOGGER = logging.getLogger(__name__) + def get_wheel( # noqa: PLR0913 distribution, @@ -50,7 +52,7 @@ def get_wheel( # noqa: PLR0913 def download_wheel(distribution, version_spec, for_py_version, search_dirs, app_data, to_folder, env): # noqa: PLR0913 to_download = f"{distribution}{version_spec or ''}" - logging.debug("download wheel %s %s to %s", to_download, for_py_version, to_folder) + LOGGER.debug("download wheel %s %s to %s", to_download, for_py_version, to_folder) cmd = [ sys.executable, "-m", @@ -75,7 +77,7 @@ def download_wheel(distribution, version_spec, for_py_version, search_dirs, app_ kwargs = {"output": out, "stderr": err} raise CalledProcessError(process.returncode, cmd, **kwargs) result = _find_downloaded_wheel(distribution, version_spec, for_py_version, to_folder, out) - logging.debug("downloaded wheel %s", result.name) + LOGGER.debug("downloaded wheel %s", result.name) return result @@ -107,7 +109,7 @@ def find_compatible_in_house(distribution, version_spec, for_py_version, in_fold def pip_wheel_env_run(search_dirs, app_data, env): env = env.copy() - env.update({"PIP_USE_WHEEL": "1", "PIP_USER": "0", "PIP_NO_INPUT": "1"}) + env.update({"PIP_USE_WHEEL": "1", "PIP_USER": "0", "PIP_NO_INPUT": "1", "PYTHONIOENCODING": "utf-8"}) wheel = get_wheel( distribution="pip", version=None, diff --git a/src/virtualenv/seed/wheels/embed/__init__.py b/src/virtualenv/seed/wheels/embed/__init__.py index f1a8da017..ef6f78e86 100644 --- a/src/virtualenv/seed/wheels/embed/__init__.py +++ b/src/virtualenv/seed/wheels/embed/__init__.py @@ -7,46 +7,48 @@ BUNDLE_FOLDER = Path(__file__).absolute().parent BUNDLE_SUPPORT = { "3.8": { - "pip": "pip-24.3.1-py3-none-any.whl", - "setuptools": "setuptools-75.3.0-py3-none-any.whl", - "wheel": "wheel-0.44.0-py3-none-any.whl", + "pip": "pip-25.0.1-py3-none-any.whl", + "setuptools": "setuptools-75.3.2-py3-none-any.whl", + "wheel": "wheel-0.45.1-py3-none-any.whl", }, "3.9": { - "pip": "pip-24.3.1-py3-none-any.whl", - "setuptools": "setuptools-75.3.0-py3-none-any.whl", - "wheel": "wheel-0.44.0-py3-none-any.whl", + "pip": "pip-25.3-py3-none-any.whl", + "setuptools": "setuptools-80.9.0-py3-none-any.whl", }, "3.10": { - "pip": "pip-24.3.1-py3-none-any.whl", - "setuptools": "setuptools-75.3.0-py3-none-any.whl", - "wheel": "wheel-0.44.0-py3-none-any.whl", + "pip": "pip-25.3-py3-none-any.whl", + "setuptools": "setuptools-80.9.0-py3-none-any.whl", }, "3.11": { - "pip": "pip-24.3.1-py3-none-any.whl", - "setuptools": "setuptools-75.3.0-py3-none-any.whl", - "wheel": "wheel-0.44.0-py3-none-any.whl", + "pip": "pip-25.3-py3-none-any.whl", + "setuptools": "setuptools-80.9.0-py3-none-any.whl", }, "3.12": { - "pip": "pip-24.3.1-py3-none-any.whl", - "setuptools": "setuptools-75.3.0-py3-none-any.whl", - "wheel": "wheel-0.44.0-py3-none-any.whl", + "pip": "pip-25.3-py3-none-any.whl", + "setuptools": "setuptools-80.9.0-py3-none-any.whl", }, "3.13": { - "pip": "pip-24.3.1-py3-none-any.whl", - "setuptools": "setuptools-75.3.0-py3-none-any.whl", - "wheel": "wheel-0.44.0-py3-none-any.whl", + "pip": "pip-25.3-py3-none-any.whl", + "setuptools": "setuptools-80.9.0-py3-none-any.whl", }, "3.14": { - "pip": "pip-24.3.1-py3-none-any.whl", - "setuptools": "setuptools-75.3.0-py3-none-any.whl", - "wheel": "wheel-0.44.0-py3-none-any.whl", + "pip": "pip-25.3-py3-none-any.whl", + "setuptools": "setuptools-80.9.0-py3-none-any.whl", + }, + "3.15": { + "pip": "pip-25.3-py3-none-any.whl", + "setuptools": "setuptools-80.9.0-py3-none-any.whl", }, } MAX = "3.8" def get_embed_wheel(distribution, for_py_version): - path = BUNDLE_FOLDER / (BUNDLE_SUPPORT.get(for_py_version, {}) or BUNDLE_SUPPORT[MAX]).get(distribution) + mapping = BUNDLE_SUPPORT.get(for_py_version, {}) or BUNDLE_SUPPORT[MAX] + wheel_file = mapping.get(distribution) + if wheel_file is None: + return None + path = BUNDLE_FOLDER / wheel_file return Wheel.from_path(path) diff --git a/src/virtualenv/seed/wheels/embed/pip-24.3.1-py3-none-any.whl b/src/virtualenv/seed/wheels/embed/pip-25.0.1-py3-none-any.whl similarity index 75% rename from src/virtualenv/seed/wheels/embed/pip-24.3.1-py3-none-any.whl rename to src/virtualenv/seed/wheels/embed/pip-25.0.1-py3-none-any.whl index 5f1d35be6..8d3b0043e 100644 Binary files a/src/virtualenv/seed/wheels/embed/pip-24.3.1-py3-none-any.whl and b/src/virtualenv/seed/wheels/embed/pip-25.0.1-py3-none-any.whl differ diff --git a/src/virtualenv/seed/wheels/embed/pip-25.3-py3-none-any.whl b/src/virtualenv/seed/wheels/embed/pip-25.3-py3-none-any.whl new file mode 100644 index 000000000..755e1aa0c Binary files /dev/null and b/src/virtualenv/seed/wheels/embed/pip-25.3-py3-none-any.whl differ diff --git a/src/virtualenv/seed/wheels/embed/setuptools-75.3.0-py3-none-any.whl b/src/virtualenv/seed/wheels/embed/setuptools-75.3.2-py3-none-any.whl similarity index 88% rename from src/virtualenv/seed/wheels/embed/setuptools-75.3.0-py3-none-any.whl rename to src/virtualenv/seed/wheels/embed/setuptools-75.3.2-py3-none-any.whl index b6a97ce19..1b66a67ff 100644 Binary files a/src/virtualenv/seed/wheels/embed/setuptools-75.3.0-py3-none-any.whl and b/src/virtualenv/seed/wheels/embed/setuptools-75.3.2-py3-none-any.whl differ diff --git a/src/virtualenv/seed/wheels/embed/setuptools-80.9.0-py3-none-any.whl b/src/virtualenv/seed/wheels/embed/setuptools-80.9.0-py3-none-any.whl new file mode 100644 index 000000000..2412ad4a3 Binary files /dev/null and b/src/virtualenv/seed/wheels/embed/setuptools-80.9.0-py3-none-any.whl differ diff --git a/src/virtualenv/seed/wheels/embed/wheel-0.44.0-py3-none-any.whl b/src/virtualenv/seed/wheels/embed/wheel-0.45.1-py3-none-any.whl similarity index 67% rename from src/virtualenv/seed/wheels/embed/wheel-0.44.0-py3-none-any.whl rename to src/virtualenv/seed/wheels/embed/wheel-0.45.1-py3-none-any.whl index 96431aa48..589308a21 100644 Binary files a/src/virtualenv/seed/wheels/embed/wheel-0.44.0-py3-none-any.whl and b/src/virtualenv/seed/wheels/embed/wheel-0.45.1-py3-none-any.whl differ diff --git a/src/virtualenv/seed/wheels/periodic_update.py b/src/virtualenv/seed/wheels/periodic_update.py index 32ffad680..ac627b3bd 100644 --- a/src/virtualenv/seed/wheels/periodic_update.py +++ b/src/virtualenv/seed/wheels/periodic_update.py @@ -22,6 +22,7 @@ from virtualenv.seed.wheels.util import Wheel from virtualenv.util.subprocess import CREATE_NO_WINDOW +LOGGER = logging.getLogger(__name__) GRACE_PERIOD_CI = timedelta(hours=1) # prevent version switch in the middle of a CI run GRACE_PERIOD_MINOR = timedelta(days=28) UPDATE_PERIOD = timedelta(days=14) @@ -45,7 +46,7 @@ def periodic_update( # noqa: PLR0913 def _update_wheel(ver): updated_wheel = Wheel(app_data.house / ver.filename) - logging.debug("using %supdated wheel %s", "periodically " if updated_wheel else "", updated_wheel) + LOGGER.debug("using %supdated wheel %s", "periodically " if updated_wheel else "", updated_wheel) return updated_wheel u_log = UpdateLog.from_app_data(app_data, distribution, for_py_version) @@ -79,10 +80,10 @@ def handle_auto_update(distribution, for_py_version, wheel, search_dirs, app_dat def add_wheel_to_update_log(wheel, for_py_version, app_data): embed_update_log = app_data.embed_update_log(wheel.distribution, for_py_version) - logging.debug("adding %s information to %s", wheel.name, embed_update_log.file) + LOGGER.debug("adding %s information to %s", wheel.name, embed_update_log.file) u_log = UpdateLog.from_dict(embed_update_log.read()) if any(version.filename == wheel.name for version in u_log.versions): - logging.warning("%s already present in %s", wheel.name, embed_update_log.file) + LOGGER.warning("%s already present in %s", wheel.name, embed_update_log.file) return # we don't need a release date for sources other than "periodic" version = NewVersion(wheel.name, datetime.now(tz=timezone.utc), None, "download") @@ -220,7 +221,7 @@ def trigger_update(distribution, for_py_version, wheel, search_dirs, app_data, e if not debug and sys.platform == "win32": kwargs["creationflags"] = CREATE_NO_WINDOW process = Popen(cmd, **kwargs) - logging.info( + LOGGER.info( "triggered periodic upgrade of %s%s (for python %s) via background process having PID %d", distribution, "" if wheel is None else f"=={wheel.version}", @@ -239,7 +240,7 @@ def do_update(distribution, for_py_version, embed_filename, app_data, search_dir try: versions = _run_do_update(app_data, distribution, embed_filename, for_py_version, periodic, search_dirs) finally: - logging.debug("done %s %s with %s", distribution, for_py_version, versions) + LOGGER.debug("done %s %s with %s", distribution, for_py_version, versions) return versions @@ -297,7 +298,7 @@ def _run_do_update( # noqa: C901, PLR0913 break release_date = release_date_for_wheel_path(dest.path) last = NewVersion(filename=dest.path.name, release_date=release_date, found_date=download_time, source=source) - logging.info("detected %s in %s", last, datetime.now(tz=timezone.utc) - download_time) + LOGGER.info("detected %s in %s", last, datetime.now(tz=timezone.utc) - download_time) versions.append(last) filenames.add(last.filename) last_wheel = last.wheel @@ -325,7 +326,7 @@ def release_date_for_wheel_path(dest): upload_time = content["releases"][wheel.version][0]["upload_time"] return datetime.strptime(upload_time, "%Y-%m-%dT%H:%M:%S").replace(tzinfo=timezone.utc) except Exception as exception: # noqa: BLE001 - logging.error("could not load release date %s because %r", content, exception) # noqa: TRY400 + LOGGER.error("could not load release date %s because %r", content, exception) # noqa: TRY400 return None @@ -353,9 +354,9 @@ def _pypi_get_distribution_info(distribution): content = json.load(file_handler) break except URLError as exception: - logging.error("failed to access %s because %r", url, exception) # noqa: TRY400 + LOGGER.error("failed to access %s because %r", url, exception) # noqa: TRY400 except Exception as exception: # noqa: BLE001 - logging.error("failed to access %s because %r", url, exception) # noqa: TRY400 + LOGGER.error("failed to access %s because %r", url, exception) # noqa: TRY400 return content @@ -386,7 +387,7 @@ def _run_manual_upgrade(app_data, distribution, for_py_version, env): do_periodic_update=False, env=env, ) - logging.warning( + LOGGER.warning( "upgrade %s for python %s with current %s", distribution, for_py_version, @@ -410,7 +411,7 @@ def _run_manual_upgrade(app_data, distribution, for_py_version, env): args.append("\n".join(f"\t{v}" for v in versions)) ver_update = "new entries found:\n%s" if versions else "no new versions found" msg = f"upgraded %s for python %s in %s {ver_update}" - logging.warning(msg, *args) + LOGGER.warning(msg, *args) __all__ = [ diff --git a/src/virtualenv/util/lock.py b/src/virtualenv/util/lock.py index b4dc66a38..b250e032f 100644 --- a/src/virtualenv/util/lock.py +++ b/src/virtualenv/util/lock.py @@ -11,6 +11,8 @@ from filelock import FileLock, Timeout +LOGGER = logging.getLogger(__name__) + class _CountedFileLock(FileLock): def __init__(self, lock_file) -> None: @@ -27,16 +29,22 @@ def acquire(self, timeout=None, poll_interval=0.05): if not self.thread_safe.acquire(timeout=-1 if timeout is None else timeout): raise Timeout(self.lock_file) if self.count == 0: - super().acquire(timeout, poll_interval) + try: + super().acquire(timeout, poll_interval) + except BaseException: + self.thread_safe.release() + raise self.count += 1 def release(self, force=False): # noqa: FBT002 with self.thread_safe: if self.count > 0: - self.thread_safe.release() - if self.count == 1: - super().release(force=force) - self.count = max(self.count - 1, 0) + if self.count == 1: + super().release(force=force) + self.count -= 1 + if self.count == 0: + # if we have no more users of this lock, release the thread lock + self.thread_safe.release() _lock_store = {} @@ -116,7 +124,7 @@ def _lock_file(self, lock, no_block=False): # noqa: FBT002 except Timeout: if no_block: raise - logging.debug("lock file %s present, will block until released", lock.lock_file) + LOGGER.debug("lock file %s present, will block until released", lock.lock_file) lock.release() # release the acquire try from above lock.acquire() diff --git a/src/virtualenv/util/path/_sync.py b/src/virtualenv/util/path/_sync.py index 78684f015..4dd5a9860 100644 --- a/src/virtualenv/util/path/_sync.py +++ b/src/virtualenv/util/path/_sync.py @@ -6,10 +6,12 @@ import sys from stat import S_IWUSR +LOGGER = logging.getLogger(__name__) + def ensure_dir(path): if not path.exists(): - logging.debug("create folder %s", str(path)) + LOGGER.debug("create folder %s", path) os.makedirs(str(path)) @@ -20,16 +22,16 @@ def ensure_safe_to_do(src, dest): if not dest.exists(): return if dest.is_dir() and not dest.is_symlink(): - logging.debug("remove directory %s", dest) + LOGGER.debug("remove directory %s", dest) safe_delete(dest) else: - logging.debug("remove file %s", dest) + LOGGER.debug("remove file %s", dest) dest.unlink() def symlink(src, dest): ensure_safe_to_do(src, dest) - logging.debug("symlink %s", _Debug(src, dest)) + LOGGER.debug("symlink %s", _Debug(src, dest)) dest.symlink_to(src, target_is_directory=src.is_dir()) @@ -37,7 +39,7 @@ def copy(src, dest): ensure_safe_to_do(src, dest) is_dir = src.is_dir() method = copytree if is_dir else shutil.copy - logging.debug("copy %s", _Debug(src, dest)) + LOGGER.debug("copy %s", _Debug(src, dest)) method(str(src), str(dest)) diff --git a/src/virtualenv/util/path/_win.py b/src/virtualenv/util/path/_win.py index aa67ca770..6404cda64 100644 --- a/src/virtualenv/util/path/_win.py +++ b/src/virtualenv/util/path/_win.py @@ -6,13 +6,13 @@ def get_short_path_name(long_name): import ctypes # noqa: PLC0415 from ctypes import wintypes # noqa: PLC0415 - _GetShortPathNameW = ctypes.windll.kernel32.GetShortPathNameW # noqa: N806 - _GetShortPathNameW.argtypes = [wintypes.LPCWSTR, wintypes.LPWSTR, wintypes.DWORD] - _GetShortPathNameW.restype = wintypes.DWORD + GetShortPathNameW = ctypes.windll.kernel32.GetShortPathNameW # noqa: N806 + GetShortPathNameW.argtypes = [wintypes.LPCWSTR, wintypes.LPWSTR, wintypes.DWORD] + GetShortPathNameW.restype = wintypes.DWORD output_buf_size = 0 while True: output_buf = ctypes.create_unicode_buffer(output_buf_size) - needed = _GetShortPathNameW(long_name, output_buf, output_buf_size) + needed = GetShortPathNameW(long_name, output_buf, output_buf_size) if output_buf_size >= needed: return output_buf.value output_buf_size = needed diff --git a/src/virtualenv/util/zipapp.py b/src/virtualenv/util/zipapp.py index 80776d010..183dd07db 100644 --- a/src/virtualenv/util/zipapp.py +++ b/src/virtualenv/util/zipapp.py @@ -1,4 +1,4 @@ -from __future__ import annotations # noqa: A005 +from __future__ import annotations import logging import os @@ -6,6 +6,8 @@ from virtualenv.info import IS_WIN, ROOT +LOGGER = logging.getLogger(__name__) + def read(full_path): sub_file = _get_path_within_zip(full_path) @@ -14,7 +16,7 @@ def read(full_path): def extract(full_path, dest): - logging.debug("extract %s to %s", full_path, dest) + LOGGER.debug("extract %s to %s", full_path, dest) sub_file = _get_path_within_zip(full_path) with zipfile.ZipFile(ROOT, "r") as zip_file: info = zip_file.getinfo(sub_file) diff --git a/tasks/make_zipapp.py b/tasks/make_zipapp.py index 4fc07dac0..608efcf8c 100644 --- a/tasks/make_zipapp.py +++ b/tasks/make_zipapp.py @@ -93,7 +93,7 @@ def __init__(self, into) -> None: self.into = into self.collected = defaultdict(lambda: defaultdict(dict)) self.pip_cmd = [str(Path(sys.executable).parent / "pip")] - self._cmd = [*self.pip_cmd, "download", "-q", "--no-deps", "--dest", str(self.into)] + self._cmd = [*self.pip_cmd, "download", "-q", "--no-deps", "--no-cache-dir", "--dest", str(self.into)] def run(self, target, versions): whl = self.build_sdist(target) diff --git a/tasks/pick_tox_env.py b/tasks/pick_tox_env.py deleted file mode 100644 index fb0088458..000000000 --- a/tasks/pick_tox_env.py +++ /dev/null @@ -1,12 +0,0 @@ -from __future__ import annotations - -import os -import sys -from pathlib import Path - -py = sys.argv[1] -if py.startswith("brew@"): - py = py[len("brew@") :] -env = f"TOXENV={py}" -with Path(os.environ["GITHUB_ENV"]).open("ta", encoding="utf-8") as file_handler: - file_handler.write(env) diff --git a/tasks/upgrade_wheels.py b/tasks/upgrade_wheels.py index 8b4a1b508..071e09b71 100644 --- a/tasks/upgrade_wheels.py +++ b/tasks/upgrade_wheels.py @@ -15,7 +15,7 @@ STRICT = "UPGRADE_ADVISORY" not in os.environ BUNDLED = ["pip", "setuptools", "wheel"] -SUPPORT = [(3, i) for i in range(8, 15)] +SUPPORT = [(3, i) for i in range(8, 16)] DEST = Path(__file__).resolve().parents[1] / "src" / "virtualenv" / "seed" / "wheels" / "embed" @@ -27,6 +27,7 @@ def download(ver, dest, package): "pip", "--disable-pip-version-check", "download", + "--no-cache-dir", "--only-binary=:all:", "--python-version", ver, @@ -37,7 +38,7 @@ def download(ver, dest, package): ) -def run(): # noqa: C901 +def run(): # noqa: C901, PLR0912 old_batch = {i.name for i in DEST.iterdir() if i.suffix == ".whl"} with TemporaryDirectory() as temp: temp_path = Path(temp) @@ -49,6 +50,8 @@ def run(): # noqa: C901 into.mkdir() folders[into] = support_ver for package in BUNDLED: + if package == "wheel" and support >= (3, 9): + continue thread = Thread(target=download, args=(support_ver, str(into), package)) targets.append(thread) thread.start() @@ -89,8 +92,10 @@ def run(): # noqa: C901 if (folder / package).exists(): support_table[version].append(package) support_table = {k: OrderedDict((i.split("-")[0], i) for i in v) for k, v in support_table.items()} - bundle = ",".join( - f"{v!r}: {{ {','.join(f'{p!r}: {f!r}' for p, f in line.items())} }}" for v, line in support_table.items() + nl = "\n" + bundle = "".join( + f"\n {v!r}: {{{nl}{''.join(f' {p!r}: {f!r},{nl}' for p, f in line.items())} }}," + for v, line in support_table.items() ) msg = dedent( f""" @@ -104,10 +109,13 @@ def run(): # noqa: C901 def get_embed_wheel(distribution, for_py_version): - path = BUNDLE_FOLDER / (BUNDLE_SUPPORT.get(for_py_version, {{}}) or BUNDLE_SUPPORT[MAX]).get(distribution) + mapping = BUNDLE_SUPPORT.get(for_py_version, {{}}) or BUNDLE_SUPPORT[MAX] + wheel_file = mapping.get(distribution) + if wheel_file is None: + return None + path = BUNDLE_FOLDER / wheel_file return Wheel.from_path(path) - __all__ = [ "get_embed_wheel", "BUNDLE_SUPPORT", @@ -119,8 +127,8 @@ def get_embed_wheel(distribution, for_py_version): ) dest_target = DEST / "__init__.py" dest_target.write_text(msg, encoding="utf-8") - subprocess.run([sys.executable, "-m", "ruff", "format", str(dest_target), "--preview"]) subprocess.run([sys.executable, "-m", "ruff", "check", str(dest_target), "--fix", "--unsafe-fixes"]) + subprocess.run([sys.executable, "-m", "ruff", "format", str(dest_target), "--preview"]) raise SystemExit(outcome) diff --git a/tests/conftest.py b/tests/conftest.py index 0310b5525..e4fc28479 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,12 +13,13 @@ from virtualenv.app_data import AppDataDiskFolder from virtualenv.discovery.py_info import PythonInfo -from virtualenv.info import IS_PYPY, IS_WIN, fs_supports_symlink +from virtualenv.info import IS_GRAALPY, IS_PYPY, IS_WIN, fs_supports_symlink from virtualenv.report import LOGGER def pytest_addoption(parser): parser.addoption("--int", action="store_true", default=False, help="run integration tests") + parser.addoption("--skip-slow", action="store_true", default=False, help="skip slow tests") def pytest_configure(config): @@ -46,6 +47,11 @@ def pytest_collection_modifyitems(config, items): if item.location[0].startswith(int_location): item.add_marker(pytest.mark.skip(reason="need --int option to run")) + if config.getoption("--skip-slow"): + for item in items: + if "slow" in [mark.name for mark in item.iter_markers()]: + item.add_marker(pytest.mark.skip(reason="skipped because --skip-slow was passed")) + @pytest.fixture(scope="session") def has_symlink_support(tmp_path_factory): # noqa: ARG001 @@ -355,7 +361,7 @@ def _skip_if_test_in_system(session_app_data): pytest.skip("test not valid if run under system") -if IS_PYPY or (IS_WIN and sys.version_info[0:2] >= (3, 13)): # https://github.com/adamchainz/time-machine/issues/456 +if IS_PYPY or IS_GRAALPY: @pytest.fixture def time_freeze(freezer): diff --git a/tests/integration/test_cachedir_tag.py b/tests/integration/test_cachedir_tag.py new file mode 100644 index 000000000..b3b6a1dfe --- /dev/null +++ b/tests/integration/test_cachedir_tag.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +import shutil +import sys +from subprocess import check_output, run +from typing import TYPE_CHECKING + +import pytest + +from virtualenv import cli_run + +if TYPE_CHECKING: + from pathlib import Path + +# gtar => gnu-tar on macOS +TAR = next((target for target in ("gtar", "tar") if shutil.which(target)), None) + + +def compatible_is_tar_present() -> bool: + return TAR and "--exclude-caches" in check_output(args=[TAR, "--help"], text=True, encoding="utf-8") + + +@pytest.mark.skipif(sys.platform == "win32", reason="Windows does not have tar") +@pytest.mark.skipif(not compatible_is_tar_present(), reason="Compatible tar is not installed") +def test_cachedir_tag_ignored_by_tag(tmp_path: Path) -> None: + venv = tmp_path / ".venv" + cli_run(["--activators", "", "--without-pip", str(venv)]) + + args = [TAR, "--create", "--file", "/dev/null", "--exclude-caches", "--verbose", venv.name] + tar_result = run(args=args, capture_output=True, text=True, encoding="utf-8", cwd=tmp_path) + assert tar_result.stdout == ".venv/\n.venv/CACHEDIR.TAG\n" + assert tar_result.stderr == f"{TAR}: .venv/: contains a cache directory tag CACHEDIR.TAG; contents not dumped\n" diff --git a/tests/integration/test_race_condition_simulation.py b/tests/integration/test_race_condition_simulation.py new file mode 100644 index 000000000..857de9aa5 --- /dev/null +++ b/tests/integration/test_race_condition_simulation.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import importlib.util +import shutil +import sys +from pathlib import Path + + +def test_race_condition_simulation(tmp_path): + """Test that simulates the race condition described in the issue. + + This test creates a temporary directory with _virtualenv.py and _virtualenv.pth, + then simulates the scenario where: + - One process imports and uses the _virtualenv module (simulating marimo) + - Another process overwrites the _virtualenv.py file (simulating uv venv) + + The test verifies that no NameError is raised for _DISTUTILS_PATCH. + """ + # Create the _virtualenv.py file + virtualenv_file = tmp_path / "_virtualenv.py" + source_file = Path(__file__).parents[2] / "src" / "virtualenv" / "create" / "via_global_ref" / "_virtualenv.py" + + shutil.copy(source_file, virtualenv_file) + + # Create the _virtualenv.pth file + pth_file = tmp_path / "_virtualenv.pth" + pth_file.write_text("import _virtualenv", encoding="utf-8") + + # Simulate the race condition by repeatedly importing + errors = [] + for _ in range(5): + # Try to import it + sys.path.insert(0, str(tmp_path)) + try: + if "_virtualenv" in sys.modules: + del sys.modules["_virtualenv"] + + import _virtualenv # noqa: F401, PLC0415 + + # Try to trigger find_spec + try: + importlib.util.find_spec("distutils.dist") + except NameError as e: + if "_DISTUTILS_PATCH" in str(e): + errors.append(str(e)) + finally: + if str(tmp_path) in sys.path: + sys.path.remove(str(tmp_path)) + + assert not errors, f"Race condition detected: {errors}" diff --git a/tests/integration/test_zipapp.py b/tests/integration/test_zipapp.py index b14344d16..c6c46c7bc 100644 --- a/tests/integration/test_zipapp.py +++ b/tests/integration/test_zipapp.py @@ -6,7 +6,6 @@ from pathlib import Path import pytest -from flaky import flaky from virtualenv.discovery.py_info import PythonInfo from virtualenv.info import fs_supports_symlink @@ -19,30 +18,33 @@ @pytest.fixture(scope="session") def zipapp_build_env(tmp_path_factory): create_env_path = None - if CURRENT.implementation != "PyPy": + if CURRENT.implementation not in {"PyPy", "GraalVM"}: exe = CURRENT.executable # guaranteed to contain a recent enough pip (tox.ini) else: create_env_path = tmp_path_factory.mktemp("zipapp-create-env") exe, found = None, False # prefer CPython as builder as pypy is slow for impl in ["cpython", ""]: - for version in range(11, 6, -1): - with suppress(Exception): - # create a virtual environment which is also guaranteed to contain a recent enough pip (bundled) - session = cli_run( - [ - "-vvv", - "-p", - f"{impl}3.{version}", - "--activators", - "", - str(create_env_path), - "--no-download", - "--no-periodic-update", - ], - ) - exe = str(session.creator.exe) - found = True + for threaded in ["", "t"]: + for version in range(11, 6, -1): + with suppress(Exception): + # create a virtual environment which is also guaranteed to contain a recent enough pip (bundled) + session = cli_run( + [ + "-vvv", + "-p", + f"{impl}3.{version}{threaded}", + "--activators", + "", + str(create_env_path), + "--no-download", + "--no-periodic-update", + ], + ) + exe = str(session.creator.exe) + found = True + break + if found: break if found: break @@ -102,13 +104,13 @@ def test_zipapp_in_symlink(capsys, call_zipapp_symlink): assert not err -@flaky(max_runs=2, min_passes=1) def test_zipapp_help(call_zipapp, capsys): call_zipapp("-h") _out, err = capsys.readouterr() assert not err +@pytest.mark.slow @pytest.mark.parametrize("seeder", ["app-data", "pip"]) def test_zipapp_create(call_zipapp, seeder): call_zipapp("--seeder", seeder) diff --git a/tests/unit/activation/conftest.py b/tests/unit/activation/conftest.py index e320038ea..53a819f96 100644 --- a/tests/unit/activation/conftest.py +++ b/tests/unit/activation/conftest.py @@ -77,7 +77,7 @@ def __call__(self, monkeypatch, tmp_path): try: process = Popen(invoke, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env) raw_, _ = process.communicate() - raw = raw_.decode() + raw = raw_.decode(errors="replace") assert process.returncode == 0, raw except subprocess.CalledProcessError as exception: output = exception.output + exception.stderr @@ -130,13 +130,13 @@ def _get_test_lines(self, activate_script): ] def assert_output(self, out, raw, tmp_path): - # pre-activation + """Compare _get_test_lines() with the expected values.""" assert out[0], raw assert out[1] == "None", raw assert out[2] == "None", raw - # post-activation - expected = self._creator.exe.parent / os.path.basename(sys.executable) - assert self.norm_path(out[3]) == self.norm_path(expected), raw + # self.activate_call(activate_script) runs at this point + python_exe = self._creator.exe.parent / os.path.basename(sys.executable) + assert self.norm_path(out[3]) == self.norm_path(python_exe), raw assert self.norm_path(out[4]) == self.norm_path(self._creator.dest).replace("\\\\", "\\"), raw assert out[5] == self._creator.env_name # Some attempts to test the prompt output print more than 1 line. @@ -212,8 +212,8 @@ def __call__(self, monkeypatch, tmp_path): stderr=subprocess.PIPE, env=env, ) - _out, _err = process.communicate() - err = _err.decode("utf-8") + _out, err_ = process.communicate() + err = err_.decode("utf-8") assert process.returncode assert self.non_source_fail_message in err @@ -232,6 +232,7 @@ def raise_on_non_source_class(): def activation_python(request, tmp_path_factory, special_char_name, current_fastest): dest = os.path.join(str(tmp_path_factory.mktemp("activation-tester-env")), special_char_name) cmd = ["--without-pip", dest, "--creator", current_fastest, "-vv", "--no-periodic-update"] + # `params` is accessed here. https://docs.pytest.org/en/stable/reference/reference.html#pytest-fixture if request.param: cmd += ["--prompt", special_char_name] session = cli_run(cmd) diff --git a/tests/unit/activation/test_bash.py b/tests/unit/activation/test_bash.py index d89f1606a..0a9c588cf 100644 --- a/tests/unit/activation/test_bash.py +++ b/tests/unit/activation/test_bash.py @@ -1,11 +1,68 @@ from __future__ import annotations +from argparse import Namespace + import pytest from virtualenv.activation import BashActivator from virtualenv.info import IS_WIN +@pytest.mark.skipif(IS_WIN, reason="Github Actions ships with WSL bash") +@pytest.mark.parametrize( + ("tcl_lib", "tk_lib", "present"), + [ + ("/path/to/tcl", "/path/to/tk", True), + (None, None, False), + ], +) +def test_bash_tkinter_generation(tmp_path, tcl_lib, tk_lib, present): + # GIVEN + class MockInterpreter: + pass + + interpreter = MockInterpreter() + interpreter.tcl_lib = tcl_lib + interpreter.tk_lib = tk_lib + + class MockCreator: + def __init__(self, dest): + self.dest = dest + self.bin_dir = dest / "bin" + self.bin_dir.mkdir() + self.interpreter = interpreter + self.pyenv_cfg = {} + self.env_name = "my-env" + + creator = MockCreator(tmp_path) + options = Namespace(prompt=None) + activator = BashActivator(options) + + # WHEN + activator.generate(creator) + content = (creator.bin_dir / "activate").read_text(encoding="utf-8") + + # THEN + # The teardown logic is always present in deactivate() + assert "unset _OLD_VIRTUAL_TCL_LIBRARY" in content + assert "unset _OLD_VIRTUAL_TK_LIBRARY" in content + + if present: + assert 'if [ /path/to/tcl != "" ]; then' in content + assert "TCL_LIBRARY=/path/to/tcl" in content + assert "export TCL_LIBRARY" in content + + assert 'if [ /path/to/tk != "" ]; then' in content + assert "TK_LIBRARY=/path/to/tk" in content + assert "export TK_LIBRARY" in content + else: + # When not present, the if condition is false, so the block is not executed + assert "if [ '' != \"\" ]; then" in content, content + assert "TCL_LIBRARY=''" in content + # The export is inside the if, so this is fine + assert "export TCL_LIBRARY" in content + + @pytest.mark.skipif(IS_WIN, reason="Github Actions ships with WSL bash") @pytest.mark.parametrize("hashing_enabled", [True, False]) def test_bash(raise_on_non_source_class, hashing_enabled, activation_tester): diff --git a/tests/unit/activation/test_batch.py b/tests/unit/activation/test_batch.py index 1d22767c3..db2595860 100644 --- a/tests/unit/activation/test_batch.py +++ b/tests/unit/activation/test_batch.py @@ -1,10 +1,57 @@ from __future__ import annotations +from argparse import Namespace + import pytest from virtualenv.activation import BatchActivator +@pytest.mark.parametrize( + ("tcl_lib", "tk_lib", "present"), + [ + ("C:\\tcl", "C:\\tk", True), + (None, None, False), + ], +) +def test_batch_tkinter_generation(tmp_path, tcl_lib, tk_lib, present): + # GIVEN + class MockInterpreter: + os = "nt" + + interpreter = MockInterpreter() + interpreter.tcl_lib = tcl_lib + interpreter.tk_lib = tk_lib + + class MockCreator: + def __init__(self, dest): + self.dest = dest + self.bin_dir = dest / "bin" + self.bin_dir.mkdir() + self.interpreter = interpreter + self.pyenv_cfg = {} + self.env_name = "my-env" + + creator = MockCreator(tmp_path) + options = Namespace(prompt=None) + activator = BatchActivator(options) + + # WHEN + activator.generate(creator) + activate_content = (creator.bin_dir / "activate.bat").read_text(encoding="utf-8") + deactivate_content = (creator.bin_dir / "deactivate.bat").read_text(encoding="utf-8") + + # THEN + if present: + assert '@if NOT "C:\\tcl"=="" @set "TCL_LIBRARY=C:\\tcl"' in activate_content + assert '@if NOT "C:\\tk"=="" @set "TK_LIBRARY=C:\\tk"' in activate_content + assert "if defined _OLD_VIRTUAL_TCL_LIBRARY" in deactivate_content + assert "if defined _OLD_VIRTUAL_TK_LIBRARY" in deactivate_content + else: + assert '@if NOT ""=="" @set "TCL_LIBRARY="' in activate_content + assert '@if NOT ""=="" @set "TK_LIBRARY="' in activate_content + + @pytest.mark.usefixtures("activation_python") def test_batch(activation_tester_class, activation_tester, tmp_path): version_script = tmp_path / "version.bat" @@ -33,3 +80,50 @@ def print_prompt(self): return 'echo "%PROMPT%"' activation_tester(Batch) + + +@pytest.mark.usefixtures("activation_python") +def test_batch_output(activation_tester_class, activation_tester, tmp_path): + version_script = tmp_path / "version.bat" + version_script.write_text("ver", encoding="utf-8") + + class Batch(activation_tester_class): + def __init__(self, session) -> None: + super().__init__(BatchActivator, session, None, "activate.bat", "bat") + self._version_cmd = [str(version_script)] + self._invoke_script = [] + self.deactivate = "call deactivate" + self.activate_cmd = "call" + self.pydoc_call = f"call {self.pydoc_call}" + self.unix_line_ending = False + + def _get_test_lines(self, activate_script): + """ + Build intermediary script which will be then called. + In the script just activate environment, call echo to get current + echo setting, and then deactivate. This ensures that echo setting + is preserved and no unwanted output appears. + """ + intermediary_script_path = str(tmp_path / "intermediary.bat") + activate_script_quoted = self.quote(str(activate_script)) + return [ + "@echo on", + f"@echo @call {activate_script_quoted} > {intermediary_script_path}", + f"@echo @echo >> {intermediary_script_path}", + f"@echo @deactivate >> {intermediary_script_path}", + f"@call {intermediary_script_path}", + ] + + def assert_output(self, out, raw, tmp_path): # noqa: ARG002 + assert out[0] == "ECHO is on.", raw + + def quote(self, s): + if '"' in s or " " in s: + text = s.replace('"', r"\"") + return f'"{text}"' + return s + + def print_prompt(self): + return 'echo "%PROMPT%"' + + activation_tester(Batch) diff --git a/tests/unit/activation/test_csh.py b/tests/unit/activation/test_csh.py index 309ae811e..5cea684ec 100644 --- a/tests/unit/activation/test_csh.py +++ b/tests/unit/activation/test_csh.py @@ -1,9 +1,66 @@ from __future__ import annotations +import sys +from argparse import Namespace +from shutil import which +from subprocess import check_output + +import pytest +from packaging.version import Version + from virtualenv.activation import CShellActivator +@pytest.mark.parametrize( + ("tcl_lib", "tk_lib", "present"), + [ + ("/path/to/tcl", "/path/to/tk", True), + (None, None, False), + ], +) +def test_cshell_tkinter_generation(tmp_path, tcl_lib, tk_lib, present): + # GIVEN + class MockInterpreter: + pass + + interpreter = MockInterpreter() + interpreter.tcl_lib = tcl_lib + interpreter.tk_lib = tk_lib + + class MockCreator: + def __init__(self, dest): + self.dest = dest + self.bin_dir = dest / "bin" + self.bin_dir.mkdir() + self.interpreter = interpreter + self.pyenv_cfg = {} + self.env_name = "my-env" + + creator = MockCreator(tmp_path) + options = Namespace(prompt=None) + activator = CShellActivator(options) + + # WHEN + activator.generate(creator) + content = (creator.bin_dir / "activate.csh").read_text(encoding="utf-8") + + if present: + assert "test $?_OLD_VIRTUAL_TCL_LIBRARY != 0" in content + assert "test $?_OLD_VIRTUAL_TK_LIBRARY != 0" in content + assert "setenv TCL_LIBRARY /path/to/tcl" in content + assert "setenv TK_LIBRARY /path/to/tk" in content + else: + assert "setenv TCL_LIBRARY ''" in content + + def test_csh(activation_tester_class, activation_tester): + exe = f"tcsh{'.exe' if sys.platform == 'win32' else ''}" + if which(exe): + version_text = check_output([exe, "--version"], text=True, encoding="utf-8") + version = Version(version_text.split(" ")[1]) + if version >= Version("6.24.14"): + pytest.skip("https://github.com/tcsh-org/tcsh/issues/117") + class Csh(activation_tester_class): def __init__(self, session) -> None: super().__init__(CShellActivator, session, "csh", "activate.csh", "csh") diff --git a/tests/unit/activation/test_fish.py b/tests/unit/activation/test_fish.py index 6a92a2790..c15a8f513 100644 --- a/tests/unit/activation/test_fish.py +++ b/tests/unit/activation/test_fish.py @@ -1,11 +1,57 @@ from __future__ import annotations +import os +import sys +from argparse import Namespace + import pytest from virtualenv.activation import FishActivator from virtualenv.info import IS_WIN +@pytest.mark.parametrize( + ("tcl_lib", "tk_lib", "present"), + [ + ("/path/to/tcl", "/path/to/tk", True), + (None, None, False), + ], +) +def test_fish_tkinter_generation(tmp_path, tcl_lib, tk_lib, present): + # GIVEN + class MockInterpreter: + pass + + interpreter = MockInterpreter() + interpreter.tcl_lib = tcl_lib + interpreter.tk_lib = tk_lib + + class MockCreator: + def __init__(self, dest): + self.dest = dest + self.bin_dir = dest / "bin" + self.bin_dir.mkdir() + self.interpreter = interpreter + self.pyenv_cfg = {} + self.env_name = "my-env" + + creator = MockCreator(tmp_path) + options = Namespace(prompt=None) + activator = FishActivator(options) + + # WHEN + activator.generate(creator) + content = (creator.bin_dir / "activate.fish").read_text(encoding="utf-8") + + # THEN + if present: + assert "set -gx TCL_LIBRARY '/path/to/tcl'" in content + assert "set -gx TK_LIBRARY '/path/to/tk'" in content + else: + assert "if test -n ''\n if set -q TCL_LIBRARY;" in content + assert "if test -n ''\n if set -q TK_LIBRARY;" in content + + @pytest.mark.skipif(IS_WIN, reason="we have not setup fish in CI yet") def test_fish(activation_tester_class, activation_tester, monkeypatch, tmp_path): monkeypatch.setenv("HOME", str(tmp_path)) @@ -20,4 +66,54 @@ def __init__(self, session) -> None: def print_prompt(self): return "fish_prompt" + def _get_test_lines(self, activate_script): + return [ + self.print_python_exe(), + self.print_os_env_var("VIRTUAL_ENV"), + self.print_os_env_var("VIRTUAL_ENV_PROMPT"), + self.print_os_env_var("PATH"), + self.activate_call(activate_script), + self.print_python_exe(), + self.print_os_env_var("VIRTUAL_ENV"), + self.print_os_env_var("VIRTUAL_ENV_PROMPT"), + self.print_os_env_var("PATH"), + self.print_prompt(), + # \\ loads documentation from the virtualenv site packages + self.pydoc_call, + self.deactivate, + self.print_python_exe(), + self.print_os_env_var("VIRTUAL_ENV"), + self.print_os_env_var("VIRTUAL_ENV_PROMPT"), + self.print_os_env_var("PATH"), + "", # just finish with an empty new line + ] + + def assert_output(self, out, raw, _): + """Compare _get_test_lines() with the expected values.""" + assert out[0], raw + assert out[1] == "None", raw + assert out[2] == "None", raw + # self.activate_call(activate_script) runs at this point + expected = self._creator.exe.parent / os.path.basename(sys.executable) + assert self.norm_path(out[4]) == self.norm_path(expected), raw + assert self.norm_path(out[5]) == self.norm_path(self._creator.dest).replace("\\\\", "\\"), raw + assert out[6] == self._creator.env_name + # Some attempts to test the prompt output print more than 1 line. + # So we need to check if the prompt exists on any of them. + prompt_text = f"({self._creator.env_name}) " + assert any(prompt_text in line for line in out[7:-5]), raw + + assert out[-5] == "wrote pydoc_test.html", raw + content = tmp_path / "pydoc_test.html" + assert content.exists(), raw + # post deactivation, same as before + assert out[-4] == out[0], raw + assert out[-3] == "None", raw + assert out[-2] == "None", raw + + # Check that the PATH is restored + assert out[3] == out[13], raw + # Check that PATH changed after activation + assert out[3] != out[8], raw + activation_tester(Fish) diff --git a/tests/unit/activation/test_nushell.py b/tests/unit/activation/test_nushell.py index fbf75e397..08c5cb1a1 100644 --- a/tests/unit/activation/test_nushell.py +++ b/tests/unit/activation/test_nushell.py @@ -1,11 +1,48 @@ from __future__ import annotations +from argparse import Namespace from shutil import which from virtualenv.activation import NushellActivator from virtualenv.info import IS_WIN +def test_nushell_tkinter_generation(tmp_path): + # GIVEN + class MockInterpreter: + pass + + interpreter = MockInterpreter() + interpreter.tcl_lib = "/path/to/tcl" + interpreter.tk_lib = "/path/to/tk" + quoted_tcl_path = NushellActivator.quote(interpreter.tcl_lib) + quoted_tk_path = NushellActivator.quote(interpreter.tk_lib) + + class MockCreator: + def __init__(self, dest): + self.dest = dest + self.bin_dir = dest / "bin" + self.bin_dir.mkdir() + self.interpreter = interpreter + self.pyenv_cfg = {} + self.env_name = "my-env" + + creator = MockCreator(tmp_path) + options = Namespace(prompt=None) + activator = NushellActivator(options) + + # WHEN + activator.generate(creator) + content = (creator.bin_dir / "activate.nu").read_text(encoding="utf-8") + + # THEN + expected_tcl = f"let $new_env = $new_env | insert TCL_LIBRARY {quoted_tcl_path}" + expected_tk = f"let $new_env = $new_env | insert TK_LIBRARY {quoted_tk_path}" + + assert expected_tcl in content + assert expected_tk in content + + def test_nushell(activation_tester_class, activation_tester): class Nushell(activation_tester_class): def __init__(self, session) -> None: diff --git a/tests/unit/activation/test_powershell.py b/tests/unit/activation/test_powershell.py index dab5748d7..2a48956cf 100644 --- a/tests/unit/activation/test_powershell.py +++ b/tests/unit/activation/test_powershell.py @@ -1,12 +1,59 @@ from __future__ import annotations import sys +from argparse import Namespace import pytest from virtualenv.activation import PowerShellActivator +@pytest.mark.parametrize( + ("tcl_lib", "tk_lib", "present"), + [ + ("C:\\tcl", "C:\\tk", True), + (None, None, False), + ], +) +def test_powershell_tkinter_generation(tmp_path, tcl_lib, tk_lib, present): + # GIVEN + class MockInterpreter: + os = "nt" + + interpreter = MockInterpreter() + interpreter.tcl_lib = tcl_lib + interpreter.tk_lib = tk_lib + + class MockCreator: + def __init__(self, dest): + self.dest = dest + self.bin_dir = dest / "bin" + self.bin_dir.mkdir() + self.interpreter = interpreter + self.pyenv_cfg = {} + self.env_name = "my-env" + + creator = MockCreator(tmp_path) + options = Namespace(prompt=None) + activator = PowerShellActivator(options) + + # WHEN + activator.generate(creator) + content = (creator.bin_dir / "activate.ps1").read_text(encoding="utf-8-sig") + + # THEN + if present: + assert "if ('C:\\tcl' -ne \"\")" in content + assert "$env:TCL_LIBRARY = 'C:\\tcl'" in content + assert "if ('C:\\tk' -ne \"\")" in content + assert "$env:TK_LIBRARY = 'C:\\tk'" in content + assert "if (Test-Path variable:_OLD_VIRTUAL_TCL_LIBRARY)" in content + assert "if (Test-Path variable:_OLD_VIRTUAL_TK_LIBRARY)" in content + else: + assert "if ('' -ne \"\")" in content + assert "$env:TCL_LIBRARY = ''" in content + + @pytest.mark.slow def test_powershell(activation_tester_class, activation_tester, monkeypatch): monkeypatch.setenv("TERM", "xterm") diff --git a/tests/unit/config/test___main__.py b/tests/unit/config/test___main__.py index fc6b3eef4..0bd2569d6 100644 --- a/tests/unit/config/test___main__.py +++ b/tests/unit/config/test___main__.py @@ -52,6 +52,16 @@ def test_fail_no_traceback(raise_on_session_done, tmp_path, capsys): assert err == "err\n" +def test_discovery_fails_no_discovery_plugin(mocker, tmp_path, capsys): + mocker.patch("virtualenv.run.plugin.discovery.Discovery.entry_points_for", return_value={}) + with pytest.raises(SystemExit) as context: + run_with_catch([str(tmp_path)]) + assert context.value.code == 1 + out, err = capsys.readouterr() + assert "RuntimeError: No discovery plugin found. Try reinstalling virtualenv to fix this issue." in out + assert not err + + def test_fail_with_traceback(raise_on_session_done, tmp_path, capsys): raise_on_session_done(TypeError("something bad")) @@ -64,7 +74,7 @@ def test_fail_with_traceback(raise_on_session_done, tmp_path, capsys): @pytest.mark.usefixtures("session_app_data") def test_session_report_full(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: - run_with_catch([str(tmp_path), "--setuptools", "bundle", "--wheel", "bundle"]) + run_with_catch([str(tmp_path), "--setuptools", "bundle"]) out, err = capsys.readouterr() assert not err lines = out.splitlines() @@ -72,7 +82,7 @@ def test_session_report_full(tmp_path: Path, capsys: pytest.CaptureFixture[str]) r"created virtual environment .* in \d+ms", r" creator .*", r" seeder .*", - r" added seed packages: .*pip==.*, setuptools==.*, wheel==.*", + r" added seed packages: .*pip==.*, setuptools==.*", r" activators .*", ] _match_regexes(lines, regexes) diff --git a/tests/unit/create/test_creator.py b/tests/unit/create/test_creator.py index ff8f08577..c72e830bc 100644 --- a/tests/unit/create/test_creator.py +++ b/tests/unit/create/test_creator.py @@ -11,6 +11,7 @@ import stat import subprocess import sys +import textwrap import zipfile from collections import OrderedDict from itertools import product @@ -24,6 +25,7 @@ from virtualenv.__main__ import run, run_with_catch from virtualenv.create.creator import DEBUG_SCRIPT, Creator, get_env_debug_info from virtualenv.create.pyenv_cfg import PyEnvCfg +from virtualenv.create.via_global_ref import api from virtualenv.create.via_global_ref.builtin.cpython.common import is_macos_brew from virtualenv.create.via_global_ref.builtin.cpython.cpython3 import CPython3Posix from virtualenv.discovery.py_info import PythonInfo @@ -223,6 +225,33 @@ def list_to_str(iterable): assert git_ignore.splitlines() == [comment, "*"] +def test_create_cachedir_tag(tmp_path): + cachedir_tag_file = tmp_path / "CACHEDIR.TAG" + cli_run([str(tmp_path), "--without-pip", "--activators", ""]) + + expected = """ + Signature: 8a477f597d28d172789f06886806bc55 + # This file is a cache directory tag created by Python virtualenv. + # For information about cache directory tags, see: + # https://bford.info/cachedir/ + """ + assert cachedir_tag_file.read_text(encoding="utf-8") == textwrap.dedent(expected).strip() + + +def test_create_cachedir_tag_exists(tmp_path: Path) -> None: + cachedir_tag_file = tmp_path / "CACHEDIR.TAG" + cachedir_tag_file.write_text("magic", encoding="utf-8") + cli_run([str(tmp_path), "--without-pip", "--activators", ""]) + assert cachedir_tag_file.read_text(encoding="utf-8") == "magic" + + +def test_create_cachedir_tag_exists_override(tmp_path: Path) -> None: + cachedir_tag_file = tmp_path / "CACHEDIR.TAG" + cachedir_tag_file.write_text("magic", encoding="utf-8") + cli_run([str(tmp_path), "--without-pip", "--activators", ""]) + assert cachedir_tag_file.read_text(encoding="utf-8") == "magic" + + def test_create_vcs_ignore_exists(tmp_path): git_ignore = tmp_path / ".gitignore" git_ignore.write_text("magic", encoding="utf-8") @@ -370,6 +399,7 @@ def test_create_long_path(tmp_path): subprocess.check_call([str(result.creator.script("pip")), "--version"]) +@pytest.mark.slow @pytest.mark.parametrize("creator", sorted(set(PythonInfo.current_system().creators().key_to_class) - {"builtin"})) @pytest.mark.usefixtures("session_app_data") def test_create_distutils_cfg(creator, tmp_path, monkeypatch): @@ -382,8 +412,6 @@ def test_create_distutils_cfg(creator, tmp_path, monkeypatch): creator, "--setuptools", "bundle", - "--wheel", - "bundle", ], ) @@ -412,7 +440,6 @@ def test_create_distutils_cfg(creator, tmp_path, monkeypatch): "--disable-pip-version-check", "install", str(dest), - "--no-use-pep517", "-vv", ] subprocess.check_call(install_demo_cmd) @@ -663,3 +690,63 @@ def func(self): cmd = [str(tmp_path), "--seeder", "app-data", "--without-pip", "--creator", "venv"] cli_run(cmd) + + +def test_fallback_to_copies_if_symlink_unsupported(tmp_path, python, mocker): + """Test that creating a virtual environment falls back to copies when filesystem has no symlink support.""" + if is_macos_brew(PythonInfo.from_exe(python)): + pytest.skip("brew python on darwin may not support copies, which is tested separately") + + # Given a filesystem that does not support symlinks + mocker.patch("virtualenv.create.via_global_ref.api.fs_supports_symlink", return_value=False) + + # When creating a virtual environment (no method specified) + cmd = [ + "-v", + "-p", + str(python), + str(tmp_path), + "--without-pip", + "--activators", + "", + ] + result = cli_run(cmd) + + # Then the creation should succeed and the creator should report it used copies + assert result.creator is not None + assert result.creator.symlinks is False + + +def test_fail_gracefully_if_no_method_supported(tmp_path, python, mocker): + """Test that virtualenv fails gracefully when no creation method is supported.""" + # Given a filesystem that does not support symlinks + mocker.patch("virtualenv.create.via_global_ref.api.fs_supports_symlink", return_value=False) + + # And a creator that does not support copying + if not is_macos_brew(PythonInfo.from_exe(python)): + original_init = api.ViaGlobalRefMeta.__init__ + + def new_init(self, *args, **kwargs): + original_init(self, *args, **kwargs) + self.copy_error = "copying is not supported" + + mocker.patch("virtualenv.create.via_global_ref.api.ViaGlobalRefMeta.__init__", new=new_init) + + # When creating a virtual environment + with pytest.raises(RuntimeError) as excinfo: + cli_run( + [ + "-p", + str(python), + str(tmp_path), + "--without-pip", + ], + ) + + # Then a RuntimeError should be raised with a detailed message + assert "neither symlink or copy method supported" in str(excinfo.value) + assert "symlink: the filesystem does not supports symlink" in str(excinfo.value) + if is_macos_brew(PythonInfo.from_exe(python)): + assert "copy: Brew disables copy creation" in str(excinfo.value) + else: + assert "copy: copying is not supported" in str(excinfo.value) diff --git a/tests/unit/create/via_global_ref/_test_race_condition_helper.py b/tests/unit/create/via_global_ref/_test_race_condition_helper.py new file mode 100644 index 000000000..8027f17d3 --- /dev/null +++ b/tests/unit/create/via_global_ref/_test_race_condition_helper.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from typing import ClassVar + + +class _Finder: + fullname = None + lock: ClassVar[list] = [] + + def find_spec(self, fullname, path, target=None): # noqa: ARG002 + # This should handle the NameError gracefully + try: + distutils_patch = _DISTUTILS_PATCH + except NameError: + return + if fullname in distutils_patch and self.fullname is None: + return + return + + @staticmethod + def exec_module(old, module): + old(module) + try: + distutils_patch = _DISTUTILS_PATCH + except NameError: + return + if module.__name__ in distutils_patch: + pass # Would call patch_dist(module) + + @staticmethod + def load_module(old, name): + module = old(name) + try: + distutils_patch = _DISTUTILS_PATCH + except NameError: + return module + if module.__name__ in distutils_patch: + pass # Would call patch_dist(module) + return module + + +finder = _Finder() diff --git a/tests/unit/create/via_global_ref/builtin/cpython/cpython3_win_embed.json b/tests/unit/create/via_global_ref/builtin/cpython/cpython3_win_embed.json index e8d0d01c9..c75c6f4fc 100644 --- a/tests/unit/create/via_global_ref/builtin/cpython/cpython3_win_embed.json +++ b/tests/unit/create/via_global_ref/builtin/cpython/cpython3_win_embed.json @@ -57,5 +57,6 @@ "system_stdlib": "c:\\path\\to\\python\\Lib", "system_stdlib_platform": "c:\\path\\to\\python\\Lib", "max_size": 9223372036854775807, - "_creators": null + "_creators": null, + "free_threaded": false } diff --git a/tests/unit/create/via_global_ref/builtin/cpython/test_cpython3_win.py b/tests/unit/create/via_global_ref/builtin/cpython/test_cpython3_win.py index f831de114..4367ebc50 100644 --- a/tests/unit/create/via_global_ref/builtin/cpython/test_cpython3_win.py +++ b/tests/unit/create/via_global_ref/builtin/cpython/test_cpython3_win.py @@ -99,3 +99,11 @@ def test_no_python_zip_if_not_exists(py_info, mock_files): sources = tuple(CPython3Windows.sources(interpreter=py_info)) assert python_zip in py_info.path assert not contains_ref(sources, python_zip) + + +@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"]) +def test_python3_exe_present(py_info, mock_files): + mock_files(CPYTHON3_PATH, [py_info.system_executable]) + sources = tuple(CPython3Windows.sources(interpreter=py_info)) + assert contains_exe(sources, py_info.system_executable, "python3.exe") + assert contains_exe(sources, py_info.system_executable, "python3") diff --git a/tests/unit/create/via_global_ref/builtin/pypy/deb_pypy37.json b/tests/unit/create/via_global_ref/builtin/pypy/deb_pypy37.json index eb694a840..4e6ca4dda 100644 --- a/tests/unit/create/via_global_ref/builtin/pypy/deb_pypy37.json +++ b/tests/unit/create/via_global_ref/builtin/pypy/deb_pypy37.json @@ -60,5 +60,6 @@ "system_stdlib": "/usr/lib/pypy3/lib-python/3.7", "system_stdlib_platform": "/usr/lib/pypy3/lib-python/3.7", "max_size": 9223372036854775807, - "_creators": null + "_creators": null, + "free_threaded": false } diff --git a/tests/unit/create/via_global_ref/builtin/pypy/deb_pypy38.json b/tests/unit/create/via_global_ref/builtin/pypy/deb_pypy38.json index 0d91a969d..070867210 100644 --- a/tests/unit/create/via_global_ref/builtin/pypy/deb_pypy38.json +++ b/tests/unit/create/via_global_ref/builtin/pypy/deb_pypy38.json @@ -60,5 +60,6 @@ "system_stdlib": "/usr/lib/pypy3.8", "system_stdlib_platform": "/usr/lib/pypy3.8", "max_size": 9223372036854775807, - "_creators": null + "_creators": null, + "free_threaded": false } diff --git a/tests/unit/create/via_global_ref/builtin/pypy/portable_pypy38.json b/tests/unit/create/via_global_ref/builtin/pypy/portable_pypy38.json index 0761455bb..136c1b4f6 100644 --- a/tests/unit/create/via_global_ref/builtin/pypy/portable_pypy38.json +++ b/tests/unit/create/via_global_ref/builtin/pypy/portable_pypy38.json @@ -59,5 +59,6 @@ "system_stdlib": "/tmp/pypy3.8-v7.3.8-linux64/lib/pypy3.8", "system_stdlib_platform": "/tmp/pypy3.8-v7.3.8-linux64/lib/pypy3.8", "max_size": 9223372036854775807, - "_creators": null + "_creators": null, + "free_threaded": false } diff --git a/tests/unit/create/via_global_ref/test_build_c_ext.py b/tests/unit/create/via_global_ref/test_build_c_ext.py index db1c16eb0..5195ff856 100644 --- a/tests/unit/create/via_global_ref/test_build_c_ext.py +++ b/tests/unit/create/via_global_ref/test_build_c_ext.py @@ -26,6 +26,7 @@ def builtin_shows_marker_missing(): return not marker.exists() +@pytest.mark.slow @pytest.mark.xfail( condition=bool(os.environ.get("CI_RUN")), strict=False, @@ -41,10 +42,19 @@ def test_can_build_c_extensions(creator, tmp_path, coverage_env): shutil.copytree(str(Path(__file__).parent.resolve() / "greet"), greet) session = cli_run(["--creator", creator, "--seeder", "app-data", str(env), "-vvv"]) coverage_env() + setuptools_index_args = () + if CURRENT.version_info >= (3, 12): + # requires to be able to install setuptools as build dependency + setuptools_index_args = ( + "--find-links", + "https://pypi.org/simple/setuptools/", + ) + cmd = [ str(session.creator.script("pip")), "install", "--no-index", + *setuptools_index_args, "--no-deps", "--disable-pip-version-check", "-vvv", diff --git a/tests/unit/create/via_global_ref/test_race_condition.py b/tests/unit/create/via_global_ref/test_race_condition.py new file mode 100644 index 000000000..3b044167c --- /dev/null +++ b/tests/unit/create/via_global_ref/test_race_condition.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import sys +from pathlib import Path + + +def test_virtualenv_py_race_condition_find_spec(tmp_path): + """Test that _Finder.find_spec handles NameError gracefully when _DISTUTILS_PATCH is not defined.""" + # Create a temporary file with partial _virtualenv.py content (simulating race condition) + venv_file = tmp_path / "_virtualenv_test.py" + + # Write a partial version of _virtualenv.py that has _Finder but not _DISTUTILS_PATCH + # This simulates the state during a race condition where the file is being rewritten + helper_file = Path(__file__).parent / "_test_race_condition_helper.py" + partial_content = helper_file.read_text(encoding="utf-8") + + venv_file.write_text(partial_content, encoding="utf-8") + + sys.path.insert(0, str(tmp_path)) + try: + import _virtualenv_test # noqa: PLC0415 + + finder = _virtualenv_test.finder + + # Try to call find_spec - this should not raise NameError + result = finder.find_spec("distutils.dist", None) + assert result is None, "find_spec should return None when _DISTUTILS_PATCH is not defined" + + # Create a mock module object + class MockModule: + __name__ = "distutils.dist" + + # Try to call exec_module - this should not raise NameError + def mock_old_exec(_x): + pass + + finder.exec_module(mock_old_exec, MockModule()) + + # Try to call load_module - this should not raise NameError + def mock_old_load(_name): + return MockModule() + + result = finder.load_module(mock_old_load, "distutils.dist") + assert result.__name__ == "distutils.dist" + + finally: + sys.path.remove(str(tmp_path)) + if "_virtualenv_test" in sys.modules: + del sys.modules["_virtualenv_test"] + + +def test_virtualenv_py_normal_operation(): + """Test that the fix doesn't break normal operation when _DISTUTILS_PATCH is defined.""" + # Read the actual _virtualenv.py file + virtualenv_py_path = ( + Path(__file__).parent.parent.parent.parent.parent + / "src" + / "virtualenv" + / "create" + / "via_global_ref" + / "_virtualenv.py" + ) + + if not virtualenv_py_path.exists(): + return # Skip if we can't find the file + + content = virtualenv_py_path.read_text(encoding="utf-8") + + # Verify the fix is present + assert "try:" in content + assert "distutils_patch = _DISTUTILS_PATCH" in content + assert "except NameError:" in content + assert "return None" in content or "return" in content diff --git a/tests/unit/create/virtualenv-16.7.9-py2.py3-none-any.whl b/tests/unit/create/virtualenv-16.7.9-py2.py3-none-any.whl deleted file mode 100644 index 46331cd04..000000000 Binary files a/tests/unit/create/virtualenv-16.7.9-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/unit/discovery/py_info/test_py_info.py b/tests/unit/discovery/py_info/test_py_info.py index 7e9994fa3..15472daab 100644 --- a/tests/unit/discovery/py_info/test_py_info.py +++ b/tests/unit/discovery/py_info/test_py_info.py @@ -14,6 +14,7 @@ import pytest +from virtualenv.create.via_global_ref.builtin.cpython.common import is_macos_brew from virtualenv.discovery import cached_py_info from virtualenv.discovery.py_info import PythonInfo, VersionInfo from virtualenv.discovery.py_spec import PythonSpec @@ -26,7 +27,9 @@ def test_current_as_json(): result = CURRENT._to_json() # noqa: SLF001 parsed = json.loads(result) a, b, c, d, e = sys.version_info + f = sysconfig.get_config_var("Py_GIL_DISABLED") == 1 assert parsed["version_info"] == {"major": a, "minor": b, "micro": c, "releaselevel": d, "serial": e} + assert parsed["free_threaded"] is f def test_bad_exe_py_info_raise(tmp_path, session_app_data): @@ -59,7 +62,7 @@ def test_bad_exe_py_info_no_raise(tmp_path, caplog, capsys, session_app_data): itertools.chain( [sys.executable], [ - f"{impl}{'.'.join(str(i) for i in ver)}{arch}" + f"{impl}{'.'.join(str(i) for i in ver)}{'t' if CURRENT.free_threaded else ''}{arch}" for impl, ver, arch in itertools.product( ( [CURRENT.implementation] @@ -90,6 +93,14 @@ def test_satisfy_not_arch(): assert matches is False +def test_satisfy_not_threaded(): + parsed_spec = PythonSpec.from_string_spec( + f"{CURRENT.implementation}{CURRENT.version_info.major}{'' if CURRENT.free_threaded else 't'}", + ) + matches = CURRENT.satisfies(parsed_spec, True) + assert matches is False + + def _generate_not_match_current_interpreter_version(): result = [] for i in range(3): @@ -144,6 +155,41 @@ def test_py_info_cache_clear(mocker, session_app_data): assert spy.call_count >= 2 * count +def test_py_info_cache_invalidation_on_py_info_change(mocker, session_app_data): + # 1. Get a PythonInfo object for the current executable, this will cache it. + PythonInfo.from_exe(sys.executable, session_app_data) + + # 2. Spy on _run_subprocess + spy = mocker.spy(cached_py_info, "_run_subprocess") + + # 3. Backup py_info.py + py_info_script = Path(cached_py_info.__file__).parent / "py_info.py" + original_content = py_info_script.read_text(encoding="utf-8") + original_stat = py_info_script.stat() + + try: + # 4. Clear the in-memory cache + mocker.patch.dict(cached_py_info._CACHE, {}, clear=True) # noqa: SLF001 + + # 5. Modify py_info.py to invalidate the cache + py_info_script.write_text(original_content + "\n# a comment", encoding="utf-8") + + # 6. Get the PythonInfo object again + info = PythonInfo.from_exe(sys.executable, session_app_data) + + # 7. Assert that _run_subprocess was called again + native_difference = 1 if info.system_executable == info.executable else 0 + if is_macos_brew(info): + assert spy.call_count + native_difference in {2, 3} + else: + assert spy.call_count + native_difference == 2 + + finally: + # 8. Restore the original content and timestamp + py_info_script.write_text(original_content, encoding="utf-8") + os.utime(str(py_info_script), (original_stat.st_atime, original_stat.st_mtime)) + + @pytest.mark.skipif(not fs_supports_symlink(), reason="symlink is not supported") @pytest.mark.xfail( # https://doc.pypy.org/en/latest/install.html?highlight=symlink#download-a-pre-built-pypy @@ -390,7 +436,9 @@ def test_custom_venv_install_scheme_is_prefered(mocker): assert pyinfo.install_path("purelib").replace(os.sep, "/") == f"lib/python{pyver}/site-packages" -@pytest.mark.skipif(not (os.name == "posix" and sys.version_info[:2] >= (3, 11)), reason="POSIX 3.11+ specific") +@pytest.mark.skipif( + IS_PYPY or not (os.name == "posix" and sys.version_info[:2] >= (3, 11)), reason="POSIX 3.11+ specific" +) def test_fallback_existent_system_executable(mocker): current = PythonInfo() # Posix may execute a "python" out of a venv but try to set the base_executable @@ -406,11 +454,11 @@ def test_fallback_existent_system_executable(mocker): mocker.patch.object(sys, "executable", current.executable) # ensure it falls back to an alternate binary name that exists - current._fast_get_system_executable() # noqa: SLF001 - assert os.path.basename(current.system_executable) in [ + system_executable = current._fast_get_system_executable() # noqa: SLF001 + assert os.path.basename(system_executable) in [ f"python{v}" for v in (current.version_info.major, f"{current.version_info.major}.{current.version_info.minor}") ] - assert os.path.exists(current.system_executable) + assert os.path.exists(system_executable) @pytest.mark.skipif(sys.version_info[:2] != (3, 10), reason="3.10 specific") diff --git a/tests/unit/discovery/py_info/test_py_info_exe_based_of.py b/tests/unit/discovery/py_info/test_py_info_exe_based_of.py index c843ca7be..90894a59c 100644 --- a/tests/unit/discovery/py_info/test_py_info_exe_based_of.py +++ b/tests/unit/discovery/py_info/test_py_info_exe_based_of.py @@ -30,7 +30,7 @@ def test_discover_ok(tmp_path, suffix, impl, version, arch, into, caplog, sessio caplog.set_level(logging.DEBUG) folder = tmp_path / into folder.mkdir(parents=True, exist_ok=True) - name = f"{impl}{version}" + name = f"{impl}{version}{'t' if CURRENT.free_threaded else ''}" if arch: name += f"-{arch}" name += suffix diff --git a/tests/unit/discovery/test_discovery.py b/tests/unit/discovery/test_discovery.py index 680131e7d..e6c78e2e3 100644 --- a/tests/unit/discovery/test_discovery.py +++ b/tests/unit/discovery/test_discovery.py @@ -2,16 +2,18 @@ import logging import os +import subprocess import sys from argparse import Namespace from pathlib import Path +from unittest.mock import patch from uuid import uuid4 import pytest -from virtualenv.discovery.builtin import Builtin, get_interpreter +from virtualenv.discovery.builtin import Builtin, LazyPathDump, get_interpreter, get_paths from virtualenv.discovery.py_info import PythonInfo -from virtualenv.info import fs_supports_symlink +from virtualenv.info import IS_WIN, fs_supports_symlink @pytest.mark.skipif(not fs_supports_symlink(), reason="symlink not supported") @@ -21,6 +23,7 @@ def test_discovery_via_path(monkeypatch, case, specificity, tmp_path, caplog, se caplog.set_level(logging.DEBUG) current = PythonInfo.current_system(session_app_data) name = "somethingVeryCryptic" + threaded = "t" if current.free_threaded else "" if case == "lower": name = name.lower() elif case == "upper": @@ -28,7 +31,7 @@ def test_discovery_via_path(monkeypatch, case, specificity, tmp_path, caplog, se if specificity == "more": # e.g. spec: python3, exe: /bin/python3.12 core_ver = current.version_info.major - exe_ver = ".".join(str(i) for i in current.version_info[0:2]) + exe_ver = ".".join(str(i) for i in current.version_info[0:2]) + threaded elif specificity == "less": # e.g. spec: python3.12.1, exe: /bin/python3 core_ver = ".".join(str(i) for i in current.version_info[0:3]) @@ -37,7 +40,7 @@ def test_discovery_via_path(monkeypatch, case, specificity, tmp_path, caplog, se # e.g. spec: python3.12.1, exe: /bin/python core_ver = ".".join(str(i) for i in current.version_info[0:3]) exe_ver = "" - core = "" if specificity == "none" else f"{name}{core_ver}" + core = "" if specificity == "none" else f"{name}{core_ver}{threaded}" exe_name = f"{name}{exe_ver}{'.exe' if sys.platform == 'win32' else ''}" target = tmp_path / current.install_path("scripts") target.mkdir(parents=True) @@ -62,6 +65,11 @@ def test_discovery_via_path_not_found(tmp_path, monkeypatch): def test_discovery_via_path_in_nonbrowseable_directory(tmp_path, monkeypatch): bad_perm = tmp_path / "bad_perm" bad_perm.mkdir(mode=0o000) + # path entry is unreadable + monkeypatch.setenv("PATH", str(bad_perm)) + interpreter = get_interpreter(uuid4().hex, []) + assert interpreter is None + # path entry parent is unreadable monkeypatch.setenv("PATH", str(bad_perm / "bin")) interpreter = get_interpreter(uuid4().hex, []) assert interpreter is None @@ -76,6 +84,68 @@ def test_relative_path(session_app_data, monkeypatch): assert result is not None +def test_uv_python(monkeypatch, tmp_path_factory, mocker): + monkeypatch.delenv("UV_PYTHON_INSTALL_DIR", raising=False) + monkeypatch.delenv("XDG_DATA_HOME", raising=False) + monkeypatch.setenv("PATH", "") + mocker.patch.object(PythonInfo, "satisfies", return_value=False) + + # UV_PYTHON_INSTALL_DIR + uv_python_install_dir = tmp_path_factory.mktemp("uv_python_install_dir") + with patch("virtualenv.discovery.builtin.PathPythonInfo.from_exe") as mock_from_exe, monkeypatch.context() as m: + m.setenv("UV_PYTHON_INSTALL_DIR", str(uv_python_install_dir)) + + get_interpreter("python", []) + mock_from_exe.assert_not_called() + + bin_path = uv_python_install_dir.joinpath("some-py-impl", "bin") + bin_path.mkdir(parents=True) + bin_path.joinpath("python").touch() + get_interpreter("python", []) + mock_from_exe.assert_called_once() + assert mock_from_exe.call_args[0][0] == str(bin_path / "python") + + # PATH takes precedence + mock_from_exe.reset_mock() + python_exe = "python.exe" if IS_WIN else "python" + dir_in_path = tmp_path_factory.mktemp("path_bin_dir") + dir_in_path.joinpath(python_exe).touch() + m.setenv("PATH", str(dir_in_path)) + get_interpreter("python", []) + mock_from_exe.assert_called_once() + assert mock_from_exe.call_args[0][0] == str(dir_in_path / python_exe) + + # XDG_DATA_HOME + xdg_data_home = tmp_path_factory.mktemp("xdg_data_home") + with patch("virtualenv.discovery.builtin.PathPythonInfo.from_exe") as mock_from_exe, monkeypatch.context() as m: + m.setenv("XDG_DATA_HOME", str(xdg_data_home)) + + get_interpreter("python", []) + mock_from_exe.assert_not_called() + + bin_path = xdg_data_home.joinpath("uv", "python", "some-py-impl", "bin") + bin_path.mkdir(parents=True) + bin_path.joinpath("python").touch() + get_interpreter("python", []) + mock_from_exe.assert_called_once() + assert mock_from_exe.call_args[0][0] == str(bin_path / "python") + + # User data path + user_data_path = tmp_path_factory.mktemp("user_data_path") + with patch("virtualenv.discovery.builtin.PathPythonInfo.from_exe") as mock_from_exe, monkeypatch.context() as m: + m.setattr("virtualenv.discovery.builtin.user_data_path", lambda x: user_data_path / x) + + get_interpreter("python", []) + mock_from_exe.assert_not_called() + + bin_path = user_data_path.joinpath("uv", "python", "some-py-impl", "bin") + bin_path.mkdir(parents=True) + bin_path.joinpath("python").touch() + get_interpreter("python", []) + mock_from_exe.assert_called_once() + assert mock_from_exe.call_args[0][0] == str(bin_path / "python") + + def test_discovery_fallback_fail(session_app_data, caplog): caplog.set_level(logging.DEBUG) builtin = Builtin( @@ -99,3 +169,163 @@ def test_discovery_fallback_ok(session_app_data, caplog): assert result.executable == sys.executable, caplog.text assert "accepted" in caplog.text + + +@pytest.fixture +def mock_get_interpreter(mocker): + return mocker.patch( + "virtualenv.discovery.builtin.get_interpreter", + lambda key, *args, **kwargs: getattr(mocker.sentinel, key), # noqa: ARG005 + ) + + +@pytest.mark.usefixtures("mock_get_interpreter") +def test_returns_first_python_specified_when_only_env_var_one_is_specified(mocker, monkeypatch, session_app_data): + monkeypatch.setenv("VIRTUALENV_PYTHON", "python_from_env_var") + builtin = Builtin( + Namespace(app_data=session_app_data, try_first_with=[], python=["python_from_env_var"], env=os.environ), + ) + + result = builtin.run() + + assert result == mocker.sentinel.python_from_env_var + + +@pytest.mark.usefixtures("mock_get_interpreter") +def test_returns_second_python_specified_when_more_than_one_is_specified_and_env_var_is_specified( + mocker, monkeypatch, session_app_data +): + monkeypatch.setenv("VIRTUALENV_PYTHON", "python_from_env_var") + builtin = Builtin( + Namespace( + app_data=session_app_data, + try_first_with=[], + python=["python_from_env_var", "python_from_cli"], + env=os.environ, + ), + ) + + result = builtin.run() + + assert result == mocker.sentinel.python_from_cli + + +def test_discovery_absolute_path_with_try_first(tmp_path, session_app_data): + good_env = tmp_path / "good" + bad_env = tmp_path / "bad" + + # Create two real virtual environments + subprocess.check_call([sys.executable, "-m", "virtualenv", str(good_env)]) + subprocess.check_call([sys.executable, "-m", "virtualenv", str(bad_env)]) + + # On Windows, the executable is in Scripts/python.exe + scripts_dir = "Scripts" if IS_WIN else "bin" + exe_name = "python.exe" if IS_WIN else "python" + good_exe = good_env / scripts_dir / exe_name + bad_exe = bad_env / scripts_dir / exe_name + + # The spec is an absolute path, this should be a hard requirement. + # The --try-first-with option should be rejected as it does not match the spec. + interpreter = get_interpreter( + str(good_exe), + try_first_with=[str(bad_exe)], + app_data=session_app_data, + ) + + assert interpreter is not None + assert Path(interpreter.executable) == good_exe + + +def test_discovery_via_path_with_file(tmp_path, monkeypatch): + a_file = tmp_path / "a_file" + a_file.touch() + monkeypatch.setenv("PATH", str(a_file)) + interpreter = get_interpreter(uuid4().hex, []) + assert interpreter is None + + +def test_absolute_path_does_not_exist(tmp_path): + """ + Test that virtualenv does not fail when an absolute path that does not exist is provided. + """ + # Create a command that uses an absolute path that does not exist + # and a valid python executable. + command = [ + sys.executable, + "-m", + "virtualenv", + "-p", + "/this/path/does/not/exist", + "-p", + sys.executable, + str(tmp_path / "dest"), + ] + + # Run the command + process = subprocess.run( + command, + capture_output=True, + text=True, + check=False, + encoding="utf-8", + ) + + # Check that the command was successful + assert process.returncode == 0, process.stderr + + +def test_absolute_path_does_not_exist_fails(tmp_path): + """ + Test that virtualenv fails when a single absolute path that does not exist is provided. + """ + # Create a command that uses an absolute path that does not exist + command = [ + sys.executable, + "-m", + "virtualenv", + "-p", + "/this/path/does/not/exist", + str(tmp_path / "dest"), + ] + + # Run the command + process = subprocess.run( + command, + capture_output=True, + text=True, + check=False, + encoding="utf-8", + ) + + # Check that the command failed + assert process.returncode != 0, process.stderr + + +def test_get_paths_no_path_env(monkeypatch): + monkeypatch.delenv("PATH", raising=False) + paths = list(get_paths({})) + assert paths + + +def test_lazy_path_dump_debug(monkeypatch, tmp_path): + monkeypatch.setenv("_VIRTUALENV_DEBUG", "1") + a_dir = tmp_path + executable_file = "a_file.exe" if IS_WIN else "a_file" + (a_dir / executable_file).touch(mode=0o755) + (a_dir / "b_file").touch(mode=0o644) + dumper = LazyPathDump(0, a_dir, os.environ) + output = repr(dumper) + assert executable_file in output + assert "b_file" not in output + + +@pytest.mark.usefixtures("mock_get_interpreter") +def test_returns_first_python_specified_when_no_env_var_is_specified(mocker, monkeypatch, session_app_data): + monkeypatch.delenv("VIRTUALENV_PYTHON", raising=False) + builtin = Builtin( + Namespace(app_data=session_app_data, try_first_with=[], python=["python_from_cli"], env=os.environ), + ) + + result = builtin.run() + + assert result == mocker.sentinel.python_from_cli diff --git a/tests/unit/discovery/test_py_spec.py b/tests/unit/discovery/test_py_spec.py index 765686645..0841019ec 100644 --- a/tests/unit/discovery/test_py_spec.py +++ b/tests/unit/discovery/test_py_spec.py @@ -45,6 +45,16 @@ def test_spec_satisfies_arch(): assert spec_2.satisfies(spec_1) is False +def test_spec_satisfies_free_threaded(): + spec_1 = PythonSpec.from_string_spec("python3.13t") + spec_2 = PythonSpec.from_string_spec("python3.13") + + assert spec_1.satisfies(spec_1) is True + assert spec_1.free_threaded is True + assert spec_2.satisfies(spec_1) is False + assert spec_2.free_threaded is False + + @pytest.mark.parametrize( ("req", "spec"), [("py", "python"), ("jython", "jython"), ("CPython", "cpython")], @@ -66,13 +76,22 @@ def test_spec_satisfies_implementation_nok(): def _version_satisfies_pairs(): target = set() version = tuple(str(i) for i in sys.version_info[0:3]) - for i in range(len(version) + 1): - req = ".".join(version[0:i]) - for j in range(i + 1): - sat = ".".join(version[0:j]) - # can be satisfied in both directions - target.add((req, sat)) - target.add((sat, req)) + for threading in (False, True): + for i in range(len(version) + 1): + req = ".".join(version[0:i]) + for j in range(i + 1): + sat = ".".join(version[0:j]) + # can be satisfied in both directions + if sat: + target.add((req, sat)) + # else: no version => no free-threading info + target.add((sat, req)) + if not threading or not sat or not req: + # free-threading info requires a version + continue + target.add((f"{req}t", f"{sat}t")) + target.add((f"{sat}t", f"{req}t")) + return sorted(target) diff --git a/tests/unit/discovery/windows/conftest.py b/tests/unit/discovery/windows/conftest.py index 21c16891c..f75278e06 100644 --- a/tests/unit/discovery/windows/conftest.py +++ b/tests/unit/discovery/windows/conftest.py @@ -65,7 +65,7 @@ def _open_key_ex(*args): mocker.patch("os.path.exists", return_value=True) -def _mock_pyinfo(major, minor, arch, exe): +def _mock_pyinfo(major, minor, arch, exe, threaded=False): """Return PythonInfo objects with essential metadata set for the given args""" from virtualenv.discovery.py_info import PythonInfo, VersionInfo # noqa: PLC0415 @@ -75,6 +75,7 @@ def _mock_pyinfo(major, minor, arch, exe): info.implementation = "CPython" info.architecture = arch info.version_info = VersionInfo(major, minor, 0, "final", 0) + info.free_threaded = threaded return info @@ -84,19 +85,21 @@ def _populate_pyinfo_cache(monkeypatch): import virtualenv.discovery.cached_py_info # noqa: PLC0415 # Data matches _mock_registry fixture + python_core_path = "C:\\Users\\user\\AppData\\Local\\Programs\\Python" interpreters = [ - ("ContinuumAnalytics", 3, 10, 32, "C:\\Users\\user\\Miniconda3\\python.exe", None), - ("ContinuumAnalytics", 3, 10, 64, "C:\\Users\\user\\Miniconda3-64\\python.exe", None), - ("PythonCore", 3, 9, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python36\\python.exe", None), - ("PythonCore", 3, 9, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python36\\python.exe", None), - ("PythonCore", 3, 5, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python35\\python.exe", None), - ("PythonCore", 3, 9, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python36\\python.exe", None), - ("PythonCore", 3, 7, 32, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python37-32\\python.exe", None), - ("PythonCore", 3, 12, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe", None), - ("PythonCore", 2, 7, 64, "C:\\Python27\\python.exe", None), - ("PythonCore", 3, 4, 64, "C:\\Python34\\python.exe", None), - ("CompanyA", 3, 6, 64, "Z:\\CompanyA\\Python\\3.6\\python.exe", None), + ("ContinuumAnalytics", 3, 10, 32, False, "C:\\Users\\user\\Miniconda3\\python.exe"), + ("ContinuumAnalytics", 3, 10, 64, False, "C:\\Users\\user\\Miniconda3-64\\python.exe"), + ("PythonCore", 3, 9, 64, False, f"{python_core_path}\\Python36\\python.exe"), + ("PythonCore", 3, 9, 64, False, f"{python_core_path}\\Python36\\python.exe"), + ("PythonCore", 3, 5, 64, False, f"{python_core_path}\\Python35\\python.exe"), + ("PythonCore", 3, 9, 64, False, f"{python_core_path}\\Python36\\python.exe"), + ("PythonCore", 3, 7, 32, False, f"{python_core_path}\\Python37-32\\python.exe"), + ("PythonCore", 3, 12, 64, False, f"{python_core_path}\\Python312\\python.exe"), + ("PythonCore", 3, 13, 64, True, f"{python_core_path}\\Python313\\python3.13t.exe"), + ("PythonCore", 2, 7, 64, False, "C:\\Python27\\python.exe"), + ("PythonCore", 3, 4, 64, False, "C:\\Python34\\python.exe"), + ("CompanyA", 3, 6, 64, False, "Z:\\CompanyA\\Python\\3.6\\python.exe"), ] - for _, major, minor, arch, exe, _ in interpreters: - info = _mock_pyinfo(major, minor, arch, exe) + for _, major, minor, arch, threaded, exe in interpreters: + info = _mock_pyinfo(major, minor, arch, exe, threaded) monkeypatch.setitem(virtualenv.discovery.cached_py_info._CACHE, Path(info.executable), info) # noqa: SLF001 diff --git a/tests/unit/discovery/windows/test_windows.py b/tests/unit/discovery/windows/test_windows.py index aca2afc14..594a1302f 100644 --- a/tests/unit/discovery/windows/test_windows.py +++ b/tests/unit/discovery/windows/test_windows.py @@ -20,11 +20,16 @@ ("python3.12", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe"), ("cpython3.12", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe"), # resolves to highest available version - ("python", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe"), - ("cpython", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe"), + ("python", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python313\\python3.13t.exe"), + ("cpython", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python313\\python3.13t.exe"), + ("python3", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe"), + ("cpython3", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe"), # Non-standard org name ("python3.6", "Z:\\CompanyA\\Python\\3.6\\python.exe"), ("cpython3.6", "Z:\\CompanyA\\Python\\3.6\\python.exe"), + # free-threaded + ("3t", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python313\\python3.13t.exe"), + ("python3.13t", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python313\\python3.13t.exe"), ], ) def test_propose_interpreters(string_spec, expected_exe): diff --git a/tests/unit/discovery/windows/test_windows_pep514.py b/tests/unit/discovery/windows/test_windows_pep514.py index 3b1c46984..0498352aa 100644 --- a/tests/unit/discovery/windows/test_windows_pep514.py +++ b/tests/unit/discovery/windows/test_windows_pep514.py @@ -13,17 +13,74 @@ def test_pep514(): interpreters = list(discover_pythons()) assert interpreters == [ - ("ContinuumAnalytics", 3, 10, 32, "C:\\Users\\user\\Miniconda3\\python.exe", None), - ("ContinuumAnalytics", 3, 10, 64, "C:\\Users\\user\\Miniconda3-64\\python.exe", None), - ("PythonCore", 3, 9, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", None), - ("PythonCore", 3, 9, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", None), - ("PythonCore", 3, 8, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python38\\python.exe", None), - ("PythonCore", 3, 9, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", None), - ("PythonCore", 3, 10, 32, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python310-32\\python.exe", None), - ("PythonCore", 3, 12, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe", None), - ("CompanyA", 3, 6, 64, "Z:\\CompanyA\\Python\\3.6\\python.exe", None), - ("PythonCore", 2, 7, 64, "C:\\Python27\\python.exe", None), - ("PythonCore", 3, 7, 64, "C:\\Python37\\python.exe", None), + ("ContinuumAnalytics", 3, 10, 32, False, "C:\\Users\\user\\Miniconda3\\python.exe", None), + ("ContinuumAnalytics", 3, 10, 64, False, "C:\\Users\\user\\Miniconda3-64\\python.exe", None), + ( + "PythonCore", + 3, + 9, + 64, + False, + "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", + None, + ), + ( + "PythonCore", + 3, + 9, + 64, + False, + "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", + None, + ), + ( + "PythonCore", + 3, + 8, + 64, + False, + "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python38\\python.exe", + None, + ), + ( + "PythonCore", + 3, + 9, + 64, + False, + "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", + None, + ), + ( + "PythonCore", + 3, + 10, + 32, + False, + "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python310-32\\python.exe", + None, + ), + ( + "PythonCore", + 3, + 12, + 64, + False, + "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe", + None, + ), + ( + "PythonCore", + 3, + 13, + 64, + True, + "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python313\\python3.13t.exe", + None, + ), + ("CompanyA", 3, 6, 64, False, "Z:\\CompanyA\\Python\\3.6\\python.exe", None), + ("PythonCore", 2, 7, 64, False, "C:\\Python27\\python.exe", None), + ("PythonCore", 3, 7, 64, False, "C:\\Python37\\python.exe", None), ] @@ -36,18 +93,19 @@ def test_pep514_run(capsys, caplog): out, err = capsys.readouterr() expected = textwrap.dedent( r""" - ('CompanyA', 3, 6, 64, 'Z:\\CompanyA\\Python\\3.6\\python.exe', None) - ('ContinuumAnalytics', 3, 10, 32, 'C:\\Users\\user\\Miniconda3\\python.exe', None) - ('ContinuumAnalytics', 3, 10, 64, 'C:\\Users\\user\\Miniconda3-64\\python.exe', None) - ('PythonCore', 2, 7, 64, 'C:\\Python27\\python.exe', None) - ('PythonCore', 3, 10, 32, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python310-32\\python.exe', None) - ('PythonCore', 3, 12, 64, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe', None) - ('PythonCore', 3, 7, 64, 'C:\\Python37\\python.exe', None) - ('PythonCore', 3, 8, 64, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python38\\python.exe', None) - ('PythonCore', 3, 9, 64, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe', None) - ('PythonCore', 3, 9, 64, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe', None) - ('PythonCore', 3, 9, 64, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe', None) - """, + ('CompanyA', 3, 6, 64, False, 'Z:\\CompanyA\\Python\\3.6\\python.exe', None) + ('ContinuumAnalytics', 3, 10, 32, False, 'C:\\Users\\user\\Miniconda3\\python.exe', None) + ('ContinuumAnalytics', 3, 10, 64, False, 'C:\\Users\\user\\Miniconda3-64\\python.exe', None) + ('PythonCore', 2, 7, 64, False, 'C:\\Python27\\python.exe', None) + ('PythonCore', 3, 10, 32, False, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python310-32\\python.exe', None) + ('PythonCore', 3, 12, 64, False, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe', None) + ('PythonCore', 3, 13, 64, True, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python313\\python3.13t.exe', None) + ('PythonCore', 3, 7, 64, False, 'C:\\Python37\\python.exe', None) + ('PythonCore', 3, 8, 64, False, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python38\\python.exe', None) + ('PythonCore', 3, 9, 64, False, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe', None) + ('PythonCore', 3, 9, 64, False, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe', None) + ('PythonCore', 3, 9, 64, False, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe', None) + """, # noqa: E501 ).strip() assert out.strip() == expected assert not err diff --git a/tests/unit/discovery/windows/winreg-mock-values.py b/tests/unit/discovery/windows/winreg-mock-values.py index da76c56b9..fa2619ba7 100644 --- a/tests/unit/discovery/windows/winreg-mock-values.py +++ b/tests/unit/discovery/windows/winreg-mock-values.py @@ -35,6 +35,8 @@ "3.11": 78700656, "3.12\\InstallPath": 78703632, "3.12": 78702608, + "3.13t\\InstallPath": 78703633, + "3.13t": 78702609, "3.X": 78703088, }, 78702960: {"2.7\\InstallPath": 78700912, "2.7": 78703136, "3.7\\InstallPath": 78703648, "3.7": 78704032}, @@ -45,27 +47,27 @@ }, } value_collect = { - 78703568: {"SysVersion": ("3.10", 1), "SysArchitecture": ("32bit", 1)}, + 78703568: {"SysVersion": ("3.10", 1), "SysArchitecture": ("32bit", 1), "DisplayName": ("Python 3.10 (32-bit)", 1)}, 78703200: { "ExecutablePath": ("C:\\Users\\user\\Miniconda3\\python.exe", 1), "ExecutableArguments": OSError(2, "The system cannot find the file specified"), }, - 78702368: {"SysVersion": ("3.10", 1), "SysArchitecture": ("64bit", 1)}, + 78702368: {"SysVersion": ("3.10", 1), "SysArchitecture": ("64bit", 1), "DisplayName": ("Python 3.10 (64-bit)", 1)}, 78703520: { "ExecutablePath": ("C:\\Users\\user\\Miniconda3-64\\python.exe", 1), "ExecutableArguments": OSError(2, "The system cannot find the file specified"), }, - 78700704: {"SysVersion": ("3.9", 1), "SysArchitecture": ("magic", 1)}, + 78700704: {"SysVersion": ("3.9", 1), "SysArchitecture": ("magic", 1), "DisplayName": ("Python 3.9 (wizardry)", 1)}, 78701824: { "ExecutablePath": ("C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", 1), "ExecutableArguments": OSError(2, "The system cannot find the file specified"), }, - 78704368: {"SysVersion": ("3.9", 1), "SysArchitecture": (100, 4)}, + 78704368: {"SysVersion": ("3.9", 1), "SysArchitecture": (100, 4), "DisplayName": ("Python 3.9 (64-bit)", 1)}, 78704048: { "ExecutablePath": ("C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", 1), "ExecutableArguments": OSError(2, "The system cannot find the file specified"), }, - 78703024: {"SysVersion": ("3.9", 1), "SysArchitecture": ("64bit", 1)}, + 78703024: {"SysVersion": ("3.9", 1), "SysArchitecture": ("64bit", 1), "DisplayName": ("Python 3.9 (64-bit)", 1)}, 78701936: { "ExecutablePath": OSError(2, "The system cannot find the file specified"), None: OSError(2, "The system cannot find the file specified"), @@ -73,17 +75,18 @@ 78701792: { "SysVersion": OSError(2, "The system cannot find the file specified"), "SysArchitecture": OSError(2, "The system cannot find the file specified"), + "DisplayName": OSError(2, "The system cannot find the file specified"), }, 78703792: { "ExecutablePath": ("C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python38\\python.exe", 1), "ExecutableArguments": OSError(2, "The system cannot find the file specified"), }, - 78703424: {"SysVersion": ("3.9", 1), "SysArchitecture": ("64bit", 1)}, + 78703424: {"SysVersion": ("3.9", 1), "SysArchitecture": ("64bit", 1), "DisplayName": ("Python 3.9 (64-bit)", 1)}, 78701888: { "ExecutablePath": ("C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", 1), "ExecutableArguments": OSError(2, "The system cannot find the file specified"), }, - 78704512: {"SysVersion": ("3.10", 1), "SysArchitecture": ("32bit", 1)}, + 78704512: {"SysVersion": ("3.10", 1), "SysArchitecture": ("32bit", 1), "DisplayName": ("Python 3.10 (32-bit)", 1)}, 78703600: { "ExecutablePath": ("C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python310-32\\python.exe", 1), "ExecutableArguments": OSError(2, "The system cannot find the file specified"), @@ -91,16 +94,31 @@ 78700656: { "SysVersion": OSError(2, "The system cannot find the file specified"), "SysArchitecture": OSError(2, "The system cannot find the file specified"), + "DisplayName": OSError(2, "The system cannot find the file specified"), + }, + 78702608: { + "SysVersion": ("magic", 1), + "SysArchitecture": ("64bit", 1), + "DisplayName": ("Python 3.12 (wizard edition)", 1), }, - 78702608: {"SysVersion": ("magic", 1), "SysArchitecture": ("64bit", 1)}, 78703632: { "ExecutablePath": ("C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe", 1), "ExecutableArguments": OSError(2, "The system cannot find the file specified"), }, + 78702609: { + "SysVersion": ("3.13", 1), + "SysArchitecture": ("64bit", 1), + "DisplayName": ("Python 3.13 (64-bit, freethreaded)", 1), + }, + 78703633: { + "ExecutablePath": ("C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python313\\python3.13t.exe", 1), + "ExecutableArguments": OSError(2, "The system cannot find the file specified"), + }, 78703088: {"SysVersion": (2778, 11)}, 78703136: { "SysVersion": OSError(2, "The system cannot find the file specified"), "SysArchitecture": OSError(2, "The system cannot find the file specified"), + "DisplayName": OSError(2, "The system cannot find the file specified"), }, 78700912: { "ExecutablePath": OSError(2, "The system cannot find the file specified"), @@ -110,6 +128,7 @@ 78704032: { "SysVersion": OSError(2, "The system cannot find the file specified"), "SysArchitecture": OSError(2, "The system cannot find the file specified"), + "DisplayName": OSError(2, "The system cannot find the file specified"), }, 78703648: { "ExecutablePath": OSError(2, "The system cannot find the file specified"), @@ -120,7 +139,11 @@ "ExecutablePath": ("Z:\\CompanyA\\Python\\3.6\\python.exe", 1), "ExecutableArguments": OSError(2, "The system cannot find the file specified"), }, - 88820000: {"SysVersion": ("3.6", 1), "SysArchitecture": ("64bit", 1)}, + 88820000: { + "SysVersion": ("3.6", 1), + "SysArchitecture": ("64bit", 1), + "DisplayName": OSError(2, "The system cannot find the file specified"), + }, } enum_collect = { 78701856: [ @@ -139,6 +162,7 @@ "3.10-32", "3.11", "3.12", + "3.13t", "3.X", OSError(22, "No more data is available", None, 259, None), ], diff --git a/tests/unit/seed/embed/test_base_embed.py b/tests/unit/seed/embed/test_base_embed.py index 255e03177..4f4fadac9 100644 --- a/tests/unit/seed/embed/test_base_embed.py +++ b/tests/unit/seed/embed/test_base_embed.py @@ -20,11 +20,46 @@ def test_download_cli_flag(args, download, tmp_path): assert session.seeder.download is download +@pytest.mark.skipif(sys.version_info[:2] == (3, 8), reason="We still bundle wheel for Python 3.8") +@pytest.mark.parametrize("flag", ["--no-wheel", "--wheel=none", "--wheel=embed", "--wheel=bundle"]) +def test_wheel_cli_flags_do_nothing(tmp_path, flag): + session = session_via_cli([flag, str(tmp_path)]) + if sys.version_info[:2] >= (3, 12): + expected = {"pip": "bundle"} + else: + expected = {"pip": "bundle", "setuptools": "bundle"} + assert session.seeder.distribution_to_versions() == expected + + +@pytest.mark.skipif(sys.version_info[:2] == (3, 8), reason="We still bundle wheel for Python 3.8") +@pytest.mark.parametrize("flag", ["--no-wheel", "--wheel=none", "--wheel=embed", "--wheel=bundle"]) +def test_wheel_cli_flags_warn(tmp_path, flag, capsys): + session_via_cli([flag, str(tmp_path)]) + out, err = capsys.readouterr() + assert "The --no-wheel and --wheel options are deprecated." in out + err + + +@pytest.mark.skipif(sys.version_info[:2] == (3, 8), reason="We still bundle wheel for Python 3.8") +def test_unused_wheel_cli_flags_dont_warn(tmp_path, capsys): + session_via_cli([str(tmp_path)]) + out, err = capsys.readouterr() + assert "The --no-wheel and --wheel options are deprecated." not in out + err + + +@pytest.mark.skipif(sys.version_info[:2] != (3, 8), reason="We only bundle wheel for Python 3.8") +@pytest.mark.parametrize("flag", ["--no-wheel", "--wheel=none", "--wheel=embed", "--wheel=bundle"]) +def test_wheel_cli_flags_dont_warn_on_38(tmp_path, flag, capsys): + session_via_cli([flag, str(tmp_path)]) + out, err = capsys.readouterr() + assert "The --no-wheel and --wheel options are deprecated." not in out + err + + def test_embed_wheel_versions(tmp_path: Path) -> None: session = session_via_cli([str(tmp_path)]) - expected = ( - {"pip": "bundle"} - if sys.version_info[:2] >= (3, 12) - else {"pip": "bundle", "setuptools": "bundle", "wheel": "bundle"} - ) + if sys.version_info[:2] >= (3, 12): + expected = {"pip": "bundle"} + elif sys.version_info[:2] >= (3, 9): + expected = {"pip": "bundle", "setuptools": "bundle"} + else: + expected = {"pip": "bundle", "setuptools": "bundle", "wheel": "bundle"} assert session.seeder.distribution_to_versions() == expected diff --git a/tests/unit/seed/embed/test_bootstrap_link_via_app_data.py b/tests/unit/seed/embed/test_bootstrap_link_via_app_data.py index 4fd3d30c9..4076573e1 100644 --- a/tests/unit/seed/embed/test_bootstrap_link_via_app_data.py +++ b/tests/unit/seed/embed/test_bootstrap_link_via_app_data.py @@ -25,7 +25,7 @@ @pytest.mark.slow @pytest.mark.parametrize("copies", [False, True] if fs_supports_symlink() else [True]) -def test_seed_link_via_app_data(tmp_path, coverage_env, current_fastest, copies): +def test_seed_link_via_app_data(tmp_path, coverage_env, current_fastest, copies, for_py_version): # noqa: PLR0915 current = PythonInfo.current_system() bundle_ver = BUNDLE_SUPPORT[current.version_release_str] create_cmd = [ @@ -45,6 +45,8 @@ def test_seed_link_via_app_data(tmp_path, coverage_env, current_fastest, copies) current_fastest, "-vv", ] + if for_py_version == "3.8": + create_cmd += ["--wheel", bundle_ver["wheel"].split("-")[1]] if not copies: create_cmd.append("--symlink-app-data") result = cli_run(create_cmd) @@ -109,7 +111,7 @@ def test_seed_link_via_app_data(tmp_path, coverage_env, current_fastest, copies) # Windows does not allow removing a executable while running it, so when uninstalling pip we need to do it via # python -m pip - remove_cmd = [str(result.creator.exe), "-m", "pip"] + remove_cmd[1:] + remove_cmd = [str(result.creator.exe), "-m", "pip", *remove_cmd[1:]] process = Popen([*remove_cmd, "pip", "wheel"]) _, __ = process.communicate() assert not process.returncode @@ -147,6 +149,7 @@ def read_only_app_data(temp_app_data): yield temp_app_data +@pytest.mark.slow @pytest.mark.skipif(sys.platform == "win32", reason="Windows only applies R/O to files") @pytest.mark.usefixtures("read_only_app_data") def test_base_bootstrap_link_via_app_data_not_writable(tmp_path, current_fastest): @@ -155,6 +158,7 @@ def test_base_bootstrap_link_via_app_data_not_writable(tmp_path, current_fastest assert result +@pytest.mark.slow @pytest.mark.skipif(sys.platform == "win32", reason="Windows only applies R/O to files") def test_populated_read_only_cache_and_symlinked_app_data(tmp_path, current_fastest, temp_app_data): dest = tmp_path / "venv" @@ -180,6 +184,7 @@ def test_populated_read_only_cache_and_symlinked_app_data(tmp_path, current_fast check_call((str(dest.joinpath("bin/python")), "-c", "import pip")) +@pytest.mark.slow @pytest.mark.skipif(sys.platform == "win32", reason="Windows only applies R/O to files") def test_populated_read_only_cache_and_copied_app_data(tmp_path, current_fastest, temp_app_data): dest = tmp_path / "venv" @@ -207,11 +212,18 @@ def test_populated_read_only_cache_and_copied_app_data(tmp_path, current_fastest @pytest.mark.slow @pytest.mark.parametrize("pkg", ["pip", "setuptools", "wheel"]) @pytest.mark.usefixtures("session_app_data", "current_fastest", "coverage_env") -def test_base_bootstrap_link_via_app_data_no(tmp_path, pkg): - create_cmd = [str(tmp_path), "--seeder", "app-data", f"--no-{pkg}", "--wheel", "bundle", "--setuptools", "bundle"] +def test_base_bootstrap_link_via_app_data_no(tmp_path, pkg, for_py_version): + if for_py_version != "3.8" and pkg == "wheel": + msg = "wheel isn't installed on Python > 3.8" + raise pytest.skip(msg) + create_cmd = [str(tmp_path), "--seeder", "app-data", f"--no-{pkg}", "--setuptools", "bundle"] + if for_py_version == "3.8": + create_cmd += ["--wheel", "bundle"] result = cli_run(create_cmd) assert not (result.creator.purelib / pkg).exists() for key in {"pip", "setuptools", "wheel"} - {pkg}: + if for_py_version != "3.8" and key == "wheel": + continue assert (result.creator.purelib / key).exists() @@ -227,7 +239,7 @@ def test_app_data_parallel_fail(tmp_path: Path, mocker: MockerFixture) -> None: exceptions = _run_parallel_threads(tmp_path) assert len(exceptions) == 2 for exception in exceptions: - assert exception.startswith("failed to build image wheel because:\nTraceback") + assert exception.startswith("failed to build image pip because:\nTraceback") assert "RuntimeError" in exception, exception @@ -236,7 +248,10 @@ def _run_parallel_threads(tmp_path): def _run(name): try: - cli_run(["--seeder", "app-data", str(tmp_path / name), "--no-pip", "--no-setuptools", "--wheel", "bundle"]) + cmd = ["--seeder", "app-data", str(tmp_path / name), "--no-setuptools"] + if sys.version_info[:2] == (3, 8): + cmd.append("--no-wheel") + cli_run(cmd) except Exception as exception: # noqa: BLE001 as_str = str(exception) exceptions.append(as_str) diff --git a/tests/unit/seed/embed/test_pip_invoke.py b/tests/unit/seed/embed/test_pip_invoke.py index d8c243e57..41f4395d2 100644 --- a/tests/unit/seed/embed/test_pip_invoke.py +++ b/tests/unit/seed/embed/test_pip_invoke.py @@ -49,7 +49,9 @@ def _execute(cmd, env): original = PipInvoke._execute # noqa: SLF001 run = mocker.patch.object(PipInvoke, "_execute", side_effect=_execute) - versions = {"pip": "embed", "setuptools": "bundle", "wheel": new["wheel"].split("-")[1]} + versions = {"pip": "embed", "setuptools": "bundle"} + if sys.version_info[:2] == (3, 8): + versions["wheel"] = new["wheel"].split("-")[1] create_cmd = [ "--seeder", @@ -86,4 +88,6 @@ def _execute(cmd, env): for key in ("pip", "setuptools", "wheel"): if key == no: continue + if sys.version_info[:2] >= (3, 9) and key == "wheel": + continue assert locals()[key] in files_post_first_create diff --git a/tests/unit/seed/wheels/test_acquire.py b/tests/unit/seed/wheels/test_acquire.py index 20979775b..27b013d38 100644 --- a/tests/unit/seed/wheels/test_acquire.py +++ b/tests/unit/seed/wheels/test_acquire.py @@ -5,7 +5,7 @@ from datetime import datetime, timezone from pathlib import Path from subprocess import CalledProcessError -from typing import TYPE_CHECKING, Callable +from typing import TYPE_CHECKING import pytest @@ -16,6 +16,7 @@ from virtualenv.seed.wheels.util import Wheel, discover_wheels if TYPE_CHECKING: + from collections.abc import Callable from unittest.mock import MagicMock from pytest_mock import MockerFixture @@ -88,6 +89,18 @@ def test_download_fails(mocker, for_py_version, session_app_data): ] == exc.cmd +def test_download_wheel_python_io_encoding(mocker, for_py_version, session_app_data): + mock_popen = mocker.patch("virtualenv.seed.wheels.acquire.Popen") + mock_popen.return_value.communicate.return_value = "Saved a-b-c.whl", "" + mock_popen.return_value.returncode = 0 + mocker.patch("pathlib.Path.absolute", return_value=Path("a-b-c.whl")) + + download_wheel("pip", "==1", for_py_version, [], session_app_data, "folder", os.environ.copy()) + + env = mock_popen.call_args[1]["env"] + assert env["PYTHONIOENCODING"] == "utf-8" + + @pytest.fixture def downloaded_wheel(mocker): wheel = Wheel.from_path(Path("setuptools-0.0.0-py2.py3-none-any.whl")) diff --git a/tests/unit/seed/wheels/test_periodic_update.py b/tests/unit/seed/wheels/test_periodic_update.py index 3172bc8e6..e11fe5e1a 100644 --- a/tests/unit/seed/wheels/test_periodic_update.py +++ b/tests/unit/seed/wheels/test_periodic_update.py @@ -74,7 +74,7 @@ def _do_update( # noqa: PLR0913 packages[args[1]["distribution"]].append(args[1]["for_py_version"]) packages = {key: sorted(value) for key, value in packages.items()} versions = sorted(BUNDLE_SUPPORT.keys()) - expected = {"setuptools": versions, "wheel": versions, "pip": versions} + expected = {"setuptools": versions, "wheel": ["3.8"], "pip": versions} assert packages == expected @@ -101,8 +101,6 @@ def test_pick_periodic_update(tmp_path, mocker, for_py_version): "--no-pip", "--setuptools", "bundle", - "--wheel", - "bundle", ], ) @@ -280,6 +278,7 @@ def test_trigger_update_no_debug(for_py_version, session_app_data, tmp_path, moc monkeypatch.delenv("_VIRTUALENV_PERIODIC_UPDATE_INLINE", raising=False) current = get_embed_wheel("setuptools", for_py_version) process = mocker.MagicMock() + process.pid = 123 process.communicate.return_value = None, None Popen = mocker.patch("virtualenv.seed.wheels.periodic_update.Popen", return_value=process) # noqa: N806 @@ -328,6 +327,7 @@ def test_trigger_update_debug(for_py_version, session_app_data, tmp_path, mocker current = get_embed_wheel("pip", for_py_version) process = mocker.MagicMock() + process.pid = 123 process.communicate.return_value = None, None Popen = mocker.patch("virtualenv.seed.wheels.periodic_update.Popen", return_value=process) # noqa: N806 diff --git a/tests/unit/seed/wheels/test_wheels_util.py b/tests/unit/seed/wheels/test_wheels_util.py index 87edab2d4..4d4ec9b9b 100644 --- a/tests/unit/seed/wheels/test_wheels_util.py +++ b/tests/unit/seed/wheels/test_wheels_util.py @@ -29,3 +29,8 @@ def test_wheel_not_support(): def test_wheel_repr(): wheel = get_embed_wheel("setuptools", MAX) assert str(wheel.path) in repr(wheel) + + +def test_unknown_distribution(): + wheel = get_embed_wheel("unknown", MAX) + assert wheel is None diff --git a/tests/unit/test_file_limit.py b/tests/unit/test_file_limit.py new file mode 100644 index 000000000..0c66561f8 --- /dev/null +++ b/tests/unit/test_file_limit.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import errno +import os +import sys + +import pytest + +from virtualenv.info import IMPLEMENTATION +from virtualenv.run import cli_run + + +@pytest.mark.skipif(sys.platform == "win32", reason="resource module not available on Windows") +def test_too_many_open_files(tmp_path): + """ + Test that we get a specific error message when we have too many open files. + """ + import resource # noqa: PLC0415 + + soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) + + # Lower the soft limit to a small number to trigger the error + try: + resource.setrlimit(resource.RLIMIT_NOFILE, (32, hard_limit)) + except ValueError: + pytest.skip("could not lower the soft limit for open files") + except AttributeError as exc: # pypy, graalpy + if "module 'resource' has no attribute 'setrlimit'" in str(exc): + pytest.skip(f"{IMPLEMENTATION} does not support resource.setrlimit") + + # Keep some file descriptors open to make it easier to trigger the error + fds = [] + try: + # JIT implementations use more file descriptors up front so we can run out early + try: + fds.extend(os.open(os.devnull, os.O_RDONLY) for _ in range(20)) + except OSError as jit_exceptions: # pypy, graalpy + assert jit_exceptions.errno == errno.EMFILE # noqa: PT017 + assert "Too many open files" in str(jit_exceptions) # noqa: PT017 + + expected_exceptions = SystemExit, OSError, RuntimeError + with pytest.raises(expected_exceptions) as too_many_open_files_exc: + cli_run([str(tmp_path / "venv")]) + + if isinstance(too_many_open_files_exc, SystemExit): + assert too_many_open_files_exc.code != 0 + else: + assert "Too many open files" in str(too_many_open_files_exc.value) + + finally: + for fd in fds: + os.close(fd) + resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, hard_limit)) diff --git a/tox.ini b/tox.ini index 591ecd9f1..7cdebdbe7 100644 --- a/tox.ini +++ b/tox.ini @@ -1,18 +1,22 @@ [tox] requires = - tox>=4.2 + tox>=4.28 env_list = fix pypy3 + 3.14 3.13 3.12 3.11 3.10 3.9 3.8 + graalpy coverage readme docs + 3.14t + 3.13t skip_missing_interpreters = true [testenv] @@ -31,19 +35,20 @@ set_env = PYTHONWARNDEFAULTENCODING = 1 _COVERAGE_SRC = {envsitepackagesdir}/virtualenv commands = - coverage erase - coverage run -m pytest {posargs:--junitxml "{toxworkdir}/junit.{envname}.xml" tests --int} - coverage combine - coverage report --skip-covered --show-missing - coverage xml -o "{toxworkdir}/coverage.{envname}.xml" - coverage html -d {envtmpdir}/htmlcov --show-contexts --title virtualenv-{envname}-coverage + !graalpy: coverage erase + !graalpy: coverage run -m pytest {posargs:--junitxml "{toxworkdir}/junit.{envname}.xml" tests --int} + !graalpy: coverage combine + !graalpy: coverage report --skip-covered --show-missing + !graalpy: coverage xml -o "{toxworkdir}/coverage.{envname}.xml" + !graalpy: coverage html -d {envtmpdir}/htmlcov --show-contexts --title virtualenv-{envname}-coverage + graalpy: pytest {posargs:--junitxml "{toxworkdir}/junit.{envname}.xml" tests --skip-slow} uv_seed = true [testenv:fix] description = format the code base to adhere to our styles, and complain about what we cannot do automatically skip_install = true deps = - pre-commit-uv>=4.1.1 + pre-commit-uv>=4.1.4 commands = pre-commit run --all-files --show-diff-on-failure @@ -51,9 +56,9 @@ commands = description = check that the long description is valid skip_install = true deps = - check-wheel-contents>=0.6 - twine>=5.1.1 - uv>=0.4.10 + check-wheel-contents>=0.6.2 + twine>=6.1 + uv>=0.8 commands = uv build --sdist --wheel --out-dir {envtmpdir} . twine check {envtmpdir}{/}* @@ -67,23 +72,29 @@ commands = sphinx-build -d "{envtmpdir}/doctree" docs "{toxworkdir}/docs_out" --color -b html {posargs:-W} python -c 'import pathlib; print("documentation available under file://\{0\}".format(pathlib.Path(r"{toxworkdir}") / "docs_out" / "index.html"))' +[testenv:3.14t] +base_python = {env:TOX_BASEPYTHON} + +[testenv:3.13t] +base_python = {env:TOX_BASEPYTHON} + [testenv:upgrade] description = upgrade pip/wheels/setuptools to latest skip_install = true deps = - ruff>=0.6.5 + ruff>=0.12.4 pass_env = UPGRADE_ADVISORY change_dir = {toxinidir}/tasks commands = - python upgrade_wheels.py + - python upgrade_wheels.py uv_seed = true [testenv:release] description = do a release, required posarg of the version number deps = - gitpython>=3.1.43 - packaging>=24.1 + gitpython>=3.1.44 + packaging>=25 towncrier>=24.8 change_dir = {toxinidir}/tasks commands = @@ -103,7 +114,7 @@ commands = description = generate a zipapp skip_install = true deps = - packaging>=24.1 + packaging>=25 commands = python tasks/make_zipapp.py uv_seed = true