feat(venv): make wheel scripts runnable in venv

CHANGELOG.md

@@ -60,6 +60,8 @@ END_UNRELEASED_TEMPLATE
 ### Changed
 * (gazelle) WORKSPACE's bazel-gazelle dependency bumped from 0.36.0 to 0.47.0.
   The go version was also bumped from 1.21.13 to 1.22.9.
+* (pypi) The data files of a wheel (bin, includes, etc) are now always included
+  as a library's data dependencies.
 
 {#v0-0-0-fixed}
 ### Fixed
@@ -74,6 +76,8 @@ END_UNRELEASED_TEMPLATE
   adding `config_setting` labels to all registered toolchains.
 * (windows) Full venv support for Windows is available. Set
   {obj}`--venvs_site_packages=yes` to enable.
+* (test/binaries) When {obj}`--venv_site_packages=yes` is enabled,
+  wheel `data`, `bin`, and `include` files are populated into the venv.
 * (runfiles) Added a pathlib-compatible API: {obj}`Runfiles.root()`
   Fixes [#3296](https://github.com/bazel-contrib/rules_python/issues/3296).
 * (toolchains) `3.13.12`, `3.14.3` Python toolchain from [20260325] release.

MODULE.bazel

@@ -246,6 +246,8 @@ use_repo(
     "somepkg_with_build_files",
     "whl_library_extras_direct_dep",
     "whl_with_build_files",
+    "whl_with_data1",
+    "whl_with_data2",
 )
 
 dev_rules_python_config = use_extension(

examples/pip_parse/pip_parse_test.py

@@ -52,16 +52,16 @@ def test_data(self):
         self.assertIsNotNone(actual)
         actual = self._remove_leading_dirs(actual.split(" "))
 
-        self.assertListEqual(
-            actual,
-            [
-                "data/share/doc/packages/s3cmd/INSTALL.md",
-                "data/share/doc/packages/s3cmd/LICENSE",
-                "data/share/doc/packages/s3cmd/NEWS",
-                "data/share/doc/packages/s3cmd/README.md",
-                "data/share/man/man1/s3cmd.1",
-            ],
-        )
+        expected = [
+            "bin/s3cmd",
+            "data/share/doc/packages/s3cmd/INSTALL.md",
+            "data/share/doc/packages/s3cmd/LICENSE",
+            "data/share/doc/packages/s3cmd/NEWS",
+            "data/share/doc/packages/s3cmd/README.md",
+            "data/share/man/man1/s3cmd.1",
+        ]
+
+        self.assertListEqual(actual, expected)
 
     def test_dist_info(self):
         actual = os.environ.get("WHEEL_DIST_INFO_CONTENTS")

internal_dev_setup.bzl

@@ -26,6 +26,8 @@ load("//python:versions.bzl", "MINOR_MAPPING", "TOOL_VERSIONS")
 load("//python/private:pythons_hub.bzl", "hub_repo")  # buildifier: disable=bzl-visibility
 load("//python/private:runtime_env_repo.bzl", "runtime_env_repo")  # buildifier: disable=bzl-visibility
 load("//python/private/pypi:deps.bzl", "pypi_deps")  # buildifier: disable=bzl-visibility
+load("//python/private/pypi:whl_library.bzl", "whl_library")  # buildifier: disable=bzl-visibility
+load("//tests/support/whl_from_dir:whl_from_dir_repo.bzl", "whl_from_dir_repo")  # buildifier: disable=bzl-visibility
 
 def rules_python_internal_setup():
     """Setup for development and testing of rules_python itself."""
@@ -59,3 +61,25 @@ def rules_python_internal_setup():
     bazel_features_deps()
     rules_shell_dependencies()
     rules_shell_toolchains()
+
+    whl_from_dir_repo(
+        name = "whl_with_data1_whl",
+        root = "//tests/repos/whl_with_data1:BUILD.bazel",
+        output = "whl_with_data1-1.0-any-none-any.whl",
+    )
+    whl_library(
+        name = "whl_with_data1",
+        whl_file = "@whl_with_data1_whl//:whl_with_data1-1.0-any-none-any.whl",
+        requirement = "whl-with-data1",
+    )
+
+    whl_from_dir_repo(
+        name = "whl_with_data2_whl",
+        root = "//tests/repos/whl_with_data2:BUILD.bazel",
+        output = "whl_with_data2-1.0-any-none-any.whl",
+    )
+    whl_library(
+        name = "whl_with_data2",
+        whl_file = "@whl_with_data2_whl//:whl_with_data2-1.0-any-none-any.whl",
+        requirement = "whl-with-data2",
+    )

python/config_settings/BUILD.bazel

@@ -229,7 +229,7 @@ string_flag(
 )
 
 config_setting(
-    name = "is_venvs_site_packages",
+    name = "_is_venvs_site_packages_yes",
     flag_values = {
         ":venvs_site_packages": VenvsSitePackages.YES,
     },

python/private/BUILD.bazel

@@ -740,6 +740,14 @@ bzl_library(
     ],
 )
 
+alias(
+    name = "venv_bin_rewriter",
+    actual = select({
+        "@platforms//os:windows": ":venv_bin_rewriter.ps1",
+        "//conditions:default": ":venv_bin_rewriter.sh",
+    }),
+)
+
 bzl_library(
     name = "venv_runfiles_bzl",
     srcs = ["venv_runfiles.bzl"],

python/private/internal_dev_deps.bzl

@@ -91,6 +91,28 @@ def _internal_dev_deps_impl(mctx):
         enable_implicit_namespace_pkgs = False,
     )
 
+    whl_from_dir_repo(
+        name = "whl_with_data1_whl",
+        root = "//tests/repos/whl_with_data1:BUILD.bazel",
+        output = "whl_with_data1-1.0-any-none-any.whl",
+    )
+    whl_library(
+        name = "whl_with_data1",
+        whl_file = "@whl_with_data1_whl//:whl_with_data1-1.0-any-none-any.whl",
+        requirement = "whl-with-data1",
+    )
+
+    whl_from_dir_repo(
+        name = "whl_with_data2_whl",
+        root = "//tests/repos/whl_with_data2:BUILD.bazel",
+        output = "whl_with_data2-1.0-any-none-any.whl",
+    )
+    whl_library(
+        name = "whl_with_data2",
+        whl_file = "@whl_with_data2_whl//:whl_with_data2-1.0-any-none-any.whl",
+        requirement = "whl-with-data2",
+    )
+
     _whl_library_from_dir(
         name = "whl_library_extras_direct_dep",
         root = "//tests/pypi/whl_library/testdata/pkg:BUILD.bazel",

python/private/py_executable.bzl

@@ -232,6 +232,11 @@ accepting arbitrary Python versions.
             default = "//python/private:uncachable_version_file",
             allow_files = True,
         ),
+        "_venv_bin_rewriter": lambda: attrb.Label(
+            default = "//python/private:venv_bin_rewriter",
+            allow_files = True,
+            cfg = "exec",
+        ),
         "_venvs_use_declare_symlink_flag": lambda: attrb.Label(
             default = labels.VENVS_USE_DECLARE_SYMLINK,
             providers = [BuildSettingInfo],
@@ -575,9 +580,13 @@ def _create_venv(ctx, output_prefix, imports, runtime_details, add_runfiles_root
         computed_substitutions = computed_subs,
     )
 
+    # See https://docs.python.org/3/library/sysconfig.html#posix-prefix
+    # for how schemes map under the venv.
     venv_dir_map = {
-        VenvSymlinkKind.BIN: venv_details.bin_dir,
+        VenvSymlinkKind.BIN: "{}/{}".format(venv_ctx_rel_root, venv_details.bin_dir),
         VenvSymlinkKind.LIB: site_packages,
+        VenvSymlinkKind.INCLUDE: "{}/{}".format(venv_ctx_rel_root, venv_details.include_dir),
+        VenvSymlinkKind.DATA: venv_ctx_rel_root,
     }
     venv_app_files = create_venv_app_files(
         ctx,
@@ -659,7 +668,7 @@ def _create_venv_unixy(ctx, *, venv_ctx_rel_root, runtime, interpreter_actual_pa
 
     recreate_venv_at_runtime = False
 
-    bin_dir = "{}/bin".format(venv_ctx_rel_root)
+    venv_bin_ctx_rel_path = "{}/bin".format(venv_ctx_rel_root)
     if create_full_venv:
         # Some wrappers around the interpreter (e.g. pyenv) use the program
         # name to decide what to do, so preserve the name.
@@ -671,15 +680,15 @@ def _create_venv_unixy(ctx, *, venv_ctx_rel_root, runtime, interpreter_actual_pa
             # When the venv symlinks are disabled, the $venv/bin/python3 file isn't
             # needed or used at runtime. However, the zip code uses the interpreter
             # File object to figure out some paths.
-            interpreter = ctx.actions.declare_file("{}/{}".format(bin_dir, py_exe_basename))
+            interpreter = ctx.actions.declare_file("{}/{}".format(venv_bin_ctx_rel_path, py_exe_basename))
             ctx.actions.write(interpreter, "actual:{}".format(interpreter_actual_path))
 
         elif runtime.interpreter:
             # Even though ctx.actions.symlink() is used, using
             # declare_symlink() is required to ensure that the resulting file
             # in runfiles is always a symlink. An RBE implementation, for example,
             # may choose to write what symlink() points to instead.
-            interpreter = ctx.actions.declare_symlink("{}/{}".format(bin_dir, py_exe_basename))
+            interpreter = ctx.actions.declare_symlink("{}/{}".format(venv_bin_ctx_rel_path, py_exe_basename))
             interpreter_runfiles.add(interpreter)
 
             rel_path = relative_path(
@@ -690,7 +699,7 @@ def _create_venv_unixy(ctx, *, venv_ctx_rel_root, runtime, interpreter_actual_pa
             )
             ctx.actions.symlink(output = interpreter, target_path = rel_path)
         else:
-            interpreter = ctx.actions.declare_symlink("{}/{}".format(bin_dir, py_exe_basename))
+            interpreter = ctx.actions.declare_symlink("{}/{}".format(venv_bin_ctx_rel_path, py_exe_basename))
             interpreter_runfiles.add(interpreter)
             ctx.actions.symlink(output = interpreter, target_path = runtime.interpreter_path)
     else:
@@ -715,7 +724,8 @@ def _create_venv_unixy(ctx, *, venv_ctx_rel_root, runtime, interpreter_actual_pa
         interpreter = interpreter,
         pyvenv_cfg = pyvenv_cfg,
         site_packages = site_packages,
-        bin_dir = bin_dir,
+        bin_dir = "bin",
+        include_dir = "include",
         recreate_venv_at_runtime = recreate_venv_at_runtime,
         interpreter_runfiles = interpreter_runfiles.build(ctx),
         interpreter_symlinks = depset(),
@@ -777,7 +787,8 @@ def _create_venv_windows(ctx, *, venv_ctx_rel_root, runtime, interpreter_actual_
         interpreter = interpreter,
         pyvenv_cfg = None,
         site_packages = site_packages,
-        bin_dir = venv_bin_ctx_rel_path,
+        bin_dir = venv_bin_rel_path,
+        include_dir = "Include",
         recreate_venv_at_runtime = True,
         interpreter_runfiles = interpreter_runfiles.build(ctx),
         interpreter_symlinks = interpreter_symlinks.build(),
@@ -789,6 +800,7 @@ def _venv_details(
         pyvenv_cfg,
         site_packages,
         bin_dir,
+        include_dir,
         recreate_venv_at_runtime,
         interpreter_runfiles,
         interpreter_symlinks):
@@ -801,8 +813,10 @@ def _venv_details(
         pyvenv_cfg = pyvenv_cfg,
         # str; venv-relative path to the site-packages directory
         site_packages = site_packages,
-        # str; ctx-relative path to the venv's bin directory.
+        # str; venv-relative path to the venv's bin directory.
         bin_dir = bin_dir,
+        # str; venv-relative-path to the venv's include directory.
+        include_dir = include_dir,
         # bool; True if the venv needs to be recreated at runtime (because the
         # build-time construction isn't sufficient). False if the build-time
         # constructed venv is sufficient.

python/private/py_info.bzl

@@ -37,6 +37,12 @@ def _VenvSymlinkKind_typedef():
 
     Indicates to create paths under the venv's include directory.
     :::
+
+    :::{field} DATA
+    :type: object
+
+    Indicates to create paths under the venv's data directory.
+    :::
     """
 
 # buildifier: disable=name-conventions
@@ -45,6 +51,7 @@ VenvSymlinkKind = struct(
     BIN = "BIN",
     LIB = "LIB",
     INCLUDE = "INCLUDE",
+    DATA = "DATA",
 )
 
 def _VenvSymlinkEntry_init(**kwargs):

python/private/pypi/whl_installer/wheel.py

@@ -316,7 +316,7 @@ def unzip(self, directory: str) -> None:
         destination = installer.destinations.SchemeDictionaryDestination(
             installation_schemes,
             # TODO Should entry_point scripts also be handled by installer rather than custom code?
-            interpreter="/dev/null",
+            interpreter="python",
             script_kind="posix",
             destdir=directory,
             bytecode_optimization_levels=[],

python/private/pypi/whl_library_targets.bzl

@@ -43,6 +43,8 @@ _BAZEL_REPO_FILE_GLOBS = [
     "WORKSPACE.bazel",
 ]
 
+_IS_VENV_SITE_PACKAGES_YES = Label("//python/config_settings:_is_venvs_site_packages_yes")
+
 def whl_library_targets_from_requires(
         *,
         name,
@@ -195,7 +197,7 @@ def whl_library_targets(
                 include = ["site-packages/*.dist-info/**"],
             ),
             DATA_LABEL: dict(
-                include = ["data/**"],
+                include = ["data/**", "bin/**", "include/**"],
             ),
         }
 
@@ -369,7 +371,7 @@ def whl_library_targets(
 
         if not enable_implicit_namespace_pkgs:
             generated_namespace_package_files = select({
-                Label("//python/config_settings:is_venvs_site_packages"): [],
+                _IS_VENV_SITE_PACKAGES_YES: [],
                 "//conditions:default": rules.create_inits(
                     srcs = srcs + data + pyi_srcs,
                     ignored_dirnames = [],  # If you need to ignore certain folders, you can patch rules_python here to do so.
@@ -379,6 +381,10 @@ def whl_library_targets(
             namespace_package_files += generated_namespace_package_files
             srcs = srcs + generated_namespace_package_files
 
+        # This is done after create_inits() is called so that the data scheme
+        # files don't have such files created in their directories.
+        data = data + [DATA_LABEL]
+
         rules.py_library(
             name = py_library_label,
             srcs = srcs,

python/private/venv_bin_rewriter.ps1

@@ -0,0 +1,28 @@
+[CmdletBinding()]
+param(
+    [Parameter(Position=0, Mandatory=$true)]
+    [string]$InFile,
+
+    [Parameter(Position=1, Mandatory=$true)]
+    [string]$OutFile
+)
+
+if ($InFile.EndsWith(".exe") -or $InFile.EndsWith(".dll")) {
+    Copy-Item -Path $InFile -Destination $OutFile
+    exit 0
+}
+
+$firstLine = Get-Content -Path $InFile -TotalCount 1 -ErrorAction SilentlyContinue
+
+if ($firstLine -match "^#!python") {
+    $content = Get-Content -Path $InFile | Select-Object -Skip 1
+    $wrapper = @'
+#!/bin/sh
+'''exec' "$(dirname "$0")/python3" "$0" "$@"
+' '''
+'@
+    Set-Content -Path $OutFile -Value $wrapper -Encoding UTF8
+    Add-Content -Path $OutFile -Value $content -Encoding UTF8
+} else {
+    Copy-Item -Path $InFile -Destination $OutFile
+}

python/private/venv_bin_rewriter.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+set -eu
+
+IN="$1"
+OUT="$2"
+
+if head -n 1 "$IN" | grep -q "^#!python"; then
+  echo "#!/bin/sh" > "$OUT"
+  # Polyglot re-exec gibberish.
+  # Shell treats first line's quotes as a quoted command to execute. It then
+  # re-execs itself with Python, which treats the triple quoted strings
+  # as plain strings and ignores them.
+  echo "'''exec' \"\$(dirname \"\$0\")/python3\" \"\$0\" \"\$@\"" >> "$OUT"
+  echo "' '''" >> "$OUT"
+  tail -n +2 "$IN" >> "$OUT"
+else
+  cp "$IN" "$OUT"
+fi
+chmod +x "$OUT"