From 85e7778a66f3d759dfa04f42b445f38f16a8deea Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 31 May 2026 08:00:26 +0000 Subject: [PATCH 1/3] feat(toolchains): support dynamic registration from remote manifest Currently, all supported Python runtime versions and their platform-specific metadata (URLs, SHA256s, strip_prefix) must be hardcoded in `python/versions.bzl`. This makes it slow and difficult to adopt new Python versions or custom builds without updating `rules_python` itself. This PR introduces the ability to dynamically fetch and register Python runtimes from a remote python-build-standalone (PBS) manifest file (e.g., `SHA256SUMS`). This is supported via two new attributes in `python.override`: - `add_runtime_manifest_urls`: A list of URLs pointing to manifest files to parse and register. - `runtime_manifest_sha`: The SHA256 hash of the manifest file. --- .bazelrc.deleted_packages | 1 + CHANGELOG.md | 3 + docs/toolchains.md | 61 +++++++ python/private/BUILD.bazel | 6 + python/private/pbs_manifest.bzl | 116 +++++++++++++ python/private/python.bzl | 103 +++++++++++- tests/integration/BUILD.bazel | 4 + tests/integration/runtime_manifests/.bazelrc | 4 + .../integration/runtime_manifests/BUILD.bazel | 7 + .../runtime_manifests/MODULE.bazel | 19 +++ tests/integration/runtime_manifests/WORKSPACE | 1 + .../runtime_manifests/basic_test.py | 27 +++ tests/python_bzlmod_ext/BUILD.bazel | 6 + .../parse_sha_manifest_tests.bzl | 155 ++++++++++++++++++ .../runtime_manifests_tests.bzl | 96 +++++++++++ tests/support/mocks/BUILD.bazel | 6 + tests/support/mocks/python_ext.bzl | 102 ++++++++++++ .../private/debug/print_defined_toolchains.sh | 14 ++ 18 files changed, 729 insertions(+), 2 deletions(-) create mode 100644 python/private/pbs_manifest.bzl create mode 100644 tests/integration/runtime_manifests/.bazelrc create mode 100644 tests/integration/runtime_manifests/BUILD.bazel create mode 100644 tests/integration/runtime_manifests/MODULE.bazel create mode 100644 tests/integration/runtime_manifests/WORKSPACE create mode 100644 tests/integration/runtime_manifests/basic_test.py create mode 100644 tests/python_bzlmod_ext/BUILD.bazel create mode 100644 tests/python_bzlmod_ext/parse_sha_manifest_tests.bzl create mode 100644 tests/python_bzlmod_ext/runtime_manifests_tests.bzl create mode 100644 tests/support/mocks/python_ext.bzl create mode 100755 tools/private/debug/print_defined_toolchains.sh diff --git a/.bazelrc.deleted_packages b/.bazelrc.deleted_packages index f4ea8527f3..b84214a1c7 100644 --- a/.bazelrc.deleted_packages +++ b/.bazelrc.deleted_packages @@ -38,6 +38,7 @@ common --deleted_packages=tests/integration/pip_parse common --deleted_packages=tests/integration/pip_parse/empty common --deleted_packages=tests/integration/pip_parse_isolated common --deleted_packages=tests/integration/py_cc_toolchain_registered +common --deleted_packages=tests/integration/runtime_manifests common --deleted_packages=tests/integration/toolchain_target_settings common --deleted_packages=tests/modules/another_module common --deleted_packages=tests/modules/other diff --git a/CHANGELOG.md b/CHANGELOG.md index bf6fe3095c..e7c56543fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -104,6 +104,9 @@ END_UNRELEASED_TEMPLATE {#v0-0-0-added} ### Added +* (toolchains) Support dynamically fetching and registering Python runtimes + from a python-build-standalone manifest file using + `python.override(add_runtime_manifest_urls = ..., runtime_manifest_sha = ...)`. * (toolchain) Added {obj}`python.override.toolchain_target_settings` to allow adding `config_setting` labels to all registered toolchains. * (windows) Full venv support for Windows is available. Set diff --git a/docs/toolchains.md b/docs/toolchains.md index 09aaed412b..b048c74a1e 100644 --- a/docs/toolchains.md +++ b/docs/toolchains.md @@ -242,6 +242,8 @@ existing attributes: {attr}`python.single_version_platform_override.coverage_tool`. * Adding additional Python versions via {bzl:obj}`python.single_version_override` or {bzl:obj}`python.single_version_platform_override`. +* Adding additional Python versions dynamically from a remote manifest file + via {attr}`python.override.add_runtime_manifest_urls`. ### Registering custom runtimes @@ -310,6 +312,65 @@ Added support for custom platform names, `target_compatible_with`, and `target_settings` with `single_version_platform_override`. ::: +### Registering runtimes from a manifest + +If you want to register multiple custom runtimes or versions at once, you can +use a python-build-standalone manifest file. This is useful if you want to +adopt new versions that are not yet built into `rules_python` without having +to manually define each one using `single_version_platform_override`. + +To do this, specify the `add_runtime_manifest_urls` and `runtime_manifest_sha` +attributes in `python.override` in your `MODULE.bazel`. + +In the example below, we register all runtimes available in a specific PBS +release manifest: + +``` +# File: MODULE.bazel +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.override( + add_runtime_manifest_urls = [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260414/SHA256SUMS", + ], + runtime_manifest_sha = "ce18fdfd47c66830a40ea9b9e314a14b1636bbfd684501bc5ca1fc6d55a7933f", +) +``` + +#### Manifest file format + +The manifest must be a plain text file where each line contains the SHA256 hash +and the location of a runtime archive, separated by whitespace: + +``` + +``` + +The `` can be either: +- A relative filename (e.g., + `cpython-3.10.20+20260414-x86_64-unknown-linux-gnu-install_only.tar.zst`). + In this case, the download URL is constructed by appending the filename to the + parent directory of each URL in `add_runtime_manifest_urls` (treating them as + mirrors). +- An absolute URL (e.g., + `https://example.com/downloads/cpython-3.10.20+20260414-x86_64-unknown-linux-gnu-install_only.tar.zst`). + In this case, the URL is used directly to download the archive. + +In both cases, the filename or the last path segment of the URL must follow the +standard python-build-standalone naming convention. `rules_python` parses this +name to extract runtime metadata (such as Python version, target architecture, +operating system, and libc). + +Notes: +- `rules_python` will download the manifest, parse it, and automatically + register toolchains for all valid Python runtimes found in it that match + supported platforms. +- Only runtimes matching known platforms in `rules_python` will be registered. + +:::{versionadded} VERSION_NEXT_FEATURE +Added support for registering runtimes from a manifest using +`add_runtime_manifest_urls` and `runtime_manifest_sha` in `python.override`. +::: + ### Using defined toolchains from WORKSPACE It is possible to use toolchains defined in `MODULE.bazel` in `WORKSPACE`. For example, diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index 6ab3d546be..b54c198069 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -252,6 +252,11 @@ bzl_library( srcs = ["normalize_name.bzl"], ) +bzl_library( + name = "pbs_manifest_bzl", + srcs = ["pbs_manifest.bzl"], +) + bzl_library( name = "precompile_bzl", srcs = ["precompile.bzl"], @@ -274,6 +279,7 @@ bzl_library( srcs = ["python.bzl"], deps = [ ":full_version_bzl", + ":pbs_manifest_bzl", ":platform_info_bzl", ":python_register_toolchains_bzl", ":pythons_hub_bzl", diff --git a/python/private/pbs_manifest.bzl b/python/private/pbs_manifest.bzl new file mode 100644 index 0000000000..495e4af907 --- /dev/null +++ b/python/private/pbs_manifest.bzl @@ -0,0 +1,116 @@ +"""Helper functions to parse python-build-standalone manifests.""" + +def parse_filename(filename): + """Parses a python-build-standalone filename (or URL) into its components. + + Example: cpython-3.10.20+20260414-x86_64_v2-unknown-linux-musl-lto-full.tar.zst + + Args: + filename: The filename or URL of the python-build-standalone release asset. + + Returns: + A dictionary of parsed components if parsed successfully, else None. + """ + basename = filename.rpartition("/")[-1] + if basename.endswith(".tar.zst"): + name = basename.removesuffix(".tar.zst") + elif basename.endswith(".tar.gz"): + name = basename.removesuffix(".tar.gz") + else: + return None + + if not name.startswith("cpython-"): + return None + name = name.removeprefix("cpython-") + + left, plus, tail = name.partition("+") + if plus: + python_version = left + build_version, sep, rest = tail.partition("-") + if not sep: + return None + else: + python_version, sep, rest = left.partition("-") + if not sep: + return None + build_version = "" + + arch, sep, rest = rest.partition("-") + if not sep: + return None + + microarch = "" + arch_base, sep_v, microarch_num = arch.partition("_v") + if sep_v: + arch = arch_base + microarch = "v" + microarch_num + + vendor, sep, rest = rest.partition("-") + if not sep: + return None + + os, sep, rest = rest.partition("-") + if not sep: + return None + + libc = "" + next_part, _, remaining = rest.partition("-") + if os == "linux" and next_part in ["gnu", "musl"]: + libc = next_part + flavor = remaining + elif os == "windows" and next_part == "msvc": + libc = next_part + flavor = remaining + else: + libc = "" + flavor = rest + + return { + "arch": arch, + "build_version": build_version, + "flavor": flavor, + "libc": libc, + "location": filename, + "microarch": microarch, + "os": os, + "python_version": python_version, + "vendor": vendor, + } + +def parse_sha_manifest(content): + """Parses the SHA256SUMS file content into a list of structs. + + Args: + content: The raw content of the manifest file. + + Returns: + A list of structs capturing the parsed components of each valid entry. + Each struct contains the following fields: + - arch: CPU architecture (e.g., "x86_64"). + - build_version: Standalone release date (e.g., "20260414"). + - location: Full package filename or URL (e.g., "cpython-3.11.15..." or "https://..."). + - flavor: Build configuration flavor (e.g., "install_only"). + - libc: C library type (e.g., "gnu", "musl", "msvc", or ""). + - microarch: Microarchitecture level (e.g., "v2", "v3", or ""). + - os: Operating system (e.g., "linux", "darwin", "windows"). + - python_version: Python semver version (e.g., "3.11.15"). + - sha256: SHA256 integrity hash of the release asset. + - vendor: Platform vendor (e.g., "unknown", "apple"). + """ + results = [] + for line in content.split("\n"): + line = line.strip() + if not line: + continue + parts = [p for p in line.split(" ") if p] + if len(parts) != 2: + continue + sha256, filename = parts + + parsed = parse_filename(filename) + if parsed: + results.append(struct( + sha256 = sha256, + **parsed + )) + return results diff --git a/python/private/python.bzl b/python/private/python.bzl index 6abc81e3d2..8f3b97a2c6 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -18,6 +18,7 @@ load("@bazel_features//:features.bzl", "bazel_features") load("//python:versions.bzl", "DEFAULT_RELEASE_BASE_URL", "PLATFORMS", "TOOL_VERSIONS") load(":auth.bzl", "AUTH_ATTRS") load(":full_version.bzl", "full_version") +load(":pbs_manifest.bzl", "parse_sha_manifest") load(":platform_info.bzl", "platform_info") load(":python_register_toolchains.bzl", "python_register_toolchains") load(":pythons_hub.bzl", "hub_repo") @@ -76,7 +77,7 @@ def parse_modules(*, module_ctx, logger = None, _fail = fail): # Map of string Major.Minor or Major.Minor.Patch to the toolchain_info struct global_toolchain_versions = {} - config = _get_toolchain_config(modules = module_ctx.modules, _fail = _fail) + config = _get_toolchain_config(mctx = module_ctx, modules = module_ctx.modules, _fail = _fail) default_python_version = _compute_default_python_version(module_ctx) @@ -741,10 +742,71 @@ def _override_defaults(*overrides, modules, _fail = fail, default): override.fn(tag = tag, _fail = _fail, default = default) -def _get_toolchain_config(*, modules, _fail = fail): +def _populate_from_pbs_manifest(*, mctx, add_runtime_manifest_urls, runtime_manifest_sha = "", available_versions, _fail): + manifest_path = mctx.path("runtime_manifest") + result = mctx.download( + url = add_runtime_manifest_urls, + output = manifest_path, + sha256 = runtime_manifest_sha, + ) + if not result.success: + _fail("Failed to download manifest from {}: {}".format(add_runtime_manifest_urls, result)) + return + + content = mctx.read(manifest_path) + base_download_urls = [url.rpartition("/")[0] for url in add_runtime_manifest_urls] + + parsed_entries = parse_sha_manifest(content) + + for entry in parsed_entries: + location = entry.location + sha256 = entry.sha256 + py_version = entry.python_version + + # Fallback to matching against PLATFORMS keys as before to ensure compatibility + # with rules_python expected platform keys. + matched_platform = None + for platform in PLATFORMS.keys(): + if platform in location: + matched_platform = platform + break + + if not matched_platform: + continue + + expects_full = matched_platform in [ + "aarch64-apple-darwin", + "aarch64-unknown-linux-gnu", + "ppc64le-unknown-linux-gnu", + "riscv64-unknown-linux-gnu", + "s390x-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", + "x86_64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", + ] + is_full = entry.flavor.endswith("-full") + if expects_full != is_full: + continue + + if "://" in location: + urls = [location] + else: + urls = ["{}/{}".format(base_url, location) for base_url in base_download_urls] + + v_dict = available_versions.setdefault(py_version, {}) + v_dict.setdefault("sha256", {})[matched_platform] = sha256 + v_dict.setdefault("url", {})[matched_platform] = urls + if is_full: + v_dict.setdefault("strip_prefix", {})[matched_platform] = "python/install" + else: + v_dict.setdefault("strip_prefix", {})[matched_platform] = "python" + +def _get_toolchain_config(*, mctx, modules, _fail = fail): """Computes the configs for toolchains. Args: + mctx: The module context. modules: The modules from module_ctx _fail: Function to call for failing; only used for testing. @@ -786,6 +848,19 @@ def _get_toolchain_config(*, modules, _fail = fail): else: available_versions[py_version]["url"] = dict(url) + # Check for add_runtime_manifest_urls in override tags in root module + root_module = modules[0] if modules else None + if root_module and root_module.is_root: + for tag in root_module.tags.override: + if tag.add_runtime_manifest_urls: + _populate_from_pbs_manifest( + mctx = mctx, + add_runtime_manifest_urls = tag.add_runtime_manifest_urls, + runtime_manifest_sha = tag.runtime_manifest_sha, + available_versions = available_versions, + _fail = _fail, + ) + default = { "base_url": DEFAULT_RELEASE_BASE_URL, "platforms": dict(PLATFORMS), # Copy so it's mutable. @@ -1111,6 +1186,21 @@ _override = tag_class( ::: """, attrs = { + "add_runtime_manifest_urls": attr.string_list( + mandatory = False, + doc = """ +URLs pointing to python-build-standalone manifest files (e.g., SHA256SUMS). + +Example: +`https://github.com/astral-sh/python-build-standalone/releases/download/20260414/SHA256SUMS` + +Note that `/latest/` can be used in place of a specific release date (e.g., `20260414`) to automatically use the latest release: +`https://github.com/astral-sh/python-build-standalone/releases/latest/download/SHA256SUMS` + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), "add_target_settings": attr.string_list( mandatory = False, doc = """\ @@ -1180,6 +1270,15 @@ The values in this mapping override the default values and do not replace them. default = {}, ), "register_all_versions": attr.bool(default = False, doc = "Add all versions"), + "runtime_manifest_sha": attr.string( + mandatory = False, + doc = """ +SHA256 hash for the add_runtime_manifest_urls. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), } | AUTH_ATTRS, ) diff --git a/tests/integration/BUILD.bazel b/tests/integration/BUILD.bazel index 9295cbb22f..81d0d13ac6 100644 --- a/tests/integration/BUILD.bazel +++ b/tests/integration/BUILD.bazel @@ -101,6 +101,10 @@ rules_python_integration_test( workspace_path = "py_cc_toolchain_registered", ) +rules_python_integration_test( + name = "runtime_manifests_test", +) + rules_python_integration_test( name = "custom_commands_test", py_main = "custom_commands_test.py", diff --git a/tests/integration/runtime_manifests/.bazelrc b/tests/integration/runtime_manifests/.bazelrc new file mode 100644 index 0000000000..d4d45a5ea7 --- /dev/null +++ b/tests/integration/runtime_manifests/.bazelrc @@ -0,0 +1,4 @@ +# Copy of fast-tests config +common:fast-tests --build_tests_only=true +common:fast-tests --build_tag_filters=-large,-enormous,-integration-test +common:fast-tests --test_tag_filters=-large,-enormous,-integration-test diff --git a/tests/integration/runtime_manifests/BUILD.bazel b/tests/integration/runtime_manifests/BUILD.bazel new file mode 100644 index 0000000000..4746fd419b --- /dev/null +++ b/tests/integration/runtime_manifests/BUILD.bazel @@ -0,0 +1,7 @@ +load("@rules_python//python:py_test.bzl", "py_test") + +py_test( + name = "basic_test", + srcs = ["basic_test.py"], + python_version = "3.11", +) diff --git a/tests/integration/runtime_manifests/MODULE.bazel b/tests/integration/runtime_manifests/MODULE.bazel new file mode 100644 index 0000000000..6891b07818 --- /dev/null +++ b/tests/integration/runtime_manifests/MODULE.bazel @@ -0,0 +1,19 @@ +module(name = "runtime_manifests") + +bazel_dep(name = "rules_python", version = "0.0.0") +local_path_override( + module_name = "rules_python", + path = "../../..", +) + +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.override( + add_runtime_manifest_urls = [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260414/SHA256SUMS", + ], + register_all_versions = True, + runtime_manifest_sha = "ce18fdfd47c66830a40ea9b9e314a14b1636bbfd684501bc5ca1fc6d55a7933f", +) +python.toolchain( + python_version = "3.11.15", +) diff --git a/tests/integration/runtime_manifests/WORKSPACE b/tests/integration/runtime_manifests/WORKSPACE new file mode 100644 index 0000000000..8277ec8090 --- /dev/null +++ b/tests/integration/runtime_manifests/WORKSPACE @@ -0,0 +1 @@ +# Workspace boundary file required by rules_bazel_integration_test diff --git a/tests/integration/runtime_manifests/basic_test.py b/tests/integration/runtime_manifests/basic_test.py new file mode 100644 index 0000000000..35f0e93b6e --- /dev/null +++ b/tests/integration/runtime_manifests/basic_test.py @@ -0,0 +1,27 @@ +import datetime +import platform +import sys +import unittest + + +class BasicTest(unittest.TestCase): + def test_basic(self): + print("Hello World from Python {}!".format(sys.version)) + print("Interpreter executable path: {}".format(sys.executable)) + + # Verify that the hermetic interpreter inside Bazel's output/sandbox tree is used + self.assertIn(".cache/bazel", sys.executable) + + # Verify that the exact custom version (3.11.15) parsed from the manifest is used + self.assertEqual(sys.version_info[:3], (3, 11, 15)) + + # Verify that the exact build version (20260414) parsed from the manifest is used + buildno, builddate = platform.python_build() + date_str = " ".join(builddate.split()[:3]) + dt = datetime.datetime.strptime(date_str, "%b %d %Y") + formatted_date = dt.strftime("%Y%m%d") + self.assertEqual(formatted_date, "20260414") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python_bzlmod_ext/BUILD.bazel b/tests/python_bzlmod_ext/BUILD.bazel new file mode 100644 index 0000000000..73bc7e0a65 --- /dev/null +++ b/tests/python_bzlmod_ext/BUILD.bazel @@ -0,0 +1,6 @@ +load(":parse_sha_manifest_tests.bzl", "parse_sha_manifest_test_suite") +load(":runtime_manifests_tests.bzl", "runtime_manifests_test_suite") + +parse_sha_manifest_test_suite(name = "parse_sha_manifest_tests") + +runtime_manifests_test_suite(name = "runtime_manifests_tests") diff --git a/tests/python_bzlmod_ext/parse_sha_manifest_tests.bzl b/tests/python_bzlmod_ext/parse_sha_manifest_tests.bzl new file mode 100644 index 0000000000..27cabab25a --- /dev/null +++ b/tests/python_bzlmod_ext/parse_sha_manifest_tests.bzl @@ -0,0 +1,155 @@ +"""Tests for manifest parsing Starlark functions.""" + +load("@bazel_skylib//lib:structs.bzl", "structs") +load("@rules_testing//lib:analysis_test.bzl", "analysis_test") +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("@rules_testing//lib:util.bzl", rt_util = "util") +load("//python/private:pbs_manifest.bzl", "parse_filename", "parse_sha_manifest") # buildifier: disable=bzl-visibility + +_tests = [] + +def _test_parse_filename_baseline(name): + """Sets up the baseline filename parsing test. + + Args: + name: The name of the test. + """ + rt_util.helper_target( + native.filegroup, + name = name + "_subject", + ) + analysis_test( + name = name, + target = name + "_subject", + impl = _test_parse_filename_baseline_impl, + ) + +def _test_parse_filename_baseline_impl(env, target): + _ = target # @unused + + # 1. Baseline + parsed1 = parse_filename("cpython-3.11.15+20260414-x86_64-unknown-linux-gnu-install_only.tar.gz") + env.expect.that_dict(parsed1).contains_exactly({ + "arch": "x86_64", + "build_version": "20260414", + "flavor": "install_only", + "libc": "gnu", + "location": "cpython-3.11.15+20260414-x86_64-unknown-linux-gnu-install_only.tar.gz", + "microarch": "", + "os": "linux", + "python_version": "3.11.15", + "vendor": "unknown", + }) + + # 2. Microarch + parsed2 = parse_filename("cpython-3.10.20+20260414-x86_64_v2-unknown-linux-musl-lto-full.tar.zst") + env.expect.that_dict(parsed2).contains_exactly({ + "arch": "x86_64", + "build_version": "20260414", + "flavor": "lto-full", + "libc": "musl", + "location": "cpython-3.10.20+20260414-x86_64_v2-unknown-linux-musl-lto-full.tar.zst", + "microarch": "v2", + "os": "linux", + "python_version": "3.10.20", + "vendor": "unknown", + }) + + # 3. Freethreaded + parsed3 = parse_filename("cpython-3.13.13+20260414-aarch64-apple-darwin-freethreaded+pgo+lto-full.tar.zst") + env.expect.that_dict(parsed3).contains_exactly({ + "arch": "aarch64", + "build_version": "20260414", + "flavor": "freethreaded+pgo+lto-full", + "libc": "", + "location": "cpython-3.13.13+20260414-aarch64-apple-darwin-freethreaded+pgo+lto-full.tar.zst", + "microarch": "", + "os": "darwin", + "python_version": "3.13.13", + "vendor": "apple", + }) + + # 4. Invalid + parsed4 = parse_filename("invalid-filename.tar.gz") + env.expect.that_bool(parsed4 == None).equals(True) + + # 5. Full URL (should return the original URL as location) + parsed5 = parse_filename("https://github.com/astral-sh/python-build-standalone/releases/download/20260414/cpython-3.11.15+20260414-x86_64-unknown-linux-gnu-install_only.tar.gz") + env.expect.that_dict(parsed5).contains_exactly({ + "arch": "x86_64", + "build_version": "20260414", + "flavor": "install_only", + "libc": "gnu", + "location": "https://github.com/astral-sh/python-build-standalone/releases/download/20260414/cpython-3.11.15+20260414-x86_64-unknown-linux-gnu-install_only.tar.gz", + "microarch": "", + "os": "linux", + "python_version": "3.11.15", + "vendor": "unknown", + }) + +_tests.append(_test_parse_filename_baseline) + +def _test_parse_sha_manifest(name): + """Sets up the manifest file parsing test. + + Args: + name: The name of the test. + """ + rt_util.helper_target( + native.filegroup, + name = name + "_subject", + ) + analysis_test( + name = name, + target = name + "_subject", + impl = _test_parse_sha_manifest_impl, + ) + +def _test_parse_sha_manifest_impl(env, target): + _ = target # @unused + content = """ +8b14030dd3af9ea7f7c51b4c90feb04afd8a8f45435727e67b875270bd08f3bc cpython-3.11.15+20260414-x86_64-unknown-linux-gnu-install_only.tar.gz +a57ffd435652092d16b30e783f9826c55e9c64b0f0a72cbae0a9f39e663137fb cpython-3.11.15+20260414-aarch64-apple-darwin-install_only.tar.gz +ce18fdfd47c66830a40ea9b9e314a14b1636bbfd684501bc5ca1fc6d55a7933f https://example.com/cpython-3.10.20+20260414-x86_64_v2-unknown-linux-musl-lto-full.tar.zst +""" + parsed = parse_sha_manifest(content) + env.expect.that_collection(parsed).has_size(3) + + env.expect.that_dict(structs.to_dict(parsed[0])).contains_exactly({ + "arch": "x86_64", + "build_version": "20260414", + "flavor": "install_only", + "libc": "gnu", + "location": "cpython-3.11.15+20260414-x86_64-unknown-linux-gnu-install_only.tar.gz", + "microarch": "", + "os": "linux", + "python_version": "3.11.15", + "sha256": "8b14030dd3af9ea7f7c51b4c90feb04afd8a8f45435727e67b875270bd08f3bc", + "vendor": "unknown", + }) + + env.expect.that_dict(structs.to_dict(parsed[2])).contains_exactly({ + "arch": "x86_64", + "build_version": "20260414", + "flavor": "lto-full", + "libc": "musl", + "location": "https://example.com/cpython-3.10.20+20260414-x86_64_v2-unknown-linux-musl-lto-full.tar.zst", + "microarch": "v2", + "os": "linux", + "python_version": "3.10.20", + "sha256": "ce18fdfd47c66830a40ea9b9e314a14b1636bbfd684501bc5ca1fc6d55a7933f", + "vendor": "unknown", + }) + +_tests.append(_test_parse_sha_manifest) + +def parse_sha_manifest_test_suite(name): + """Defines the test suite for manifest parsing. + + Args: + name: The name of the test suite. + """ + test_suite( + name = name, + tests = _tests, + ) diff --git a/tests/python_bzlmod_ext/runtime_manifests_tests.bzl b/tests/python_bzlmod_ext/runtime_manifests_tests.bzl new file mode 100644 index 0000000000..45c7406375 --- /dev/null +++ b/tests/python_bzlmod_ext/runtime_manifests_tests.bzl @@ -0,0 +1,96 @@ +"""Starlark unit tests for dynamic toolchain registration via manifests.""" + +load("@rules_testing//lib:analysis_test.bzl", "analysis_test") +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("@rules_testing//lib:util.bzl", rt_util = "util") +load("//python/private:python.bzl", "parse_modules") # buildifier: disable=bzl-visibility +load("//python/private:repo_utils.bzl", "repo_utils") # buildifier: disable=bzl-visibility +load("//tests/support/mocks:mocks.bzl", "mocks") # buildifier: disable=bzl-visibility +load("//tests/support/mocks:python_ext.bzl", "python_ext") # buildifier: disable=bzl-visibility + +_tests = [] + +_mock_logger = repo_utils.logger( + name = "mock", + verbosity_level = "ERROR", +) + +def _test_dynamic_manifest_toolchains(name): + rt_util.helper_target( + native.filegroup, + name = name + "_subject", + ) + analysis_test( + name = name, + target = name + "_subject", + impl = _test_dynamic_manifest_toolchains_impl, + ) + +def _test_dynamic_manifest_toolchains_impl(env, target): + _ = target # @unused + + # Construct Bzlmod mock module locally inside the test execution block. + # We test using virtual patch version "3.11.99" (not present in TOOL_VERSIONS) + # so that the populated config contains ONLY our dynamically parsed manifest keys + # without any pre-populated multi-platform templates, allowing exact dictionary match! + root_module = python_ext.module( + name = "runtime_manifests", + override = [ + python_ext.override( + add_runtime_manifest_urls = [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260414/SHA256SUMS", + ], + runtime_manifest_sha = "ce18fdfd47c66830a40ea9b9e314a14b1636bbfd684501bc5ca1fc6d55a7933f", + register_all_versions = True, + ), + ], + defaults = [ + python_ext.defaults( + python_version = "3.11.99", + ), + ], + ) + + # Pre-populate mock_files directly to bypass download output struct key mismatch in mock read lookups. + mock_mctx = mocks.mctx( + modules = [root_module], + mock_files = { + "runtime_manifest": """ +01e607cf764b97d4d5d6f69fd1ff3d8a9a162513dde5c39e98260fce40fe220a cpython-3.11.99+20260414-x86_64-unknown-linux-gnu-pgo+lto-full.tar.zst +8b14030dd3af9ea7f7c51b4c90feb04afd8a8f45435727e67b875270bd08f3bc cpython-3.11.99+20260414-x86_64-unknown-linux-gnu-install_only.tar.gz +""", + }, + ) + + res = parse_modules( + module_ctx = mock_mctx, + logger = _mock_logger, + ) + + tool_versions = res.config.default["tool_versions"] + env.expect.that_bool("3.11.99" in tool_versions).equals(True) + + version_info = tool_versions["3.11.99"] + + # Assert on the entire dictionary at once! + env.expect.that_dict(version_info).contains_exactly({ + "sha256": { + "x86_64-unknown-linux-gnu": "01e607cf764b97d4d5d6f69fd1ff3d8a9a162513dde5c39e98260fce40fe220a", + }, + "strip_prefix": { + "x86_64-unknown-linux-gnu": "python/install", + }, + "url": { + "x86_64-unknown-linux-gnu": [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260414/cpython-3.11.99+20260414-x86_64-unknown-linux-gnu-pgo+lto-full.tar.zst", + ], + }, + }) + +_tests.append(_test_dynamic_manifest_toolchains) + +def runtime_manifests_test_suite(name): + test_suite( + name = name, + tests = _tests, + ) diff --git a/tests/support/mocks/BUILD.bazel b/tests/support/mocks/BUILD.bazel index 45949fc578..07bdb2bffc 100644 --- a/tests/support/mocks/BUILD.bazel +++ b/tests/support/mocks/BUILD.bazel @@ -10,4 +10,10 @@ bzl_library( srcs = ["mocks.bzl"], ) +bzl_library( + name = "python_ext_bzl", + srcs = ["python_ext.bzl"], + deps = [":mocks_bzl"], +) + mocks_test_suite(name = "mocks_tests") diff --git a/tests/support/mocks/python_ext.bzl b/tests/support/mocks/python_ext.bzl new file mode 100644 index 0000000000..6f76ec785b --- /dev/null +++ b/tests/support/mocks/python_ext.bzl @@ -0,0 +1,102 @@ +"""Helper for defining a mock module for the python bzlmod extension.""" + +load("//tests/support/mocks:mocks.bzl", "mocks") # buildifier: disable=bzl-visibility + +def _module(name = "rules_python", is_root = True, **tags): + """Creates a mock Bzlmod module struct with defaulted tag lists. + + Args: + name: The module name. + is_root: Whether this is the root module. + **tags: Lists of tag objects. + + Returns: + A mock module struct. + """ + defaulted_tags = { + "defaults": [], + "override": [], + "single_version_override": [], + "single_version_platform_override": [], + "toolchain": [], + } + defaulted_tags.update(tags) + return mocks.module(name = name, is_root = is_root, **defaulted_tags) + +def _override(**kwargs): + """Creates a mock python.override tag with default values.""" + attrs = { + "add_runtime_manifest_urls": [], + "add_target_settings": [], + "available_python_versions": [], + "base_url": "https://github.com/astral-sh/python-build-standalone/releases/download", + "ignore_root_user_error": True, + "minor_mapping": {}, + "register_all_versions": False, + "runtime_manifest_sha": "", + } + attrs.update(kwargs) + return mocks.tag(**attrs) + +def _defaults(**kwargs): + """Creates a mock python.defaults tag with default values.""" + attrs = { + "python_version": "", + "python_version_env": "", + } + attrs.update(kwargs) + return mocks.tag(**attrs) + +def _single_version_override(**kwargs): + """Creates a mock python.single_version_override tag with default values.""" + attrs = { + "distutils": None, + "distutils_content": "", + "patch_strip": 0, + "patches": [], + "python_version": "", + "sha256": {}, + "strip_prefix": "python", + "urls": [], + } + attrs.update(kwargs) + return mocks.tag(**attrs) + +def _single_version_platform_override(**kwargs): + """Creates a mock python.single_version_platform_override tag with default values.""" + attrs = { + "arch": "", + "coverage_tool": None, + "os_name": "", + "patch_strip": 0, + "patches": [], + "platform": "", + "python_version": "", + "sha256": "", + "strip_prefix": "python", + "target_compatible_with": [], + "target_settings": [], + "urls": [], + } + attrs.update(kwargs) + return mocks.tag(**attrs) + +def _toolchain(**kwargs): + """Creates a mock python.toolchain tag with default values.""" + attrs = { + "configure_coverage_tool": False, + "ignore_root_user_error": True, + "is_default": False, + "python_version": "", + } + attrs.update(kwargs) + return mocks.tag(**attrs) + +python_ext = struct( + defaults = _defaults, + module = _module, + override = _override, + single_version_override = _single_version_override, + single_version_platform_override = _single_version_platform_override, + toolchain = _toolchain, +) diff --git a/tools/private/debug/print_defined_toolchains.sh b/tools/private/debug/print_defined_toolchains.sh new file mode 100755 index 0000000000..1694dc975c --- /dev/null +++ b/tools/private/debug/print_defined_toolchains.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Programmatically probe which repository target name is resolved successfully inside this workspace +if bazel query @pythons_hub//... >/dev/null 2>&1; then + HUB_REPO="@pythons_hub" +elif bazel query @@rules_python++python+pythons_hub//... >/dev/null 2>&1; then + HUB_REPO="@@rules_python++python+pythons_hub" +else + HUB_REPO="@@+python+pythons_hub" +fi + +# Query standard toolchains inside the resolved hub repository, excluding CC and Exec Tools toolchains. +bazel query "kind('toolchain', ${HUB_REPO}//...) - filter('_py_cc_toolchain$', ${HUB_REPO}//...) - filter('_py_exec_tools_toolchain$', ${HUB_REPO}//...)" "$@" From 7533be344aada763617b982e9780829076095b10 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 31 May 2026 21:58:00 +0000 Subject: [PATCH 2/3] test(python): bypass bzlmod extension unit tests under legacy workspace mode Workspace builds running under older Bazel versions do not support Bzlmod module extensions. This commit introduces Starlark helper macros to conditionally gate and register Bzlmod-specific unit tests, resolving test suite loading crashes in workspace CI jobs. --- tests/python/BUILD.bazel | 4 +- tests/python/python_tests.bzl | 6 ++- tests/python/test_helpers.bzl | 32 ++++++++++++++++ tests/python_bzlmod_ext/BUILD.bazel | 11 +++--- tests/python_bzlmod_ext/test_helpers.bzl | 48 ++++++++++++++++++++++++ 5 files changed, 93 insertions(+), 8 deletions(-) create mode 100644 tests/python/test_helpers.bzl create mode 100644 tests/python_bzlmod_ext/test_helpers.bzl diff --git a/tests/python/BUILD.bazel b/tests/python/BUILD.bazel index 2553536b63..92970d4a9c 100644 --- a/tests/python/BUILD.bazel +++ b/tests/python/BUILD.bazel @@ -12,6 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -load(":python_tests.bzl", "python_test_suite") +load(":test_helpers.bzl", "register_python_tests") -python_test_suite(name = "python_tests") +register_python_tests(name = "python_tests") diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index 60e7311c00..e753c1787b 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl @@ -54,7 +54,9 @@ def _override( minor_mapping = {}, netrc = "", register_all_versions = False, - add_target_settings = []): + add_target_settings = [], + add_runtime_manifest_urls = [], + runtime_manifest_sha = None): return struct( auth_patterns = auth_patterns, available_python_versions = available_python_versions, @@ -63,6 +65,8 @@ def _override( netrc = netrc, register_all_versions = register_all_versions, add_target_settings = add_target_settings, + add_runtime_manifest_urls = add_runtime_manifest_urls, + runtime_manifest_sha = runtime_manifest_sha, ) def _rules_python_module(is_root = False): diff --git a/tests/python/test_helpers.bzl b/tests/python/test_helpers.bzl new file mode 100644 index 0000000000..58059ce212 --- /dev/null +++ b/tests/python/test_helpers.bzl @@ -0,0 +1,32 @@ +# Copyright 2026 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helpers to conditionally register tests depending on Bzlmod enablement.""" + +load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility +load(":python_tests.bzl", "python_test_suite") + +def register_python_tests(name): + """Registers the python tests if Bzlmod is enabled, otherwise defines an empty test_suite. + + Args: + name: The name of the test target. + """ + if BZLMOD_ENABLED: + python_test_suite(name = name) + else: + native.test_suite( + name = name, + tests = [], + ) diff --git a/tests/python_bzlmod_ext/BUILD.bazel b/tests/python_bzlmod_ext/BUILD.bazel index 73bc7e0a65..0add4e7690 100644 --- a/tests/python_bzlmod_ext/BUILD.bazel +++ b/tests/python_bzlmod_ext/BUILD.bazel @@ -1,6 +1,7 @@ -load(":parse_sha_manifest_tests.bzl", "parse_sha_manifest_test_suite") -load(":runtime_manifests_tests.bzl", "runtime_manifests_test_suite") +load(":test_helpers.bzl", "register_python_bzlmod_ext_tests") -parse_sha_manifest_test_suite(name = "parse_sha_manifest_tests") - -runtime_manifests_test_suite(name = "runtime_manifests_tests") +register_python_bzlmod_ext_tests( + name = "python_bzlmod_ext_tests", + parse_sha_manifest_name = "parse_sha_manifest_tests", + runtime_manifests_name = "runtime_manifests_tests", +) diff --git a/tests/python_bzlmod_ext/test_helpers.bzl b/tests/python_bzlmod_ext/test_helpers.bzl new file mode 100644 index 0000000000..f6c177650a --- /dev/null +++ b/tests/python_bzlmod_ext/test_helpers.bzl @@ -0,0 +1,48 @@ +# Copyright 2026 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helpers to conditionally register tests depending on Bzlmod enablement.""" + +load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility +load(":parse_sha_manifest_tests.bzl", "parse_sha_manifest_test_suite") +load(":runtime_manifests_tests.bzl", "runtime_manifests_test_suite") + +def register_python_bzlmod_ext_tests(name, parse_sha_manifest_name, runtime_manifests_name): + """Registers the Bzlmod extension tests if Bzlmod is enabled, otherwise defines empty test_suites. + + Args: + name: The name of the master test_suite target. + parse_sha_manifest_name: The name of the parse_sha_manifest test target. + runtime_manifests_name: The name of the runtime_manifests test target. + """ + if BZLMOD_ENABLED: + parse_sha_manifest_test_suite(name = parse_sha_manifest_name) + runtime_manifests_test_suite(name = runtime_manifests_name) + else: + native.test_suite( + name = parse_sha_manifest_name, + tests = [], + ) + native.test_suite( + name = runtime_manifests_name, + tests = [], + ) + + native.test_suite( + name = name, + tests = [ + parse_sha_manifest_name, + runtime_manifests_name, + ], + ) From 4cbfe5e911927fe4b330b247769ba35f70070112 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 31 May 2026 21:58:22 +0000 Subject: [PATCH 3/3] refactor(test): inline python test helpers into python_tests.bzl Removes the external test_helpers.bzl under tests/python/ and inlines the register_python_tests macro directly inside python_tests.bzl. This simplifies the test suite layout while keeping legacy workspace gating completely intact. --- tests/python/BUILD.bazel | 2 +- tests/python/python_tests.bzl | 15 +++++++++++++++ tests/python/test_helpers.bzl | 32 -------------------------------- 3 files changed, 16 insertions(+), 33 deletions(-) delete mode 100644 tests/python/test_helpers.bzl diff --git a/tests/python/BUILD.bazel b/tests/python/BUILD.bazel index 92970d4a9c..887fe969b5 100644 --- a/tests/python/BUILD.bazel +++ b/tests/python/BUILD.bazel @@ -12,6 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -load(":test_helpers.bzl", "register_python_tests") +load(":python_tests.bzl", "register_python_tests") register_python_tests(name = "python_tests") diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index e753c1787b..c2a32400db 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl @@ -16,6 +16,7 @@ load("@pythons_hub//:versions.bzl", "MINOR_MAPPING") load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility load("//python/private:python.bzl", "parse_modules") # buildifier: disable=bzl-visibility load("//python/private:repo_utils.bzl", "repo_utils") # buildifier: disable=bzl-visibility load("//tests/support/mocks:mocks.bzl", "mocks") @@ -893,3 +894,17 @@ def python_test_suite(name): name: the name of the test suite """ test_suite(name = name, basic_tests = _tests) + +def register_python_tests(name): + """Registers the python tests if Bzlmod is enabled, otherwise defines an empty test_suite. + + Args: + name: The name of the test target. + """ + if BZLMOD_ENABLED: + python_test_suite(name = name) + else: + native.test_suite( + name = name, + tests = [], + ) diff --git a/tests/python/test_helpers.bzl b/tests/python/test_helpers.bzl deleted file mode 100644 index 58059ce212..0000000000 --- a/tests/python/test_helpers.bzl +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2026 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Helpers to conditionally register tests depending on Bzlmod enablement.""" - -load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility -load(":python_tests.bzl", "python_test_suite") - -def register_python_tests(name): - """Registers the python tests if Bzlmod is enabled, otherwise defines an empty test_suite. - - Args: - name: The name of the test target. - """ - if BZLMOD_ENABLED: - python_test_suite(name = name) - else: - native.test_suite( - name = name, - tests = [], - )