diff --git a/.bazelrc.deleted_packages b/.bazelrc.deleted_packages index f4ea8527f3..b84214a1c7 100644 --- a/.bazelrc.deleted_packages +++ b/.bazelrc.deleted_packages @@ -38,6 +38,7 @@ common --deleted_packages=tests/integration/pip_parse common --deleted_packages=tests/integration/pip_parse/empty common --deleted_packages=tests/integration/pip_parse_isolated common --deleted_packages=tests/integration/py_cc_toolchain_registered +common --deleted_packages=tests/integration/runtime_manifests common --deleted_packages=tests/integration/toolchain_target_settings common --deleted_packages=tests/modules/another_module common --deleted_packages=tests/modules/other diff --git a/CHANGELOG.md b/CHANGELOG.md index bf6fe3095c..e7c56543fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -104,6 +104,9 @@ END_UNRELEASED_TEMPLATE {#v0-0-0-added} ### Added +* (toolchains) Support dynamically fetching and registering Python runtimes + from a python-build-standalone manifest file using + `python.override(add_runtime_manifest_urls = ..., runtime_manifest_sha = ...)`. * (toolchain) Added {obj}`python.override.toolchain_target_settings` to allow adding `config_setting` labels to all registered toolchains. * (windows) Full venv support for Windows is available. Set diff --git a/docs/toolchains.md b/docs/toolchains.md index 09aaed412b..b048c74a1e 100644 --- a/docs/toolchains.md +++ b/docs/toolchains.md @@ -242,6 +242,8 @@ existing attributes: {attr}`python.single_version_platform_override.coverage_tool`. * Adding additional Python versions via {bzl:obj}`python.single_version_override` or {bzl:obj}`python.single_version_platform_override`. +* Adding additional Python versions dynamically from a remote manifest file + via {attr}`python.override.add_runtime_manifest_urls`. ### Registering custom runtimes @@ -310,6 +312,65 @@ Added support for custom platform names, `target_compatible_with`, and `target_settings` with `single_version_platform_override`. ::: +### Registering runtimes from a manifest + +If you want to register multiple custom runtimes or versions at once, you can +use a python-build-standalone manifest file. This is useful if you want to +adopt new versions that are not yet built into `rules_python` without having +to manually define each one using `single_version_platform_override`. + +To do this, specify the `add_runtime_manifest_urls` and `runtime_manifest_sha` +attributes in `python.override` in your `MODULE.bazel`. + +In the example below, we register all runtimes available in a specific PBS +release manifest: + +``` +# File: MODULE.bazel +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.override( + add_runtime_manifest_urls = [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260414/SHA256SUMS", + ], + runtime_manifest_sha = "ce18fdfd47c66830a40ea9b9e314a14b1636bbfd684501bc5ca1fc6d55a7933f", +) +``` + +#### Manifest file format + +The manifest must be a plain text file where each line contains the SHA256 hash +and the location of a runtime archive, separated by whitespace: + +``` + +``` + +The `` can be either: +- A relative filename (e.g., + `cpython-3.10.20+20260414-x86_64-unknown-linux-gnu-install_only.tar.zst`). + In this case, the download URL is constructed by appending the filename to the + parent directory of each URL in `add_runtime_manifest_urls` (treating them as + mirrors). +- An absolute URL (e.g., + `https://example.com/downloads/cpython-3.10.20+20260414-x86_64-unknown-linux-gnu-install_only.tar.zst`). + In this case, the URL is used directly to download the archive. + +In both cases, the filename or the last path segment of the URL must follow the +standard python-build-standalone naming convention. `rules_python` parses this +name to extract runtime metadata (such as Python version, target architecture, +operating system, and libc). + +Notes: +- `rules_python` will download the manifest, parse it, and automatically + register toolchains for all valid Python runtimes found in it that match + supported platforms. +- Only runtimes matching known platforms in `rules_python` will be registered. + +:::{versionadded} VERSION_NEXT_FEATURE +Added support for registering runtimes from a manifest using +`add_runtime_manifest_urls` and `runtime_manifest_sha` in `python.override`. +::: + ### Using defined toolchains from WORKSPACE It is possible to use toolchains defined in `MODULE.bazel` in `WORKSPACE`. For example, diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index 6ab3d546be..b54c198069 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -252,6 +252,11 @@ bzl_library( srcs = ["normalize_name.bzl"], ) +bzl_library( + name = "pbs_manifest_bzl", + srcs = ["pbs_manifest.bzl"], +) + bzl_library( name = "precompile_bzl", srcs = ["precompile.bzl"], @@ -274,6 +279,7 @@ bzl_library( srcs = ["python.bzl"], deps = [ ":full_version_bzl", + ":pbs_manifest_bzl", ":platform_info_bzl", ":python_register_toolchains_bzl", ":pythons_hub_bzl", diff --git a/python/private/pbs_manifest.bzl b/python/private/pbs_manifest.bzl new file mode 100644 index 0000000000..495e4af907 --- /dev/null +++ b/python/private/pbs_manifest.bzl @@ -0,0 +1,116 @@ +"""Helper functions to parse python-build-standalone manifests.""" + +def parse_filename(filename): + """Parses a python-build-standalone filename (or URL) into its components. + + Example: cpython-3.10.20+20260414-x86_64_v2-unknown-linux-musl-lto-full.tar.zst + + Args: + filename: The filename or URL of the python-build-standalone release asset. + + Returns: + A dictionary of parsed components if parsed successfully, else None. + """ + basename = filename.rpartition("/")[-1] + if basename.endswith(".tar.zst"): + name = basename.removesuffix(".tar.zst") + elif basename.endswith(".tar.gz"): + name = basename.removesuffix(".tar.gz") + else: + return None + + if not name.startswith("cpython-"): + return None + name = name.removeprefix("cpython-") + + left, plus, tail = name.partition("+") + if plus: + python_version = left + build_version, sep, rest = tail.partition("-") + if not sep: + return None + else: + python_version, sep, rest = left.partition("-") + if not sep: + return None + build_version = "" + + arch, sep, rest = rest.partition("-") + if not sep: + return None + + microarch = "" + arch_base, sep_v, microarch_num = arch.partition("_v") + if sep_v: + arch = arch_base + microarch = "v" + microarch_num + + vendor, sep, rest = rest.partition("-") + if not sep: + return None + + os, sep, rest = rest.partition("-") + if not sep: + return None + + libc = "" + next_part, _, remaining = rest.partition("-") + if os == "linux" and next_part in ["gnu", "musl"]: + libc = next_part + flavor = remaining + elif os == "windows" and next_part == "msvc": + libc = next_part + flavor = remaining + else: + libc = "" + flavor = rest + + return { + "arch": arch, + "build_version": build_version, + "flavor": flavor, + "libc": libc, + "location": filename, + "microarch": microarch, + "os": os, + "python_version": python_version, + "vendor": vendor, + } + +def parse_sha_manifest(content): + """Parses the SHA256SUMS file content into a list of structs. + + Args: + content: The raw content of the manifest file. + + Returns: + A list of structs capturing the parsed components of each valid entry. + Each struct contains the following fields: + - arch: CPU architecture (e.g., "x86_64"). + - build_version: Standalone release date (e.g., "20260414"). + - location: Full package filename or URL (e.g., "cpython-3.11.15..." or "https://..."). + - flavor: Build configuration flavor (e.g., "install_only"). + - libc: C library type (e.g., "gnu", "musl", "msvc", or ""). + - microarch: Microarchitecture level (e.g., "v2", "v3", or ""). + - os: Operating system (e.g., "linux", "darwin", "windows"). + - python_version: Python semver version (e.g., "3.11.15"). + - sha256: SHA256 integrity hash of the release asset. + - vendor: Platform vendor (e.g., "unknown", "apple"). + """ + results = [] + for line in content.split("\n"): + line = line.strip() + if not line: + continue + parts = [p for p in line.split(" ") if p] + if len(parts) != 2: + continue + sha256, filename = parts + + parsed = parse_filename(filename) + if parsed: + results.append(struct( + sha256 = sha256, + **parsed + )) + return results diff --git a/python/private/python.bzl b/python/private/python.bzl index 6abc81e3d2..8f3b97a2c6 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -18,6 +18,7 @@ load("@bazel_features//:features.bzl", "bazel_features") load("//python:versions.bzl", "DEFAULT_RELEASE_BASE_URL", "PLATFORMS", "TOOL_VERSIONS") load(":auth.bzl", "AUTH_ATTRS") load(":full_version.bzl", "full_version") +load(":pbs_manifest.bzl", "parse_sha_manifest") load(":platform_info.bzl", "platform_info") load(":python_register_toolchains.bzl", "python_register_toolchains") load(":pythons_hub.bzl", "hub_repo") @@ -76,7 +77,7 @@ def parse_modules(*, module_ctx, logger = None, _fail = fail): # Map of string Major.Minor or Major.Minor.Patch to the toolchain_info struct global_toolchain_versions = {} - config = _get_toolchain_config(modules = module_ctx.modules, _fail = _fail) + config = _get_toolchain_config(mctx = module_ctx, modules = module_ctx.modules, _fail = _fail) default_python_version = _compute_default_python_version(module_ctx) @@ -741,10 +742,71 @@ def _override_defaults(*overrides, modules, _fail = fail, default): override.fn(tag = tag, _fail = _fail, default = default) -def _get_toolchain_config(*, modules, _fail = fail): +def _populate_from_pbs_manifest(*, mctx, add_runtime_manifest_urls, runtime_manifest_sha = "", available_versions, _fail): + manifest_path = mctx.path("runtime_manifest") + result = mctx.download( + url = add_runtime_manifest_urls, + output = manifest_path, + sha256 = runtime_manifest_sha, + ) + if not result.success: + _fail("Failed to download manifest from {}: {}".format(add_runtime_manifest_urls, result)) + return + + content = mctx.read(manifest_path) + base_download_urls = [url.rpartition("/")[0] for url in add_runtime_manifest_urls] + + parsed_entries = parse_sha_manifest(content) + + for entry in parsed_entries: + location = entry.location + sha256 = entry.sha256 + py_version = entry.python_version + + # Fallback to matching against PLATFORMS keys as before to ensure compatibility + # with rules_python expected platform keys. + matched_platform = None + for platform in PLATFORMS.keys(): + if platform in location: + matched_platform = platform + break + + if not matched_platform: + continue + + expects_full = matched_platform in [ + "aarch64-apple-darwin", + "aarch64-unknown-linux-gnu", + "ppc64le-unknown-linux-gnu", + "riscv64-unknown-linux-gnu", + "s390x-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", + "x86_64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", + ] + is_full = entry.flavor.endswith("-full") + if expects_full != is_full: + continue + + if "://" in location: + urls = [location] + else: + urls = ["{}/{}".format(base_url, location) for base_url in base_download_urls] + + v_dict = available_versions.setdefault(py_version, {}) + v_dict.setdefault("sha256", {})[matched_platform] = sha256 + v_dict.setdefault("url", {})[matched_platform] = urls + if is_full: + v_dict.setdefault("strip_prefix", {})[matched_platform] = "python/install" + else: + v_dict.setdefault("strip_prefix", {})[matched_platform] = "python" + +def _get_toolchain_config(*, mctx, modules, _fail = fail): """Computes the configs for toolchains. Args: + mctx: The module context. modules: The modules from module_ctx _fail: Function to call for failing; only used for testing. @@ -786,6 +848,19 @@ def _get_toolchain_config(*, modules, _fail = fail): else: available_versions[py_version]["url"] = dict(url) + # Check for add_runtime_manifest_urls in override tags in root module + root_module = modules[0] if modules else None + if root_module and root_module.is_root: + for tag in root_module.tags.override: + if tag.add_runtime_manifest_urls: + _populate_from_pbs_manifest( + mctx = mctx, + add_runtime_manifest_urls = tag.add_runtime_manifest_urls, + runtime_manifest_sha = tag.runtime_manifest_sha, + available_versions = available_versions, + _fail = _fail, + ) + default = { "base_url": DEFAULT_RELEASE_BASE_URL, "platforms": dict(PLATFORMS), # Copy so it's mutable. @@ -1111,6 +1186,21 @@ _override = tag_class( ::: """, attrs = { + "add_runtime_manifest_urls": attr.string_list( + mandatory = False, + doc = """ +URLs pointing to python-build-standalone manifest files (e.g., SHA256SUMS). + +Example: +`https://github.com/astral-sh/python-build-standalone/releases/download/20260414/SHA256SUMS` + +Note that `/latest/` can be used in place of a specific release date (e.g., `20260414`) to automatically use the latest release: +`https://github.com/astral-sh/python-build-standalone/releases/latest/download/SHA256SUMS` + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), "add_target_settings": attr.string_list( mandatory = False, doc = """\ @@ -1180,6 +1270,15 @@ The values in this mapping override the default values and do not replace them. default = {}, ), "register_all_versions": attr.bool(default = False, doc = "Add all versions"), + "runtime_manifest_sha": attr.string( + mandatory = False, + doc = """ +SHA256 hash for the add_runtime_manifest_urls. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), } | AUTH_ATTRS, ) diff --git a/tests/integration/BUILD.bazel b/tests/integration/BUILD.bazel index 9295cbb22f..81d0d13ac6 100644 --- a/tests/integration/BUILD.bazel +++ b/tests/integration/BUILD.bazel @@ -101,6 +101,10 @@ rules_python_integration_test( workspace_path = "py_cc_toolchain_registered", ) +rules_python_integration_test( + name = "runtime_manifests_test", +) + rules_python_integration_test( name = "custom_commands_test", py_main = "custom_commands_test.py", diff --git a/tests/integration/runtime_manifests/.bazelrc b/tests/integration/runtime_manifests/.bazelrc new file mode 100644 index 0000000000..d4d45a5ea7 --- /dev/null +++ b/tests/integration/runtime_manifests/.bazelrc @@ -0,0 +1,4 @@ +# Copy of fast-tests config +common:fast-tests --build_tests_only=true +common:fast-tests --build_tag_filters=-large,-enormous,-integration-test +common:fast-tests --test_tag_filters=-large,-enormous,-integration-test diff --git a/tests/integration/runtime_manifests/BUILD.bazel b/tests/integration/runtime_manifests/BUILD.bazel new file mode 100644 index 0000000000..4746fd419b --- /dev/null +++ b/tests/integration/runtime_manifests/BUILD.bazel @@ -0,0 +1,7 @@ +load("@rules_python//python:py_test.bzl", "py_test") + +py_test( + name = "basic_test", + srcs = ["basic_test.py"], + python_version = "3.11", +) diff --git a/tests/integration/runtime_manifests/MODULE.bazel b/tests/integration/runtime_manifests/MODULE.bazel new file mode 100644 index 0000000000..6891b07818 --- /dev/null +++ b/tests/integration/runtime_manifests/MODULE.bazel @@ -0,0 +1,19 @@ +module(name = "runtime_manifests") + +bazel_dep(name = "rules_python", version = "0.0.0") +local_path_override( + module_name = "rules_python", + path = "../../..", +) + +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.override( + add_runtime_manifest_urls = [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260414/SHA256SUMS", + ], + register_all_versions = True, + runtime_manifest_sha = "ce18fdfd47c66830a40ea9b9e314a14b1636bbfd684501bc5ca1fc6d55a7933f", +) +python.toolchain( + python_version = "3.11.15", +) diff --git a/tests/integration/runtime_manifests/WORKSPACE b/tests/integration/runtime_manifests/WORKSPACE new file mode 100644 index 0000000000..8277ec8090 --- /dev/null +++ b/tests/integration/runtime_manifests/WORKSPACE @@ -0,0 +1 @@ +# Workspace boundary file required by rules_bazel_integration_test diff --git a/tests/integration/runtime_manifests/basic_test.py b/tests/integration/runtime_manifests/basic_test.py new file mode 100644 index 0000000000..35f0e93b6e --- /dev/null +++ b/tests/integration/runtime_manifests/basic_test.py @@ -0,0 +1,27 @@ +import datetime +import platform +import sys +import unittest + + +class BasicTest(unittest.TestCase): + def test_basic(self): + print("Hello World from Python {}!".format(sys.version)) + print("Interpreter executable path: {}".format(sys.executable)) + + # Verify that the hermetic interpreter inside Bazel's output/sandbox tree is used + self.assertIn(".cache/bazel", sys.executable) + + # Verify that the exact custom version (3.11.15) parsed from the manifest is used + self.assertEqual(sys.version_info[:3], (3, 11, 15)) + + # Verify that the exact build version (20260414) parsed from the manifest is used + buildno, builddate = platform.python_build() + date_str = " ".join(builddate.split()[:3]) + dt = datetime.datetime.strptime(date_str, "%b %d %Y") + formatted_date = dt.strftime("%Y%m%d") + self.assertEqual(formatted_date, "20260414") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/BUILD.bazel b/tests/python/BUILD.bazel index 2553536b63..887fe969b5 100644 --- a/tests/python/BUILD.bazel +++ b/tests/python/BUILD.bazel @@ -12,6 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -load(":python_tests.bzl", "python_test_suite") +load(":python_tests.bzl", "register_python_tests") -python_test_suite(name = "python_tests") +register_python_tests(name = "python_tests") diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index 60e7311c00..c2a32400db 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl @@ -16,6 +16,7 @@ load("@pythons_hub//:versions.bzl", "MINOR_MAPPING") load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility load("//python/private:python.bzl", "parse_modules") # buildifier: disable=bzl-visibility load("//python/private:repo_utils.bzl", "repo_utils") # buildifier: disable=bzl-visibility load("//tests/support/mocks:mocks.bzl", "mocks") @@ -54,7 +55,9 @@ def _override( minor_mapping = {}, netrc = "", register_all_versions = False, - add_target_settings = []): + add_target_settings = [], + add_runtime_manifest_urls = [], + runtime_manifest_sha = None): return struct( auth_patterns = auth_patterns, available_python_versions = available_python_versions, @@ -63,6 +66,8 @@ def _override( netrc = netrc, register_all_versions = register_all_versions, add_target_settings = add_target_settings, + add_runtime_manifest_urls = add_runtime_manifest_urls, + runtime_manifest_sha = runtime_manifest_sha, ) def _rules_python_module(is_root = False): @@ -889,3 +894,17 @@ def python_test_suite(name): name: the name of the test suite """ test_suite(name = name, basic_tests = _tests) + +def register_python_tests(name): + """Registers the python tests if Bzlmod is enabled, otherwise defines an empty test_suite. + + Args: + name: The name of the test target. + """ + if BZLMOD_ENABLED: + python_test_suite(name = name) + else: + native.test_suite( + name = name, + tests = [], + ) diff --git a/tests/python_bzlmod_ext/BUILD.bazel b/tests/python_bzlmod_ext/BUILD.bazel new file mode 100644 index 0000000000..0add4e7690 --- /dev/null +++ b/tests/python_bzlmod_ext/BUILD.bazel @@ -0,0 +1,7 @@ +load(":test_helpers.bzl", "register_python_bzlmod_ext_tests") + +register_python_bzlmod_ext_tests( + name = "python_bzlmod_ext_tests", + parse_sha_manifest_name = "parse_sha_manifest_tests", + runtime_manifests_name = "runtime_manifests_tests", +) diff --git a/tests/python_bzlmod_ext/parse_sha_manifest_tests.bzl b/tests/python_bzlmod_ext/parse_sha_manifest_tests.bzl new file mode 100644 index 0000000000..27cabab25a --- /dev/null +++ b/tests/python_bzlmod_ext/parse_sha_manifest_tests.bzl @@ -0,0 +1,155 @@ +"""Tests for manifest parsing Starlark functions.""" + +load("@bazel_skylib//lib:structs.bzl", "structs") +load("@rules_testing//lib:analysis_test.bzl", "analysis_test") +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("@rules_testing//lib:util.bzl", rt_util = "util") +load("//python/private:pbs_manifest.bzl", "parse_filename", "parse_sha_manifest") # buildifier: disable=bzl-visibility + +_tests = [] + +def _test_parse_filename_baseline(name): + """Sets up the baseline filename parsing test. + + Args: + name: The name of the test. + """ + rt_util.helper_target( + native.filegroup, + name = name + "_subject", + ) + analysis_test( + name = name, + target = name + "_subject", + impl = _test_parse_filename_baseline_impl, + ) + +def _test_parse_filename_baseline_impl(env, target): + _ = target # @unused + + # 1. Baseline + parsed1 = parse_filename("cpython-3.11.15+20260414-x86_64-unknown-linux-gnu-install_only.tar.gz") + env.expect.that_dict(parsed1).contains_exactly({ + "arch": "x86_64", + "build_version": "20260414", + "flavor": "install_only", + "libc": "gnu", + "location": "cpython-3.11.15+20260414-x86_64-unknown-linux-gnu-install_only.tar.gz", + "microarch": "", + "os": "linux", + "python_version": "3.11.15", + "vendor": "unknown", + }) + + # 2. Microarch + parsed2 = parse_filename("cpython-3.10.20+20260414-x86_64_v2-unknown-linux-musl-lto-full.tar.zst") + env.expect.that_dict(parsed2).contains_exactly({ + "arch": "x86_64", + "build_version": "20260414", + "flavor": "lto-full", + "libc": "musl", + "location": "cpython-3.10.20+20260414-x86_64_v2-unknown-linux-musl-lto-full.tar.zst", + "microarch": "v2", + "os": "linux", + "python_version": "3.10.20", + "vendor": "unknown", + }) + + # 3. Freethreaded + parsed3 = parse_filename("cpython-3.13.13+20260414-aarch64-apple-darwin-freethreaded+pgo+lto-full.tar.zst") + env.expect.that_dict(parsed3).contains_exactly({ + "arch": "aarch64", + "build_version": "20260414", + "flavor": "freethreaded+pgo+lto-full", + "libc": "", + "location": "cpython-3.13.13+20260414-aarch64-apple-darwin-freethreaded+pgo+lto-full.tar.zst", + "microarch": "", + "os": "darwin", + "python_version": "3.13.13", + "vendor": "apple", + }) + + # 4. Invalid + parsed4 = parse_filename("invalid-filename.tar.gz") + env.expect.that_bool(parsed4 == None).equals(True) + + # 5. Full URL (should return the original URL as location) + parsed5 = parse_filename("https://github.com/astral-sh/python-build-standalone/releases/download/20260414/cpython-3.11.15+20260414-x86_64-unknown-linux-gnu-install_only.tar.gz") + env.expect.that_dict(parsed5).contains_exactly({ + "arch": "x86_64", + "build_version": "20260414", + "flavor": "install_only", + "libc": "gnu", + "location": "https://github.com/astral-sh/python-build-standalone/releases/download/20260414/cpython-3.11.15+20260414-x86_64-unknown-linux-gnu-install_only.tar.gz", + "microarch": "", + "os": "linux", + "python_version": "3.11.15", + "vendor": "unknown", + }) + +_tests.append(_test_parse_filename_baseline) + +def _test_parse_sha_manifest(name): + """Sets up the manifest file parsing test. + + Args: + name: The name of the test. + """ + rt_util.helper_target( + native.filegroup, + name = name + "_subject", + ) + analysis_test( + name = name, + target = name + "_subject", + impl = _test_parse_sha_manifest_impl, + ) + +def _test_parse_sha_manifest_impl(env, target): + _ = target # @unused + content = """ +8b14030dd3af9ea7f7c51b4c90feb04afd8a8f45435727e67b875270bd08f3bc cpython-3.11.15+20260414-x86_64-unknown-linux-gnu-install_only.tar.gz +a57ffd435652092d16b30e783f9826c55e9c64b0f0a72cbae0a9f39e663137fb cpython-3.11.15+20260414-aarch64-apple-darwin-install_only.tar.gz +ce18fdfd47c66830a40ea9b9e314a14b1636bbfd684501bc5ca1fc6d55a7933f https://example.com/cpython-3.10.20+20260414-x86_64_v2-unknown-linux-musl-lto-full.tar.zst +""" + parsed = parse_sha_manifest(content) + env.expect.that_collection(parsed).has_size(3) + + env.expect.that_dict(structs.to_dict(parsed[0])).contains_exactly({ + "arch": "x86_64", + "build_version": "20260414", + "flavor": "install_only", + "libc": "gnu", + "location": "cpython-3.11.15+20260414-x86_64-unknown-linux-gnu-install_only.tar.gz", + "microarch": "", + "os": "linux", + "python_version": "3.11.15", + "sha256": "8b14030dd3af9ea7f7c51b4c90feb04afd8a8f45435727e67b875270bd08f3bc", + "vendor": "unknown", + }) + + env.expect.that_dict(structs.to_dict(parsed[2])).contains_exactly({ + "arch": "x86_64", + "build_version": "20260414", + "flavor": "lto-full", + "libc": "musl", + "location": "https://example.com/cpython-3.10.20+20260414-x86_64_v2-unknown-linux-musl-lto-full.tar.zst", + "microarch": "v2", + "os": "linux", + "python_version": "3.10.20", + "sha256": "ce18fdfd47c66830a40ea9b9e314a14b1636bbfd684501bc5ca1fc6d55a7933f", + "vendor": "unknown", + }) + +_tests.append(_test_parse_sha_manifest) + +def parse_sha_manifest_test_suite(name): + """Defines the test suite for manifest parsing. + + Args: + name: The name of the test suite. + """ + test_suite( + name = name, + tests = _tests, + ) diff --git a/tests/python_bzlmod_ext/runtime_manifests_tests.bzl b/tests/python_bzlmod_ext/runtime_manifests_tests.bzl new file mode 100644 index 0000000000..45c7406375 --- /dev/null +++ b/tests/python_bzlmod_ext/runtime_manifests_tests.bzl @@ -0,0 +1,96 @@ +"""Starlark unit tests for dynamic toolchain registration via manifests.""" + +load("@rules_testing//lib:analysis_test.bzl", "analysis_test") +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("@rules_testing//lib:util.bzl", rt_util = "util") +load("//python/private:python.bzl", "parse_modules") # buildifier: disable=bzl-visibility +load("//python/private:repo_utils.bzl", "repo_utils") # buildifier: disable=bzl-visibility +load("//tests/support/mocks:mocks.bzl", "mocks") # buildifier: disable=bzl-visibility +load("//tests/support/mocks:python_ext.bzl", "python_ext") # buildifier: disable=bzl-visibility + +_tests = [] + +_mock_logger = repo_utils.logger( + name = "mock", + verbosity_level = "ERROR", +) + +def _test_dynamic_manifest_toolchains(name): + rt_util.helper_target( + native.filegroup, + name = name + "_subject", + ) + analysis_test( + name = name, + target = name + "_subject", + impl = _test_dynamic_manifest_toolchains_impl, + ) + +def _test_dynamic_manifest_toolchains_impl(env, target): + _ = target # @unused + + # Construct Bzlmod mock module locally inside the test execution block. + # We test using virtual patch version "3.11.99" (not present in TOOL_VERSIONS) + # so that the populated config contains ONLY our dynamically parsed manifest keys + # without any pre-populated multi-platform templates, allowing exact dictionary match! + root_module = python_ext.module( + name = "runtime_manifests", + override = [ + python_ext.override( + add_runtime_manifest_urls = [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260414/SHA256SUMS", + ], + runtime_manifest_sha = "ce18fdfd47c66830a40ea9b9e314a14b1636bbfd684501bc5ca1fc6d55a7933f", + register_all_versions = True, + ), + ], + defaults = [ + python_ext.defaults( + python_version = "3.11.99", + ), + ], + ) + + # Pre-populate mock_files directly to bypass download output struct key mismatch in mock read lookups. + mock_mctx = mocks.mctx( + modules = [root_module], + mock_files = { + "runtime_manifest": """ +01e607cf764b97d4d5d6f69fd1ff3d8a9a162513dde5c39e98260fce40fe220a cpython-3.11.99+20260414-x86_64-unknown-linux-gnu-pgo+lto-full.tar.zst +8b14030dd3af9ea7f7c51b4c90feb04afd8a8f45435727e67b875270bd08f3bc cpython-3.11.99+20260414-x86_64-unknown-linux-gnu-install_only.tar.gz +""", + }, + ) + + res = parse_modules( + module_ctx = mock_mctx, + logger = _mock_logger, + ) + + tool_versions = res.config.default["tool_versions"] + env.expect.that_bool("3.11.99" in tool_versions).equals(True) + + version_info = tool_versions["3.11.99"] + + # Assert on the entire dictionary at once! + env.expect.that_dict(version_info).contains_exactly({ + "sha256": { + "x86_64-unknown-linux-gnu": "01e607cf764b97d4d5d6f69fd1ff3d8a9a162513dde5c39e98260fce40fe220a", + }, + "strip_prefix": { + "x86_64-unknown-linux-gnu": "python/install", + }, + "url": { + "x86_64-unknown-linux-gnu": [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260414/cpython-3.11.99+20260414-x86_64-unknown-linux-gnu-pgo+lto-full.tar.zst", + ], + }, + }) + +_tests.append(_test_dynamic_manifest_toolchains) + +def runtime_manifests_test_suite(name): + test_suite( + name = name, + tests = _tests, + ) diff --git a/tests/python_bzlmod_ext/test_helpers.bzl b/tests/python_bzlmod_ext/test_helpers.bzl new file mode 100644 index 0000000000..f6c177650a --- /dev/null +++ b/tests/python_bzlmod_ext/test_helpers.bzl @@ -0,0 +1,48 @@ +# Copyright 2026 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helpers to conditionally register tests depending on Bzlmod enablement.""" + +load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility +load(":parse_sha_manifest_tests.bzl", "parse_sha_manifest_test_suite") +load(":runtime_manifests_tests.bzl", "runtime_manifests_test_suite") + +def register_python_bzlmod_ext_tests(name, parse_sha_manifest_name, runtime_manifests_name): + """Registers the Bzlmod extension tests if Bzlmod is enabled, otherwise defines empty test_suites. + + Args: + name: The name of the master test_suite target. + parse_sha_manifest_name: The name of the parse_sha_manifest test target. + runtime_manifests_name: The name of the runtime_manifests test target. + """ + if BZLMOD_ENABLED: + parse_sha_manifest_test_suite(name = parse_sha_manifest_name) + runtime_manifests_test_suite(name = runtime_manifests_name) + else: + native.test_suite( + name = parse_sha_manifest_name, + tests = [], + ) + native.test_suite( + name = runtime_manifests_name, + tests = [], + ) + + native.test_suite( + name = name, + tests = [ + parse_sha_manifest_name, + runtime_manifests_name, + ], + ) diff --git a/tests/support/mocks/BUILD.bazel b/tests/support/mocks/BUILD.bazel index 45949fc578..07bdb2bffc 100644 --- a/tests/support/mocks/BUILD.bazel +++ b/tests/support/mocks/BUILD.bazel @@ -10,4 +10,10 @@ bzl_library( srcs = ["mocks.bzl"], ) +bzl_library( + name = "python_ext_bzl", + srcs = ["python_ext.bzl"], + deps = [":mocks_bzl"], +) + mocks_test_suite(name = "mocks_tests") diff --git a/tests/support/mocks/python_ext.bzl b/tests/support/mocks/python_ext.bzl new file mode 100644 index 0000000000..6f76ec785b --- /dev/null +++ b/tests/support/mocks/python_ext.bzl @@ -0,0 +1,102 @@ +"""Helper for defining a mock module for the python bzlmod extension.""" + +load("//tests/support/mocks:mocks.bzl", "mocks") # buildifier: disable=bzl-visibility + +def _module(name = "rules_python", is_root = True, **tags): + """Creates a mock Bzlmod module struct with defaulted tag lists. + + Args: + name: The module name. + is_root: Whether this is the root module. + **tags: Lists of tag objects. + + Returns: + A mock module struct. + """ + defaulted_tags = { + "defaults": [], + "override": [], + "single_version_override": [], + "single_version_platform_override": [], + "toolchain": [], + } + defaulted_tags.update(tags) + return mocks.module(name = name, is_root = is_root, **defaulted_tags) + +def _override(**kwargs): + """Creates a mock python.override tag with default values.""" + attrs = { + "add_runtime_manifest_urls": [], + "add_target_settings": [], + "available_python_versions": [], + "base_url": "https://github.com/astral-sh/python-build-standalone/releases/download", + "ignore_root_user_error": True, + "minor_mapping": {}, + "register_all_versions": False, + "runtime_manifest_sha": "", + } + attrs.update(kwargs) + return mocks.tag(**attrs) + +def _defaults(**kwargs): + """Creates a mock python.defaults tag with default values.""" + attrs = { + "python_version": "", + "python_version_env": "", + } + attrs.update(kwargs) + return mocks.tag(**attrs) + +def _single_version_override(**kwargs): + """Creates a mock python.single_version_override tag with default values.""" + attrs = { + "distutils": None, + "distutils_content": "", + "patch_strip": 0, + "patches": [], + "python_version": "", + "sha256": {}, + "strip_prefix": "python", + "urls": [], + } + attrs.update(kwargs) + return mocks.tag(**attrs) + +def _single_version_platform_override(**kwargs): + """Creates a mock python.single_version_platform_override tag with default values.""" + attrs = { + "arch": "", + "coverage_tool": None, + "os_name": "", + "patch_strip": 0, + "patches": [], + "platform": "", + "python_version": "", + "sha256": "", + "strip_prefix": "python", + "target_compatible_with": [], + "target_settings": [], + "urls": [], + } + attrs.update(kwargs) + return mocks.tag(**attrs) + +def _toolchain(**kwargs): + """Creates a mock python.toolchain tag with default values.""" + attrs = { + "configure_coverage_tool": False, + "ignore_root_user_error": True, + "is_default": False, + "python_version": "", + } + attrs.update(kwargs) + return mocks.tag(**attrs) + +python_ext = struct( + defaults = _defaults, + module = _module, + override = _override, + single_version_override = _single_version_override, + single_version_platform_override = _single_version_platform_override, + toolchain = _toolchain, +) diff --git a/tools/private/debug/print_defined_toolchains.sh b/tools/private/debug/print_defined_toolchains.sh new file mode 100755 index 0000000000..1694dc975c --- /dev/null +++ b/tools/private/debug/print_defined_toolchains.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Programmatically probe which repository target name is resolved successfully inside this workspace +if bazel query @pythons_hub//... >/dev/null 2>&1; then + HUB_REPO="@pythons_hub" +elif bazel query @@rules_python++python+pythons_hub//... >/dev/null 2>&1; then + HUB_REPO="@@rules_python++python+pythons_hub" +else + HUB_REPO="@@+python+pythons_hub" +fi + +# Query standard toolchains inside the resolved hub repository, excluding CC and Exec Tools toolchains. +bazel query "kind('toolchain', ${HUB_REPO}//...) - filter('_py_cc_toolchain$', ${HUB_REPO}//...) - filter('_py_exec_tools_toolchain$', ${HUB_REPO}//...)" "$@"