From d48286fdd0125cfae313bf782a58c54742444a77 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sat, 16 Aug 2025 18:35:18 -0700 Subject: [PATCH 1/6] docs: tell how to do and request patch releases/backports (#3185) We've had several backport requests of late. It's tenable for users to do the more time consuming working of backporting the code, so give steps on how to do so. --- .../ISSUE_TEMPLATE/patch_release_request.md | 22 +++++++++++++ docs/devguide.md | 32 +++++++++++++++++++ docs/support.md | 16 ++++++++++ 3 files changed, 70 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/patch_release_request.md diff --git a/.github/ISSUE_TEMPLATE/patch_release_request.md b/.github/ISSUE_TEMPLATE/patch_release_request.md new file mode 100644 index 0000000000..7b38ec40df --- /dev/null +++ b/.github/ISSUE_TEMPLATE/patch_release_request.md @@ -0,0 +1,22 @@ +--- +name: "🏗️ Patch release or backport" +about: Request a patch release or backport of a fix to a release. +title: 'Patch release: MAJOR.MINOR.PATCH' +labels: 'type: process' +--- + + + +**What version of `rules_python` do you want to patch?** + + +**What pull requests do you want to backport?** + +Please provide a list of pull request numbers. + +- # +- # diff --git a/docs/devguide.md b/docs/devguide.md index 43120bf2a1..afb990588b 100644 --- a/docs/devguide.md +++ b/docs/devguide.md @@ -116,3 +116,35 @@ to have everything self-documented, we have a special target, `//private:requirements.update`, which uses `rules_multirun` to run all of the requirement-updating scripts in sequence in one go. This can be done once per release as we prepare for releases. + +## Creating Backport PRs + +The steps to create a backport PR are: + +1. Create an issue for the patch release; use the [patch relase + template][patch-release-issue]. +2. Create a fork of `rules_python`. +3. Checkout the `release/X.Y` branch. +4. Use `git cherry-pick -x` to cherry pick the desired fixes. +5. Update the release's `CHANGELOG.md` file: + * Add a Major.Minor.Patch section if one doesn't exist + * Copy the changelog text from `main` to the release's changelog. +6. Send a PR with the backport's changes. + * The title should be `backport: PR#N to Major.Minor` + * The body must preserve the original PR's number, commit hash, description, + and authorship. + Use the following format (`git cherry-pick` will use this format): + ``` + + + + (cherry picked from commit ) + ----- + Co-authored-by: + ``` + * If the PR contains multiple backport commits, separate each's description + with `-----`. +7. Send a PR to update the `main` branch's `CHANGELOG.md` to reflect the + changes done in the patched release. + +[patch-release-issue]: https://github.com/bazelbuild/rules_python/issues/new?template=patch_release.md diff --git a/docs/support.md b/docs/support.md index ad943b3845..8728540804 100644 --- a/docs/support.md +++ b/docs/support.md @@ -14,6 +14,22 @@ the willingness of volunteers. If you want or need particular functionality backported, then the best way is to open a PR to demonstrate the feasibility of the backport. +### Backports and Patch Releases + +Backports and patch releases are provided on a best-effort basis. Only fixes are +backported. Features are not backported. + +Backports can be done to older releases, but only if newer releases also have +the fix backported. For example, if the current release is 1.5, in order to +patch 1.4, version 1.5 must be patched first. + +Backports can be requested by [creating an issue with the patch release +template][patch-release-issue] or by sending a pull request performing the backport. +See the dev guide for [how to create a backport PR][backport-pr]. + +[patch-release-issue]: https://github.com/bazelbuild/rules_python/issues/new?template=patch_release_request.md +[backport-pr]: devguide.html#creating-backport-prs + ## Supported Bazel Versions The supported Bazel versions are: From 8c33aa6d64898fcf4513fae0de5982903a3b8ee8 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 17 Aug 2025 10:36:23 +0900 Subject: [PATCH 2/6] fix(pypi): pull fewer wheels with experimental_index_url (#3058) Before this we would pull all of the wheels that the user target configuration would be compatible with and that meant that it was not customizable. This also meant that there were a lot of footguns in the configuration where the select statements were not really foolproof. With this PR we select only those sources that need to be for the declared configurations. Freethreaded support should be done by defining extra freethreaded platforms using the new builder API. It is done as a followup in #3063. This is also changing the default platforms to be only the fully supported platforms. This makes the testing easier and avoids us running into compatibility issues during the rollout. Work towards #2747 Fixes #2759 Fixes #2849 --- CHANGELOG.md | 8 + MODULE.bazel | 69 ++-- examples/bzlmod/MODULE.bazel | 17 + python/private/pypi/BUILD.bazel | 5 +- python/private/pypi/evaluate_markers.bzl | 6 +- python/private/pypi/extension.bzl | 242 ++++++++++---- python/private/pypi/parse_requirements.bzl | 94 +++--- python/private/pypi/whl_target_platforms.bzl | 132 -------- tests/pypi/extension/extension_tests.bzl | 157 ++++----- .../parse_requirements_tests.bzl | 159 +++++++-- tests/pypi/whl_target_platforms/BUILD.bazel | 3 - .../whl_target_platforms/select_whl_tests.bzl | 314 ------------------ 12 files changed, 523 insertions(+), 683 deletions(-) delete mode 100644 tests/pypi/whl_target_platforms/select_whl_tests.bzl diff --git a/CHANGELOG.md b/CHANGELOG.md index 2dc235fbf6..9d45aa53c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,10 @@ END_UNRELEASED_TEMPLATE * (toolchain) Python 3.13 now references 3.13.6 * (gazelle) Switched back to smacker/go-tree-sitter, fixing [#2630](https://github.com/bazel-contrib/rules_python/issues/2630) +* (pypi) From now on the list of default platforms only includes `linux_x86_64`, `linux_aarch64`, + `osx_x86_64`, `osx_aarch64` and `windows_x86_64`. If you are on other platforms, you need to + use the `pip.default` to configure it yourself. If you are interested in graduating the + platform, consider helping set us up CI for them and update the documentation. * (ci) We are now testing on Ubuntu 22.04 for RBE and non-RBE configurations. * (core) `#!/usr/bin/env bash` is now used as a shebang in the stage1 bootstrap template. * (gazelle:docs) The Gazelle docs have been migrated from {gh-path}`gazelle/README.md` to @@ -90,6 +94,10 @@ END_UNRELEASED_TEMPLATE ([#2503](https://github.com/bazel-contrib/rules_python/issues/2503)). * (pypi) The pipstar `defaults` configuration now supports any custom platform name. +* (pypi) The selection of the whls has been changed and should no longer result + in ambiguous select matches ({gh-issue}`2759`) and should be much more efficient + when running `bazel query` due to fewer repositories being included + ({gh-issue}`2849`). * Multi-line python imports (e.g. with escaped newlines) are now correctly processed by Gazelle. * (toolchains) `local_runtime_repo` works with multiarch Debian with Python 3.8 ([#3099](https://github.com/bazel-contrib/rules_python/issues/3099)). diff --git a/MODULE.bazel b/MODULE.bazel index b8d8c16a0c..66297b99a1 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -74,16 +74,18 @@ pip = use_extension("//python/extensions:pip.bzl", "pip") env = {"platform_version": "0"}, os_name = "linux", platform = "linux_{}".format(cpu), + whl_abi_tags = [ + "abi3", + "cp{major}{minor}", + ], + whl_platform_tags = [ + "linux_{}".format(cpu), + "manylinux_*_{}".format(cpu), + ], ) for cpu in [ "x86_64", "aarch64", - # TODO @aignas 2025-05-19: only leave tier 0-1 cpus when stabilizing the - # `pip.default` extension. i.e. drop the below values - users will have to - # define themselves if they need them. - "arm", - "ppc", - "s390x", ] ] @@ -99,26 +101,53 @@ pip = use_extension("//python/extensions:pip.bzl", "pip") env = {"platform_version": "14.0"}, os_name = "osx", platform = "osx_{}".format(cpu), + whl_abi_tags = [ + "abi3", + "cp{major}{minor}", + ], + whl_platform_tags = [ + "macosx_*_{}".format(suffix) + for suffix in platform_tag_cpus + ], ) - for cpu in [ - "aarch64", - "x86_64", - ] + for cpu, platform_tag_cpus in { + "aarch64": [ + "universal2", + "arm64", + ], + "x86_64": [ + "universal2", + "x86_64", + ], + }.items() +] + +[ + pip.default( + arch_name = cpu, + config_settings = [ + "@platforms//cpu:{}".format(cpu), + "@platforms//os:windows", + ], + env = {"platform_version": "0"}, + os_name = "windows", + platform = "windows_{}".format(cpu), + whl_abi_tags = [ + "abi3", + "cp{major}{minor}", + ], + whl_platform_tags = whl_platform_tags, + ) + for cpu, whl_platform_tags in { + "x86_64": ["win_amd64"], + }.items() ] -pip.default( - arch_name = "x86_64", - config_settings = [ - "@platforms//cpu:x86_64", - "@platforms//os:windows", - ], - env = {"platform_version": "0"}, - os_name = "windows", - platform = "windows_x86_64", -) pip.parse( # NOTE @aignas 2024-10-26: We have an integration test that depends on us # being able to build sdists for this hub, so explicitly set this to False. + # + # how do we test sdists? Maybe just worth adding a single sdist somewhere? download_only = False, experimental_index_url = "https://pypi.org/simple", hub_name = "rules_python_publish_deps", diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index 841c096dcf..95e1090f53 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -158,6 +158,23 @@ pip.whl_mods( ) use_repo(pip, "whl_mods_hub") +# Because below we are using `windows_aarch64` platform, we have to define various +# properties for it. +pip.default( + arch_name = "aarch64", + config_settings = [ + "@platforms//os:windows", + "@platforms//cpu:aarch64", + ], + env = { + "platform_version": "0", + }, + os_name = "windows", + platform = "windows_aarch64", + whl_abi_tags = [], # default to all ABIs + whl_platform_tags = ["win_amd64"], +) + # To fetch pip dependencies, use pip.parse. We can pass in various options, # but typically we pass requirements and the Python version. The Python # version must have been configured by a corresponding `python.toolchain()` diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 4b56b73284..847c85d634 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -116,11 +116,11 @@ bzl_library( ":parse_whl_name_bzl", ":pep508_env_bzl", ":pip_repository_attrs_bzl", + ":python_tag_bzl", ":simpleapi_download_bzl", ":whl_config_setting_bzl", ":whl_library_bzl", ":whl_repo_name_bzl", - ":whl_target_platforms_bzl", "//python/private:full_version_bzl", "//python/private:normalize_name_bzl", "//python/private:version_bzl", @@ -209,7 +209,7 @@ bzl_library( ":parse_requirements_txt_bzl", ":pypi_repo_utils_bzl", ":requirements_files_by_platform_bzl", - ":whl_target_platforms_bzl", + ":select_whl_bzl", "//python/private:normalize_name_bzl", "//python/private:repo_utils_bzl", ], @@ -438,5 +438,4 @@ bzl_library( bzl_library( name = "whl_target_platforms_bzl", srcs = ["whl_target_platforms.bzl"], - deps = [":parse_whl_name_bzl"], ) diff --git a/python/private/pypi/evaluate_markers.bzl b/python/private/pypi/evaluate_markers.bzl index 6167cdbc96..4d6a39a1df 100644 --- a/python/private/pypi/evaluate_markers.bzl +++ b/python/private/pypi/evaluate_markers.bzl @@ -43,11 +43,11 @@ def evaluate_markers(*, requirements, platforms): for req_string, platform_strings in requirements.items(): req = requirement(req_string) for platform_str in platform_strings: - env = platforms.get(platform_str) - if not env: + plat = platforms.get(platform_str) + if not plat: fail("Please define platform: '{}'".format(platform_str)) - if evaluate(req.marker, env = env): + if evaluate(req.marker, env = plat.env): ret.setdefault(req_string, []).append(platform_str) return ret diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 59e77d13e4..2c7aa8a0e5 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -31,6 +31,7 @@ load(":parse_requirements.bzl", "parse_requirements") load(":parse_whl_name.bzl", "parse_whl_name") load(":pep508_env.bzl", "env") load(":pip_repository_attrs.bzl", "ATTRS") +load(":python_tag.bzl", "python_tag") load(":requirements_files_by_platform.bzl", "requirements_files_by_platform") load(":simpleapi_download.bzl", "simpleapi_download") load(":whl_config_setting.bzl", "whl_config_setting") @@ -68,19 +69,40 @@ def _whl_mods_impl(whl_mods_dict): def _platforms(*, python_version, minor_mapping, config): platforms = {} - python_version = full_version( - version = python_version, - minor_mapping = minor_mapping, + python_version = version.parse( + full_version( + version = python_version, + minor_mapping = minor_mapping, + ), + strict = True, ) - abi = "cp3{}".format(python_version[2:]) for platform, values in config.platforms.items(): - key = "{}_{}".format(abi, platform) - platforms[key] = env( - env = values.env, - os = values.os_name, - arch = values.arch_name, - python_version = python_version, + # TODO @aignas 2025-07-07: this is probably doing the parsing of the version too + # many times. + key = "{}{}{}.{}_{}".format( + python_tag(values.env["implementation_name"]), + python_version.release[0], + python_version.release[1], + python_version.release[2], + platform, + ) + + platforms[key] = struct( + env = env( + env = values.env, + os = values.os_name, + arch = values.arch_name, + python_version = python_version.string, + ), + whl_abi_tags = [ + v.format( + major = python_version.release[0], + minor = python_version.release[1], + ) + for v in values.whl_abi_tags + ], + whl_platform_tags = values.whl_platform_tags, ) return platforms @@ -153,6 +175,8 @@ def _create_whl_repos( )) python_interpreter_target = available_interpreters[python_name] + # TODO @aignas 2025-06-29: we should not need the version in the pip_name if + # we are using pipstar and we are downloading the wheel using the downloader pip_name = "{}_{}".format( hub_name, version_label(pip_attr.python_version), @@ -231,12 +255,21 @@ def _create_whl_repos( ), logger = logger, ), + platforms = _platforms( + python_version = pip_attr.python_version, + minor_mapping = minor_mapping, + config = config, + ), extra_pip_args = pip_attr.extra_pip_args, get_index_urls = get_index_urls, evaluate_markers = evaluate_markers, logger = logger, ) + use_downloader = { + normalize_name(s): False + for s in pip_attr.simpleapi_skip + } exposed_packages = {} for whl in requirements_by_platform: if whl.is_exposed: @@ -290,11 +323,20 @@ def _create_whl_repos( whl_library_args = whl_library_args, download_only = pip_attr.download_only, netrc = pip_attr.netrc, + use_downloader = use_downloader.get( + whl.name, + get_index_urls != None, # defaults to True if the get_index_urls is defined + ), auth_patterns = pip_attr.auth_patterns, python_version = major_minor, is_multiple_versions = whl.is_multiple_versions, enable_pipstar = config.enable_pipstar, ) + if repo == None: + # NOTE @aignas 2025-07-07: we guard against an edge-case where there + # are more platforms defined than there are wheels for and users + # disallow building from sdist. + continue repo_name = "{}_{}".format(pip_name, repo.repo_name) if repo_name in whl_libraries: @@ -313,7 +355,17 @@ def _create_whl_repos( whl_libraries = whl_libraries, ) -def _whl_repo(*, src, whl_library_args, is_multiple_versions, download_only, netrc, auth_patterns, python_version, enable_pipstar = False): +def _whl_repo( + *, + src, + whl_library_args, + is_multiple_versions, + download_only, + netrc, + auth_patterns, + python_version, + use_downloader, + enable_pipstar = False): args = dict(whl_library_args) args["requirement"] = src.requirement_line is_whl = src.filename.endswith(".whl") @@ -326,19 +378,24 @@ def _whl_repo(*, src, whl_library_args, is_multiple_versions, download_only, net args["extra_pip_args"] = src.extra_pip_args if not src.url or (not is_whl and download_only): - # Fallback to a pip-installed wheel - target_platforms = src.target_platforms if is_multiple_versions else [] - return struct( - repo_name = pypi_repo_name( - normalize_name(src.distribution), - *target_platforms - ), - args = args, - config_setting = whl_config_setting( - version = python_version, - target_platforms = target_platforms or None, - ), - ) + if download_only and use_downloader: + # If the user did not allow using sdists and we are using the downloader + # and we are not using simpleapi_skip for this + return None + else: + # Fallback to a pip-installed wheel + target_platforms = src.target_platforms if is_multiple_versions else [] + return struct( + repo_name = pypi_repo_name( + normalize_name(src.distribution), + *target_platforms + ), + args = args, + config_setting = whl_config_setting( + version = python_version, + target_platforms = target_platforms or None, + ), + ) # This is no-op because pip is not used to download the wheel. args.pop("download_only", None) @@ -360,30 +417,37 @@ def _whl_repo(*, src, whl_library_args, is_multiple_versions, download_only, net for p in src.target_platforms ] - # Pure python wheels or sdists may need to have a platform here - target_platforms = None - if is_whl and not src.filename.endswith("-any.whl"): - pass - elif is_multiple_versions: - target_platforms = src.target_platforms - return struct( repo_name = whl_repo_name(src.filename, src.sha256), args = args, config_setting = whl_config_setting( version = python_version, - filename = src.filename, - target_platforms = target_platforms, + target_platforms = src.target_platforms, ), ) -def _plat(*, name, arch_name, os_name, config_settings = [], env = {}): +def _plat(*, name, arch_name, os_name, config_settings = [], env = {}, whl_abi_tags = [], whl_platform_tags = []): + # NOTE @aignas 2025-07-08: the least preferred is the first item in the list + if "any" not in whl_platform_tags: + # the lowest priority one needs to be the first one + whl_platform_tags = ["any"] + whl_platform_tags + + whl_abi_tags = whl_abi_tags or ["abi3", "cp{major}{minor}"] + if "none" not in whl_abi_tags: + # the lowest priority one needs to be the first one + whl_abi_tags = ["none"] + whl_abi_tags + return struct( name = name, arch_name = arch_name, os_name = os_name, config_settings = config_settings, - env = env, + env = { + # defaults for env + "implementation_name": "cpython", + } | env, + whl_abi_tags = whl_abi_tags, + whl_platform_tags = whl_platform_tags, ) def _configure(config, *, override = False, **kwargs): @@ -430,10 +494,12 @@ def build_config( env = tag.env, os_name = tag.os_name, name = platform.replace("-", "_").lower(), + whl_abi_tags = tag.whl_abi_tags, + whl_platform_tags = tag.whl_platform_tags, override = mod.is_root, ) - if platform and not (tag.arch_name or tag.config_settings or tag.env or tag.os_name): + if platform and not (tag.arch_name or tag.config_settings or tag.env or tag.os_name or tag.whl_abi_tags or tag.whl_platform_tags): defaults["platforms"].pop(platform) # TODO @aignas 2025-05-19: add more attr groups: @@ -441,8 +507,6 @@ def build_config( # attribute. # * for index/downloader config. This includes all of those attributes for # overrides, etc. Index overrides per platform could be also used here. - # * for whl selection - selecting preferences of which `platform_tag`s we should use - # for what. We could also model the `cp313t` freethreaded as separate platforms. return struct( platforms = { @@ -630,6 +694,7 @@ You cannot use both the additive_build_content and additive_build_content_file a extra_aliases.setdefault(hub_name, {}) for whl_name, aliases in out.extra_aliases.items(): extra_aliases[hub_name].setdefault(whl_name, {}).update(aliases) + if hub_name not in exposed_packages: exposed_packages[hub_name] = out.exposed_packages else: @@ -640,6 +705,14 @@ You cannot use both the additive_build_content and additive_build_content_file a intersection[pkg] = None exposed_packages[hub_name] = intersection whl_libraries.update(out.whl_libraries) + for whl_name, lib in out.whl_libraries.items(): + if enable_pipstar: + whl_libraries.setdefault(whl_name, lib) + elif whl_name in lib: + fail("'{}' already in created".format(whl_name)) + else: + # replicate whl_libraries.update(out.whl_libraries) + whl_libraries[whl_name] = lib # TODO @aignas 2024-04-05: how do we support different requirement # cycles for different abis/oses? For now we will need the users to @@ -790,6 +863,7 @@ _default_attrs = { "arch_name": attr.string( doc = """\ The CPU architecture name to be used. +You can use any cpu name from the `@platforms//cpu:` package. :::{note} Either this or {attr}`env` `platform_machine` key should be specified. @@ -803,25 +877,6 @@ The list of labels to `config_setting` targets that need to be matched for the p selected. """, ), - "os_name": attr.string( - doc = """\ -The OS name to be used. - -:::{note} -Either this or the appropriate `env` keys should be specified. -::: -""", - ), - "platform": attr.string( - doc = """\ -A platform identifier which will be used as the unique identifier within the extension evaluation. -If you are defining custom platforms in your project and don't want things to clash, use extension -[isolation] feature. - -[isolation]: https://bazel.build/rules/lib/globals/module#use_extension.isolate -""", - ), -} | { "env": attr.string_dict( doc = """\ The values to use for environment markers when evaluating an expression. @@ -847,6 +902,79 @@ This is only used if the {envvar}`RULES_PYTHON_ENABLE_PIPSTAR` is enabled. """, ), # The values for PEP508 env marker evaluation during the lock file parsing + "os_name": attr.string( + doc = """\ +The OS name to be used. +You can use any OS name from the `@platforms//os:` package. + +:::{note} +Either this or the appropriate `env` keys should be specified. +::: +""", + ), + "platform": attr.string( + doc = """\ +A platform identifier which will be used as the unique identifier within the extension evaluation. +If you are defining custom platforms in your project and don't want things to clash, use extension +[isolation] feature. + +[isolation]: https://bazel.build/rules/lib/globals/module#use_extension.isolate +""", + ), + "whl_abi_tags": attr.string_list( + doc = """\ +A list of ABIs to select wheels for. The values can be either strings or include template +parameters like `{major}` and `{minor}` which will be replaced with python version parts. e.g. +`cp{major}{minor}` will result in `cp313` given the full python version is `3.13.5`. +Will always include `"none"` even if it is not specified. + +:::{note} +We select a single wheel and the last match will take precedence. +::: + +:::{seealso} +See official [docs](https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#abi-tag) for more information. +::: +""", + ), + "whl_platform_tags": attr.string_list( + doc = """\ +A list of `platform_tag` matchers so that we can select the best wheel based on the user +preference. +Will always include `"any"` even if it is not specified. + +The items in this list can contain a single `*` character that is equivalent to matching the +latest available version component in the platform_tag. Note, if the wheel platform tag does not +have a version component, e.g. `linux_x86_64` or `win_amd64`, then `*` will act as a regular +character. + +We will always select the highest available `platform_tag` version that is compatible with the +target platform. + +:::{note} +We select a single wheel and the last match will take precedence, if the platform_tag that we +match has a version component (e.g. `android_x_arch`, then the version `x` will be used in the +matching algorithm). + +If the matcher you provide has `*`, then we will match a wheel with the highest available target platform, i.e. if `musllinux_1_1_arch` and `musllinux_1_2_arch` are both present, then we will select `musllinux_1_2_arch`. +Otherwise we will select the highest available version that is equal or lower to the specifier, i.e. if `manylinux_2_12` and `manylinux_2_17` wheels are present and the matcher is `manylinux_2_15`, then we will match `manylinux_2_12` but not `manylinux_2_17`. +::: + +:::{note} +The following tag prefixes should be used instead of the legacy equivalents: +* `manylinux_2_5` instead of `manylinux1` +* `manylinux_2_12` instead of `manylinux2010` +* `manylinux_2_17` instead of `manylinux2014` + +When parsing the whl filenames `rules_python` will automatically transform wheel filenames to the +latest format. +::: + +:::{seealso} +See official [docs](https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#platform-tag) for more information. +::: +""", + ), } _SUPPORTED_PEP508_KEYS = [ diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl index 9c610f11d3..ebd447d95d 100644 --- a/python/private/pypi/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl @@ -31,13 +31,14 @@ load("//python/private:repo_utils.bzl", "repo_utils") load(":index_sources.bzl", "index_sources") load(":parse_requirements_txt.bzl", "parse_requirements_txt") load(":pep508_requirement.bzl", "requirement") -load(":whl_target_platforms.bzl", "select_whls") +load(":select_whl.bzl", "select_whl") def parse_requirements( ctx, *, requirements_by_platform = {}, extra_pip_args = [], + platforms = {}, get_index_urls = None, evaluate_markers = None, extract_url_srcs = True, @@ -46,6 +47,7 @@ def parse_requirements( Args: ctx: A context that has .read function that would read contents from a label. + platforms: The target platform descriptions. requirements_by_platform (label_keyed_string_dict): a way to have different package versions (or different packages) for different os, arch combinations. @@ -88,7 +90,7 @@ def parse_requirements( requirements = {} for file, plats in requirements_by_platform.items(): if logger: - logger.debug(lambda: "Using {} for {}".format(file, plats)) + logger.trace(lambda: "Using {} for {}".format(file, plats)) contents = ctx.read(file) # Parse the requirements file directly in starlark to get the information @@ -161,7 +163,7 @@ def parse_requirements( # VCS package references. env_marker_target_platforms = evaluate_markers(ctx, reqs_with_env_markers) if logger: - logger.debug(lambda: "Evaluated env markers from:\n{}\n\nTo:\n{}".format( + logger.trace(lambda: "Evaluated env markers from:\n{}\n\nTo:\n{}".format( reqs_with_env_markers, env_marker_target_platforms, )) @@ -196,6 +198,7 @@ def parse_requirements( name = name, reqs = reqs, index_urls = index_urls, + platforms = platforms, env_marker_target_platforms = env_marker_target_platforms, extract_url_srcs = extract_url_srcs, logger = logger, @@ -203,7 +206,7 @@ def parse_requirements( ) ret.append(item) if not item.is_exposed and logger: - logger.debug(lambda: "Package '{}' will not be exposed because it is only present on a subset of platforms: {} out of {}".format( + logger.trace(lambda: "Package '{}' will not be exposed because it is only present on a subset of platforms: {} out of {}".format( name, sorted(requirement_target_platforms), sorted(requirements), @@ -219,38 +222,43 @@ def _package_srcs( name, reqs, index_urls, + platforms, logger, env_marker_target_platforms, extract_url_srcs): """A function to return sources for a particular package.""" srcs = {} for r in sorted(reqs.values(), key = lambda r: r.requirement_line): - whls, sdist = _add_dists( - requirement = r, - index_urls = index_urls.get(name), - logger = logger, - ) - target_platforms = env_marker_target_platforms.get(r.requirement_line, r.target_platforms) - target_platforms = sorted(target_platforms) + extra_pip_args = tuple(r.extra_pip_args) - all_dists = [] + whls - if sdist: - all_dists.append(sdist) + for target_platform in target_platforms: + if platforms and target_platform not in platforms: + fail("The target platform '{}' could not be found in {}".format( + target_platform, + platforms.keys(), + )) - if extract_url_srcs and all_dists: - req_line = r.srcs.requirement - else: - all_dists = [struct( - url = "", - filename = "", - sha256 = "", - yanked = False, - )] - req_line = r.srcs.requirement_line + dist = _add_dists( + requirement = r, + target_platform = platforms.get(target_platform), + index_urls = index_urls.get(name), + logger = logger, + ) + if logger: + logger.debug(lambda: "The whl dist is: {}".format(dist.filename if dist else dist)) + + if extract_url_srcs and dist: + req_line = r.srcs.requirement + else: + dist = struct( + url = "", + filename = "", + sha256 = "", + yanked = False, + ) + req_line = r.srcs.requirement_line - extra_pip_args = tuple(r.extra_pip_args) - for dist in all_dists: key = ( dist.filename, req_line, @@ -269,9 +277,9 @@ def _package_srcs( yanked = dist.yanked, ), ) - for p in target_platforms: - if p not in entry.target_platforms: - entry.target_platforms.append(p) + + if target_platform not in entry.target_platforms: + entry.target_platforms.append(target_platform) return srcs.values() @@ -325,7 +333,7 @@ def host_platform(ctx): repo_utils.get_platforms_cpu_name(ctx), ) -def _add_dists(*, requirement, index_urls, logger = None): +def _add_dists(*, requirement, index_urls, target_platform, logger = None): """Populate dists based on the information from the PyPI index. This function will modify the given requirements_by_platform data structure. @@ -333,6 +341,7 @@ def _add_dists(*, requirement, index_urls, logger = None): Args: requirement: The result of parse_requirements function. index_urls: The result of simpleapi_download. + target_platform: The target_platform information. logger: A logger for printing diagnostic info. """ @@ -342,7 +351,7 @@ def _add_dists(*, requirement, index_urls, logger = None): logger.debug(lambda: "Could not detect the filename from the URL, falling back to pip: {}".format( requirement.srcs.url, )) - return [], None + return None # Handle direct URLs in requirements dist = struct( @@ -353,12 +362,12 @@ def _add_dists(*, requirement, index_urls, logger = None): ) if dist.filename.endswith(".whl"): - return [dist], None + return dist else: - return [], dist + return dist if not index_urls: - return [], None + return None whls = [] sdist = None @@ -401,11 +410,16 @@ def _add_dists(*, requirement, index_urls, logger = None): for reason, dists in yanked.items() ])) - # Filter out the wheels that are incompatible with the target_platforms. - whls = select_whls( + if not target_platform: + # The pipstar platforms are undefined here, so we cannot do any matching + return sdist + + # Select a single wheel that can work on the target_platform + return select_whl( whls = whls, - want_platforms = requirement.target_platforms, + python_version = target_platform.env["python_full_version"], + implementation_name = target_platform.env["implementation_name"], + whl_abi_tags = target_platform.whl_abi_tags, + whl_platform_tags = target_platform.whl_platform_tags, logger = logger, - ) - - return whls, sdist + ) or sdist diff --git a/python/private/pypi/whl_target_platforms.bzl b/python/private/pypi/whl_target_platforms.bzl index 6ea3f120c3..6c3dd5da83 100644 --- a/python/private/pypi/whl_target_platforms.bzl +++ b/python/private/pypi/whl_target_platforms.bzl @@ -16,8 +16,6 @@ A starlark implementation of the wheel platform tag parsing to get the target platform. """ -load(":parse_whl_name.bzl", "parse_whl_name") - # The order of the dictionaries is to keep definitions with their aliases next to each # other _CPU_ALIASES = { @@ -46,136 +44,6 @@ _OS_PREFIXES = { "win": "windows", } # buildifier: disable=unsorted-dict-items -def select_whls(*, whls, want_platforms = [], logger = None): - """Select a subset of wheels suitable for target platforms from a list. - - Args: - whls(list[struct]): A list of candidates which have a `filename` - attribute containing the `whl` filename. - want_platforms(str): The platforms in "{abi}_{os}_{cpu}" or "{os}_{cpu}" format. - logger: A logger for printing diagnostic messages. - - Returns: - A filtered list of items from the `whls` arg where `filename` matches - the selected criteria. If no match is found, an empty list is returned. - """ - if not whls: - return [] - - want_abis = { - "abi3": None, - "none": None, - } - - _want_platforms = {} - version_limit = None - - for p in want_platforms: - if not p.startswith("cp3"): - fail("expected all platforms to start with ABI, but got: {}".format(p)) - - abi, _, os_cpu = p.partition("_") - abi, _, _ = abi.partition(".") - _want_platforms[os_cpu] = None - - # TODO @aignas 2025-04-20: add a test - _want_platforms["{}_{}".format(abi, os_cpu)] = None - - version_limit_candidate = int(abi[3:]) - if not version_limit: - version_limit = version_limit_candidate - if version_limit and version_limit != version_limit_candidate: - fail("Only a single python version is supported for now") - - # For some legacy implementations the wheels may target the `cp3xm` ABI - _want_platforms["{}m_{}".format(abi, os_cpu)] = None - want_abis[abi] = None - want_abis[abi + "m"] = None - - # Also add freethreaded wheels if we find them since we started supporting them - _want_platforms["{}t_{}".format(abi, os_cpu)] = None - want_abis[abi + "t"] = None - - want_platforms = sorted(_want_platforms) - - candidates = {} - for whl in whls: - parsed = parse_whl_name(whl.filename) - - if logger: - logger.trace(lambda: "Deciding whether to use '{}'".format(whl.filename)) - - supported_implementations = {} - whl_version_min = 0 - for tag in parsed.python_tag.split("."): - supported_implementations[tag[:2]] = None - - if tag.startswith("cp3") or tag.startswith("py3"): - version = int(tag[len("..3"):] or 0) - else: - # In this case it should be eithor "cp2" or "py2" and we will default - # to `whl_version_min` = 0 - continue - - if whl_version_min == 0 or version < whl_version_min: - whl_version_min = version - - if not ("cp" in supported_implementations or "py" in supported_implementations): - if logger: - logger.trace(lambda: "Discarding the whl because the whl does not support CPython, whl supported implementations are: {}".format(supported_implementations)) - continue - - if want_abis and parsed.abi_tag not in want_abis: - # Filter out incompatible ABIs - if logger: - logger.trace(lambda: "Discarding the whl because the whl abi did not match") - continue - - if whl_version_min > version_limit: - if logger: - logger.trace(lambda: "Discarding the whl because the whl supported python version is too high") - continue - - compatible = False - if parsed.platform_tag == "any": - compatible = True - else: - for p in whl_target_platforms(parsed.platform_tag, abi_tag = parsed.abi_tag.strip("m") if parsed.abi_tag.startswith("cp") else None): - if p.target_platform in want_platforms: - compatible = True - break - - if not compatible: - if logger: - logger.trace(lambda: "Discarding the whl because the whl does not support the desired platforms: {}".format(want_platforms)) - continue - - for implementation in supported_implementations: - candidates.setdefault( - ( - parsed.abi_tag, - parsed.platform_tag, - ), - {}, - ).setdefault( - ( - # prefer cp implementation - implementation == "cp", - # prefer higher versions - whl_version_min, - # prefer abi3 over none - parsed.abi_tag != "none", - # prefer cpx abi over abi3 - parsed.abi_tag != "abi3", - ), - [], - ).append(whl) - - return [ - candidates[key][sorted(v)[-1]][-1] - for key, v in candidates.items() - ] - def whl_target_platforms(platform_tag, abi_tag = ""): """Parse the wheel abi and platform tags and return (os, cpu) tuples. diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index d115546b63..72cbb61d81 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -65,13 +65,14 @@ def _mod(*, name, default = [], parse = [], override = [], whl_mods = [], is_roo "@platforms//os:{}".format(os), "@platforms//cpu:{}".format(cpu), ], + whl_platform_tags = whl_platform_tags, ) - for os, cpu in [ - ("linux", "x86_64"), - ("linux", "aarch64"), - ("osx", "aarch64"), - ("windows", "aarch64"), - ] + for (os, cpu), whl_platform_tags in { + ("linux", "x86_64"): ["linux_*_x86_64", "manylinux_*_x86_64"], + ("linux", "aarch64"): ["linux_*_aarch64", "manylinux_*_aarch64"], + ("osx", "aarch64"): ["macosx_*_arm64"], + ("windows", "aarch64"): ["win_arm64"], + }.items() ], ), is_root = is_root, @@ -105,21 +106,22 @@ def _build_config(env, enable_pipstar = 0, **kwargs): ) def _default( + *, arch_name = None, config_settings = None, os_name = None, platform = None, + whl_platform_tags = None, env = None, - whl_limit = None, - whl_platforms = None): + whl_abi_tags = None): return struct( arch_name = arch_name, os_name = os_name, platform = platform, + whl_platform_tags = whl_platform_tags or [], config_settings = config_settings, env = env or {}, - whl_platforms = whl_platforms, - whl_limit = whl_limit, + whl_abi_tags = whl_abi_tags or [], ) def _parse( @@ -515,18 +517,21 @@ def _test_torch_experimental_index_url(env): "@platforms//os:{}".format(os), "@platforms//cpu:{}".format(cpu), ], + whl_platform_tags = whl_platform_tags, ) - for os, cpu in [ - ("linux", "aarch64"), - ("linux", "x86_64"), - ("osx", "aarch64"), - ("windows", "x86_64"), - ] + for (os, cpu), whl_platform_tags in { + ("linux", "x86_64"): ["linux_x86_64", "manylinux_*_x86_64"], + ("linux", "aarch64"): ["linux_aarch64", "manylinux_*_aarch64"], + ("osx", "aarch64"): ["macosx_*_arm64"], + ("windows", "x86_64"): ["win_amd64"], + ("windows", "aarch64"): ["win_arm64"], # this should be ignored + }.items() ], parse = [ _parse( hub_name = "pypi", python_version = "3.12", + download_only = True, experimental_index_url = "https://torch.index", requirements_lock = "universal.txt", ), @@ -583,25 +588,25 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "torch": { "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": [ whl_config_setting( - filename = "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl", + target_platforms = ["cp312_linux_x86_64"], version = "3.12", ), ], "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": [ whl_config_setting( - filename = "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + target_platforms = ["cp312_linux_aarch64"], version = "3.12", ), ], "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": [ whl_config_setting( - filename = "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl", + target_platforms = ["cp312_windows_x86_64"], version = "3.12", ), ], "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": [ whl_config_setting( - filename = "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl", + target_platforms = ["cp312_osx_aarch64"], version = "3.12", ), ], @@ -610,10 +615,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ pypi.whl_libraries().contains_exactly({ "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_x86_64", - "windows_x86_64", - ], + "experimental_target_platforms": ["linux_x86_64"], "filename": "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1+cpu", @@ -622,10 +624,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ }, "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_aarch64", - "osx_aarch64", - ], + "experimental_target_platforms": ["linux_aarch64"], "filename": "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1", @@ -634,10 +633,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ }, "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_x86_64", - "windows_x86_64", - ], + "experimental_target_platforms": ["windows_x86_64"], "filename": "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1+cpu", @@ -646,10 +642,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ }, "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_aarch64", - "osx_aarch64", - ], + "experimental_target_platforms": ["osx_aarch64"], "filename": "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1", @@ -856,78 +849,79 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "pypi": { "direct_sdist_without_sha": { "pypi_315_any_name": [ - struct( - config_setting = None, - filename = "any-name.tar.gz", - target_platforms = None, + whl_config_setting( + target_platforms = ( + "cp315_linux_aarch64", + "cp315_linux_x86_64", + "cp315_osx_aarch64", + "cp315_windows_aarch64", + ), version = "3.15", ), ], }, "direct_without_sha": { "pypi_315_direct_without_sha_0_0_1_py3_none_any": [ - struct( - config_setting = None, - filename = "direct_without_sha-0.0.1-py3-none-any.whl", - target_platforms = None, + whl_config_setting( + target_platforms = ( + "cp315_linux_aarch64", + "cp315_linux_x86_64", + "cp315_osx_aarch64", + "cp315_windows_aarch64", + ), version = "3.15", ), ], }, "git_dep": { "pypi_315_git_dep": [ - struct( - config_setting = None, - filename = None, - target_platforms = None, + whl_config_setting( version = "3.15", ), ], }, "pip_fallback": { "pypi_315_pip_fallback": [ - struct( - config_setting = None, - filename = None, - target_platforms = None, + whl_config_setting( version = "3.15", ), ], }, "simple": { "pypi_315_simple_py3_none_any_deadb00f": [ - struct( - config_setting = None, - filename = "simple-0.0.1-py3-none-any.whl", - target_platforms = None, - version = "3.15", - ), - ], - "pypi_315_simple_sdist_deadbeef": [ - struct( - config_setting = None, - filename = "simple-0.0.1.tar.gz", - target_platforms = None, + whl_config_setting( + target_platforms = ( + "cp315_linux_aarch64", + "cp315_linux_x86_64", + "cp315_osx_aarch64", + "cp315_windows_aarch64", + ), version = "3.15", ), ], }, "some_other_pkg": { "pypi_315_some_py3_none_any_deadb33f": [ - struct( - config_setting = None, - filename = "some-other-pkg-0.0.1-py3-none-any.whl", - target_platforms = None, + whl_config_setting( + target_platforms = ( + "cp315_linux_aarch64", + "cp315_linux_x86_64", + "cp315_osx_aarch64", + "cp315_windows_aarch64", + ), version = "3.15", ), ], }, "some_pkg": { "pypi_315_some_pkg_py3_none_any_deadbaaf": [ - struct( - config_setting = None, - filename = "some_pkg-0.0.1-py3-none-any.whl", - target_platforms = None, + whl_config_setting( + target_platforms = ( + "cp315_linux_aarch64", + "cp315_linux_x86_64", + "cp315_osx_aarch64", + "cp315_windows_aarch64", + ), version = "3.15", ), ], @@ -990,21 +984,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "sha256": "deadb00f", "urls": ["example2.org"], }, - "pypi_315_simple_sdist_deadbeef": { - "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_aarch64", - "linux_x86_64", - "osx_aarch64", - "windows_aarch64", - ], - "extra_pip_args": ["--extra-args-for-sdist-building"], - "filename": "simple-0.0.1.tar.gz", - "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "simple==0.0.1", - "sha256": "deadbeef", - "urls": ["example.org"], - }, "pypi_315_some_pkg_py3_none_any_deadbaaf": { "dep_template": "@pypi//{name}:{target}", "experimental_target_platforms": [ @@ -1260,7 +1239,9 @@ def _test_build_pipstar_platform(env): "@platforms//os:linux", "@platforms//cpu:x86_64", ], - env = {}, + env = {"implementation_name": "cpython"}, + whl_abi_tags = ["none", "abi3", "cp{major}{minor}"], + whl_platform_tags = ["any"], ), }) diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl index b14467bc84..249af90114 100644 --- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl +++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl @@ -15,7 +15,10 @@ "" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private/pypi:parse_requirements.bzl", "parse_requirements", "select_requirement") # buildifier: disable=bzl-visibility +load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "REPO_VERBOSITY_ENV_VAR", "repo_utils") # buildifier: disable=bzl-visibility +load("//python/private/pypi:evaluate_markers.bzl", "evaluate_markers") # buildifier: disable=bzl-visibility +load("//python/private/pypi:parse_requirements.bzl", "select_requirement", _parse_requirements = "parse_requirements") # buildifier: disable=bzl-visibility +load("//python/private/pypi:pep508_env.bzl", pep508_env = "env") # buildifier: disable=bzl-visibility def _mock_ctx(): testdata = { @@ -64,6 +67,12 @@ foo[extra]==0.0.1 --hash=sha256:deadbeef "requirements_marker": """\ foo[extra]==0.0.1 ;marker --hash=sha256:deadbeef bar==0.0.1 --hash=sha256:deadbeef +""", + "requirements_multi_version": """\ +foo==0.0.1; python_full_version < '3.10.0' \ + --hash=sha256:deadbeef +foo==0.0.2; python_full_version >= '3.10.0' \ + --hash=sha256:deadb11f """, "requirements_optional_hash": """ foo==0.0.4 @ https://example.org/foo-0.0.4.whl @@ -96,9 +105,22 @@ bar==0.0.1 --hash=sha256:deadb00f _tests = [] +def parse_requirements(debug = False, **kwargs): + return _parse_requirements( + ctx = _mock_ctx(), + logger = repo_utils.logger(struct( + os = struct( + environ = { + REPO_DEBUG_ENV_VAR: "1", + REPO_VERBOSITY_ENV_VAR: "TRACE" if debug else "INFO", + }, + ), + ), "unit-test"), + **kwargs + ) + def _test_simple(env): got = parse_requirements( - ctx = _mock_ctx(), requirements_by_platform = { "requirements_lock": ["linux_x86_64", "windows_x86_64"], }, @@ -131,7 +153,6 @@ _tests.append(_test_simple) def _test_direct_urls_integration(env): """Check that we are using the filename from index_sources.""" got = parse_requirements( - ctx = _mock_ctx(), requirements_by_platform = { "requirements_direct": ["linux_x86_64"], "requirements_direct_sdist": ["osx_x86_64"], @@ -171,7 +192,6 @@ _tests.append(_test_direct_urls_integration) def _test_extra_pip_args(env): got = parse_requirements( - ctx = _mock_ctx(), requirements_by_platform = { "requirements_extra_args": ["linux_x86_64"], }, @@ -203,7 +223,6 @@ _tests.append(_test_extra_pip_args) def _test_dupe_requirements(env): got = parse_requirements( - ctx = _mock_ctx(), requirements_by_platform = { "requirements_lock_dupe": ["linux_x86_64"], }, @@ -232,7 +251,6 @@ _tests.append(_test_dupe_requirements) def _test_multi_os(env): got = parse_requirements( - ctx = _mock_ctx(), requirements_by_platform = { "requirements_linux": ["linux_x86_64"], "requirements_windows": ["windows_x86_64"], @@ -296,7 +314,6 @@ _tests.append(_test_multi_os) def _test_multi_os_legacy(env): got = parse_requirements( - ctx = _mock_ctx(), requirements_by_platform = { "requirements_linux_download_only": ["cp39_linux_x86_64"], "requirements_osx_download_only": ["cp39_osx_aarch64"], @@ -377,7 +394,6 @@ def _test_env_marker_resolution(env): return ret got = parse_requirements( - ctx = _mock_ctx(), requirements_by_platform = { "requirements_marker": ["cp311_linux_super_exotic", "cp311_windows_x86_64"], }, @@ -424,7 +440,6 @@ _tests.append(_test_env_marker_resolution) def _test_different_package_version(env): got = parse_requirements( - ctx = _mock_ctx(), requirements_by_platform = { "requirements_different_package_version": ["linux_x86_64"], }, @@ -463,7 +478,6 @@ _tests.append(_test_different_package_version) def _test_optional_hash(env): got = parse_requirements( - ctx = _mock_ctx(), requirements_by_platform = { "requirements_optional_hash": ["linux_x86_64"], }, @@ -502,7 +516,6 @@ _tests.append(_test_optional_hash) def _test_git_sources(env): got = parse_requirements( - ctx = _mock_ctx(), requirements_by_platform = { "requirements_git": ["linux_x86_64"], }, @@ -531,11 +544,30 @@ _tests.append(_test_git_sources) def _test_overlapping_shas_with_index_results(env): got = parse_requirements( - ctx = _mock_ctx(), requirements_by_platform = { "requirements_linux": ["cp39_linux_x86_64"], "requirements_osx": ["cp39_osx_x86_64"], }, + platforms = { + "cp39_linux_x86_64": struct( + env = pep508_env( + python_version = "3.9.0", + os = "linux", + arch = "x86_64", + ), + whl_abi_tags = ["none"], + whl_platform_tags = ["any"], + ), + "cp39_osx_x86_64": struct( + env = pep508_env( + python_version = "3.9.0", + os = "osx", + arch = "x86_64", + ), + whl_abi_tags = ["none"], + whl_platform_tags = ["macosx_*_x86_64"], + ), + }, get_index_urls = lambda _, __: { "foo": struct( sdists = { @@ -566,10 +598,9 @@ def _test_overlapping_shas_with_index_results(env): env.expect.that_collection(got).contains_exactly([ struct( - name = "foo", is_exposed = True, - # TODO @aignas 2025-05-25: how do we rename this? is_multiple_versions = True, + name = "foo", srcs = [ struct( distribution = "foo", @@ -577,27 +608,109 @@ def _test_overlapping_shas_with_index_results(env): filename = "foo-0.0.1-py3-none-any.whl", requirement_line = "foo==0.0.3", sha256 = "deadbaaf", - target_platforms = ["cp39_linux_x86_64", "cp39_osx_x86_64"], + target_platforms = ["cp39_linux_x86_64"], url = "super2", yanked = False, ), struct( distribution = "foo", extra_pip_args = [], - filename = "foo-0.0.1.tar.gz", + filename = "foo-0.0.1-py3-none-macosx_14_0_x86_64.whl", requirement_line = "foo==0.0.3", - sha256 = "5d15t", - target_platforms = ["cp39_linux_x86_64", "cp39_osx_x86_64"], - url = "sdist", + sha256 = "deadb11f", + target_platforms = ["cp39_osx_x86_64"], + url = "super2", yanked = False, ), + ], + ), + ]) + +_tests.append(_test_overlapping_shas_with_index_results) + +def _test_get_index_urls_different_versions(env): + got = parse_requirements( + requirements_by_platform = { + "requirements_multi_version": [ + "cp39_linux_x86_64", + "cp310_linux_x86_64", + ], + }, + platforms = { + "cp310_linux_x86_64": struct( + env = pep508_env( + python_version = "3.9.0", + os = "linux", + arch = "x86_64", + ), + whl_abi_tags = ["none"], + whl_platform_tags = ["any"], + ), + "cp39_linux_x86_64": struct( + env = pep508_env( + python_version = "3.9.0", + os = "linux", + arch = "x86_64", + ), + whl_abi_tags = ["none"], + whl_platform_tags = ["any"], + ), + }, + get_index_urls = lambda _, __: { + "foo": struct( + sdists = {}, + whls = { + "deadb11f": struct( + url = "super2", + sha256 = "deadb11f", + filename = "foo-0.0.2-py3-none-any.whl", + yanked = False, + ), + "deadbaaf": struct( + url = "super2", + sha256 = "deadbaaf", + filename = "foo-0.0.1-py3-none-any.whl", + yanked = False, + ), + }, + ), + }, + evaluate_markers = lambda _, requirements: evaluate_markers( + requirements = requirements, + platforms = { + "cp310_linux_x86_64": struct( + env = {"python_full_version": "3.10.0"}, + ), + "cp39_linux_x86_64": struct( + env = {"python_full_version": "3.9.0"}, + ), + }, + ), + ) + + env.expect.that_collection(got).contains_exactly([ + struct( + is_exposed = True, + is_multiple_versions = True, + name = "foo", + srcs = [ struct( distribution = "foo", extra_pip_args = [], - filename = "foo-0.0.1-py3-none-macosx_14_0_x86_64.whl", - requirement_line = "foo==0.0.3", + filename = "", + requirement_line = "foo==0.0.1 --hash=sha256:deadbeef", + sha256 = "", + target_platforms = ["cp39_linux_x86_64"], + url = "", + yanked = False, + ), + struct( + distribution = "foo", + extra_pip_args = [], + filename = "foo-0.0.2-py3-none-any.whl", + requirement_line = "foo==0.0.2", sha256 = "deadb11f", - target_platforms = ["cp39_osx_x86_64"], + target_platforms = ["cp310_linux_x86_64"], url = "super2", yanked = False, ), @@ -605,7 +718,7 @@ def _test_overlapping_shas_with_index_results(env): ), ]) -_tests.append(_test_overlapping_shas_with_index_results) +_tests.append(_test_get_index_urls_different_versions) def parse_requirements_test_suite(name): """Create the test suite. diff --git a/tests/pypi/whl_target_platforms/BUILD.bazel b/tests/pypi/whl_target_platforms/BUILD.bazel index 6c35b08d32..fec25af033 100644 --- a/tests/pypi/whl_target_platforms/BUILD.bazel +++ b/tests/pypi/whl_target_platforms/BUILD.bazel @@ -12,9 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -load(":select_whl_tests.bzl", "select_whl_test_suite") load(":whl_target_platforms_tests.bzl", "whl_target_platforms_test_suite") -select_whl_test_suite(name = "select_whl_tests") - whl_target_platforms_test_suite(name = "whl_target_platforms_tests") diff --git a/tests/pypi/whl_target_platforms/select_whl_tests.bzl b/tests/pypi/whl_target_platforms/select_whl_tests.bzl deleted file mode 100644 index 1674ac5ef2..0000000000 --- a/tests/pypi/whl_target_platforms/select_whl_tests.bzl +++ /dev/null @@ -1,314 +0,0 @@ -# Copyright 2024 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"" - -load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "REPO_VERBOSITY_ENV_VAR", "repo_utils") # buildifier: disable=bzl-visibility -load("//python/private/pypi:whl_target_platforms.bzl", "select_whls") # buildifier: disable=bzl-visibility - -WHL_LIST = [ - "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl", - "pkg-0.0.1-cp311-cp311-macosx_10_9_x86_64.whl", - "pkg-0.0.1-cp311-cp311-macosx_11_0_arm64.whl", - "pkg-0.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "pkg-0.0.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", - "pkg-0.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", - "pkg-0.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "pkg-0.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", - "pkg-0.0.1-cp313-cp313t-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-cp313-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-abi3-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-none-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp311-cp311-musllinux_1_1_aarch64.whl", - "pkg-0.0.1-cp311-cp311-musllinux_1_1_i686.whl", - "pkg-0.0.1-cp311-cp311-musllinux_1_1_ppc64le.whl", - "pkg-0.0.1-cp311-cp311-musllinux_1_1_s390x.whl", - "pkg-0.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp311-cp311-win32.whl", - "pkg-0.0.1-cp311-cp311-win_amd64.whl", - "pkg-0.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", - "pkg-0.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "pkg-0.0.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", - "pkg-0.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", - "pkg-0.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "pkg-0.0.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", - "pkg-0.0.1-cp37-cp37m-musllinux_1_1_aarch64.whl", - "pkg-0.0.1-cp37-cp37m-musllinux_1_1_i686.whl", - "pkg-0.0.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", - "pkg-0.0.1-cp37-cp37m-musllinux_1_1_s390x.whl", - "pkg-0.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp37-cp37m-win32.whl", - "pkg-0.0.1-cp37-cp37m-win_amd64.whl", - "pkg-0.0.1-cp39-cp39-macosx_10_9_universal2.whl", - "pkg-0.0.1-cp39-cp39-macosx_10_9_x86_64.whl", - "pkg-0.0.1-cp39-cp39-macosx_11_0_arm64.whl", - "pkg-0.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "pkg-0.0.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", - "pkg-0.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", - "pkg-0.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "pkg-0.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", - "pkg-0.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", - "pkg-0.0.1-cp39-cp39-musllinux_1_1_i686.whl", - "pkg-0.0.1-cp39-cp39-musllinux_1_1_ppc64le.whl", - "pkg-0.0.1-cp39-cp39-musllinux_1_1_s390x.whl", - "pkg-0.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp39-cp39-win32.whl", - "pkg-0.0.1-cp39-cp39-win_amd64.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py310-abi3-any.whl", - "pkg-0.0.1-py3-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", -] - -def _match(env, got, *want_filenames): - if not want_filenames: - env.expect.that_collection(got).has_size(len(want_filenames)) - return - - got_filenames = [g.filename for g in got] - env.expect.that_collection(got_filenames).contains_exactly(want_filenames) - - if got: - # Check that we pass the original structs - env.expect.that_str(got[0].other).equals("dummy") - -def _select_whls(whls, debug = False, **kwargs): - return select_whls( - whls = [ - struct( - filename = f, - other = "dummy", - ) - for f in whls - ], - logger = repo_utils.logger(struct( - os = struct( - environ = { - REPO_DEBUG_ENV_VAR: "1", - REPO_VERBOSITY_ENV_VAR: "TRACE" if debug else "INFO", - }, - ), - ), "unit-test"), - **kwargs - ) - -_tests = [] - -def _test_simplest(env): - got = _select_whls( - whls = [ - "pkg-0.0.1-py2.py3-abi3-any.whl", - "pkg-0.0.1-py3-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ], - want_platforms = ["cp30_ignored"], - ) - _match( - env, - got, - "pkg-0.0.1-py3-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ) - -_tests.append(_test_simplest) - -def _test_select_by_supported_py_version(env): - for minor_version, match in { - 8: "pkg-0.0.1-py3-abi3-any.whl", - 11: "pkg-0.0.1-py311-abi3-any.whl", - }.items(): - got = _select_whls( - whls = [ - "pkg-0.0.1-py2.py3-abi3-any.whl", - "pkg-0.0.1-py3-abi3-any.whl", - "pkg-0.0.1-py311-abi3-any.whl", - ], - want_platforms = ["cp3{}_ignored".format(minor_version)], - ) - _match(env, got, match) - -_tests.append(_test_select_by_supported_py_version) - -def _test_select_by_supported_cp_version(env): - for minor_version, match in { - 11: "pkg-0.0.1-cp311-abi3-any.whl", - 8: "pkg-0.0.1-py3-abi3-any.whl", - }.items(): - got = _select_whls( - whls = [ - "pkg-0.0.1-py2.py3-abi3-any.whl", - "pkg-0.0.1-py3-abi3-any.whl", - "pkg-0.0.1-py311-abi3-any.whl", - "pkg-0.0.1-cp311-abi3-any.whl", - ], - want_platforms = ["cp3{}_ignored".format(minor_version)], - ) - _match(env, got, match) - -_tests.append(_test_select_by_supported_cp_version) - -def _test_supported_cp_version_manylinux(env): - for minor_version, match in { - 8: "pkg-0.0.1-py3-none-manylinux_x86_64.whl", - 11: "pkg-0.0.1-cp311-none-manylinux_x86_64.whl", - }.items(): - got = _select_whls( - whls = [ - "pkg-0.0.1-py2.py3-none-manylinux_x86_64.whl", - "pkg-0.0.1-py3-none-manylinux_x86_64.whl", - "pkg-0.0.1-py311-none-manylinux_x86_64.whl", - "pkg-0.0.1-cp311-none-manylinux_x86_64.whl", - ], - want_platforms = ["cp3{}_linux_x86_64".format(minor_version)], - ) - _match(env, got, match) - -_tests.append(_test_supported_cp_version_manylinux) - -def _test_ignore_unsupported(env): - got = _select_whls( - whls = [ - "pkg-0.0.1-xx3-abi3-any.whl", - ], - want_platforms = ["cp30_ignored"], - ) - _match(env, got) - -_tests.append(_test_ignore_unsupported) - -def _test_match_abi_and_not_py_version(env): - # Check we match the ABI and not the py version - got = _select_whls(whls = WHL_LIST, want_platforms = ["cp37_linux_x86_64"]) - _match( - env, - got, - "pkg-0.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "pkg-0.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-py3-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ) - -_tests.append(_test_match_abi_and_not_py_version) - -def _test_select_filename_with_many_tags(env): - # Check we can select a filename with many platform tags - got = _select_whls(whls = WHL_LIST, want_platforms = ["cp39_linux_x86_32"]) - _match( - env, - got, - "pkg-0.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", - "pkg-0.0.1-cp39-cp39-musllinux_1_1_i686.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ) - -_tests.append(_test_select_filename_with_many_tags) - -def _test_osx_prefer_arch_specific(env): - # Check that we prefer the specific wheel - got = _select_whls( - whls = WHL_LIST, - want_platforms = ["cp311_osx_x86_64", "cp311_osx_x86_32"], - ) - _match( - env, - got, - "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl", - "pkg-0.0.1-cp311-cp311-macosx_10_9_x86_64.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ) - - got = _select_whls(whls = WHL_LIST, want_platforms = ["cp311_osx_aarch64"]) - _match( - env, - got, - "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl", - "pkg-0.0.1-cp311-cp311-macosx_11_0_arm64.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ) - -_tests.append(_test_osx_prefer_arch_specific) - -def _test_osx_fallback_to_universal2(env): - # Check that we can use the universal2 if the arm wheel is not available - got = _select_whls( - whls = [w for w in WHL_LIST if "arm64" not in w], - want_platforms = ["cp311_osx_aarch64"], - ) - _match( - env, - got, - "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ) - -_tests.append(_test_osx_fallback_to_universal2) - -def _test_prefer_manylinux_wheels(env): - # Check we prefer platform specific wheels - got = _select_whls(whls = WHL_LIST, want_platforms = ["cp39_linux_x86_64"]) - _match( - env, - got, - "pkg-0.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "pkg-0.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ) - -_tests.append(_test_prefer_manylinux_wheels) - -def _test_freethreaded_wheels(env): - # Check we prefer platform specific wheels - got = _select_whls(whls = WHL_LIST, want_platforms = ["cp313_linux_x86_64"]) - _match( - env, - got, - "pkg-0.0.1-cp313-cp313t-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-cp313-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-abi3-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-none-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ) - -_tests.append(_test_freethreaded_wheels) - -def _test_micro_version_freethreaded(env): - # Check we prefer platform specific wheels - got = _select_whls(whls = WHL_LIST, want_platforms = ["cp313.3_linux_x86_64"]) - _match( - env, - got, - "pkg-0.0.1-cp313-cp313t-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-cp313-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-abi3-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-none-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ) - -_tests.append(_test_micro_version_freethreaded) - -def select_whl_test_suite(name): - """Create the test suite. - - Args: - name: the name of the test suite - """ - test_suite(name = name, basic_tests = _tests) From e2295aba9aae8c6aef60eae6be9597052349e8d0 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 17 Aug 2025 11:08:03 +0900 Subject: [PATCH 3/6] feat(pypi): builder for netrc and auth_patterns (#3136) With this we move closer towards starting playing with the API to fully replace `pip.parse` with `pip.configure` builder pattern for better expressiveness. Work towards #2747 --- CHANGELOG.md | 3 ++- python/private/pypi/extension.bzl | 23 +++++++++++++++-------- tests/pypi/extension/extension_tests.bzl | 22 +++++++++++++++++----- 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d45aa53c3..0e8ad65a5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -128,7 +128,8 @@ END_UNRELEASED_TEMPLATE ([#3114](https://github.com/bazel-contrib/rules_python/pull/3114)). * (pypi) To configure the environment for `requirements.txt` evaluation, use the newly added developer preview of the `pip.default` tag class. Only `rules_python` and root modules can use - this feature. You can also configure custom `config_settings` using `pip.default`. + this feature. You can also configure custom `config_settings` using `pip.default`. It + can also be used to set the global `netrc` or `auth_patterns` variables. * (pypi) PyPI dependencies now expose an `:extracted_whl_files` filegroup target of all the files extracted from the wheel. This can be used in lieu of {obj}`whl_filegroup` to avoid copying/extracting wheel multiple times to diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 2c7aa8a0e5..0c06dea2ff 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -322,12 +322,12 @@ def _create_whl_repos( src = src, whl_library_args = whl_library_args, download_only = pip_attr.download_only, - netrc = pip_attr.netrc, + netrc = config.netrc or pip_attr.netrc, use_downloader = use_downloader.get( whl.name, get_index_urls != None, # defaults to True if the get_index_urls is defined ), - auth_patterns = pip_attr.auth_patterns, + auth_patterns = config.auth_patterns or pip_attr.auth_patterns, python_version = major_minor, is_multiple_versions = whl.is_multiple_versions, enable_pipstar = config.enable_pipstar, @@ -502,13 +502,20 @@ def build_config( if platform and not (tag.arch_name or tag.config_settings or tag.env or tag.os_name or tag.whl_abi_tags or tag.whl_platform_tags): defaults["platforms"].pop(platform) - # TODO @aignas 2025-05-19: add more attr groups: - # * for AUTH - the default `netrc` usage could be configured through a common - # attribute. - # * for index/downloader config. This includes all of those attributes for - # overrides, etc. Index overrides per platform could be also used here. + _configure( + defaults, + override = mod.is_root, + # extra values that we just add + auth_patterns = tag.auth_patterns, + netrc = tag.netrc, + # TODO @aignas 2025-05-19: add more attr groups: + # * for index/downloader config. This includes all of those attributes for + # overrides, etc. Index overrides per platform could be also used here. + ) return struct( + auth_patterns = defaults.get("auth_patterns", {}), + netrc = defaults.get("netrc", None), platforms = { name: _plat(**values) for name, values in defaults["platforms"].items() @@ -975,7 +982,7 @@ See official [docs](https://packaging.python.org/en/latest/specifications/platfo ::: """, ), -} +} | AUTH_ATTRS _SUPPORTED_PEP508_KEYS = [ "implementation_name", diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 72cbb61d81..ab8362ef0c 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -100,28 +100,34 @@ def _build_config(env, enable_pipstar = 0, **kwargs): **kwargs ), attrs = dict( - platforms = subjects.dict, + auth_patterns = subjects.dict, enable_pipstar = subjects.bool, + netrc = subjects.str, + platforms = subjects.dict, ), ) def _default( *, arch_name = None, + auth_patterns = None, config_settings = None, + env = None, + netrc = None, os_name = None, platform = None, whl_platform_tags = None, - env = None, whl_abi_tags = None): return struct( arch_name = arch_name, - os_name = os_name, - platform = platform, - whl_platform_tags = whl_platform_tags or [], + auth_patterns = auth_patterns or {}, config_settings = config_settings, env = env or {}, + netrc = netrc, + os_name = os_name, + platform = platform, whl_abi_tags = whl_abi_tags or [], + whl_platform_tags = whl_platform_tags or [], ) def _parse( @@ -1224,11 +1230,17 @@ def _test_build_pipstar_platform(env): ], ), _default(platform = "myplat2"), + _default( + netrc = "my_netrc", + auth_patterns = {"foo": "bar"}, + ), ], ), ), enable_pipstar = True, ) + config.auth_patterns().contains_exactly({"foo": "bar"}) + config.netrc().equals("my_netrc") config.enable_pipstar().equals(True) config.platforms().contains_exactly({ "myplat": struct( From 8684dd98df009c321e7a5525f224b2eb61a8fa2d Mon Sep 17 00:00:00 2001 From: Ivo List Date: Sun, 17 Aug 2025 04:19:35 +0200 Subject: [PATCH 4/6] refactor: Use the linkstamps from linker_inputs instead of from cc_linking_context for to support upcoming CcInfo changes (#3075) This change allows Bazel to remove linkstamp from CcLinkingContext. The change is a no-op. Co-authored-by: Richard Levasseur Co-authored-by: Ignas Anikevicius <240938+aignas@users.noreply.github.com> --- python/private/py_executable.bzl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index 9927975aa8..30f18b5e64 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -1578,7 +1578,11 @@ def _create_shared_native_deps_dso( feature_configuration, requested_features, cc_toolchain): - linkstamps = py_internal.linking_context_linkstamps(cc_info.linking_context) + linkstamps = [ + py_internal.linkstamp_file(linkstamp) + for linker_input in cc_info.linking_context.linker_inputs.to_list() + for linkstamp in linker_input.linkstamps + ] partially_disabled_thin_lto = ( cc_common.is_enabled( @@ -1602,10 +1606,7 @@ def _create_shared_native_deps_dso( for input in cc_info.linking_context.linker_inputs.to_list() for flag in input.user_link_flags ], - linkstamps = [ - py_internal.linkstamp_file(linkstamp) - for linkstamp in linkstamps.to_list() - ], + linkstamps = linkstamps, build_info_artifacts = _get_build_info(ctx, cc_toolchain) if linkstamps else [], features = requested_features, is_test_target_partially_disabled_thin_lto = is_test and partially_disabled_thin_lto, From 51d9b98bf9f03804280a4c8e8a97ab0de5c2ae5f Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 17 Aug 2025 11:21:27 +0900 Subject: [PATCH 5/6] chore(pypi): remove unused config setting code (#3065) Summary: - remove config settings generation - remove unnecessary params - remove BUILD.bazel generating unnecessary params --- python/private/pypi/BUILD.bazel | 4 - python/private/pypi/config_settings.bzl | 328 +---------- python/private/pypi/hub_repository.bzl | 4 - python/private/pypi/pkg_aliases.bzl | 243 +------- python/private/pypi/render_pkg_aliases.bzl | 53 +- python/private/pypi/whl_config_setting.bzl | 12 +- .../config_settings/config_settings_tests.bzl | 551 ------------------ tests/pypi/extension/extension_tests.bzl | 26 +- tests/pypi/integration/BUILD.bazel | 13 - tests/pypi/pkg_aliases/pkg_aliases_test.bzl | 198 +------ .../render_pkg_aliases_test.bzl | 263 +-------- 11 files changed, 72 insertions(+), 1623 deletions(-) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 847c85d634..cb3408a191 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -320,8 +320,6 @@ bzl_library( srcs = ["pkg_aliases.bzl"], deps = [ ":labels_bzl", - ":parse_whl_name_bzl", - ":whl_target_platforms_bzl", "//python/private:text_util_bzl", "@bazel_skylib//lib:selects", ], @@ -349,9 +347,7 @@ bzl_library( srcs = ["render_pkg_aliases.bzl"], deps = [ ":generate_group_library_build_bazel_bzl", - ":parse_whl_name_bzl", ":whl_config_setting_bzl", - ":whl_target_platforms_bzl", "//python/private:normalize_name_bzl", "//python/private:text_util_bzl", ], diff --git a/python/private/pypi/config_settings.bzl b/python/private/pypi/config_settings.bzl index f4826007f8..dcb6779d5b 100644 --- a/python/private/pypi/config_settings.bzl +++ b/python/private/pypi/config_settings.bzl @@ -17,101 +17,17 @@ The {obj}`config_settings` macro is used to create the config setting targets that can be used in the {obj}`pkg_aliases` macro for selecting the compatible repositories. -Bazel's selects work by selecting the most-specialized configuration setting -that matches the target platform, which is further described in [bazel documentation][docs]. -We can leverage this fact to ensure that the most specialized matches are used -by default with the users being able to configure string_flag values to select -the less specialized ones. - -[docs]: https://bazel.build/docs/configurable-attributes - -The config settings in the order from the least specialized to the most -specialized is as follows: -* `:is_cp3` -* `:is_cp3_sdist` -* `:is_cp3_py_none_any` -* `:is_cp3_py3_none_any` -* `:is_cp3_py3_abi3_any` -* `:is_cp3_none_any` -* `:is_cp3_any_any` -* `:is_cp3_cp3_any` and `:is_cp3_cp3t_any` -* `:is_cp3_py_none_` -* `:is_cp3_py3_none_` -* `:is_cp3_py3_abi3_` -* `:is_cp3_none_` -* `:is_cp3_abi3_` -* `:is_cp3_cp3_` and `:is_cp3_cp3t_` - -Optionally instead of `` there sometimes may be `.` used in order to fully specify the versions - -The specialization of free-threaded vs non-free-threaded wheels is the same as -they are just variants of each other. The same goes for the specialization of -`musllinux` vs `manylinux`. - -The goal of this macro is to provide config settings that provide unambigous -matches if any pair of them is used together for any target configuration -setting. We achieve this by using dummy internal `flag_values` keys to force the -items further down the list to appear to be more specialized than the ones above. - -What is more, the names of the config settings are as similar to the platform wheel -specification as possible. How the wheel names map to the config setting names defined -in here is described in {obj}`pkg_aliases` documentation. - -:::{note} -Right now the specialization of adjacent config settings where one is with -`constraint_values` and one is without is ambiguous. I.e. `py_none_any` and -`sdist_linux_x86_64` have the same specialization from bazel point of view -because one has one `flag_value` entry and `constraint_values` and the -other has 2 flag_value entries. And unfortunately there is no way to disambiguate -it, because we are essentially in two dimensions here (`flag_values` and -`constraint_values`). Hence, when using the `config_settings` from here, -either have all of them with empty `suffix` or all of them with a non-empty -suffix. -::: +The config settings are of the form `:is_cp3`. Suffix is a +normalized user provided value for the platform name. The goal of this macro is to +ensure that we can incorporate the user provided `config_setting` targets to create +our composite config_setting targets. """ load("@bazel_skylib//lib:selects.bzl", "selects") -load("//python/private:flags.bzl", "LibcFlag") -load(":flags.bzl", "INTERNAL_FLAGS", "UniversalWhlFlag") - -FLAGS = struct( - **{ - f: str(Label("//python/config_settings:" + f)) - for f in [ - "is_pip_whl_auto", - "is_pip_whl_no", - "is_pip_whl_only", - "_is_py_freethreaded_yes", - "_is_py_freethreaded_no", - "pip_whl_glibc_version", - "pip_whl_muslc_version", - "pip_whl_osx_arch", - "pip_whl_osx_version", - "py_linux_libc", - "python_version", - ] - } -) - -_DEFAULT = "//conditions:default" -_INCOMPATIBLE = "@platforms//:incompatible" - -# Here we create extra string flags that are just to work with the select -# selecting the most specialized match. We don't allow the user to change -# them. -_flags = struct( - **{ - f: str(Label("//python/config_settings:_internal_pip_" + f)) - for f in INTERNAL_FLAGS - } -) def config_settings( *, python_versions = [], - glibc_versions = [], - muslc_versions = [], - osx_versions = [], name = None, platform_config_settings = {}, **kwargs): @@ -121,12 +37,6 @@ def config_settings( name (str): Currently unused. python_versions (list[str]): The list of python versions to configure config settings for. - glibc_versions (list[str]): The list of glibc version of the wheels to - configure config settings for. - muslc_versions (list[str]): The list of musl version of the wheels to - configure config settings for. - osx_versions (list[str]): The list of OSX OS versions to configure - config settings for. platform_config_settings: {type}`dict[str, list[str]]` the constraint values to use instead of the default ones. Key are platform names (a human-friendly platform string). Values are lists of @@ -134,212 +44,46 @@ def config_settings( **kwargs: Other args passed to the underlying implementations, such as {obj}`native`. """ - - glibc_versions = [""] + glibc_versions - muslc_versions = [""] + muslc_versions - osx_versions = [""] + osx_versions target_platforms = { "": [], - # TODO @aignas 2025-06-15: allowing universal2 and platform specific wheels in one - # closure is making things maybe a little bit too complicated. - "osx_universal2": ["@platforms//os:osx"], } | platform_config_settings - for python_version in python_versions: - for platform_name, config_settings in target_platforms.items(): - suffix = "_{}".format(platform_name) if platform_name else "" - os, _, cpu = platform_name.partition("_") + for platform_name, config_settings in target_platforms.items(): + suffix = "_{}".format(platform_name) if platform_name else "" + + # We parse the target settings and if there is a "platforms//os" or + # "platforms//cpu" value in here, we also add it into the constraint_values + # + # this is to ensure that we can still pass all of the unit tests for config + # setting specialization. + # + # TODO @aignas 2025-07-23: is this the right way? Maybe we should drop these + # and remove the tests? + constraint_values = [] + for setting in config_settings: + setting_label = Label(setting) + if setting_label.repo_name == "platforms" and setting_label.package in ["os", "cpu"]: + constraint_values.append(setting) + + for python_version in python_versions: + cpv = "cp" + python_version.replace(".", "") + prefix = "is_{}".format(cpv) - # We parse the target settings and if there is a "platforms//os" or - # "platforms//cpu" value in here, we also add it into the constraint_values - # - # this is to ensure that we can still pass all of the unit tests for config - # setting specialization. - constraint_values = [] - for setting in config_settings: - setting_label = Label(setting) - if setting_label.repo_name == "platforms" and setting_label.package in ["os", "cpu"]: - constraint_values.append(setting) - - _dist_config_settings( - suffix = suffix, - plat_flag_values = _plat_flag_values( - os = os, - cpu = cpu, - osx_versions = osx_versions, - glibc_versions = glibc_versions, - muslc_versions = muslc_versions, - ), + _dist_config_setting( + name = prefix + suffix, + flag_values = { + Label("//python/config_settings:python_version_major_minor"): python_version, + }, config_settings = config_settings, constraint_values = constraint_values, - python_version = python_version, **kwargs ) -def _dist_config_settings(*, suffix, plat_flag_values, python_version, **kwargs): - flag_values = { - Label("//python/config_settings:python_version_major_minor"): python_version, - } - - cpv = "cp" + python_version.replace(".", "") - prefix = "is_{}".format(cpv) - - _dist_config_setting( - name = prefix + suffix, - flag_values = flag_values, - **kwargs - ) - - flag_values[_flags.dist] = "" - - # First create an sdist, we will be building upon the flag values, which - # will ensure that each sdist config setting is the least specialized of - # all. However, we need at least one flag value to cover the case where we - # have `sdist` for any platform, hence we have a non-empty `flag_values` - # here. - _dist_config_setting( - name = "{}_sdist{}".format(prefix, suffix), - flag_values = flag_values, - compatible_with = (FLAGS.is_pip_whl_no, FLAGS.is_pip_whl_auto), - **kwargs - ) - - used_flags = {} - - # NOTE @aignas 2024-12-01: the abi3 is not compatible with freethreaded - # builds as per PEP703 (https://peps.python.org/pep-0703/#backwards-compatibility) - # - # The discussion here also reinforces this notion: - # https://discuss.python.org/t/pep-703-making-the-global-interpreter-lock-optional-3-12-updates/26503/99 - - for name, f, compatible_with in [ - ("py_none", _flags.whl, None), - ("py3_none", _flags.whl_py3, None), - ("py3_abi3", _flags.whl_py3_abi3, (FLAGS._is_py_freethreaded_no,)), - ("none", _flags.whl_pycp3x, None), - ("abi3", _flags.whl_pycp3x_abi3, (FLAGS._is_py_freethreaded_no,)), - # The below are not specializations of one another, they are variants - (cpv, _flags.whl_pycp3x_abicp, (FLAGS._is_py_freethreaded_no,)), - (cpv + "t", _flags.whl_pycp3x_abicp, (FLAGS._is_py_freethreaded_yes,)), - ]: - if (f, compatible_with) in used_flags: - # This should never happen as all of the different whls should have - # unique flag values - fail("BUG: the flag {} is attempted to be added twice to the list".format(f)) - else: - flag_values[f] = "yes" if f == _flags.whl else "" - used_flags[(f, compatible_with)] = True - - _dist_config_setting( - name = "{}_{}_any{}".format(prefix, name, suffix), - flag_values = flag_values, - compatible_with = compatible_with, - **kwargs - ) - - generic_flag_values = flag_values - generic_used_flags = used_flags - - for (suffix, flag_values) in plat_flag_values: - used_flags = {(f, None): True for f in flag_values} | generic_used_flags - flag_values = flag_values | generic_flag_values - - for name, f, compatible_with in [ - ("py_none", _flags.whl_plat, None), - ("py3_none", _flags.whl_plat_py3, None), - ("py3_abi3", _flags.whl_plat_py3_abi3, (FLAGS._is_py_freethreaded_no,)), - ("none", _flags.whl_plat_pycp3x, None), - ("abi3", _flags.whl_plat_pycp3x_abi3, (FLAGS._is_py_freethreaded_no,)), - # The below are not specializations of one another, they are variants - (cpv, _flags.whl_plat_pycp3x_abicp, (FLAGS._is_py_freethreaded_no,)), - (cpv + "t", _flags.whl_plat_pycp3x_abicp, (FLAGS._is_py_freethreaded_yes,)), - ]: - if (f, compatible_with) in used_flags: - # This should never happen as all of the different whls should have - # unique flag values. - fail("BUG: the flag {} is attempted to be added twice to the list".format(f)) - else: - flag_values[f] = "" - used_flags[(f, compatible_with)] = True - - _dist_config_setting( - name = "{}_{}_{}".format(prefix, name, suffix), - flag_values = flag_values, - compatible_with = compatible_with, - **kwargs - ) - -def _to_version_string(version, sep = "."): - if not version: - return "" - - return "{}{}{}".format(version[0], sep, version[1]) - -def _plat_flag_values(os, cpu, osx_versions, glibc_versions, muslc_versions): - ret = [] - if os == "": - return [] - elif os == "windows": - ret.append(("{}_{}".format(os, cpu), {})) - elif os == "osx": - for osx_version in osx_versions: - flags = { - FLAGS.pip_whl_osx_version: _to_version_string(osx_version), - } - if cpu != "universal2": - flags[FLAGS.pip_whl_osx_arch] = UniversalWhlFlag.ARCH - - if not osx_version: - suffix = "{}_{}".format(os, cpu) - else: - suffix = "{}_{}_{}".format(os, _to_version_string(osx_version, "_"), cpu) - - ret.append((suffix, flags)) - - elif os == "linux": - for os_prefix, linux_libc in { - os: LibcFlag.GLIBC, - "many" + os: LibcFlag.GLIBC, - "musl" + os: LibcFlag.MUSL, - }.items(): - if linux_libc == LibcFlag.GLIBC: - libc_versions = glibc_versions - libc_flag = FLAGS.pip_whl_glibc_version - elif linux_libc == LibcFlag.MUSL: - libc_versions = muslc_versions - libc_flag = FLAGS.pip_whl_muslc_version - else: - fail("Unsupported libc type: {}".format(linux_libc)) - - for libc_version in libc_versions: - if libc_version and os_prefix == os: - continue - elif libc_version: - suffix = "{}_{}_{}".format(os_prefix, _to_version_string(libc_version, "_"), cpu) - else: - suffix = "{}_{}".format(os_prefix, cpu) - - ret.append(( - suffix, - { - FLAGS.py_linux_libc: linux_libc, - libc_flag: _to_version_string(libc_version), - }, - )) - else: - fail("Unsupported os: {}".format(os)) - - return ret - -def _dist_config_setting(*, name, compatible_with = None, selects = selects, native = native, config_settings = None, **kwargs): +def _dist_config_setting(*, name, selects = selects, native = native, config_settings = None, **kwargs): """A macro to create a target for matching Python binary and source distributions. Args: name: The name of the public target. - compatible_with: {type}`tuple[Label]` A collection of config settings that are - compatible with the given dist config setting. For example, if only - non-freethreaded python builds are allowed, add - FLAGS._is_py_freethreaded_no here. config_settings: {type}`list[str | Label]` the list of target settings that must be matched before we try to evaluate the config_setting that we may create in this function. @@ -352,18 +96,6 @@ def _dist_config_setting(*, name, compatible_with = None, selects = selects, nat **kwargs: The kwargs passed to the config_setting rule. Visibility of the main alias target is also taken from the kwargs. """ - if compatible_with: - dist_config_setting_name = "_" + name - native.alias( - name = name, - actual = select( - {setting: dist_config_setting_name for setting in compatible_with} | { - _DEFAULT: _INCOMPATIBLE, - }, - ), - visibility = kwargs.get("visibility"), - ) - name = dist_config_setting_name # first define the config setting that has all of the constraint values _name = "_" + name diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index 75f3ec98d7..1d572d09e2 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -142,10 +142,6 @@ def whl_config_settings_to_json(repo_mapping): def _whl_config_setting_dict(a): ret = {} - if a.config_setting: - ret["config_setting"] = a.config_setting - if a.filename: - ret["filename"] = a.filename if a.target_platforms: ret["target_platforms"] = a.target_platforms if a.version: diff --git a/python/private/pypi/pkg_aliases.bzl b/python/private/pypi/pkg_aliases.bzl index 4d3cc61590..67ce297466 100644 --- a/python/private/pypi/pkg_aliases.bzl +++ b/python/private/pypi/pkg_aliases.bzl @@ -23,54 +23,12 @@ Definitions: :suffix: Can be either empty or `__`, which is usually used to distinguish multiple versions used for different target platforms. :os: OS identifier that exists in `@platforms//os:`. :cpu: CPU architecture identifier that exists in `@platforms//cpu:`. -:python_tag: The Python tag as defined by the [Python Packaging Authority][packaging_spec]. E.g. `py2.py3`, `py3`, `py311`, `cp311`. -:abi_tag: The ABI tag as defined by the [Python Packaging Authority][packaging_spec]. E.g. `none`, `abi3`, `cp311`, `cp311t`. -:platform_tag: The Platform tag as defined by the [Python Packaging Authority][packaging_spec]. E.g. `manylinux_2_17_x86_64`. -:platform_suffix: is a derivative of the `platform_tag` and is used to implement selection based on `libc` or `osx` version. All of the config settings used by this macro are generated by {obj}`config_settings`, for more detailed documentation on what each config setting maps to and their precedence, refer to documentation on that page. -The first group of config settings that are as follows: - -* `//_config:is_cp3` is used to select legacy `pip` - based `whl` and `sdist` {obj}`whl_library` instances. Whereas other config - settings are created when {obj}`pip.parse.experimental_index_url` is used. -* `//_config:is_cp3_sdist` is for wheels built from - `sdist` in {obj}`whl_library`. -* `//_config:is_cp3_py__any` for wheels with - `py2.py3` `python_tag` value. -* `//_config:is_cp3_py3__any` for wheels with - `py3` `python_tag` value. -* `//_config:is_cp3__any` for any other wheels. -* `//_config:is_cp3_py__` for - platform-specific wheels with `py2.py3` `python_tag` value. -* `//_config:is_cp3_py3__` for - platform-specific wheels with `py3` `python_tag` value. -* `//_config:is_cp3__` for any other - platform-specific wheels. - -Note that wheels with `abi3` or `none` `abi_tag` values and `python_tag` values -other than `py2.py3` or `py3` are compatible with the python version that is -equal or higher than the one denoted in the `python_tag`. For example: `py37` -and `cp37` wheels are compatible with Python 3.7 and above and in the case of -the target python version being `3.11`, `rules_python` will use -`//_config:is_cp311__any` config settings. - -For platform-specific wheels, i.e. the ones that have their `platform_tag` as -something else than `any`, we treat them as below: -* `linux_` tags assume that the target `libc` flavour is `glibc`, so this - is in many ways equivalent to it being `manylinux`, but with an unspecified - `libc` version. -* For `osx` and `linux` OSes wheel filename will be mapped to multiple config settings: - * `osx_` and `osx___` where - `major_version` and `minor_version` are the compatible OSX versions. - * `linux_` and - `linux___` where the version - identifiers are the compatible libc versions. - -[packaging_spec]: https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/ +`//_config:is_cp3` is used to select any target platforms. """ load("@bazel_skylib//lib:selects.bzl", "selects") @@ -85,14 +43,6 @@ load( "WHEEL_FILE_IMPL_LABEL", "WHEEL_FILE_PUBLIC_LABEL", ) -load(":parse_whl_name.bzl", "parse_whl_name") -load(":whl_target_platforms.bzl", "whl_target_platforms") - -# This value is used as sentinel value in the alias/config setting machinery -# for libc and osx versions. If we encounter this version in this part of the -# code, then it means that we have a bug in rules_python and that we should fix -# it. It is more of an internal consistency check. -_VERSION_NONE = (0, 0) _NO_MATCH_ERROR_TEMPLATE = """\ No matching wheel for current configuration's Python version. @@ -137,7 +87,7 @@ def pkg_aliases( to bazel skylib's `selects.with_or`, so they can be tuples as well. group_name: {type}`str` The group name that the pkg belongs to. extra_aliases: {type}`list[str]` The extra aliases to be created. - **kwargs: extra kwargs to pass to {bzl:obj}`get_filename_config_settings`. + **kwargs: extra kwargs to pass to {bzl:obj}`get_config_settings`. """ alias = kwargs.pop("native", native).alias select = kwargs.pop("select", selects.with_or) @@ -219,21 +169,9 @@ def pkg_aliases( actual = "//_groups:{}_whl".format(group_name), ) -def _normalize_versions(name, versions): - if not versions: - return [] - - if _VERSION_NONE in versions: - fail("a sentinel version found in '{}', check render_pkg_aliases for bugs".format(name)) - - return sorted(versions) - def multiplatform_whl_aliases( *, - aliases = [], - glibc_versions = [], - muslc_versions = [], - osx_versions = []): + aliases = []): """convert a list of aliases from filename to config_setting ones. Exposed only for unit tests. @@ -243,12 +181,6 @@ def multiplatform_whl_aliases( to process. Any aliases that have the filename set will be converted to a dict of config settings to repo names. The struct is created by {func}`whl_config_setting`. - glibc_versions: {type}`list[tuple[int, int]]` list of versions that can be - used in this hub repo. - muslc_versions: {type}`list[tuple[int, int]]` list of versions that can be - used in this hub repo. - osx_versions: {type}`list[tuple[int, int]]` list of versions that can be - used in this hub repo. Returns: A dict with of config setting labels to repo names or the repo name itself. @@ -258,207 +190,54 @@ def multiplatform_whl_aliases( # We don't have any aliases, this is a repo name return aliases - # TODO @aignas 2024-11-17: we might be able to use FeatureFlagInfo and some - # code gen to create a version_lt_x target, which would allow us to check - # if the libc version is in a particular range. - glibc_versions = _normalize_versions("glibc_versions", glibc_versions) - muslc_versions = _normalize_versions("muslc_versions", muslc_versions) - osx_versions = _normalize_versions("osx_versions", osx_versions) - ret = {} - versioned_additions = {} for alias, repo in aliases.items(): if type(alias) != "struct": ret[alias] = repo continue - elif not (alias.filename or alias.target_platforms): - # This is an internal consistency check - fail("Expected to have either 'filename' or 'target_platforms' set, got: {}".format(alias)) - config_settings, all_versioned_settings = get_filename_config_settings( - filename = alias.filename or "", + config_settings = get_config_settings( target_platforms = alias.target_platforms, python_version = alias.version, - # If we have multiple platforms but no wheel filename, lets use different - # config settings. - non_whl_prefix = "sdist" if alias.filename else "", - glibc_versions = glibc_versions, - muslc_versions = muslc_versions, - osx_versions = osx_versions, ) for setting in config_settings: ret["//_config" + setting] = repo - # Now for the versioned platform config settings, we need to select one - # that best fits the bill and if there are multiple wheels, e.g. - # manylinux_2_17_x86_64 and manylinux_2_28_x86_64, then we need to select - # the former when the glibc is in the range of [2.17, 2.28) and then chose - # the later if it is [2.28, ...). If the 2.28 wheel was not present in - # the hub, then we would need to use 2.17 for all the glibc version - # configurations. - # - # Here we add the version settings to a dict where we key the range of - # versions that the whl spans. If the wheel supports musl and glibc at - # the same time, we do this for each supported platform, hence the - # double dict. - for default_setting, versioned in all_versioned_settings.items(): - versions = sorted(versioned) - min_version = versions[0] - max_version = versions[-1] - - versioned_additions.setdefault(default_setting, {})[(min_version, max_version)] = struct( - repo = repo, - settings = versioned, - ) - - versioned = {} - for default_setting, candidates in versioned_additions.items(): - # Sort the candidates by the range of versions the span, so that we - # start with the lowest version. - for _, candidate in sorted(candidates.items()): - # Set the default with the first candidate, which gives us the highest - # compatibility. If the users want to use a higher-version than the default - # they can configure the glibc_version flag. - versioned.setdefault("//_config" + default_setting, candidate.repo) - - # We will be overwriting previously added entries, but that is intended. - for _, setting in candidate.settings.items(): - versioned["//_config" + setting] = candidate.repo - - ret.update(versioned) return ret -def get_filename_config_settings( +def get_config_settings( *, - filename, target_platforms, - python_version, - glibc_versions = None, - muslc_versions = None, - osx_versions = None, - non_whl_prefix = "sdist"): + python_version): """Get the filename config settings. Exposed only for unit tests. Args: - filename: the distribution filename (can be a whl or an sdist). target_platforms: list[str], target platforms in "{abi}_{os}_{cpu}" format. - glibc_versions: list[tuple[int, int]], list of versions. - muslc_versions: list[tuple[int, int]], list of versions. - osx_versions: list[tuple[int, int]], list of versions. python_version: the python version to generate the config_settings for. - non_whl_prefix: the prefix of the config setting when the whl we don't have - a filename ending with ".whl". Returns: A tuple: * A list of config settings that are generated by ./pip_config_settings.bzl * The list of default version settings. """ - prefixes = [] - suffixes = [] - setting_supported_versions = {} - - if filename.endswith(".whl"): - parsed = parse_whl_name(filename) - if parsed.python_tag == "py2.py3": - py = "py_" - elif parsed.python_tag == "py3": - py = "py3_" - elif parsed.python_tag.startswith("cp"): - py = "" - else: - py = "py3_" - abi = parsed.abi_tag - - # TODO @aignas 2025-04-20: test - abi, _, _ = abi.partition(".") - - if parsed.platform_tag == "any": - prefixes = ["{}{}_any".format(py, abi)] - else: - prefixes = ["{}{}".format(py, abi)] - suffixes = _whl_config_setting_suffixes( - platform_tag = parsed.platform_tag, - glibc_versions = glibc_versions, - muslc_versions = muslc_versions, - osx_versions = osx_versions, - setting_supported_versions = setting_supported_versions, - ) - else: - prefixes = [non_whl_prefix or ""] - - py = "cp{}".format(python_version).replace(".", "") prefixes = [ - "{}_{}".format(py, prefix) if prefix else py - for prefix in prefixes + "cp{}".format(python_version).replace(".", ""), ] - versioned = { - ":is_{}_{}".format(prefix, suffix): { - version: ":is_{}_{}".format(prefix, setting) - for version, setting in versions.items() - } - for prefix in prefixes - for suffix, versions in setting_supported_versions.items() - } - - if suffixes or target_platforms or versioned: + if target_platforms: target_platforms = target_platforms or [] - suffixes = suffixes or [_non_versioned_platform(p) for p in target_platforms] + suffixes = [_non_versioned_platform(p) for p in target_platforms] return [ ":is_{}_{}".format(prefix, suffix) for prefix in prefixes for suffix in suffixes - ], versioned + ] else: - return [":is_{}".format(p) for p in prefixes], setting_supported_versions - -def _whl_config_setting_suffixes( - platform_tag, - glibc_versions, - muslc_versions, - osx_versions, - setting_supported_versions): - suffixes = [] - for platform_tag in platform_tag.split("."): - for p in whl_target_platforms(platform_tag): - prefix = p.os - suffix = p.cpu - if "manylinux" in platform_tag: - prefix = "manylinux" - versions = glibc_versions - elif "musllinux" in platform_tag: - prefix = "musllinux" - versions = muslc_versions - elif p.os in ["linux", "windows"]: - versions = [(0, 0)] - elif p.os == "osx": - versions = osx_versions - if "universal2" in platform_tag: - suffix = "universal2" - else: - fail("Unsupported whl os: {}".format(p.os)) - - default_version_setting = "{}_{}".format(prefix, suffix) - supported_versions = {} - for v in versions: - if v == (0, 0): - suffixes.append(default_version_setting) - elif v >= p.version: - supported_versions[v] = "{}_{}_{}_{}".format( - prefix, - v[0], - v[1], - suffix, - ) - if supported_versions: - setting_supported_versions[default_version_setting] = supported_versions - - return suffixes + return [":is_{}".format(p) for p in prefixes] def _non_versioned_platform(p, *, strict = False): """A small utility function that converts 'cp311_linux_x86_64' to 'linux_x86_64'. diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index e743fc20f7..0a1c328491 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -22,8 +22,6 @@ load( ":generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel", ) # buildifier: disable=bzl-visibility -load(":parse_whl_name.bzl", "parse_whl_name") -load(":whl_target_platforms.bzl", "whl_target_platforms") NO_MATCH_ERROR_MESSAGE_TEMPLATE = """\ No matching wheel for current configuration's Python version. @@ -48,19 +46,17 @@ def _repr_dict(*, value_repr = repr, **kwargs): return {k: value_repr(v) for k, v in kwargs.items() if v} def _repr_config_setting(alias): - if alias.filename or alias.target_platforms: + if alias.target_platforms: return render.call( "whl_config_setting", **_repr_dict( - filename = alias.filename, target_platforms = alias.target_platforms, - config_setting = alias.config_setting, version = alias.version, ) ) else: return repr( - alias.config_setting or "//_config:is_cp{}".format(alias.version.replace(".", "")), + "//_config:is_cp{}".format(alias.version.replace(".", "")), ) def _repr_actual(aliases): @@ -179,15 +175,9 @@ def render_multiplatform_pkg_aliases(*, aliases, platform_config_settings = {}, contents = render_pkg_aliases( aliases = aliases, - glibc_versions = flag_versions.get("glibc_versions", []), - muslc_versions = flag_versions.get("muslc_versions", []), - osx_versions = flag_versions.get("osx_versions", []), **kwargs ) contents["_config/BUILD.bazel"] = _render_config_settings( - glibc_versions = flag_versions.get("glibc_versions", []), - muslc_versions = flag_versions.get("muslc_versions", []), - osx_versions = flag_versions.get("osx_versions", []), python_versions = _major_minor_versions(flag_versions.get("python_versions", [])), platform_config_settings = platform_config_settings, visibility = ["//:__subpackages__"], @@ -219,54 +209,21 @@ def get_whl_flag_versions(settings): * python_versions """ python_versions = {} - glibc_versions = {} target_platforms = {} - muslc_versions = {} - osx_versions = {} for setting in settings: - if not setting.version and not setting.filename: + if not setting.version: continue if setting.version: python_versions[setting.version] = None - if setting.filename and setting.filename.endswith(".whl") and not setting.filename.endswith("-any.whl"): - parsed = parse_whl_name(setting.filename) - else: - for plat in setting.target_platforms or []: - target_platforms[_non_versioned_platform(plat)] = None - continue - - for platform_tag in parsed.platform_tag.split("."): - parsed = whl_target_platforms(platform_tag) - - for p in parsed: - target_platforms[p.target_platform] = None - - if platform_tag.startswith("win") or platform_tag.startswith("linux"): - continue - - head, _, tail = platform_tag.partition("_") - major, _, tail = tail.partition("_") - minor, _, tail = tail.partition("_") - if tail: - version = (int(major), int(minor)) - if "many" in head: - glibc_versions[version] = None - elif "musl" in head: - muslc_versions[version] = None - elif "mac" in head: - osx_versions[version] = None - else: - fail(platform_tag) + for plat in setting.target_platforms or []: + target_platforms[_non_versioned_platform(plat)] = None return { k: sorted(v) for k, v in { - "glibc_versions": glibc_versions, - "muslc_versions": muslc_versions, - "osx_versions": osx_versions, "python_versions": python_versions, "target_platforms": target_platforms, }.items() diff --git a/python/private/pypi/whl_config_setting.bzl b/python/private/pypi/whl_config_setting.bzl index 3b81e4694f..1d868b1b65 100644 --- a/python/private/pypi/whl_config_setting.bzl +++ b/python/private/pypi/whl_config_setting.bzl @@ -14,7 +14,7 @@ "A small function to create an alias for a whl distribution" -def whl_config_setting(*, version = None, config_setting = None, filename = None, target_platforms = None): +def whl_config_setting(*, version = None, target_platforms = None): """The bzl_packages value used by by the render_pkg_aliases function. This contains the minimum amount of information required to generate correct @@ -25,15 +25,17 @@ def whl_config_setting(*, version = None, config_setting = None, filename = None whl alias is for. If not set, then non-version aware aliases will be constructed. This is mainly used for better error messages when there is no match found during a select. - config_setting: {type}`str | Label | None` the config setting that we should use. Defaults - to "//_config:is_python_{version}". - filename: {type}`str | None` the distribution filename to derive the config_setting. target_platforms: {type}`list[str] | None` the list of target_platforms for this distribution. Returns: a struct with the validated and parsed values. """ + + # FIXME @aignas 2025-07-26: There is still a potential that there will be ambigous match + # if the user is trying to have different packages for 3.X.Y and 3.X.Z versions of python + # in the same hub repository. Consider just removing this processing as I am not sure it + # has much value. if target_platforms: target_platforms_input = target_platforms target_platforms = [] @@ -50,8 +52,6 @@ def whl_config_setting(*, version = None, config_setting = None, filename = None target_platforms.append("{}_{}".format(abi, tail)) return struct( - config_setting = config_setting, - filename = filename, # Make the struct hashable target_platforms = tuple(target_platforms) if target_platforms else None, version = version, diff --git a/tests/pypi/config_settings/config_settings_tests.bzl b/tests/pypi/config_settings/config_settings_tests.bzl index a15f6b4d32..b3e6ada9e8 100644 --- a/tests/pypi/config_settings/config_settings_tests.bzl +++ b/tests/pypi/config_settings/config_settings_tests.bzl @@ -97,554 +97,6 @@ def _test_legacy_with_constraint_values(name): _tests.append(_test_legacy_with_constraint_values) -# Tests when we only have an `sdist` present. - -def _test_sdist_default(name): - _analysis_test( - name = name, - dist = { - "is_cp37_sdist": "sdist", - }, - want = "sdist", - ) - -_tests.append(_test_sdist_default) - -def _test_legacy_less_specialized_than_sdist(name): - _analysis_test( - name = name, - dist = { - "is_cp37": "legacy", - "is_cp37_sdist": "sdist", - }, - want = "sdist", - ) - -_tests.append(_test_legacy_less_specialized_than_sdist) - -def _test_sdist_no_whl(name): - _analysis_test( - name = name, - dist = { - "is_cp37_sdist": "sdist", - }, - config_settings = [ - _flag.platform("linux_aarch64"), - _flag.pip_whl("no"), - ], - want = "sdist", - ) - -_tests.append(_test_sdist_no_whl) - -def _test_sdist_no_sdist(name): - _analysis_test( - name = name, - dist = { - "is_cp37_sdist": "sdist", - }, - config_settings = [ - _flag.platform("linux_aarch64"), - _flag.pip_whl("only"), - ], - # We will use `no_match_error` in the real case to indicate that `sdist` is not - # allowed to be used. - want = "no_match", - ) - -_tests.append(_test_sdist_no_sdist) - -def _test_basic_whl_default(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py_none_any": "whl", - "is_cp37_sdist": "sdist", - }, - want = "whl", - ) - -_tests.append(_test_basic_whl_default) - -def _test_basic_whl_nowhl(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py_none_any": "whl", - "is_cp37_sdist": "sdist", - }, - config_settings = [ - _flag.platform("linux_aarch64"), - _flag.pip_whl("no"), - ], - want = "sdist", - ) - -_tests.append(_test_basic_whl_nowhl) - -def _test_basic_whl_nosdist(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py_none_any": "whl", - "is_cp37_sdist": "sdist", - }, - config_settings = [ - _flag.platform("linux_aarch64"), - _flag.pip_whl("only"), - ], - want = "whl", - ) - -_tests.append(_test_basic_whl_nosdist) - -def _test_whl_default(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py3_none_any": "whl", - "is_cp37_py_none_any": "basic_whl", - }, - want = "whl", - ) - -_tests.append(_test_whl_default) - -def _test_whl_nowhl(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py3_none_any": "whl", - "is_cp37_py_none_any": "basic_whl", - }, - config_settings = [ - _flag.platform("linux_aarch64"), - _flag.pip_whl("no"), - ], - want = "no_match", - ) - -_tests.append(_test_whl_nowhl) - -def _test_whl_nosdist(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py3_none_any": "whl", - }, - config_settings = [ - _flag.platform("linux_aarch64"), - _flag.pip_whl("only"), - ], - want = "whl", - ) - -_tests.append(_test_whl_nosdist) - -def _test_abi_whl_is_prefered(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py3_abi3_any": "abi_whl", - "is_cp37_py3_none_any": "whl", - }, - want = "abi_whl", - ) - -_tests.append(_test_abi_whl_is_prefered) - -def _test_whl_with_constraints_is_prefered(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py3_none_any": "default_whl", - "is_cp37_py3_none_any_linux_aarch64": "whl", - "is_cp37_py3_none_any_linux_x86_64": "amd64_whl", - }, - want = "whl", - ) - -_tests.append(_test_whl_with_constraints_is_prefered) - -def _test_cp_whl_is_prefered_over_py3(name): - _analysis_test( - name = name, - dist = { - "is_cp37_none_any": "cp", - "is_cp37_py3_abi3_any": "py3_abi3", - "is_cp37_py3_none_any": "py3", - }, - want = "cp", - ) - -_tests.append(_test_cp_whl_is_prefered_over_py3) - -def _test_cp_abi_whl_is_prefered_over_py3(name): - _analysis_test( - name = name, - dist = { - "is_cp37_abi3_any": "cp", - "is_cp37_py3_abi3_any": "py3", - }, - want = "cp", - ) - -_tests.append(_test_cp_abi_whl_is_prefered_over_py3) - -def _test_cp_version_is_selected_when_python_version_is_specified(name): - _analysis_test( - name = name, - dist = { - "is_cp310_none_any": "cp310", - "is_cp38_none_any": "cp38", - "is_cp39_none_any": "cp39", - }, - want = "cp310", - config_settings = [ - _flag.python_version("3.10.9"), - _flag.platform("linux_aarch64"), - ], - ) - -_tests.append(_test_cp_version_is_selected_when_python_version_is_specified) - -def _test_py_none_any_versioned(name): - _analysis_test( - name = name, - dist = { - "is_cp310_py_none_any": "whl", - "is_cp39_py_none_any": "too-low", - }, - want = "whl", - config_settings = [ - _flag.python_version("3.10.9"), - _flag.platform("linux_aarch64"), - ], - ) - -_tests.append(_test_py_none_any_versioned) - -def _test_cp_whl_is_not_prefered_over_py3_non_freethreaded(name): - _analysis_test( - name = name, - dist = { - "is_cp37_abi3_any": "py3_abi3", - "is_cp37_cp37t_any": "cp", - "is_cp37_none_any": "py3", - }, - want = "py3_abi3", - config_settings = [ - _flag.py_freethreaded("no"), - ], - ) - -_tests.append(_test_cp_whl_is_not_prefered_over_py3_non_freethreaded) - -def _test_cp_whl_is_not_prefered_over_py3_freethreaded(name): - _analysis_test( - name = name, - dist = { - "is_cp37_abi3_any": "py3_abi3", - "is_cp37_cp37_any": "cp", - "is_cp37_none_any": "py3", - }, - want = "py3", - config_settings = [ - _flag.py_freethreaded("yes"), - ], - ) - -_tests.append(_test_cp_whl_is_not_prefered_over_py3_freethreaded) - -def _test_cp_cp_whl(name): - _analysis_test( - name = name, - dist = { - "is_cp310_cp310_linux_aarch64": "whl", - }, - want = "whl", - config_settings = [ - _flag.python_version("3.10.9"), - _flag.platform("linux_aarch64"), - ], - ) - -_tests.append(_test_cp_cp_whl) - -def _test_cp_version_sdist_is_selected(name): - _analysis_test( - name = name, - dist = { - "is_cp310_sdist": "sdist", - }, - want = "sdist", - config_settings = [ - _flag.python_version("3.10.9"), - _flag.platform("linux_aarch64"), - ], - ) - -_tests.append(_test_cp_version_sdist_is_selected) - -# NOTE: Right now there is no way to get the following behaviour without -# breaking other tests. We need to choose either ta have the correct -# specialization behaviour between `is_cp37_cp37_any` and -# `is_cp37_cp37_any_linux_aarch64` or this commented out test case. -# -# I think having this behaviour not working is fine because the `suffix` -# will be either present on all of config settings of the same platform -# or none, because we use it as a way to select a separate version of the -# wheel for a single platform only. -# -# If we can think of a better way to handle it, then we can lift this -# limitation. -# -# def _test_any_whl_with_suffix_specialization(name): -# _analysis_test( -# name = name, -# dist = { -# "is_cp37_abi3_any_linux_aarch64": "abi3", -# "is_cp37_cp37_any": "cp37", -# }, -# want = "cp37", -# ) -# -# _tests.append(_test_any_whl_with_suffix_specialization) - -def _test_platform_vs_any_with_suffix_specialization(name): - _analysis_test( - name = name, - dist = { - "is_cp37_cp37_any_linux_aarch64": "any", - "is_cp37_py3_none_linux_aarch64": "platform_whl", - }, - want = "platform_whl", - ) - -_tests.append(_test_platform_vs_any_with_suffix_specialization) - -def _test_platform_whl_is_prefered_over_any_whl_with_constraints(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py3_abi3_any": "better_default_whl", - "is_cp37_py3_abi3_any_linux_aarch64": "better_default_any_whl", - "is_cp37_py3_none_any": "default_whl", - "is_cp37_py3_none_any_linux_aarch64": "whl", - "is_cp37_py3_none_linux_aarch64": "platform_whl", - }, - want = "platform_whl", - ) - -_tests.append(_test_platform_whl_is_prefered_over_any_whl_with_constraints) - -def _test_abi3_platform_whl_preference(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py3_abi3_linux_aarch64": "abi3_platform", - "is_cp37_py3_none_linux_aarch64": "platform", - }, - want = "abi3_platform", - ) - -_tests.append(_test_abi3_platform_whl_preference) - -def _test_glibc(name): - _analysis_test( - name = name, - dist = { - "is_cp37_cp37_manylinux_aarch64": "glibc", - "is_cp37_py3_abi3_linux_aarch64": "abi3_platform", - }, - want = "glibc", - ) - -_tests.append(_test_glibc) - -def _test_glibc_versioned(name): - _analysis_test( - name = name, - dist = { - "is_cp37_cp37_manylinux_2_14_aarch64": "glibc", - "is_cp37_cp37_manylinux_2_17_aarch64": "glibc", - "is_cp37_py3_abi3_linux_aarch64": "abi3_platform", - }, - want = "glibc", - config_settings = [ - _flag.py_linux_libc("glibc"), - _flag.pip_whl_glibc_version("2.17"), - _flag.platform("linux_aarch64"), - ], - ) - -_tests.append(_test_glibc_versioned) - -def _test_glibc_compatible_exists(name): - _analysis_test( - name = name, - dist = { - # Code using the conditions will need to construct selects, which - # do the version matching correctly. - "is_cp37_cp37_manylinux_2_14_aarch64": "2_14_whl_via_2_14_branch", - "is_cp37_cp37_manylinux_2_17_aarch64": "2_14_whl_via_2_17_branch", - }, - want = "2_14_whl_via_2_17_branch", - config_settings = [ - _flag.py_linux_libc("glibc"), - _flag.pip_whl_glibc_version("2.17"), - _flag.platform("linux_aarch64"), - ], - ) - -_tests.append(_test_glibc_compatible_exists) - -def _test_musl(name): - _analysis_test( - name = name, - dist = { - "is_cp37_cp37_musllinux_aarch64": "musl", - }, - want = "musl", - config_settings = [ - _flag.py_linux_libc("musl"), - _flag.platform("linux_aarch64"), - ], - ) - -_tests.append(_test_musl) - -def _test_windows(name): - _analysis_test( - name = name, - dist = { - "is_cp37_cp37_windows_x86_64": "whl", - "is_cp37_cp37t_windows_x86_64": "whl_freethreaded", - }, - want = "whl", - config_settings = [ - _flag.platform("windows_x86_64"), - ], - ) - -_tests.append(_test_windows) - -def _test_windows_freethreaded(name): - _analysis_test( - name = name, - dist = { - "is_cp37_cp37_windows_x86_64": "whl", - "is_cp37_cp37t_windows_x86_64": "whl_freethreaded", - }, - want = "whl_freethreaded", - config_settings = [ - _flag.platform("windows_x86_64"), - _flag.py_freethreaded("yes"), - ], - ) - -_tests.append(_test_windows_freethreaded) - -def _test_osx(name): - _analysis_test( - name = name, - dist = { - # We prefer arch specific whls over universal - "is_cp37_cp37_osx_universal2": "universal_whl", - "is_cp37_cp37_osx_x86_64": "whl", - }, - want = "whl", - config_settings = [ - _flag.platform("mac_x86_64"), - ], - ) - -_tests.append(_test_osx) - -def _test_osx_universal_default(name): - _analysis_test( - name = name, - dist = { - # We default to universal if only that exists - "is_cp37_cp37_osx_universal2": "whl", - }, - want = "whl", - config_settings = [ - _flag.platform("mac_x86_64"), - ], - ) - -_tests.append(_test_osx_universal_default) - -def _test_osx_universal_only(name): - _analysis_test( - name = name, - dist = { - # If we prefer universal, then we use that - "is_cp37_cp37_osx_universal2": "universal", - "is_cp37_cp37_osx_x86_64": "whl", - }, - want = "universal", - config_settings = [ - _flag.pip_whl_osx_arch("universal"), - _flag.platform("mac_x86_64"), - ], - ) - -_tests.append(_test_osx_universal_only) - -def _test_osx_os_version(name): - _analysis_test( - name = name, - dist = { - # Similarly to the libc version, the user of the config settings will have to - # construct the select so that the version selection is correct. - "is_cp37_cp37_osx_10_9_x86_64": "whl", - }, - want = "whl", - config_settings = [ - _flag.pip_whl_osx_version("10.9"), - _flag.platform("mac_x86_64"), - ], - ) - -_tests.append(_test_osx_os_version) - -def _test_all(name): - _analysis_test( - name = name, - dist = { - "is_cp37_" + f: f - for f in [ - "{py}{abi}_{plat}".format(py = valid_py, abi = valid_abi, plat = valid_plat) - # we have py2.py3, py3, cp3 - for valid_py in ["py_", "py3_", ""] - # cp abi usually comes with a version and we only need one - # config setting variant for all of them because the python - # version will discriminate between different versions. - for valid_abi in ["none", "abi3", "cp37"] - for valid_plat in [ - "any", - "manylinux_2_17_x86_64", - "manylinux_2_17_aarch64", - "osx_x86_64", - "windows_x86_64", - ] - if not ( - valid_abi == "abi3" and valid_py == "py_" or - valid_abi == "cp37" and valid_py != "" - ) - ] - }, - want = "cp37_manylinux_2_17_x86_64", - config_settings = [ - _flag.pip_whl_glibc_version("2.17"), - _flag.platform("linux_x86_64"), - ], - ) - -_tests.append(_test_all) - def config_settings_test_suite(name): # buildifier: disable=function-docstring test_suite( name = name, @@ -654,9 +106,6 @@ def config_settings_test_suite(name): # buildifier: disable=function-docstring config_settings( name = "dummy", python_versions = ["3.7", "3.8", "3.9", "3.10"], - glibc_versions = [(2, 14), (2, 17)], - muslc_versions = [(1, 1)], - osx_versions = [(10, 9), (11, 0)], platform_config_settings = { "linux_aarch64": [ "@platforms//cpu:aarch64", diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index ab8362ef0c..b85414528d 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -594,25 +594,25 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "torch": { "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": [ whl_config_setting( - target_platforms = ["cp312_linux_x86_64"], + target_platforms = ("cp312_linux_x86_64",), version = "3.12", ), ], "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": [ whl_config_setting( - target_platforms = ["cp312_linux_aarch64"], + target_platforms = ("cp312_linux_aarch64",), version = "3.12", ), ], "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": [ whl_config_setting( - target_platforms = ["cp312_windows_x86_64"], + target_platforms = ("cp312_windows_x86_64",), version = "3.12", ), ], "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": [ whl_config_setting( - target_platforms = ["cp312_osx_aarch64"], + target_platforms = ("cp312_osx_aarch64",), version = "3.12", ), ], @@ -1085,8 +1085,6 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' "cp315_linux_aarch64", "cp315_linux_x86_64", ], - config_setting = None, - filename = None, ), ], "pypi_315_optimum_osx_aarch64": [ @@ -1095,8 +1093,6 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' target_platforms = [ "cp315_osx_aarch64", ], - config_setting = None, - filename = None, ), ], }, @@ -1127,7 +1123,7 @@ def _test_pipstar_platforms(env): name = "rules_python", default = [ _default( - platform = "my{}_{}".format(os, cpu), + platform = "my{}{}".format(os, cpu), os_name = os, arch_name = cpu, config_settings = [ @@ -1167,19 +1163,19 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' pypi.hub_whl_map().contains_exactly({ "pypi": { "optimum": { - "pypi_315_optimum_mylinux_x86_64": [ + "pypi_315_optimum_mylinuxx86_64": [ whl_config_setting( version = "3.15", target_platforms = [ - "cp315_mylinux_x86_64", + "cp315_mylinuxx86_64", ], ), ], - "pypi_315_optimum_myosx_aarch64": [ + "pypi_315_optimum_myosxaarch64": [ whl_config_setting( version = "3.15", target_platforms = [ - "cp315_myosx_aarch64", + "cp315_myosxaarch64", ], ), ], @@ -1188,12 +1184,12 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' }) pypi.whl_libraries().contains_exactly({ - "pypi_315_optimum_mylinux_x86_64": { + "pypi_315_optimum_mylinuxx86_64": { "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "optimum[onnxruntime-gpu]==1.17.1", }, - "pypi_315_optimum_myosx_aarch64": { + "pypi_315_optimum_myosxaarch64": { "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "optimum[onnxruntime]==1.17.1", diff --git a/tests/pypi/integration/BUILD.bazel b/tests/pypi/integration/BUILD.bazel index 9ea8dcebe4..316abe8271 100644 --- a/tests/pypi/integration/BUILD.bazel +++ b/tests/pypi/integration/BUILD.bazel @@ -1,20 +1,7 @@ load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@rules_python_publish_deps//:requirements.bzl", "all_requirements") -load(":transitions.bzl", "transition_rule") build_test( name = "all_requirements_build_test", targets = all_requirements, ) - -# Rule that transitions dependencies to be built from sdist -transition_rule( - name = "all_requirements_from_sdist", - testonly = True, - deps = all_requirements, -) - -build_test( - name = "all_requirements_from_sdist_build_test", - targets = ["all_requirements_from_sdist"], -) diff --git a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl index 3fd08c393c..6248261ed4 100644 --- a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl +++ b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl @@ -159,18 +159,12 @@ def _test_multiplatform_whl_aliases(env): name = "bar_baz", actual = { whl_config_setting( - filename = "foo-0.0.0-py3-none-any.whl", version = "3.9", - ): "filename_repo", + ): "version_repo", whl_config_setting( - filename = "foo-0.0.0-py3-none-any.whl", version = "3.9", target_platforms = ["cp39_linux_x86_64"], - ): "filename_repo_for_platform", - whl_config_setting( - version = "3.9", - target_platforms = ["cp39_linux_x86_64"], - ): "bzlmod_repo_for_a_particular_platform", + ): "version_platform_repo", "//:my_config_setting": "bzlmod_repo", }, extra_aliases = [], @@ -178,18 +172,14 @@ def _test_multiplatform_whl_aliases(env): alias = lambda *, name, actual, visibility = None, tags = None: got.update({name: actual}), ), select = mock_select, - glibc_versions = [], - muslc_versions = [], - osx_versions = [], ) # buildifier: disable=unsorted-dict-items want = { "pkg": { "//:my_config_setting": "@bzlmod_repo//:pkg", - "//_config:is_cp39_linux_x86_64": "@bzlmod_repo_for_a_particular_platform//:pkg", - "//_config:is_cp39_py3_none_any": "@filename_repo//:pkg", - "//_config:is_cp39_py3_none_any_linux_x86_64": "@filename_repo_for_platform//:pkg", + "//_config:is_cp39": "@version_repo//:pkg", + "//_config:is_cp39_linux_x86_64": "@version_platform_repo//:pkg", "//conditions:default": "_no_matching_repository", }, } @@ -198,9 +188,8 @@ def _test_multiplatform_whl_aliases(env): env.expect.that_str(actual_no_match_error[0]).contains("""\ configuration settings: //:my_config_setting + //_config:is_cp39 //_config:is_cp39_linux_x86_64 - //_config:is_cp39_py3_none_any - //_config:is_cp39_py3_none_any_linux_x86_64 """) @@ -279,7 +268,6 @@ _tests.append(_test_multiplatform_whl_aliases_nofilename) def _test_multiplatform_whl_aliases_nofilename_target_platforms(env): aliases = { whl_config_setting( - config_setting = "//:ignored", version = "3.1", target_platforms = [ "cp31_linux_x86_64", @@ -298,102 +286,6 @@ def _test_multiplatform_whl_aliases_nofilename_target_platforms(env): _tests.append(_test_multiplatform_whl_aliases_nofilename_target_platforms) -def _test_multiplatform_whl_aliases_filename(env): - aliases = { - whl_config_setting( - filename = "foo-0.0.3-py3-none-any.whl", - version = "3.2", - ): "foo-py3-0.0.3", - whl_config_setting( - filename = "foo-0.0.1-py3-none-any.whl", - version = "3.1", - ): "foo-py3-0.0.1", - whl_config_setting( - filename = "foo-0.0.1-cp313-cp313-any.whl", - version = "3.13", - ): "foo-cp-0.0.1", - whl_config_setting( - filename = "foo-0.0.1-cp313-cp313t-any.whl", - version = "3.13", - ): "foo-cpt-0.0.1", - whl_config_setting( - filename = "foo-0.0.2-py3-none-any.whl", - version = "3.1", - target_platforms = [ - "cp31_linux_x86_64", - "cp31_linux_aarch64", - ], - ): "foo-0.0.2", - } - got = multiplatform_whl_aliases( - aliases = aliases, - glibc_versions = [], - muslc_versions = [], - osx_versions = [], - ) - want = { - "//_config:is_cp313_cp313_any": "foo-cp-0.0.1", - "//_config:is_cp313_cp313t_any": "foo-cpt-0.0.1", - "//_config:is_cp31_py3_none_any": "foo-py3-0.0.1", - "//_config:is_cp31_py3_none_any_linux_aarch64": "foo-0.0.2", - "//_config:is_cp31_py3_none_any_linux_x86_64": "foo-0.0.2", - "//_config:is_cp32_py3_none_any": "foo-py3-0.0.3", - } - env.expect.that_dict(got).contains_exactly(want) - -_tests.append(_test_multiplatform_whl_aliases_filename) - -def _test_multiplatform_whl_aliases_filename_versioned(env): - aliases = { - whl_config_setting( - filename = "foo-0.0.1-py3-none-manylinux_2_17_x86_64.whl", - version = "3.1", - ): "glibc-2.17", - whl_config_setting( - filename = "foo-0.0.1-py3-none-manylinux_2_18_x86_64.whl", - version = "3.1", - ): "glibc-2.18", - whl_config_setting( - filename = "foo-0.0.1-py3-none-musllinux_1_1_x86_64.whl", - version = "3.1", - ): "musl-1.1", - } - got = multiplatform_whl_aliases( - aliases = aliases, - glibc_versions = [(2, 17), (2, 18)], - muslc_versions = [(1, 1), (1, 2)], - osx_versions = [], - ) - want = { - # This could just work with: - # select({ - # "//_config:is_gt_eq_2.18": "//_config:is_cp3.1_py3_none_manylinux_x86_64", - # "//conditions:default": "//_config:is_gt_eq_2.18", - # }): "glibc-2.18", - # select({ - # "//_config:is_range_2.17_2.18": "//_config:is_cp3.1_py3_none_manylinux_x86_64", - # "//_config:is_glibc_default": "//_config:is_cp3.1_py3_none_manylinux_x86_64", - # "//conditions:default": "//_config:is_glibc_default", - # }): "glibc-2.17", - # ( - # "//_config:is_gt_musl_1.1": "musl-1.1", - # "//_config:is_musl_default": "musl-1.1", - # ): "musl-1.1", - # - # For this to fully work we need to have the pypi:config_settings.bzl to generate the - # extra targets that use the FeatureFlagInfo and this to generate extra aliases for the - # config settings. - "//_config:is_cp31_py3_none_manylinux_2_17_x86_64": "glibc-2.17", - "//_config:is_cp31_py3_none_manylinux_2_18_x86_64": "glibc-2.18", - "//_config:is_cp31_py3_none_manylinux_x86_64": "glibc-2.17", - "//_config:is_cp31_py3_none_musllinux_1_1_x86_64": "musl-1.1", - "//_config:is_cp31_py3_none_musllinux_1_2_x86_64": "musl-1.1", - "//_config:is_cp31_py3_none_musllinux_x86_64": "musl-1.1", - } - env.expect.that_dict(got).contains_exactly(want) - -_tests.append(_test_multiplatform_whl_aliases_filename_versioned) - def _mock_alias(container): return lambda name, **kwargs: container.append(name) @@ -451,86 +343,6 @@ def _test_config_settings_exist_legacy(env): _tests.append(_test_config_settings_exist_legacy) -def _test_config_settings_exist(env): - for py_tag in ["py2.py3", "py3", "py311", "cp311"]: - if py_tag == "py2.py3": - abis = ["none"] - elif py_tag.startswith("py"): - abis = ["none", "abi3"] - else: - abis = ["none", "abi3", "cp311"] - - for abi_tag in abis: - for platform_tag, kwargs in { - "any": {}, - "macosx_11_0_arm64": { - "osx_versions": [(11, 0)], - "platform_config_settings": { - "osx_aarch64": [ - "@platforms//cpu:aarch64", - "@platforms//os:osx", - ], - }, - }, - "manylinux_2_17_x86_64": { - "glibc_versions": [(2, 17), (2, 18)], - "platform_config_settings": { - "linux_x86_64": [ - "@platforms//cpu:x86_64", - "@platforms//os:linux", - ], - }, - }, - "manylinux_2_18_x86_64": { - "glibc_versions": [(2, 17), (2, 18)], - "platform_config_settings": { - "linux_x86_64": [ - "@platforms//cpu:x86_64", - "@platforms//os:linux", - ], - }, - }, - "musllinux_1_1_aarch64": { - "muslc_versions": [(1, 2), (1, 1), (1, 0)], - "platform_config_settings": { - "linux_aarch64": [ - "@platforms//cpu:aarch64", - "@platforms//os:linux", - ], - }, - }, - }.items(): - aliases = { - whl_config_setting( - filename = "foo-0.0.1-{}-{}-{}.whl".format(py_tag, abi_tag, platform_tag), - version = "3.11", - ): "repo", - } - available_config_settings = [] - config_settings( - python_versions = ["3.11"], - native = struct( - alias = _mock_alias(available_config_settings), - config_setting = _mock_config_setting([]), - ), - selects = struct( - config_setting_group = _mock_config_setting_group(available_config_settings), - ), - **kwargs - ) - - got_aliases = multiplatform_whl_aliases( - aliases = aliases, - glibc_versions = kwargs.get("glibc_versions", []), - muslc_versions = kwargs.get("muslc_versions", []), - osx_versions = kwargs.get("osx_versions", []), - ) - got = [a.partition(":")[-1] for a in got_aliases] - - env.expect.that_collection(available_config_settings).contains_at_least(got) - -_tests.append(_test_config_settings_exist) - def pkg_aliases_test_suite(name): """Create the test suite. diff --git a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl index ad7f36aed6..9114f59279 100644 --- a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -15,10 +15,6 @@ """render_pkg_aliases tests""" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load( - "//python/private/pypi:pkg_aliases.bzl", - "get_filename_config_settings", -) # buildifier: disable=bzl-visibility load( "//python/private/pypi:render_pkg_aliases.bzl", "get_whl_flag_versions", @@ -70,26 +66,13 @@ def _test_bzlmod_aliases(env): whl_config_setting( # Add one with micro version to mimic construction in the extension version = "3.2.2", - config_setting = "//:my_config_setting", ): "pypi_32_bar_baz", whl_config_setting( version = "3.2", - config_setting = "//:my_config_setting", target_platforms = [ "cp32_linux_x86_64", ], ): "pypi_32_bar_baz_linux_x86_64", - whl_config_setting( - version = "3.2", - filename = "foo-0.0.0-py3-none-any.whl", - ): "filename_repo", - whl_config_setting( - version = "3.2.2", - filename = "foo-0.0.0-py3-none-any.whl", - target_platforms = [ - "cp32.2_linux_x86_64", - ], - ): "filename_repo_linux_x86_64", }, }, extra_hub_aliases = {"bar_baz": ["foo"]}, @@ -111,21 +94,11 @@ package(default_visibility = ["//visibility:public"]) pkg_aliases( name = "bar_baz", actual = { - "//:my_config_setting": "pypi_32_bar_baz", + "//_config:is_cp322": "pypi_32_bar_baz", whl_config_setting( target_platforms = ("cp32_linux_x86_64",), - config_setting = "//:my_config_setting", version = "3.2", ): "pypi_32_bar_baz_linux_x86_64", - whl_config_setting( - filename = "foo-0.0.0-py3-none-any.whl", - version = "3.2", - ): "filename_repo", - whl_config_setting( - filename = "foo-0.0.0-py3-none-any.whl", - target_platforms = ("cp32_linux_x86_64",), - version = "3.2.2", - ): "filename_repo_linux_x86_64", }, extra_aliases = ["foo"], )""" @@ -256,252 +229,24 @@ def _test_get_python_versions_with_target_platforms(env): _tests.append(_test_get_python_versions_with_target_platforms) -def _test_get_python_versions_from_filenames(env): - got = get_whl_flag_versions( - settings = [ - whl_config_setting( - version = "3.3", - filename = "foo-0.0.0-py3-none-" + plat + ".whl", - ) - for plat in [ - "linux_x86_64", - "manylinux_2_17_x86_64", - "manylinux_2_14_aarch64.musllinux_1_1_aarch64", - "musllinux_1_0_x86_64", - "manylinux2014_x86_64.manylinux_2_17_x86_64", - "macosx_11_0_arm64", - "macosx_10_9_x86_64", - "macosx_10_9_universal2", - "windows_x86_64", - ] - ], - ) - want = { - "glibc_versions": [(2, 14), (2, 17)], - "muslc_versions": [(1, 0), (1, 1)], - "osx_versions": [(10, 9), (11, 0)], - "python_versions": ["3.3"], - "target_platforms": [ - "linux_aarch64", - "linux_x86_64", - "osx_aarch64", - "osx_x86_64", - "windows_x86_64", - ], - } - env.expect.that_dict(got).contains_exactly(want) - -_tests.append(_test_get_python_versions_from_filenames) - def _test_get_flag_versions_from_alias_target_platforms(env): got = get_whl_flag_versions( settings = [ + whl_config_setting(version = "3.3"), whl_config_setting( version = "3.3", - filename = "foo-0.0.0-py3-none-" + plat + ".whl", - ) - for plat in [ - "windows_x86_64", - ] - ] + [ - whl_config_setting( - version = "3.3", - filename = "foo-0.0.0-py3-none-any.whl", - target_platforms = [ - "cp33_linux_x86_64", - ], + target_platforms = ["cp33_linux_x86_64"], ), ], ) want = { "python_versions": ["3.3"], - "target_platforms": [ - "linux_x86_64", - "windows_x86_64", - ], + "target_platforms": ["linux_x86_64"], } env.expect.that_dict(got).contains_exactly(want) _tests.append(_test_get_flag_versions_from_alias_target_platforms) -def _test_config_settings( - env, - *, - filename, - want, - python_version, - want_versions = {}, - target_platforms = [], - glibc_versions = [], - muslc_versions = [], - osx_versions = []): - got, got_default_version_settings = get_filename_config_settings( - filename = filename, - target_platforms = target_platforms, - glibc_versions = glibc_versions, - muslc_versions = muslc_versions, - osx_versions = osx_versions, - python_version = python_version, - ) - env.expect.that_collection(got).contains_exactly(want) - env.expect.that_dict(got_default_version_settings).contains_exactly(want_versions) - -def _test_sdist(env): - # Do the first test for multiple extensions - for ext in [".tar.gz", ".zip"]: - _test_config_settings( - env, - filename = "foo-0.0.1" + ext, - python_version = "3.2", - want = [":is_cp32_sdist"], - ) - - ext = ".zip" - _test_config_settings( - env, - filename = "foo-0.0.1" + ext, - python_version = "3.2", - target_platforms = [ - "linux_aarch64", - "linux_x86_64", - ], - want = [ - ":is_cp32_sdist_linux_aarch64", - ":is_cp32_sdist_linux_x86_64", - ], - ) - -_tests.append(_test_sdist) - -def _test_py2_py3_none_any(env): - _test_config_settings( - env, - filename = "foo-0.0.1-py2.py3-none-any.whl", - python_version = "3.2", - want = [ - ":is_cp32_py_none_any", - ], - ) - - _test_config_settings( - env, - filename = "foo-0.0.1-py2.py3-none-any.whl", - python_version = "3.2", - target_platforms = [ - "osx_x86_64", - ], - want = [":is_cp32_py_none_any_osx_x86_64"], - ) - -_tests.append(_test_py2_py3_none_any) - -def _test_py3_none_any(env): - _test_config_settings( - env, - filename = "foo-0.0.1-py3-none-any.whl", - python_version = "3.1", - want = [":is_cp31_py3_none_any"], - ) - - _test_config_settings( - env, - filename = "foo-0.0.1-py3-none-any.whl", - python_version = "3.1", - target_platforms = ["linux_x86_64"], - want = [":is_cp31_py3_none_any_linux_x86_64"], - ) - -_tests.append(_test_py3_none_any) - -def _test_py3_none_macosx_10_9_universal2(env): - _test_config_settings( - env, - filename = "foo-0.0.1-py3-none-macosx_10_9_universal2.whl", - python_version = "3.1", - osx_versions = [ - (10, 9), - (11, 0), - ], - want = [], - want_versions = { - ":is_cp31_py3_none_osx_universal2": { - (10, 9): ":is_cp31_py3_none_osx_10_9_universal2", - (11, 0): ":is_cp31_py3_none_osx_11_0_universal2", - }, - }, - ) - -_tests.append(_test_py3_none_macosx_10_9_universal2) - -def _test_cp37_abi3_linux_x86_64(env): - _test_config_settings( - env, - filename = "foo-0.0.1-cp37-abi3-linux_x86_64.whl", - python_version = "3.7", - want = [":is_cp37_abi3_linux_x86_64"], - ) - -_tests.append(_test_cp37_abi3_linux_x86_64) - -def _test_cp37_abi3_windows_x86_64(env): - _test_config_settings( - env, - filename = "foo-0.0.1-cp37-abi3-windows_x86_64.whl", - python_version = "3.7", - want = [":is_cp37_abi3_windows_x86_64"], - ) - -_tests.append(_test_cp37_abi3_windows_x86_64) - -def _test_cp37_abi3_manylinux_2_17_x86_64(env): - _test_config_settings( - env, - filename = "foo-0.0.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", - python_version = "3.7", - glibc_versions = [ - (2, 16), - (2, 17), - (2, 18), - ], - want = [], - want_versions = { - ":is_cp37_abi3_manylinux_x86_64": { - (2, 17): ":is_cp37_abi3_manylinux_2_17_x86_64", - (2, 18): ":is_cp37_abi3_manylinux_2_18_x86_64", - }, - }, - ) - -_tests.append(_test_cp37_abi3_manylinux_2_17_x86_64) - -def _test_cp37_abi3_manylinux_2_17_musllinux_1_1_aarch64(env): - # I've seen such a wheel being built for `uv` - _test_config_settings( - env, - filename = "foo-0.0.1-cp37-cp37-manylinux_2_17_arm64.musllinux_1_1_arm64.whl", - python_version = "3.7", - glibc_versions = [ - (2, 16), - (2, 17), - (2, 18), - ], - muslc_versions = [ - (1, 1), - ], - want = [], - want_versions = { - ":is_cp37_cp37_manylinux_aarch64": { - (2, 17): ":is_cp37_cp37_manylinux_2_17_aarch64", - (2, 18): ":is_cp37_cp37_manylinux_2_18_aarch64", - }, - ":is_cp37_cp37_musllinux_aarch64": { - (1, 1): ":is_cp37_cp37_musllinux_1_1_aarch64", - }, - }, - ) - -_tests.append(_test_cp37_abi3_manylinux_2_17_musllinux_1_1_aarch64) - def render_pkg_aliases_test_suite(name): """Create the test suite. From e09a6f434ab6634a857b5303d088246ad0f940c2 Mon Sep 17 00:00:00 2001 From: Jason Bedard Date: Sat, 16 Aug 2025 20:28:03 -0700 Subject: [PATCH 6/6] deps(gazelle): upgrade rules_go to remove patching of tree-sitter (#3179) Update rules_go to include https://github.com/bazel-contrib/rules_go/pull/4298 Update gazelle to align with the version the rules_go bzlmod will bring in, and ensure the go.mod version is the same as bzlmod version. Update go to 1.21 to include the `slices` library that some of the go.mod updates depend on. Fixes #2956. --------- Co-authored-by: Douglas Thor --- CHANGELOG.md | 3 + examples/build_file_generation/WORKSPACE | 14 +-- .../bzlmod_build_file_generation/MODULE.bazel | 2 +- gazelle/MODULE.bazel | 15 +-- gazelle/WORKSPACE | 14 +-- gazelle/deps.bzl | 109 ++++++++++-------- gazelle/go.mod | 18 +-- gazelle/go.sum | 32 ++--- gazelle/internal/smacker_BUILD.bazel | 80 ------------- gazelle/python/BUILD.bazel | 6 +- .../testdata/respect_kind_mapping/BUILD.out | 2 +- 11 files changed, 109 insertions(+), 186 deletions(-) delete mode 100644 gazelle/internal/smacker_BUILD.bazel diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e8ad65a5e..0fa5e373ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,9 @@ END_UNRELEASED_TEMPLATE {#v0-0-0-changed} ### Changed +* (gazelle) update minimum gazelle version to 0.36.0 - may cause BUILD file changes +* (gazelle) update minimum rules_go version to 0.55.1 +* (gazelle) remove custom go-tree-sitter module BUILD file * (gazelle) For package mode, resolve dependencies when imports are relative to the package path. This is enabled via the `# gazelle:python_experimental_allow_relative_imports` true directive ({gh-issue}`2203`). diff --git a/examples/build_file_generation/WORKSPACE b/examples/build_file_generation/WORKSPACE index 6681ad6861..27f6ec071c 100644 --- a/examples/build_file_generation/WORKSPACE +++ b/examples/build_file_generation/WORKSPACE @@ -20,20 +20,20 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") http_archive( name = "io_bazel_rules_go", - sha256 = "278b7ff5a826f3dc10f04feaf0b70d48b68748ccd512d7f98bf442077f043fe3", + sha256 = "9d72f7b8904128afb98d46bbef82ad7223ec9ff3718d419afb355fddd9f9484a", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.41.0/rules_go-v0.41.0.zip", - "https://github.com/bazelbuild/rules_go/releases/download/v0.41.0/rules_go-v0.41.0.zip", + "https://mirror.bazel.build/github.com/bazel-contrib/rules_go/releases/download/v0.55.1/rules_go-v0.55.1.zip", + "https://github.com/bazel-contrib/rules_go/releases/download/v0.55.1/rules_go-v0.55.1.zip", ], ) # Download the bazel_gazelle ruleset. http_archive( name = "bazel_gazelle", - sha256 = "d3fa66a39028e97d76f9e2db8f1b0c11c099e8e01bf363a923074784e451f809", + sha256 = "75df288c4b31c81eb50f51e2e14f4763cb7548daae126817247064637fd9ea62", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.33.0/bazel-gazelle-v0.33.0.tar.gz", - "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.33.0/bazel-gazelle-v0.33.0.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.36.0/bazel-gazelle-v0.36.0.tar.gz", + "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.36.0/bazel-gazelle-v0.36.0.tar.gz", ], ) @@ -49,7 +49,7 @@ go_rules_dependencies() # go_rules_dependencies is a function that registers external dependencies # needed by the Go rules. # See: https://github.com/bazelbuild/rules_go/blob/master/go/dependencies.rst#go_rules_dependencies -go_register_toolchains(version = "1.19.4") +go_register_toolchains(version = "1.21.13") # The following call configured the gazelle dependencies, Go environment and Go SDK. gazelle_dependencies() diff --git a/examples/bzlmod_build_file_generation/MODULE.bazel b/examples/bzlmod_build_file_generation/MODULE.bazel index b9b428d365..3436fbf0af 100644 --- a/examples/bzlmod_build_file_generation/MODULE.bazel +++ b/examples/bzlmod_build_file_generation/MODULE.bazel @@ -38,7 +38,7 @@ local_path_override( # The following stanza defines the dependency for gazelle # See here https://github.com/bazelbuild/bazel-gazelle/releases/ for the # latest version. -bazel_dep(name = "gazelle", version = "0.30.0", repo_name = "bazel_gazelle") +bazel_dep(name = "gazelle", version = "0.36.0", repo_name = "bazel_gazelle") # The following stanze returns a proxy object representing a module extension; # its methods can be invoked to create module extension tags. diff --git a/gazelle/MODULE.bazel b/gazelle/MODULE.bazel index 51352a0ba6..1560e73d7b 100644 --- a/gazelle/MODULE.bazel +++ b/gazelle/MODULE.bazel @@ -6,8 +6,8 @@ module( bazel_dep(name = "bazel_skylib", version = "1.6.1") bazel_dep(name = "rules_python", version = "0.18.0") -bazel_dep(name = "rules_go", version = "0.41.0", repo_name = "io_bazel_rules_go") -bazel_dep(name = "gazelle", version = "0.33.0", repo_name = "bazel_gazelle") +bazel_dep(name = "rules_go", version = "0.55.1", repo_name = "io_bazel_rules_go") +bazel_dep(name = "gazelle", version = "0.36.0", repo_name = "bazel_gazelle") bazel_dep(name = "rules_cc", version = "0.0.16") local_path_override( @@ -23,21 +23,12 @@ use_repo( "com_github_bmatcuk_doublestar_v4", "com_github_emirpasic_gods", "com_github_ghodss_yaml", + "com_github_smacker_go_tree_sitter", "com_github_stretchr_testify", "in_gopkg_yaml_v2", "org_golang_x_sync", ) -http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -http_archive( - name = "com_github_smacker_go_tree_sitter", - build_file = "//:internal/smacker_BUILD.bazel", - integrity = "sha256-4AkDY4Rh5Auu9Kwzhj5XYSirMLlhmd6ClMWo/r0kmu4=", - strip_prefix = "go-tree-sitter-dd81d9e9be82a8cac96ed1d50c7389c5f1997c02", - url = "https://github.com/smacker/go-tree-sitter/archive/dd81d9e9be82a8cac96ed1d50c7389c5f1997c02.zip", -) - python_stdlib_list = use_extension("//python:extensions.bzl", "python_stdlib_list") use_repo( python_stdlib_list, diff --git a/gazelle/WORKSPACE b/gazelle/WORKSPACE index ad428b10cd..ec0532c3f6 100644 --- a/gazelle/WORKSPACE +++ b/gazelle/WORKSPACE @@ -4,19 +4,19 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") http_archive( name = "io_bazel_rules_go", - sha256 = "278b7ff5a826f3dc10f04feaf0b70d48b68748ccd512d7f98bf442077f043fe3", + sha256 = "9d72f7b8904128afb98d46bbef82ad7223ec9ff3718d419afb355fddd9f9484a", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.41.0/rules_go-v0.41.0.zip", - "https://github.com/bazelbuild/rules_go/releases/download/v0.41.0/rules_go-v0.41.0.zip", + "https://mirror.bazel.build/github.com/bazel-contrib/rules_go/releases/download/v0.55.1/rules_go-v0.55.1.zip", + "https://github.com/bazel-contrib/rules_go/releases/download/v0.55.1/rules_go-v0.55.1.zip", ], ) http_archive( name = "bazel_gazelle", - sha256 = "29d5dafc2a5582995488c6735115d1d366fcd6a0fc2e2a153f02988706349825", + sha256 = "75df288c4b31c81eb50f51e2e14f4763cb7548daae126817247064637fd9ea62", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.31.0/bazel-gazelle-v0.31.0.tar.gz", - "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.31.0/bazel-gazelle-v0.31.0.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.36.0/bazel-gazelle-v0.36.0.tar.gz", + "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.36.0/bazel-gazelle-v0.36.0.tar.gz", ], ) @@ -25,7 +25,7 @@ load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_depe go_rules_dependencies() -go_register_toolchains(version = "1.19.4") +go_register_toolchains(version = "1.21.13") gazelle_dependencies() diff --git a/gazelle/deps.bzl b/gazelle/deps.bzl index 8c4c055e9b..a7b5990c3b 100644 --- a/gazelle/deps.bzl +++ b/gazelle/deps.bzl @@ -46,31 +46,28 @@ def go_deps(): go_repository( name = "com_github_bazelbuild_bazel_gazelle", importpath = "github.com/bazelbuild/bazel-gazelle", - sum = "h1:ROyUyUHzoEdvoOs1e0haxJx1l5EjZX6AOqiKdVlaBbg=", - version = "v0.31.1", + sum = "h1:n41ODckCkU9D2BEwBxYN+xu5E92Vd0gaW6QmsIW9l00=", + version = "v0.36.0", ) - go_repository( name = "com_github_bazelbuild_buildtools", build_naming_convention = "go_default_library", importpath = "github.com/bazelbuild/buildtools", - sum = "h1:HTepWP/jhtWTC1gvK0RnvKCgjh4gLqiwaOwGozAXcbw=", - version = "v0.0.0-20231103205921-433ea8554e82", + sum = "h1:VNqmvOfFzn2Hrtoni8vqgXlIQ4C2Zt22fxeZ9gOOkp0=", + version = "v0.0.0-20240313121412-66c605173954", ) go_repository( name = "com_github_bazelbuild_rules_go", importpath = "github.com/bazelbuild/rules_go", - sum = "h1:JzlRxsFNhlX+g4drDRPhIaU5H5LnI978wdMJ0vK4I+k=", - version = "v0.41.0", + sum = "h1:cQYGcunY8myOB+0Ym6PGQRhc/milkRcNv0my3XgxaDU=", + version = "v0.55.1", ) - go_repository( name = "com_github_bmatcuk_doublestar_v4", importpath = "github.com/bmatcuk/doublestar/v4", sum = "h1:fdDeAqgT47acgwd9bd9HxJRDmc9UAmPpc+2m0CXv75Q=", version = "v4.7.1", ) - go_repository( name = "com_github_burntsushi_toml", importpath = "github.com/BurntSushi/toml", @@ -134,16 +131,21 @@ def go_deps(): go_repository( name = "com_github_fsnotify_fsnotify", importpath = "github.com/fsnotify/fsnotify", - sum = "h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=", - version = "v1.6.0", + sum = "h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=", + version = "v1.7.0", ) - go_repository( name = "com_github_ghodss_yaml", importpath = "github.com/ghodss/yaml", sum = "h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=", version = "v1.0.0", ) + go_repository( + name = "com_github_gogo_protobuf", + importpath = "github.com/gogo/protobuf", + sum = "h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=", + version = "v1.3.2", + ) go_repository( name = "com_github_golang_glog", importpath = "github.com/golang/glog", @@ -153,20 +155,20 @@ def go_deps(): go_repository( name = "com_github_golang_mock", importpath = "github.com/golang/mock", - sum = "h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=", - version = "v1.6.0", + sum = "h1:YojYx61/OLFsiv6Rw1Z96LpldJIy31o+UHmwAUMJ6/U=", + version = "v1.7.0-rc.1", ) go_repository( name = "com_github_golang_protobuf", importpath = "github.com/golang/protobuf", - sum = "h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=", - version = "v1.5.2", + sum = "h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=", + version = "v1.5.4", ) go_repository( name = "com_github_google_go_cmp", importpath = "github.com/google/go-cmp", - sum = "h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=", - version = "v0.5.9", + sum = "h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=", + version = "v0.6.0", ) go_repository( name = "com_github_pmezard_go_difflib", @@ -180,12 +182,11 @@ def go_deps(): sum = "h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=", version = "v0.0.0-20190812154241-14fe0d1b01d4", ) - http_archive( + go_repository( name = "com_github_smacker_go_tree_sitter", - build_file = Label("//:internal/smacker_BUILD.bazel"), - integrity = "sha256-4AkDY4Rh5Auu9Kwzhj5XYSirMLlhmd6ClMWo/r0kmu4=", - strip_prefix = "go-tree-sitter-dd81d9e9be82a8cac96ed1d50c7389c5f1997c02", - url = "https://github.com/smacker/go-tree-sitter/archive/dd81d9e9be82a8cac96ed1d50c7389c5f1997c02.zip", + importpath = "github.com/smacker/go-tree-sitter", + sum = "h1:6C8qej6f1bStuePVkLSFxoU22XBS165D3klxlzRg8F4=", + version = "v0.0.0-20240827094217-dd81d9e9be82", ) go_repository( name = "com_github_stretchr_objx", @@ -199,13 +200,6 @@ def go_deps(): sum = "h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=", version = "v1.9.0", ) - - go_repository( - name = "com_github_yuin_goldmark", - importpath = "github.com/yuin/goldmark", - sum = "h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE=", - version = "v1.4.13", - ) go_repository( name = "com_google_cloud_go", importpath = "cloud.google.com/go", @@ -230,7 +224,6 @@ def go_deps(): sum = "h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=", version = "v3.0.1", ) - go_repository( name = "net_starlark_go", importpath = "go.starlark.net", @@ -246,20 +239,32 @@ def go_deps(): go_repository( name = "org_golang_google_genproto", importpath = "google.golang.org/genproto", - sum = "h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=", - version = "v0.0.0-20200526211855-cb27e3aa2013", + sum = "h1:387Y+JbxF52bmesc8kq1NyYIp33dnxCw6eiA7JMsTmw=", + version = "v0.0.0-20250115164207-1a7da9e5054f", + ) + go_repository( + name = "org_golang_google_genproto_googleapis_rpc", + importpath = "google.golang.org/genproto/googleapis/rpc", + sum = "h1:3UsHvIr4Wc2aW4brOaSCmcxh9ksica6fHEr8P1XhkYw=", + version = "v0.0.0-20250106144421-5f5ef82da422", ) go_repository( name = "org_golang_google_grpc", importpath = "google.golang.org/grpc", - sum = "h1:fPVVDxY9w++VjTZsYvXWqEf9Rqar/e+9zYfxKK+W+YU=", - version = "v1.50.0", + sum = "h1:OgPcDAFKHnH8X3O4WcO4XUc8GRDeKsKReqbQtiCj7N8=", + version = "v1.67.3", + ) + go_repository( + name = "org_golang_google_grpc_cmd_protoc_gen_go_grpc", + importpath = "google.golang.org/grpc/cmd/protoc-gen-go-grpc", + sum = "h1:F29+wU6Ee6qgu9TddPgooOdaqsxTMunOoj8KA5yuS5A=", + version = "v1.5.1", ) go_repository( name = "org_golang_google_protobuf", importpath = "google.golang.org/protobuf", - sum = "h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=", - version = "v1.28.0", + sum = "h1:82DV7MYdb8anAVi3qge1wSnMDrnKK7ebr+I0hHRN1BU=", + version = "v1.36.3", ) go_repository( name = "org_golang_x_crypto", @@ -282,14 +287,14 @@ def go_deps(): go_repository( name = "org_golang_x_mod", importpath = "golang.org/x/mod", - sum = "h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=", - version = "v0.10.0", + sum = "h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM=", + version = "v0.23.0", ) go_repository( name = "org_golang_x_net", importpath = "golang.org/x/net", - sum = "h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=", - version = "v0.10.0", + sum = "h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8=", + version = "v0.35.0", ) go_repository( name = "org_golang_x_oauth2", @@ -300,20 +305,20 @@ def go_deps(): go_repository( name = "org_golang_x_sync", importpath = "golang.org/x/sync", - sum = "h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=", - version = "v0.2.0", + sum = "h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=", + version = "v0.11.0", ) go_repository( name = "org_golang_x_sys", importpath = "golang.org/x/sys", - sum = "h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=", - version = "v0.8.0", + sum = "h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=", + version = "v0.30.0", ) go_repository( name = "org_golang_x_text", importpath = "golang.org/x/text", - sum = "h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=", - version = "v0.3.3", + sum = "h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=", + version = "v0.22.0", ) go_repository( name = "org_golang_x_tools", @@ -321,8 +326,14 @@ def go_deps(): "gazelle:exclude **/testdata/**/*", ], importpath = "golang.org/x/tools", - sum = "h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo=", - version = "v0.9.1", + sum = "h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY=", + version = "v0.30.0", + ) + go_repository( + name = "org_golang_x_tools_go_vcs", + importpath = "golang.org/x/tools/go/vcs", + sum = "h1:cOIJqWBl99H1dH5LWizPa+0ImeeJq3t3cJjaeOWUAL4=", + version = "v0.1.0-deprecated", ) go_repository( name = "org_golang_x_xerrors", diff --git a/gazelle/go.mod b/gazelle/go.mod index 6f65ffbc7e..7623079af9 100644 --- a/gazelle/go.mod +++ b/gazelle/go.mod @@ -1,26 +1,26 @@ module github.com/bazel-contrib/rules_python/gazelle -go 1.19 +go 1.21.13 require ( - github.com/bazelbuild/bazel-gazelle v0.31.1 - github.com/bazelbuild/buildtools v0.0.0-20231103205921-433ea8554e82 - github.com/bazelbuild/rules_go v0.41.0 + github.com/bazelbuild/bazel-gazelle v0.36.0 + github.com/bazelbuild/buildtools v0.0.0-20240313121412-66c605173954 + github.com/bazelbuild/rules_go v0.55.1 github.com/bmatcuk/doublestar/v4 v4.7.1 github.com/emirpasic/gods v1.18.1 github.com/ghodss/yaml v1.0.0 github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82 github.com/stretchr/testify v1.9.0 - golang.org/x/sync v0.2.0 + golang.org/x/sync v0.11.0 gopkg.in/yaml.v2 v2.4.0 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect - github.com/google/go-cmp v0.5.9 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/mod v0.10.0 // indirect - golang.org/x/sys v0.8.0 // indirect - golang.org/x/tools v0.9.1 // indirect + golang.org/x/mod v0.23.0 // indirect + golang.org/x/sys v0.30.0 // indirect + golang.org/x/tools/go/vcs v0.1.0-deprecated // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/gazelle/go.sum b/gazelle/go.sum index 0aaa186620..5a4d42d46a 100644 --- a/gazelle/go.sum +++ b/gazelle/go.sum @@ -1,11 +1,11 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/bazelbuild/bazel-gazelle v0.31.1 h1:ROyUyUHzoEdvoOs1e0haxJx1l5EjZX6AOqiKdVlaBbg= -github.com/bazelbuild/bazel-gazelle v0.31.1/go.mod h1:Ul0pqz50f5wxz0QNzsZ+mrEu4AVAVJZEB5xLnHgIG9c= -github.com/bazelbuild/buildtools v0.0.0-20231103205921-433ea8554e82 h1:HTepWP/jhtWTC1gvK0RnvKCgjh4gLqiwaOwGozAXcbw= -github.com/bazelbuild/buildtools v0.0.0-20231103205921-433ea8554e82/go.mod h1:689QdV3hBP7Vo9dJMmzhoYIyo/9iMhEmHkJcnaPRCbo= -github.com/bazelbuild/rules_go v0.41.0 h1:JzlRxsFNhlX+g4drDRPhIaU5H5LnI978wdMJ0vK4I+k= -github.com/bazelbuild/rules_go v0.41.0/go.mod h1:TMHmtfpvyfsxaqfL9WnahCsXMWDMICTw7XeK9yVb+YU= +github.com/bazelbuild/bazel-gazelle v0.36.0 h1:n41ODckCkU9D2BEwBxYN+xu5E92Vd0gaW6QmsIW9l00= +github.com/bazelbuild/bazel-gazelle v0.36.0/go.mod h1:5wGHbkRpDUdz4LxREtPYwXstrWfnkV+oDmOuxNAxW1s= +github.com/bazelbuild/buildtools v0.0.0-20240313121412-66c605173954 h1:VNqmvOfFzn2Hrtoni8vqgXlIQ4C2Zt22fxeZ9gOOkp0= +github.com/bazelbuild/buildtools v0.0.0-20240313121412-66c605173954/go.mod h1:689QdV3hBP7Vo9dJMmzhoYIyo/9iMhEmHkJcnaPRCbo= +github.com/bazelbuild/rules_go v0.55.1 h1:cQYGcunY8myOB+0Ym6PGQRhc/milkRcNv0my3XgxaDU= +github.com/bazelbuild/rules_go v0.55.1/go.mod h1:T90Gpyq4HDFlsrvtQa2CBdHNJ2P4rAu/uUTmQbanzf0= github.com/bmatcuk/doublestar/v4 v4.7.1 h1:fdDeAqgT47acgwd9bd9HxJRDmc9UAmPpc+2m0CXv75Q= github.com/bmatcuk/doublestar/v4 v4.7.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -38,8 +38,8 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -53,8 +53,8 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= -golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM= +golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -63,20 +63,20 @@ golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAG golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= -golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools/go/vcs v0.1.0-deprecated h1:cOIJqWBl99H1dH5LWizPa+0ImeeJq3t3cJjaeOWUAL4= +golang.org/x/tools/go/vcs v0.1.0-deprecated/go.mod h1:zUrvATBAvEI9535oC0yWYsLsHIV4Z7g63sNPVMtuBy8= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= diff --git a/gazelle/internal/smacker_BUILD.bazel b/gazelle/internal/smacker_BUILD.bazel deleted file mode 100644 index 3ec96760e8..0000000000 --- a/gazelle/internal/smacker_BUILD.bazel +++ /dev/null @@ -1,80 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -filegroup( - name = "common_libs", - srcs = [ - "alloc.h", - "api.h", - "array.h", - ], - visibility = [":__subpackages__"], -) - -go_library( - name = "go-tree-sitter", - srcs = [ - "alloc.c", - "alloc.h", - "api.h", - "array.h", - "atomic.h", - "bindings.c", - "bindings.go", - "bindings.h", - "bits.h", - "clock.h", - "error_costs.h", - "get_changed_ranges.c", - "get_changed_ranges.h", - "host.h", - "iter.go", - "language.c", - "language.h", - "length.h", - "lexer.c", - "lexer.h", - "node.c", - "parser.c", - "parser.h", - "point.h", - "ptypes.h", - "query.c", - "reduce_action.h", - "reusable_node.h", - "stack.c", - "stack.h", - "subtree.c", - "subtree.h", - "test_grammar.go", - "tree.c", - "tree.h", - "tree_cursor.c", - "tree_cursor.h", - "umachine.h", - "unicode.h", - "urename.h", - "utf.h", - "utf16.h", - "utf8.h", - "wasm_store.c", - "wasm_store.h", - ], - cgo = True, - importpath = "github.com/smacker/go-tree-sitter", - visibility = ["//visibility:public"], -) - -go_library( - name = "python", - srcs = [ - "python/binding.go", - "python/parser.c", - "python/parser.h", - "python/scanner.c", - ":common_libs", - ], - cgo = True, - importpath = "github.com/smacker/go-tree-sitter/python", - visibility = ["//visibility:public"], - deps = [":go-tree-sitter"], -) diff --git a/gazelle/python/BUILD.bazel b/gazelle/python/BUILD.bazel index 1a7c54f4b2..b6ca8adef5 100644 --- a/gazelle/python/BUILD.bazel +++ b/gazelle/python/BUILD.bazel @@ -29,22 +29,20 @@ go_library( importpath = "github.com/bazel-contrib/rules_python/gazelle/python", visibility = ["//visibility:public"], deps = [ - "//manifest", "//pythonconfig", "@bazel_gazelle//config:go_default_library", "@bazel_gazelle//label:go_default_library", "@bazel_gazelle//language:go_default_library", - "@bazel_gazelle//language/proto:go_default_library", "@bazel_gazelle//repo:go_default_library", "@bazel_gazelle//resolve:go_default_library", "@bazel_gazelle//rule:go_default_library", - "@com_github_bazelbuild_buildtools//build:go_default_library", + "@com_github_bazelbuild_buildtools//build", "@com_github_bmatcuk_doublestar_v4//:doublestar", "@com_github_emirpasic_gods//lists/singlylinkedlist", "@com_github_emirpasic_gods//sets/treeset", "@com_github_emirpasic_gods//utils", "@com_github_smacker_go_tree_sitter//:go-tree-sitter", - "@com_github_smacker_go_tree_sitter//:python", + "@com_github_smacker_go_tree_sitter//python", "@org_golang_x_sync//errgroup", ], ) diff --git a/gazelle/python/testdata/respect_kind_mapping/BUILD.out b/gazelle/python/testdata/respect_kind_mapping/BUILD.out index 7c5fb0bd20..fa06e2af12 100644 --- a/gazelle/python/testdata/respect_kind_mapping/BUILD.out +++ b/gazelle/python/testdata/respect_kind_mapping/BUILD.out @@ -1,5 +1,5 @@ -load(":mytest.bzl", "my_test") load("@rules_python//python:defs.bzl", "py_library") +load(":mytest.bzl", "my_test") # gazelle:map_kind py_test my_test :mytest.bzl