From a44299774d01126aa772d933954939208c8bcc89 Mon Sep 17 00:00:00 2001 From: thesayyn Date: Thu, 23 Nov 2023 13:03:52 -0800 Subject: [PATCH 01/10] refactor: toolchainize py_proto_library --- examples/py_proto_library/WORKSPACE | 31 ++++++++++++----------- internal_deps.bzl | 10 ++++---- python/private/proto/py_proto_library.bzl | 11 ++------ 3 files changed, 23 insertions(+), 29 deletions(-) diff --git a/examples/py_proto_library/WORKSPACE b/examples/py_proto_library/WORKSPACE index bf38112f98..90a348452a 100644 --- a/examples/py_proto_library/WORKSPACE +++ b/examples/py_proto_library/WORKSPACE @@ -22,27 +22,28 @@ python_register_toolchains( # Then we need to setup dependencies in order to use py_proto_library load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + http_archive( name = "rules_proto", - sha256 = "dc3fb206a2cb3441b485eb1e423165b231235a1ea9b031b4433cf7bc1fa460dd", - strip_prefix = "rules_proto-5.3.0-21.7", - urls = [ - "https://github.com/bazelbuild/rules_proto/archive/refs/tags/5.3.0-21.7.tar.gz", - ], + sha256 = "903af49528dc37ad2adbb744b317da520f133bc1cbbecbdd2a6c546c9ead080b", + strip_prefix = "rules_proto-6.0.0-rc0", + url = "https://github.com/bazelbuild/rules_proto/releases/download/6.0.0-rc0/rules_proto-6.0.0-rc0.tar.gz", ) + +load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") +rules_proto_dependencies() +rules_proto_toolchains() + + http_archive( name = "com_google_protobuf", - sha256 = "75be42bd736f4df6d702a0e4e4d30de9ee40eac024c4b845d17ae4cc831fe4ae", - strip_prefix = "protobuf-21.7", - urls = [ - "https://mirror.bazel.build/github.com/protocolbuffers/protobuf/archive/v21.7.tar.gz", - "https://github.com/protocolbuffers/protobuf/archive/v21.7.tar.gz", - ], + # sha256 = "8632d7e8a6cbbd7e58f40e3f62fed77bff3df6e4dcac73498f0a68ba5a38d892", + strip_prefix = "protobuf-d1c5e885b7b7d3ac888e89cf4d45cb0c9fa6b4ed", + urls = ["https://github.com/protocolbuffers/protobuf/archive/d1c5e885b7b7d3ac888e89cf4d45cb0c9fa6b4ed.tar.gz"], ) -load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") +load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps", "protobuf_register_toolchains") +protobuf_deps() +protobuf_register_toolchains() -rules_proto_dependencies() - -rules_proto_toolchains() diff --git a/internal_deps.bzl b/internal_deps.bzl index 9931933396..e704d9fa61 100644 --- a/internal_deps.bzl +++ b/internal_deps.bzl @@ -173,15 +173,15 @@ def rules_python_internal_deps(): ], ) + http_archive( name = "rules_proto", - sha256 = "dc3fb206a2cb3441b485eb1e423165b231235a1ea9b031b4433cf7bc1fa460dd", - strip_prefix = "rules_proto-5.3.0-21.7", - urls = [ - "https://github.com/bazelbuild/rules_proto/archive/refs/tags/5.3.0-21.7.tar.gz", - ], + sha256 = "903af49528dc37ad2adbb744b317da520f133bc1cbbecbdd2a6c546c9ead080b", + strip_prefix = "rules_proto-6.0.0-rc0", + url = "https://github.com/bazelbuild/rules_proto/releases/download/6.0.0-rc0/rules_proto-6.0.0-rc0.tar.gz", ) + http_archive( name = "com_google_protobuf", sha256 = "75be42bd736f4df6d702a0e4e4d30de9ee40eac024c4b845d17ae4cc831fe4ae", diff --git a/python/private/proto/py_proto_library.bzl b/python/private/proto/py_proto_library.bzl index 91faa2dc60..18f558dae7 100644 --- a/python/private/proto/py_proto_library.bzl +++ b/python/private/proto/py_proto_library.bzl @@ -17,8 +17,6 @@ load("@rules_proto//proto:defs.bzl", "ProtoInfo", "proto_common") load("//python:defs.bzl", "PyInfo") -ProtoLangToolchainInfo = proto_common.ProtoLangToolchainInfo - _PyProtoInfo = provider( doc = "Encapsulates information needed by the Python proto rules.", fields = { @@ -51,7 +49,6 @@ def _py_proto_aspect_impl(target, ctx): ([_PyProtoInfo]) Providers collecting transitive information about generated files. """ - _proto_library = ctx.rule.attr # Check Proto file names @@ -61,7 +58,7 @@ def _py_proto_aspect_impl(target, ctx): proto.path, )) - proto_lang_toolchain_info = ctx.attr._aspect_proto_toolchain[ProtoLangToolchainInfo] + proto_lang_toolchain_info = ctx.toolchains["@rules_python//python/proto:toolchain_type"].proto api_deps = [proto_lang_toolchain_info.runtime] generated_sources = [] @@ -123,14 +120,10 @@ def _py_proto_aspect_impl(target, ctx): _py_proto_aspect = aspect( implementation = _py_proto_aspect_impl, - attrs = { - "_aspect_proto_toolchain": attr.label( - default = ":python_toolchain", - ), - }, attr_aspects = ["deps"], required_providers = [ProtoInfo], provides = [_PyProtoInfo], + toolchains = ["@rules_python//python/proto:toolchain_type"] ) def _py_proto_library_rule(ctx): From b33e7ea94f89e6ac12af2190f0aa306f81897ecd Mon Sep 17 00:00:00 2001 From: thesayyn Date: Mon, 4 Dec 2023 16:10:34 -0800 Subject: [PATCH 02/10] use toolchains helpers --- examples/py_proto_library/WORKSPACE | 7 +++---- internal_deps.bzl | 8 +++----- python/private/proto/py_proto_library.bzl | 12 ++++++++++-- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/examples/py_proto_library/WORKSPACE b/examples/py_proto_library/WORKSPACE index 90a348452a..47057f39ef 100644 --- a/examples/py_proto_library/WORKSPACE +++ b/examples/py_proto_library/WORKSPACE @@ -25,12 +25,11 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") http_archive( name = "rules_proto", - sha256 = "903af49528dc37ad2adbb744b317da520f133bc1cbbecbdd2a6c546c9ead080b", - strip_prefix = "rules_proto-6.0.0-rc0", - url = "https://github.com/bazelbuild/rules_proto/releases/download/6.0.0-rc0/rules_proto-6.0.0-rc0.tar.gz", + sha256 = "904a8097fae42a690c8e08d805210e40cccb069f5f9a0f6727cf4faa7bed2c9c", + strip_prefix = "rules_proto-6.0.0-rc1", + url = "https://github.com/bazelbuild/rules_proto/releases/download/6.0.0-rc1/rules_proto-6.0.0-rc1.tar.gz", ) - load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") rules_proto_dependencies() rules_proto_toolchains() diff --git a/internal_deps.bzl b/internal_deps.bzl index e704d9fa61..f6f2460a70 100644 --- a/internal_deps.bzl +++ b/internal_deps.bzl @@ -173,15 +173,13 @@ def rules_python_internal_deps(): ], ) - http_archive( name = "rules_proto", - sha256 = "903af49528dc37ad2adbb744b317da520f133bc1cbbecbdd2a6c546c9ead080b", - strip_prefix = "rules_proto-6.0.0-rc0", - url = "https://github.com/bazelbuild/rules_proto/releases/download/6.0.0-rc0/rules_proto-6.0.0-rc0.tar.gz", + sha256 = "904a8097fae42a690c8e08d805210e40cccb069f5f9a0f6727cf4faa7bed2c9c", + strip_prefix = "rules_proto-6.0.0-rc1", + url = "https://github.com/bazelbuild/rules_proto/releases/download/6.0.0-rc1/rules_proto-6.0.0-rc1.tar.gz", ) - http_archive( name = "com_google_protobuf", sha256 = "75be42bd736f4df6d702a0e4e4d30de9ee40eac024c4b845d17ae4cc831fe4ae", diff --git a/python/private/proto/py_proto_library.bzl b/python/private/proto/py_proto_library.bzl index 18f558dae7..b1d7cd2e64 100644 --- a/python/private/proto/py_proto_library.bzl +++ b/python/private/proto/py_proto_library.bzl @@ -15,8 +15,11 @@ """The implementation of the `py_proto_library` rule and its aspect.""" load("@rules_proto//proto:defs.bzl", "ProtoInfo", "proto_common") +load("@rules_proto//proto:proto_common.bzl", "toolchains") load("//python:defs.bzl", "PyInfo") +PY_PROTO_TOOLCHAIN = "@rules_python//python/proto:toolchain_type" + _PyProtoInfo = provider( doc = "Encapsulates information needed by the Python proto rules.", fields = { @@ -58,7 +61,7 @@ def _py_proto_aspect_impl(target, ctx): proto.path, )) - proto_lang_toolchain_info = ctx.toolchains["@rules_python//python/proto:toolchain_type"].proto + proto_lang_toolchain_info = toolchains.find_toolchain(ctx, "_aspect_proto_toolchain", PY_PROTO_TOOLCHAIN) api_deps = [proto_lang_toolchain_info.runtime] generated_sources = [] @@ -120,10 +123,15 @@ def _py_proto_aspect_impl(target, ctx): _py_proto_aspect = aspect( implementation = _py_proto_aspect_impl, + attrs = toolchains.if_legacy_toolchain({ + "_aspect_proto_toolchain": attr.label( + default = ":python_toolchain", + ), + }), attr_aspects = ["deps"], required_providers = [ProtoInfo], provides = [_PyProtoInfo], - toolchains = ["@rules_python//python/proto:toolchain_type"] + toolchains = toolchains.use_toolchain(PY_PROTO_TOOLCHAIN) ) def _py_proto_library_rule(ctx): From ab3ea2c5a3291ea7896a8e595173619bb63c39f3 Mon Sep 17 00:00:00 2001 From: thesayyn Date: Mon, 8 Jan 2024 13:25:38 -0800 Subject: [PATCH 03/10] rules_proto version --- MODULE.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MODULE.bazel b/MODULE.bazel index e89b8ef679..14ac1920e4 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -9,7 +9,7 @@ bazel_dep(name = "bazel_skylib", version = "1.3.0") bazel_dep(name = "platforms", version = "0.0.4") # Those are loaded only when using py_proto_library -bazel_dep(name = "rules_proto", version = "5.3.0-21.7") +bazel_dep(name = "rules_proto", version = "6.0.0-rc1") bazel_dep(name = "protobuf", version = "21.7", repo_name = "com_google_protobuf") internal_deps = use_extension("@rules_python//python/private/bzlmod:internal_deps.bzl", "internal_deps") From b1b4cd2e9eaf94abc1ccaa205c835fc4bf7c4d41 Mon Sep 17 00:00:00 2001 From: Alex Eagle Date: Tue, 5 Mar 2024 16:03:30 -0800 Subject: [PATCH 04/10] fix: use public API for rules_proto proto_common toolchains Relies on https://github.com/bazelbuild/rules_proto/pull/207 landing and being available in a rules_proto release. --- python/private/proto/py_proto_library.bzl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/private/proto/py_proto_library.bzl b/python/private/proto/py_proto_library.bzl index b1d7cd2e64..f17c06ec7c 100644 --- a/python/private/proto/py_proto_library.bzl +++ b/python/private/proto/py_proto_library.bzl @@ -14,8 +14,7 @@ """The implementation of the `py_proto_library` rule and its aspect.""" -load("@rules_proto//proto:defs.bzl", "ProtoInfo", "proto_common") -load("@rules_proto//proto:proto_common.bzl", "toolchains") +load("@rules_proto//proto:defs.bzl", "ProtoInfo", "proto_common", "toolchains") load("//python:defs.bzl", "PyInfo") PY_PROTO_TOOLCHAIN = "@rules_python//python/proto:toolchain_type" From e5ef7cd170e474159e53463c8c13010e33e23a73 Mon Sep 17 00:00:00 2001 From: Alex Eagle Date: Tue, 5 Mar 2024 16:40:13 -0800 Subject: [PATCH 05/10] refactor: don't use toolchains helper at all It has a bug with bzlmod where it tries to reference the label of the users toolchain_type, leading to an error like com.google.devtools.build.lib.packages.BuildFileNotFoundException: no such package '@@[unknown repo 'rules_python' requested from @@rules_proto~]//python/proto': The repository '@@[unknown repo 'rules_python' requested from @@rules_proto~]' could not be resolved: No repository visible as '@rules_python' from repository '@@rules_proto~' --- python/private/proto/py_proto_library.bzl | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/python/private/proto/py_proto_library.bzl b/python/private/proto/py_proto_library.bzl index f17c06ec7c..e123ff8476 100644 --- a/python/private/proto/py_proto_library.bzl +++ b/python/private/proto/py_proto_library.bzl @@ -14,7 +14,7 @@ """The implementation of the `py_proto_library` rule and its aspect.""" -load("@rules_proto//proto:defs.bzl", "ProtoInfo", "proto_common", "toolchains") +load("@rules_proto//proto:defs.bzl", "ProtoInfo", "proto_common") load("//python:defs.bzl", "PyInfo") PY_PROTO_TOOLCHAIN = "@rules_python//python/proto:toolchain_type" @@ -35,6 +35,9 @@ _PyProtoInfo = provider( def _filter_provider(provider, *attrs): return [dep[provider] for attr in attrs for dep in attr if provider in dep] +def _incompatible_toolchains_enabled(): + return getattr(proto_common, "INCOMPATIBLE_ENABLE_PROTO_TOOLCHAIN_RESOLUTION", False) + def _py_proto_aspect_impl(target, ctx): """Generates and compiles Python code for a proto_library. @@ -60,7 +63,14 @@ def _py_proto_aspect_impl(target, ctx): proto.path, )) - proto_lang_toolchain_info = toolchains.find_toolchain(ctx, "_aspect_proto_toolchain", PY_PROTO_TOOLCHAIN) + if _incompatible_toolchains_enabled(): + toolchain = ctx.toolchains[PY_PROTO_TOOLCHAIN] + if not toolchain: + fail("No toolchains registered for '%s'." % PY_PROTO_TOOLCHAIN) + proto_lang_toolchain_info = toolchain.proto + else: + proto_lang_toolchain_info = getattr(ctx.attr, "_aspect_proto_toolchain")[proto_common.ProtoLangToolchainInfo] + api_deps = [proto_lang_toolchain_info.runtime] generated_sources = [] @@ -122,15 +132,15 @@ def _py_proto_aspect_impl(target, ctx): _py_proto_aspect = aspect( implementation = _py_proto_aspect_impl, - attrs = toolchains.if_legacy_toolchain({ + attrs = {} if _incompatible_toolchains_enabled() else { "_aspect_proto_toolchain": attr.label( default = ":python_toolchain", ), - }), + }, attr_aspects = ["deps"], required_providers = [ProtoInfo], provides = [_PyProtoInfo], - toolchains = toolchains.use_toolchain(PY_PROTO_TOOLCHAIN) + toolchains = [PY_PROTO_TOOLCHAIN] if _incompatible_toolchains_enabled() else [], ) def _py_proto_library_rule(ctx): From 89f0a561c48e000fe6e531fc76c7cff1459d3f03 Mon Sep 17 00:00:00 2001 From: Alex Eagle Date: Tue, 5 Mar 2024 17:17:57 -0800 Subject: [PATCH 06/10] chore: green up example --- examples/py_proto_library/WORKSPACE | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/examples/py_proto_library/WORKSPACE b/examples/py_proto_library/WORKSPACE index 47057f39ef..f7f9636fdf 100644 --- a/examples/py_proto_library/WORKSPACE +++ b/examples/py_proto_library/WORKSPACE @@ -22,7 +22,6 @@ python_register_toolchains( # Then we need to setup dependencies in order to use py_proto_library load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - http_archive( name = "rules_proto", sha256 = "904a8097fae42a690c8e08d805210e40cccb069f5f9a0f6727cf4faa7bed2c9c", @@ -31,18 +30,17 @@ http_archive( ) load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") + rules_proto_dependencies() -rules_proto_toolchains() +rules_proto_toolchains() http_archive( name = "com_google_protobuf", - # sha256 = "8632d7e8a6cbbd7e58f40e3f62fed77bff3df6e4dcac73498f0a68ba5a38d892", - strip_prefix = "protobuf-d1c5e885b7b7d3ac888e89cf4d45cb0c9fa6b4ed", - urls = ["https://github.com/protocolbuffers/protobuf/archive/d1c5e885b7b7d3ac888e89cf4d45cb0c9fa6b4ed.tar.gz"], + strip_prefix = "protobuf-25.3", + urls = ["https://github.com/protocolbuffers/protobuf/archive/v25.3.tar.gz"], ) -load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps", "protobuf_register_toolchains") -protobuf_deps() -protobuf_register_toolchains() +load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") +protobuf_deps() From f95798242a641877f4d606a448671016c110549f Mon Sep 17 00:00:00 2001 From: thesayyn Date: Wed, 10 Apr 2024 12:43:39 -0700 Subject: [PATCH 07/10] update protobuf --- examples/py_proto_library/WORKSPACE | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/examples/py_proto_library/WORKSPACE b/examples/py_proto_library/WORKSPACE index f7f9636fdf..7d119bb4c2 100644 --- a/examples/py_proto_library/WORKSPACE +++ b/examples/py_proto_library/WORKSPACE @@ -22,6 +22,7 @@ python_register_toolchains( # Then we need to setup dependencies in order to use py_proto_library load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + http_archive( name = "rules_proto", sha256 = "904a8097fae42a690c8e08d805210e40cccb069f5f9a0f6727cf4faa7bed2c9c", @@ -30,17 +31,15 @@ http_archive( ) load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") - rules_proto_dependencies() - rules_proto_toolchains() http_archive( name = "com_google_protobuf", - strip_prefix = "protobuf-25.3", - urls = ["https://github.com/protocolbuffers/protobuf/archive/v25.3.tar.gz"], + strip_prefix = "protobuf-26.1", + sha256 = "4fc5ff1b2c339fb86cd3a25f0b5311478ab081e65ad258c6789359cd84d421f8", + urls = ["https://github.com/protocolbuffers/protobuf/archive/v26.1.tar.gz"], ) load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") - -protobuf_deps() +protobuf_deps() \ No newline at end of file From 4e8079421d39fe09a5e2ecb00fc6ebf39bfab83e Mon Sep 17 00:00:00 2001 From: thesayyn Date: Wed, 10 Apr 2024 12:48:20 -0700 Subject: [PATCH 08/10] buildifier --- examples/py_proto_library/WORKSPACE | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/examples/py_proto_library/WORKSPACE b/examples/py_proto_library/WORKSPACE index 7d119bb4c2..ab3ed49b3f 100644 --- a/examples/py_proto_library/WORKSPACE +++ b/examples/py_proto_library/WORKSPACE @@ -22,7 +22,6 @@ python_register_toolchains( # Then we need to setup dependencies in order to use py_proto_library load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - http_archive( name = "rules_proto", sha256 = "904a8097fae42a690c8e08d805210e40cccb069f5f9a0f6727cf4faa7bed2c9c", @@ -31,15 +30,18 @@ http_archive( ) load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") + rules_proto_dependencies() + rules_proto_toolchains() http_archive( name = "com_google_protobuf", - strip_prefix = "protobuf-26.1", sha256 = "4fc5ff1b2c339fb86cd3a25f0b5311478ab081e65ad258c6789359cd84d421f8", + strip_prefix = "protobuf-26.1", urls = ["https://github.com/protocolbuffers/protobuf/archive/v26.1.tar.gz"], ) load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") -protobuf_deps() \ No newline at end of file + +protobuf_deps() From 59677d4ad1541be04e5821945cb20b897292b40b Mon Sep 17 00:00:00 2001 From: thesayyn Date: Wed, 10 Apr 2024 12:57:57 -0700 Subject: [PATCH 09/10] shorter workspace --- examples/py_proto_library/WORKSPACE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/py_proto_library/WORKSPACE b/examples/py_proto_library/WORKSPACE index ab3ed49b3f..0ba65752b8 100644 --- a/examples/py_proto_library/WORKSPACE +++ b/examples/py_proto_library/WORKSPACE @@ -1,4 +1,4 @@ -workspace(name = "rules_python_py_proto_library_example") +workspace(name = "p") # The following local_path_override is only needed to run this example as part of our CI. local_repository( From 4b1831d05510a5ad019313478848cadf89a199f5 Mon Sep 17 00:00:00 2001 From: Alex Eagle Date: Thu, 11 Apr 2024 11:16:06 -0700 Subject: [PATCH 10/10] Update WORKSPACE --- examples/py_proto_library/WORKSPACE | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/py_proto_library/WORKSPACE b/examples/py_proto_library/WORKSPACE index 0ba65752b8..81f189dbbf 100644 --- a/examples/py_proto_library/WORKSPACE +++ b/examples/py_proto_library/WORKSPACE @@ -1,3 +1,5 @@ +# NB: short workspace name is required to workaround PATH length limitation, see +# https://github.com/bazelbuild/bazel/issues/18683#issuecomment-1843857373 workspace(name = "p") # The following local_path_override is only needed to run this example as part of our CI.