From 272c9c605634e52622104001afda0cef1b7b31c3 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 6 Dec 2023 08:57:38 -0800 Subject: [PATCH 1/4] feat: expose runtime's shared libraries through toolchain This exposes the runtime's C libraries throught the py cc toolchain. This allows tools to embed the Python runtime or otherwise link against it. Work towards https://github.com/bazelbuild/rules_python/issues/824 --- internal_deps.bzl | 6 +- python/private/py_cc_toolchain_info.bzl | 21 +++++++ python/private/py_cc_toolchain_rule.bzl | 12 ++++ python/repositories.bzl | 1 + tests/cc/BUILD.bazel | 8 +++ .../py_cc_toolchain/py_cc_toolchain_tests.bzl | 15 ++++- tests/cc_info_subject.bzl | 55 +++++++++++++++++++ tests/py_cc_toolchain_info_subject.bzl | 10 ++++ 8 files changed, 124 insertions(+), 4 deletions(-) diff --git a/internal_deps.bzl b/internal_deps.bzl index 3835cd61b2..9931933396 100644 --- a/internal_deps.bzl +++ b/internal_deps.bzl @@ -57,9 +57,9 @@ def rules_python_internal_deps(): http_archive( name = "rules_testing", - sha256 = "8df0a8eb21739ea4b0a03f5dc79e68e245a45c076cfab404b940cc205cb62162", - strip_prefix = "rules_testing-0.4.0", - url = "https://github.com/bazelbuild/rules_testing/releases/download/v0.4.0/rules_testing-v0.4.0.tar.gz", + sha256 = "b84ed8546f1969d700ead4546de9f7637e0f058d835e47e865dcbb13c4210aed", + strip_prefix = "rules_testing-0.5.0", + url = "https://github.com/bazelbuild/rules_testing/releases/download/v0.5.0/rules_testing-v0.5.0.tar.gz", ) http_archive( diff --git a/python/private/py_cc_toolchain_info.bzl b/python/private/py_cc_toolchain_info.bzl index a2e62a8783..a47a6a560d 100644 --- a/python/private/py_cc_toolchain_info.bzl +++ b/python/private/py_cc_toolchain_info.bzl @@ -31,6 +31,27 @@ PyCcToolchainInfo = provider( target (typically a `cc_library`) to be propagated to consumers (directly exposing a Target object can cause memory issues and is an anti-pattern). + When consuming this map, it's suggested to use `providers_map.values()` to + return all providers; or copy the map and filter out or replace keys as + appropriate. Note that any keys beginning with `_` (underscore) are + considered private and should be forward along as-is (this better allows + e.g. `:current_py_cc_headers` to act as the underlying headers target it + represents). +""", + "libs": """\ +(struct) Information about C libraries, with fields: + * providers_map: A dict of string to provider instances. The key should be + a fully qualified name (e.g. `@rules_foo//bar:baz.bzl#MyInfo`) of the + provider to uniquely identify its type. + + The following keys are always present: + * CcInfo: the CcInfo provider instance for the libraries. + * DefaultInfo: the DefaultInfo provider instance for the headers. + + A map is used to allow additional providers from the originating libraries + target (typically a `cc_library`) to be propagated to consumers (directly + exposing a Target object can cause memory issues and is an anti-pattern). + When consuming this map, it's suggested to use `providers_map.values()` to return all providers; or copy the map and filter out or replace keys as appropriate. Note that any keys beginning with `_` (underscore) are diff --git a/python/private/py_cc_toolchain_rule.bzl b/python/private/py_cc_toolchain_rule.bzl index c80f845065..abb3fb6e9f 100644 --- a/python/private/py_cc_toolchain_rule.bzl +++ b/python/private/py_cc_toolchain_rule.bzl @@ -28,6 +28,12 @@ def _py_cc_toolchain_impl(ctx): "DefaultInfo": ctx.attr.headers[DefaultInfo], }, ), + libs = struct( + providers_map = { + "CcInfo": ctx.attr.libs[CcInfo], + "DefaultInfo": ctx.attr.libs[DefaultInfo], + }, + ), python_version = ctx.attr.python_version, ) return [platform_common.ToolchainInfo( @@ -43,6 +49,12 @@ py_cc_toolchain = rule( providers = [CcInfo], mandatory = True, ), + "libs": attr.label( + doc = ("Target that provides the Python runtime libraries for linking. " + + "Typically this is a cc_library target of `.so` files."), + providers = [CcInfo], + mandatory = True, + ), "python_version": attr.string( doc = "The Major.minor Python version, e.g. 3.11", mandatory = True, diff --git a/python/repositories.bzl b/python/repositories.bzl index bfba86d09a..b2712770ab 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -353,6 +353,7 @@ py_runtime_pair( py_cc_toolchain( name = "py_cc_toolchain", headers = ":python_headers", + libs = ":libpython", python_version = "{python_version}", ) """.format( diff --git a/tests/cc/BUILD.bazel b/tests/cc/BUILD.bazel index ef64d6dbef..234e580379 100644 --- a/tests/cc/BUILD.bazel +++ b/tests/cc/BUILD.bazel @@ -31,6 +31,7 @@ toolchain( py_cc_toolchain( name = "fake_py_cc_toolchain_impl", headers = ":fake_headers", + libs = ":fake_libs", python_version = "3.999", tags = PREVENT_IMPLICIT_BUILDING_TAGS, ) @@ -44,6 +45,13 @@ cc_library( tags = PREVENT_IMPLICIT_BUILDING_TAGS, ) +cc_library( + name = "fake_libs", + srcs = ["libpython3.so"], + data = ["libdata.txt"], + tags = PREVENT_IMPLICIT_BUILDING_TAGS, +) + cc_toolchain_suite( name = "cc_toolchain_suite", tags = ["manual"], diff --git a/tests/cc/py_cc_toolchain/py_cc_toolchain_tests.bzl b/tests/cc/py_cc_toolchain/py_cc_toolchain_tests.bzl index 609518da78..0321ea1282 100644 --- a/tests/cc/py_cc_toolchain/py_cc_toolchain_tests.bzl +++ b/tests/cc/py_cc_toolchain/py_cc_toolchain_tests.bzl @@ -15,7 +15,7 @@ """Tests for py_cc_toolchain.""" load("@rules_testing//lib:analysis_test.bzl", "analysis_test", "test_suite") -load("@rules_testing//lib:truth.bzl", "matching") +load("@rules_testing//lib:truth.bzl", "matching", "subjects") load("//tests:cc_info_subject.bzl", "cc_info_subject") load("//tests:default_info_subject.bzl", "default_info_subject") load("//tests:py_cc_toolchain_info_subject.bzl", "PyCcToolchainInfoSubject") @@ -74,6 +74,19 @@ def _py_cc_toolchain_test_impl(env, target): matching.str_matches("*/cc/data.txt"), ) + libs_providers = toolchain.libs().providers_map() + libs_providers.keys().contains_exactly(["CcInfo", "DefaultInfo"]) + + cc_info = libs_providers.get("CcInfo", factory = cc_info_subject) + + cc_info.linking_context().linker_inputs().has_size(2) + + default_info = libs_providers.get("DefaultInfo", factory = subjects.default_info) + default_info.runfiles().contains("_main/tests/cc/libdata.txt") + default_info.runfiles().contains_predicate( + matching.str_matches("/libpython3."), + ) + _tests.append(_py_cc_toolchain_test) def py_cc_toolchain_test_suite(name): diff --git a/tests/cc_info_subject.bzl b/tests/cc_info_subject.bzl index 31ac03a035..e33ccb8262 100644 --- a/tests/cc_info_subject.bzl +++ b/tests/cc_info_subject.bzl @@ -29,7 +29,9 @@ def cc_info_subject(info, *, meta): # buildifier: disable=uninitialized public = struct( # go/keep-sorted start + actual = info, compilation_context = lambda *a, **k: _cc_info_subject_compilation_context(self, *a, **k), + linking_context = lambda *a, **k: _cc_info_subject_linking_context(self, *a, **k), # go/keep-sorted end ) self = struct( @@ -52,6 +54,20 @@ def _cc_info_subject_compilation_context(self): meta = self.meta.derive("compilation_context()"), ) +def _cc_info_subject_linking_context(self): + """Returns the CcInfo.linking_context as a subject. + + Args: + self: implicitly added. + + Returns: + [`LinkingContextSubject`] instance. + """ + return _linking_context_subject_new( + self.actual.linking_context, + meta = self.meta.derive("linking_context()"), + ) + def _compilation_context_subject_new(info, *, meta): """Creates a CompilationContextSubject. @@ -126,3 +142,42 @@ def _compilation_context_subject_system_includes(self): container_name = "includes", element_plural_name = "include paths", ) + +def _linking_context_subject_new(info, meta): + """Creates a LinkingContextSubject. + + Args: + info: ([`LinkingContext`]) object instance. + meta: rules_testing `ExpectMeta` instance. + + Returns: + [`LinkingContextSubject`] object. + """ + + # buildifier: disable=uninitialized + public = struct( + # go/keep-sorted start + linker_inputs = lambda *a, **k: _linking_context_subject_linker_inputs(self, *a, **k), + # go/keep-sorted end + ) + self = struct( + actual = info, + meta = meta, + ) + return public + +def _linking_context_subject_linker_inputs(self): + """Returns the linker inputs. + + Args: + self: implicitly added + + Returns: + [`CollectionSubject`] of the linker inputs. + """ + return subjects.collection( + self.actual.linker_inputs.to_list(), + meta = self.meta.derive("linker_inputs()"), + container_name = "linker_inputs", + element_plural_name = "linker input values", + ) diff --git a/tests/py_cc_toolchain_info_subject.bzl b/tests/py_cc_toolchain_info_subject.bzl index ab9d1b8266..4d3647c53e 100644 --- a/tests/py_cc_toolchain_info_subject.bzl +++ b/tests/py_cc_toolchain_info_subject.bzl @@ -19,6 +19,7 @@ def _py_cc_toolchain_info_subject_new(info, *, meta): # buildifier: disable=uninitialized public = struct( headers = lambda *a, **k: _py_cc_toolchain_info_subject_headers(self, *a, **k), + libs = lambda *a, **k: _py_cc_toolchain_info_subject_libs(self, *a, **k), python_version = lambda *a, **k: _py_cc_toolchain_info_subject_python_version(self, *a, **k), actual = info, ) @@ -34,6 +35,15 @@ def _py_cc_toolchain_info_subject_headers(self): ), ) +def _py_cc_toolchain_info_subject_libs(self): + return subjects.struct( + self.actual.libs, + meta = self.meta.derive("libs()"), + attrs = dict( + providers_map = subjects.dict, + ), + ) + def _py_cc_toolchain_info_subject_python_version(self): return subjects.str( self.actual.python_version, From ee62a81d3aa7cfd2c835b5a1805a6140097d22be Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Mon, 22 Jan 2024 18:41:56 -0800 Subject: [PATCH 2/4] Add current_py_cc_libs and more tests --- python/cc/BUILD.bazel | 12 +++ python/private/current_py_cc_libs.bzl | 41 ++++++++++ tests/cc/BUILD.bazel | 6 ++ tests/cc/current_py_cc_libs/BUILD.bazel | 17 ++++ .../current_py_cc_libs_tests.bzl | 77 +++++++++++++++++++ 5 files changed, 153 insertions(+) create mode 100644 python/private/current_py_cc_libs.bzl create mode 100644 tests/cc/current_py_cc_libs/BUILD.bazel create mode 100644 tests/cc/current_py_cc_libs/current_py_cc_libs_tests.bzl diff --git a/python/cc/BUILD.bazel b/python/cc/BUILD.bazel index 0d90e15225..d384d0538f 100644 --- a/python/cc/BUILD.bazel +++ b/python/cc/BUILD.bazel @@ -3,6 +3,7 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") load("//python/private:current_py_cc_headers.bzl", "current_py_cc_headers") +load("//python/private:current_py_cc_libs.bzl", "current_py_cc_libs") package( default_visibility = ["//:__subpackages__"], @@ -19,6 +20,17 @@ current_py_cc_headers( visibility = ["//visibility:public"], ) +# This target provides the C libraries for whatever the current toolchain is for +# the consuming rule. It basically acts like a cc_library by forwarding on the +# providers for the underlying cc_library that the toolchain is using. +current_py_cc_libs( + name = "current_py_cc_libs", + # Building this directly will fail unless a py cc toolchain is registered, + # and it's only under bzlmod that one is registered by default. + tags = [] if BZLMOD_ENABLED else ["manual"], + visibility = ["//visibility:public"], +) + toolchain_type( name = "toolchain_type", visibility = ["//visibility:public"], diff --git a/python/private/current_py_cc_libs.bzl b/python/private/current_py_cc_libs.bzl new file mode 100644 index 0000000000..863e59a927 --- /dev/null +++ b/python/private/current_py_cc_libs.bzl @@ -0,0 +1,41 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Implementation of current_py_cc_libs rule.""" + +def _current_py_cc_libs_impl(ctx): + py_cc_toolchain = ctx.toolchains["//python/cc:toolchain_type"].py_cc_toolchain + return py_cc_toolchain.libs.providers_map.values() + +current_py_cc_libs = rule( + implementation = _current_py_cc_libs_impl, + toolchains = ["//python/cc:toolchain_type"], + provides = [CcInfo], + doc = """\ +Provides the currently active Python toolchain's C libraries. + +This is a wrapper around the underlying `cc_library()` for the +C libraries for the consuming target's currently active Python toolchain. + +To use, simply depend on this target where you would have wanted the +toolchain's underlying `:libpython` target: + +```starlark +cc_library( + name = "foo", + deps = ["@rules_python//python/cc:current_py_cc_libs"] +) +``` +""", +) diff --git a/tests/cc/BUILD.bazel b/tests/cc/BUILD.bazel index 234e580379..cec3e3faac 100644 --- a/tests/cc/BUILD.bazel +++ b/tests/cc/BUILD.bazel @@ -21,6 +21,12 @@ package(default_visibility = ["//:__subpackages__"]) exports_files(["fake_header.h"]) +filegroup( + name = "libpython", + srcs = ["libpython-fake.so"], + tags = PREVENT_IMPLICIT_BUILDING_TAGS, +) + toolchain( name = "fake_py_cc_toolchain", tags = PREVENT_IMPLICIT_BUILDING_TAGS, diff --git a/tests/cc/current_py_cc_libs/BUILD.bazel b/tests/cc/current_py_cc_libs/BUILD.bazel new file mode 100644 index 0000000000..1e108c3132 --- /dev/null +++ b/tests/cc/current_py_cc_libs/BUILD.bazel @@ -0,0 +1,17 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load(":current_py_cc_libs_tests.bzl", "current_py_cc_libs_test_suite") + +current_py_cc_libs_test_suite(name = "current_py_cc_libs_tests") diff --git a/tests/cc/current_py_cc_libs/current_py_cc_libs_tests.bzl b/tests/cc/current_py_cc_libs/current_py_cc_libs_tests.bzl new file mode 100644 index 0000000000..5699b75cc1 --- /dev/null +++ b/tests/cc/current_py_cc_libs/current_py_cc_libs_tests.bzl @@ -0,0 +1,77 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for current_py_cc_libs.""" + +load("@rules_testing//lib:analysis_test.bzl", "analysis_test", "test_suite") +load("@rules_testing//lib:truth.bzl", "matching") +load("//tests:cc_info_subject.bzl", "cc_info_subject") + +_tests = [] + +def _test_current_toolchain_libs(name): + analysis_test( + name = name, + impl = _test_current_toolchain_libs_impl, + target = "//python/cc:current_py_cc_libs", + config_settings = { + "//command_line_option:extra_toolchains": [str(Label("//tests/cc:all"))], + }, + attrs = { + "lib": attr.label( + default = "//tests/cc:libpython", + allow_single_file = True, + ), + }, + ) + +def _test_current_toolchain_libs_impl(env, target): + # Check that the forwarded CcInfo looks vaguely correct. + cc_info = env.expect.that_target(target).provider( + CcInfo, + factory = cc_info_subject, + ) + cc_info.linking_context().linker_inputs().has_size(2) + + # Check that the forward DefaultInfo looks correct + env.expect.that_target(target).runfiles().contains_predicate( + matching.str_matches("*/libdata.txt"), + ) + + # The shared library should also end up in runfiles + # The `_solib` directory is a special directory CC rules put + # libraries into. + env.expect.that_target(target).runfiles().contains_predicate( + matching.str_matches("*_solib*/libpython3.so"), + ) + +_tests.append(_test_current_toolchain_libs) + +def _test_toolchain_is_registered_by_default(name): + analysis_test( + name = name, + impl = _test_toolchain_is_registered_by_default_impl, + target = "//python/cc:current_py_cc_libs", + ) + +def _test_toolchain_is_registered_by_default_impl(env, target): + env.expect.that_target(target).has_provider(CcInfo) + +_tests.append(_test_toolchain_is_registered_by_default) + +def current_py_cc_libs_test_suite(name): + test_suite( + name = name, + tests = _tests, + ) From 0687031077f3abee5aec092ed507f047dac122e8 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 24 Jan 2024 11:57:26 -0800 Subject: [PATCH 3/4] disable native-cc warning --- tests/cc/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/cc/BUILD.bazel b/tests/cc/BUILD.bazel index cec3e3faac..889f9e02d2 100644 --- a/tests/cc/BUILD.bazel +++ b/tests/cc/BUILD.bazel @@ -51,6 +51,7 @@ cc_library( tags = PREVENT_IMPLICIT_BUILDING_TAGS, ) +# buildifier: disable=native-cc cc_library( name = "fake_libs", srcs = ["libpython3.so"], From c9961aa6e282e232065aba9a3286fcfe450b9b25 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 24 Jan 2024 13:07:45 -0800 Subject: [PATCH 4/4] make test pass for workspace builds --- tests/cc/py_cc_toolchain/py_cc_toolchain_tests.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cc/py_cc_toolchain/py_cc_toolchain_tests.bzl b/tests/cc/py_cc_toolchain/py_cc_toolchain_tests.bzl index 0321ea1282..fe83bf2e2d 100644 --- a/tests/cc/py_cc_toolchain/py_cc_toolchain_tests.bzl +++ b/tests/cc/py_cc_toolchain/py_cc_toolchain_tests.bzl @@ -82,7 +82,7 @@ def _py_cc_toolchain_test_impl(env, target): cc_info.linking_context().linker_inputs().has_size(2) default_info = libs_providers.get("DefaultInfo", factory = subjects.default_info) - default_info.runfiles().contains("_main/tests/cc/libdata.txt") + default_info.runfiles().contains("{workspace}/tests/cc/libdata.txt") default_info.runfiles().contains_predicate( matching.str_matches("/libpython3."), )