From 3959bea342ee692891b790062089b742f2002830 Mon Sep 17 00:00:00 2001 From: Nicolas Lopez Date: Fri, 8 Nov 2024 15:35:22 -0500 Subject: [PATCH 1/9] setting up Buck2 cpp RE EngFlow example --- buck2/cpp/.buckconfig | 36 +++++++++++++++++++++++ buck2/cpp/.buckroot | 0 buck2/cpp/.gitignore | 1 + buck2/cpp/BUCK | 21 ++++++++++++++ buck2/cpp/README.md | 49 ++++++++++++++++++++++++++++++++ buck2/cpp/main.cpp | 4 +++ buck2/cpp/platforms/BUCK | 18 ++++++++++++ buck2/cpp/platforms/defs.bzl | 52 ++++++++++++++++++++++++++++++++++ buck2/cpp/toolchains/BUCK | 38 +++++++++++++++++++++++++ buck2/cpp/toolchains/tools.bzl | 41 +++++++++++++++++++++++++++ 10 files changed, 260 insertions(+) create mode 100644 buck2/cpp/.buckconfig create mode 100644 buck2/cpp/.buckroot create mode 100644 buck2/cpp/.gitignore create mode 100644 buck2/cpp/BUCK create mode 100644 buck2/cpp/README.md create mode 100644 buck2/cpp/main.cpp create mode 100644 buck2/cpp/platforms/BUCK create mode 100644 buck2/cpp/platforms/defs.bzl create mode 100644 buck2/cpp/toolchains/BUCK create mode 100644 buck2/cpp/toolchains/tools.bzl diff --git a/buck2/cpp/.buckconfig b/buck2/cpp/.buckconfig new file mode 100644 index 00000000..1e37b597 --- /dev/null +++ b/buck2/cpp/.buckconfig @@ -0,0 +1,36 @@ +[cells] + root = . + prelude = prelude + toolchains = toolchains + none = none + +[cell_aliases] + config = prelude + ovr_config = prelude + fbcode = none + fbsource = none + fbcode_macros = none + buck = none + +# Uses a copy of the prelude bundled with the buck2 binary. You can alternatively delete this +# section and vendor a copy of the prelude to the `prelude` directory of your project. +[external_cells] + prelude = bundled + +[parser] + target_platform_detector_spec = target:root//...->prelude//platforms:default + +[buck2] +digest_algorithms = SHA256 + +[buck2_re_client] +engine_address = .cluster.engflow.com +action_cache_address = .cluster.engflow.com +cas_address = .cluster.engflow.com +tls_client_cert = x-engflow-auth-method:jwt-v0,x-engflow-auth-token:LONG_JWT_STRING + +[build] + execution_platforms = root//platforms:remote_platform + +[project] + ignore = .git diff --git a/buck2/cpp/.buckroot b/buck2/cpp/.buckroot new file mode 100644 index 00000000..e69de29b diff --git a/buck2/cpp/.gitignore b/buck2/cpp/.gitignore new file mode 100644 index 00000000..0a0ddb2e --- /dev/null +++ b/buck2/cpp/.gitignore @@ -0,0 +1 @@ +/buck-out diff --git a/buck2/cpp/BUCK b/buck2/cpp/BUCK new file mode 100644 index 00000000..14e9285c --- /dev/null +++ b/buck2/cpp/BUCK @@ -0,0 +1,21 @@ +# Copyright 2022 EngFlow Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cxx_binary( + name = "main", + srcs = ["main.cpp"], + link_style = "static", + exec_compatible_with = ["//platforms:remote_platform"], + default_target_platform = "//platforms:remote", +) diff --git a/buck2/cpp/README.md b/buck2/cpp/README.md new file mode 100644 index 00000000..2c7d59f7 --- /dev/null +++ b/buck2/cpp/README.md @@ -0,0 +1,49 @@ +# EngFlow RE + Buck2 cpp example + +This example demonstrates use of EngFlow RE for a simple C++ project built with Buck2 using the prelude. + +It is based on two existing samples in the Buck2 upstream repo: + +* Simple cpp Hello World project - https://github.com/facebook/buck2/tree/804d62242214455d51787f7c8c96a1e12c75ec32/examples/hello_world +* Buck2 remote execution integration with EngFlow - https://github.com/facebook/buck2/tree/804d62242214455d51787f7c8c96a1e12c75ec32/examples/remote_execution/engflow + +### Example structure + +In the `platforms` cell we specify: +* The platform used for remote execution in this project `root//platforms:remote_platform`, which includes the definition of the Docker image used for remote execution. +* The `execution_platform`, `root//platforms:remote` that defines constraints for targets to run in the remote execution environment. + +In the `toolchains` cell we specify: + +* The c++ toolchain `root//toolchains:cxx_tools_info_toolchain` that is compatible with the remote execution environment. +* The clang tools, `root//toolchains:path_clang_tools, which is used by the c++ toolchain, and specifies the tools installed in the Docker image. + +The main `BUCK` file defines: + +* A `cxx_binary` binary target that has the `exec_compatible_with` attr pointing to the `root//platforms:remote_platform` target and the `default_target_platform` attr pointing to the `root//platforms:remote` target. + +### Relevant configs in `.buckconfig` + +The EngFlow endpoint and certificate should be configured as the +following: + +```ini +[buck2_re_client] +engine_address = .cluster.engflow.com +action_cache_address = .cluster.engflow.com +cas_address = .cluster.engflow.com +tls_client_cert = x-engflow-auth-method:jwt-v0,x-engflow-auth-token:LONG_JWT_STRING + ``` + + To obtain the value of `LONG_JWT_STRING`, log into https://.cluster.engflow.com/gettingstarted and use the value of `x-engflow-auth-token` in section `Method 2: JWT`. + +### Usage instructions + +Clone the repository and replace the relevant configs in `.buckconfig`. + +Build and run the example: + +``` +buck2 build //:main +buck2 run -v 4 //:main +``` diff --git a/buck2/cpp/main.cpp b/buck2/cpp/main.cpp new file mode 100644 index 00000000..6b7adc90 --- /dev/null +++ b/buck2/cpp/main.cpp @@ -0,0 +1,4 @@ +#include +int main() { + std::cout << "Hello from a C++ Buck2 program!" << std::endl; +} \ No newline at end of file diff --git a/buck2/cpp/platforms/BUCK b/buck2/cpp/platforms/BUCK new file mode 100644 index 00000000..d5cd757d --- /dev/null +++ b/buck2/cpp/platforms/BUCK @@ -0,0 +1,18 @@ +load(":defs.bzl", "platforms") +load("@prelude//platforms:defs.bzl", "execution_platform") + +# This platform configures details of remote execution. +platforms( + name = "remote_platform", + cpu_configuration = "config//cpu:x86_64", + os_configuration = "config//os:linux", +) + +# Simple execution platform for the remote worker env (x86_64 / linux) +execution_platform( + name = "remote", + cpu_configuration = "config//cpu:x86_64", + os_configuration = "config//os:linux", + use_windows_path_separators = False, + visibility = ["PUBLIC"], +) diff --git a/buck2/cpp/platforms/defs.bzl b/buck2/cpp/platforms/defs.bzl new file mode 100644 index 00000000..7d0fe4ec --- /dev/null +++ b/buck2/cpp/platforms/defs.bzl @@ -0,0 +1,52 @@ +# Copyright 2022 EngFlow Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This platform is essentially the same as the one provided in https://github.com/facebook/buck2/blob/804d62242214455d51787f7c8c96a1e12c75ec32/examples/remote_execution/engflow/platforms/defs.bzl +# The main difference is we enable passing CPU and OS constraints and we use the sample EngFlow RE image. +def _platforms(ctx): + constraints = dict() + constraints.update(ctx.attrs.cpu_configuration[ConfigurationInfo].constraints) + constraints.update(ctx.attrs.os_configuration[ConfigurationInfo].constraints) + configuration = ConfigurationInfo( + constraints = constraints, + values = {}, + ) + + # The sample EngFlow RE image. + image = "docker://gcr.io/bazel-public/ubuntu2004-java11@sha256:69a78f121230c6d5cbfe2f4af8ce65481aa3f2acaaaf8e899df335f1ac1b35b5" + platform = ExecutionPlatformInfo( + label = ctx.label.raw_target(), + configuration = configuration, + executor_config = CommandExecutorConfig( + local_enabled = False, + remote_enabled = True, + use_limited_hybrid = False, + remote_execution_properties = { + "container-image": image, + }, + remote_execution_use_case = "buck2-default", + # TODO: Use output_paths + remote_output_paths = "strict", + ), + ) + + return [DefaultInfo(), ExecutionPlatformRegistrationInfo(platforms = [platform]), configuration] + +platforms = rule( + attrs = { + "cpu_configuration": attrs.dep(providers = [ConfigurationInfo]), + "os_configuration": attrs.dep(providers = [ConfigurationInfo]), + }, + impl = _platforms +) diff --git a/buck2/cpp/toolchains/BUCK b/buck2/cpp/toolchains/BUCK new file mode 100644 index 00000000..e724e77c --- /dev/null +++ b/buck2/cpp/toolchains/BUCK @@ -0,0 +1,38 @@ +# Copyright 2022 EngFlow Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +load("@prelude//toolchains:python.bzl", "system_python_bootstrap_toolchain") +load("@prelude//toolchains:cxx.bzl", "cxx_tools_info_toolchain") +load("tools.bzl", "path_clang_tools") + +# We use the default system python toolchain in this example, focusing only on configuring the cpp toolchain. +system_python_bootstrap_toolchain( + name = "python_bootstrap", + visibility = ["PUBLIC"], +) + +# Custom clang tools that use g++ for linking. +path_clang_tools( + name = "clang_tools", + target_compatible_with = ["config//os:linux"], + visibility = ["PUBLIC"], +) + +# Custom cpp toolchain that is compatible with the remote worker environment. +cxx_tools_info_toolchain( + name = "cxx", + target_compatible_with = ["config//os:linux"], + cxx_tools_info = ":clang_tools", + link_style = "static", + visibility = ["PUBLIC"], +) \ No newline at end of file diff --git a/buck2/cpp/toolchains/tools.bzl b/buck2/cpp/toolchains/tools.bzl new file mode 100644 index 00000000..b62e65bd --- /dev/null +++ b/buck2/cpp/toolchains/tools.bzl @@ -0,0 +1,41 @@ +# Copyright 2022 EngFlow Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@prelude//cxx:cxx_toolchain_types.bzl", "LinkerType") +load("@prelude//toolchains:cxx.bzl", "CxxToolsInfo") + +# This def is similar to the one in https://github.com/facebook/buck2/blob/804d62242214455d51787f7c8c96a1e12c75ec32/prelude/toolchains/cxx/clang/tools.bzl +# The main difference is we set linker to "g++" to match the cpp tools installed in the sample docker container. +def _path_clang_tools_impl(_ctx) -> list[Provider]: + return [ + DefaultInfo(), + CxxToolsInfo( + compiler = "clang", + compiler_type = "clang", + cxx_compiler = "clang++", + asm_compiler = "clang", + asm_compiler_type = "clang", + rc_compiler = None, + cvtres_compiler = None, + archiver = "ar", + archiver_type = "gnu", + linker = "g++", + linker_type = LinkerType("gnu"), + ), + ] + +path_clang_tools = rule( + impl = _path_clang_tools_impl, + attrs = {}, +) \ No newline at end of file From 60e7ad2446b4ef78d90a6259653e23553489bd70 Mon Sep 17 00:00:00 2001 From: Nicolas Lopez Date: Fri, 8 Nov 2024 15:36:39 -0500 Subject: [PATCH 2/9] add newlines --- buck2/cpp/main.cpp | 2 +- buck2/cpp/toolchains/BUCK | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buck2/cpp/main.cpp b/buck2/cpp/main.cpp index 6b7adc90..5b3849e1 100644 --- a/buck2/cpp/main.cpp +++ b/buck2/cpp/main.cpp @@ -1,4 +1,4 @@ #include int main() { std::cout << "Hello from a C++ Buck2 program!" << std::endl; -} \ No newline at end of file +} diff --git a/buck2/cpp/toolchains/BUCK b/buck2/cpp/toolchains/BUCK index e724e77c..b4e48701 100644 --- a/buck2/cpp/toolchains/BUCK +++ b/buck2/cpp/toolchains/BUCK @@ -35,4 +35,4 @@ cxx_tools_info_toolchain( cxx_tools_info = ":clang_tools", link_style = "static", visibility = ["PUBLIC"], -) \ No newline at end of file +) From d6008a48f7a73358f8845f0e33e3ff50405427d7 Mon Sep 17 00:00:00 2001 From: Nicolas Lopez Date: Fri, 8 Nov 2024 15:37:30 -0500 Subject: [PATCH 3/9] Add Buck2 link --- buck2/cpp/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buck2/cpp/README.md b/buck2/cpp/README.md index 2c7d59f7..e770390a 100644 --- a/buck2/cpp/README.md +++ b/buck2/cpp/README.md @@ -1,6 +1,6 @@ # EngFlow RE + Buck2 cpp example -This example demonstrates use of EngFlow RE for a simple C++ project built with Buck2 using the prelude. +This example demonstrates use of EngFlow RE for a simple C++ project built with [Buck2](https://github.com/facebook/buck2) using the prelude. It is based on two existing samples in the Buck2 upstream repo: From 6e4a34038c93fb9313ae6ddb14fa953c2ac1f037 Mon Sep 17 00:00:00 2001 From: Nicolas Lopez Date: Tue, 12 Nov 2024 14:05:12 -0500 Subject: [PATCH 4/9] simplify platforms --- buck2/cpp/BUCK | 2 +- buck2/cpp/platforms/BUCK | 10 ---------- buck2/cpp/platforms/defs.bzl | 8 +++++++- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/buck2/cpp/BUCK b/buck2/cpp/BUCK index 14e9285c..6e91ee49 100644 --- a/buck2/cpp/BUCK +++ b/buck2/cpp/BUCK @@ -17,5 +17,5 @@ cxx_binary( srcs = ["main.cpp"], link_style = "static", exec_compatible_with = ["//platforms:remote_platform"], - default_target_platform = "//platforms:remote", + default_target_platform = "//platforms:remote_platform", ) diff --git a/buck2/cpp/platforms/BUCK b/buck2/cpp/platforms/BUCK index d5cd757d..790deec2 100644 --- a/buck2/cpp/platforms/BUCK +++ b/buck2/cpp/platforms/BUCK @@ -1,5 +1,4 @@ load(":defs.bzl", "platforms") -load("@prelude//platforms:defs.bzl", "execution_platform") # This platform configures details of remote execution. platforms( @@ -7,12 +6,3 @@ platforms( cpu_configuration = "config//cpu:x86_64", os_configuration = "config//os:linux", ) - -# Simple execution platform for the remote worker env (x86_64 / linux) -execution_platform( - name = "remote", - cpu_configuration = "config//cpu:x86_64", - os_configuration = "config//os:linux", - use_windows_path_separators = False, - visibility = ["PUBLIC"], -) diff --git a/buck2/cpp/platforms/defs.bzl b/buck2/cpp/platforms/defs.bzl index 7d0fe4ec..9da90572 100644 --- a/buck2/cpp/platforms/defs.bzl +++ b/buck2/cpp/platforms/defs.bzl @@ -25,6 +25,7 @@ def _platforms(ctx): # The sample EngFlow RE image. image = "docker://gcr.io/bazel-public/ubuntu2004-java11@sha256:69a78f121230c6d5cbfe2f4af8ce65481aa3f2acaaaf8e899df335f1ac1b35b5" + name = ctx.label.raw_target() platform = ExecutionPlatformInfo( label = ctx.label.raw_target(), configuration = configuration, @@ -41,7 +42,12 @@ def _platforms(ctx): ), ) - return [DefaultInfo(), ExecutionPlatformRegistrationInfo(platforms = [platform]), configuration] + return [ + DefaultInfo(), + ExecutionPlatformRegistrationInfo(platforms = [platform]), + configuration, + PlatformInfo(label = str(name), configuration = configuration), + ] platforms = rule( attrs = { From 31d827c645406da6c763acec2a257bb7e6ee8cb6 Mon Sep 17 00:00:00 2001 From: Nicolas Lopez Date: Wed, 13 Nov 2024 10:18:03 -0500 Subject: [PATCH 5/9] add cc test example --- buck2/cpp/.buckconfig | 14 ++++++++++++++ buck2/cpp/BUCK | 31 +++++++++++++++++++++++++++++++ buck2/cpp/hello.cc | 7 +++++++ buck2/cpp/hello.h | 8 ++++++++ buck2/cpp/hello_test.cc | 11 +++++++++++ buck2/cpp/main.cc | 7 +++++++ buck2/cpp/platforms/BUCK | 24 ++++++++++++++++++++++++ buck2/cpp/platforms/defs.bzl | 16 ++++++++++++++++ buck2/cpp/toolchains/BUCK | 19 ++++++++++++++++++- 9 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 buck2/cpp/hello.cc create mode 100644 buck2/cpp/hello.h create mode 100644 buck2/cpp/hello_test.cc create mode 100644 buck2/cpp/main.cc diff --git a/buck2/cpp/.buckconfig b/buck2/cpp/.buckconfig index 1e37b597..82ba3911 100644 --- a/buck2/cpp/.buckconfig +++ b/buck2/cpp/.buckconfig @@ -1,3 +1,17 @@ +# Copyright 2022 EngFlow Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + [cells] root = . prelude = prelude diff --git a/buck2/cpp/BUCK b/buck2/cpp/BUCK index 6e91ee49..2c02a9b5 100644 --- a/buck2/cpp/BUCK +++ b/buck2/cpp/BUCK @@ -19,3 +19,34 @@ cxx_binary( exec_compatible_with = ["//platforms:remote_platform"], default_target_platform = "//platforms:remote_platform", ) + +cxx_binary( + name = "cpp", + srcs = ["main.cc"], + deps = [":cpp_lib"], + exec_compatible_with = ["//platforms:remote_platform"], + default_target_platform = "//platforms:remote_platform", +) + +cxx_library( + name = "cpp_lib", + srcs = [ + "hello.cc", + ], + exported_headers = glob(["**/*.h"]), + visibility = ["PUBLIC"], + exec_compatible_with = ["//platforms:remote_platform"], + default_target_platform = "//platforms:remote_platform", +) + +cxx_test( + name = "cpp_test", + srcs = ["hello_test.cc"], + deps = [ + ":cpp_lib", + ], + exec_compatible_with = ["//platforms:remote_platform"], + default_target_platform = "//platforms:remote_platform", + remote_execution_action_key_providers = "//platforms:remote_execution_action_keys", +) + diff --git a/buck2/cpp/hello.cc b/buck2/cpp/hello.cc new file mode 100644 index 00000000..631a7fd0 --- /dev/null +++ b/buck2/cpp/hello.cc @@ -0,0 +1,7 @@ +#include "hello.h" + +#include + +std::string hello() { + return "hello"; +} diff --git a/buck2/cpp/hello.h b/buck2/cpp/hello.h new file mode 100644 index 00000000..8df9418b --- /dev/null +++ b/buck2/cpp/hello.h @@ -0,0 +1,8 @@ +#ifndef cpp_hello_h +#define cpp_hello_h + +#include + +std::string hello(); + +#endif diff --git a/buck2/cpp/hello_test.cc b/buck2/cpp/hello_test.cc new file mode 100644 index 00000000..97b455a2 --- /dev/null +++ b/buck2/cpp/hello_test.cc @@ -0,0 +1,11 @@ +#include +#include "hello.h" + +int main() { + auto got = hello(); + if (got != "hello") { + std::cerr << "got '" << got << "', want 'hello'" << std::endl; + return 1; + } + return 0; +} diff --git a/buck2/cpp/main.cc b/buck2/cpp/main.cc new file mode 100644 index 00000000..57010349 --- /dev/null +++ b/buck2/cpp/main.cc @@ -0,0 +1,7 @@ +#include +#include "hello.h" + +int main() { + std::cout << hello() << std::endl; + return 0; +} diff --git a/buck2/cpp/platforms/BUCK b/buck2/cpp/platforms/BUCK index 790deec2..8e4f659a 100644 --- a/buck2/cpp/platforms/BUCK +++ b/buck2/cpp/platforms/BUCK @@ -1,4 +1,19 @@ +# Copyright 2022 EngFlow Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + load(":defs.bzl", "platforms") +load(":defs.bzl", "action_keys") # This platform configures details of remote execution. platforms( @@ -6,3 +21,12 @@ platforms( cpu_configuration = "config//cpu:x86_64", os_configuration = "config//os:linux", ) + +# This action_key provides a default BuildModeInfo that is needed for RE of tests to function properly. +# The values in `cell` and `mode` can be used, in practice, to create cache silos. Any values can be given to these attributes. +action_keys( + name = "remote_execution_action_keys", + cell = "standard", + mode = "standard", + visibility = ["PUBLIC"], +) diff --git a/buck2/cpp/platforms/defs.bzl b/buck2/cpp/platforms/defs.bzl index 9da90572..830d53fb 100644 --- a/buck2/cpp/platforms/defs.bzl +++ b/buck2/cpp/platforms/defs.bzl @@ -14,6 +14,8 @@ # This platform is essentially the same as the one provided in https://github.com/facebook/buck2/blob/804d62242214455d51787f7c8c96a1e12c75ec32/examples/remote_execution/engflow/platforms/defs.bzl # The main difference is we enable passing CPU and OS constraints and we use the sample EngFlow RE image. +load("@prelude//:build_mode.bzl", "BuildModeInfo") + def _platforms(ctx): constraints = dict() constraints.update(ctx.attrs.cpu_configuration[ConfigurationInfo].constraints) @@ -49,6 +51,12 @@ def _platforms(ctx): PlatformInfo(label = str(name), configuration = configuration), ] +def _action_keys(ctx): + return [ + DefaultInfo(), + BuildModeInfo(cell = ctx.attrs.cell, mode = ctx.attrs.mode), + ] + platforms = rule( attrs = { "cpu_configuration": attrs.dep(providers = [ConfigurationInfo]), @@ -56,3 +64,11 @@ platforms = rule( }, impl = _platforms ) + +action_keys = rule( + attrs = { + "cell": attrs.string(), + "mode": attrs.string(), + }, + impl = _action_keys +) diff --git a/buck2/cpp/toolchains/BUCK b/buck2/cpp/toolchains/BUCK index b4e48701..3b13b551 100644 --- a/buck2/cpp/toolchains/BUCK +++ b/buck2/cpp/toolchains/BUCK @@ -13,6 +13,7 @@ # limitations under the License. load("@prelude//toolchains:python.bzl", "system_python_bootstrap_toolchain") load("@prelude//toolchains:cxx.bzl", "cxx_tools_info_toolchain") +load("@prelude//toolchains:remote_test_execution.bzl", "remote_test_execution_toolchain") load("tools.bzl", "path_clang_tools") # We use the default system python toolchain in this example, focusing only on configuring the cpp toolchain. @@ -33,6 +34,22 @@ cxx_tools_info_toolchain( name = "cxx", target_compatible_with = ["config//os:linux"], cxx_tools_info = ":clang_tools", - link_style = "static", visibility = ["PUBLIC"], ) + +# Default toolchain for remote execution of tests. +# Note it defines a profile with a capability that defines the `container-image` that matches the one defined in //platforms:remote_platform. +# Capabilities are passed to the RE service to find workers that match them as Platform options. +remote_test_execution_toolchain( + name = "remote_test_execution", + visibility = ["PUBLIC"], + default_profile = "cxx_re_toolchain", + profiles = { + "cxx_re_toolchain": { + "use_case": "cxx-testing", + "capabilities": { + "container-image" : "docker://gcr.io/bazel-public/ubuntu2004-java11@sha256:69a78f121230c6d5cbfe2f4af8ce65481aa3f2acaaaf8e899df335f1ac1b35b5", + }, + } + }, +) From 8c548f367ec37e6d12b145ee0e4be09ffdcc40ce Mon Sep 17 00:00:00 2001 From: Nicolas Lopez Date: Wed, 13 Nov 2024 10:27:33 -0500 Subject: [PATCH 6/9] update readme --- buck2/cpp/README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/buck2/cpp/README.md b/buck2/cpp/README.md index e770390a..28c2311c 100644 --- a/buck2/cpp/README.md +++ b/buck2/cpp/README.md @@ -10,17 +10,20 @@ It is based on two existing samples in the Buck2 upstream repo: ### Example structure In the `platforms` cell we specify: -* The platform used for remote execution in this project `root//platforms:remote_platform`, which includes the definition of the Docker image used for remote execution. -* The `execution_platform`, `root//platforms:remote` that defines constraints for targets to run in the remote execution environment. +* The platform used for remote execution in this project `root//platforms:remote_platform`, which includes the definition of the Docker image used for remote execution, and that defines constraints for targets to run in the remote execution environment. This platform provides an `ExecutionPlatformRegistrationInfo` a `ConfigurationInfo` and a `PlatformInfo` to be able to be used in the `.buckconfig`, and in the `exec_compatible_with` and `default_target_platform` of `cxx_*` rules. +* The `root//platforms:remote_execution_action_keys` target that provides a `BuildModeInfo` which is necessary for the prelude to correctly configure remote execution of tests. It defines two attributes that can be used as cache silo keys. In the `toolchains` cell we specify: * The c++ toolchain `root//toolchains:cxx_tools_info_toolchain` that is compatible with the remote execution environment. * The clang tools, `root//toolchains:path_clang_tools, which is used by the c++ toolchain, and specifies the tools installed in the Docker image. +* The remote test execution toolchain, `root//toolchains:remote_test_execution_toolchain`. This toolchain defines platform options in the form of `capabilities`. Critically these include the `container-image`. The main `BUCK` file defines: -* A `cxx_binary` binary target that has the `exec_compatible_with` attr pointing to the `root//platforms:remote_platform` target and the `default_target_platform` attr pointing to the `root//platforms:remote` target. +* A `cxx_binary` target that has the `exec_compatible_with` as well as the `default_target_platform` attrs pointing to the `root//platforms:remote_platform`. +* A `cxx_library`target that has the `exec_compatible_with` as well as the `default_target_platform` attrs pointing to the `root//platforms:remote_platform`. +* A `cxx_test` target that has the `exec_compatible_with` as well as the `default_target_platform` attrs pointing to the `root//platforms:remote_platform`. It also has a `remote_execution_action_key_providers` attr that points to the `root//platforms:remote_execution_action_keys` target. ### Relevant configs in `.buckconfig` From f5e4dc6824a70f8995aff7c8b9ac2d5884c0ab72 Mon Sep 17 00:00:00 2001 From: Nicolas Lopez Date: Wed, 13 Nov 2024 10:28:38 -0500 Subject: [PATCH 7/9] update readme --- buck2/cpp/README.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/buck2/cpp/README.md b/buck2/cpp/README.md index 28c2311c..5cfaba17 100644 --- a/buck2/cpp/README.md +++ b/buck2/cpp/README.md @@ -44,9 +44,14 @@ tls_client_cert = x-engflow-auth-method:jwt-v0,x-engflow-auth-token:LONG_JW Clone the repository and replace the relevant configs in `.buckconfig`. -Build and run the example: +Build the example: ``` -buck2 build //:main -buck2 run -v 4 //:main +buck2 build //:cpp_lib +``` + +Test the example: + +``` +buck2 test //:cpp_test ``` From d3650f9e820698fb27f000da170297e02f02a39c Mon Sep 17 00:00:00 2001 From: Nicolas Lopez Date: Wed, 13 Nov 2024 13:28:02 -0500 Subject: [PATCH 8/9] fix readme for .buckconfig --- buck2/cpp/.buckconfig | 2 +- buck2/cpp/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buck2/cpp/.buckconfig b/buck2/cpp/.buckconfig index 82ba3911..c3ac155e 100644 --- a/buck2/cpp/.buckconfig +++ b/buck2/cpp/.buckconfig @@ -41,7 +41,7 @@ digest_algorithms = SHA256 engine_address = .cluster.engflow.com action_cache_address = .cluster.engflow.com cas_address = .cluster.engflow.com -tls_client_cert = x-engflow-auth-method:jwt-v0,x-engflow-auth-token:LONG_JWT_STRING +http_headers = x-engflow-auth-method:jwt-v0,x-engflow-auth-token:LONG_JWT_STRING [build] execution_platforms = root//platforms:remote_platform diff --git a/buck2/cpp/README.md b/buck2/cpp/README.md index 5cfaba17..f75bbd7f 100644 --- a/buck2/cpp/README.md +++ b/buck2/cpp/README.md @@ -35,7 +35,7 @@ following: engine_address = .cluster.engflow.com action_cache_address = .cluster.engflow.com cas_address = .cluster.engflow.com -tls_client_cert = x-engflow-auth-method:jwt-v0,x-engflow-auth-token:LONG_JWT_STRING +http_headers = x-engflow-auth-method:jwt-v0,x-engflow-auth-token:LONG_JWT_STRING ``` To obtain the value of `LONG_JWT_STRING`, log into https://.cluster.engflow.com/gettingstarted and use the value of `x-engflow-auth-token` in section `Method 2: JWT`. From 12ba01b55a55877d99418ff27f564ad9d4863f5b Mon Sep 17 00:00:00 2001 From: Nicolas Lopez Date: Thu, 14 Nov 2024 16:44:53 -0500 Subject: [PATCH 9/9] fix for newer buck2 --- buck2/cpp/toolchains/BUCK | 7 +++++++ buck2/cpp/toolchains/tools.bzl | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/buck2/cpp/toolchains/BUCK b/buck2/cpp/toolchains/BUCK index 3b13b551..96b30a3b 100644 --- a/buck2/cpp/toolchains/BUCK +++ b/buck2/cpp/toolchains/BUCK @@ -14,6 +14,7 @@ load("@prelude//toolchains:python.bzl", "system_python_bootstrap_toolchain") load("@prelude//toolchains:cxx.bzl", "cxx_tools_info_toolchain") load("@prelude//toolchains:remote_test_execution.bzl", "remote_test_execution_toolchain") +load("@prelude//tests:test_toolchain.bzl", "noop_test_toolchain") load("tools.bzl", "path_clang_tools") # We use the default system python toolchain in this example, focusing only on configuring the cpp toolchain. @@ -53,3 +54,9 @@ remote_test_execution_toolchain( } }, ) + +# In newer versions of buck2 this toolchain is needed `noop_test_toolchain`. +noop_test_toolchain( + name = "test", + visibility = ["PUBLIC"], +) diff --git a/buck2/cpp/toolchains/tools.bzl b/buck2/cpp/toolchains/tools.bzl index b62e65bd..1c3bc720 100644 --- a/buck2/cpp/toolchains/tools.bzl +++ b/buck2/cpp/toolchains/tools.bzl @@ -38,4 +38,4 @@ def _path_clang_tools_impl(_ctx) -> list[Provider]: path_clang_tools = rule( impl = _path_clang_tools_impl, attrs = {}, -) \ No newline at end of file +)