Update cross compile toolchains

WORKSPACE

@@ -3,16 +3,6 @@ workspace(name = "prysm")
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
 
-http_archive(
-    name = "bazel_toolchains",
-    sha256 = "8e0633dfb59f704594f19ae996a35650747adc621ada5e8b9fb588f808c89cb0",
-    strip_prefix = "bazel-toolchains-3.7.0",
-    urls = [
-        "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/releases/download/3.7.0/bazel-toolchains-3.7.0.tar.gz",
-        "https://github.com/bazelbuild/bazel-toolchains/releases/download/3.7.0/bazel-toolchains-3.7.0.tar.gz",
-    ],
-)
-
 http_archive(
     name = "com_grail_bazel_toolchain",
     sha256 = "b210fc8e58782ef171f428bfc850ed7179bdd805543ebd1aa144b9c93489134f",
@@ -39,10 +29,6 @@ load("@prysm//tools/cross-toolchain:prysm_toolchains.bzl", "configure_prysm_tool
 
 configure_prysm_toolchains()
 
-load("@prysm//tools/cross-toolchain:rbe_toolchains_config.bzl", "rbe_toolchains_config")
-
-rbe_toolchains_config()
-
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
 http_archive(

beacon-chain/package/BUILD.bazel

@@ -46,6 +46,7 @@ pkg_deb(
     package = "prysm-beacon-chain",
     postinst = "postinst.sh",
     preinst = "preinst.sh",
+    tags = ["no-remote"],
     version_file = "//runtime:version_file",
     visibility = ["//beacon-chain:__pkg__"],
 )

build/bazelrc/cross.bazelrc

@@ -2,8 +2,10 @@
 -----------------------------------------------
 
 build:cross --crosstool_top=@prysm_toolchains//:multiarch_toolchain
-build:cross --host_platform=@io_bazel_rules_go//go/toolchain:linux_amd64
+build:cross --extra_execution_platforms=@//tools/cross-toolchain/configs/config:platform
+build:cross --extra_toolchains=@prysm_toolchains//:cc-toolchain-multiarch
 build:cross --host_crosstool_top=@prysm_toolchains//:hostonly_toolchain
+build:cross --platforms=@//tools/cross-toolchain/configs/config:platform
 
 # linux_amd64 config for cross compiler toolchain, not strictly necessary since host/exec env is amd64
 build:linux_amd64 --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64_cgo
@@ -21,10 +23,10 @@ build:osx_arm64 --cpu=aarch64
 
 # windows
 build:windows_amd64 --config=cross
-build:windows_amd64  --platforms=@io_bazel_rules_go//go/toolchain:windows_amd64_cgo
+build:windows_amd64 --platforms=@io_bazel_rules_go//go/toolchain:windows_amd64_cgo
 build:windows_amd64 --compiler=mingw-w64
 
-# linux_arm64 conifg for cross compiler toolchain
+# linux_arm64 config for cross compiler toolchain
 build:linux_arm64 --config=cross
 build:linux_arm64 --platforms=@io_bazel_rules_go//go/toolchain:linux_arm64_cgo
 build:linux_arm64 --copt=-funsafe-math-optimizations
@@ -38,20 +40,17 @@ build:linux_arm64 --copt=-march=armv8-a
 # Docker Sandbox Configs
 #-----------------------
 # Note all docker sandbox configs must run from a linux x86_64 host
-# build:docker-sandbox --experimental_docker_image=gcr.io/prysmaticlabs/rbe-worker:latest
+build:docker-sandbox --unconditional_warning="⚠️  Docker sandbox is extremely slow! Consider a remote executor, if possible. See tools/cross-toolchain/README.md#Caveats ⚠️"
 build:docker-sandbox --spawn_strategy=docker --strategy=Javac=docker --genrule_strategy=docker
 build:docker-sandbox --define=EXECUTOR=remote
 build:docker-sandbox --experimental_docker_verbose
 build:docker-sandbox --experimental_enable_docker_sandbox
-build:docker-sandbox --crosstool_top=@rbe_ubuntu_clang//cc:toolchain 
-build:docker-sandbox --host_javabase=@rbe_ubuntu_clang//java:jdk 
-build:docker-sandbox --javabase=@rbe_ubuntu_clang//java:jdk 
-build:docker-sandbox --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 
-build:docker-sandbox --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 
-build:docker-sandbox --extra_execution_platforms=@rbe_ubuntu_clang//config:platform 
-build:docker-sandbox --host_platform=@rbe_ubuntu_clang//config:platform 
-build:docker-sandbox --platforms=@rbe_ubuntu_clang//config:platform
-build:docker-sandbox --extra_toolchains=@prysm_toolchains//:cc-toolchain-multiarch
+
+# Remote with docker sandbox fallback
+build:remote --spawn_strategy=remote,docker,linux-sandbox --strategy=Javac=remote,docker,linux-sandbox --genrule_strategy=remote,docker,linux-sandbox
+build:remote --define=EXECUTOR=remote
+build:remote --experimental_docker_verbose
+build:remote --experimental_enable_docker_sandbox
 
 # windows_amd64 docker sandbox build config
 build:windows_amd64_docker --config=docker-sandbox --config=windows_amd64

tools/cross-toolchain/Dockerfile

@@ -1,15 +1,6 @@
 # Prysmatic Labs Remote Build Execution Image with Cross Compile Support.
 #
-# Update instructions.
-# - Build the docker image.
-# - Push to gcr.io.
-# - Update _PRYSM_BUILD_IMAGE_DIGEST in //tools/cross-toolchain/rbe_toolchains_config.bzl
-# - Run ./tools/cross-toolchain/regenerate.sh
-# - Add and commit the newly generated configs.
-# - Done!
-#
-# docker build -t gcr.io/prysmaticlabs/rbe-worker:latest .
-# gcloud docker -- push gcr.io/prysmaticlabs/rbe-worker:latest 
+# See README.md for update instructions.
 
 FROM debian:bullseye-slim as build
 
@@ -22,7 +13,7 @@ RUN apt-get update && \
         gcc-aarch64-linux-gnu g++-aarch64-linux-gnu \
         cmake libssl-dev libxml2-dev vim apt-transport-https \
         zip unzip libtinfo5 patch zlib1g-dev autoconf libtool \
-        pkg-config make docker.io gnupg2 libgmp-dev python
+        pkg-config make docker.io gnupg2 libgmp-dev python3 openjdk-17-jdk-headless
 
 # install llvm/clang cross-build toolchains
 ENV INSTALL_LLVM_VERSION=12.0.0
@@ -38,6 +29,7 @@ RUN /tmp/link_osxcross.sh
 
 # containerized development environment
 FROM build as devel
-RUN mkdir /workdir
-WORKDIR /workdir
+
+ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64/
+
 RUN echo 'PS1="\[$(tput setaf 3)$(tput bold)[\]devel@\\h:\\W]#\[$(tput sgr0) \]"' >> /root/.bashrc

tools/cross-toolchain/README.md

@@ -11,24 +11,71 @@ This package declares a c++ toolchain suite with cross compilers for targeting f
 
 This toolchain suite describes cross compile configuration with a Dockerfile with the appropriate host dependencies. These toolchains can be used locally (see [caveats](#caveats)), [Remote Build Execution (RBE)](https://docs.bazel.build/versions/master/remote-execution.html), and in a docker sandbox (like RBE, but local).
 
+### Updating the toolchain suite
+
+1) Build and push the worker docker image, if necessary.
+
+```bash
+docker build -t gcr.io/prysmaticlabs/rbe-worker:latest tools/cross-toolchain/.
+docker push gcr.io/prysmaticlabs/rbe-worker:latest 
+```
+
+Note: You must configured your gcr access credentials to push to gcr.io/prysmaticlabs. Run `gcloud auth configure-docker` or contact SRE team for push access.
+
+2) Note the docker image sha256 digest from the recently pushed image or use the latest one available.
+
+3) Download and run [rbe_configs_gen](https://github.com/bazelbuild/bazel-toolchains#rbe_configs_gen---cli-tool-to-generate-configs) CLI tool.
+
+```bash
+# Run from the root of the Prysm repo.
+rbe_configs_gen \
+  --bazel_version=$(cat .bazelversion) \
+  --target_os=linux \
+  --exec_os=linux \
+  --output_src_root=tools/cross-toolchain/configs \
+  --generate_cpp_configs=true \
+  --generate_java_configs=true \
+  --cpp_env_json=tools/cross-toolchain/cpp_env_clang.json \
+  --toolchain_container=gcr.io/prysmaticlabs/rbe-worker@sha256:9a30c5ff7dcd1fbc5cb5b5f320782dec755a31cd2f6811e238a8584d94b04e78 # The sha256 digest from step 2.
+```
+
+4) Test the builds work locally for all supported platforms.
+
+```bash
+bazel build --config=release --config=linux_amd64 --config=llvm //cmd/beacon-chain //cmd/validator //cmd/client-stats //cmd/prysmctl
+bazel build --config=release --config=linux_arm64_docker //cmd/beacon-chain //cmd/validator //cmd/client-stats //cmd/prysmctl
+bazel build --config=release --config=osx_amd64_docker //cmd/beacon-chain //cmd/validator //cmd/client-stats //cmd/prysmctl
+bazel build --config=release --config=osx_arm64_docker //cmd/beacon-chain //cmd/validator //cmd/client-stats //cmd/prysmctl 
+bazel build --config=release --config=windows_amd64_docker //cmd/beacon-chain //cmd/validator //cmd/client-stats //cmd/prysmctl
+```
+
+5) Run gazelle.
+
+```bash
+bazel run //:gazelle
+```
+
+6) Add and commit the newly generated configs.
+
+7) Done!
 
 ### Cross compile target support
 
-| target           | linux_amd64 | linux_arm64 | osx_amd64 | osx_arm64 | windows_amd64                     |
-|------------------|-------------------|------------------|-----------------|-----------------|-----------------------------------|
+| target           | linux_amd64                                                        | linux_arm64                                | osx_amd64                          | osx_arm64                          | windows_amd64                      |
+|------------------|--------------------------------------------------------------------|--------------------------------------------|------------------------------------|------------------------------------|------------------------------------|
 | `//beacon-chain` | :heavy_check_mark:  docker-sandbox and RBE, supported locally only | :heavy_check_mark:  docker-sandbox and RBE | :heavy_check_mark:  docker-sandbox | :heavy_check_mark:  docker-sandbox | :heavy_check_mark:  docker-sandbox |
-| `//validator`    | :heavy_check_mark:  docker-sandbox and RBE | :heavy_check_mark: docker-sandbox and RBE | :heavy_check_mark:  docker-sandbox | :heavy_check_mark:  docker-sandbox | :heavy_check_mark:                 |
+| `//validator`    | :heavy_check_mark:  docker-sandbox and RBE                         | :heavy_check_mark: docker-sandbox and RBE  | :heavy_check_mark:  docker-sandbox | :heavy_check_mark:  docker-sandbox | :heavy_check_mark:                 |
 
 The configurations above are enforced via pull request presubmit checks.
 
 ### Bazel config flag values
 
 Use these values with `--config=<flag>`, multiple times if more than one value is defined in the table. Example: `bazel build //beacon-chain --config=windows_amd64_docker` to build windows binary in a docker sandbox.
 
-| Config                        | linux_amd64 | linux_arm64 | osx_amd64                 | osx_arm64                 | windows_amd64                |
-|-------------------------------|-------------------|------------------|---------------------------|---------------------------|------------------------------|
-| Local run                     | `linux_amd64` | `linux_arm64` | `osx_amd64`               | `osx_arm64`               | `windows_amd64`              | 
-| Docker sandbox                | `linux_amd64_docker` | `linux_arm64_docker` | `osx_amd64_docker`        | `osx_arm64_docker`        | `windows_amd64_docker `      |
+| Config                        | linux_amd64                | linux_arm64                 | osx_amd64                 | osx_arm64                 | windows_amd64                 |
+|-------------------------------|----------------------------|-----------------------------|---------------------------|---------------------------|-------------------------------|
+| Local run                     | `linux_amd64`              | `linux_arm64`               | `osx_amd64`               | `osx_arm64`               | `windows_amd64`               |
+| Docker sandbox                | `linux_amd64_docker`       | `linux_arm64_docker`        | `osx_amd64_docker`        | `osx_arm64_docker`        | `windows_amd64_docker `       |
 | RBE (See [Caveats](#caveats)) | `linux_amd64` and `remote` | `linux_arm64`  and `remote` | `osx_amd64`  and `remote` | `osx_arm64`  and `remote` | `windows_amd64`  and `remote` |
 
 ### Caveats