Migrate to Bzlmod (#626)

The new builds bootstrap the crate_universe binary which increases compatibility with various platforms. New supported builds: - Native Bazel on MacOS - Nix-wrapped Cargo fon MacOS - Native Bazel builds with zig-cc on Ubuntu 20.04 and 22.04 In addition, add a new `--config=linux_zig` which allows building nativelink with a more hermetic (but not perfectly hermetic) toolchain on systems that can't use the LRE toolchains. The zig-cc toolchain runs a lot slower than the LRE toolchains but is more similar to the LRE toolchain and at least somewhat reduces variation between compilers. This commit is a slight regression in terms of hermeticity and reproducibility. The bzlmod migration moved the Cargo.Bazel.lock file to an autogenerated MODULE.bazel.lock, but we have to gitignore that file for now since rules_rust currently doesn't create the same lockfiles on Linux and MacOS.
TraceMachina · Jan 28, 2024 · 2a89ce6 · 2a89ce6
1 parent 63826f2
commit 2a89ce6
Show file tree

Hide file tree

Showing 43 changed files with 3,512 additions and 21,540 deletions.
diff --git a/.bazelignore b/.bazelignore
@@ -9,3 +9,4 @@ bazel-remote-nativelink
 bazel-root
 bazel-testlogs
 bazel-nativelink
+local-remote-execution
diff --git a/.bazelrc b/.bazelrc
@@ -12,7 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Use blake3 as digest function instead of the much slower default sha256.
 startup --digest_function=blake3
+common --enable_platform_specific_config
+
+# TODO(aaronmondal): Remove this once we get the build working without it.
+common --noincompatible_sandbox_hermetic_tmp
+
+# Don't leak PATH and LD_LIBRARY_PATH into the build.
+build --incompatible_strict_action_env
+
+# Don't use legacy toolchain resolution.
+build --incompatible_enable_cc_toolchain_resolution
 
 # Use the earliest supported C++ version for protoc.
 build --cxxopt=-std=c++14 --host_cxxopt=-std=c++14
@@ -57,19 +68,39 @@ startup --windows_enable_symlinks
 build:windows --cxxopt=/std:c++14 --host_cxxopt=/std:c++14
 build:windows --enable_runfiles
 
-# Local remote execution flags.
+# Global rust toolchain configuration. Deferred to here so that the cc
+# toolchains are resolved before the rust toolchains.
+build --extra_toolchains=@rust_toolchains//:all
 
-# TODO(aaronmondal): rules_rust doesn't work with this enabled.
+# Option to test the zig toolchain on Linux. Prefer the default `linux`
+# toolchain which builds cc targets roughly twice as fast.
+#
+# WARNING:
 #
-# Tracked in: https://github.com/TraceMachina/nativelink/issues/477.
-build:lre --incompatible_enable_cc_toolchain_resolution
+# We're using an incredibly old target glibc here. Builds created with this
+# toolchain have maximum compatibility (theoretically down to Ubuntu 18), but
+# miss out on half a decade of optimizations. Don't use this for production
+# builds if you're running a non-ancient OS and care about performance.
+#
+# TODO(aaronmondal): Migrate to a statically linked musl as soon as rules_rust
+#                    supports it. This way we get to keep (or even improve)
+#                    backwards compatibility without sacrificing performance.
+build:linux_zig --host_platform=@zig_sdk//platform:linux_amd64
+build:linux_zig --extra_toolchains=@zig_sdk//libc_aware/toolchain:linux_amd64_gnu.2.28
+build:linux_zig --repo_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
 
-# TODO(aaronmondal): Remove when https://github.com/bazelbuild/bazel/issues/7254
-#                    is resolved.
-build:lre --define=EXECUTOR=remote
+
+# Local Remote Execution.
+build:lre --host_platform=@local-remote-execution//generated/config:platform
+build:lre --extra_toolchains=@local-remote-execution//generated/config:cc-toolchain"
+build:lre --extra_toolchains=@local-remote-execution//generated/java:all"
 
 # See: https://github.com/bazelbuild/bazel/issues/19714#issuecomment-1745604978
 build:lre --repo_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
 
+# TODO(aaronmondal): Remove when https://github.com/bazelbuild/bazel/issues/7254
+#                    is resolved.
+build:lre --define=EXECUTOR=remote
+
 # Allow user-side customization.
 try-import %workspace%/.bazelrc.user
diff --git a/.bazelversion b/.bazelversion
@@ -1 +1 @@
-6.4.0
+7.0.0
diff --git a/.github/workflows/lre.yaml b/.github/workflows/lre.yaml
@@ -37,7 +37,7 @@ jobs:
           bash -c "bazel run \
            --config=lre \
            --verbose_failures \
-           //local-remote-execution/examples:hello_lre"
+           @local-remote-execution//examples:hello_lre"
 
   remote:
     strategy:
@@ -105,4 +105,4 @@ jobs:
             --remote_cache=grpc://$cache_ip:50051 \
             --remote_executor=grpc://$scheduler_ip:50052 \
             --verbose_failures \
-            //local-remote-execution/examples:hello_lre"
+            @local-remote-execution//examples:hello_lre"
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -50,15 +50,15 @@ jobs:
         cd ../../ && \
         docker run --rm --net=host -w /root/nativelink -v $PWD:/root/nativelink trace_machina/nativelink:builder sh -c ' \
           bazel clean && \
-          bazel test //... \
+          bazel test --config=linux_zig //... \
           --remote_instance_name=main \
           --remote_cache=grpc://127.0.0.1:50051 \
           --remote_executor=grpc://127.0.0.1:50052 \
           --remote_default_exec_properties=cpu_count=1 \
         ' && \
         docker run --rm --net=host -w /root/nativelink -v $PWD:/root/nativelink trace_machina/nativelink:builder sh -c ' \
           bazel clean && \
-          bazel test //... \
+          bazel test --config=linux_zig //... \
           --remote_instance_name=main \
           --remote_cache=grpc://127.0.0.1:50051 \
           --remote_executor=grpc://127.0.0.1:50052 \

diff --git a/.github/workflows/native-bazel.yaml b/.github/workflows/native-bazel.yaml
@@ -14,7 +14,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, ubuntu-22.04]
+        os: [ubuntu-22.04, macos-13]
     name: ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     steps:
@@ -67,3 +67,35 @@ jobs:
             exit 1
           fi
         shell: bash
+
+  zig-cc-unit-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-20.04, ubuntu-22.04]
+    name: zig-cc ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    steps:
+
+      - name: Checkout
+        uses: >- # v4.1.1
+          actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+
+      - name: Setup Bazelisk
+        uses: >- # Custom commit at 2023-10-23
+          bazelbuild/setup-bazelisk@5bc144ec07bd24eaa9b5c3e8c9af4006b033b629
+
+      - name: Mount bazel cache
+        uses: >- # v3.3.2
+          actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84
+        with:
+          path: |
+            ~/.cache/bazel
+          key: |
+            ${{ matrix.os }}-bazel-legacy-${{ hashFiles('.bazelversion', '.bazelrc', 'WORKSPACE.bazel', 'MODULE.bazel') }}
+          restore-keys: |
+            ${{ matrix.os }}-bazel-legacy-
+
+      - name: Run Bazel tests
+        run: |
+          bazel test --config=linux_zig //... --verbose_failures
diff --git a/.github/workflows/nix.yaml b/.github/workflows/nix.yaml
@@ -39,15 +39,24 @@ jobs:
           key: ${{ runner.os }}-bazel-nix
 
       - name: Invoke Bazel build in Nix shell
-        run: >
-          nix develop --impure --command
-          bash -c "bazel test ... --verbose_failures"
+        run: |
+          if [ "$RUNNER_OS" == "Linux" ]; then
+            nix develop --impure --command \
+            bash -c "bazel test --config=linux_zig ... --verbose_failures"
+          elif [ "$RUNNER_OS" == "macOS" ]; then
+            nix develop --impure --command \
+            bash -c "bazel test //... --verbose_failures"
+          else
+            echo "Unsupported runner OS: $RUNNER_OS"
+            exit 1
+          fi
+        shell: bash
 
   nix-cargo:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04]
+        os: [ubuntu-22.04, macos-13]
     name: Cargo Dev / ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     steps:

diff --git a/.github/workflows/sanitizers.yaml b/.github/workflows/sanitizers.yaml
@@ -36,5 +36,5 @@ jobs:
       #     key: ${{ matrix.os }}-bazel-${{ matrix.sanitizer }}
 
       - name: Run Bazel tests
-        run: bazel test --config=${{ matrix.sanitizer }} //...
+        run: bazel test --config=${{ matrix.sanitizer }} --verbose_failures //...
         shell: bash
diff --git a/.gitignore b/.gitignore
@@ -12,3 +12,4 @@ __pycache__
 .pre-commit-config.yaml
 result
 .bazelrc.user
+MODULE.bazel.lock
diff --git a/BUILD.bazel b/BUILD.bazel
@@ -21,23 +21,23 @@ rust_binary(
         "//nativelink-store",
         "//nativelink-util",
         "//nativelink-worker",
-        "@crate_index//:async-lock",
-        "@crate_index//:axum",
-        "@crate_index//:clap",
-        "@crate_index//:console-subscriber",
-        "@crate_index//:futures",
-        "@crate_index//:hyper",
-        "@crate_index//:parking_lot",
-        "@crate_index//:prometheus-client",
-        "@crate_index//:rustls-pemfile",
-        "@crate_index//:scopeguard",
-        "@crate_index//:serde_json5",
-        "@crate_index//:tokio",
-        "@crate_index//:tokio-rustls",
-        "@crate_index//:tonic",
-        "@crate_index//:tower",
-        "@crate_index//:tracing",
-        "@crate_index//:tracing-subscriber",
+        "@crates//:async-lock",
+        "@crates//:axum",
+        "@crates//:clap",
+        "@crates//:console-subscriber",
+        "@crates//:futures",
+        "@crates//:hyper",
+        "@crates//:parking_lot",
+        "@crates//:prometheus-client",
+        "@crates//:rustls-pemfile",
+        "@crates//:scopeguard",
+        "@crates//:serde_json5",
+        "@crates//:tokio",
+        "@crates//:tokio-rustls",
+        "@crates//:tonic",
+        "@crates//:tower",
+        "@crates//:tracing",
+        "@crates//:tracing-subscriber",
     ],
 )
 

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -64,19 +64,6 @@ various Cloud CLIs for you:
     @rules_rust//:rustfmt
   ```
 
-## Updating Rust dependencies
-
-After modifying the corresponding `Cargo.toml` file in either the top level or
-one of the crate subdirectories run the following command to rebuild the crate
-index:
-
-```
-CARGO_BAZEL_REPIN=1 bazel sync --only=crate_index
-```
-
-This updates `Cargo.lock` and `Cargo.Bazel.lock` with the new dependency
-information.
-
 ## Conduct
 
 Native Link Code of Conduct is available in the