Test path mapping in Bazel (bazelbuild#18155)]

Add end-to-end tests to verify that `--experimental_output_paths=strip` works with sandboxed, sandboxed worker, and remote execution and results in the desired cache hits. Also adapt and enable `config_stripped_outputs_test` for Bazel (e.g. make it work on macOS).

This requires a minor changes: We ensure that full `.jdeps` are rewritten to contain unmapped paths even after a fallback to full classpath.

Closes bazelbuild#18155.

PiperOrigin-RevId: 582014351
Change-Id: If9c51acb090d9fc4bc86052143ad9960e44fd8a7

src/main/java/com/google/devtools/build/lib/rules/java/JavaCompileAction.java

@@ -489,6 +489,12 @@ public ActionResult execute(ActionExecutionContext actionExecutionContext)
           .insertDependencies(
               outputDepsProto,
               readFullOutputDeps(fallbackResults, actionExecutionContext, spawn.getPathMapper()));
+    } else if (!spawn.getPathMapper().isNoop()) {
+      // As a side effect, readFullOutputDeps rewrites the on-disk .jdeps file from mapped to
+      // unmapped paths. To make path mapping fully transparent to consumers of this action's
+      // output, we ensure that the file always contains unmapped paths.
+      var unused =
+          readFullOutputDeps(fallbackResults, actionExecutionContext, spawn.getPathMapper());
     }
     return ActionResult.create(
         ImmutableList.copyOf(Iterables.concat(primaryResults, fallbackResults)));

src/test/shell/bazel/BUILD

@@ -1306,6 +1306,20 @@ sh_test(
     deps = ["@bazel_tools//tools/bash/runfiles"],
 )
 
+sh_test(
+    name = "path_mapping_test",
+    srcs = ["path_mapping_test.sh"],
+    data = [
+        ":test-deps",
+        "//src/tools/remote:worker",
+    ],
+    tags = ["no_windows"],
+    deps = [
+        "//src/test/shell/bazel/remote:remote_utils",
+        "@bazel_tools//tools/bash/runfiles",
+    ],
+)
+
 sh_test(
     name = "platforms_test",
     srcs = ["platforms_test.sh"],

src/test/shell/bazel/path_mapping_test.sh

@@ -0,0 +1,289 @@
+#!/bin/bash
+#
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Tests path mapping support of Bazel's executors.
+
+set -euo pipefail
+
+# --- begin runfiles.bash initialization ---
+if [[ ! -d "${RUNFILES_DIR:-/dev/null}" && ! -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
+  if [[ -f "$0.runfiles_manifest" ]]; then
+    export RUNFILES_MANIFEST_FILE="$0.runfiles_manifest"
+  elif [[ -f "$0.runfiles/MANIFEST" ]]; then
+    export RUNFILES_MANIFEST_FILE="$0.runfiles/MANIFEST"
+  elif [[ -f "$0.runfiles/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
+    export RUNFILES_DIR="$0.runfiles"
+  fi
+fi
+if [[ -f "${RUNFILES_DIR:-/dev/null}/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
+  source "${RUNFILES_DIR}/bazel_tools/tools/bash/runfiles/runfiles.bash"
+elif [[ -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
+  source "$(grep -m1 "^bazel_tools/tools/bash/runfiles/runfiles.bash " \
+            "$RUNFILES_MANIFEST_FILE" | cut -d ' ' -f 2-)"
+else
+  echo >&2 "ERROR: cannot find @bazel_tools//tools/bash/runfiles:runfiles.bash"
+  exit 1
+fi
+# --- end runfiles.bash initialization ---
+
+source "$(rlocation "io_bazel/src/test/shell/integration_test_setup.sh")" \
+  || { echo "integration_test_setup.sh not found!" >&2; exit 1; }
+source "$(rlocation "io_bazel/src/test/shell/bazel/remote_helpers.sh")" \
+  || { echo "remote_helpers.sh not found!" >&2; exit 1; }
+source "$(rlocation "io_bazel/src/test/shell/bazel/remote/remote_utils.sh")" \
+  || { echo "remote_utils.sh not found!" >&2; exit 1; }
+
+function set_up() {
+  start_worker
+
+  mkdir -p src/main/java/com/example
+  cat > src/main/java/com/example/BUILD <<'EOF'
+java_binary(
+    name = "Main",
+    srcs = ["Main.java"],
+    deps = [":lib"],
+)
+java_library(
+    name = "lib",
+    srcs = ["Lib.java"],
+)
+EOF
+  cat > src/main/java/com/example/Main.java <<'EOF'
+package com.example;
+public class Main {
+  public static void main(String[] args) {
+    System.out.println(Lib.getGreeting());
+  }
+}
+EOF
+  cat > src/main/java/com/example/Lib.java <<'EOF'
+package com.example;
+public class Lib {
+  public static String getGreeting() {
+    return "Hello, World!";
+  }
+}
+EOF
+}
+
+function tear_down() {
+  bazel clean >& $TEST_log
+  stop_worker
+}
+
+function test_path_stripping_sandboxed() {
+  if is_windows; then
+    echo "Skipping test_path_stripping_sandboxed on Windows as it requires sandboxing"
+    return
+  fi
+
+  cache_dir=$(mktemp -d)
+
+  bazel run -c fastbuild \
+    --disk_cache=$cache_dir \
+    --experimental_output_paths=strip \
+    --strategy=Javac=sandboxed \
+    //src/main/java/com/example:Main &> $TEST_log || fail "run failed unexpectedly"
+  expect_log 'Hello, World!'
+  # JavaToolchainCompileBootClasspath, JavaToolchainCompileClasses, 1x header compilation and 2x
+  # actual compilation.
+  expect_log '5 \(linux\|darwin\|processwrapper\)-sandbox'
+  expect_not_log 'disk cache hit'
+
+  bazel run -c opt \
+    --disk_cache=$cache_dir \
+    --experimental_output_paths=strip \
+    --strategy=Javac=sandboxed \
+    //src/main/java/com/example:Main &> $TEST_log || fail "run failed unexpectedly"
+  expect_log 'Hello, World!'
+  expect_log '5 disk cache hit'
+  expect_not_log '[0-9] \(linux\|darwin\|processwrapper\)-sandbox'
+}
+
+function test_path_stripping_singleplex_worker() {
+  if is_windows; then
+    echo "Skipping test_path_stripping_singleplex_worker on Windows as it requires sandboxing"
+    return
+  fi
+
+  cache_dir=$(mktemp -d)
+
+  bazel run -c fastbuild \
+    --disk_cache=$cache_dir \
+    --experimental_output_paths=strip \
+    --strategy=Javac=worker \
+    --worker_sandboxing \
+    --noexperimental_worker_multiplex \
+    //src/main/java/com/example:Main &> $TEST_log || fail "run failed unexpectedly"
+  expect_log 'Hello, World!'
+  # JavaToolchainCompileBootClasspath, JavaToolchainCompileClasses and header compilation.
+  expect_log '3 \(linux\|darwin\|processwrapper\)-sandbox'
+  # Actual compilation actions.
+  expect_log '2 worker'
+  expect_not_log 'disk cache hit'
+
+  bazel run -c opt \
+    --disk_cache=$cache_dir \
+    --experimental_output_paths=strip \
+    --strategy=Javac=worker \
+    --worker_sandboxing \
+    --noexperimental_worker_multiplex \
+    //src/main/java/com/example:Main &> $TEST_log || fail "run failed unexpectedly"
+  expect_log 'Hello, World!'
+  expect_log '5 disk cache hit'
+  expect_not_log '[0-9] \(linux\|darwin\|processwrapper\)-sandbox'
+  expect_not_log '[0-9] worker'
+}
+
+function test_path_stripping_remote() {
+  bazel run -c fastbuild \
+    --experimental_output_paths=strip \
+    --remote_executor=grpc://localhost:${worker_port} \
+    //src/main/java/com/example:Main &> $TEST_log || fail "run failed unexpectedly"
+  expect_log 'Hello, World!'
+  # JavaToolchainCompileBootClasspath, JavaToolchainCompileClasses, 1x header compilation and 2x
+  # actual compilation.
+  expect_log '5 remote'
+  expect_not_log 'remote cache hit'
+
+  bazel run -c opt \
+    --experimental_output_paths=strip \
+    --remote_executor=grpc://localhost:${worker_port} \
+    //src/main/java/com/example:Main &> $TEST_log || fail "run failed unexpectedly"
+  expect_log 'Hello, World!'
+  expect_log '5 remote cache hit'
+  # Do not match "5 remote cache hit", which is expected.
+  expect_not_log '[0-9] remote[^ ]'
+}
+
+function test_path_stripping_remote_multiple_configs() {
+  mkdir rules
+  cat > rules/defs.bzl <<'EOF'
+LocationInfo = provider(fields = ["location"])
+
+def _location_setting_impl(ctx):
+    return LocationInfo(location = ctx.build_setting_value)
+
+location_setting = rule(
+    implementation = _location_setting_impl,
+    build_setting = config.string(),
+)
+
+def _location_transition_impl(settings, attr):
+    return {"//rules:location": attr.location}
+
+_location_transition = transition(
+    implementation = _location_transition_impl,
+    inputs = [],
+    outputs = ["//rules:location"],
+)
+
+def _bazelcon_greeting_impl(ctx):
+    content = """
+package com.example.{package};
+
+public class Lib {{
+  public static String getGreeting() {{
+    return String.format("Hello, BazelCon {location}!");
+  }}
+}}
+""".format(
+        package = ctx.attr.name,
+        location = ctx.attr.location,
+    )
+    file = ctx.actions.declare_file("Lib.java")
+    ctx.actions.write(file, content)
+    return [
+        DefaultInfo(files = depset([file])),
+    ]
+
+bazelcon_greeting = rule(
+    _bazelcon_greeting_impl,
+    cfg = _location_transition,
+    attrs = {
+        "location": attr.string(),
+    },
+)
+EOF
+  cat > rules/BUILD << 'EOF'
+load("//rules:defs.bzl", "location_setting")
+
+location_setting(
+    name = "location",
+    build_setting_default = "",
+)
+EOF
+
+  mkdir -p src/main/java/com/example
+  cat > src/main/java/com/example/BUILD <<'EOF'
+load("//rules:defs.bzl", "bazelcon_greeting")
+java_binary(
+    name = "Main",
+    srcs = ["Main.java"],
+    deps = [":lib"],
+)
+java_library(
+    name = "lib",
+    srcs = [
+        ":munich",
+        ":new_york",
+    ],
+)
+bazelcon_greeting(
+    name = "munich",
+    location = "Munich",
+)
+bazelcon_greeting(
+    name = "new_york",
+    location = "New York",
+)
+EOF
+  cat > src/main/java/com/example/Main.java <<'EOF'
+package com.example;
+public class Main {
+  public static void main(String[] args) {
+    System.out.println(com.example.new_york.Lib.getGreeting());
+    System.out.println(com.example.munich.Lib.getGreeting());
+  }
+}
+EOF
+
+  bazel run -c fastbuild \
+    --experimental_output_paths=strip \
+    --remote_executor=grpc://localhost:${worker_port} \
+    //src/main/java/com/example:Main &> $TEST_log || fail "run failed unexpectedly"
+  expect_log 'Hello, BazelCon New York!'
+  expect_log 'Hello, BazelCon Munich!'
+  # JavaToolchainCompileBootClasspath, JavaToolchainCompileClasses, 1x header compilation and 2x
+  # actual compilation.
+  expect_log '5 remote'
+  expect_not_log 'remote cache hit'
+
+  bazel run -c opt \
+    --experimental_output_paths=strip \
+    --remote_executor=grpc://localhost:${worker_port} \
+    //src/main/java/com/example:Main &> $TEST_log || fail "run failed unexpectedly"
+  expect_log 'Hello, BazelCon New York!'
+  expect_log 'Hello, BazelCon Munich!'
+  # JavaToolchainCompileBootClasspath, JavaToolchainCompileClasses and compilation of the binary.
+  expect_log '3 remote cache hit'
+  # Do not match "[0-9] remote cache hit", which is expected separately.
+  # Header and actual compilation of the library, which doesn't use path stripping as it would
+  # result in ambiguous paths due to the multiple configs.
+  expect_log '2 remote[^ ]'
+}
+
+run_suite "path mapping tests"

src/test/shell/bazel/remote/BUILD

@@ -14,6 +14,7 @@ sh_library(
         "//src/tools/remote:worker",
         "//src/tools/remote:worker_deploy.jar",
     ],
+    visibility = ["//src/test/shell/bazel:__pkg__"],
 )
 
 sh_test(

src/test/shell/integration/BUILD

@@ -877,6 +877,8 @@ sh_test(
         ":test-deps",
         "@bazel_tools//tools/bash/runfiles",
     ],
+    # Bazel test requires sandboxing
+    tags = ["no_windows"],
 )
 
 sh_library(

src/test/shell/integration/config_stripped_outputs_lib.sh

@@ -40,7 +40,10 @@ function assert_paths_stripped() {
   found_identifying_output=0
 
   # Check every output path in the action command line is stripped.
-  for o in $(echo $cmd | xargs -d' ' -n 1 | egrep -o "${bazel_out}[^)]*"); do
+  # Note: keep subshells out of for loop headers as otherwise failures aren't propagated by
+  # "set -e" and silently ignored.
+  output_paths=$(echo "$cmd" | tr -s ' ' '\n' | xargs -n 1 | grep -E -o "${bazel_out}[^)]*")
+  for o in $output_paths; do
     echo "$o" | grep -v "${bazel_out}/bin" \
       && fail "expected all \"${bazel_out}\" paths to start with " \
       "\"${bazel_out}/bin.*\": $o"
@@ -49,11 +52,18 @@ function assert_paths_stripped() {
     fi
   done
 
-  # Check every output path in every .params file is stripped.
-  for o in $(echo $cmd | xargs -d' ' -n 1 | egrep -o  "${bazel_out}[^)]*.params"); do
+  # Check every output path in every .params file is stripped. Do not fail if there are no param
+  # files.
+  param_files=$(echo "$cmd" | tr -s ' ' '\n' | xargs -n 1 | grep -E -o "${bazel_out}[^)]*.params" || true)
+  for o in $param_files; do
     bin_relative_path=$(echo $o | sed -r "s|${bazel_out}/bin/||")
     local_path="${bazel_out:0:5}-bin/${bin_relative_path}"
-    for k in $(grep "${bazel_out}" $local_path); do
+    # Don't fail if the file doesn't contain any output paths, but do fail if it doesn't exist.
+    if [[ ! -f "$local_path" ]]; then
+      fail "expected param file to exist: $local_path"
+    fi
+    param_file_paths=$(grep "${bazel_out}" $local_path || true)
+    for k in $param_file_paths; do
       echo "$k" | grep -v "${bazel_out}/bin" \
         && fail "$local_path: expected all \"${bazel_out}\" paths to start " \
         "with \"${bazel_out}/bin.*\": $k"
@@ -65,3 +75,15 @@ function assert_paths_stripped() {
 
   assert_equals 1 "$found_identifying_output"
 }
+
+# Asserts a file contains no stripped paths.
+#
+# Arguments:
+#   Path to the file.
+function assert_contains_no_stripped_path() {
+  # For "bazel-out/x86-fastbuild/bin/...", return "bazel-out".
+  output_path=$(bazel info | grep '^output_path:')
+  stripped_bin="${output_path##*/}/bin"
+
+  assert_not_contains "$stripped_bin" "$1" "Stripped path found in $1"
+}