Add flag experimental_throttle_remote_action_building

to allow users temporarily disable remote action building throttle. Workaround for #20478. Closes #20558. PiperOrigin-RevId: 597445193 Change-Id: Ib2c7133adf86139b35156d94e39cbf9e17906439
bazelbuild · Jan 11, 2024 · 294c904 · 294c904
1 parent f0ade80
commit 294c904
Show file tree

Hide file tree

Showing 2 changed files with 37 additions and 10 deletions.
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java
@@ -535,13 +535,29 @@ private ToolSignature getToolSignature(Spawn spawn, SpawnExecutionContext contex
         : null;
   }
 
+  private void maybeAcquireRemoteActionBuildingSemaphore(ProfilerTask task)
+      throws InterruptedException {
+    if (!remoteOptions.throttleRemoteActionBuilding) {
+      return;
+    }
+
+    try (var c = Profiler.instance().profile(task, "acquiring semaphore")) {
+      remoteActionBuildingSemaphore.acquire();
+    }
+  }
+
+  private void maybeReleaseRemoteActionBuildingSemaphore() {
+    if (!remoteOptions.throttleRemoteActionBuilding) {
+      return;
+    }
+
+    remoteActionBuildingSemaphore.release();
+  }
+
   /** Creates a new {@link RemoteAction} instance from spawn. */
   public RemoteAction buildRemoteAction(Spawn spawn, SpawnExecutionContext context)
       throws IOException, ExecException, ForbiddenActionInputException, InterruptedException {
-    try (SilentCloseable c =
-        Profiler.instance().profile(ProfilerTask.REMOTE_SETUP, "acquiring semaphore")) {
-      remoteActionBuildingSemaphore.acquire();
-    }
+    maybeAcquireRemoteActionBuildingSemaphore(ProfilerTask.REMOTE_SETUP);
     try {
       // Create a remote path resolver that is aware of the spawn's path mapper, which rewrites
       // the paths of the inputs and outputs as well as paths appearing in the command line for
@@ -603,7 +619,7 @@ public RemoteAction buildRemoteAction(Spawn spawn, SpawnExecutionContext context
           actionKey,
           remoteOptions.remoteDiscardMerkleTrees);
     } finally {
-      remoteActionBuildingSemaphore.release();
+      maybeReleaseRemoteActionBuildingSemaphore();
     }
   }
 
@@ -1436,10 +1452,7 @@ public void uploadInputsIfNotPresent(RemoteAction action, boolean force)
     // concurrency. This prevents memory exhaustion. We assume that
     // ensureInputsPresent() provides enough parallelism to saturate the
     // network connection.
-    try (SilentCloseable c =
-        Profiler.instance().profile(ProfilerTask.UPLOAD_TIME, "acquiring semaphore")) {
-      remoteActionBuildingSemaphore.acquire();
-    }
+    maybeAcquireRemoteActionBuildingSemaphore(ProfilerTask.UPLOAD_TIME);
     try {
       MerkleTree merkleTree = action.getMerkleTree();
       if (merkleTree == null) {
@@ -1462,7 +1475,7 @@ public void uploadInputsIfNotPresent(RemoteAction action, boolean force)
           additionalInputs,
           force);
     } finally {
-      remoteActionBuildingSemaphore.release();
+      maybeReleaseRemoteActionBuildingSemaphore();
     }
   }
 

diff --git a/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java b/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java
@@ -28,6 +28,7 @@
 import com.google.devtools.common.options.Converter;
 import com.google.devtools.common.options.Converters;
 import com.google.devtools.common.options.Converters.AssignmentConverter;
+import com.google.devtools.common.options.Converters.BooleanConverter;
 import com.google.devtools.common.options.EnumConverter;
 import com.google.devtools.common.options.Option;
 import com.google.devtools.common.options.OptionDocumentationCategory;
@@ -725,6 +726,19 @@ public RemoteOutputsStrategyConverter() {
               + " variables).")
   public Scrubber scrubber;
 
+  @Option(
+      name = "experimental_throttle_remote_action_building",
+      defaultValue = "true",
+      converter = BooleanConverter.class,
+      documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+      metadataTags = OptionMetadataTag.EXPERIMENTAL,
+      effectTags = {OptionEffectTag.EXECUTION},
+      help =
+          "Whether to throttle the building of remote action to avoid OOM. Defaults to true.\n\n"
+              + "This is a temporary flag to allow users switch off the behaviour. Once Bazel is"
+              + " smart enough about the RAM/CPU usages, this flag will be removed.")
+  public boolean throttleRemoteActionBuilding;
+
   private static final class ScrubberConverter extends Converter.Contextless<Scrubber> {
 
     @Override