Log source artifact bytes read during this build to BEP.

Suggested by @jfield.

PiperOrigin-RevId: 353789327

Author: janakdr

src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto

@@ -740,6 +740,10 @@ message BuildMetrics {
     // This includes any remote cache hits, but excludes
     // local action cache hits.
     int64 actions_executed = 2;
+
+    // Total size of all source files newly read this build. Will not include
+    // unchanged sources on incremental builds.
+    int64 source_artifact_bytes_read = 5;
   }
   ActionSummary action_summary = 1;
 

src/main/java/com/google/devtools/build/lib/metrics/BUILD

@@ -38,6 +38,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/analysis:analysis_phase_started_event",
         "//src/main/java/com/google/devtools/build/lib/buildeventstream/proto:build_event_stream_java_proto",
         "//src/main/java/com/google/devtools/build/lib/profiler",
+        "//src/main/java/com/google/devtools/build/lib/skyframe:execution_finished_event",
         "//src/main/java/com/google/devtools/common/options",
         "//third_party:guava",
     ],

src/main/java/com/google/devtools/build/lib/metrics/MetricsCollector.java

@@ -31,6 +31,7 @@
 import com.google.devtools.build.lib.metrics.PostGCMemoryUseRecorder.PeakHeap;
 import com.google.devtools.build.lib.profiler.Profiler;
 import com.google.devtools.build.lib.runtime.CommandEnvironment;
+import com.google.devtools.build.lib.skyframe.ExecutionFinishedEvent;
 import java.lang.management.ManagementFactory;
 import java.lang.management.MemoryMXBean;
 import java.time.Duration;
@@ -49,7 +50,8 @@ class MetricsCollector {
   private int targetsLoaded;
   private int targetsConfigured;
   private int packagesLoaded;
-  private static long analysisTimeInMs;
+  private long sourceArtifactBytesReadPerBuild;
+  private long analysisTimeInMs;
 
   private MetricsCollector(CommandEnvironment env) {
     this.env = env;
@@ -92,6 +94,12 @@ public void onActionComplete(ActionCompletionEvent event) {
     executedActionCount.incrementAndGet();
   }
 
+  @SuppressWarnings("unused")
+  @Subscribe
+  public void onExecutionComplete(ExecutionFinishedEvent event) {
+    this.sourceArtifactBytesReadPerBuild = event.sourceArtifactBytesRead();
+  }
+
   @Subscribe
   public void onBuildComplete(BuildPrecompleteEvent event) {
     env.getEventBus().post(new BuildMetricsEvent(createBuildMetrics()));
@@ -112,6 +120,7 @@ private ActionSummary createActionSummary() {
     return ActionSummary.newBuilder()
         .setActionsCreated(actionsConstructed)
         .setActionsExecuted(executedActionCount.get())
+        .setSourceArtifactBytesRead(sourceArtifactBytesReadPerBuild)
         .build();
   }
 
@@ -147,7 +156,7 @@ private CumulativeMetrics createCumulativeMetrics() {
         .build();
   }
 
-  private static TimingMetrics createTimingMetrics() {
+  private TimingMetrics createTimingMetrics() {
     TimingMetrics.Builder timingMetricsBuilder = TimingMetrics.newBuilder();
     Duration elapsedWallTime = Profiler.elapsedTimeMaybe();
     if (elapsedWallTime != null) {

src/main/java/com/google/devtools/build/lib/skyframe/ArtifactFunction.java

@@ -54,6 +54,7 @@
 import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Supplier;
 import javax.annotation.Nullable;
 
@@ -65,6 +66,7 @@
  */
 class ArtifactFunction implements SkyFunction {
   private final Supplier<Boolean> mkdirForTreeArtifacts;
+  private final AtomicLong sourceArtifactBytesReadThisBuild;
 
   public static final class MissingFileArtifactValue implements SkyValue {
     private final DetailedExitCode detailedExitCode;
@@ -87,8 +89,10 @@ public String toString() {
     }
   }
 
-  public ArtifactFunction(Supplier<Boolean> mkdirForTreeArtifacts) {
+  public ArtifactFunction(
+      Supplier<Boolean> mkdirForTreeArtifacts, AtomicLong sourceArtifactBytesReadThisBuild) {
     this.mkdirForTreeArtifacts = mkdirForTreeArtifacts;
+    this.sourceArtifactBytesReadThisBuild = sourceArtifactBytesReadThisBuild;
   }
 
   @Override
@@ -251,7 +255,7 @@ private static TreeArtifactValue createTreeArtifactValueFromActionKey(
     return tree;
   }
 
-  private static SkyValue createSourceValue(Artifact artifact, Environment env)
+  private SkyValue createSourceValue(Artifact artifact, Environment env)
       throws IOException, InterruptedException {
     RootedPath path = RootedPath.toRootedPath(artifact.getRoot().getRoot(), artifact.getPath());
     SkyKey fileSkyKey = FileValue.key(path);
@@ -268,51 +272,54 @@ private static SkyValue createSourceValue(Artifact artifact, Environment env)
       return makeMissingSourceInputFileValue(artifact);
     }
 
-    // For directory artifacts that are not Filesets, we initiate a directory traversal here, and
-    // compute a hash from the directory structure.
-    if (fileValue.isDirectory() && TrackSourceDirectoriesFlag.trackSourceDirectories()) {
-      // We rely on the guarantees of RecursiveFilesystemTraversalFunction for correctness.
-      //
-      // This approach may have unexpected interactions with --package_path. In particular, the exec
-      // root is setup from the loading / analysis phase, and it is now too late to change it;
-      // therefore, this may traverse a different set of files depending on which targets are built
-      // at the same time and what the package-path layout is (this may be moot if there is only one
-      // entry). Or this may return a set of files that's inconsistent with those actually available
-      // to the action (for local execution).
-      //
-      // In the future, we need to make this result the source of truth for the files available to
-      // the action so that we at least have consistency.
-      TraversalRequest request =
-          TraversalRequest.create(
-              DirectTraversalRoot.forRootedPath(path),
-              /*isRootGenerated=*/ false,
-              PackageBoundaryMode.CROSS,
-              /*strictOutputFiles=*/ true,
-              /*skipTestingForSubpackage=*/ true,
-              /*errorInfo=*/ "Directory artifact " + artifact.prettyPrint());
-      RecursiveFilesystemTraversalValue value;
-      try {
-        value =
-            (RecursiveFilesystemTraversalValue) env.getValueOrThrow(
-                request, RecursiveFilesystemTraversalException.class);
-      } catch (RecursiveFilesystemTraversalException e) {
-        throw new IOException(e);
-      }
-      if (value == null) {
-        return null;
+    if (!fileValue.isDirectory() || !TrackSourceDirectoriesFlag.trackSourceDirectories()) {
+      if (fileValue.isFile()) {
+        sourceArtifactBytesReadThisBuild.addAndGet(fileValue.getSize());
       }
-      Fingerprint fp = new Fingerprint();
-      for (ResolvedFile file : value.getTransitiveFiles().toList()) {
-        fp.addString(file.getNameInSymlinkTree().getPathString());
-        fp.addBytes(file.getMetadata().getDigest());
+      try {
+        return FileArtifactValue.createForSourceArtifact(artifact, fileValue);
+      } catch (IOException e) {
+        return makeIOExceptionSourceInputFileValue(artifact, e);
       }
-      return FileArtifactValue.createForDirectoryWithHash(fp.digestAndReset());
     }
+    // For directory artifacts that are not Filesets, we initiate a directory traversal here, and
+    // compute a hash from the directory structure.
+    // We rely on the guarantees of RecursiveFilesystemTraversalFunction for correctness.
+    //
+    // This approach may have unexpected interactions with --package_path. In particular, the exec
+    // root is setup from the loading / analysis phase, and it is now too late to change it;
+    // therefore, this may traverse a different set of files depending on which targets are built
+    // at the same time and what the package-path layout is (this may be moot if there is only one
+    // entry). Or this may return a set of files that's inconsistent with those actually available
+    // to the action (for local execution).
+    //
+    // In the future, we need to make this result the source of truth for the files available to
+    // the action so that we at least have consistency.
+    TraversalRequest request =
+        TraversalRequest.create(
+            DirectTraversalRoot.forRootedPath(path),
+            /*isRootGenerated=*/ false,
+            PackageBoundaryMode.CROSS,
+            /*strictOutputFiles=*/ true,
+            /*skipTestingForSubpackage=*/ true,
+            /*errorInfo=*/ "Directory artifact " + artifact.prettyPrint());
+    RecursiveFilesystemTraversalValue value;
     try {
-      return FileArtifactValue.createForSourceArtifact(artifact, fileValue);
-    } catch (IOException e) {
-      return makeIOExceptionSourceInputFileValue(artifact, e);
+      value =
+          (RecursiveFilesystemTraversalValue)
+              env.getValueOrThrow(request, RecursiveFilesystemTraversalException.class);
+    } catch (RecursiveFilesystemTraversalException e) {
+      throw new IOException(e);
+    }
+    if (value == null) {
+      return null;
+    }
+    Fingerprint fp = new Fingerprint();
+    for (ResolvedFile file : value.getTransitiveFiles().toList()) {
+      fp.addString(file.getNameInSymlinkTree().getPathString());
+      fp.addBytes(file.getMetadata().getDigest());
     }
+    return FileArtifactValue.createForDirectoryWithHash(fp.digestAndReset());
   }
 
   static MissingFileArtifactValue makeMissingSourceInputFileValue(Artifact artifact) {

src/main/java/com/google/devtools/build/lib/skyframe/ExecutionFinishedEvent.java

@@ -22,14 +22,16 @@
  */
 @AutoValue
 public abstract class ExecutionFinishedEvent {
-  public static final ExecutionFinishedEvent EMPTY =
-      builder()
-          .setOutputDirtyFiles(0)
-          .setOutputModifiedFilesDuringPreviousBuild(0)
-          .setSourceDiffCheckingDuration(Duration.ZERO)
-          .setNumSourceFilesCheckedBecauseOfMissingDiffs(0)
-          .setOutputTreeDiffCheckingDuration(Duration.ZERO)
-          .build();
+  // AutoValue Builders require that all fields are populated, so we provide a default.
+  public static ExecutionFinishedEvent.Builder builderWithDefaults() {
+    return builder()
+        .setOutputDirtyFiles(0)
+        .setOutputModifiedFilesDuringPreviousBuild(0)
+        .setSourceDiffCheckingDuration(Duration.ZERO)
+        .setNumSourceFilesCheckedBecauseOfMissingDiffs(0)
+        .setOutputTreeDiffCheckingDuration(Duration.ZERO)
+        .setSourceArtifactBytesRead(0L);
+  }
 
   public abstract int outputDirtyFiles();
 
@@ -41,6 +43,8 @@ public abstract class ExecutionFinishedEvent {
 
   public abstract Duration outputTreeDiffCheckingDuration();
 
+  public abstract long sourceArtifactBytesRead();
+
   static Builder builder() {
     return new AutoValue_ExecutionFinishedEvent.Builder();
   }
@@ -59,6 +63,8 @@ abstract Builder setNumSourceFilesCheckedBecauseOfMissingDiffs(
 
     abstract Builder setOutputTreeDiffCheckingDuration(Duration outputTreeDiffCheckingDuration);
 
+    abstract Builder setSourceArtifactBytesRead(long sourceArtifactBytesRead);
+
     abstract ExecutionFinishedEvent build();
   }
 }

src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java

@@ -912,21 +912,20 @@ public Void call() throws InterruptedException {
   }
 
   @Override
-  public ExecutionFinishedEvent createExecutionFinishedEvent() {
-    ExecutionFinishedEvent result =
+  protected ExecutionFinishedEvent.Builder createExecutionFinishedEventInternal() {
+    ExecutionFinishedEvent.Builder builder =
         ExecutionFinishedEvent.builder()
             .setOutputDirtyFiles(outputDirtyFiles)
             .setOutputModifiedFilesDuringPreviousBuild(modifiedFilesDuringPreviousBuild)
             .setSourceDiffCheckingDuration(sourceDiffCheckingDuration)
             .setNumSourceFilesCheckedBecauseOfMissingDiffs(
                 numSourceFilesCheckedBecauseOfMissingDiffs)
-            .setOutputTreeDiffCheckingDuration(outputTreeDiffCheckingDuration)
-            .build();
+            .setOutputTreeDiffCheckingDuration(outputTreeDiffCheckingDuration);
     outputDirtyFiles = 0;
     modifiedFilesDuringPreviousBuild = 0;
     sourceDiffCheckingDuration = Duration.ZERO;
     outputTreeDiffCheckingDuration = Duration.ZERO;
-    return result;
+    return builder;
   }
 
   @Override

src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java

@@ -217,6 +217,7 @@
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 import java.util.function.Supplier;
@@ -249,6 +250,12 @@ public abstract class SkyframeExecutor implements WalkableGraphFactory, Configur
   protected final BlazeDirectories directories;
   protected final ExternalFilesHelper externalFilesHelper;
   private final GraphInconsistencyReceiver graphInconsistencyReceiver;
+  /**
+   * Tracks the accumulated size of source artifacts read this build. Does not include cached
+   * artifacts, so is not useful on incremental builds.
+   */
+  private final AtomicLong sourceArtifactBytesReadThisBuild = new AtomicLong();
+
   @Nullable protected OutputService outputService;
 
   // TODO(bazel-team): Figure out how to handle value builders that block internally. Blocking
@@ -584,7 +591,8 @@ private ImmutableMap<SkyFunctionName, SkyFunction> skyFunctions(PackageFactory p
     map.put(
         Artifact.ARTIFACT,
         new ArtifactFunction(
-            () -> !skyframeActionExecutor.actionFileSystemType().inMemoryFileSystem()));
+            () -> !skyframeActionExecutor.actionFileSystemType().inMemoryFileSystem(),
+            sourceArtifactBytesReadThisBuild));
     map.put(
         SkyFunctions.BUILD_INFO_COLLECTION,
         new BuildInfoCollectionFunction(actionKeyContext, artifactFactory));
@@ -2694,6 +2702,7 @@ protected final void syncPackageLoadingInternal(
 
     incrementalBuildMonitor = new SkyframeIncrementalBuildMonitor();
     invalidateTransientErrors();
+    sourceArtifactBytesReadThisBuild.set(0L);
   }
 
   private void getActionEnvFromOptions(CoreOptions opt) {
@@ -3014,7 +3023,16 @@ public void evaluated(
     }
   }
 
-  public abstract ExecutionFinishedEvent createExecutionFinishedEvent();
+  public final ExecutionFinishedEvent createExecutionFinishedEvent() {
+    return createExecutionFinishedEventInternal()
+        .setSourceArtifactBytesRead(sourceArtifactBytesReadThisBuild.getAndSet(0L))
+        .build();
+  }
+
+  @ForOverride
+  protected ExecutionFinishedEvent.Builder createExecutionFinishedEventInternal() {
+    return ExecutionFinishedEvent.builderWithDefaults();
+  }
 
   protected Iterable<ActionLookupValue> getActionLookupValuesInBuild(
       List<ConfiguredTargetKey> topLevelCtKeys, List<AspectValueKey> aspectKeys)

src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTestCase.java

@@ -52,6 +52,7 @@
 import java.io.IOException;
 import java.util.LinkedHashSet;
 import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 import org.junit.Before;
 
@@ -106,7 +107,7 @@ public void baseSetUp() throws Exception {
                         new AtomicReference<>(UnixGlob.DEFAULT_SYSCALLS),
                         externalFilesHelper))
                 .put(FileValue.FILE, new FileFunction(pkgLocator))
-                .put(Artifact.ARTIFACT, new ArtifactFunction(() -> true))
+                .put(Artifact.ARTIFACT, new ArtifactFunction(() -> true, new AtomicLong()))
                 .put(SkyFunctions.ACTION_EXECUTION, new SimpleActionExecutionFunction())
                 .put(
                     SkyFunctions.PACKAGE,

src/test/java/com/google/devtools/build/lib/skyframe/TimestampBuilderTestCase.java

@@ -126,6 +126,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 import javax.annotation.Nullable;
 import org.junit.Before;
@@ -260,7 +261,7 @@ protected BuilderWithResult createBuilder(
                         new AtomicReference<>(UnixGlob.DEFAULT_SYSCALLS),
                         externalFilesHelper))
                 .put(FileValue.FILE, new FileFunction(pkgLocator))
-                .put(Artifact.ARTIFACT, new ArtifactFunction(() -> true))
+                .put(Artifact.ARTIFACT, new ArtifactFunction(() -> true, new AtomicLong()))
                 .put(
                     SkyFunctions.ACTION_EXECUTION,
                     new ActionExecutionFunction(skyframeActionExecutor, directories, tsgmRef))