Output contents of RepoMappingManifest files when using --include_fil…

…e_write_contents This is so that we can write out the files in bazel for android, which converts the aquery results to a ninja file. PiperOrigin-RevId: 543837984 Change-Id: I9926a282214b293eb46ac376f42b21104e6af43b
bazelbuild · Jun 27, 2023 · 604a9ef · 604a9ef
1 parent 24db557
commit 604a9ef
Show file tree

Hide file tree

Showing 13 changed files with 188 additions and 47 deletions.
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/BUILD b/src/main/java/com/google/devtools/build/lib/analysis/BUILD
@@ -1048,6 +1048,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/actions:commandline_item",
         "//src/main/java/com/google/devtools/build/lib/cmdline",
         "//src/main/java/com/google/devtools/build/lib/collect/nestedset",
+        "//src/main/java/com/google/devtools/build/lib/events",
         "//src/main/java/com/google/devtools/build/lib/packages",
         "//src/main/java/com/google/devtools/build/lib/util",
         "//src/main/java/net/starlark/java/eval",
@@ -1400,6 +1401,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/actions",
         "//src/main/java/com/google/devtools/build/lib/actions:artifacts",
         "//src/main/java/com/google/devtools/build/lib/collect/nestedset",
+        "//src/main/java/com/google/devtools/build/lib/events",
         "//third_party:guava",
         "//third_party:jsr305",
     ],

diff --git a/src/main/java/com/google/devtools/build/lib/analysis/RepoMappingManifestAction.java b/src/main/java/com/google/devtools/build/lib/analysis/RepoMappingManifestAction.java
@@ -15,6 +15,7 @@
 
 import static com.google.common.collect.ImmutableSortedMap.toImmutableSortedMap;
 import static java.nio.charset.StandardCharsets.ISO_8859_1;
+import static java.nio.charset.StandardCharsets.UTF_8;
 import static java.util.Comparator.comparing;
 
 import com.google.common.collect.ImmutableSet;
@@ -26,7 +27,6 @@
 import com.google.devtools.build.lib.actions.Artifact.ArtifactExpander;
 import com.google.devtools.build.lib.actions.CommandLineExpansionException;
 import com.google.devtools.build.lib.actions.CommandLineItem.MapFn;
-import com.google.devtools.build.lib.actions.ExecException;
 import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction;
 import com.google.devtools.build.lib.analysis.actions.DeterministicWriter;
 import com.google.devtools.build.lib.cmdline.Label;
@@ -35,16 +35,20 @@
 import com.google.devtools.build.lib.collect.nestedset.NestedSet;
 import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
 import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.events.EventHandler;
 import com.google.devtools.build.lib.packages.Package;
 import com.google.devtools.build.lib.util.Fingerprint;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
 import java.io.PrintWriter;
 import java.util.Map.Entry;
 import java.util.UUID;
 import javax.annotation.Nullable;
 import net.starlark.java.eval.EvalException;
 
 /** Creates a manifest file describing the repos and mappings relevant for a runfile tree. */
-public final class RepoMappingManifestAction extends AbstractFileWriteAction {
+public final class RepoMappingManifestAction extends AbstractFileWriteAction
+    implements AbstractFileWriteAction.FileContentsProvider {
 
   private static final UUID MY_UUID = UUID.fromString("458e351c-4d30-433d-b927-da6cddd4737f");
 
@@ -119,9 +123,24 @@ protected void computeKey(
     fp.addString(workspaceName);
   }
 
+  /**
+   * Get the contents of a file internally using an in memory output stream.
+   *
+   * @return returns the file contents as a string.
+   */
   @Override
-  public DeterministicWriter newDeterministicWriter(ActionExecutionContext ctx)
-      throws InterruptedException, ExecException {
+  public String getFileContents(@Nullable EventHandler eventHandler) throws IOException {
+    ByteArrayOutputStream stream = new ByteArrayOutputStream();
+    newDeterministicWriter().writeOutputFile(stream);
+    return stream.toString(UTF_8);
+  }
+
+  @Override
+  public DeterministicWriter newDeterministicWriter(ActionExecutionContext ctx) {
+    return newDeterministicWriter();
+  }
+
+  public DeterministicWriter newDeterministicWriter() {
     return out -> {
       PrintWriter writer = new PrintWriter(out, /* autoFlush= */ false, ISO_8859_1);
 

diff --git a/src/main/java/com/google/devtools/build/lib/analysis/SourceManifestAction.java b/src/main/java/com/google/devtools/build/lib/analysis/SourceManifestAction.java
@@ -55,7 +55,8 @@
  * <p>This action carefully avoids building the manifest content in memory because it can be large.
  */
 @Immutable // if all ManifestWriter implementations are immutable
-public final class SourceManifestAction extends AbstractFileWriteAction {
+public final class SourceManifestAction extends AbstractFileWriteAction
+    implements AbstractFileWriteAction.FileContentsProvider {
 
   private static final String GUID = "07459553-a3d0-4d37-9d78-18ed942470f4";
 
@@ -193,15 +194,16 @@ public void writeOutputFile(OutputStream out, @Nullable EventHandler eventHandle
    *
    * @return returns the file contents as a string.
    */
-  public String getFileContentsAsString(@Nullable EventHandler eventHandler) throws IOException {
+  @Override
+  public String getFileContents(@Nullable EventHandler eventHandler) throws IOException {
     ByteArrayOutputStream stream = new ByteArrayOutputStream();
     writeOutputFile(stream, eventHandler);
     return stream.toString(UTF_8);
   }
 
   @Override
   public String getStarlarkContent() throws IOException {
-    return getFileContentsAsString(null);
+    return getFileContents(null);
   }
 
   @Override

diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/AbstractFileWriteAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/AbstractFileWriteAction.java
@@ -25,6 +25,9 @@
 import com.google.devtools.build.lib.actions.ExecException;
 import com.google.devtools.build.lib.actions.SpawnResult;
 import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.events.EventHandler;
+import java.io.IOException;
+import javax.annotation.Nullable;
 
 /**
  * Abstract Action to write to a file.
@@ -104,4 +107,11 @@ public boolean isRemotable() {
     return true;
   }
 
+  /**
+   * This interface is used to get the contents of the file to output to aquery when using
+   * --include_file_write_contents.
+   */
+  public interface FileContentsProvider {
+    String getFileContents(@Nullable EventHandler eventHandler) throws IOException;
+  }
 }
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/FileWriteAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/FileWriteAction.java
@@ -25,6 +25,7 @@
 import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
 import com.google.devtools.build.lib.collect.nestedset.Order;
 import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.events.EventHandler;
 import com.google.devtools.build.lib.util.Fingerprint;
 import com.google.devtools.build.lib.util.OnDemandString;
 import java.io.ByteArrayInputStream;
@@ -49,7 +50,8 @@
  * BinaryFileWriteAction}.
  */
 @Immutable // if fileContents is immutable
-public final class FileWriteAction extends AbstractFileWriteAction {
+public final class FileWriteAction extends AbstractFileWriteAction
+    implements AbstractFileWriteAction.FileContentsProvider {
 
   private static final String GUID = "332877c7-ca9f-4731-b387-54f620408522";
 
@@ -206,6 +208,14 @@ public String toString() {
     }
   }
 
+  /**
+   * @see #getFilecontents()
+   */
+  @Override
+  public String getFileContents(@Nullable EventHandler eventHandler) {
+    return getFileContents();
+  }
+
   /**
    * Returns the string contents to be written.
    *

diff --git a/src/main/java/com/google/devtools/build/lib/query2/BUILD b/src/main/java/com/google/devtools/build/lib/query2/BUILD
@@ -30,6 +30,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/actions:artifacts",
         "//src/main/java/com/google/devtools/build/lib/actions:commandline_item",
         "//src/main/java/com/google/devtools/build/lib/actions:file_metadata",
+        "//src/main/java/com/google/devtools/build/lib/analysis:actions/abstract_file_write_action",
         "//src/main/java/com/google/devtools/build/lib/analysis:actions/parameter_file_write_action",
         "//src/main/java/com/google/devtools/build/lib/analysis:actions/substitution",
         "//src/main/java/com/google/devtools/build/lib/analysis:actions/template_expansion_action",

diff --git a/...a/com/google/devtools/build/lib/query2/aquery/ActionGraphTextOutputFormatterCallback.java b/...a/com/google/devtools/build/lib/query2/aquery/ActionGraphTextOutputFormatterCallback.java
@@ -31,8 +31,7 @@
 import com.google.devtools.build.lib.actions.CommandLineExpansionException;
 import com.google.devtools.build.lib.analysis.AspectValue;
 import com.google.devtools.build.lib.analysis.ConfiguredTargetValue;
-import com.google.devtools.build.lib.analysis.SourceManifestAction;
-import com.google.devtools.build.lib.analysis.actions.FileWriteAction;
+import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction;
 import com.google.devtools.build.lib.analysis.actions.ParameterFileWriteAction;
 import com.google.devtools.build.lib.analysis.actions.Substitution;
 import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction;
@@ -336,22 +335,13 @@ private void writeAction(ActionAnalysisMetadata action, PrintStream printStream)
       stringBuilder.append("  ]\n");
     }
 
-    if (options.includeFileWriteContents && action instanceof FileWriteAction) {
-      FileWriteAction fileWriteAction = (FileWriteAction) action;
+    if (options.includeFileWriteContents
+        && action instanceof AbstractFileWriteAction.FileContentsProvider) {
+      String contents =
+          ((AbstractFileWriteAction.FileContentsProvider) action).getFileContents(eventHandler);
       stringBuilder
           .append("  FileWriteContents: [")
-          .append(
-              Base64.getEncoder().encodeToString(fileWriteAction.getFileContents().getBytes(UTF_8)))
-          .append("]\n");
-    }
-    if (options.includeFileWriteContents && action instanceof SourceManifestAction) {
-      SourceManifestAction sourceManifestAction = (SourceManifestAction) action;
-      stringBuilder
-          .append("  FileWriteContents: [")
-          .append(
-              Base64.getEncoder()
-                  .encodeToString(
-                      sourceManifestAction.getFileContentsAsString(eventHandler).getBytes(UTF_8)))
+          .append(Base64.getEncoder().encodeToString(contents.getBytes(UTF_8)))
           .append("]\n");
     }
 

diff --git a/src/main/java/com/google/devtools/build/lib/query2/aquery/AqueryOptions.java b/src/main/java/com/google/devtools/build/lib/query2/aquery/AqueryOptions.java
@@ -64,8 +64,8 @@ public class AqueryOptions extends CommonQueryOptions {
       documentationCategory = OptionDocumentationCategory.QUERY,
       effectTags = {OptionEffectTag.TERMINAL_OUTPUT},
       help =
-          "Include the file contents for the FileWrite and SourceSymlinkManifest actions"
-              + " (potentially large). ")
+          "Include the file contents for the FileWrite, SourceSymlinkManifest, and "
+              + "RepoMappingManifest actions (potentially large). ")
   public boolean includeFileWriteContents;
 
   @Option(

diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/ActionGraphDump.java b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/ActionGraphDump.java
@@ -30,8 +30,7 @@
 import com.google.devtools.build.lib.analysis.AspectValue;
 import com.google.devtools.build.lib.analysis.ConfiguredTarget;
 import com.google.devtools.build.lib.analysis.ConfiguredTargetValue;
-import com.google.devtools.build.lib.analysis.SourceManifestAction;
-import com.google.devtools.build.lib.analysis.actions.FileWriteAction;
+import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction;
 import com.google.devtools.build.lib.analysis.actions.ParameterFileWriteAction;
 import com.google.devtools.build.lib.analysis.actions.Substitution;
 import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction;
@@ -198,14 +197,11 @@ private void dumpSingleAction(ConfiguredTarget configuredTarget, ActionAnalysisM
       actionBuilder.addAllArguments(commandAction.getArguments());
     }
 
-    if (includeFileWriteContents && action instanceof FileWriteAction) {
-      FileWriteAction fileWriteAction = (FileWriteAction) action;
-      actionBuilder.setFileContents(fileWriteAction.getFileContents());
-    }
-
-    if (includeFileWriteContents && action instanceof SourceManifestAction) {
-      SourceManifestAction sourceManifestAction = (SourceManifestAction) action;
-      actionBuilder.setFileContents(sourceManifestAction.getFileContentsAsString(eventHandler));
+    if (includeFileWriteContents
+        && action instanceof AbstractFileWriteAction.FileContentsProvider) {
+      String contents =
+          ((AbstractFileWriteAction.FileContentsProvider) action).getFileContents(eventHandler);
+      actionBuilder.setFileContents(contents);
     }
 
     // Include the content of param files in output.

diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/BUILD b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/BUILD
@@ -18,6 +18,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/actions",
         "//src/main/java/com/google/devtools/build/lib/actions:artifacts",
         "//src/main/java/com/google/devtools/build/lib/actions:commandline_item",
+        "//src/main/java/com/google/devtools/build/lib/analysis:actions/abstract_file_write_action",
         "//src/main/java/com/google/devtools/build/lib/analysis:actions/parameter_file_write_action",
         "//src/main/java/com/google/devtools/build/lib/analysis:actions/substitution",
         "//src/main/java/com/google/devtools/build/lib/analysis:actions/template_expansion_action",

diff --git a/src/test/java/com/google/devtools/build/lib/analysis/SourceManifestActionTest.java b/src/test/java/com/google/devtools/build/lib/analysis/SourceManifestActionTest.java
@@ -163,18 +163,20 @@ public boolean isRemotable() {
   @Test
   public void testManifestWriterIntegration() throws Exception {
     MockManifestWriter mockWriter = new MockManifestWriter();
-    new SourceManifestAction(
-            mockWriter,
-            NULL_ACTION_OWNER,
-            manifestOutputFile,
-            new Runfiles.Builder("TESTING", false).addSymlinks(fakeManifest).build())
-        .getFileContentsAsString(reporter);
+    String manifestContents =
+        new SourceManifestAction(
+                mockWriter,
+                NULL_ACTION_OWNER,
+                manifestOutputFile,
+                new Runfiles.Builder("TESTING", false).addSymlinks(fakeManifest).build())
+            .getFileContents(reporter);
     assertThat(mockWriter.unconsumedInputs()).isEqualTo(0);
+    assertThat(manifestContents).isEmpty();
   }
 
   @Test
   public void testSimpleFileWriting() throws Exception {
-    String manifestContents = createSymlinkAction().getFileContentsAsString(reporter);
+    String manifestContents = createSymlinkAction().getFileContents(reporter);
     assertThat(manifestContents)
         .isEqualTo(
             "TESTING/trivial/BUILD /workspace/trivial/BUILD\n"
@@ -188,7 +190,7 @@ public void testSimpleFileWriting() throws Exception {
    */
   @Test
   public void testSourceOnlyFormatting() throws Exception {
-    String manifestContents = createSourceOnlyAction().getFileContentsAsString(reporter);
+    String manifestContents = createSourceOnlyAction().getFileContents(reporter);
     assertThat(manifestContents)
         .isEqualTo(
             "TESTING/trivial/BUILD\n"
@@ -207,7 +209,7 @@ public void testSwigLibrariesTriggerInitDotPyInclusion() throws Exception {
     Path swiggedFile = scratch.file("swig/fakeLib.so");
     Artifact swigDotSO = ActionsTestUtil.createArtifact(swiggedLibPath, swiggedFile);
     fakeManifest.put(swiggedFile.relativeTo(rootDirectory), swigDotSO);
-    String manifestContents = createSymlinkAction().getFileContentsAsString(reporter);
+    String manifestContents = createSymlinkAction().getFileContents(reporter);
     assertThat(manifestContents).containsMatch(".*TESTING/swig/__init__.py .*");
     assertThat(manifestContents).containsMatch("fakeLib.so");
   }
@@ -219,7 +221,7 @@ public void testNoPythonOrSwigLibrariesDoNotTriggerInitDotPyInclusion() throws E
     Path nonPythonFile = scratch.file("not_python/blob_of_data");
     Artifact nonPython = ActionsTestUtil.createArtifact(nonPythonPath, nonPythonFile);
     fakeManifest.put(nonPythonFile.relativeTo(rootDirectory), nonPython);
-    String manifestContents = createSymlinkAction().getFileContentsAsString(reporter);
+    String manifestContents = createSymlinkAction().getFileContents(reporter);
     assertThat(manifestContents).doesNotContain("not_python/__init__.py \n");
     assertThat(manifestContents).containsMatch("blob_of_data");
   }
@@ -367,7 +369,7 @@ public void testUnresolvedSymlink() throws Exception {
     assertThat(inputs).isEqualTo(action.getInputs());
     assertThat(inputs.toList()).isEqualTo(action.getInputs().toList());
 
-    assertThat(action.getFileContentsAsString(reporter))
+    assertThat(action.getFileContents(reporter))
         .isEqualTo(
             "TESTING/BUILD /workspace/trivial/BUILD\n"
                 + "TESTING/absolute_symlink /absolute/path\n"

diff --git a/src/test/shell/integration/BUILD b/src/test/shell/integration/BUILD
@@ -320,6 +320,16 @@ sh_test(
     ],
 )
 
+sh_test(
+    name = "bazel_aquery_test",
+    size = "large",
+    srcs = ["bazel_aquery_test.sh"],
+    data = [
+        ":test-deps",
+        "@bazel_tools//tools/bash/runfiles",
+    ],
+)
+
 sh_test(
     name = "bazel_command_log_test",
     size = "medium",