Watch arbitrary file in repo rules

* Starlark API changes
  * New method `rctx.watch()` that watches a path. As of right now, said path must exist and be a regular file, but that will change in follow-ups.
  * Existing method `rctx.read()` gets a new optional string parameter `watch` that defaults to `auto`. This causes the file read to be watched as well by default, even if it's not addressed by a label. See `rctx.read()` docs for the semantics of `watch`.
  * Both changes apply to `mctx` as well, except that no files outside the current Bazel workspace may be watched for `mctx`.
* Marker file format changes
  * `FILE:` marker file entries can now be `FILE:` followed by either an absolute path (starts with a singular '/'), or a label-like ... thing (which is a label but has a slash in place of a colon). This is because we might be watching something inside a repo that is _not_ under a package.
  * Same change will happen to the `accumulatedFileDigests` sections of the module lockfile.
* Code changes
  * `RepoRecordedInput.File` subclasses are renamed to `FileOutsideWorkspace` (absolute paths) and `FileInsideWorkspace` (label-like things) instead to better reflect their meaning.
  * Unified the file watching/digest checking logic in SingleExtensionEvalFunction with the RepoRecordedInput stuff. Follow-ups will unify more.
  * Moved RepoRecordedInput to its own BUILD target as it's being used outside the repo fetching machinery (namely, module extensions)
  * Since `FileOutsideWorkspace` has to be an absolute path (never relative), we don't need a "base directory" for the RepoRecordedInput parser. What we really need is just the FileSystem object, which we can get from the PathPackageLocator available in Skyframe. Refactored code to reflect that.

Work towards bazelbuild#20952.

Closes bazelbuild#21230.

RELNOTES: Added a new method, `repository_ctx.watch()`, which asks Bazel to watch for changes to an arbitrary file. When said file changes, Bazel will refetch the repo.
PiperOrigin-RevId: 607097422
Change-Id: Ib96a49461deddd7f4c786bd1c227de18b2dd71d7

MODULE.bazel.lock

@@ -2741,7 +2741,7 @@
       "general": {
         "bzlTransitiveDigest": "r8gQnSLwon27gWD77J8mb3DIe4v3gtn7J/rsic53Qyw=",
         "accumulatedFileDigests": {
-          "@@//src/test/tools/bzlmod:MODULE.bazel.lock": "f4fd09267e9d5e7e6648096a68b2607793c9d34dcfcaadadb83f6ed2b636e508",
+          "@@//src/test/tools/bzlmod:MODULE.bazel.lock": "e06d281fdc21f520f6c489fd021f6393bb5f94d6fb3dc6aeaa3214bd614f8266",
           "@@//:MODULE.bazel": "c440045e7f8f6ff70a870646e1d3502afe2b63b4f51d794d0b6bfa6cc4b3c007"
         },
         "envVariables": {},

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD

@@ -155,6 +155,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/cmdline",
         "//src/main/java/com/google/devtools/build/lib/events",
         "//src/main/java/com/google/devtools/build/lib/packages",
+        "//src/main/java/com/google/devtools/build/lib/rules:repository/repo_recorded_input",
         "//src/main/java/com/google/devtools/build/lib/skyframe:sky_functions",
         "//src/main/java/com/google/devtools/build/lib/skyframe/serialization:visible-for-serialization",
         "//src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec",
@@ -217,6 +218,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/events",
         "//src/main/java/com/google/devtools/build/lib/packages",
         "//src/main/java/com/google/devtools/build/lib/profiler",
+        "//src/main/java/com/google/devtools/build/lib/rules:repository/repo_recorded_input",
         "//src/main/java/com/google/devtools/build/lib/rules:repository/repository_directory_value",
         "//src/main/java/com/google/devtools/build/lib/rules:repository/repository_function",
         "//src/main/java/com/google/devtools/build/lib/skyframe:bzl_load_failed_exception",

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java

@@ -37,7 +37,7 @@
 @GenerateTypeAdapter
 public abstract class BazelLockFileValue implements SkyValue, Postable {
 
-  public static final int LOCK_FILE_VERSION = 4;
+  public static final int LOCK_FILE_VERSION = 5;
 
   @SerializationConstant public static final SkyKey KEY = () -> SkyFunctions.BAZEL_LOCK_FILE;
 

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java

@@ -29,6 +29,7 @@
 import com.google.devtools.build.lib.cmdline.Label;
 import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
 import com.google.devtools.build.lib.cmdline.RepositoryName;
+import com.google.devtools.build.lib.rules.repository.RepoRecordedInput;
 import com.google.devtools.build.lib.vfs.Path;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
@@ -444,6 +445,20 @@ public Location read(JsonReader jsonReader) throws IOException {
     }
   }
 
+  private static final TypeAdapter<RepoRecordedInput.File> REPO_RECORDED_INPUT_FILE_TYPE_ADAPTER =
+      new TypeAdapter<>() {
+        @Override
+        public void write(JsonWriter jsonWriter, RepoRecordedInput.File value) throws IOException {
+          jsonWriter.value(value.toStringInternal());
+        }
+
+        @Override
+        public RepoRecordedInput.File read(JsonReader jsonReader) throws IOException {
+          return (RepoRecordedInput.File)
+              RepoRecordedInput.File.PARSER.parse(jsonReader.nextString());
+        }
+      };
+
   public static Gson createLockFileGson(Path moduleFilePath, Path workspaceRoot) {
     return new GsonBuilder()
         .setPrettyPrinting()
@@ -468,6 +483,7 @@ public static Gson createLockFileGson(Path moduleFilePath, Path workspaceRoot) {
         .registerTypeAdapter(ModuleExtensionId.IsolationKey.class, ISOLATION_KEY_TYPE_ADAPTER)
         .registerTypeAdapter(AttributeValues.class, new AttributeValuesAdapter())
         .registerTypeAdapter(byte[].class, BYTE_ARRAY_TYPE_ADAPTER)
+        .registerTypeAdapter(RepoRecordedInput.File.class, REPO_RECORDED_INPUT_FILE_TYPE_ADAPTER)
         .create();
   }
 

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/LockFileModuleExtension.java

@@ -17,9 +17,9 @@
 import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableTable;
-import com.google.devtools.build.lib.cmdline.Label;
 import com.google.devtools.build.lib.cmdline.RepositoryName;
 import com.google.devtools.build.lib.events.ExtendedEventHandler.Postable;
+import com.google.devtools.build.lib.rules.repository.RepoRecordedInput;
 import com.ryanharter.auto.value.gson.GenerateTypeAdapter;
 import java.util.Optional;
 
@@ -41,7 +41,7 @@ public static Builder builder() {
   @SuppressWarnings("mutable")
   public abstract byte[] getBzlTransitiveDigest();
 
-  public abstract ImmutableMap<Label, String> getAccumulatedFileDigests();
+  public abstract ImmutableMap<RepoRecordedInput.File, String> getRecordedFileInputs();
 
   public abstract ImmutableMap<String, String> getEnvVariables();
 
@@ -65,7 +65,8 @@ public abstract static class Builder {
 
     public abstract Builder setBzlTransitiveDigest(byte[] digest);
 
-    public abstract Builder setAccumulatedFileDigests(ImmutableMap<Label, String> value);
+    public abstract Builder setRecordedFileInputs(
+        ImmutableMap<RepoRecordedInput.File, String> value);
 
     public abstract Builder setEnvVariables(ImmutableMap<String, String> value);
 

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionContext.java

@@ -16,6 +16,7 @@
 
 import com.google.common.collect.ImmutableMap;
 import com.google.devtools.build.docgen.annot.DocCategory;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
 import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager;
 import com.google.devtools.build.lib.bazel.repository.starlark.StarlarkBaseExternalContext;
 import com.google.devtools.build.lib.runtime.ProcessWrapper;
@@ -50,6 +51,7 @@ public class ModuleExtensionContext extends StarlarkBaseExternalContext {
 
   protected ModuleExtensionContext(
       Path workingDirectory,
+      BlazeDirectories directories,
       Environment env,
       Map<String, String> envVariables,
       DownloadManager downloadManager,
@@ -62,13 +64,15 @@ protected ModuleExtensionContext(
       boolean rootModuleHasNonDevDependency) {
     super(
         workingDirectory,
+        directories,
         env,
         envVariables,
         downloadManager,
         timeoutScaling,
         processWrapper,
         starlarkSemantics,
-        remoteExecutor);
+        remoteExecutor,
+        /* allowWatchingFilesOutsideWorkspace= */ false);
     this.extensionId = extensionId;
     this.modules = modules;
     this.rootModuleHasNonDevDependency = rootModuleHasNonDevDependency;

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java

@@ -29,7 +29,6 @@
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Table;
-import com.google.devtools.build.lib.actions.FileValue;
 import com.google.devtools.build.lib.analysis.BlazeDirectories;
 import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode;
 import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager;
@@ -48,6 +47,7 @@
 import com.google.devtools.build.lib.profiler.ProfilerTask;
 import com.google.devtools.build.lib.profiler.SilentCloseable;
 import com.google.devtools.build.lib.rules.repository.NeedsSkyframeRestartException;
+import com.google.devtools.build.lib.rules.repository.RepoRecordedInput;
 import com.google.devtools.build.lib.rules.repository.RepositoryFunction;
 import com.google.devtools.build.lib.runtime.ProcessWrapper;
 import com.google.devtools.build.lib.runtime.RepositoryRemoteExecutor;
@@ -61,7 +61,6 @@
 import com.google.devtools.build.lib.util.Fingerprint;
 import com.google.devtools.build.lib.util.OS;
 import com.google.devtools.build.lib.vfs.Path;
-import com.google.devtools.build.lib.vfs.RootedPath;
 import com.google.devtools.build.skyframe.SkyFunction;
 import com.google.devtools.build.skyframe.SkyFunctionException;
 import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
@@ -71,7 +70,6 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Map;
@@ -219,7 +217,7 @@ public SkyValue compute(SkyKey skyKey, Environment env)
                   extension.getEvalFactors(),
                   LockFileModuleExtension.builder()
                       .setBzlTransitiveDigest(extension.getBzlTransitiveDigest())
-                      .setAccumulatedFileDigests(moduleExtensionResult.getAccumulatedFileDigests())
+                      .setRecordedFileInputs(moduleExtensionResult.getRecordedFileInputs())
                       .setEnvVariables(extension.getEnvVars())
                       .setGeneratedRepoSpecs(generatedRepoSpecs)
                       .setModuleExtensionMetadata(moduleExtensionMetadata)
@@ -291,7 +289,7 @@ private SingleExtensionEvalValue tryGettingValueFromLockFile(
                 + extensionId
                 + "' have changed");
       }
-      if (didFilesChange(env, lockedExtension.getAccumulatedFileDigests())) {
+      if (didFilesChange(env, directories, lockedExtension.getRecordedFileInputs())) {
         diffRecorder.record(
             "One or more files the extension '" + extensionId + "' is using have changed");
       }
@@ -393,39 +391,15 @@ private static boolean didRepoMappingsChange(
   }
 
   private static boolean didFilesChange(
-      Environment env, ImmutableMap<Label, String> accumulatedFileDigests)
+      Environment env,
+      BlazeDirectories directories,
+      ImmutableMap<RepoRecordedInput.File, String> recordedFileInputs)
       throws InterruptedException, NeedsSkyframeRestartException {
-    // Turn labels into FileValue keys & get those values
-    Map<Label, FileValue.Key> fileKeys = new HashMap<>();
-    for (Label label : accumulatedFileDigests.keySet()) {
-      try {
-        RootedPath rootedPath = RepositoryFunction.getRootedPathFromLabel(label, env);
-        fileKeys.put(label, FileValue.key(rootedPath));
-      } catch (NeedsSkyframeRestartException e) {
-        throw e;
-      } catch (EvalException e) {
-        // Consider those exception to be a cause for invalidation
-        return true;
-      }
-    }
-    SkyframeLookupResult result = env.getValuesAndExceptions(fileKeys.values());
+    boolean upToDate = RepoRecordedInput.areAllValuesUpToDate(env, directories, recordedFileInputs);
     if (env.valuesMissing()) {
       throw new NeedsSkyframeRestartException();
     }
-
-    // Compare the collected file values with the hashes stored in the lockfile
-    for (Entry<Label, String> entry : accumulatedFileDigests.entrySet()) {
-      FileValue fileValue = (FileValue) result.get(fileKeys.get(entry.getKey()));
-      try {
-        if (!entry.getValue().equals(RepositoryFunction.fileValueToMarkerValue(fileValue))) {
-          return true;
-        }
-      } catch (IOException e) {
-        // Consider those exception to be a cause for invalidation
-        return true;
-      }
-    }
-    return false;
+    return !upToDate;
   }
 
   /**
@@ -956,7 +930,7 @@ public RunModuleExtensionResult run(
         }
       }
       return RunModuleExtensionResult.create(
-          moduleContext.getAccumulatedFileDigests(),
+          moduleContext.getRecordedFileInputs(),
           threadContext.getGeneratedRepoSpecs(),
           moduleExtensionMetadata,
           repoMappingRecorder.recordedEntries());
@@ -992,6 +966,7 @@ private ModuleExtensionContext createContext(
           rootUsage != null && rootUsage.getHasNonDevUseExtension();
       return new ModuleExtensionContext(
           workingDirectory,
+          directories,
           env,
           clientEnvironmentSupplier.get(),
           downloadManager,
@@ -1016,7 +991,7 @@ static final class SingleExtensionEvalFunctionException extends SkyFunctionExcep
   @AutoValue
   abstract static class RunModuleExtensionResult {
 
-    abstract ImmutableMap<Label, String> getAccumulatedFileDigests();
+    abstract ImmutableMap<RepoRecordedInput.File, String> getRecordedFileInputs();
 
     abstract ImmutableMap<String, RepoSpec> getGeneratedRepoSpecs();
 
@@ -1025,12 +1000,12 @@ abstract static class RunModuleExtensionResult {
     abstract ImmutableTable<RepositoryName, String, RepositoryName> getRecordedRepoMappingEntries();
 
     static RunModuleExtensionResult create(
-        ImmutableMap<Label, String> accumulatedFileDigests,
+        ImmutableMap<RepoRecordedInput.File, String> recordedFileInputs,
         ImmutableMap<String, RepoSpec> generatedRepoSpecs,
         Optional<ModuleExtensionMetadata> moduleExtensionMetadata,
         ImmutableTable<RepositoryName, String, RepositoryName> recordedRepoMappingEntries) {
       return new AutoValue_SingleExtensionEvalFunction_RunModuleExtensionResult(
-          accumulatedFileDigests,
+          recordedFileInputs,
           generatedRepoSpecs,
           moduleExtensionMetadata,
           recordedRepoMappingEntries);

src/main/java/com/google/devtools/build/lib/bazel/repository/BUILD

@@ -37,6 +37,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/events",
         "//src/main/java/com/google/devtools/build/lib/packages",
         "//src/main/java/com/google/devtools/build/lib/repository:repository_events",
+        "//src/main/java/com/google/devtools/build/lib/rules:repository/repo_recorded_input",
         "//src/main/java/com/google/devtools/build/lib/rules:repository/repository_directory_value",
         "//src/main/java/com/google/devtools/build/lib/rules:repository/repository_function",
         "//src/main/java/com/google/devtools/build/lib/rules:repository/resolved_file_value",

src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/BUILD

@@ -35,6 +35,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/profiler",
         "//src/main/java/com/google/devtools/build/lib/repository:repository_events",
         "//src/main/java/com/google/devtools/build/lib/repository:request_repository_information_event",
+        "//src/main/java/com/google/devtools/build/lib/rules:repository/repo_recorded_input",
         "//src/main/java/com/google/devtools/build/lib/rules:repository/repository_directory_value",
         "//src/main/java/com/google/devtools/build/lib/rules:repository/repository_function",
         "//src/main/java/com/google/devtools/build/lib/rules:repository/workspace_attribute_mapper",
@@ -46,7 +47,6 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/starlarkbuildapi/repository",
         "//src/main/java/com/google/devtools/build/lib/util",
         "//src/main/java/com/google/devtools/build/lib/util:string",
-        "//src/main/java/com/google/devtools/build/lib/util/io",
         "//src/main/java/com/google/devtools/build/lib/util/io:out-err",
         "//src/main/java/com/google/devtools/build/lib/vfs",
         "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",