[6.4.0] Lockfile updates & fixes (#19153) (#19175)

* Add 'environ' attribute to module extensions and update lockfile

- Add 'environ' to module extensions
- Store the variables and their values in the lockfile
- Compare the variables and values with the lockfile and re-evaluate or error if they differ

PiperOrigin-RevId: 548964193
Change-Id: Ic2d52fe3332e93095c414d8bca4c9b4312bca8c2

# Conflicts:
#	src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtension.java
#	src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java
#	src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryModule.java
#	src/main/java/com/google/devtools/build/lib/skyframe/ActionEnvironmentFunction.java
#	src/main/java/com/google/devtools/build/lib/skyframe/ClientEnvironmentFunction.java
#	src/main/java/com/google/devtools/build/lib/starlarkbuildapi/repository/RepositoryModuleApi.java
#	src/main/java/com/google/devtools/build/skydoc/fakebuildapi/repository/FakeRepositoryModule.java
#	src/test/java/com/google/devtools/build/lib/bazel/bzlmod/StarlarkBazelModuleTest.java
#	src/test/py/bazel/bzlmod/bazel_lockfile_test.py

* Remove extra changes

* Re-run extension if its files change
fixes #19068

PiperOrigin-RevId: 552823534
Change-Id: I87256b2cf954b932e24c70e22386020599f21a6f

* Do not fail on new fields added to lockfile data
Fixes #19105

PiperOrigin-RevId: 553068023
Change-Id: I877bc8ece0641c01119a9295e09175a2d0a3a0c1

---------

Co-authored-by: Salma Samy <salmasamy@google.com>

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java

@@ -109,25 +109,34 @@ public ImmutableList<String> getModuleAndFlagsDiff(
     return moduleDiff.build();
   }
 
+  /** Returns the differences between an extension and its locked data */
   public ImmutableList<String> getModuleExtensionDiff(
-      LockFileModuleExtension lockedExtension,
-      ImmutableMap<ModuleKey, ModuleExtensionUsage> lockedExtensionUsages,
       ModuleExtensionId extensionId,
       byte[] transitiveDigest,
-      ImmutableMap<ModuleKey, ModuleExtensionUsage> extensionUsages) {
+      boolean filesChanged,
+      ImmutableMap<String, String> envVariables,
+      ImmutableMap<ModuleKey, ModuleExtensionUsage> extensionUsages,
+      ImmutableMap<ModuleKey, ModuleExtensionUsage> lockedExtensionUsages) {
+    LockFileModuleExtension lockedExtension = getModuleExtensions().get(extensionId);
+
     ImmutableList.Builder<String> extDiff = new ImmutableList.Builder<>();
-    if (lockedExtension == null) {
-      extDiff.add("The module extension '" + extensionId + "' does not exist in the lockfile");
-    } else {
-      if (!Arrays.equals(transitiveDigest, lockedExtension.getBzlTransitiveDigest())) {
+    if (!Arrays.equals(transitiveDigest, lockedExtension.getBzlTransitiveDigest())) {
         extDiff.add(
             "The implementation of the extension '"
                 + extensionId
                 + "' or one of its transitive .bzl files has changed");
-      }
-      if (!extensionUsages.equals(lockedExtensionUsages)) {
-        extDiff.add("The usages of the extension named '" + extensionId + "' has changed");
-      }
+    }
+    if (filesChanged) {
+      extDiff.add("One or more files the extension '" + extensionId + "' is using have changed");
+    }
+    if (!extensionUsages.equals(lockedExtensionUsages)) {
+      extDiff.add("The usages of the extension '" + extensionId + "' has changed");
+    }
+    if (!envVariables.equals(lockedExtension.getEnvVariables())) {
+      extDiff.add(
+          "The environment variables the extension '"
+              + extensionId
+              + "' depends on (or their values) have changed");
     }
     return extDiff.build();
   }

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java

@@ -91,6 +91,19 @@ public ModuleKey read(JsonReader jsonReader) throws IOException {
         }
       };
 
+  public static final TypeAdapter<Label> LABEL_TYPE_ADAPTER =
+      new TypeAdapter<Label>() {
+        @Override
+        public void write(JsonWriter jsonWriter, Label label) throws IOException {
+          jsonWriter.value(label.getUnambiguousCanonicalForm());
+        }
+
+        @Override
+        public Label read(JsonReader jsonReader) throws IOException {
+          return Label.parseCanonicalUnchecked(jsonReader.nextString());
+        }
+      };
+
   public static final TypeAdapter<ModuleExtensionId> MODULE_EXTENSION_ID_TYPE_ADAPTER =
       new TypeAdapter<>() {
         @Override
@@ -308,6 +321,7 @@ public static Gson createLockFileGson(Path moduleFilePath) {
         .registerTypeAdapterFactory(IMMUTABLE_SET)
         .registerTypeAdapterFactory(OPTIONAL)
         .registerTypeAdapterFactory(new LocationTypeAdapterFactory(moduleFilePath))
+        .registerTypeAdapter(Label.class, LABEL_TYPE_ADAPTER)
         .registerTypeAdapter(Version.class, VERSION_TYPE_ADAPTER)
         .registerTypeAdapter(ModuleKey.class, MODULE_KEY_TYPE_ADAPTER)
         .registerTypeAdapter(ModuleExtensionId.class, MODULE_EXTENSION_ID_TYPE_ADAPTER)

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/LockFileModuleExtension.java

@@ -16,6 +16,7 @@
 
 import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.cmdline.Label;
 import com.google.devtools.build.lib.events.ExtendedEventHandler.Postable;
 import com.ryanharter.auto.value.gson.GenerateTypeAdapter;
 
@@ -28,13 +29,36 @@
 @GenerateTypeAdapter
 public abstract class LockFileModuleExtension implements Postable {
 
+  public static Builder builder() {
+    return new AutoValue_LockFileModuleExtension.Builder()
+        // TODO(salmasamy) can be removed when updating lockfile version
+        .setEnvVariables(ImmutableMap.of())
+        .setAccumulatedFileDigests(ImmutableMap.of());
+  }
+
   @SuppressWarnings("mutable")
   public abstract byte[] getBzlTransitiveDigest();
 
+  public abstract ImmutableMap<Label, String> getAccumulatedFileDigests();
+
+  public abstract ImmutableMap<String, String> getEnvVariables();
+
   public abstract ImmutableMap<String, RepoSpec> getGeneratedRepoSpecs();
 
-  public static LockFileModuleExtension create(
-      byte[] transitiveDigest, ImmutableMap<String, RepoSpec> generatedRepoSpecs) {
-    return new AutoValue_LockFileModuleExtension(transitiveDigest, generatedRepoSpecs);
+  public abstract Builder toBuilder();
+
+  /** Builder type for {@link LockFileModuleExtension}. */
+  @AutoValue.Builder
+  public abstract static class Builder {
+
+    public abstract Builder setBzlTransitiveDigest(byte[] digest);
+
+    public abstract Builder setAccumulatedFileDigests(ImmutableMap<Label, String> value);
+
+    public abstract Builder setEnvVariables(ImmutableMap<String, String> value);
+
+    public abstract Builder setGeneratedRepoSpecs(ImmutableMap<String, RepoSpec> value);
+
+    public abstract LockFileModuleExtension build();
   }
 }

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtension.java

@@ -16,21 +16,23 @@
 
 import com.google.auto.value.AutoValue;
 import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.devtools.build.lib.cmdline.Label;
 import com.google.devtools.build.lib.events.EventHandler;
 import com.google.devtools.build.lib.packages.StarlarkExportable;
 import javax.annotation.Nullable;
 import net.starlark.java.annot.StarlarkBuiltin;
 import net.starlark.java.eval.StarlarkCallable;
+import net.starlark.java.eval.StarlarkValue;
 import net.starlark.java.syntax.Location;
 
 /**
  * A module extension object, which can be used to perform arbitrary logic in order to create repos
  * or register toolchains and execution platforms.
  */
 @AutoValue
-public abstract class ModuleExtension {
+public abstract class ModuleExtension implements StarlarkValue {
   public abstract String getName();
 
   public abstract StarlarkCallable getImplementation();
@@ -43,6 +45,8 @@ public abstract class ModuleExtension {
 
   public abstract Location getLocation();
 
+  public abstract ImmutableList<String> getEnvVariables();
+
   public static Builder builder() {
     return new AutoValue_ModuleExtension.Builder();
   }
@@ -63,6 +67,8 @@ public abstract static class Builder {
 
     public abstract Builder setTagClasses(ImmutableMap<String, TagClass> value);
 
+    public abstract Builder setEnvVariables(ImmutableList<String> value);
+
     public abstract ModuleExtension build();
   }