bzlmod: IndexRegistry

(#13316)

This CL implements the basic "Index" registry handling (which is what the Bazel Central Registry will be using). A registry object does two things right now: one is fetching the module file (used during ModuleFileFunction), and the other is telling later stages how to instantiate a repo rule to fetch the contents of the module (used during BzlmodRepoRuleFunction).

PiperOrigin-RevId: 382739746

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java

@@ -0,0 +1,80 @@
+// Copyright 2021 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package com.google.devtools.build.lib.bazel.bzlmod;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import net.starlark.java.eval.StarlarkInt;
+
+/**
+ * Builder for a {@link RepoSpec} object that indicates how to materialize a repo corresponding to
+ * an {@code http_archive} repo rule call.
+ */
+public class ArchiveRepoSpecBuilder {
+  private final ImmutableMap.Builder<String, Object> attrBuilder;
+
+  public ArchiveRepoSpecBuilder() {
+    attrBuilder = new ImmutableMap.Builder<>();
+  }
+
+  public ArchiveRepoSpecBuilder setRepoName(String repoName) {
+    attrBuilder.put("name", repoName);
+    return this;
+  }
+
+  public ArchiveRepoSpecBuilder setUrls(ImmutableList<String> urls) {
+    attrBuilder.put("urls", urls);
+    return this;
+  }
+
+  public ArchiveRepoSpecBuilder setIntegrity(String integrity) {
+    attrBuilder.put("integrity", integrity);
+    return this;
+  }
+
+  public ArchiveRepoSpecBuilder setStripPrefix(String stripPrefix) {
+    attrBuilder.put("strip_prefix", stripPrefix);
+    return this;
+  }
+
+  public ArchiveRepoSpecBuilder setPatches(ImmutableList<String> patches) {
+    attrBuilder.put("patches", patches);
+    return this;
+  }
+
+  public ArchiveRepoSpecBuilder setPatchStrip(int patchStrip) {
+    attrBuilder.put("patch_args", ImmutableList.of("-p" + patchStrip));
+    return this;
+  }
+
+  public ArchiveRepoSpecBuilder setRemotePatches(ImmutableMap<String, String> remotePatches) {
+    attrBuilder.put("remote_patches", remotePatches);
+    return this;
+  }
+
+  public ArchiveRepoSpecBuilder setRemotePatchStrip(int remotePatchStrip) {
+    attrBuilder.put("remote_patch_strip", StarlarkInt.of(remotePatchStrip));
+    return this;
+  }
+
+  public RepoSpec build() {
+    return RepoSpec.builder()
+        .setBzlFile("@bazel_tools//tools/build_defs/repo:http.bzl")
+        .setRuleClassName("http_archive")
+        .setAttributes(attrBuilder.build())
+        .build();
+  }
+}

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD

@@ -15,10 +15,12 @@ filegroup(
 java_library(
     name = "common",
     srcs = [
+        "ArchiveRepoSpecBuilder.java",
         "ModuleKey.java",
         "RepoSpec.java",
     ],
     deps = [
+        "//src/main/java/net/starlark/java/eval",
         "//third_party:auto_value",
         "//third_party:guava",
     ],
@@ -27,12 +29,17 @@ java_library(
 java_library(
     name = "registry",
     srcs = [
+        "IndexRegistry.java",
         "Registry.java",
         "RegistryFactory.java",
+        "RegistryFactoryImpl.java",
     ],
     deps = [
         ":common",
+        "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
         "//src/main/java/com/google/devtools/build/lib/events",
+        "//third_party:gson",
+        "//third_party:guava",
     ],
 )
 

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java

@@ -0,0 +1,180 @@
+// Copyright 2021 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package com.google.devtools.build.lib.bazel.bzlmod;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader;
+import com.google.devtools.build.lib.events.ExtendedEventHandler;
+import com.google.gson.FieldNamingPolicy;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URL;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * Represents a Bazel module registry that serves a list of module metadata from a static HTTP
+ * server or a local file path.
+ */
+// TODO(wyv): Insert "For details, see ..." when we have public documentation.
+public class IndexRegistry implements Registry {
+  private final URI uri;
+  private final HttpDownloader httpDownloader;
+  private final Map<String, String> clientEnv;
+  private final Gson gson;
+
+  public IndexRegistry(URI uri, HttpDownloader httpDownloader, Map<String, String> clientEnv) {
+    this.uri = uri;
+    this.httpDownloader = httpDownloader;
+    this.clientEnv = clientEnv;
+    this.gson =
+        new GsonBuilder()
+            .setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES)
+            .create();
+  }
+
+  @Override
+  public String getUrl() {
+    return uri.toString();
+  }
+
+  private String constructUrl(String base, String... segments) {
+    StringBuilder url = new StringBuilder(base);
+    for (String segment : segments) {
+      if (url.charAt(url.length() - 1) != '/' && !segment.startsWith("/")) {
+        url.append('/');
+      }
+      url.append(segment);
+    }
+    return url.toString();
+  }
+
+  /** Grabs a file from the given URL. Returns {@link Optional#empty} if the file doesn't exist. */
+  private Optional<byte[]> grabFile(String url, ExtendedEventHandler eventHandler)
+      throws IOException, InterruptedException {
+    try {
+      return Optional.of(
+          httpDownloader.downloadAndReadOneUrl(new URL(url), eventHandler, clientEnv));
+    } catch (FileNotFoundException e) {
+      return Optional.empty();
+    }
+  }
+
+  @Override
+  public Optional<byte[]> getModuleFile(ModuleKey key, ExtendedEventHandler eventHandler)
+      throws IOException, InterruptedException {
+    return grabFile(
+        constructUrl(getUrl(), "modules", key.getName(), key.getVersion(), "MODULE.bazel"),
+        eventHandler);
+  }
+
+  /** Represents fields available in {@code bazel_registry.json} for the registry. */
+  private static class BazelRegistryJson {
+    String[] mirrors;
+  }
+
+  /** Represents fields available in {@code source.json} for each version of a module. */
+  private static class SourceJson {
+    URL url;
+    String integrity;
+    String stripPrefix;
+    Map<String, String> patches;
+    int patchStrip;
+  }
+
+  /**
+   * Grabs a JSON file from the given URL, and returns it as a parsed object with fields in {@code
+   * T}. Returns {@link Optional#empty} if the file doesn't exist.
+   */
+  private <T> Optional<T> grabJson(String url, Class<T> klass, ExtendedEventHandler eventHandler)
+      throws IOException, InterruptedException {
+    Optional<byte[]> bytes = grabFile(url, eventHandler);
+    if (!bytes.isPresent()) {
+      return Optional.empty();
+    }
+    String jsonString = new String(bytes.get(), UTF_8);
+    return Optional.of(gson.fromJson(jsonString, klass));
+  }
+
+  @Override
+  public RepoSpec getRepoSpec(ModuleKey key, String repoName, ExtendedEventHandler eventHandler)
+      throws IOException, InterruptedException {
+    Optional<BazelRegistryJson> bazelRegistryJson =
+        grabJson(
+            constructUrl(getUrl(), "bazel_registry.json"), BazelRegistryJson.class, eventHandler);
+    Optional<SourceJson> sourceJson =
+        grabJson(
+            constructUrl(getUrl(), "modules", key.getName(), key.getVersion(), "source.json"),
+            SourceJson.class,
+            eventHandler);
+    if (!sourceJson.isPresent()) {
+      throw new FileNotFoundException(
+          String.format("Module %s's source information not found in registry %s", key, getUrl()));
+    }
+    URL sourceUrl = sourceJson.get().url;
+    if (sourceUrl == null) {
+      throw new IOException(String.format("Missing source URL for module %s", key));
+    }
+    if (sourceJson.get().integrity == null) {
+      throw new IOException(String.format("Missing integrity for module %s", key));
+    }
+
+    ImmutableList.Builder<String> urls = new ImmutableList.Builder<>();
+    // For each mirror specified in bazel_registry.json, add a URL that's essentially the mirror
+    // URL concatenated with the source URL.
+    if (bazelRegistryJson.isPresent() && bazelRegistryJson.get().mirrors != null) {
+      for (String mirror : bazelRegistryJson.get().mirrors) {
+        try {
+          new URL(mirror);
+        } catch (MalformedURLException e) {
+          throw new IOException("Malformed mirror URL", e);
+        }
+
+        urls.add(constructUrl(mirror, sourceUrl.getAuthority(), sourceUrl.getFile()));
+      }
+    }
+    // Finally add the original source URL itself.
+    urls.add(sourceUrl.toString());
+
+    // Build remote patches as key-value pairs of "url" => "integrity".
+    ImmutableMap.Builder<String, String> remotePatches = new ImmutableMap.Builder<>();
+    if (sourceJson.get().patches != null) {
+      for (Map.Entry<String, String> entry : sourceJson.get().patches.entrySet()) {
+        remotePatches.put(
+            constructUrl(
+                getUrl(), "modules", key.getName(), key.getVersion(), "patches", entry.getKey()),
+            entry.getValue());
+      }
+    }
+
+    return new ArchiveRepoSpecBuilder()
+        .setRepoName(repoName)
+        .setUrls(urls.build())
+        .setIntegrity(sourceJson.get().integrity)
+        .setStripPrefix(Strings.nullToEmpty(sourceJson.get().stripPrefix))
+        .setRemotePatches(remotePatches.build())
+        .setRemotePatchStrip(sourceJson.get().patchStrip)
+        .build();
+  }
+}

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileGlobals.java

@@ -42,6 +42,8 @@ public void module(String name, String version, String compatibilityLevel) throw
       throw Starlark.errorf("the module() directive can only be called once");
     }
     moduleCalled = true;
+    // TODO(wyv): add validation logic for name (alphanumerical) and version (use ParsedVersion) &
+    //   others in the future
     module.setName(name).setVersion(version);
     // TODO(wyv): compatibility level
   }
@@ -51,6 +53,8 @@ public void bazelDep(String name, String version, String repoName) throws EvalEx
     if (repoName.isEmpty()) {
       repoName = name;
     }
+    // TODO(wyv): add validation logic for name (alphanumerical), version (use ParsedVersion),
+    //   and repoName (RepositoryName?)
     if (deps.putIfAbsent(repoName, ModuleKey.create(name, version)) != null) {
       throw Starlark.errorf("a bazel_dep with the repo name %s already exists", repoName);
     }

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Registry.java

@@ -32,5 +32,10 @@ public interface Registry {
   Optional<byte[]> getModuleFile(ModuleKey key, ExtendedEventHandler eventHandler)
       throws IOException, InterruptedException;
 
-  // TODO(wyv): getRepoSpec
+  /**
+   * Retrieves the {@link RepoSpec} object that indicates how the contents of the module identified
+   * by {@code key} should be materialized as a repo (with name {@code repoName}).
+   */
+  RepoSpec getRepoSpec(ModuleKey key, String repoName, ExtendedEventHandler eventHandler)
+      throws IOException, InterruptedException;
 }

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java

@@ -0,0 +1,47 @@
+// Copyright 2021 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package com.google.devtools.build.lib.bazel.bzlmod;
+
+import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Map;
+import java.util.function.Supplier;
+
+/** Prod implementation of {@link RegistryFactory}. */
+public class RegistryFactoryImpl implements RegistryFactory {
+  private final HttpDownloader httpDownloader;
+  private final Supplier<Map<String, String>> clientEnvironmentSupplier;
+
+  public RegistryFactoryImpl(
+      HttpDownloader httpDownloader, Supplier<Map<String, String>> clientEnvironmentSupplier) {
+    this.httpDownloader = httpDownloader;
+    this.clientEnvironmentSupplier = clientEnvironmentSupplier;
+  }
+
+  @Override
+  public Registry getRegistryWithUrl(String url) throws URISyntaxException {
+    URI uri = new URI(url);
+    switch (uri.getScheme()) {
+      case "http":
+      case "https":
+      case "file":
+        return new IndexRegistry(uri, httpDownloader, clientEnvironmentSupplier.get());
+      default:
+        throw new IllegalArgumentException("Unrecognized registry URL protocol: " + url);
+    }
+  }
+}

src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/BUILD

@@ -23,7 +23,6 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/vfs",
         "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
         "//src/main/java/net/starlark/java/syntax",
-        "//third_party:flogger",
         "//third_party:guava",
         "//third_party:jsr305",
     ],

src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD

@@ -24,6 +24,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:common",
         "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:registry",
         "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution",
+        "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
         "//src/main/java/com/google/devtools/build/lib/events",
         "//src/main/java/com/google/devtools/build/lib/pkgcache",
         "//src/main/java/com/google/devtools/build/lib/skyframe:file_function",