Bridged --experimental_repository_cache value to HttpDownloader. Crea…

…ted HttpCache skeleton to implement caching logic of HttpDownloadValues as the first step (more types of caches will come later).

Having RepositoryDelegatorFunction initialize the cache in the respective RepositoryFunction handlers decouples the cache implementation from itself. It delegates the choice of Cache classes to the respective RepositoryFunctions, and let them decide what to do with the PathFragment of the cache location.

Continuation of commit 239d995.

A follow up CL will contain the implementation of HttpCache. For now, it's the empty interface of com.google.common.cache.Cache.

GITHUB: #1752

--
MOS_MIGRATED_REVID=135400724

src/main/java/com/google/devtools/build/lib/BUILD

@@ -18,6 +18,7 @@ filegroup(
         "//src/main/java/com/google/devtools/build/docgen:srcs",
         "//src/main/java/com/google/devtools/build/lib/actions:srcs",
         "//src/main/java/com/google/devtools/build/lib/bazel/dash:srcs",
+        "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache:srcs",
         "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader:srcs",
         "//src/main/java/com/google/devtools/build/lib/buildeventstream/proto:srcs",
         "//src/main/java/com/google/devtools/build/lib/buildeventstream/transports:srcs",
@@ -603,6 +604,7 @@ java_library(
         ":vfs",
         "//src/main/java/com/google/devtools/build/lib/actions",
         "//src/main/java/com/google/devtools/build/lib/bazel/dash",
+        "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache",
         "//src/main/java/com/google/devtools/build/lib/remote",
         "//src/main/java/com/google/devtools/build/lib/sandbox",
         "//src/main/java/com/google/devtools/build/lib/ssd",
@@ -668,6 +670,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib:shell",
         "//src/main/java/com/google/devtools/build/lib:skylarkinterface",
         "//src/main/java/com/google/devtools/build/lib:vfs",
+        "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache",
         "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
         "//src/main/java/com/google/devtools/build/skyframe",
         "//src/main/java/com/google/devtools/common/options",

src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java

@@ -30,6 +30,7 @@
 import com.google.devtools.build.lib.bazel.repository.NewGitRepositoryFunction;
 import com.google.devtools.build.lib.bazel.repository.NewHttpArchiveFunction;
 import com.google.devtools.build.lib.bazel.repository.RepositoryOptions;
+import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache;
 import com.google.devtools.build.lib.bazel.repository.skylark.SkylarkRepositoryFunction;
 import com.google.devtools.build.lib.bazel.repository.skylark.SkylarkRepositoryModule;
 import com.google.devtools.build.lib.bazel.rules.android.AndroidNdkRepositoryFunction;
@@ -69,7 +70,7 @@
 
 import java.util.Map.Entry;
 import java.util.concurrent.atomic.AtomicBoolean;
-
+import java.util.concurrent.atomic.AtomicReference;
 import javax.annotation.Nullable;
 
 /**
@@ -83,12 +84,13 @@ public class BazelRepositoryModule extends BlazeModule {
   private final SkylarkRepositoryFunction skylarkRepositoryFunction =
       new SkylarkRepositoryFunction();
   private final RepositoryDelegatorFunction delegator;
+  private final AtomicReference<RepositoryCache> repositoryCache = new AtomicReference<>();
 
   public BazelRepositoryModule() {
     this.repositoryHandlers =
         ImmutableMap.<String, RepositoryFunction>builder()
             .put(LocalRepositoryRule.NAME, new LocalRepositoryFunction())
-            .put(HttpArchiveRule.NAME, new HttpArchiveFunction())
+            .put(HttpArchiveRule.NAME, new HttpArchiveFunction(repositoryCache))
             .put(GitRepositoryRule.NAME, new GitRepositoryFunction())
             .put(HttpJarRule.NAME, new HttpJarFunction())
             .put(HttpFileRule.NAME, new HttpFileFunction())
@@ -166,6 +168,11 @@ public void initializeRuleClasses(ConfiguredRuleClassProvider.Builder builder) {
   public void handleOptions(OptionsProvider optionsProvider) {
     PackageCacheOptions pkgOptions = optionsProvider.getOptions(PackageCacheOptions.class);
     isFetch.set(pkgOptions != null && pkgOptions.fetch);
+
+    RepositoryOptions repoOptions = optionsProvider.getOptions(RepositoryOptions.class);
+    if (repoOptions != null && repoOptions.experimentalRepositoryCache != null) {
+      repositoryCache.set(new RepositoryCache(repoOptions.experimentalRepositoryCache));
+    }
   }
 
   @Override

src/main/java/com/google/devtools/build/lib/bazel/repository/HttpArchiveFunction.java

@@ -16,6 +16,7 @@
 
 import com.google.devtools.build.lib.analysis.BlazeDirectories;
 import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache;
 import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader;
 import com.google.devtools.build.lib.bazel.rules.workspace.HttpArchiveRule;
 import com.google.devtools.build.lib.packages.Rule;
@@ -31,12 +32,26 @@
 import com.google.devtools.build.skyframe.SkyValue;
 
 import java.io.IOException;
+import java.util.concurrent.atomic.AtomicReference;
 
 /**
  * Downloads a file over HTTP.
  */
 public class HttpArchiveFunction extends RepositoryFunction {
 
+  private final AtomicReference<RepositoryCache> repositoryCache;
+
+  protected HttpArchiveFunction() {
+    repositoryCache = new AtomicReference<>();
+  }
+
+  /**
+   * @param repositoryCache the reference to the repository cache.
+   */
+  public HttpArchiveFunction(AtomicReference<RepositoryCache> repositoryCache) {
+    this.repositoryCache = repositoryCache;
+  }
+
   @Override
   public boolean isLocal(Rule rule) {
     return false;
@@ -63,8 +78,10 @@ public SkyValue fetch(
     //
     // This would download png.tar.gz to .external-repository/png/png.tar.gz.
     createDirectory(outputDirectory);
-    Path downloadedPath = HttpDownloader.download(
-        rule, outputDirectory, env.getListener(), clientEnvironment);
+    Path downloadedPath = (repositoryCache == null || repositoryCache.get() == null)
+        ? HttpDownloader.download(rule, outputDirectory, env.getListener(), clientEnvironment)
+        : HttpDownloader.download(rule, outputDirectory, env.getListener(), clientEnvironment,
+            repositoryCache.get());
 
     DecompressorValue.decompress(getDescriptor(rule, downloadedPath, outputDirectory));
     return RepositoryDirectoryValue.create(outputDirectory);

src/main/java/com/google/devtools/build/lib/bazel/repository/cache/BUILD

@@ -0,0 +1,14 @@
+package(
+    default_visibility = ["//src:__subpackages__"],
+)
+
+filegroup(
+    name = "srcs",
+    srcs = glob(["*"]),
+)
+
+java_library(
+    name = "cache",
+    srcs = ["RepositoryCache.java"],
+    deps = ["//src/main/java/com/google/devtools/build/lib:vfs"],
+)

src/main/java/com/google/devtools/build/lib/bazel/repository/cache/RepositoryCache.java

@@ -0,0 +1,33 @@
+// Copyright 2016 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository.cache;
+
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+/**
+ * The cache implementation of HttpDownloadValues.
+ */
+public class RepositoryCache {
+
+  /**
+   * @param repositoryCachePath The base location of the repository cache.
+   */
+  @SuppressWarnings("unused")
+  public RepositoryCache(PathFragment repositoryCachePath) {
+    // TODO(jingwen): Preload in-memory set of cached values here.
+    // Create any cache subdirectories here as well.
+  }
+
+}

src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/BUILD

@@ -16,6 +16,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib:packages-internal",
         "//src/main/java/com/google/devtools/build/lib:syntax",
         "//src/main/java/com/google/devtools/build/lib:vfs",
+        "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache",
         "//src/main/java/com/google/devtools/build/skyframe",
         "//third_party:guava",
         "//third_party:jsr305",

src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java

@@ -16,6 +16,7 @@
 
 import com.google.common.hash.Hasher;
 import com.google.common.hash.Hashing;
+import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache;
 import com.google.devtools.build.lib.events.Event;
 import com.google.devtools.build.lib.events.EventHandler;
 import com.google.devtools.build.lib.packages.Rule;
@@ -97,6 +98,22 @@ public static Path download(
     }
   }
 
+  @SuppressWarnings("unused")
+  @Nullable
+  public static Path download(
+      Rule rule, Path outputDirectory, EventHandler eventHandler, Map<String, String> clientEnv,
+      RepositoryCache repositoryCache)
+      throws RepositoryFunctionException, InterruptedException {
+
+    /* TODO(jingwen): No-op with the cache controller for now.
+     * This will be the place where we detect cache hits with the sha256 value
+     * and copy the values accordingly.
+     * 
+     * For now, delegating operation to the original download method.
+     */
+    return download(rule, outputDirectory, eventHandler, clientEnv);
+  }
+
   @Nullable
   public static Path download(
       String url, String sha256, String type, Path output, EventHandler eventHandler, Map<String,