Reorganize build steps for parallelization. (#34)

Most of the code just wraps each build step in a ListenableFuture. The inputs to each build step then become the ListenableFutures for their dependencies (the steps they depend on). The code should be refactored to reduce the mess of ListenableFutures everywhere.
GoogleCloudPlatform · Feb 1, 2018 · 674fd56 · 674fd56
1 parent 197abdb
commit 674fd56
Show file tree

Hide file tree

Showing 28 changed files with 1,089 additions and 490 deletions.
diff --git a/jib-core/build.gradle b/jib-core/build.gradle
@@ -36,6 +36,7 @@ dependencies {
     compile 'org.apache.commons:commons-compress:1.15'
     compile 'com.google.guava:guava:23.5-jre'
     compile 'com.fasterxml.jackson.core:jackson-databind:2.9.2'
+    compile 'org.slf4j:slf4j-api:1.7.25'
     testCompile 'junit:junit:4.12'
     testCompile 'org.mockito:mockito-core:2.12.0'
 }

diff --git a/...egration-test/java/com/google/cloud/tools/jib/builder/BuildImageStepsIntegrationTest.java b/...egration-test/java/com/google/cloud/tools/jib/builder/BuildImageStepsIntegrationTest.java
@@ -16,40 +16,29 @@
 
 package com.google.cloud.tools.jib.builder;
 
-import com.google.cloud.tools.jib.cache.CacheMetadataCorruptedException;
-import com.google.cloud.tools.jib.image.DuplicateLayerException;
-import com.google.cloud.tools.jib.image.LayerCountMismatchException;
-import com.google.cloud.tools.jib.image.LayerPropertyNotFoundException;
 import com.google.cloud.tools.jib.registry.LocalRegistry;
-import com.google.cloud.tools.jib.registry.NonexistentDockerCredentialHelperException;
-import com.google.cloud.tools.jib.registry.NonexistentServerUrlDockerCredentialHelperException;
-import com.google.cloud.tools.jib.registry.RegistryAuthenticationFailedException;
-import com.google.cloud.tools.jib.registry.RegistryException;
 import com.google.common.io.CharStreams;
-import java.io.IOException;
 import java.io.InputStreamReader;
-import java.net.URISyntaxException;
 import java.nio.charset.StandardCharsets;
 import org.junit.Assert;
 import org.junit.ClassRule;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /** Integration tests for {@link BuildImageSteps}. */
 public class BuildImageStepsIntegrationTest {
 
   @ClassRule public static LocalRegistry localRegistry = new LocalRegistry(5000);
 
+  private static final Logger logger = LoggerFactory.getLogger(TestBuildLogger.class);
+
   @Rule public TemporaryFolder temporaryCacheDirectory = new TemporaryFolder();
 
   @Test
-  public void testSteps()
-      throws DuplicateLayerException, LayerPropertyNotFoundException, RegistryException,
-          LayerCountMismatchException, IOException, CacheMetadataCorruptedException,
-          RegistryAuthenticationFailedException,
-          NonexistentServerUrlDockerCredentialHelperException,
-          NonexistentDockerCredentialHelperException, URISyntaxException, InterruptedException {
+  public void testSteps() throws Exception {
     SourceFilesConfiguration sourceFilesConfiguration = new TestSourceFilesConfiguration();
     BuildConfiguration buildConfiguration =
         BuildConfiguration.builder()
@@ -60,6 +49,7 @@ public void testSteps()
             .setTargetImageName("testimage")
             .setTargetTag("testtag")
             .setMainClass("HelloWorld")
+            .setBuildLogger(new TestBuildLogger())
             .build();
 
     BuildImageSteps buildImageSteps =
@@ -69,11 +59,11 @@ public void testSteps()
             temporaryCacheDirectory.getRoot().toPath());
 
     long lastTime = System.nanoTime();
-    buildImageSteps.run();
-    System.out.println("Initial build time: " + ((System.nanoTime() - lastTime) / 1_000_000));
+    buildImageSteps.runAsync();
+    logger.info("Initial build time: " + ((System.nanoTime() - lastTime) / 1_000_000));
     lastTime = System.nanoTime();
-    buildImageSteps.run();
-    System.out.println("Secondary build time: " + ((System.nanoTime() - lastTime) / 1_000_000));
+    buildImageSteps.runAsync();
+    logger.info("Secondary build time: " + ((System.nanoTime() - lastTime) / 1_000_000));
 
     // TODO: Put this in a utility function.
     Runtime.getRuntime().exec("docker pull localhost:5000/testimage:testtag").waitFor();

diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/Timer.java b/jib-core/src/main/java/com/google/cloud/tools/jib/Timer.java
@@ -16,58 +16,73 @@
 
 package com.google.cloud.tools.jib;
 
+import com.google.cloud.tools.jib.builder.BuildLogger;
 import java.io.Closeable;
-import java.util.Stack;
 
 /**
  * Times execution intervals. This is only for testing purposes and will be removed before the first
  * release.
  */
 public class Timer implements Closeable {
 
-  private static Stack<String> labels = new Stack<>();
-  private static Stack<Long> times = new Stack<>();
+  private final BuildLogger buildLogger;
+  private final int depth;
 
-  private static StringBuilder log = new StringBuilder();
+  private String label;
+  private long startTime = System.nanoTime();
 
-  public static void print() {
-    System.out.println(log);
+  public Timer(BuildLogger buildLogger, String label) {
+    this(buildLogger, label, 0);
   }
 
-  public static Timer push(String label) {
-    logTabs();
-    log.append("TIMING\t");
-    log.append(label);
-    log.append("\n");
-    labels.push(label);
-    times.push(System.currentTimeMillis());
-    return new Timer();
+  private Timer(BuildLogger buildLogger, String label, int depth) {
+    this.buildLogger = buildLogger;
+    this.label = label;
+    this.depth = depth;
+
+    if (buildLogger != null) {
+      buildLogger.debug(getTabs().append("TIMING\t").append(label));
+      if (depth == 0) {
+        buildLogger.info("RUNNING\t" + label);
+      }
+    }
   }
 
-  public static void time(String label) {
-    pop();
-    push(label);
+  public Timer subTimer(String label) {
+    return new Timer(buildLogger, label, depth + 1);
   }
 
-  private static void pop() {
-    String label = labels.pop();
-    long time = times.pop();
-    logTabs();
-    log.append("TIMED\t");
-    log.append(label);
-    log.append(" : ");
-    log.append(System.currentTimeMillis() - time);
-    log.append("\n");
+  public void lap(String label) {
+    if (this.label == null) {
+      throw new IllegalStateException("Tried to lap Timer after closing");
+    }
+    if (buildLogger != null) {
+      double timeInMillis = (System.nanoTime() - startTime) / 1000 / 1000.0;
+      buildLogger.debug(
+          getTabs()
+              .append("TIMED\t")
+              .append(this.label)
+              .append(" : ")
+              .append(timeInMillis)
+              .append(" ms"));
+      if (depth == 0) {
+        buildLogger.info(this.label + " : " + timeInMillis + " ms");
+      }
+    }
+    this.label = label;
+    startTime = System.nanoTime();
   }
 
-  private static void logTabs() {
-    for (int i = 0; i < labels.size(); i++) {
-      log.append("\t");
+  private StringBuilder getTabs() {
+    StringBuilder tabs = new StringBuilder();
+    for (int i = 0; i < depth; i++) {
+      tabs.append("\t");
     }
+    return tabs;
   }
 
   @Override
   public void close() {
-    Timer.pop();
+    lap(null);
   }
 }
diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/AuthenticatePullStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/AuthenticatePullStep.java
@@ -16,6 +16,7 @@
 
 package com.google.cloud.tools.jib.builder;
 
+import com.google.cloud.tools.jib.Timer;
 import com.google.cloud.tools.jib.http.Authorization;
 import com.google.cloud.tools.jib.registry.RegistryAuthenticationFailedException;
 import com.google.cloud.tools.jib.registry.RegistryAuthenticators;
@@ -26,17 +27,22 @@
 /** Retrieves credentials to push from the base image registry. */
 class AuthenticatePullStep implements Callable<Authorization> {
 
+  private static final String DESCRIPTION = "Authenticating with base image registry";
+
   private final BuildConfiguration buildConfiguration;
 
   AuthenticatePullStep(BuildConfiguration buildConfiguration) {
     this.buildConfiguration = buildConfiguration;
   }
 
+  /** Depends on nothing. */
   @Override
   public Authorization call()
       throws RegistryAuthenticationFailedException, IOException, RegistryException {
-    return RegistryAuthenticators.forOther(
-            buildConfiguration.getBaseImageServerUrl(), buildConfiguration.getBaseImageName())
-        .authenticate();
+    try (Timer ignored = new Timer(buildConfiguration.getBuildLogger(), DESCRIPTION)) {
+      return RegistryAuthenticators.forOther(
+              buildConfiguration.getBaseImageServerUrl(), buildConfiguration.getBaseImageName())
+          .authenticate();
+    }
   }
 }
diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/AuthenticatePushStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/AuthenticatePushStep.java
@@ -16,6 +16,7 @@
 
 package com.google.cloud.tools.jib.builder;
 
+import com.google.cloud.tools.jib.Timer;
 import com.google.cloud.tools.jib.http.Authorization;
 import com.google.cloud.tools.jib.registry.DockerCredentialRetriever;
 import com.google.cloud.tools.jib.registry.NonexistentDockerCredentialHelperException;
@@ -27,25 +28,37 @@
 /** Retrieves credentials to push to a target registry. */
 class AuthenticatePushStep implements Callable<Authorization> {
 
+  private static final String DESCRIPTION = "Authenticating with %s using docker-credential-%s";
+
   private final BuildConfiguration buildConfiguration;
 
   AuthenticatePushStep(BuildConfiguration buildConfiguration) {
     this.buildConfiguration = buildConfiguration;
   }
 
+  /** Depends on nothing. */
   @Override
   @Nullable
   public Authorization call()
       throws NonexistentServerUrlDockerCredentialHelperException,
           NonexistentDockerCredentialHelperException, IOException {
-    if (buildConfiguration.getCredentialHelperName() == null) {
-      return null;
-    }
+    try (Timer ignored =
+        new Timer(
+            buildConfiguration.getBuildLogger(),
+            String.format(
+                DESCRIPTION,
+                buildConfiguration.getTargetServerUrl(),
+                buildConfiguration.getCredentialHelperName()))) {
+      if (buildConfiguration.getCredentialHelperName() == null) {
+        return null;
+      }
 
-    DockerCredentialRetriever dockerCredentialRetriever =
-        new DockerCredentialRetriever(
-            buildConfiguration.getTargetServerUrl(), buildConfiguration.getCredentialHelperName());
+      DockerCredentialRetriever dockerCredentialRetriever =
+          new DockerCredentialRetriever(
+              buildConfiguration.getTargetServerUrl(),
+              buildConfiguration.getCredentialHelperName());
 
-    return dockerCredentialRetriever.retrieve();
+      return dockerCredentialRetriever.retrieve();
+    }
   }
 }