Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adds useOnlyProjectCache jib-maven-plugin configuration. #134

Merged
merged 14 commits into from
Mar 19, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,10 @@
package com.google.cloud.tools.jib.builder;

import com.google.cloud.tools.jib.Command;
import com.google.cloud.tools.jib.cache.Caches;
import com.google.cloud.tools.jib.image.ImageReference;
import com.google.cloud.tools.jib.registry.LocalRegistry;
import java.nio.file.Path;
import org.junit.Assert;
import org.junit.ClassRule;
import org.junit.Rule;
Expand All @@ -44,11 +46,12 @@ public void testSteps() throws Exception {
.setMainClass("HelloWorld")
.build();

Path cacheDirectory = temporaryCacheDirectory.newFolder().toPath();
BuildImageSteps buildImageSteps =
new BuildImageSteps(
buildConfiguration,
sourceFilesConfiguration,
temporaryCacheDirectory.getRoot().toPath());
Caches.newInitializer(cacheDirectory).setBaseCacheDirectory(cacheDirectory));

long lastTime = System.nanoTime();
buildImageSteps.run();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,16 +19,17 @@
import com.google.cloud.tools.jib.Timer;
import com.google.cloud.tools.jib.blob.BlobDescriptor;
import com.google.cloud.tools.jib.cache.Cache;
import com.google.cloud.tools.jib.cache.CacheDirectoryNotOwnedException;
import com.google.cloud.tools.jib.cache.CacheMetadataCorruptedException;
import com.google.cloud.tools.jib.cache.CachedLayer;
import com.google.cloud.tools.jib.cache.Caches;
import com.google.cloud.tools.jib.http.Authorization;
import com.google.cloud.tools.jib.image.Image;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
Expand All @@ -40,24 +41,24 @@ public class BuildImageSteps {

private final BuildConfiguration buildConfiguration;
private final SourceFilesConfiguration sourceFilesConfiguration;
private final Path cacheDirectory;
private final Caches.Initializer cachesInitializer;

public BuildImageSteps(
BuildConfiguration buildConfiguration,
SourceFilesConfiguration sourceFilesConfiguration,
Path cacheDirectory) {
Caches.Initializer cachesInitializer) {
this.buildConfiguration = buildConfiguration;
this.sourceFilesConfiguration = sourceFilesConfiguration;
this.cacheDirectory = cacheDirectory;
this.cachesInitializer = cachesInitializer;
}

public BuildConfiguration getBuildConfiguration() {
return buildConfiguration;
}

public void run()
throws InterruptedException, ExecutionException, CacheMetadataCorruptedException,
IOException {
throws InterruptedException, ExecutionException, CacheMetadataCorruptedException, IOException,
CacheDirectoryNotOwnedException {
List<String> entrypoint =
EntrypointBuilder.makeEntrypoint(
sourceFilesConfiguration,
Expand All @@ -69,7 +70,10 @@ public void run()
ListeningExecutorService listeningExecutorService =
MoreExecutors.listeningDecorator(Executors.newCachedThreadPool());

try (Cache cache = Cache.init(cacheDirectory)) {
try (Caches caches = cachesInitializer.init()) {
Cache baseLayersCache = caches.getBaseCache();
Cache applicationLayersCache = caches.getApplicationCache();

timer2.lap("Setting up credential retrieval");
ListenableFuture<Authorization> retrieveTargetRegistryCredentialsFuture =
listeningExecutorService.submit(
Expand Down Expand Up @@ -112,7 +116,7 @@ public void run()
.call(
new PullAndCacheBaseImageLayersStep(
buildConfiguration,
cache,
baseLayersCache,
listeningExecutorService,
authenticatePullFuture,
pullBaseImageFuture),
Expand All @@ -134,7 +138,10 @@ public void run()
// Builds the application layers.
List<ListenableFuture<CachedLayer>> buildAndCacheApplicationLayerFutures =
new BuildAndCacheApplicationLayersStep(
buildConfiguration, sourceFilesConfiguration, cache, listeningExecutorService)
buildConfiguration,
sourceFilesConfiguration,
applicationLayersCache,
listeningExecutorService)
.call();

timer2.lap("Setting up container configuration push");
Expand Down
30 changes: 19 additions & 11 deletions jib-core/src/main/java/com/google/cloud/tools/jib/cache/Caches.java
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import java.io.Closeable;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.NotDirectoryException;
import java.nio.file.Path;

/**
Expand All @@ -47,13 +46,13 @@ public static class Initializer {
/** A file to store in the default base image layers cache to check ownership by Jib. */
private static final String OWNERSHIP_FILE_NAME = ".jib";

@VisibleForTesting
/**
* Ensures ownership of {@code cacheDirectory} by checking for the existence of {@link
* #OWNERSHIP_FILE_NAME}.
*
* <p>This is a safety check to make sure we are not writing to a directory not created by Jib.
*/
@VisibleForTesting
static void ensureOwnership(Path cacheDirectory)
throws CacheDirectoryNotOwnedException, IOException {
Path ownershipFile = cacheDirectory.resolve(OWNERSHIP_FILE_NAME);
Expand Down Expand Up @@ -107,22 +106,31 @@ public static Initializer newInitializer(Path applicationCacheDirectory) {

/** Instantiate with {@link Initializer#init}. */
private Caches(Path baseCacheDirectory, Path applicationCacheDirectory)
throws CacheMetadataCorruptedException, NotDirectoryException {
baseCache = Cache.init(baseCacheDirectory);
throws CacheMetadataCorruptedException, IOException {
applicationCache = Cache.init(applicationCacheDirectory);
}

@Override
public void close() throws IOException {
baseCache.close();
applicationCache.close();
// Ensures that only one Cache is initialized if using the same directory.
if (Files.isSameFile(baseCacheDirectory, applicationCacheDirectory)) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

just curious, does this need to be syncrhonized or made threadsafe somehow?

Copy link
Contributor Author

@coollog coollog Mar 14, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, most things the codebase are designed to be thread-local and/or immutable but the CacheMetadata in a Cache is shared between threads. The only method that mutates the CacheMetadata state is CacheMetadata#addLayer, which is synchronized.

baseCache = applicationCache;
} else {
baseCache = Cache.init(baseCacheDirectory);
}
}

Cache getBaseCache() {
public Cache getBaseCache() {
return baseCache;
}

Cache getApplicationCache() {
public Cache getApplicationCache() {
return applicationCache;
}

@Override
public void close() throws IOException {
applicationCache.close();

if (baseCache != applicationCache) {
baseCache.close();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,9 @@
import com.google.cloud.tools.jib.builder.BuildConfiguration;
import com.google.cloud.tools.jib.builder.BuildImageSteps;
import com.google.cloud.tools.jib.builder.SourceFilesConfiguration;
import com.google.cloud.tools.jib.cache.CacheDirectoryNotOwnedException;
import com.google.cloud.tools.jib.cache.CacheMetadataCorruptedException;
import com.google.cloud.tools.jib.cache.Caches;
import com.google.cloud.tools.jib.http.Authorization;
import com.google.cloud.tools.jib.http.Authorizations;
import com.google.cloud.tools.jib.image.ImageReference;
Expand Down Expand Up @@ -111,6 +113,9 @@ private Class<? extends BuildableManifestTemplate> getManifestTemplateClass() {
@Parameter(defaultValue = "Docker", required = true)
private ImageFormat imageFormat;

@Parameter(defaultValue = "false", required = true)
private boolean useOnlyProjectCache;

@Override
public void execute() throws MojoExecutionException, MojoFailureException {
validateParameters();
Expand Down Expand Up @@ -173,6 +178,10 @@ public void execute() throws MojoExecutionException, MojoFailureException {
throw new MojoExecutionException("Could not create cache directory: " + cacheDirectory, ex);
}
}
Caches.Initializer cachesInitializer = Caches.newInitializer(cacheDirectory);
if (useOnlyProjectCache) {
cachesInitializer.setBaseCacheDirectory(cacheDirectory);
}

getLog().info("");
getLog().info("Pushing image as " + targetImageReference);
Expand All @@ -186,7 +195,8 @@ public void execute() throws MojoExecutionException, MojoFailureException {

RegistryClient.setUserAgentSuffix(USER_AGENT_SUFFIX);

buildImage(new BuildImageSteps(buildConfiguration, sourceFilesConfiguration, cacheDirectory));
buildImage(
new BuildImageSteps(buildConfiguration, sourceFilesConfiguration, cachesInitializer));

getLog().info("");
getLog().info("Built and pushed image as " + targetImageReference);
Expand Down Expand Up @@ -232,6 +242,14 @@ void buildImage(BuildImageSteps buildImageSteps) throws MojoExecutionException {
getLog().error(ex);
// TODO: Add more suggestions for various build failures.
throwMojoExecutionExceptionWithHelpMessage(ex, null);

} catch (CacheDirectoryNotOwnedException ex) {
throwMojoExecutionExceptionWithHelpMessage(
ex,
"check that '"
+ ex.getCacheDirectory()
+ "' is not used by another application or set the `useOnlyProjectCache` "
+ "configuration");
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,12 @@
import com.google.api.client.http.HttpStatusCodes;
import com.google.cloud.tools.jib.builder.BuildConfiguration;
import com.google.cloud.tools.jib.builder.BuildImageSteps;
import com.google.cloud.tools.jib.cache.CacheDirectoryNotOwnedException;
import com.google.cloud.tools.jib.cache.CacheMetadataCorruptedException;
import com.google.cloud.tools.jib.registry.RegistryUnauthorizedException;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collections;
import java.util.concurrent.ExecutionException;
import org.apache.http.conn.HttpHostConnectException;
Expand Down Expand Up @@ -60,8 +63,8 @@ public void testBuildImage_pass() throws MojoExecutionException {

@Test
public void testBuildImage_cacheMetadataCorruptedException()
throws InterruptedException, ExecutionException, CacheMetadataCorruptedException,
IOException {
throws InterruptedException, ExecutionException, CacheMetadataCorruptedException, IOException,
CacheDirectoryNotOwnedException {
CacheMetadataCorruptedException mockCacheMetadataCorruptedException =
Mockito.mock(CacheMetadataCorruptedException.class);
Mockito.doThrow(mockCacheMetadataCorruptedException).when(mockBuildImageSteps).run();
Expand All @@ -80,8 +83,8 @@ public void testBuildImage_cacheMetadataCorruptedException()

@Test
public void testBuildImage_executionException_httpHostConnectException()
throws InterruptedException, ExecutionException, CacheMetadataCorruptedException,
IOException {
throws InterruptedException, ExecutionException, CacheMetadataCorruptedException, IOException,
CacheDirectoryNotOwnedException {
HttpHostConnectException mockHttpHostConnectException =
Mockito.mock(HttpHostConnectException.class);
Mockito.when(mockExecutionException.getCause()).thenReturn(mockHttpHostConnectException);
Expand All @@ -101,8 +104,8 @@ public void testBuildImage_executionException_httpHostConnectException()

@Test
public void testBuildImage_executionException_registryUnauthorizedException_statusCodeForbidden()
throws InterruptedException, ExecutionException, CacheMetadataCorruptedException,
IOException {
throws InterruptedException, ExecutionException, CacheMetadataCorruptedException, IOException,
CacheDirectoryNotOwnedException {
Mockito.when(mockRegistryUnauthorizedException.getHttpResponseException())
.thenReturn(mockHttpResponseException);
Mockito.when(mockRegistryUnauthorizedException.getImageReference())
Expand All @@ -127,8 +130,8 @@ public void testBuildImage_executionException_registryUnauthorizedException_stat

@Test
public void testBuildImage_executionException_registryUnauthorizedException_noCredentials()
throws InterruptedException, ExecutionException, CacheMetadataCorruptedException,
IOException {
throws InterruptedException, ExecutionException, CacheMetadataCorruptedException, IOException,
CacheDirectoryNotOwnedException {
Mockito.when(mockRegistryUnauthorizedException.getHttpResponseException())
.thenReturn(mockHttpResponseException);
Mockito.when(mockRegistryUnauthorizedException.getRegistry()).thenReturn("someregistry");
Expand All @@ -151,8 +154,8 @@ public void testBuildImage_executionException_registryUnauthorizedException_noCr

@Test
public void testBuildImage_executionException_registryUnauthorizedException_other()
throws InterruptedException, ExecutionException, CacheMetadataCorruptedException,
IOException {
throws InterruptedException, ExecutionException, CacheMetadataCorruptedException, IOException,
CacheDirectoryNotOwnedException {
Mockito.when(mockRegistryUnauthorizedException.getHttpResponseException())
.thenReturn(mockHttpResponseException);
Mockito.when(mockRegistryUnauthorizedException.getRegistry()).thenReturn("someregistry");
Expand All @@ -178,8 +181,8 @@ public void testBuildImage_executionException_registryUnauthorizedException_othe

@Test
public void testBuildImage_executionException_other()
throws InterruptedException, ExecutionException, CacheMetadataCorruptedException,
IOException {
throws InterruptedException, ExecutionException, CacheMetadataCorruptedException, IOException,
CacheDirectoryNotOwnedException {
Throwable throwable = new Throwable();
Mockito.when(mockExecutionException.getCause()).thenReturn(throwable);
Mockito.doThrow(mockExecutionException).when(mockBuildImageSteps).run();
Expand All @@ -196,8 +199,8 @@ public void testBuildImage_executionException_other()

@Test
public void testBuildImage_otherException()
throws InterruptedException, ExecutionException, CacheMetadataCorruptedException,
IOException {
throws InterruptedException, ExecutionException, CacheMetadataCorruptedException, IOException,
CacheDirectoryNotOwnedException {
IOException ioException = new IOException();
Mockito.doThrow(ioException).when(mockBuildImageSteps).run();

Expand All @@ -215,4 +218,30 @@ public void testBuildImage_otherException()
Mockito.verify(mockLog).error(ioException);
}
}

@Test
public void testBuildImage_cacheDirectoryNotOwnedException()
throws InterruptedException, ExecutionException, CacheDirectoryNotOwnedException,
CacheMetadataCorruptedException, IOException {
Path expectedCacheDirectory = Paths.get("some/path");

CacheDirectoryNotOwnedException mockCacheDirectoryNotOwnedException =
Mockito.mock(CacheDirectoryNotOwnedException.class);
Mockito.when(mockCacheDirectoryNotOwnedException.getCacheDirectory())
.thenReturn(expectedCacheDirectory);
Mockito.doThrow(mockCacheDirectoryNotOwnedException).when(mockBuildImageSteps).run();

try {
testBuildImageMojo.buildImage(mockBuildImageSteps);
Assert.fail("buildImage should have thrown an exception");

} catch (MojoExecutionException ex) {
Assert.assertEquals(
"Build image failed, perhaps you should check that '"
+ expectedCacheDirectory
+ "' is not used by another application or set the `useOnlyProjectCache` configuration",
ex.getMessage());
Assert.assertEquals(mockCacheDirectoryNotOwnedException, ex.getCause());
}
}
}