feat(toolkit): improve docker build time in CI (#1776)

When running in CI, try to pull the latest image first and use it as cache
for the build. CI is detected by the presence of the `CI` environment variable.

Closes #1748

Author: jogold

packages/aws-cdk/bin/cdk.ts

@@ -61,6 +61,7 @@ async function parseCommandLineArguments() {
     .command('deploy [STACKS..]', 'Deploys the stack(s) named STACKS into your AWS account', yargs => yargs
       .option('exclusively', { type: 'boolean', alias: 'e', desc: 'only deploy requested stacks, don\'t include dependencies' })
       .option('require-approval', { type: 'string', choices: [RequireApproval.Never, RequireApproval.AnyChange, RequireApproval.Broadening], desc: 'what security-sensitive changes need manual approval' }))
+      .option('ci', { type: 'boolean', desc: 'Force CI detection. Use --no-ci to disable CI autodetection.', default: process.env.CI !== undefined })
     .command('destroy [STACKS..]', 'Destroy the stack(s) named STACKS', yargs => yargs
       .option('exclusively', { type: 'boolean', alias: 'x', desc: 'only deploy requested stacks, don\'t include dependees' })
       .option('force', { type: 'boolean', alias: 'f', desc: 'Do not ask for confirmation before destroying the stacks' }))
@@ -172,7 +173,7 @@ async function initCommandLine() {
         return await cliBootstrap(args.ENVIRONMENTS, toolkitStackName, args.roleArn);
 
       case 'deploy':
-        return await cliDeploy(args.STACKS, args.exclusively, toolkitStackName, args.roleArn, configuration.combined.get(['requireApproval']));
+        return await cliDeploy(args.STACKS, args.exclusively, toolkitStackName, args.roleArn, configuration.combined.get(['requireApproval']), args.ci);
 
       case 'destroy':
         return await cliDestroy(args.STACKS, args.exclusively, args.force, args.roleArn);
@@ -324,7 +325,8 @@ async function initCommandLine() {
                            exclusively: boolean,
                            toolkitStackName: string,
                            roleArn: string | undefined,
-                           requireApproval: RequireApproval) {
+                           requireApproval: RequireApproval,
+                           ci: boolean) {
     if (requireApproval === undefined) { requireApproval = RequireApproval.Broadening; }
 
     const stacks = await appStacks.selectStacks(stackNames, exclusively ? ExtendedStackSelection.None : ExtendedStackSelection.Upstream);
@@ -362,7 +364,7 @@ async function initCommandLine() {
       }
 
       try {
-        const result = await deployStack({ stack, sdk: aws, toolkitInfo, deployName, roleArn });
+        const result = await deployStack({ stack, sdk: aws, toolkitInfo, deployName, roleArn, ci });
         const message = result.noOp
           ? ` ✅  %s (no changes)`
           : ` ✅  %s`;

packages/aws-cdk/lib/api/deploy-stack.ts

@@ -30,6 +30,7 @@ export interface DeployStackOptions {
   roleArn?: string;
   deployName?: string;
   quiet?: boolean;
+  ci?: boolean;
 }
 
 const LARGE_TEMPLATE_SIZE_KB = 50;
@@ -39,7 +40,7 @@ export async function deployStack(options: DeployStackOptions): Promise<DeploySt
     throw new Error(`The stack ${options.stack.name} does not have an environment`);
   }
 
-  const params = await prepareAssets(options.stack, options.toolkitInfo);
+  const params = await prepareAssets(options.stack, options.toolkitInfo, options.ci);
 
   const deployName = options.deployName || options.stack.name;
 

packages/aws-cdk/lib/api/toolkit-info.ts

@@ -99,11 +99,11 @@ export class ToolkitInfo {
   /**
    * Prepare an ECR repository for uploading to using Docker
    */
-  public async prepareEcrRepository(id: string, imageTag: string): Promise<EcrRepositoryInfo> {
+  public async prepareEcrRepository(assetId: string): Promise<EcrRepositoryInfo> {
     const ecr = await this.props.sdk.ecr(this.props.environment, Mode.ForWriting);
 
-    // Create the repository if it doesn't exist yet
-    const repositoryName = 'cdk/' + id.replace(/[:/]/g, '-').toLowerCase();
+    // Repository name based on asset id
+    const repositoryName = 'cdk/' + assetId.replace(/[:/]/g, '-').toLowerCase();
 
     let repository;
     try {
@@ -115,32 +115,34 @@ export class ToolkitInfo {
     }
 
     if (repository) {
-      try {
-        debug(`${repositoryName}: checking for image ${imageTag}`);
-        await ecr.describeImages({ repositoryName, imageIds: [{ imageTag }] }).promise();
-
-        // If we got here, the image already exists. Nothing else needs to be done.
-        return {
-          alreadyExists: true,
-          repositoryUri: repository.repositoryUri!,
-          repositoryName
-        };
-      } catch (e) {
-        if (e.code !== 'ImageNotFoundException') { throw e; }
-      }
-    } else {
-      debug(`${repositoryName}: creating`);
-      const response = await ecr.createRepository({ repositoryName }).promise();
-      repository = response.repository!;
-
-      // Better put a lifecycle policy on this so as to not cost too much money
-      await ecr.putLifecyclePolicy({
-        repositoryName,
-        lifecyclePolicyText: JSON.stringify(DEFAULT_REPO_LIFECYCLE)
-      }).promise();
+      return {
+        repositoryUri: repository.repositoryUri!,
+        repositoryName
+      };
     }
 
-    // The repo exists, image just needs to be uploaded. Get auth to do so.
+    debug(`${repositoryName}: creating`);
+    const response = await ecr.createRepository({ repositoryName }).promise();
+    repository = response.repository!;
+
+    // Better put a lifecycle policy on this so as to not cost too much money
+    await ecr.putLifecyclePolicy({
+      repositoryName,
+      lifecyclePolicyText: JSON.stringify(DEFAULT_REPO_LIFECYCLE)
+    }).promise();
+
+    return {
+      repositoryUri: repository.repositoryUri!,
+      repositoryName
+    };
+  }
+
+  /**
+   * Get ECR credentials
+   */
+  public async getEcrCredentials(): Promise<EcrCredentials> {
+    const ecr = await this.props.sdk.ecr(this.props.environment, Mode.ForReading);
+
     debug(`Fetching ECR authorization token`);
     const authData =  (await ecr.getAuthorizationToken({ }).promise()).authorizationData || [];
     if (authData.length === 0) {
@@ -150,28 +152,38 @@ export class ToolkitInfo {
     const [username, password] = token.split(':');
 
     return {
-      alreadyExists: false,
-      repositoryUri: repository.repositoryUri!,
-      repositoryName,
       username,
       password,
       endpoint: authData[0].proxyEndpoint!,
     };
   }
-}
 
-export type EcrRepositoryInfo = CompleteEcrRepositoryInfo | UploadableEcrRepositoryInfo;
+  /**
+   * Check if image already exists in ECR repository
+   */
+  public async checkEcrImage(repositoryName: string, imageTag: string): Promise<boolean> {
+    const ecr = await this.props.sdk.ecr(this.props.environment, Mode.ForReading);
 
-export interface CompleteEcrRepositoryInfo {
-  repositoryUri: string;
-  repositoryName: string;
-  alreadyExists: true;
+    try {
+      debug(`${repositoryName}: checking for image ${imageTag}`);
+      await ecr.describeImages({ repositoryName, imageIds: [{ imageTag }] }).promise();
+
+      // If we got here, the image already exists. Nothing else needs to be done.
+      return true;
+    } catch (e) {
+      if (e.code !== 'ImageNotFoundException') { throw e; }
+    }
+
+    return false;
+  }
 }
 
-export interface UploadableEcrRepositoryInfo {
+export interface EcrRepositoryInfo {
   repositoryUri: string;
   repositoryName: string;
-  alreadyExists: false;
+}
+
+export interface EcrCredentials {
   username: string;
   password: string;
   endpoint: string;

packages/aws-cdk/lib/assets.ts

@@ -10,7 +10,7 @@ import { zipDirectory } from './archive';
 import { prepareContainerAsset } from './docker';
 import { debug, success } from './logging';
 
-export async function prepareAssets(stack: SynthesizedStack, toolkitInfo?: ToolkitInfo): Promise<CloudFormation.Parameter[]> {
+export async function prepareAssets(stack: SynthesizedStack, toolkitInfo?: ToolkitInfo, ci?: boolean): Promise<CloudFormation.Parameter[]> {
   const assets = findAssets(stack.metadata);
   if (assets.length === 0) {
     return [];
@@ -26,21 +26,21 @@ export async function prepareAssets(stack: SynthesizedStack, toolkitInfo?: Toolk
   for (const asset of assets) {
     debug(` - ${asset.path} (${asset.packaging})`);
 
-    params = params.concat(await prepareAsset(asset, toolkitInfo));
+    params = params.concat(await prepareAsset(asset, toolkitInfo, ci));
   }
 
   return params;
 }
 
-async function prepareAsset(asset: AssetMetadataEntry, toolkitInfo: ToolkitInfo): Promise<CloudFormation.Parameter[]> {
+async function prepareAsset(asset: AssetMetadataEntry, toolkitInfo: ToolkitInfo, ci?: boolean): Promise<CloudFormation.Parameter[]> {
   debug('Preparing asset', JSON.stringify(asset));
   switch (asset.packaging) {
     case 'zip':
       return await prepareZipAsset(asset, toolkitInfo);
     case 'file':
       return await prepareFileAsset(asset, toolkitInfo);
     case 'container-image':
-      return await prepareContainerAsset(asset, toolkitInfo);
+      return await prepareContainerAsset(asset, toolkitInfo, ci);
     default:
       // tslint:disable-next-line:max-line-length
       throw new Error(`Unsupported packaging type: ${(asset as any).packaging}. You might need to upgrade your aws-cdk toolkit to support this asset type.`);

packages/aws-cdk/lib/docker.ts

@@ -18,38 +18,71 @@ import { PleaseHold } from './util/please-hold';
  *
  * As a workaround, we calculate our own digest over parts of the manifest that
  * are unlikely to change, and tag based on that.
+ *
+ * When running in CI, we pull the latest image first and use it as cache for
+ * the build. Generally pulling will be faster than building, especially for
+ * Dockerfiles with lots of OS/code packages installation or changes only in
+ * the bottom layers. When running locally chances are that we already have
+ * layers cache available.
+ *
+ * CI is detected by the presence of the `CI` environment variable or
+ * the `--ci` command line option.
  */
-export async function prepareContainerAsset(asset: ContainerImageAssetMetadataEntry, toolkitInfo: ToolkitInfo): Promise<CloudFormation.Parameter[]> {
+export async function prepareContainerAsset(asset: ContainerImageAssetMetadataEntry,
+                                            toolkitInfo: ToolkitInfo,
+                                            ci?: boolean): Promise<CloudFormation.Parameter[]> {
   debug(' 👑  Preparing Docker image asset:', asset.path);
 
   const buildHold = new PleaseHold(` ⌛ Building Docker image for ${asset.path}; this may take a while.`);
   try {
+    const ecr = await toolkitInfo.prepareEcrRepository(asset.id);
+    const latest = `${ecr.repositoryUri}:latest`;
+
+    let loggedIn = false;
+
+    // In CI we try to pull latest first
+    if (ci) {
+      await dockerLogin(toolkitInfo);
+      loggedIn = true;
+
+      try {
+        await shell(['docker', 'pull', latest]);
+      } catch (e) {
+        debug('Failed to pull latest image from ECR repository');
+      }
+    }
+
     buildHold.start();
 
-    const command = ['docker',
+    const baseCommand = ['docker',
       'build',
       '--quiet',
       asset.path];
+    const command = ci
+      ? [...baseCommand, '--cache-from', latest] // This does not fail if latest is not available
+      : baseCommand;
     const imageId = (await shell(command, { quiet: true })).trim();
+
     buildHold.stop();
 
     const tag = await calculateImageFingerprint(imageId);
 
-    debug(` ⌛  Image has tag ${tag}, preparing ECR repository`);
-    const ecr = await toolkitInfo.prepareEcrRepository(asset.id, tag);
+    debug(` ⌛  Image has tag ${tag}, checking ECR repository`);
+    const imageExists = await toolkitInfo.checkEcrImage(ecr.repositoryName, tag);
 
-    if (ecr.alreadyExists) {
+    if (imageExists) {
       debug(' 👑  Image already uploaded.');
     } else {
       // Login and push
       debug(` ⌛  Image needs to be uploaded first.`);
 
-      await shell(['docker', 'login',
-        '--username', ecr.username,
-        '--password', ecr.password,
-        ecr.endpoint]);
+      if (!loggedIn) { // We could be already logged in if in CI
+        await dockerLogin(toolkitInfo);
+        loggedIn = true;
+      }
 
       const qualifiedImageName = `${ecr.repositoryUri}:${tag}`;
+
       await shell(['docker', 'tag', imageId, qualifiedImageName]);
 
       // There's no way to make this quiet, so we can't use a PleaseHold. Print a header message.
@@ -58,6 +91,14 @@ export async function prepareContainerAsset(asset: ContainerImageAssetMetadataEn
       debug(` 👑  Docker image for ${asset.path} pushed.`);
     }
 
+    if (!loggedIn) { // We could be already logged in if in CI or if image did not exist
+      await dockerLogin(toolkitInfo);
+    }
+
+    // Always tag and push latest
+    await shell(['docker', 'tag', imageId, latest]);
+    await shell(['docker', 'push', latest]);
+
     return [
       { ParameterKey: asset.imageNameParameter, ParameterValue: `${ecr.repositoryName}:${tag}` },
     ];
@@ -72,6 +113,17 @@ export async function prepareContainerAsset(asset: ContainerImageAssetMetadataEn
   }
 }
 
+/**
+ * Get credentials from ECR and run docker login
+ */
+async function dockerLogin(toolkitInfo: ToolkitInfo) {
+  const credentials = await toolkitInfo.getEcrCredentials();
+  await shell(['docker', 'login',
+  '--username', credentials.username,
+  '--password', credentials.password,
+  credentials.endpoint]);
+}
+
 /**
  * Calculate image fingerprint.
  *