feat: stage assets under .cdk.assets (#2182)

To ensure that assets are available for the toolchain to deploy after the CDK app
exists, the CLI will, by default, request that the app will stage the assets under
the `.cdk.assets` directory (relative to working directory).

The CDK will then *copy* all assets from their source locations to this staging
directory and will refer to the staging location as the asset path. Assets will
be stored using their content fingerprint (md5 hash) so they will never be copied
twice unless they change.

Docker build context directories will also be staged.

Staging is disabled by default and in cdk-integ.

Added .cdk.staging to all .gitignore files in cdk init templates.

Fixes #1716
Fixes #2096

Author: Elad Ben-Israel

packages/@aws-cdk/assets-docker/lib/image-asset.ts

@@ -1,3 +1,4 @@
+import assets = require('@aws-cdk/assets');
 import ecr = require('@aws-cdk/aws-ecr');
 import cdk = require('@aws-cdk/cdk');
 import cxapi = require('@aws-cdk/cx-api');
@@ -49,14 +50,20 @@ export class DockerImageAsset extends cdk.Construct {
     super(scope, id);
 
     // resolve full path
-    this.directory = path.resolve(props.directory);
-    if (!fs.existsSync(this.directory)) {
-      throw new Error(`Cannot find image directory at ${this.directory}`);
+    const dir = path.resolve(props.directory);
+    if (!fs.existsSync(dir)) {
+      throw new Error(`Cannot find image directory at ${dir}`);
     }
-    if (!fs.existsSync(path.join(this.directory, 'Dockerfile'))) {
-      throw new Error(`No 'Dockerfile' found in ${this.directory}`);
+    if (!fs.existsSync(path.join(dir, 'Dockerfile'))) {
+      throw new Error(`No 'Dockerfile' found in ${dir}`);
     }
 
+    const staging = new assets.Staging(this, 'Staging', {
+      sourcePath: dir
+    });
+
+    this.directory = staging.stagedPath;
+
     const imageNameParameter = new cdk.CfnParameter(this, 'ImageName', {
       type: 'String',
       description: `ECR repository name and tag asset "${this.node.path}"`,

packages/@aws-cdk/assets-docker/package-lock.json

@@ -69,13 +69,15 @@
     "@aws-cdk/aws-iam": "^0.28.0",
     "@aws-cdk/aws-lambda": "^0.28.0",
     "@aws-cdk/aws-s3": "^0.28.0",
+    "@aws-cdk/assets": "^0.28.0",
     "@aws-cdk/cdk": "^0.28.0",
     "@aws-cdk/cx-api": "^0.28.0"
   },
   "homepage": "https://github.com/awslabs/aws-cdk",
   "peerDependencies": {
     "@aws-cdk/aws-ecr": "^0.28.0",
     "@aws-cdk/aws-iam": "^0.28.0",
+    "@aws-cdk/assets": "^0.28.0",
     "@aws-cdk/aws-s3": "^0.28.0",
     "@aws-cdk/cdk": "^0.28.0"
   },

packages/@aws-cdk/assets-docker/package.json

@@ -1,7 +1,10 @@
 import { expect, haveResource, SynthUtils } from '@aws-cdk/assert';
 import iam = require('@aws-cdk/aws-iam');
 import cdk = require('@aws-cdk/cdk');
+import cxapi = require('@aws-cdk/cx-api');
+import fs = require('fs');
 import { Test } from 'nodeunit';
+import os = require('os');
 import path = require('path');
 import { DockerImageAsset } from '../lib';
 
@@ -143,5 +146,30 @@ export = {
       });
     }, /No 'Dockerfile' found in/);
     test.done();
+  },
+
+  'docker directory is staged if asset staging is enabled'(test: Test) {
+    const workdir = mkdtempSync();
+    process.chdir(workdir);
+
+    const app = new cdk.App({
+      context: { [cxapi.ASSET_STAGING_DIR_CONTEXT]: '.stage-me' }
+    });
+
+    const stack = new cdk.Stack(app, 'stack');
+
+    new DockerImageAsset(stack, 'MyAsset', {
+      directory: path.join(__dirname, 'demo-image')
+    });
+
+    app.run();
+
+    test.ok(fs.existsSync('.stage-me/96e3ffe92a19cbaa6c558942f7a60246/Dockerfile'));
+    test.ok(fs.existsSync('.stage-me/96e3ffe92a19cbaa6c558942f7a60246/index.py'));
+    test.done();
   }
 };
+
+function mkdtempSync() {
+  return fs.mkdtempSync(path.join(os.tmpdir(), 'test.assets'));
+}

packages/@aws-cdk/assets-docker/test/test.image-asset.ts

@@ -4,6 +4,7 @@ import cdk = require('@aws-cdk/cdk');
 import cxapi = require('@aws-cdk/cx-api');
 import fs = require('fs');
 import path = require('path');
+import { Staging } from './staging';
 
 /**
  * Defines the way an asset is packaged before it is uploaded to S3.
@@ -61,7 +62,10 @@ export class Asset extends cdk.Construct {
   public readonly s3Url: string;
 
   /**
-   * Resolved full-path location of this asset.
+   * The path to the asset (stringinfied token).
+   *
+   * If asset staging is disabled, this will just be the original path.
+   * If asset staging is enabled it will be the staged path.
    */
   public readonly assetPath: string;
 
@@ -84,16 +88,20 @@ export class Asset extends cdk.Construct {
   constructor(scope: cdk.Construct, id: string, props: GenericAssetProps) {
     super(scope, id);
 
-    // resolve full path
-    this.assetPath = path.resolve(props.path);
+    // stage the asset source (conditionally).
+    const staging = new Staging(this, 'Stage', {
+      sourcePath: path.resolve(props.path)
+    });
+
+    this.assetPath = staging.stagedPath;
 
     // sets isZipArchive based on the type of packaging and file extension
     const allowedExtensions: string[] = ['.jar', '.zip'];
     this.isZipArchive = props.packaging === AssetPackaging.ZipDirectory
       ? true
-      : allowedExtensions.some(ext => this.assetPath.toLowerCase().endsWith(ext));
+      : allowedExtensions.some(ext => staging.sourcePath.toLowerCase().endsWith(ext));
 
-    validateAssetOnDisk(this.assetPath, props.packaging);
+    validateAssetOnDisk(staging.sourcePath, props.packaging);
 
     // add parameters for s3 bucket and s3 key. those will be set by
     // the toolkit or by CI/CD when the stack is deployed and will include

packages/@aws-cdk/assets/lib/asset.ts

@@ -0,0 +1,89 @@
+import fs = require('fs');
+import minimatch = require('minimatch');
+import path = require('path');
+import { FollowMode } from './follow-mode';
+
+export interface CopyOptions {
+  /**
+   * @default External only follows symlinks that are external to the source directory
+   */
+  follow?: FollowMode;
+
+  /**
+   * glob patterns to exclude from the copy.
+   */
+  exclude?: string[];
+}
+
+export function copyDirectory(srcDir: string, destDir: string, options: CopyOptions = { }, rootDir?: string) {
+  const follow = options.follow !== undefined ? options.follow : FollowMode.External;
+  const exclude = options.exclude || [];
+
+  rootDir = rootDir || srcDir;
+
+  if (!fs.statSync(srcDir).isDirectory()) {
+    throw new Error(`${srcDir} is not a directory`);
+  }
+
+  const files = fs.readdirSync(srcDir);
+  for (const file of files) {
+    const sourceFilePath = path.join(srcDir, file);
+
+    if (shouldExclude(path.relative(rootDir, sourceFilePath))) {
+      continue;
+    }
+
+    const destFilePath = path.join(destDir, file);
+
+    let stat: fs.Stats | undefined = follow === FollowMode.Always
+      ? fs.statSync(sourceFilePath)
+      : fs.lstatSync(sourceFilePath);
+
+    if (stat && stat.isSymbolicLink()) {
+      const target = fs.readlinkSync(sourceFilePath);
+
+      // determine if this is an external link (i.e. the target's absolute path
+      // is outside of the root directory).
+      const targetPath = path.normalize(path.resolve(srcDir, target));
+      const rootPath = path.normalize(rootDir);
+      const external = !targetPath.startsWith(rootPath);
+
+      if (follow === FollowMode.External && external) {
+        stat = fs.statSync(sourceFilePath);
+      } else {
+        fs.symlinkSync(target, destFilePath);
+        stat = undefined;
+      }
+    }
+
+    if (stat && stat.isDirectory()) {
+      fs.mkdirSync(destFilePath);
+      copyDirectory(sourceFilePath, destFilePath, options, rootDir);
+      stat = undefined;
+    }
+
+    if (stat && stat.isFile()) {
+      fs.copyFileSync(sourceFilePath, destFilePath);
+      stat = undefined;
+    }
+  }
+
+  function shouldExclude(filePath: string): boolean {
+    let excludeOutput = false;
+
+    for (const pattern of exclude) {
+      const negate = pattern.startsWith('!');
+      const match = minimatch(filePath, pattern, { matchBase: true, flipNegate: true });
+
+      if (!negate && match) {
+        excludeOutput = true;
+      }
+
+      if (negate && match) {
+        excludeOutput = false;
+      }
+    }
+
+    return excludeOutput;
+  }
+}

packages/@aws-cdk/assets/lib/fs/copy.ts

@@ -0,0 +1,86 @@
+import crypto = require('crypto');
+import fs = require('fs');
+import path = require('path');
+import { FollowMode } from './follow-mode';
+
+const BUFFER_SIZE = 8 * 1024;
+
+export interface FingerprintOptions {
+  /**
+   * Extra information to encode into the fingerprint (e.g. build instructions
+   * and other inputs)
+   */
+  extra?: string;
+
+  /**
+   * List of exclude patterns (see `CopyOptions`)
+   * @default include all files
+   */
+  exclude?: string[];
+
+  /**
+   * What to do when we encounter symlinks.
+   * @default External only follows symlinks that are external to the source
+   * directory
+   */
+  follow?: FollowMode;
+}
+
+/**
+ * Produces fingerprint based on the contents of a single file or an entire directory tree.
+ *
+ * The fingerprint will also include:
+ * 1. An extra string if defined in `options.extra`.
+ * 2. The set of exclude patterns, if defined in `options.exclude`
+ * 3. The symlink follow mode value.
+ *
+ * @param fileOrDirectory The directory or file to fingerprint
+ * @param options Fingerprinting options
+ */
+export function fingerprint(fileOrDirectory: string, options: FingerprintOptions = { }) {
+  const follow = options.follow !== undefined ? options.follow : FollowMode.External;
+  const hash = crypto.createHash('md5');
+  addToHash(fileOrDirectory);
+
+  hash.update(`==follow==${follow}==\n\n`);
+
+  if (options.extra) {
+    hash.update(`==extra==${options.extra}==\n\n`);
+  }
+
+  for (const ex of options.exclude || []) {
+    hash.update(`==exclude==${ex}==\n\n`);
+  }
+
+  return hash.digest('hex');
+
+  function addToHash(pathToAdd: string) {
+    hash.update('==\n');
+    const relativePath = path.relative(fileOrDirectory, pathToAdd);
+    hash.update(relativePath + '\n');
+    hash.update('~~~~~~~~~~~~~~~~~~\n');
+    const stat = fs.statSync(pathToAdd);
+
+    if (stat.isSymbolicLink()) {
+      const target = fs.readlinkSync(pathToAdd);
+      hash.update(target);
+    } else if (stat.isDirectory()) {
+      for (const file of fs.readdirSync(pathToAdd)) {
+        addToHash(path.join(pathToAdd, file));
+      }
+    } else {
+      const file = fs.openSync(pathToAdd, 'r');
+      const buffer = Buffer.alloc(BUFFER_SIZE);
+
+      try {
+        let bytesRead;
+        do {
+          bytesRead = fs.readSync(file, buffer, 0, BUFFER_SIZE, null);
+          hash.update(buffer.slice(0, bytesRead));
+        } while (bytesRead === BUFFER_SIZE);
+      } finally {
+        fs.closeSync(file);
+      }
+    }
+  }
+}

packages/@aws-cdk/assets/lib/fs/fingerprint.ts

@@ -0,0 +1,29 @@
+export enum FollowMode {
+  /**
+   * Never follow symlinks.
+   */
+  Never = 'never',
+
+  /**
+   * Materialize all symlinks, whether they are internal or external to the source directory.
+   */
+  Always = 'always',
+
+  /**
+   * Only follows symlinks that are external to the source directory.
+   */
+  External = 'external',
+
+  // ----------------- TODO::::::::::::::::::::::::::::::::::::::::::::
+  /**
+   * Forbids source from having any symlinks pointing outside of the source
+   * tree.
+   *
+   * This is the safest mode of operation as it ensures that copy operations
+   * won't materialize files from the user's file system. Internal symlinks are
+   * not followed.
+   *
+   * If the copy operation runs into an external symlink, it will fail.
+   */
+  BlockExternal = 'internal-only',
+}

packages/@aws-cdk/assets/lib/fs/follow-mode.ts

@@ -0,0 +1,3 @@
+export * from './fingerprint';
+export * from './follow-mode';
+export * from './copy';

packages/@aws-cdk/assets/lib/fs/index.ts

@@ -1 +1,2 @@
 export * from './asset';
+export * from './staging';