feat: stage assets under .cdk.assets

packages/@aws-cdk/assets-docker/lib/image-asset.ts

@@ -1,3 +1,4 @@
+import assets = require('@aws-cdk/assets');
 import ecr = require('@aws-cdk/aws-ecr');
 import cdk = require('@aws-cdk/cdk');
 import cxapi = require('@aws-cdk/cx-api');
@@ -49,14 +50,20 @@ export class DockerImageAsset extends cdk.Construct {
     super(scope, id);
 
     // resolve full path
-    this.directory = path.resolve(props.directory);
-    if (!fs.existsSync(this.directory)) {
-      throw new Error(`Cannot find image directory at ${this.directory}`);
+    const dir = path.resolve(props.directory);
+    if (!fs.existsSync(dir)) {
+      throw new Error(`Cannot find image directory at ${dir}`);
     }
-    if (!fs.existsSync(path.join(this.directory, 'Dockerfile'))) {
-      throw new Error(`No 'Dockerfile' found in ${this.directory}`);
+    if (!fs.existsSync(path.join(dir, 'Dockerfile'))) {
+      throw new Error(`No 'Dockerfile' found in ${dir}`);
     }
 
+    const staging = new assets.Staging(this, 'Staging', {
+      sourcePath: dir
+    });
+
+    this.directory = staging.stagedPath;
+
     const imageNameParameter = new cdk.CfnParameter(this, 'ImageName', {
       type: 'String',
       description: `ECR repository name and tag asset "${this.node.path}"`,

packages/@aws-cdk/assets-docker/package.json

@@ -69,13 +69,15 @@
     "@aws-cdk/aws-iam": "^0.28.0",
     "@aws-cdk/aws-lambda": "^0.28.0",
     "@aws-cdk/aws-s3": "^0.28.0",
+    "@aws-cdk/assets": "^0.28.0",
     "@aws-cdk/cdk": "^0.28.0",
     "@aws-cdk/cx-api": "^0.28.0"
   },
   "homepage": "https://github.com/awslabs/aws-cdk",
   "peerDependencies": {
     "@aws-cdk/aws-ecr": "^0.28.0",
     "@aws-cdk/aws-iam": "^0.28.0",
+    "@aws-cdk/assets": "^0.28.0",
     "@aws-cdk/aws-s3": "^0.28.0",
     "@aws-cdk/cdk": "^0.28.0"
   },

packages/@aws-cdk/assets-docker/test/test.image-asset.ts

@@ -1,7 +1,10 @@
 import { expect, haveResource, SynthUtils } from '@aws-cdk/assert';
 import iam = require('@aws-cdk/aws-iam');
 import cdk = require('@aws-cdk/cdk');
+import cxapi = require('@aws-cdk/cx-api');
+import fs = require('fs');
 import { Test } from 'nodeunit';
+import os = require('os');
 import path = require('path');
 import { DockerImageAsset } from '../lib';
 
@@ -143,5 +146,30 @@ export = {
       });
     }, /No 'Dockerfile' found in/);
     test.done();
+  },
+
+  'docker directory is staged if asset staging is enabled'(test: Test) {
+    const workdir = mkdtempSync();
+    process.chdir(workdir);
+
+    const app = new cdk.App({
+      context: { [cxapi.ASSET_STAGING_DIR_CONTEXT]: '.stage-me' }
+    });
+
+    const stack = new cdk.Stack(app, 'stack');
+
+    new DockerImageAsset(stack, 'MyAsset', {
+      directory: path.join(__dirname, 'demo-image')
+    });
+
+    app.run();
+
+    test.ok(fs.existsSync('.stage-me/96e3ffe92a19cbaa6c558942f7a60246/Dockerfile'));
+    test.ok(fs.existsSync('.stage-me/96e3ffe92a19cbaa6c558942f7a60246/index.py'));
+    test.done();
   }
 };
+
+function mkdtempSync() {
+  return fs.mkdtempSync(path.join(os.tmpdir(), 'test.assets'));
+}

packages/@aws-cdk/assets/lib/asset.ts

@@ -4,6 +4,7 @@ import cdk = require('@aws-cdk/cdk');
 import cxapi = require('@aws-cdk/cx-api');
 import fs = require('fs');
 import path = require('path');
+import { Staging } from './staging';
 
 /**
  * Defines the way an asset is packaged before it is uploaded to S3.
@@ -61,7 +62,10 @@ export class Asset extends cdk.Construct {
   public readonly s3Url: string;
 
   /**
-   * Resolved full-path location of this asset.
+   * The path to the asset (stringinfied token).
+   *
+   * If asset staging is disabled, this will just be the original path.
+   * If asset staging is enabled it will be the staged path.
    */
   public readonly assetPath: string;
 
@@ -84,16 +88,20 @@ export class Asset extends cdk.Construct {
   constructor(scope: cdk.Construct, id: string, props: GenericAssetProps) {
     super(scope, id);
 
-    // resolve full path
-    this.assetPath = path.resolve(props.path);
+    // stage the asset source (conditionally).
+    const staging = new Staging(this, 'Stage', {
+      sourcePath: path.resolve(props.path)
+    });
+
+    this.assetPath = staging.stagedPath;
 
     // sets isZipArchive based on the type of packaging and file extension
     const allowedExtensions: string[] = ['.jar', '.zip'];
     this.isZipArchive = props.packaging === AssetPackaging.ZipDirectory
       ? true
-      : allowedExtensions.some(ext => this.assetPath.toLowerCase().endsWith(ext));
+      : allowedExtensions.some(ext => staging.sourcePath.toLowerCase().endsWith(ext));
 
-    validateAssetOnDisk(this.assetPath, props.packaging);
+    validateAssetOnDisk(staging.sourcePath, props.packaging);
 
     // add parameters for s3 bucket and s3 key. those will be set by
     // the toolkit or by CI/CD when the stack is deployed and will include

packages/@aws-cdk/assets/lib/fs/copy.ts

@@ -0,0 +1,89 @@
+import fs = require('fs');
+import minimatch = require('minimatch');
+import path = require('path');
+import { FollowMode } from './follow-mode';
+
+export interface CopyOptions {
+  /**
+   * @default External only follows symlinks that are external to the source directory
+   */
+  follow?: FollowMode;
+
+  /**
+   * glob patterns to exclude from the copy.
+   */
+  exclude?: string[];
+}
+
+export function copyDirectory(srcDir: string, destDir: string, options: CopyOptions = { }, rootDir?: string) {
+  const follow = options.follow !== undefined ? options.follow : FollowMode.External;
+  const exclude = options.exclude || [];
+
+  rootDir = rootDir || srcDir;
+
+  if (!fs.statSync(srcDir).isDirectory()) {
+    throw new Error(`${srcDir} is not a directory`);
+  }
+
+  const files = fs.readdirSync(srcDir);
+  for (const file of files) {
+    const sourceFilePath = path.join(srcDir, file);
+
+    if (shouldExclude(path.relative(rootDir, sourceFilePath))) {
+      continue;
+    }
+
+    const destFilePath = path.join(destDir, file);
+
+    let stat: fs.Stats | undefined = follow === FollowMode.Always
+      ? fs.statSync(sourceFilePath)
+      : fs.lstatSync(sourceFilePath);
+
+    if (stat && stat.isSymbolicLink()) {
+      const target = fs.readlinkSync(sourceFilePath);
+
+      // determine if this is an external link (i.e. the target's absolute path
+      // is outside of the root directory).
+      const targetPath = path.normalize(path.resolve(srcDir, target));
+      const rootPath = path.normalize(rootDir);
+      const external = !targetPath.startsWith(rootPath);
+
+      if (follow === FollowMode.External && external) {
+        stat = fs.statSync(sourceFilePath);
+      } else {
+        fs.symlinkSync(target, destFilePath);
+        stat = undefined;
+      }
+    }
+
+    if (stat && stat.isDirectory()) {
+      fs.mkdirSync(destFilePath);
+      copyDirectory(sourceFilePath, destFilePath, options, rootDir);
+      stat = undefined;
+    }
+
+    if (stat && stat.isFile()) {
+      fs.copyFileSync(sourceFilePath, destFilePath);
+      stat = undefined;
+    }
+  }
+
+  function shouldExclude(filePath: string): boolean {
+    let excludeOutput = false;
+
+    for (const pattern of exclude) {
+      const negate = pattern.startsWith('!');
+      const match = minimatch(filePath, pattern, { matchBase: true, flipNegate: true });
+
+      if (!negate && match) {
+        excludeOutput = true;
+      }
+
+      if (negate && match) {
+        excludeOutput = false;
+      }
+    }
+
+    return excludeOutput;
+  }
+}

packages/@aws-cdk/assets/lib/fs/fingerprint.ts

@@ -0,0 +1,86 @@
+import crypto = require('crypto');
+import fs = require('fs');
+import path = require('path');
+import { FollowMode } from './follow-mode';
+
+const BUFFER_SIZE = 8 * 1024;
+
+export interface FingerprintOptions {
+  /**
+   * Extra information to encode into the fingerprint (e.g. build instructions
+   * and other inputs)
+   */
+  extra?: string;
+
+  /**
+   * List of exclude patterns (see `CopyOptions`)
+   * @default include all files
+   */
+  exclude?: string[];
+
+  /**
+   * What to do when we encounter symlinks.
+   * @default External only follows symlinks that are external to the source
+   * directory
+   */
+  follow?: FollowMode;
+}
+
+/**
+ * Produces fingerprint based on the contents of a single file or an entire directory tree.
+ *
+ * The fingerprint will also include:
+ * 1. An extra string if defined in `options.extra`.
+ * 2. The set of exclude patterns, if defined in `options.exclude`
+ * 3. The symlink follow mode value.
+ *
+ * @param fileOrDirectory The directory or file to fingerprint
+ * @param options Fingerprinting options
+ */
+export function fingerprint(fileOrDirectory: string, options: FingerprintOptions = { }) {
+  const follow = options.follow !== undefined ? options.follow : FollowMode.External;
+  const hash = crypto.createHash('md5');
+  addToHash(fileOrDirectory);
+
+  hash.update(`==follow==${follow}==\n\n`);
+
+  if (options.extra) {
+    hash.update(`==extra==${options.extra}==\n\n`);
+  }
+
+  for (const ex of options.exclude || []) {
+    hash.update(`==exclude==${ex}==\n\n`);
+  }
+
+  return hash.digest('hex');
+
+  function addToHash(pathToAdd: string) {
+    hash.update('==\n');
+    const relativePath = path.relative(fileOrDirectory, pathToAdd);
+    hash.update(relativePath + '\n');
+    hash.update('~~~~~~~~~~~~~~~~~~\n');
+    const stat = fs.statSync(pathToAdd);
+
+    if (stat.isSymbolicLink()) {
+      const target = fs.readlinkSync(pathToAdd);
+      hash.update(target);
+    } else if (stat.isDirectory()) {
+      for (const file of fs.readdirSync(pathToAdd)) {
+        addToHash(path.join(pathToAdd, file));
+      }
+    } else {
+      const file = fs.openSync(pathToAdd, 'r');
+      const buffer = Buffer.alloc(BUFFER_SIZE);
+
+      try {
+        let bytesRead;
+        do {
+          bytesRead = fs.readSync(file, buffer, 0, BUFFER_SIZE, null);
+          hash.update(buffer.slice(0, bytesRead));
+        } while (bytesRead === BUFFER_SIZE);
+      } finally {
+        fs.closeSync(file);
+      }
+    }
+  }
+}

packages/@aws-cdk/assets/lib/fs/follow-mode.ts

@@ -0,0 +1,29 @@
+export enum FollowMode {
+  /**
+   * Never follow symlinks.
+   */
+  Never = 'never',
+
+  /**
+   * Materialize all symlinks, whether they are internal or external to the source directory.
+   */
+  Always = 'always',
+
+  /**
+   * Only follows symlinks that are external to the source directory.
+   */
+  External = 'external',
+
+  // ----------------- TODO::::::::::::::::::::::::::::::::::::::::::::
+  /**
+   * Forbids source from having any symlinks pointing outside of the source
+   * tree.
+   *
+   * This is the safest mode of operation as it ensures that copy operations
+   * won't materialize files from the user's file system. Internal symlinks are
+   * not followed.
+   *
+   * If the copy operation runs into an external symlink, it will fail.
+   */
+  BlockExternal = 'internal-only',
+}

packages/@aws-cdk/assets/lib/fs/index.ts

@@ -0,0 +1,3 @@
+export * from './fingerprint';
+export * from './follow-mode';
+export * from './copy';

packages/@aws-cdk/assets/lib/index.ts

@@ -1 +1,2 @@
 export * from './asset';
+export * from './staging';