feat(lambda-nodejs): external and install modules (#8681)

Add support for:
* external modules: modules that should not be bundled. Defaults to
`aws-sdk`.
* install modules: modules that should not be bundled but included in
the `node_modules` folder of the Lambda package. Those modules are
installed in a Lambda compatible Docker image with the right installer
(`npm` or `yarn`) based on lock file detection.

Closes #6323
Closes #7912 

----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

.gitignore

@@ -35,3 +35,6 @@ cdk.out/
 
 # Yarn error log
 yarn-error.log
+
+# Parcel default cache directory
+.parcel-cache

packages/@aws-cdk/aws-lambda-nodejs/.gitignore

@@ -14,10 +14,6 @@ nyc.config.js
 .LAST_PACKAGE
 *.snk
 
-# Parcel
-.build
-.cache
-
 !test/integ-handlers/js-handler.js
 !test/function.test.handler2.js
 !.eslintrc.js

packages/@aws-cdk/aws-lambda-nodejs/README.md

@@ -53,10 +53,41 @@ new lambda.NodejsFunction(this, 'my-handler', {
 ```
 
 ### Configuring Parcel
-The `NodejsFunction` construct exposes some [Parcel](https://parceljs.org/) options via properties: `minify`, `sourceMaps`,
-`buildDir` and `cacheDir`.
+The `NodejsFunction` construct exposes some [Parcel](https://parceljs.org/) options via properties: `minify`, `sourceMaps` and `cacheDir`.
 
 Parcel transpiles your code (every internal module) with [@babel/preset-env](https://babeljs.io/docs/en/babel-preset-env) and uses the
 runtime version of your Lambda function as target.
 
 Configuring Babel with Parcel is possible via a `.babelrc` or a `babel` config in `package.json`.
+
+### Working with modules
+
+#### Externals
+By default, all node modules are bundled except for `aws-sdk`. This can be configured by specifying
+the `externalModules` prop.
+
+```ts
+new lambda.NodejsFunction(this, 'my-handler', {
+  externalModules: [
+    'aws-sdk', // Use the 'aws-sdk' available in the Lambda runtime
+    'cool-module', // 'cool-module' is already available in a Layer
+  ],
+});
+```
+
+#### Install modules
+By default, all node modules referenced in your Lambda code will be bundled by Parcel.
+Use the `nodeModules` prop to specify a list of modules that should not be bundled
+but instead included in the `node_modules` folder of the Lambda package. This is useful
+when working with native dependencies or when Parcel fails to bundle a module.
+
+```ts
+new lambda.NodejsFunction(this, 'my-handler', {
+  nodeModules: ['native-module', 'other-module']
+});
+```
+
+The modules listed in `nodeModules` must be present in the `package.json`'s dependencies. The
+same version will be used for installation. If a lock file is detected (`package-lock.json` or
+`yarn.lock`) it will be used along with the right installer (`npm` or `yarn`). The modules are
+installed in a [Lambda compatible Docker container](https://github.com/lambci/docker-lambda).

packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts

@@ -2,136 +2,213 @@ import * as lambda from '@aws-cdk/aws-lambda';
 import * as cdk from '@aws-cdk/core';
 import * as fs from 'fs';
 import * as path from 'path';
-import { findPkgPath } from './util';
+import { PackageJsonManager } from './package-json-manager';
+import { findClosestPathContaining } from './util';
 
 /**
- * Options for Parcel bundling
+ * Base options for Parcel bundling
  */
-export interface ParcelOptions {
+export interface ParcelBaseOptions {
   /**
-   * Entry file
+   * Whether to minify files when bundling.
+   *
+   * @default false
    */
-  readonly entry: string;
+  readonly minify?: boolean;
 
   /**
-   * Expose modules as UMD under this name
+   * Whether to include source maps when bundling.
+   *
+   * @default false
    */
-  readonly global: string;
+  readonly sourceMaps?: boolean;
 
   /**
-   * Minify
+   * The cache directory
+   *
+   * Parcel uses a filesystem cache for fast rebuilds.
+   *
+   * @default - `.cache` in the root directory
    */
-  readonly minify?: boolean;
+  readonly cacheDir?: string;
 
   /**
-   * Include source maps
+   * The root of the project. This will be used as the source for the volume
+   * mounted in the Docker container. If you specify this prop, ensure that
+   * this path includes `entry` and any module/dependencies used by your
+   * function otherwise bundling will not be possible.
+   *
+   * @default - the closest path containing a .git folder
    */
-  readonly sourceMaps?: boolean;
+  readonly projectRoot?: string;
 
   /**
-   * The cache directory
+   * Environment variables defined when Parcel runs.
+   *
+   * @default - no environment variables are defined.
    */
-  readonly cacheDir?: string;
+  readonly parcelEnvironment?: { [key: string]: string; };
 
   /**
-   * The node version to use as target for Babel
+   * A list of modules that should be considered as externals (already available
+   * in the runtime).
+   *
+   * @default ['aws-sdk']
    */
-  readonly nodeVersion: string;
+  readonly externalModules?: string[];
 
   /**
-   * The docker tag of the node base image to use in the parcel-bundler docker image
+   * A list of modules that should be installed instead of bundled. Modules are
+   * installed in a Lambda compatible environnment.
    *
-   * @see https://hub.docker.com/_/node/?tab=tags
+   * @default - all modules are bundled
    */
-  readonly nodeDockerTag: string;
+  readonly nodeModules?: string[];
 
   /**
-   * The root of the project. This will be used as the source for the volume
-   * mounted in the Docker container.
+   * The version of Parcel to use.
+   *
+   * @default - 2.0.0-beta.1
+   */
+  readonly parcelVersion?: string;
+}
+
+/**
+ * Options for Parcel bundling
+ */
+export interface ParcelOptions extends ParcelBaseOptions {
+  /**
+   * Entry file
    */
-  readonly projectRoot: string;
+  readonly entry: string;
 
   /**
-   * The environment variables to pass to the container running Parcel.
-   *
-   * @default - no environment variables are passed to the container
+   * The runtime of the lambda function
    */
-  readonly environment?: { [key: string]: string; };
+  readonly runtime: lambda.Runtime;
 }
 
 /**
- * Parcel code
+ * Bundling
  */
 export class Bundling {
+  /**
+   * Parcel bundled Lambda asset code
+   */
   public static parcel(options: ParcelOptions): lambda.AssetCode {
-    // Original package.json path and content
-    let pkgPath = findPkgPath();
-    if (!pkgPath) {
-      throw new Error('Cannot find a `package.json` in this project.');
+    // Find project root
+    const projectRoot = options.projectRoot ?? findClosestPathContaining(`.git${path.sep}`);
+    if (!projectRoot) {
+      throw new Error('Cannot find project root. Please specify it with `projectRoot`.');
     }
-    pkgPath = path.join(pkgPath, 'package.json');
-    const originalPkg = fs.readFileSync(pkgPath);
-    const originalPkgJson = JSON.parse(originalPkg.toString());
 
-    // Update engines.node in package.json to set the right Babel target
-    setEngines(options.nodeVersion, pkgPath, originalPkgJson);
+    // Bundling image derived from runtime bundling image (lambci)
+    const image = cdk.BundlingDockerImage.fromAsset(path.join(__dirname, '../parcel'), {
+      buildArgs: {
+        IMAGE: options.runtime.bundlingDockerImage.image,
+        PARCEL_VERSION: options.parcelVersion ?? '2.0.0-beta.1',
+      },
+    });
+
+    const packageJsonManager = new PackageJsonManager();
+
+    // Collect external and install modules
+    let includeNodeModules: { [key: string]: boolean } | undefined;
+    let dependencies: { [key: string]: string } | undefined;
+    const externalModules = options.externalModules ?? ['aws-sdk'];
+    if (externalModules || options.nodeModules) {
+      const modules = [...externalModules, ...options.nodeModules ?? []];
+      includeNodeModules = {};
+      for (const mod of modules) {
+        includeNodeModules[mod] = false;
+      }
+      if (options.nodeModules) {
+        dependencies = packageJsonManager.getVersions(options.nodeModules);
+      }
+    }
 
-    // Entry file path relative to container path
-    const containerEntryPath = path.join(cdk.AssetStaging.BUNDLING_INPUT_DIR, path.relative(options.projectRoot, path.resolve(options.entry)));
-
-    try {
-      const command = [
-        'parcel', 'build', containerEntryPath.replace(/\\/g, '/'), // Always use POSIX paths in the container
-        '--out-dir', cdk.AssetStaging.BUNDLING_OUTPUT_DIR,
-        '--out-file', 'index.js',
-        '--global', options.global,
-        '--target', 'node',
-        '--bundle-node-modules',
-        '--log-level', '2',
-        !options.minify && '--no-minify',
-        !options.sourceMaps && '--no-source-maps',
-        ...(options.cacheDir ? ['--cache-dir', '/parcel-cache'] : []),
-      ].filter(Boolean) as string[];
-
-      return lambda.Code.fromAsset(options.projectRoot, {
-        assetHashType: cdk.AssetHashType.BUNDLE,
-        bundling: {
-          image: cdk.BundlingDockerImage.fromAsset(path.join(__dirname, '../parcel-bundler'), {
-            buildArgs: {
-              NODE_TAG: options.nodeDockerTag ?? `${process.versions.node}-alpine`,
-            },
-          }),
-          environment: options.environment,
-          volumes: options.cacheDir
-            ? [{ containerPath: '/parcel-cache', hostPath: options.cacheDir }]
-            : [],
-          workingDirectory: path.dirname(containerEntryPath).replace(/\\/g, '/'), // Always use POSIX paths in the container
-          command,
+    // Configure target in package.json for Parcel
+    packageJsonManager.update({
+      'cdk-lambda': `${cdk.AssetStaging.BUNDLING_OUTPUT_DIR}/index.js`,
+      'targets': {
+        'cdk-lambda': {
+          context: 'node',
+          includeNodeModules: includeNodeModules ?? true,
+          sourceMap: options.sourceMaps ?? false,
+          minify: options.minify ?? false,
+          engines: {
+            node: `>= ${runtimeVersion(options.runtime)}`,
+          },
         },
-      });
-    } finally {
-      restorePkg(pkgPath, originalPkg);
+      },
+    });
+
+    // Entry file path relative to container path
+    const containerEntryPath = path.join(cdk.AssetStaging.BUNDLING_INPUT_DIR, path.relative(projectRoot, path.resolve(options.entry)));
+    const parcelCommand = `parcel build ${containerEntryPath.replace(/\\/g, '/')} --target cdk-lambda${options.cacheDir ? ' --cache-dir /parcel-cache' : ''}`;
+
+    let installer = Installer.NPM;
+    let lockfile: string | undefined;
+    let depsCommand = '';
+
+    if (dependencies) {
+      // Create a dummy package.json for dependencies that we need to install
+      fs.writeFileSync(
+        path.join(projectRoot, '.package.json'),
+        JSON.stringify({ dependencies }),
+      );
+
+      // Use npm unless we have a yarn.lock.
+      if (fs.existsSync(path.join(projectRoot, LockFile.YARN))) {
+        installer = Installer.YARN;
+        lockfile = LockFile.YARN;
+      } else if (fs.existsSync(path.join(projectRoot, LockFile.NPM))) {
+        lockfile = LockFile.NPM;
+      }
+
+      // Move dummy package.json and lock file then install
+      depsCommand = chain([
+        `mv ${cdk.AssetStaging.BUNDLING_INPUT_DIR}/.package.json ${cdk.AssetStaging.BUNDLING_OUTPUT_DIR}/package.json`,
+        lockfile ? `cp ${cdk.AssetStaging.BUNDLING_INPUT_DIR}/${lockfile} ${cdk.AssetStaging.BUNDLING_OUTPUT_DIR}/${lockfile}` : '',
+        `cd ${cdk.AssetStaging.BUNDLING_OUTPUT_DIR} && ${installer} install`,
+      ]);
     }
+
+    return lambda.Code.fromAsset(projectRoot, {
+      assetHashType: cdk.AssetHashType.BUNDLE,
+      bundling: {
+        image,
+        command: ['bash', '-c', chain([parcelCommand, depsCommand])],
+        environment: options.parcelEnvironment,
+        volumes: options.cacheDir
+          ? [{ containerPath: '/parcel-cache', hostPath: options.cacheDir }]
+          : [],
+        workingDirectory: path.dirname(containerEntryPath).replace(/\\/g, '/'), // Always use POSIX paths in the container
+      },
+    });
   }
 }
 
-function setEngines(nodeVersion: string, pkgPath: string, originalPkgJson: any): void {
-  // Update engines.node (Babel target)
-  const updateData = {
-    engines: {
-      node: `>= ${nodeVersion}`,
-    },
-  };
-
-  // Write new package.json
-  if (Object.keys(updateData).length !== 0) {
-    fs.writeFileSync(pkgPath, JSON.stringify({
-      ...originalPkgJson,
-      ...updateData,
-    }, null, 2));
+enum Installer {
+  NPM = 'npm',
+  YARN = 'yarn',
+}
+
+enum LockFile {
+  NPM = 'package-lock.json',
+  YARN = 'yarn.lock'
+}
+
+function runtimeVersion(runtime: lambda.Runtime): string {
+  const match = runtime.name.match(/nodejs(\d+)/);
+
+  if (!match) {
+    throw new Error('Cannot extract version from runtime.');
   }
+
+  return match[1];
 }
 
-function restorePkg(pkgPath: string, originalPkg: Buffer): void {
-  fs.writeFileSync(pkgPath, originalPkg);
+function chain(commands: string[]): string {
+  return commands.filter(c => !!c).join(' && ');
 }