feat(batch): add batchAwsCustomCompiler option to duck.config.js

README.md

@@ -116,7 +116,7 @@ module.exports = {
   soyJarPath: "lib/closure-templates.jar",
   // (Required) Directories where Closure Templates .soy files are stored
   soyFileRoots: ["src/soy"],
-  // Classpaths for Closure Templates 
+  // Classpaths for Closure Templates
   soyClasspaths: ["lib/plugin.jar"],
   // CLI options for Closure Templates
   soyOptions: {
@@ -130,6 +130,11 @@ module.exports = {
   concurrency: 4,
   // Build in batch mode with faast.js on "aws" for production or "local" for debug (default: disabled)
   batch: "aws",
+  // Custom Closure Compiler published npm pacakge in AWS batch (default: google-closure-compiler-linux)
+  batchAwsCustomCompiler: {
+    name: "my-custom-closure-compiler",
+    version: "^1.0.0",
+  },
   // Options for faast.js in batch mode. See https://faastjs.org/docs/api/faastjs.awsoptions
   batchOptions: {},
   // Reporters (choose from "json", "text" or "xunit")
@@ -153,7 +158,7 @@ module.exports = {
   // Hostname for serve command (default: 0.0.0.0)
   host: "localhost",
   // Port number for serve command (default: 9810)
-  port:  1234,
+  port: 1234,
   // Use HTTP/2 in serve command (deafult: false)
   http2: true,
   // Settings for HTTPS (HTTP/2) (default: not specified, HTTP is used)
@@ -163,7 +168,7 @@ module.exports = {
     // A path to a self-signed certificate
     certPath: "path/to/cert.pem",
   },
-}
+};
 ```
 
 Also see [`examples`](examples).
@@ -186,27 +191,30 @@ duck provides batch mode that compiles all entry points simultaneously in parall
 
 1. Setting AWS credentials in Node.js (See [AWS document](https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/setting-credentials-node.html))
 2. Configure `batchOptions` in `duck.config.js`. It's used for faast.js as [AWSOptions](https://faastjs.org/docs/api/faastjs.awsoptions).
+
 ```js
-const closureVersion = require('google-closure-compiler/package.json').version;
+const closureVersion = require("google-closure-compiler/package.json").version;
 
 module.exports = {
   batchOptions: {
-    region: 'ap-northeast-1',
+    region: "ap-northeast-1",
     awsLambdaOptions: {
-      Runtime: 'nodejs10.x',
+      Runtime: "nodejs10.x",
     },
-    include: ['path/to/your/source/**/*.js'],
-    exclude: ['**/*_spec.js'],
+    include: ["path/to/your/source/**/*.js"],
+    exclude: ["**/*_spec.js"],
     packageJson: {
       dependencies: {
-        'google-closure-compiler-linux': closureVersion,
-        'google-closure-library': closureVersion,
+        "google-closure-compiler-linux": closureVersion,
+        "google-closure-library": closureVersion,
       },
     },
   },
 };
 ```
+
 3. Run `build` or `build:js` command with `--batch aws`.
+
 ```console
 $ duck build --batch aws
 ```
@@ -216,7 +224,7 @@ $ duck build --batch aws
 - Use `--batch local` for [local debugging](https://faastjs.org/docs/local)
 - Use `DEBUG=faast:info` or [other log level](https://faastjs.org/docs/workflow#debug-environment-variable) to get more debug information
 - Get `logUrl` from debug info and view it in CloudWatch logs
-- Use `FAAST_PACKAGE_DIR=foo/bar` to investigate a package sent to Lambda 
+- Use `FAAST_PACKAGE_DIR=foo/bar` to investigate a package sent to Lambda
 
 Also see [faast.js document](https://faastjs.org/docs/api/faastjs.awsoptions) for more information.
 
@@ -234,9 +242,9 @@ Then specify them and enable `http2` in `duck.config.js`.
 module.exports = {
   http2: true,
   https: {
-    keyPath: './path/to/duck-key.pem',
-    certPath: './path/to/duck-cert.pem'
-  }
+    keyPath: "./path/to/duck-key.pem",
+    certPath: "./path/to/duck-cert.pem",
+  },
 };
 ```
 

examples/build-and-serve/duck.config.js

@@ -9,4 +9,12 @@ module.exports = {
   //   keyPath: "./duck-key.pem",
   //   certPath: "./duck-cert.pem",
   // },
+  batchOptions: {
+    region: "ap-northeast-1",
+    awsLambdaOptions: {
+      Runtime: "nodejs16.x",
+    },
+    include: ["js", "node_modules/google-closure-library"],
+    exclude: [],
+  },
 };

src/batch.ts

@@ -49,18 +49,32 @@ function getBatchOptions(config: DuckConfig): AwsOptions | LocalOptions {
   );
 }
 
+function getNativeCompilerPackageForBatch(config: DuckConfig): {
+  name: string;
+  version: string;
+} {
+  const { batch, batchAwsCustomCompiler } = config;
+  if (batch === "aws" && batchAwsCustomCompiler) {
+    return { ...batchAwsCustomCompiler };
+  }
+  const name = `google-closure-compiler-${getOsForNativeImage(config)}`;
+  // This is run in local context and it (osx/windows) is probably different
+  // platform from AWS (linux). `google-closure-compiler-linux` is not
+  // available in macOS / Windows, so use the version of the parent package
+  // `google-closure-compiler`.
+  const { version } = require("google-closure-compiler/package.json");
+  const major = semver.major(version);
+  return { name, version: `^${major}.0.0` };
+}
+
 function defaultBatchOptions(config: DuckConfig): AwsOptions {
-  const closureVersion =
-    require("google-closure-compiler/package.json").version;
-  const major = semver.major(closureVersion);
+  const compiler = getNativeCompilerPackageForBatch(config);
   return {
     packageJson: {
       // To suppress npm warnings
       private: true,
       dependencies: {
-        [`google-closure-compiler-${getOsForNativeImage(
-          config
-        )}`]: `^${major}.0.0`,
+        [compiler.name]: compiler.version,
       },
     },
     webpackOptions: {

src/compiler-core.ts

@@ -2,10 +2,13 @@ import fs from "fs";
 import { compiler as ClosureCompiler } from "google-closure-compiler";
 import { dirname } from "path";
 import * as tempy from "tempy";
+import { DuckConfig } from "./duckconfig";
 import { WarningsWhitelistItem } from "./entryconfig";
 import { logger } from "./logger";
 import { CompileErrorItem } from "./report";
 
+declare const __non_webpack_require__: NodeRequire;
+
 export interface CompilerOptions {
   [idx: string]: any;
   /**
@@ -53,11 +56,10 @@ export type CompilerOptionsFormattingType =
   | "PRINT_INPUT_DELIMITER"
   | "SINGLE_QUOTES";
 
-export interface ExtendedCompilerOptions {
+export type ExtendedCompilerOptions = {
   compilerOptions: CompilerOptions;
-  batch?: "aws" | "local";
   warningsWhitelist?: WarningsWhitelistItem[];
-}
+} & Pick<DuckConfig, "batch" | "batchAwsCustomCompiler">;
 
 export interface CompilerOutput {
   path: string;
@@ -109,16 +111,7 @@ async function compile(
   const compiler = new ClosureCompiler(opts as any);
   if (extendedOpts.batch) {
     compiler.JAR_PATH = null;
-    try {
-      const { getNativeImagePath } = await import(
-        "google-closure-compiler/lib/utils.js"
-      );
-      compiler.javaPath = getNativeImagePath();
-    } catch {
-      throw new Error(
-        "Installed google-closure-compiler is too old for batch mode."
-      );
-    }
+    compiler.javaPath = getNativeImagePath(extendedOpts);
   }
   return new Promise((resolve, reject) => {
     compiler.run((exitCode: number, stdout: string, stderr?: string) => {
@@ -135,23 +128,37 @@ function isInAwsLambda(): boolean {
 }
 
 function rewriteNodePathForAwsLambda(options: CompilerOptions): void {
+  assertRunInWebpack();
   if (options.js) {
-    // google-closure-library is installed as a Lambda Layer.
+    // google-closure-library maybe installed as a Lambda Layer.
     // It's extracted in /opt/nodejs/node_modules/google-closure-library.
     // But working directory is /var/task in Lambda.
     // The relative path is different from local environment.
     // So convert options.js paths.
     const closureLibraryDir = dirname(
-      // use `eval` to avoid webpack replacement
-      // eslint-disable-next-line no-eval
-      eval("require.resolve")("google-closure-library/package.json")
+      __non_webpack_require__.resolve("google-closure-library/package.json")
     );
     options.js = options.js.map((js) =>
       js.replace(/^node_modules\/google-closure-library/, closureLibraryDir)
     );
   }
 }
 
+function getNativeImagePath({
+  batch,
+  batchAwsCustomCompiler,
+}: ExtendedCompilerOptions): string {
+  assertRunInWebpack();
+  if (batch === "aws" && batchAwsCustomCompiler) {
+    return __non_webpack_require__(batchAwsCustomCompiler.name);
+  } else if (process.platform === "darwin") {
+    return __non_webpack_require__(`google-closure-compiler-osx`);
+  } else if (process.platform === "win32") {
+    return __non_webpack_require__(`google-closure-compiler-windows`);
+  }
+  return __non_webpack_require__(`google-closure-compiler-linux`);
+}
+
 export class CompilerError extends Error {
   exitCode: number;
   constructor(msg: string, exitCode: number) {
@@ -161,6 +168,12 @@ export class CompilerError extends Error {
   }
 }
 
+function assertRunInWebpack(): void {
+  if (typeof __non_webpack_require__ === undefined) {
+    throw new TypeError("This function must be run in webpack context.");
+  }
+}
+
 /**
  * To avoid "spawn E2BIG" errors on a large scale project,
  * transfer compiler options via a flagfile instead of CLI arguments.

src/compiler.ts

@@ -201,17 +201,12 @@ export function createCompilerOptionsForPage(
   if (wrapper && wrapper !== wrapperMarker) {
     compilerOptions.output_wrapper = wrapper;
   }
-  const extendedOpts: ExtendedCompilerOptions = { compilerOptions };
-  if (entryConfig.warningsWhitelist) {
-    extendedOpts.warningsWhitelist = createWarningsWhitelist(
-      entryConfig.warningsWhitelist,
-      duckConfig
-    );
-  }
-  if (duckConfig.batch) {
-    extendedOpts.batch = duckConfig.batch;
-  }
-  return extendedOpts;
+  const options: ExtendedCompilerOptions = createExtendedCompilerOptions(
+    compilerOptions,
+    duckConfig,
+    entryConfig
+  );
+  return options;
 }
 
 export async function createCompilerOptionsForChunks(
@@ -269,6 +264,21 @@ export async function createCompilerOptionsForChunks(
     convertCompilerOptionsToRelative(compilerOptions);
   }
 
+  const options: ExtendedCompilerOptions = createExtendedCompilerOptions(
+    compilerOptions,
+    duckConfig,
+    entryConfig
+  );
+  return { options, sortedChunkIds, rootChunkId: sortedChunkIds[0] };
+}
+
+const wrapperMarker = "%output%";
+
+function createExtendedCompilerOptions(
+  compilerOptions: CompilerOptions,
+  duckConfig: DuckConfig,
+  entryConfig: EntryConfig
+) {
   const options: ExtendedCompilerOptions = {
     compilerOptions,
   };
@@ -281,11 +291,12 @@ export async function createCompilerOptionsForChunks(
   if (duckConfig.batch) {
     options.batch = duckConfig.batch;
   }
-  return { options, sortedChunkIds, rootChunkId: sortedChunkIds[0] };
+  if (duckConfig.batchAwsCustomCompiler) {
+    options.batchAwsCustomCompiler = duckConfig.batchAwsCustomCompiler;
+  }
+  return options;
 }
 
-const wrapperMarker = "%output%";
-
 function createOutputWrapper(
   entryConfig: EntryConfig,
   level: CompilationLevel

src/duckconfig.ts

@@ -20,6 +20,10 @@ export interface DuckConfig {
   soyFileRoots?: readonly string[];
   concurrency?: number;
   batch?: "aws" | "local";
+  batchAwsCustomCompiler?: {
+    name: string;
+    version: string;
+  };
   batchOptions?: import("faastjs").AwsOptions | import("faastjs").LocalOptions;
   reporters?: Array<"json" | "text" | "xunit">;
   reporterOptions?: {
@@ -92,6 +96,18 @@ export function loadConfig(opts: any = {}): DuckConfig {
     }
   }
 
+  if (result.batchAwsCustomCompiler) {
+    if (!result.batchAwsCustomCompiler.name) {
+      throw new TypeError(
+        "batchAwsCustomCompiler.name is required in duck.config"
+      );
+    }
+    if (!result.batchAwsCustomCompiler.version) {
+      throw new TypeError(
+        "batchAwsCustomCompiler.version is required in duck.config"
+      );
+    }
+  }
   return result;
 }