Use jest worker to parallelize compilation

lib/compile.js

@@ -1,130 +1,57 @@
 'use strict';
 
 const _ = require('lodash');
-const BbPromise = require('bluebird');
-const webpack = require('webpack');
-const tty = require('tty');
-const isBuiltinModule = require('is-builtin-module');
-
-const defaultStatsConfig = {
-  colors: tty.isatty(process.stdout.fd),
-  hash: false,
-  version: false,
-  chunks: false,
-  children: false
-};
+const { Worker } = require('jest-worker');
+const lib = require('./index');
 
 function ensureArray(obj) {
   return _.isArray(obj) ? obj : [obj];
 }
 
-function getStatsLogger(statsConfig, consoleLog) {
-  return stats => {
-    const statsOutput = stats.toString(statsConfig || defaultStatsConfig);
-    if (statsOutput) {
-      consoleLog(statsOutput);
-    }
-  };
-}
-
-function getExternalModuleName(module) {
-  const path = /^external "(.*)"$/.exec(module.identifier())[1];
-  const pathComponents = path.split('/');
-  const main = pathComponents[0];
-
-  // this is a package within a namespace
-  if (main.charAt(0) == '@') {
-    return `${main}/${pathComponents[1]}`;
-  }
-
-  return main;
-}
-
-function isExternalModule(module) {
-  return _.startsWith(module.identifier(), 'external ') && !isBuiltinModule(getExternalModuleName(module));
-}
-
-/**
- * Gets the module issuer. The ModuleGraph api does not exists in webpack@4
- * so falls back to using module.issuer.
- */
-function getIssuerCompat(moduleGraph, module) {
-  if (moduleGraph) {
-    return moduleGraph.getIssuer(module);
-  }
-
-  return module.issuer;
-}
-
-/**
- * Find the original module that required the transient dependency. Returns
- * undefined if the module is a first level dependency.
- * @param {Object} moduleGraph - Webpack module graph
- * @param {Object} issuer - Module issuer
- */
-function findExternalOrigin(moduleGraph, issuer) {
-  if (!_.isNil(issuer) && _.startsWith(issuer.rawRequest, './')) {
-    return findExternalOrigin(moduleGraph, getIssuerCompat(moduleGraph, issuer));
-  }
-  return issuer;
-}
-
-function getExternalModules({ compilation }) {
-  const externals = new Set();
-  for (const module of compilation.modules) {
-    if (isExternalModule(module)) {
-      externals.add({
-        origin: _.get(
-          findExternalOrigin(compilation.moduleGraph, getIssuerCompat(compilation.moduleGraph, module)),
-          'rawRequest'
-        ),
-        external: getExternalModuleName(module)
-      });
-    }
-  }
-  return Array.from(externals);
-}
-
-function webpackCompile(config, logStats) {
-  return BbPromise.fromCallback(cb => webpack(config).run(cb)).then(stats => {
-    // ensure stats in any array in the case of concurrent build.
-    stats = stats.stats ? stats.stats : [stats];
-
-    _.forEach(stats, compileStats => {
-      logStats(compileStats);
-      if (compileStats.hasErrors()) {
-        throw new Error('Webpack compilation error, see stats above');
-      }
-    });
-
-    return _.map(stats, compileStats => ({
-      outputPath: compileStats.compilation.compiler.outputPath,
-      externalModules: getExternalModules(compileStats)
-    }));
+async function webpackConcurrentCompile(webpackConfigFilePath, configs, concurrency) {
+  const worker = new Worker(require.resolve('./compileWorker'), {
+    numWorkers: concurrency,
+    enableWorkerThreads: true,
+    maxRetries: 0,
+    exposedMethods: ['compile']
   });
-}
-
-function webpackConcurrentCompile(configs, logStats, concurrency) {
-  return BbPromise.map(configs, config => webpackCompile(config, logStats), { concurrency }).then(stats =>
-    _.flatten(stats)
-  );
+  worker.getStdout().pipe(process.stdout);
+
+  try {
+    const stats = await Promise.all(
+      _.map(configs, config => {
+        return worker.compile({
+          webpackConfigFilePath,
+          configOverrides: {
+            entry: config.entry,
+            output: config.output,
+            context: config.context,
+            node: config.node,
+            target: config.target
+          },
+          entries: lib.entries,
+          options: lib.options
+        });
+      })
+    );
+    return _.flatten(stats);
+  } finally {
+    await worker.end();
+  }
 }
 
 module.exports = {
-  compile() {
+  async compile() {
     this.serverless.cli.log('Bundling with Webpack...');
 
     const configs = ensureArray(this.webpackConfig);
-    const logStats = getStatsLogger(configs[0].stats, this.serverless.cli.consoleLog);
 
     if (!this.configuration) {
-      return BbPromise.reject('Missing plugin configuration');
+      throw new Error('Missing plugin configuration');
     }
     const concurrency = this.configuration.concurrency;
 
-    return webpackConcurrentCompile(configs, logStats, concurrency).then(stats => {
-      this.compileStats = { stats };
-      return BbPromise.resolve();
-    });
+    const stats = await webpackConcurrentCompile(this.webpackConfigFilePath, configs, concurrency);
+    this.compileStats = { stats };
   }
 };

lib/compileWorker.js

@@ -0,0 +1,133 @@
+'use strict';
+
+const _ = require('lodash');
+const webpack = require('webpack');
+const BbPromise = require('bluebird');
+const path = require('path');
+const rechoir = require('rechoir');
+const interpret = require('interpret');
+const tty = require('tty');
+const isBuiltinModule = require('is-builtin-module');
+const log = require('@serverless/utils/log');
+const lib = require('./index');
+
+const defaultStatsConfig = {
+  colors: tty.isatty(process.stdout.fd),
+  hash: false,
+  version: false,
+  chunks: false,
+  children: false
+};
+
+function getStatsLogger(statsConfig) {
+  return stats => {
+    const statsOutput = stats.toString(statsConfig || defaultStatsConfig);
+    if (statsOutput) {
+      console.log(statsOutput);
+    }
+  };
+}
+
+function getExternalModuleName(module) {
+  const path = /^external "(.*)"$/.exec(module.identifier())[1];
+  const pathComponents = path.split('/');
+  const main = pathComponents[0];
+
+  // this is a package within a namespace
+  if (main.charAt(0) == '@') {
+    return `${main}/${pathComponents[1]}`;
+  }
+
+  return main;
+}
+
+function isExternalModule(module) {
+  return _.startsWith(module.identifier(), 'external ') && !isBuiltinModule(getExternalModuleName(module));
+}
+
+/**
+ * Gets the module issuer. The ModuleGraph api does not exists in webpack@4
+ * so falls back to using module.issuer.
+ */
+function getIssuerCompat(moduleGraph, module) {
+  if (moduleGraph) {
+    return moduleGraph.getIssuer(module);
+  }
+
+  return module.issuer;
+}
+
+/**
+ * Find the original module that required the transient dependency. Returns
+ * undefined if the module is a first level dependency.
+ * @param {Object} moduleGraph - Webpack module graph
+ * @param {Object} issuer - Module issuer
+ */
+function findExternalOrigin(moduleGraph, issuer) {
+  if (!_.isNil(issuer) && _.startsWith(issuer.rawRequest, './')) {
+    return findExternalOrigin(moduleGraph, getIssuerCompat(moduleGraph, issuer));
+  }
+  return issuer;
+}
+
+function getExternalModules({ compilation }) {
+  const externals = new Set();
+  for (const module of compilation.modules) {
+    if (isExternalModule(module)) {
+      externals.add({
+        origin: _.get(
+          findExternalOrigin(compilation.moduleGraph, getIssuerCompat(compilation.moduleGraph, module)),
+          'rawRequest'
+        ),
+        external: getExternalModuleName(module)
+      });
+    }
+  }
+  return Array.from(externals);
+}
+
+async function webpackCompile({ webpackConfigFilePath, configOverrides, options, entries }) {
+  const functionName = configOverrides.output.path.split(path.sep).pop();
+  log(`Start compiling ${functionName}`);
+
+  // Setup globals for webpack config.
+  lib.options = options;
+  lib.entries = entries;
+
+  rechoir.prepare(
+    {
+      ...interpret.extensions,
+      ..._.mapKeys(interpret.extensions, (value, key) => `.config${key}`)
+    },
+    webpackConfigFilePath
+  );
+
+  let webpackConfig = require(webpackConfigFilePath);
+  if (webpackConfig.default) {
+    webpackConfig = webpackConfig.default;
+  }
+  if (_.isFunction(webpackConfig.then)) {
+    webpackConfig = await webpackConfig;
+  }
+  const config = { ...webpackConfig, ...configOverrides };
+  const logStats = getStatsLogger(config.stats);
+  let stats = await BbPromise.fromCallback(cb => webpack(config).run(cb));
+  // ensure stats in any array in the case of concurrent build.
+  stats = stats.stats ? stats.stats : [stats];
+
+  _.forEach(stats, compileStats => {
+    logStats(compileStats);
+    if (compileStats.hasErrors()) {
+      throw new Error('Webpack compilation error, see stats above');
+    }
+  });
+
+  log(`Finished compiling ${functionName}`);
+
+  return _.map(stats, compileStats => ({
+    outputPath: compileStats.compilation.compiler.outputPath,
+    externalModules: getExternalModules(compileStats)
+  }));
+}
+
+module.exports = { compile: webpackCompile };

lib/validate.js

@@ -154,6 +154,7 @@ module.exports = {
         this.serverless.cli.log(`Could not load webpack config '${webpackConfigFilePath}'`);
         return BbPromise.reject(err);
       }
+      this.webpackConfigFilePath = webpackConfigFilePath;
     }
 
     // Intermediate function to handle async webpack config