New: Use woker-farm to handle webpack multi compile

lib/compile.js

@@ -2,41 +2,58 @@
 
 const _ = require('lodash');
 const BbPromise = require('bluebird');
-const webpack = require('webpack');
 const tty = require('tty');
 
+const { compiler } = require('./compiler');
+const { multiCompiler } = require('./multiCompiler/compiler');
+
 module.exports = {
   compile() {
     this.serverless.cli.log('Bundling with Webpack...');
+    const consoleStats = this.webpackConfig.stats ||
+      _.get(this, 'webpackConfig[0].stats') || {
+      colors: tty.isatty(process.stdout.fd),
+      hash: false,
+      version: false,
+      chunks: false,
+      children: false
+    };
+
+    const configOptions = {
+      servicePath: this.serverless.config.servicePath,
+      out: this.options.out
+    };
+
+    const compileOptions = {
+      webpackConfigFilePath: this.webpackConfigFilePath,
+      webpackConfig: this.webpackConfig,
+
+      entryFunctions: this.entryFunctions,
+
+      configOptions,
+      consoleStats
+    };
 
-    const compiler = webpack(this.webpackConfig);
+    if (this.multiCompile) {
+      this.options.verbose && this.serverless.cli.log('Using multi-thread function compiler');
+    }
 
-    return BbPromise.fromCallback(cb => compiler.run(cb)).then(stats => {
-      if (!this.multiCompile) {
-        stats = { stats: [stats] };
-      }
+    const webpackCompiler = this.multiCompile ? multiCompiler(compileOptions) : compiler(compileOptions);
 
+    return webpackCompiler.then(stats => {
       const compileOutputPaths = [];
-      const consoleStats = this.webpackConfig.stats ||
-        _.get(this, 'webpackConfig[0].stats') || {
-        colors: tty.isatty(process.stdout.fd),
-        hash: false,
-        version: false,
-        chunks: false,
-        children: false
-      };
 
       _.forEach(stats.stats, compileStats => {
-        const statsOutput = compileStats.toString(consoleStats);
+        const statsOutput = compileStats.cliOutput;
         if (statsOutput) {
           this.serverless.cli.consoleLog(statsOutput);
         }
 
-        if (compileStats.compilation.errors.length) {
+        if (compileStats.errors.length) {
           throw new Error('Webpack compilation error, see above');
         }
 
-        compileOutputPaths.push(compileStats.compilation.compiler.outputPath);
+        compileOutputPaths.push(compileStats.outputPath);
       });
 
       this.compileOutputPaths = compileOutputPaths;

lib/compiler.js

@@ -0,0 +1,27 @@
+'use strict';
+
+const BbPromise = require('bluebird');
+const webpack = require('webpack');
+
+const { processWebpackStats } = require('./processWebpackStats');
+const { setOptionsOnConfig } = require('./processConfig');
+
+module.exports = {
+  compiler(options) {
+    const webpackConfig = options.webpackConfig;
+    const configOptions = options.configOptions;
+    const consoleStats = options.consoleStats;
+
+    const config = setOptionsOnConfig(webpackConfig, configOptions);
+
+    const compiler = webpack(config);
+
+    return BbPromise.fromCallback(cb => compiler.run(cb)).then(stats => {
+      const result = processWebpackStats(stats, consoleStats);
+
+      return {
+        stats: [result]
+      };
+    });
+  }
+};

lib/multiCompiler/compiler.js

@@ -0,0 +1,45 @@
+'use strict';
+
+const _ = require('lodash');
+const BbPromise = require('bluebird');
+const workerFarm = require('worker-farm');
+
+module.exports = {
+  multiCompiler(options) {
+    const workerOptions = {
+      maxCallsPerWorker: 1
+    };
+    const workerThreadSourcePath = require.resolve('./workerThread');
+    const methods = ['runWebpack'];
+
+    const workers = workerFarm(workerOptions, workerThreadSourcePath, methods);
+
+    const configOptions = options.configOptions;
+
+    const threads = _.map(options.entryFunctions, entryFunc => {
+      const webpackConfigFilePath = options.webpackConfigFilePath;
+      const workerOptions = {
+        webpackConfigFilePath,
+        configOptions: {
+          ...configOptions,
+          entryFunc
+        },
+        consoleStats: options.consoleStats
+      };
+
+      if (!webpackConfigFilePath) {
+        workerOptions.webpackConfig = options.webpackConfig;
+      }
+
+      return BbPromise.fromCallback(cb => {
+        workers.runWebpack(workerOptions, cb);
+      });
+    });
+
+    return Promise.all(threads).then(stats => {
+      return {
+        stats
+      };
+    });
+  }
+};

lib/multiCompiler/workerThread.js

@@ -0,0 +1,43 @@
+'use strict';
+
+const webpack = require('webpack');
+const { setOptionsOnConfig } = require('../processConfig');
+const { processWebpackStats } = require('../processWebpackStats');
+
+module.exports = {
+  runWebpack(options, callback) {
+    const webpackConfigFilePath = options.webpackConfigFilePath;
+    const configOptions = options.configOptions;
+    const consoleStats = options.consoleStats;
+
+    let webpackConfig = null;
+    try {
+      if (webpackConfigFilePath) {
+        webpackConfig = require(webpackConfigFilePath);
+      } else if (options.webpackConfig) {
+        webpackConfig = options.webpackConfig;
+      } else {
+        throw new Error('Missing config');
+      }
+    } catch (error) {
+      callback(new Error('Failed to load config'));
+      return;
+    }
+
+    const entryConfig = setOptionsOnConfig(webpackConfig, configOptions);
+
+    const compiler = webpack(entryConfig);
+
+    compiler.run((error, stats) => {
+      if (error) {
+        callback(new Error('Failed to compile'));
+
+        return;
+      }
+
+      const result = processWebpackStats(stats, consoleStats);
+
+      callback(null, result);
+    });
+  }
+};

lib/packExternalModules.js

@@ -4,7 +4,6 @@ const BbPromise = require('bluebird');
 const _ = require('lodash');
 const path = require('path');
 const fse = require('fs-extra');
-const isBuiltinModule = require('is-builtin-module');
 
 const Packagers = require('./packagers');
 
@@ -143,58 +142,6 @@ function getProdModules(externalModules, packagePath, dependencyGraph, forceExcl
   return prodModules;
 }
 
-function getExternalModuleName(module) {
-  const path = /^external "(.*)"$/.exec(module.identifier())[1];
-  const pathComponents = path.split('/');
-  const main = pathComponents[0];
-
-  // this is a package within a namespace
-  if (main.charAt(0) == '@') {
-    return `${main}/${pathComponents[1]}`;
-  }
-
-  return main;
-}
-
-function isExternalModule(module) {
-  return _.startsWith(module.identifier(), 'external ') && !isBuiltinModule(getExternalModuleName(module));
-}
-
-/**
- * Find the original module that required the transient dependency. Returns
- * undefined if the module is a first level dependency.
- * @param {Object} issuer - Module issuer
- */
-function findExternalOrigin(issuer) {
-  if (!_.isNil(issuer) && _.startsWith(issuer.rawRequest, './')) {
-    return findExternalOrigin(issuer.issuer);
-  }
-  return issuer;
-}
-
-function getExternalModules(stats) {
-  if (!stats.compilation.chunks) {
-    return [];
-  }
-  const externals = new Set();
-  for (const chunk of stats.compilation.chunks) {
-    if (!chunk.modulesIterable) {
-      continue;
-    }
-
-    // Explore each module within the chunk (built inputs):
-    for (const module of chunk.modulesIterable) {
-      if (isExternalModule(module)) {
-        externals.add({
-          origin: _.get(findExternalOrigin(module.issuer), 'rawRequest'),
-          external: getExternalModuleName(module)
-        });
-      }
-    }
-  }
-  return Array.from(externals);
-}
-
 module.exports = {
   /**
    * We need a performant algorithm to install the packages for each single
@@ -259,7 +206,7 @@ module.exports = {
         const compositeModules = _.uniq(
           _.flatMap(stats.stats, compileStats => {
             const externalModules = _.concat(
-              getExternalModules.call(this, compileStats),
+              compileStats.externalModules,
               _.map(packageForceIncludes, whitelistedPackage => ({
                 external: whitelistedPackage
               }))
@@ -328,7 +275,7 @@ module.exports = {
               .return(stats.stats);
           })
           .mapSeries(compileStats => {
-            const modulePath = compileStats.compilation.compiler.outputPath;
+            const modulePath = compileStats.outputPath;
 
             // Create package.json
             const modulePackageJson = path.join(modulePath, 'package.json');
@@ -346,7 +293,7 @@ module.exports = {
             const prodModules = getProdModules.call(
               this,
               _.concat(
-                getExternalModules.call(this, compileStats),
+                compileStats.externalModules,
                 _.map(packageForceIncludes, whitelistedPackage => ({
                   external: whitelistedPackage
                 }))

lib/packageModules.js

@@ -76,7 +76,7 @@ module.exports = {
     return BbPromise.mapSeries(stats.stats, (compileStats, index) => {
       const entryFunction = _.get(this.entryFunctions, index, {});
       const filename = `${entryFunction.funcName || this.serverless.service.getServiceObject().name}.zip`;
-      const modulePath = compileStats.compilation.compiler.outputPath;
+      const modulePath = compileStats.outputPath;
 
       const startZip = _.now();
       return zip

lib/processConfig.js

@@ -0,0 +1,47 @@
+'use strict';
+
+const _ = require('lodash');
+const path = require('path');
+
+module.exports = {
+  setOptionsOnConfig(webpackConfig, options) {
+    // Default context
+    if (!webpackConfig.context) {
+      webpackConfig.context = options.servicePath;
+    }
+
+    // Default target
+    if (!webpackConfig.target) {
+      webpackConfig.target = 'node';
+    }
+
+    // Default output
+    if (!webpackConfig.output || _.isEmpty(webpackConfig.output)) {
+      const outputPath = path.join(options.servicePath, '.webpack');
+      webpackConfig.output = {
+        libraryTarget: 'commonjs',
+        path: outputPath,
+        filename: '[name].js'
+      };
+    }
+
+    // Custom output path
+    if (options.out) {
+      webpackConfig.output.path = path.join(options.servicePath, options.out);
+    }
+
+    // In case of individual packaging we have to create a separate config for each function
+    if (options.entryFunc) {
+      const entryFunc = options.entryFunc;
+      webpackConfig.entry = {
+        [entryFunc.entry.key]: entryFunc.entry.value
+      };
+      const compileName = entryFunc.funcName || _.camelCase(entryFunc.entry.key);
+      webpackConfig.output.path = path.join(webpackConfig.output.path, compileName);
+    } else {
+      webpackConfig.output.path = path.join(webpackConfig.output.path, 'service');
+    }
+
+    return webpackConfig;
+  }
+};