Reduce memory usage by releasing webpack stats objects after compile

lib/compile.js

@@ -4,6 +4,7 @@ const _ = require('lodash');
 const BbPromise = require('bluebird');
 const webpack = require('webpack');
 const tty = require('tty');
+const isBuiltinModule = require('is-builtin-module');
 
 const defaultStatsConfig = {
   colors: tty.isatty(process.stdout.fd),
@@ -26,6 +27,64 @@ function getStatsLogger(statsConfig, consoleLog) {
   };
 }
 
+function getExternalModuleName(module) {
+  const path = /^external "(.*)"$/.exec(module.identifier())[1];
+  const pathComponents = path.split('/');
+  const main = pathComponents[0];
+
+  // this is a package within a namespace
+  if (main.charAt(0) == '@') {
+    return `${main}/${pathComponents[1]}`;
+  }
+
+  return main;
+}
+
+function isExternalModule(module) {
+  return _.startsWith(module.identifier(), 'external ') && !isBuiltinModule(getExternalModuleName(module));
+}
+
+/**
+ * Gets the module issuer. The ModuleGraph api does not exists in webpack@4
+ * so falls back to using module.issuer.
+ */
+function getIssuerCompat(moduleGraph, module) {
+  if (moduleGraph) {
+    return moduleGraph.getIssuer(module);
+  }
+
+  return module.issuer;
+}
+
+/**
+ * Find the original module that required the transient dependency. Returns
+ * undefined if the module is a first level dependency.
+ * @param {Object} moduleGraph - Webpack module graph
+ * @param {Object} issuer - Module issuer
+ */
+function findExternalOrigin(moduleGraph, issuer) {
+  if (!_.isNil(issuer) && _.startsWith(issuer.rawRequest, './')) {
+    return findExternalOrigin(moduleGraph, getIssuerCompat(moduleGraph, issuer));
+  }
+  return issuer;
+}
+
+function getExternalModules({ compilation }) {
+  const externals = new Set();
+  for (const module of compilation.modules) {
+    if (isExternalModule(module)) {
+      externals.add({
+        origin: _.get(
+          findExternalOrigin(compilation.moduleGraph, getIssuerCompat(compilation.moduleGraph, module)),
+          'rawRequest'
+        ),
+        external: getExternalModuleName(module)
+      });
+    }
+  }
+  return Array.from(externals);
+}
+
 function webpackCompile(config, logStats) {
   return BbPromise.fromCallback(cb => webpack(config).run(cb)).then(stats => {
     // ensure stats in any array in the case of concurrent build.
@@ -38,7 +97,10 @@ function webpackCompile(config, logStats) {
       }
     });
 
-    return stats;
+    return _.map(stats, compileStats => ({
+      outputPath: compileStats.compilation.compiler.outputPath,
+      externalModules: getExternalModules(compileStats)
+    }));
   });
 }
 

lib/packExternalModules.js

@@ -4,7 +4,6 @@ const BbPromise = require('bluebird');
 const _ = require('lodash');
 const path = require('path');
 const fse = require('fs-extra');
-const isBuiltinModule = require('is-builtin-module');
 
 const Packagers = require('./packagers');
 
@@ -178,64 +177,6 @@ function getProdModules(externalModules, packagePath, nodeModulesRelativeDir, de
   return prodModules;
 }
 
-function getExternalModuleName(module) {
-  const path = /^external "(.*)"$/.exec(module.identifier())[1];
-  const pathComponents = path.split('/');
-  const main = pathComponents[0];
-
-  // this is a package within a namespace
-  if (main.charAt(0) == '@') {
-    return `${main}/${pathComponents[1]}`;
-  }
-
-  return main;
-}
-
-function isExternalModule(module) {
-  return _.startsWith(module.identifier(), 'external ') && !isBuiltinModule(getExternalModuleName(module));
-}
-
-/**
- * Gets the module issuer. The ModuleGraph api does not exists in webpack@4
- * so falls back to using module.issuer.
- */
-function getIssuerCompat(moduleGraph, module) {
-  if (moduleGraph) {
-    return moduleGraph.getIssuer(module);
-  }
-
-  return module.issuer;
-}
-
-/**
- * Find the original module that required the transient dependency. Returns
- * undefined if the module is a first level dependency.
- * @param {Object} moduleGraph - Webpack module graph
- * @param {Object} issuer - Module issuer
- */
-function findExternalOrigin(moduleGraph, issuer) {
-  if (!_.isNil(issuer) && _.startsWith(issuer.rawRequest, './')) {
-    return findExternalOrigin(moduleGraph, getIssuerCompat(moduleGraph, issuer));
-  }
-  return issuer;
-}
-
-function getExternalModules({ compilation }) {
-  const externals = new Set();
-  for (const module of compilation.modules) {
-    if (isExternalModule(module)) {
-      externals.add({
-        origin: _.get(
-          findExternalOrigin(compilation.moduleGraph, getIssuerCompat(compilation.moduleGraph, module)),
-          'rawRequest'
-        ),
-        external: getExternalModuleName(module)
-      });
-    }
-  }
-  return Array.from(externals);
-}
-
 module.exports = {
   /**
    * We need a performant algorithm to install the packages for each single
@@ -305,7 +246,7 @@ module.exports = {
         const compositeModules = _.uniq(
           _.flatMap(stats.stats, compileStats => {
             const externalModules = _.concat(
-              getExternalModules.call(this, compileStats),
+              compileStats.externalModules,
               _.map(packageForceIncludes, whitelistedPackage => ({
                 external: whitelistedPackage
               }))
@@ -381,7 +322,7 @@ module.exports = {
               .return(stats.stats);
           })
           .mapSeries(compileStats => {
-            const modulePath = compileStats.compilation.compiler.outputPath;
+            const modulePath = compileStats.outputPath;
 
             // Create package.json
             const modulePackageJson = path.join(modulePath, 'package.json');
@@ -399,7 +340,7 @@ module.exports = {
             const prodModules = getProdModules.call(
               this,
               _.concat(
-                getExternalModules.call(this, compileStats),
+                compileStats.externalModules,
                 _.map(packageForceIncludes, whitelistedPackage => ({
                   external: whitelistedPackage
                 }))

lib/packageModules.js

@@ -131,7 +131,7 @@ module.exports = {
     return BbPromise.mapSeries(stats.stats, (compileStats, index) => {
       const entryFunction = _.get(this.entryFunctions, index, {});
       const filename = getArtifactName.call(this, entryFunction);
-      const modulePath = compileStats.compilation.compiler.outputPath;
+      const modulePath = compileStats.outputPath;
 
       const startZip = _.now();
       return zip

tests/compile.test.js

@@ -97,7 +97,8 @@ describe('compile', () => {
             errors: [],
             compiler: {
               outputPath: 'statsMock-outputPath'
-            }
+            },
+            modules: []
           },
           toString: sandbox.stub().returns('testStats'),
           hasErrors: _.constant(false)
@@ -124,7 +125,8 @@ describe('compile', () => {
             errors: [],
             compiler: {
               outputPath: 'statsMock-outputPath'
-            }
+            },
+            modules: []
           },
           toString: sandbox.stub().returns('testStats'),
           hasErrors: _.constant(false)
@@ -152,7 +154,8 @@ describe('compile', () => {
         errors: [],
         compiler: {
           outputPath: 'statsMock-outputPath'
-        }
+        },
+        modules: []
       },
       toString: sandbox.stub().returns('testStats'),
       hasErrors: _.constant(false)
@@ -175,4 +178,93 @@ describe('compile', () => {
         return null;
       });
   });
+
+  it('should set stats outputPath', () => {
+    const testWebpackConfig = 'testconfig';
+    const multiStats = {
+      stats: [
+        {
+          compilation: {
+            errors: [],
+            compiler: {
+              outputPath: 'compileStats-outputPath'
+            },
+            modules: []
+          },
+          toString: sandbox.stub().returns('testStats'),
+          hasErrors: _.constant(false)
+        }
+      ]
+    };
+    module.webpackConfig = testWebpackConfig;
+    module.configuration = { concurrency: 1 };
+    webpackMock.compilerMock.run.reset();
+    webpackMock.compilerMock.run.yields(null, multiStats);
+    return expect(module.compile()).to.be.fulfilled.then(() => {
+      expect(module.compileStats.stats[0].outputPath).to.equal('compileStats-outputPath');
+      return null;
+    });
+  });
+
+  it('should set stats externals', () => {
+    const testWebpackConfig = 'testconfig';
+    const multiStats = {
+      stats: [
+        {
+          compilation: {
+            errors: [],
+            compiler: {
+              outputPath: 'compileStats-outputPath'
+            },
+            modules: [
+              {
+                identifier: _.constant('"crypto"')
+              },
+              {
+                identifier: _.constant('"uuid/v4"')
+              },
+              {
+                identifier: _.constant('"mockery"')
+              },
+              {
+                identifier: _.constant('"@scoped/vendor/module1"')
+              },
+              {
+                identifier: _.constant('external "@scoped/vendor/module2"')
+              },
+              {
+                identifier: _.constant('external "uuid/v4"')
+              },
+              {
+                identifier: _.constant('external "localmodule"')
+              },
+              {
+                identifier: _.constant('external "bluebird"')
+              },
+              {
+                identifier: _.constant('external "aws-sdk"')
+              }
+            ]
+          },
+          toString: sandbox.stub().returns('testStats'),
+          hasErrors: _.constant(false)
+        }
+      ]
+    };
+    module.webpackConfig = testWebpackConfig;
+    module.configuration = { concurrency: 1 };
+    webpackMock.compilerMock.run.reset();
+    webpackMock.compilerMock.run.yields(null, multiStats);
+    return expect(module.compile()).to.be.fulfilled.then(() => {
+      console.log(JSON.stringify(module.compileStats.stats[0].externalModules));
+      expect(module.compileStats.stats[0].externalModules).to.eql([
+        { external: '@scoped/vendor', origin: undefined },
+        { external: 'uuid', origin: undefined },
+        { external: 'localmodule', origin: undefined },
+        { external: 'bluebird', origin: undefined },
+        { external: 'aws-sdk', origin: undefined }
+      ]);
+      return null;
+    });
+  });
 });