Skip to content

Commit

Permalink
Merge pull request #858 from janicduplessis/perf
Browse files Browse the repository at this point in the history 
Reduce memory usage by releasing webpack stats objects after compile
  • Loading branch information
@j0k3r
j0k3r committed Jun 8, 2021
2 parents 9a0b400 + fa2370d commit 9b5c290
Show file tree
Hide file tree
Showing 7 changed files with 197 additions and 336 deletions.
64 changes: 63 additions & 1 deletion lib/compile.js
View file
Expand Up @@ -4,6 +4,7 @@ const _ = require('lodash');
const BbPromise = require('bluebird');
const webpack = require('webpack');
const tty = require('tty');
const isBuiltinModule = require('is-builtin-module');

const defaultStatsConfig = {
colors: tty.isatty(process.stdout.fd),
Expand All @@ -26,6 +27,64 @@ function getStatsLogger(statsConfig, consoleLog) {
};
}

function getExternalModuleName(module) {
const path = /^external "(.*)"$/.exec(module.identifier())[1];
const pathComponents = path.split('/');
const main = pathComponents[0];

// this is a package within a namespace
if (main.charAt(0) == '@') {
return `${main}/${pathComponents[1]}`;
}

return main;
}

function isExternalModule(module) {
return _.startsWith(module.identifier(), 'external ') && !isBuiltinModule(getExternalModuleName(module));
}

/**
* Gets the module issuer. The ModuleGraph api does not exists in webpack@4
* so falls back to using module.issuer.
*/
function getIssuerCompat(moduleGraph, module) {
if (moduleGraph) {
return moduleGraph.getIssuer(module);
}

return module.issuer;
}

/**
* Find the original module that required the transient dependency. Returns
* undefined if the module is a first level dependency.
* @param {Object} moduleGraph - Webpack module graph
* @param {Object} issuer - Module issuer
*/
function findExternalOrigin(moduleGraph, issuer) {
if (!_.isNil(issuer) && _.startsWith(issuer.rawRequest, './')) {
return findExternalOrigin(moduleGraph, getIssuerCompat(moduleGraph, issuer));
}
return issuer;
}

function getExternalModules({ compilation }) {
const externals = new Set();
for (const module of compilation.modules) {
if (isExternalModule(module)) {
externals.add({
origin: _.get(
findExternalOrigin(compilation.moduleGraph, getIssuerCompat(compilation.moduleGraph, module)),
'rawRequest'
),
external: getExternalModuleName(module)
});
}
}
return Array.from(externals);
}

function webpackCompile(config, logStats) {
return BbPromise.fromCallback(cb => webpack(config).run(cb)).then(stats => {
// ensure stats in any array in the case of concurrent build.
Expand All @@ -38,7 +97,10 @@ function webpackCompile(config, logStats) {
}
});

return stats;
return _.map(stats, compileStats => ({
outputPath: compileStats.compilation.compiler.outputPath,
externalModules: getExternalModules(compileStats)
}));
});
}

Expand Down
65 changes: 3 additions & 62 deletions lib/packExternalModules.js
View file
Expand Up @@ -4,7 +4,6 @@ const BbPromise = require('bluebird');
const _ = require('lodash');
const path = require('path');
const fse = require('fs-extra');
const isBuiltinModule = require('is-builtin-module');

const Packagers = require('./packagers');

Expand Down Expand Up @@ -178,64 +177,6 @@ function getProdModules(externalModules, packagePath, nodeModulesRelativeDir, de
return prodModules;
}

function getExternalModuleName(module) {
const path = /^external "(.*)"$/.exec(module.identifier())[1];
const pathComponents = path.split('/');
const main = pathComponents[0];

// this is a package within a namespace
if (main.charAt(0) == '@') {
return `${main}/${pathComponents[1]}`;
}

return main;
}

function isExternalModule(module) {
return _.startsWith(module.identifier(), 'external ') && !isBuiltinModule(getExternalModuleName(module));
}

/**
* Gets the module issuer. The ModuleGraph api does not exists in webpack@4
* so falls back to using module.issuer.
*/
function getIssuerCompat(moduleGraph, module) {
if (moduleGraph) {
return moduleGraph.getIssuer(module);
}

return module.issuer;
}

/**
* Find the original module that required the transient dependency. Returns
* undefined if the module is a first level dependency.
* @param {Object} moduleGraph - Webpack module graph
* @param {Object} issuer - Module issuer
*/
function findExternalOrigin(moduleGraph, issuer) {
if (!_.isNil(issuer) && _.startsWith(issuer.rawRequest, './')) {
return findExternalOrigin(moduleGraph, getIssuerCompat(moduleGraph, issuer));
}
return issuer;
}

function getExternalModules({ compilation }) {
const externals = new Set();
for (const module of compilation.modules) {
if (isExternalModule(module)) {
externals.add({
origin: _.get(
findExternalOrigin(compilation.moduleGraph, getIssuerCompat(compilation.moduleGraph, module)),
'rawRequest'
),
external: getExternalModuleName(module)
});
}
}
return Array.from(externals);
}

module.exports = {
/**
* We need a performant algorithm to install the packages for each single
Expand Down Expand Up @@ -305,7 +246,7 @@ module.exports = {
const compositeModules = _.uniq(
_.flatMap(stats.stats, compileStats => {
const externalModules = _.concat(
getExternalModules.call(this, compileStats),
compileStats.externalModules,
_.map(packageForceIncludes, whitelistedPackage => ({
external: whitelistedPackage
}))
Expand Down Expand Up @@ -381,7 +322,7 @@ module.exports = {
.return(stats.stats);
})
.mapSeries(compileStats => {
const modulePath = compileStats.compilation.compiler.outputPath;
const modulePath = compileStats.outputPath;

// Create package.json
const modulePackageJson = path.join(modulePath, 'package.json');
Expand All @@ -399,7 +340,7 @@ module.exports = {
const prodModules = getProdModules.call(
this,
_.concat(
getExternalModules.call(this, compileStats),
compileStats.externalModules,
_.map(packageForceIncludes, whitelistedPackage => ({
external: whitelistedPackage
}))
Expand Down
2 changes: 1 addition & 1 deletion lib/packageModules.js
View file
Expand Up @@ -131,7 +131,7 @@ module.exports = {
return BbPromise.mapSeries(stats.stats, (compileStats, index) => {
const entryFunction = _.get(this.entryFunctions, index, {});
const filename = getArtifactName.call(this, entryFunction);
const modulePath = compileStats.compilation.compiler.outputPath;
const modulePath = compileStats.outputPath;

const startZip = _.now();
return zip
Expand Down
98 changes: 95 additions & 3 deletions tests/compile.test.js
View file
Expand Up @@ -97,7 +97,8 @@ describe('compile', () => {
errors: [],
compiler: {
outputPath: 'statsMock-outputPath'
}
},
modules: []
},
toString: sandbox.stub().returns('testStats'),
hasErrors: _.constant(false)
Expand All @@ -124,7 +125,8 @@ describe('compile', () => {
errors: [],
compiler: {
outputPath: 'statsMock-outputPath'
}
},
modules: []
},
toString: sandbox.stub().returns('testStats'),
hasErrors: _.constant(false)
Expand Down Expand Up @@ -152,7 +154,8 @@ describe('compile', () => {
errors: [],
compiler: {
outputPath: 'statsMock-outputPath'
}
},
modules: []
},
toString: sandbox.stub().returns('testStats'),
hasErrors: _.constant(false)
Expand All @@ -175,4 +178,93 @@ describe('compile', () => {
return null;
});
});

it('should set stats outputPath', () => {
const testWebpackConfig = 'testconfig';
const multiStats = {
stats: [
{
compilation: {
errors: [],
compiler: {
outputPath: 'compileStats-outputPath'
},
modules: []
},
toString: sandbox.stub().returns('testStats'),
hasErrors: _.constant(false)
}
]
};
module.webpackConfig = testWebpackConfig;
module.configuration = { concurrency: 1 };
webpackMock.compilerMock.run.reset();
webpackMock.compilerMock.run.yields(null, multiStats);
return expect(module.compile()).to.be.fulfilled.then(() => {
expect(module.compileStats.stats[0].outputPath).to.equal('compileStats-outputPath');
return null;
});
});

it('should set stats externals', () => {
const testWebpackConfig = 'testconfig';
const multiStats = {
stats: [
{
compilation: {
errors: [],
compiler: {
outputPath: 'compileStats-outputPath'
},
modules: [
{
identifier: _.constant('"crypto"')
},
{
identifier: _.constant('"uuid/v4"')
},
{
identifier: _.constant('"mockery"')
},
{
identifier: _.constant('"@scoped/vendor/module1"')
},
{
identifier: _.constant('external "@scoped/vendor/module2"')
},
{
identifier: _.constant('external "uuid/v4"')
},
{
identifier: _.constant('external "localmodule"')
},
{
identifier: _.constant('external "bluebird"')
},
{
identifier: _.constant('external "aws-sdk"')
}
]
},
toString: sandbox.stub().returns('testStats'),
hasErrors: _.constant(false)
}
]
};
module.webpackConfig = testWebpackConfig;
module.configuration = { concurrency: 1 };
webpackMock.compilerMock.run.reset();
webpackMock.compilerMock.run.yields(null, multiStats);
return expect(module.compile()).to.be.fulfilled.then(() => {
console.log(JSON.stringify(module.compileStats.stats[0].externalModules));
expect(module.compileStats.stats[0].externalModules).to.eql([
{ external: '@scoped/vendor', origin: undefined },
{ external: 'uuid', origin: undefined },
{ external: 'localmodule', origin: undefined },
{ external: 'bluebird', origin: undefined },
{ external: 'aws-sdk', origin: undefined }
]);
return null;
});
});
});

0 comments on commit 9b5c290

Please sign in to comment.