From 1bb6166e7891af4d22aa97facbfe9d039cb7b41d Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Mon, 27 Sep 2021 19:03:55 +0200 Subject: [PATCH] move unsafe cache from NormalModuleFactory to Compilation to skip more processing --- lib/Compilation.js | 309 ++++++++++++++++++++++++++----------- lib/Compiler.js | 6 +- lib/NormalModuleFactory.js | 39 ----- types.d.ts | 7 +- 4 files changed, 226 insertions(+), 135 deletions(-) diff --git a/lib/Compilation.js b/lib/Compilation.js index d8846b966c0..5fc89df87d3 100644 --- a/lib/Compilation.js +++ b/lib/Compilation.js @@ -92,6 +92,7 @@ const { isSourceEqual } = require("./util/source"); /** @typedef {import("./CacheFacade")} CacheFacade */ /** @typedef {import("./ChunkGroup").ChunkGroupOptions} ChunkGroupOptions */ /** @typedef {import("./Compiler")} Compiler */ +/** @typedef {import("./Compiler").CompilationParams} CompilationParams */ /** @typedef {import("./DependenciesBlock")} DependenciesBlock */ /** @typedef {import("./Dependency")} Dependency */ /** @typedef {import("./Dependency").DependencyLocation} DependencyLocation */ @@ -101,6 +102,7 @@ const { isSourceEqual } = require("./util/source"); /** @typedef {import("./Module").CodeGenerationResult} CodeGenerationResult */ /** @typedef {import("./ModuleFactory")} ModuleFactory */ /** @typedef {import("./ModuleFactory").ModuleFactoryCreateDataContextInfo} ModuleFactoryCreateDataContextInfo */ +/** @typedef {import("./ModuleFactory").ModuleFactoryResult} ModuleFactoryResult */ /** @typedef {import("./RequestShortener")} RequestShortener */ /** @typedef {import("./RuntimeModule")} RuntimeModule */ /** @typedef {import("./Template").RenderManifestEntry} RenderManifestEntry */ @@ -400,12 +402,19 @@ const byLocation = compareSelect(err => err.loc, compareLocations); const compareErrors = concatComparators(byModule, byLocation, byMessage); +/** @type {WeakMap} */ +const unsafeCacheDependencies = new WeakMap(); + +/** @type {WeakMap} */ +const unsafeCacheData = new WeakMap(); + class Compilation { /** * Creates an instance of Compilation. * @param {Compiler} compiler the compiler which created the compilation + * @param {CompilationParams} params the compilation parameters */ - constructor(compiler) { + constructor(compiler, params) { const getNormalModuleLoader = () => deprecatedNormalModuleLoaderHook(this); /** @typedef {{ additionalAssets?: true | Function }} ProcessAssetsAdditionalOptions */ /** @type {AsyncSeriesHook<[CompilationAssets], ProcessAssetsAdditionalOptions>} */ @@ -880,6 +889,7 @@ BREAKING CHANGE: Asset processing hooks in Compilation has been merged into a si /** @type {boolean} */ this.profile = (options && options.profile) || false; + this.params = params; this.mainTemplate = new MainTemplate(this.outputOptions, this); this.chunkTemplate = new ChunkTemplate(this.outputOptions, this); this.runtimeTemplate = new RuntimeTemplate( @@ -999,6 +1009,8 @@ BREAKING CHANGE: Asset processing hooks in Compilation has been merged into a si this.usedModuleIds = null; /** @type {boolean} */ this.needAdditionalPass = false; + /** @type {Set} */ + this._restoredUnsafeCacheEntries = new Set(); /** @type {WeakSet} */ this.builtModules = new WeakSet(); /** @type {WeakSet} */ @@ -1031,6 +1043,11 @@ BREAKING CHANGE: Asset processing hooks in Compilation has been merged into a si this._modulesCache = this.getCache("Compilation/modules"); this._assetsCache = this.getCache("Compilation/assets"); this._codeGenerationCache = this.getCache("Compilation/codeGeneration"); + + const unsafeCache = options.module.unsafeCache; + this._unsafeCache = !!unsafeCache; + this._unsafeCachePredicate = + typeof unsafeCache === "function" ? unsafeCache : () => true; } getStats() { @@ -1415,12 +1432,44 @@ BREAKING CHANGE: Asset processing hooks in Compilation has been merged into a si /** @type {Dependency[]} */ let listCacheValue; + const unsafeRestoredModules = new Set(); + /** * @param {Dependency} dep dependency * @returns {void} */ const processDependency = dep => { this.moduleGraph.setParents(dep, currentBlock, module); + if (this._unsafeCache) { + try { + const cachedModule = unsafeCacheDependencies.get(dep); + if (cachedModule === null) return; + if (cachedModule !== undefined) { + if (!this._restoredUnsafeCacheEntries.has(cachedModule)) { + const data = unsafeCacheData.get(cachedModule); + cachedModule.restoreFromUnsafeCache( + data, + this.params.normalModuleFactory, + this.params + ); + this._restoredUnsafeCacheEntries.add(cachedModule); + if (!this.modules.has(cachedModule)) { + this._handleNewModuleFromUnsafeCache(module, dep, cachedModule); + unsafeRestoredModules.add(cachedModule); + return; + } + } + this._handleExistingModuleFromUnsafeCache( + module, + dep, + cachedModule + ); + return; + } + } catch (e) { + console.error(e); + } + } const resourceIdent = dep.getResourceIdentifier(); if (resourceIdent !== undefined && resourceIdent !== null) { const category = dep.category; @@ -1504,7 +1553,7 @@ BREAKING CHANGE: Asset processing hooks in Compilation has been merged into a si return callback(e); } - if (sortedDependencies.length === 0) { + if (sortedDependencies.length === 0 && unsafeRestoredModules.size === 0) { callback(); return; } @@ -1512,27 +1561,78 @@ BREAKING CHANGE: Asset processing hooks in Compilation has been merged into a si // This is nested so we need to allow one additional task this.processDependenciesQueue.increaseParallelism(); - asyncLib.forEach( - sortedDependencies, - (item, callback) => { - this.handleModuleCreation(item, err => { - // In V8, the Error objects keep a reference to the functions on the stack. These warnings & - // errors are created inside closures that keep a reference to the Compilation, so errors are - // leaking the Compilation object. - if (err && this.bail) { - // eslint-disable-next-line no-self-assign - err.stack = err.stack; - return callback(err); - } - callback(); - }); - }, - err => { - this.processDependenciesQueue.decreaseParallelism(); + const processSortedDependency = (item, callback) => { + this.handleModuleCreation(item, err => { + // In V8, the Error objects keep a reference to the functions on the stack. These warnings & + // errors are created inside closures that keep a reference to the Compilation, so errors are + // leaking the Compilation object. + if (err && this.bail) { + // eslint-disable-next-line no-self-assign + err.stack = err.stack; + return callback(err); + } + callback(); + }); + }; - return callback(err); - } + const processUnsafeRestoredModule = (item, callback) => { + this._handleModuleBuildAndDependencies(module, item, true, callback); + }; + + const finalCallback = err => { + this.processDependenciesQueue.decreaseParallelism(); + + return callback(err); + }; + + if (sortedDependencies.length === 0) { + asyncLib.forEach( + unsafeRestoredModules, + processUnsafeRestoredModule, + finalCallback + ); + } else if (unsafeRestoredModules.size === 0) { + asyncLib.forEach( + sortedDependencies, + processSortedDependency, + finalCallback + ); + } else { + asyncLib.parallel( + [ + cb => + asyncLib.forEach( + unsafeRestoredModules, + processUnsafeRestoredModule, + cb + ), + cb => + asyncLib.forEach(sortedDependencies, processSortedDependency, cb) + ], + finalCallback + ); + } + } + + _handleNewModuleFromUnsafeCache(originModule, dependency, module) { + const moduleGraph = this.moduleGraph; + + moduleGraph.setResolvedModule(originModule, dependency, module); + + moduleGraph.setIssuerIfUnset( + module, + originModule !== undefined ? originModule : null ); + + this._modules.set(module.identifier(), module); + this.modules.add(module); + ModuleGraph.setModuleGraphForModule(module, this.moduleGraph); + } + + _handleExistingModuleFromUnsafeCache(originModule, dependency, module) { + const moduleGraph = this.moduleGraph; + + moduleGraph.setResolvedModule(originModule, dependency, module); } /** @@ -1605,13 +1705,35 @@ BREAKING CHANGE: Asset processing hooks in Compilation has been merged into a si return callback(err); } - for (let i = 0; i < dependencies.length; i++) { - const dependency = dependencies[i]; - moduleGraph.setResolvedModule( - connectOrigin ? originModule : null, - dependency, - module - ); + if ( + this._unsafeCache && + /** @type {any} */ (module).restoreFromUnsafeCache && + this._unsafeCachePredicate(module) + ) { + for (let i = 0; i < dependencies.length; i++) { + const dependency = dependencies[i]; + moduleGraph.setResolvedModule( + connectOrigin ? originModule : null, + dependency, + module + ); + unsafeCacheDependencies.set( + dependency, + /** @type {any} */ (module) + ); + if (!unsafeCacheData.has(module)) { + unsafeCacheData.set(module, module.getUnsafeCacheData()); + } + } + } else { + for (let i = 0; i < dependencies.length; i++) { + const dependency = dependencies[i]; + moduleGraph.setResolvedModule( + connectOrigin ? originModule : null, + dependency, + module + ); + } } moduleGraph.setIssuerIfUnset( @@ -1629,75 +1751,84 @@ BREAKING CHANGE: Asset processing hooks in Compilation has been merged into a si } } - // Check for cycles when build is trigger inside another build - let creatingModuleDuringBuildSet = undefined; - if (!recursive && this.buildQueue.isProcessing(originModule)) { - // Track build dependency - creatingModuleDuringBuildSet = - this.creatingModuleDuringBuild.get(originModule); - if (creatingModuleDuringBuildSet === undefined) { - creatingModuleDuringBuildSet = new Set(); - this.creatingModuleDuringBuild.set( - originModule, - creatingModuleDuringBuildSet - ); - } - creatingModuleDuringBuildSet.add(originModule); - - // When building is blocked by another module - // search for a cycle, cancel the cycle by throwing - // an error (otherwise this would deadlock) - const blockReasons = this.creatingModuleDuringBuild.get(module); - if (blockReasons !== undefined) { - const set = new Set(blockReasons); - for (const item of set) { - const blockReasons = this.creatingModuleDuringBuild.get(item); - if (blockReasons !== undefined) { - for (const m of blockReasons) { - if (m === module) { - return callback(new BuildCycleError(module)); - } - set.add(m); - } - } + this._handleModuleBuildAndDependencies( + originModule, + module, + recursive, + callback + ); + }); + } + ); + } + + _handleModuleBuildAndDependencies(originModule, module, recursive, callback) { + // Check for cycles when build is trigger inside another build + let creatingModuleDuringBuildSet = undefined; + if (!recursive && this.buildQueue.isProcessing(originModule)) { + // Track build dependency + creatingModuleDuringBuildSet = + this.creatingModuleDuringBuild.get(originModule); + if (creatingModuleDuringBuildSet === undefined) { + creatingModuleDuringBuildSet = new Set(); + this.creatingModuleDuringBuild.set( + originModule, + creatingModuleDuringBuildSet + ); + } + creatingModuleDuringBuildSet.add(originModule); + + // When building is blocked by another module + // search for a cycle, cancel the cycle by throwing + // an error (otherwise this would deadlock) + const blockReasons = this.creatingModuleDuringBuild.get(module); + if (blockReasons !== undefined) { + const set = new Set(blockReasons); + for (const item of set) { + const blockReasons = this.creatingModuleDuringBuild.get(item); + if (blockReasons !== undefined) { + for (const m of blockReasons) { + if (m === module) { + return callback(new BuildCycleError(module)); } + set.add(m); } } + } + } + } - this.buildModule(module, err => { - if (creatingModuleDuringBuildSet !== undefined) { - creatingModuleDuringBuildSet.delete(module); - } - if (err) { - if (!err.module) { - err.module = module; - } - this.errors.push(err); - - return callback(err); - } + this.buildModule(module, err => { + if (creatingModuleDuringBuildSet !== undefined) { + creatingModuleDuringBuildSet.delete(module); + } + if (err) { + if (!err.module) { + err.module = module; + } + this.errors.push(err); - if (!recursive) { - this.processModuleDependenciesNonRecursive(module); - callback(null, module); - return; - } + return callback(err); + } - // This avoids deadlocks for circular dependencies - if (this.processDependenciesQueue.isProcessing(module)) { - return callback(); - } + if (!recursive) { + this.processModuleDependenciesNonRecursive(module); + callback(null, module); + return; + } - this.processModuleDependencies(module, err => { - if (err) { - return callback(err); - } - callback(null, module); - }); - }); - }); + // This avoids deadlocks for circular dependencies + if (this.processDependenciesQueue.isProcessing(module)) { + return callback(); } - ); + + this.processModuleDependencies(module, err => { + if (err) { + return callback(err); + } + callback(null, module); + }); + }); } /** diff --git a/lib/Compiler.js b/lib/Compiler.js index f5d1d3039ee..4a6f30207f1 100644 --- a/lib/Compiler.js +++ b/lib/Compiler.js @@ -1035,9 +1035,9 @@ ${other}`); return !!this.parentCompilation; } - createCompilation() { + createCompilation(params) { this._cleanupLastCompilation(); - return (this._lastCompilation = new Compilation(this)); + return (this._lastCompilation = new Compilation(this, params)); } /** @@ -1045,7 +1045,7 @@ ${other}`); * @returns {Compilation} the created compilation */ newCompilation(params) { - const compilation = this.createCompilation(); + const compilation = this.createCompilation(params); compilation.name = this.name; compilation.records = this.records; this.hooks.thisCompilation.call(compilation, params); diff --git a/lib/NormalModuleFactory.js b/lib/NormalModuleFactory.js index cc233876d44..ed3f0f99c3c 100644 --- a/lib/NormalModuleFactory.js +++ b/lib/NormalModuleFactory.js @@ -167,12 +167,6 @@ const deprecationChangedHookMessage = (name, hook) => { ); }; -/** @type {WeakMap} */ -const unsafeCacheDependencies = new WeakMap(); - -/** @type {WeakMap} */ -const unsafeCacheData = new WeakMap(); - const ruleSetCompiler = new RuleSetCompiler([ new BasicMatcherRulePlugin("test", "resource"), new BasicMatcherRulePlugin("scheme"), @@ -256,11 +250,6 @@ class NormalModuleFactory extends ModuleFactory { rules: options.rules } ]); - this.unsafeCache = !!options.unsafeCache; - this.cachePredicate = - typeof options.unsafeCache === "function" - ? options.unsafeCache - : () => true; this.context = context || ""; this.fs = fs; this._globalParserOptions = options.parser; @@ -745,18 +734,6 @@ class NormalModuleFactory extends ModuleFactory { */ create(data, callback) { const dependencies = /** @type {ModuleDependency[]} */ (data.dependencies); - if (this.unsafeCache) { - const cacheEntry = unsafeCacheDependencies.get(dependencies[0]); - if (cacheEntry) { - const { module } = cacheEntry; - if (!this._restoredUnsafeCacheEntries.has(module)) { - const data = unsafeCacheData.get(module); - module.restoreFromUnsafeCache(data, this); - this._restoredUnsafeCacheEntries.add(module); - } - return callback(null, cacheEntry); - } - } const context = data.context || this.context; const resolveOptions = data.resolveOptions || EMPTY_RESOLVE_OPTIONS; const dependency = dependencies[0]; @@ -825,22 +802,6 @@ class NormalModuleFactory extends ModuleFactory { contextDependencies }; - if ( - this.unsafeCache && - resolveData.cacheable && - module && - module.restoreFromUnsafeCache && - this.cachePredicate(module) - ) { - for (const d of dependencies) { - unsafeCacheDependencies.set(d, factoryResult); - } - if (!unsafeCacheData.has(module)) { - unsafeCacheData.set(module, module.getUnsafeCacheData()); - } - this._restoredUnsafeCacheEntries.add(module); - } - callback(null, factoryResult); }); }); diff --git a/types.d.ts b/types.d.ts index 50a81734698..9a2bab9a659 100644 --- a/types.d.ts +++ b/types.d.ts @@ -1299,7 +1299,7 @@ declare class Compilation { /** * Creates an instance of Compilation. */ - constructor(compiler: Compiler); + constructor(compiler: Compiler, params: CompilationParams); hooks: Readonly<{ buildModule: SyncHook<[Module]>; rebuildModule: SyncHook<[Module]>; @@ -1455,6 +1455,7 @@ declare class Compilation { outputOptions: OutputNormalized; bail: boolean; profile: boolean; + params: CompilationParams; mainTemplate: MainTemplate; chunkTemplate: ChunkTemplate; runtimeTemplate: RuntimeTemplate; @@ -1921,7 +1922,7 @@ declare class Compiler { plugins?: WebpackPluginInstance[] ): Compiler; isChild(): boolean; - createCompilation(): Compilation; + createCompilation(params?: any): Compilation; newCompilation(params: CompilationParams): Compilation; createNormalModuleFactory(): NormalModuleFactory; createContextModuleFactory(): ContextModuleFactory; @@ -7366,8 +7367,6 @@ declare abstract class NormalModuleFactory extends ModuleFactory { }>; resolverFactory: ResolverFactory; ruleSet: RuleSet; - unsafeCache: boolean; - cachePredicate: Function; context: string; fs: InputFileSystem; parserCache: Map>;