From 84ec7708c2d87a37410d01d29e7b26947ee36416 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Tue, 23 Feb 2021 00:25:38 +0100 Subject: [PATCH 1/3] improve performance of getModuleRuntimes --- lib/ChunkGraph.js | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/lib/ChunkGraph.js b/lib/ChunkGraph.js index 66c352d7d96..9b2d9be1703 100644 --- a/lib/ChunkGraph.js +++ b/lib/ChunkGraph.js @@ -69,6 +69,18 @@ const getArray = set => { return Array.from(set); }; +/** + * @param {SortableSet} chunks the chunks + * @returns {RuntimeSpecSet} runtimes + */ +const getModuleRuntimes = chunks => { + const runtimes = new RuntimeSpecSet(); + for (const chunk of chunks) { + runtimes.add(chunk.runtime); + } + return runtimes; +}; + /** * @param {SortableSet} set the set * @returns {Map>} modules by source type @@ -510,11 +522,7 @@ class ChunkGraph { */ getModuleRuntimes(module) { const cgm = this._getChunkGraphModule(module); - const runtimes = new RuntimeSpecSet(); - for (const chunk of cgm.chunks) { - runtimes.add(chunk.runtime); - } - return runtimes; + return cgm.chunks.getFromUnorderedCache(getModuleRuntimes); } /** From 365a36252230ad7554cfc0f078cf4ae1edac5aa8 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Tue, 23 Feb 2021 10:59:59 +0100 Subject: [PATCH 2/3] performance optimization add statistics to ModuleConcatenationPlugin --- lib/CaseSensitiveModulesWarning.js | 6 +- lib/Module.js | 8 +- lib/ModuleGraph.js | 48 +++++++- lib/ModuleGraphConnection.js | 4 +- lib/async-modules/InferAsyncModulesPlugin.js | 16 ++- lib/optimize/ModuleConcatenationPlugin.js | 108 +++++++++++++----- .../__snapshots__/StatsTestCases.test.js.snap | 4 +- types.d.ts | 13 ++- 8 files changed, 153 insertions(+), 54 deletions(-) diff --git a/lib/CaseSensitiveModulesWarning.js b/lib/CaseSensitiveModulesWarning.js index 9b8c43e6d1e..65898f4d830 100644 --- a/lib/CaseSensitiveModulesWarning.js +++ b/lib/CaseSensitiveModulesWarning.js @@ -37,12 +37,12 @@ const createModulesListMessage = (modules, moduleGraph) => { .map(m => { let message = `* ${m.identifier()}`; const validReasons = Array.from( - moduleGraph.getIncomingConnections(m) - ).filter(reason => reason.originModule); + moduleGraph.getIncomingConnectionsByOriginModule(m).keys() + ).filter(x => x); if (validReasons.length > 0) { message += `\n Used by ${validReasons.length} module(s), i. e.`; - message += `\n ${validReasons[0].originModule.identifier()}`; + message += `\n ${validReasons[0].identifier()}`; } return message; }) diff --git a/lib/Module.js b/lib/Module.js index 7eb5a00bb18..7dd4780aa96 100644 --- a/lib/Module.js +++ b/lib/Module.js @@ -631,9 +631,11 @@ class Module extends DependenciesBlock { */ hasReasonForChunk(chunk, moduleGraph, chunkGraph) { // check for each reason if we need the chunk - for (const connection of moduleGraph.getIncomingConnections(this)) { - if (!connection.isTargetActive(chunk.runtime)) continue; - const fromModule = connection.originModule; + for (const [ + fromModule, + connections + ] of moduleGraph.getIncomingConnectionsByOriginModule(this)) { + if (!connections.some(c => c.isTargetActive(chunk.runtime))) continue; for (const originChunk of chunkGraph.getModuleChunksIterable( fromModule )) { diff --git a/lib/ModuleGraph.js b/lib/ModuleGraph.js index ae0b893f486..7bc8148927f 100644 --- a/lib/ModuleGraph.js +++ b/lib/ModuleGraph.js @@ -8,6 +8,7 @@ const util = require("util"); const ExportsInfo = require("./ExportsInfo"); const ModuleGraphConnection = require("./ModuleGraphConnection"); +const SortableSet = require("./util/SortableSet"); /** @typedef {import("./DependenciesBlock")} DependenciesBlock */ /** @typedef {import("./Dependency")} Dependency */ @@ -15,7 +16,6 @@ const ModuleGraphConnection = require("./ModuleGraphConnection"); /** @typedef {import("./Module")} Module */ /** @typedef {import("./ModuleProfile")} ModuleProfile */ /** @typedef {import("./RequestShortener")} RequestShortener */ -/** @template T @typedef {import("./util/SortableSet")} SortableSet */ /** @typedef {import("./util/runtime").RuntimeSpec} RuntimeSpec */ /** @@ -26,10 +26,40 @@ const ModuleGraphConnection = require("./ModuleGraphConnection"); const EMPTY_ARRAY = []; +/** + * @param {SortableSet} set input + * @returns {readonly Map} mapped by origin module + */ +const getConnectionsByOriginModule = set => { + const map = new Map(); + /** @type {Module | 0} */ + let lastModule = 0; + /** @type {ModuleGraphConnection[]} */ + let lastList = undefined; + for (const connection of set) { + const { originModule } = connection; + if (lastModule === originModule) { + lastList.push(connection); + } else { + lastModule = originModule; + const list = map.get(originModule); + if (list !== undefined) { + lastList = list; + list.push(connection); + } else { + const list = [connection]; + lastList = list; + map.set(originModule, list); + } + } + } + return map; +}; + class ModuleGraphModule { constructor() { - /** @type {Set} */ - this.incomingConnections = new Set(); + /** @type {SortableSet} */ + this.incomingConnections = new SortableSet(); /** @type {Set | undefined} */ this.outgoingConnections = undefined; /** @type {Module | null} */ @@ -319,9 +349,6 @@ class ModuleGraph { newConnections.add(newConnection); if (newConnection.module !== undefined) { const otherMgm = this._getModuleGraphModule(newConnection.module); - if (otherMgm.incomingConnections === undefined) { - otherMgm.incomingConnections = new Set(); - } otherMgm.incomingConnections.add(newConnection); } } @@ -402,6 +429,15 @@ class ModuleGraph { return connections === undefined ? EMPTY_ARRAY : connections; } + /** + * @param {Module} module the module + * @returns {readonly Map} reasons why a module is included, in a map by source module + */ + getIncomingConnectionsByOriginModule(module) { + const connections = this._getModuleGraphModule(module).incomingConnections; + return connections.getFromUnorderedCache(getConnectionsByOriginModule); + } + /** * @param {Module} module the module * @returns {ModuleProfile | null} the module profile diff --git a/lib/ModuleGraphConnection.js b/lib/ModuleGraphConnection.js index ec63b196dd9..94f971953e8 100644 --- a/lib/ModuleGraphConnection.js +++ b/lib/ModuleGraphConnection.js @@ -51,8 +51,8 @@ const intersectConnectionStates = (a, b) => { class ModuleGraphConnection { /** - * @param {Module|undefined} originModule the referencing module - * @param {Dependency|undefined} dependency the referencing dependency + * @param {Module|null} originModule the referencing module + * @param {Dependency|null} dependency the referencing dependency * @param {Module} module the referenced module * @param {string=} explanation some extra detail * @param {boolean=} weak the reference is weak diff --git a/lib/async-modules/InferAsyncModulesPlugin.js b/lib/async-modules/InferAsyncModulesPlugin.js index a578f9baf32..9e64972e483 100644 --- a/lib/async-modules/InferAsyncModulesPlugin.js +++ b/lib/async-modules/InferAsyncModulesPlugin.js @@ -31,14 +31,18 @@ class InferAsyncModulesPlugin { } for (const module of queue) { moduleGraph.setAsync(module); - const connections = moduleGraph.getIncomingConnections(module); - for (const connection of connections) { - const dep = connection.dependency; + for (const [ + originModule, + connections + ] of moduleGraph.getIncomingConnectionsByOriginModule(module)) { if ( - dep instanceof HarmonyImportDependency && - connection.isTargetActive(undefined) + connections.some( + c => + c.dependency instanceof HarmonyImportDependency && + c.isTargetActive(undefined) + ) ) { - queue.add(connection.originModule); + queue.add(originModule); } } } diff --git a/lib/optimize/ModuleConcatenationPlugin.js b/lib/optimize/ModuleConcatenationPlugin.js index 4738cbfc509..9d5272b2739 100644 --- a/lib/optimize/ModuleConcatenationPlugin.js +++ b/lib/optimize/ModuleConcatenationPlugin.js @@ -28,6 +28,19 @@ const ConcatenatedModule = require("./ConcatenatedModule"); /** @typedef {import("../RequestShortener")} RequestShortener */ /** @typedef {import("../util/runtime").RuntimeSpec} RuntimeSpec */ +/** + * @typedef {Object} Statistics + * @property {number} cached + * @property {number} alreadyInConfig + * @property {number} invalidModule + * @property {number} incorrectChunks + * @property {number} incorrectDependency + * @property {number} incorrectChunksOfImporter + * @property {number} incorrectRuntimeCondition + * @property {number} importerFailed + * @property {number} added + */ + const formatBailoutReason = msg => { return "ModuleConcatenation bailout: " + msg; }; @@ -210,12 +223,29 @@ class ModuleConcatenationPlugin { }); logger.timeEnd("sort relevant modules"); + /** @type {Statistics} */ + const stats = { + cached: 0, + alreadyInConfig: 0, + invalidModule: 0, + incorrectChunks: 0, + incorrectDependency: 0, + incorrectChunksOfImporter: 0, + incorrectRuntimeCondition: 0, + importerFailed: 0, + added: 0 + }; + let statsCandidates = 0; + let statsSizeSum = 0; + let statsEmptyConfigurations = 0; + logger.time("find modules to concatenate"); const concatConfigurations = []; const usedAsInner = new Set(); for (const currentRoot of relevantModules) { // when used by another configuration as inner: // the other configuration is better and we can skip this one + // TODO reconsider that when it's only used in a different runtime if (usedAsInner.has(currentRoot)) continue; let chunkRuntime = undefined; @@ -267,7 +297,8 @@ class ModuleConcatenationPlugin { impCandidates, failureCache, chunkGraph, - true + true, + stats ); if (problem) { failureCache.set(imp, problem); @@ -278,14 +309,18 @@ class ModuleConcatenationPlugin { } } } + statsCandidates += candidates.size; if (!currentConfiguration.isEmpty()) { + const modules = currentConfiguration.getModules(); + statsSizeSum += modules.size; concatConfigurations.push(currentConfiguration); - for (const module of currentConfiguration.getModules()) { + for (const module of modules) { if (module !== currentConfiguration.rootModule) { usedAsInner.add(module); } } } else { + statsEmptyConfigurations++; const optimizationBailouts = moduleGraph.getOptimizationBailout( currentRoot ); @@ -298,7 +333,14 @@ class ModuleConcatenationPlugin { } logger.timeEnd("find modules to concatenate"); logger.debug( - `${concatConfigurations.length} concat configurations` + `${ + concatConfigurations.length + } successful concat configurations (avg size: ${ + statsSizeSum / concatConfigurations.length + }), ${statsEmptyConfigurations} bailed out completely` + ); + logger.debug( + `${statsCandidates} candidates were considered for adding (${stats.cached} cached failure, ${stats.alreadyInConfig} already in config, ${stats.invalidModule} invalid module, ${stats.incorrectChunks} incorrect chunks, ${stats.incorrectDependency} incorrect dependency, ${stats.incorrectChunksOfImporter} incorrect chunks of importer, ${stats.incorrectRuntimeCondition} incorrect runtime condition, ${stats.importerFailed} importer failed, ${stats.added} added)` ); // HACK: Sort configurations by length and start with the longest one // to get the biggest groups possible. Used modules are marked with usedModules @@ -502,6 +544,7 @@ class ModuleConcatenationPlugin { * @param {Map} failureCache cache for problematic modules to be more performant * @param {ChunkGraph} chunkGraph the chunk graph * @param {boolean} avoidMutateOnFailure avoid mutating the config when adding fails + * @param {Statistics} statistics gathering metrics * @returns {Module | function(RequestShortener): string} the problematic module */ _tryToAdd( @@ -514,20 +557,24 @@ class ModuleConcatenationPlugin { candidates, failureCache, chunkGraph, - avoidMutateOnFailure + avoidMutateOnFailure, + statistics ) { const cacheEntry = failureCache.get(module); if (cacheEntry) { + statistics.cached++; return cacheEntry; } // Already added? if (config.has(module)) { + statistics.alreadyInConfig++; return null; } // Not possible to add? if (!possibleModules.has(module)) { + statistics.invalidModule++; failureCache.set(module, module); // cache failures for performance return module; } @@ -535,24 +582,26 @@ class ModuleConcatenationPlugin { // Module must be in the correct chunks const missingChunks = Array.from( chunkGraph.getModuleChunksIterable(config.rootModule) - ) - .filter(chunk => !chunkGraph.isModuleInChunk(module, chunk)) - .map(chunk => chunk.name || "unnamed chunk(s)"); + ).filter(chunk => !chunkGraph.isModuleInChunk(module, chunk)); if (missingChunks.length > 0) { - const missingChunksList = Array.from(new Set(missingChunks)).sort(); - const chunks = Array.from( - new Set( - Array.from(chunkGraph.getModuleChunksIterable(module)).map( - chunk => chunk.name || "unnamed chunk(s)" + const problem = requestShortener => { + const missingChunksList = Array.from( + new Set(missingChunks.map(chunk => chunk.name || "unnamed chunk(s)")) + ).sort(); + const chunks = Array.from( + new Set( + Array.from(chunkGraph.getModuleChunksIterable(module)).map( + chunk => chunk.name || "unnamed chunk(s)" + ) ) - ) - ).sort(); - const problem = requestShortener => - `Module ${module.readableIdentifier( + ).sort(); + return `Module ${module.readableIdentifier( requestShortener )} is not in the same chunk(s) (expected in chunk(s) ${missingChunksList.join( ", " )}, module is in chunk(s) ${chunks.join(", ")})`; + }; + statistics.incorrectChunks++; failureCache.set(module, problem); // cache failures for performance return problem; } @@ -642,27 +691,29 @@ class ModuleConcatenationPlugin { )} is referenced in a unsupported way`; } }; + statistics.incorrectDependency++; failureCache.set(module, problem); // cache failures for performance return problem; } + const incomingModules = Array.from( + new Set(incomingConnections.map(c => c.originModule)) + ); + // Module must be in the same chunks like the referencing module - const otherChunkConnections = incomingConnections.filter(connection => { + const otherChunkModules = incomingModules.filter(originModule => { for (const chunk of chunkGraph.getModuleChunksIterable( config.rootModule )) { - if (!chunkGraph.isModuleInChunk(connection.originModule, chunk)) { + if (!chunkGraph.isModuleInChunk(originModule, chunk)) { return true; } } return false; }); - if (otherChunkConnections.length > 0) { + if (otherChunkModules.length > 0) { const problem = requestShortener => { - const importingModules = new Set( - otherChunkConnections.map(c => c.originModule) - ); - const names = Array.from(importingModules) + const names = otherChunkModules .map(m => m.readableIdentifier(requestShortener)) .sort(); return `Module ${module.readableIdentifier( @@ -671,6 +722,7 @@ class ModuleConcatenationPlugin { ", " )}`; }; + statistics.incorrectChunksOfImporter++; failureCache.set(module, problem); // cache failures for performance return problem; } @@ -714,6 +766,7 @@ class ModuleConcatenationPlugin { )})` ).join(", ")}`; }; + statistics.incorrectRuntimeCondition++; failureCache.set(module, problem); // cache failures for performance return problem; } @@ -727,9 +780,7 @@ class ModuleConcatenationPlugin { // Add the module config.add(module); - const incomingModules = Array.from( - new Set(incomingConnections.map(c => c.originModule)) - ).sort(compareModulesByIdentifier); + incomingModules.sort(compareModulesByIdentifier); // Every module which depends on the added module must be in the configuration too. for (const originModule of incomingModules) { @@ -743,10 +794,12 @@ class ModuleConcatenationPlugin { candidates, failureCache, chunkGraph, - false + false, + statistics ); if (problem) { if (backup !== undefined) config.rollback(backup); + statistics.importerFailed++; failureCache.set(module, problem); // cache failures for performance return problem; } @@ -756,6 +809,7 @@ class ModuleConcatenationPlugin { for (const imp of this._getImports(compilation, module, runtime)) { candidates.add(imp); } + statistics.added++; return null; } } diff --git a/test/__snapshots__/StatsTestCases.test.js.snap b/test/__snapshots__/StatsTestCases.test.js.snap index b3b0587f0a0..f9ea900f912 100644 --- a/test/__snapshots__/StatsTestCases.test.js.snap +++ b/test/__snapshots__/StatsTestCases.test.js.snap @@ -2423,7 +2423,7 @@ LOG from webpack.ModuleConcatenationPlugin find modules to concatenate: X ms sort concat configurations: X ms create concatenated modules: X ms -+ 2 hidden lines ++ 3 hidden lines LOG from webpack.FileSystemInfo 6 new snapshots created @@ -3032,7 +3032,7 @@ cacheable modules 975 bytes modules by path ./*.js 106 bytes ./vendor.js 25 bytes [built] [code generated] ./lazy_shared.js 56 bytes [built] [code generated] - ModuleConcatenation bailout: Cannot concat with ./common_lazy_shared.js: Module ./common_lazy_shared.js is referenced from different chunks by these modules: ./lazy_first.js + 1 modules, ./lazy_second.js + 1 modules + ModuleConcatenation bailout: Cannot concat with ./common_lazy_shared.js: Module ./common_lazy_shared.js is referenced from different chunks by these modules: ./lazy_first.js, ./lazy_second.js ./common_lazy_shared.js 25 bytes [built] [code generated] ./first.js + 2 modules 292 bytes [built] [code generated] ModuleConcatenation bailout: Cannot concat with ./vendor.js: Module ./vendor.js is not in the same chunk(s) (expected in chunk(s) first, module is in chunk(s) vendor) diff --git a/types.d.ts b/types.d.ts index ad0974d2843..3371f1b150f 100644 --- a/types.d.ts +++ b/types.d.ts @@ -5979,6 +5979,9 @@ declare class ModuleGraph { getResolvedOrigin(dependency: Dependency): Module; getIncomingConnections(module: Module): Iterable; getOutgoingConnections(module: Module): Iterable; + getIncomingConnectionsByOriginModule( + module: Module + ): Map>; getProfile(module: Module): null | ModuleProfile; setProfile(module: Module, profile: null | ModuleProfile): void; getIssuer(module: Module): null | Module; @@ -6025,8 +6028,8 @@ declare class ModuleGraph { } declare class ModuleGraphConnection { constructor( - originModule: undefined | Module, - dependency: undefined | Dependency, + originModule: null | Module, + dependency: null | Dependency, module: Module, explanation?: string, weak?: boolean, @@ -6034,9 +6037,9 @@ declare class ModuleGraphConnection { | false | ((arg0: ModuleGraphConnection, arg1: RuntimeSpec) => ConnectionState) ); - originModule?: Module; - resolvedOriginModule?: Module; - dependency?: Dependency; + originModule: null | Module; + resolvedOriginModule: null | Module; + dependency: null | Dependency; resolvedModule: Module; module: Module; weak: boolean; From 5c42b918b438916add040531266f8d88c604427d Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Tue, 23 Feb 2021 12:03:26 +0100 Subject: [PATCH 3/3] further optimizations --- lib/ids/OccurrenceModuleIdsPlugin.js | 45 +++-- lib/javascript/JavascriptModulesPlugin.js | 10 +- lib/optimize/ModuleConcatenationPlugin.js | 230 ++++++++++++---------- 3 files changed, 157 insertions(+), 128 deletions(-) diff --git a/lib/ids/OccurrenceModuleIdsPlugin.js b/lib/ids/OccurrenceModuleIdsPlugin.js index 08667c18ba3..7bc4baaf82a 100644 --- a/lib/ids/OccurrenceModuleIdsPlugin.js +++ b/lib/ids/OccurrenceModuleIdsPlugin.js @@ -66,32 +66,43 @@ class OccurrenceModuleIdsPlugin { } /** - * @param {Iterable} connections connections + * @param {Module} module module * @returns {number} count of occurs */ - const countOccursInEntry = connections => { + const countOccursInEntry = module => { let sum = 0; - for (const c of connections) { - if (!c.isTargetActive(undefined)) continue; - if (!c.originModule) continue; - sum += initialChunkChunkMap.get(c.originModule); + for (const [ + originModule, + connections + ] of moduleGraph.getIncomingConnectionsByOriginModule(module)) { + if (!originModule) continue; + if (!connections.some(c => c.isTargetActive(undefined))) continue; + sum += initialChunkChunkMap.get(originModule); } return sum; }; /** - * @param {Iterable} connections connections + * @param {Module} module module * @returns {number} count of occurs */ - const countOccurs = connections => { + const countOccurs = module => { let sum = 0; - for (const c of connections) { - if (!c.isTargetActive(undefined)) continue; - if (!c.originModule) continue; - if (!c.dependency) continue; - const factor = c.dependency.getNumberOfIdOccurrences(); - if (factor === 0) continue; - sum += factor * chunkGraph.getNumberOfModuleChunks(c.originModule); + for (const [ + originModule, + connections + ] of moduleGraph.getIncomingConnectionsByOriginModule(module)) { + if (!originModule) continue; + const chunkModules = chunkGraph.getNumberOfModuleChunks( + originModule + ); + for (const c of connections) { + if (!c.isTargetActive(undefined)) continue; + if (!c.dependency) continue; + const factor = c.dependency.getNumberOfIdOccurrences(); + if (factor === 0) continue; + sum += factor * chunkModules; + } } return sum; }; @@ -99,7 +110,7 @@ class OccurrenceModuleIdsPlugin { if (prioritiseInitial) { for (const m of modulesInOccurrenceOrder) { const result = - countOccursInEntry(moduleGraph.getIncomingConnections(m)) + + countOccursInEntry(m) + initialChunkChunkMap.get(m) + entryCountMap.get(m); occursInInitialChunksMap.set(m, result); @@ -108,7 +119,7 @@ class OccurrenceModuleIdsPlugin { for (const m of modules) { const result = - countOccurs(moduleGraph.getIncomingConnections(m)) + + countOccurs(m) + chunkGraph.getNumberOfModuleChunks(m) + entryCountMap.get(m); occursInAllChunksMap.set(m, result); diff --git a/lib/javascript/JavascriptModulesPlugin.js b/lib/javascript/JavascriptModulesPlugin.js index 838aed3b906..cdb10f980f7 100644 --- a/lib/javascript/JavascriptModulesPlugin.js +++ b/lib/javascript/JavascriptModulesPlugin.js @@ -901,12 +901,12 @@ class JavascriptModulesPlugin { if ( result.allowInlineStartup && someInIterable( - moduleGraph.getIncomingConnections(entryModule), - c => - c.originModule && - c.isTargetActive(chunk.runtime) && + moduleGraph.getIncomingConnectionsByOriginModule(entryModule), + ([originModule, connections]) => + originModule && + connections.some(c => c.isTargetActive(chunk.runtime)) && someInIterable( - chunkGraph.getModuleRuntimes(c.originModule), + chunkGraph.getModuleRuntimes(originModule), runtime => intersectRuntime(runtime, chunk.runtime) !== undefined ) diff --git a/lib/optimize/ModuleConcatenationPlugin.js b/lib/optimize/ModuleConcatenationPlugin.js index 9d5272b2739..a00d4098c23 100644 --- a/lib/optimize/ModuleConcatenationPlugin.js +++ b/lib/optimize/ModuleConcatenationPlugin.js @@ -35,6 +35,7 @@ const ConcatenatedModule = require("./ConcatenatedModule"); * @property {number} invalidModule * @property {number} incorrectChunks * @property {number} incorrectDependency + * @property {number} incorrectModuleDependency * @property {number} incorrectChunksOfImporter * @property {number} incorrectRuntimeCondition * @property {number} importerFailed @@ -230,6 +231,7 @@ class ModuleConcatenationPlugin { invalidModule: 0, incorrectChunks: 0, incorrectDependency: 0, + incorrectModuleDependency: 0, incorrectChunksOfImporter: 0, incorrectRuntimeCondition: 0, importerFailed: 0, @@ -340,7 +342,7 @@ class ModuleConcatenationPlugin { }), ${statsEmptyConfigurations} bailed out completely` ); logger.debug( - `${statsCandidates} candidates were considered for adding (${stats.cached} cached failure, ${stats.alreadyInConfig} already in config, ${stats.invalidModule} invalid module, ${stats.incorrectChunks} incorrect chunks, ${stats.incorrectDependency} incorrect dependency, ${stats.incorrectChunksOfImporter} incorrect chunks of importer, ${stats.incorrectRuntimeCondition} incorrect runtime condition, ${stats.importerFailed} importer failed, ${stats.added} added)` + `${statsCandidates} candidates were considered for adding (${stats.cached} cached failure, ${stats.alreadyInConfig} already in config, ${stats.invalidModule} invalid module, ${stats.incorrectChunks} incorrect chunks, ${stats.incorrectDependency} incorrect dependency, ${stats.incorrectChunksOfImporter} incorrect chunks of importer, ${stats.incorrectModuleDependency} incorrect module dependency, ${stats.incorrectRuntimeCondition} incorrect runtime condition, ${stats.importerFailed} importer failed, ${stats.added} added)` ); // HACK: Sort configurations by length and start with the longest one // to get the biggest groups possible. Used modules are marked with usedModules @@ -608,97 +610,65 @@ class ModuleConcatenationPlugin { const moduleGraph = compilation.moduleGraph; - const incomingConnections = Array.from( - moduleGraph.getIncomingConnections(module) - ).filter(connection => { - // We are not interested in inactive connections - if (!connection.isActive(runtime)) return false; - - // Include, but do not analyse further, connections from non-modules - if (!connection.originModule) return true; - - // Ignore connection from orphan modules - if (chunkGraph.getNumberOfModuleChunks(connection.originModule) === 0) - return false; + const incomingConnections = moduleGraph.getIncomingConnectionsByOriginModule( + module + ); - // We don't care for connections from other runtimes - let originRuntime = undefined; - for (const r of chunkGraph.getModuleRuntimes(connection.originModule)) { - originRuntime = mergeRuntimeOwned(originRuntime, r); + const incomingConnectionsFromNonModules = + incomingConnections.get(null) || incomingConnections.get(undefined); + if (incomingConnectionsFromNonModules) { + const activeNonModulesConnections = incomingConnectionsFromNonModules.filter( + connection => { + // We are not interested in inactive connections + // or connections without dependency + return connection.isActive(runtime) || connection.dependency; + } + ); + if (activeNonModulesConnections.length > 0) { + const problem = requestShortener => { + const importingExplanations = new Set( + activeNonModulesConnections.map(c => c.explanation).filter(Boolean) + ); + const explanations = Array.from(importingExplanations).sort(); + return `Module ${module.readableIdentifier( + requestShortener + )} is referenced ${ + explanations.length > 0 + ? `by: ${explanations.join(", ")}` + : "in an unsupported way" + }`; + }; + statistics.incorrectDependency++; + failureCache.set(module, problem); // cache failures for performance + return problem; } + } - return intersectRuntime(runtime, originRuntime); - }); + /** @type {Map} */ + const incomingConnectionsFromModules = new Map(); + for (const [originModule, connections] of incomingConnections) { + if (originModule) { + // Ignore connection from orphan modules + if (chunkGraph.getNumberOfModuleChunks(originModule) === 0) continue; + + // We don't care for connections from other runtimes + let originRuntime = undefined; + for (const r of chunkGraph.getModuleRuntimes(originModule)) { + originRuntime = mergeRuntimeOwned(originRuntime, r); + } - const nonHarmonyConnections = incomingConnections.filter( - connection => - !connection.originModule || - !connection.dependency || - !(connection.dependency instanceof HarmonyImportDependency) - ); - if (nonHarmonyConnections.length > 0) { - const problem = requestShortener => { - const importingModules = new Set( - nonHarmonyConnections.map(c => c.originModule).filter(Boolean) - ); - const importingExplanations = new Set( - nonHarmonyConnections.map(c => c.explanation).filter(Boolean) - ); - const importingModuleTypes = new Map( - Array.from(importingModules).map( - m => - /** @type {[Module, Set]} */ ([ - m, - new Set( - nonHarmonyConnections - .filter(c => c.originModule === m) - .map(c => c.dependency.type) - .sort() - ) - ]) - ) + if (!intersectRuntime(runtime, originRuntime)) continue; + + // We are not interested in inactive connections + const activeConnections = connections.filter(connection => + connection.isActive(runtime) ); - const names = Array.from(importingModules) - .map( - m => - `${m.readableIdentifier( - requestShortener - )} (referenced with ${Array.from( - importingModuleTypes.get(m) - ).join(", ")})` - ) - .sort(); - const explanations = Array.from(importingExplanations).sort(); - if (names.length > 0 && explanations.length === 0) { - return `Module ${module.readableIdentifier( - requestShortener - )} is referenced from these modules with unsupported syntax: ${names.join( - ", " - )}`; - } else if (names.length === 0 && explanations.length > 0) { - return `Module ${module.readableIdentifier( - requestShortener - )} is referenced by: ${explanations.join(", ")}`; - } else if (names.length > 0 && explanations.length > 0) { - return `Module ${module.readableIdentifier( - requestShortener - )} is referenced from these modules with unsupported syntax: ${names.join( - ", " - )} and by: ${explanations.join(", ")}`; - } else { - return `Module ${module.readableIdentifier( - requestShortener - )} is referenced in a unsupported way`; - } - }; - statistics.incorrectDependency++; - failureCache.set(module, problem); // cache failures for performance - return problem; + if (activeConnections.length > 0) + incomingConnectionsFromModules.set(originModule, activeConnections); + } } - const incomingModules = Array.from( - new Set(incomingConnections.map(c => c.originModule)) - ); + const incomingModules = Array.from(incomingConnectionsFromModules.keys()); // Module must be in the same chunks like the referencing module const otherChunkModules = incomingModules.filter(originModule => { @@ -727,37 +697,85 @@ class ModuleConcatenationPlugin { return problem; } + /** @type {Map} */ + const nonHarmonyConnections = new Map(); + for (const [originModule, connections] of incomingConnectionsFromModules) { + const selected = connections.filter( + connection => + !connection.dependency || + !(connection.dependency instanceof HarmonyImportDependency) + ); + if (selected.length > 0) + nonHarmonyConnections.set(originModule, connections); + } + if (nonHarmonyConnections.size > 0) { + const problem = requestShortener => { + const names = Array.from(nonHarmonyConnections) + .map(([originModule, connections]) => { + return `${originModule.readableIdentifier( + requestShortener + )} (referenced with ${Array.from( + new Set( + connections + .map(c => c.dependency && c.dependency.type) + .filter(Boolean) + ) + ) + .sort() + .join(", ")})`; + }) + .sort(); + return `Module ${module.readableIdentifier( + requestShortener + )} is referenced from these modules with unsupported syntax: ${names.join( + ", " + )}`; + }; + statistics.incorrectModuleDependency++; + failureCache.set(module, problem); // cache failures for performance + return problem; + } + if (runtime !== undefined && typeof runtime !== "string") { // Module must be consistently referenced in the same runtimes - /** @type {Map} */ - const runtimeConditionMap = new Map(); - for (const connection of incomingConnections) { - const runtimeCondition = filterRuntime(runtime, runtime => { - return connection.isTargetActive(runtime); - }); - if (runtimeCondition === false) continue; - const old = runtimeConditionMap.get(connection.originModule) || false; - if (old === true) continue; - if (old !== false && runtimeCondition !== true) { - runtimeConditionMap.set( - connection.originModule, - mergeRuntime(old, runtimeCondition) - ); - } else { - runtimeConditionMap.set(connection.originModule, runtimeCondition); + /** @type {{ originModule: Module, runtimeCondition: RuntimeSpec }[]} */ + const otherRuntimeConnections = []; + outer: for (const [ + originModule, + connections + ] of incomingConnectionsFromModules) { + /** @type {false | RuntimeSpec} */ + let currentRuntimeCondition = false; + for (const connection of connections) { + const runtimeCondition = filterRuntime(runtime, runtime => { + return connection.isTargetActive(runtime); + }); + if (runtimeCondition === false) continue; + if (runtimeCondition === true) continue outer; + if (currentRuntimeCondition !== false) { + currentRuntimeCondition = mergeRuntime( + currentRuntimeCondition, + runtimeCondition + ); + } else { + currentRuntimeCondition = runtimeCondition; + } + } + if (currentRuntimeCondition !== false) { + otherRuntimeConnections.push({ + originModule, + runtimeCondition: currentRuntimeCondition + }); } } - const otherRuntimeConnections = Array.from(runtimeConditionMap).filter( - ([, runtimeCondition]) => typeof runtimeCondition !== "boolean" - ); if (otherRuntimeConnections.length > 0) { const problem = requestShortener => { return `Module ${module.readableIdentifier( requestShortener )} is runtime-dependent referenced by these modules: ${Array.from( otherRuntimeConnections, - ([module, runtimeCondition]) => - `${module.readableIdentifier( + ({ originModule, runtimeCondition }) => + `${originModule.readableIdentifier( requestShortener )} (expected runtime ${runtimeToString( runtime