Skip to content

Commit

Permalink
add memCache2 for mem caching with module/chunk ids
Browse files Browse the repository at this point in the history
  • Loading branch information
@sokra
sokra committed Oct 5, 2021
1 parent 508eded commit 91b6972
Show file tree
Hide file tree
Showing 23 changed files with 301 additions and 65 deletions.
203 changes: 154 additions & 49 deletions lib/Compilation.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -917,8 +917,10 @@ BREAKING CHANGE: Asset processing hooks in Compilation has been merged into a si
};
defineRemovedModuleTemplates(this.moduleTemplates);

/** @type {WeakMap<Module, WeakTupleMap<any, any>> | undefined} */
/** @type {Map<Module, WeakTupleMap<any, any>> | undefined} */
this.moduleMemCaches = undefined;
/** @type {Map<Module, WeakTupleMap<any, any>> | undefined} */
this.moduleMemCaches2 = undefined;
this.moduleGraph = new ModuleGraph();
/** @type {ChunkGraph} */
this.chunkGraph = undefined;
Expand Down Expand Up @@ -2169,7 +2171,7 @@ BREAKING CHANGE: Asset processing hooks in Compilation has been merged into a si
const moduleMemCacheCache = this.compiler.moduleMemCaches;
if (!moduleMemCacheCache) return;
if (!this.moduleMemCaches) {
this.moduleMemCaches = new WeakMap();
this.moduleMemCaches = new Map();
this.moduleGraph.setModuleMemCaches(this.moduleMemCaches);
}
const { moduleGraph, moduleMemCaches } = this;
Expand Down Expand Up @@ -2211,48 +2213,63 @@ BREAKING CHANGE: Asset processing hooks in Compilation has been merged into a si
return true;
};

for (const module of modules) {
const hash = module.buildInfo && module.buildInfo.hash;
if (typeof hash === "string") {
const cachedMemCache = moduleMemCacheCache.get(module);
if (cachedMemCache === undefined) {
// create a new entry
const memCache = new WeakTupleMap();
moduleMemCacheCache.set(module, {
hash: hash,
references: computeReferences(module),
memCache
});
moduleMemCaches.set(module, memCache);
affectedModules.add(module);
statNew++;
} else if (cachedMemCache.hash !== hash) {
// use a new one
const memCache = new WeakTupleMap();
moduleMemCaches.set(module, memCache);
affectedModules.add(module);
cachedMemCache.hash = hash;
cachedMemCache.references = computeReferences(module);
cachedMemCache.memCache = memCache;
statChanged++;
} else if (!compareReferences(module, cachedMemCache.references)) {
// use a new one
const memCache = new WeakTupleMap();
moduleMemCaches.set(module, memCache);
affectedModules.add(module);
cachedMemCache.references = computeReferences(module);
cachedMemCache.memCache = memCache;
statReferencesChanged++;
const modulesWithoutCache = new Set(modules);
for (const [module, cachedMemCache] of moduleMemCacheCache) {
if (modulesWithoutCache.has(module)) {
const hash = module.buildInfo && module.buildInfo.hash;
if (typeof hash === "string") {
if (cachedMemCache.hash !== hash) {
// use a new one
const memCache = new WeakTupleMap();
moduleMemCaches.set(module, memCache);
affectedModules.add(module);
cachedMemCache.hash = hash;
cachedMemCache.references = computeReferences(module);
cachedMemCache.memCache = memCache;
statChanged++;
} else if (!compareReferences(module, cachedMemCache.references)) {
// use a new one
const memCache = new WeakTupleMap();
moduleMemCaches.set(module, memCache);
affectedModules.add(module);
cachedMemCache.references = computeReferences(module);
cachedMemCache.memCache = memCache;
statReferencesChanged++;
} else {
// keep the old mem cache
moduleMemCaches.set(module, cachedMemCache.memCache);
statUnchanged++;
}
} else {
// keep the old mem cache
moduleMemCaches.set(module, cachedMemCache.memCache);
statUnchanged++;
infectedModules.add(module);
moduleMemCacheCache.delete(module);
statWithoutHash++;
}
modulesWithoutCache.delete(module);
} else {
moduleMemCacheCache.delete(module);
}
}

for (const module of modulesWithoutCache) {
const hash = module.buildInfo && module.buildInfo.hash;
if (typeof hash === "string") {
// create a new entry
const memCache = new WeakTupleMap();
moduleMemCacheCache.set(module, {
hash: hash,
references: computeReferences(module),
memCache
});
moduleMemCaches.set(module, memCache);
affectedModules.add(module);
statNew++;
} else {
infectedModules.add(module);
statWithoutHash++;
}
}

const reduceAffectType = connections => {
let affected = false;
for (const { dependency } of connections) {
Expand Down Expand Up @@ -2317,6 +2334,99 @@ BREAKING CHANGE: Asset processing hooks in Compilation has been merged into a si
);
}

_updateAffectedModulesWithIds() {
const { moduleMemCaches } = this;
if (!moduleMemCaches) return;
const moduleMemCaches2 = (this.moduleMemCaches2 = new Map());
const { moduleGraph, chunkGraph } = this;
const key = "memCache2";
/**
* @param {Module} module module
* @returns {{ modules?: Map<Module, string | number | undefined>, blocks?: (string | number)[] }} references
*/
const computeReferences = module => {
/** @type {Map<Module, string | number | undefined>} */
let modules = undefined;
/** @type {(string | number)[] | undefined} */
let blocks = undefined;
const outgoing = moduleGraph.getOutgoingConnectionsByModule(module);
if (outgoing !== undefined) {
for (const m of outgoing.keys()) {
if (!m) continue;
if (modules === undefined) modules = new Map();
modules.set(m, chunkGraph.getModuleId(module));
}
}
if (module.blocks.length > 0) {
blocks = [];
const queue = Array.from(module.blocks);
for (const block of queue) {
const chunkGroup = chunkGraph.getBlockChunkGroup(block);
if (chunkGroup) {
for (const chunk of chunkGroup.chunks) {
blocks.push(chunk.id);
}
} else {
blocks.push(null);
}
queue.push.apply(queue, block.blocks);
}
}
return { modules, blocks };
};
/**
* @param {Module} module module
* @param {Object} references references
* @param {Map<Module, string | number>=} references.modules modules
* @param {(string | number)[]=} references.blocks blocks
* @returns {boolean} ok?
*/
const compareReferences = (module, { modules, blocks }) => {
if (modules !== undefined) {
for (const [module, id] of modules) {
if (chunkGraph.getModuleId(module) !== id) return false;
}
}
if (blocks !== undefined) {
const queue = Array.from(module.blocks);
let i = 0;
for (const block of queue) {
const chunkGroup = chunkGraph.getBlockChunkGroup(block);
if (chunkGroup) {
for (const chunk of chunkGroup.chunks) {
if (i >= blocks.length || blocks[i++] !== chunk.id) return false;
}
} else {
if (i >= blocks.length || blocks[i++] !== null) return false;
}
queue.push.apply(queue, block.blocks);
}
if (i !== blocks.length) return false;
}
return true;
};

for (const [module, memCache] of moduleMemCaches) {
/** @type {{ references: { modules?: Map<Module, string | number | undefined>, blocks?: (string | number)[]}, memCache: WeakTupleMap<any[], any> }} */
const cache = memCache.get(key);
if (cache === undefined) {
const memCache2 = new WeakTupleMap();
memCache.set(key, {
references: computeReferences(module),
memCache: memCache2
});
moduleMemCaches2.set(module, memCache2);
} else if (!compareReferences(module, cache.references)) {
const memCache = new WeakTupleMap();
cache.references = computeReferences(module);
cache.memCache = memCache;
moduleMemCaches2.set(module, memCache);
} else {
moduleMemCaches2.set(module, cache.memCache);
}
}
}

finish(callback) {
this.factorizeQueue.clear();
if (this.profile) {
Expand Down Expand Up @@ -2561,6 +2671,7 @@ BREAKING CHANGE: Asset processing hooks in Compilation has been merged into a si
this.assetsInfo.clear();
this.moduleGraph.removeAllModuleAttributes();
this.moduleGraph.unfreeze();
this.moduleMemCaches2 = undefined;
}

/**
Expand Down Expand Up @@ -2778,6 +2889,8 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o

this.assignRuntimeIds();

this._updateAffectedModulesWithIds();

this.sortItemsWithChunkIds();

if (shouldRecord) {
Expand Down Expand Up @@ -3121,18 +3234,14 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o
chunkGraphEntries = this._getChunkGraphEntries()
} = {}) {
const context = { chunkGraph, codeGenerationResults };
const { moduleMemCaches } = this;
const { moduleMemCaches2 } = this;
this.logger.time("runtime requirements.modules");
const additionalModuleRuntimeRequirements =
this.hooks.additionalModuleRuntimeRequirements;
const runtimeRequirementInModule = this.hooks.runtimeRequirementInModule;
for (const module of modules) {
if (chunkGraph.getNumberOfModuleChunks(module) > 0) {
const memCache =
moduleMemCaches &&
// modules with async blocks depend on the chunk graph and can't be cached that way
module.blocks.length === 0 &&
moduleMemCaches.get(module);
const memCache = moduleMemCaches2 && moduleMemCaches2.get(module);
for (const runtime of chunkGraph.getModuleRuntimes(module)) {
if (memCache) {
const cached = memCache.get(
Expand Down Expand Up @@ -3589,14 +3698,10 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o
createModuleHashes() {
let statModulesHashed = 0;
let statModulesFromCache = 0;
const { chunkGraph, runtimeTemplate, moduleMemCaches } = this;
const { chunkGraph, runtimeTemplate, moduleMemCaches2 } = this;
const { hashFunction, hashDigest, hashDigestLength } = this.outputOptions;
for (const module of this.modules) {
const memCache =
moduleMemCaches &&
// modules with async blocks depend on the chunk graph and can't be cached that way
module.blocks.length === 0 &&
moduleMemCaches.get(module);
const memCache = moduleMemCaches2 && moduleMemCaches2.get(module);
for (const runtime of chunkGraph.getModuleRuntimes(module)) {
if (memCache) {
const digest = memCache.get(`moduleHash-${getRuntimeKey(runtime)}`);
Expand Down
2 changes: 1 addition & 1 deletion lib/Compiler.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,7 +249,7 @@ class Compiler {

this.cache = new Cache();

/** @type {WeakMap<Module, { hash: string, references: WeakMap<Dependency, Module>, memCache: WeakTupleMap }> | undefined} */
/** @type {Map<Module, { hash: string, references: WeakMap<Dependency, Module>, memCache: WeakTupleMap }> | undefined} */
this.moduleMemCaches = undefined;

this.compilerPath = "";
Expand Down
7 changes: 6 additions & 1 deletion lib/FlagDependencyUsagePlugin.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,15 @@ class FlagDependencyUsagePlugin {
stage: STAGE_DEFAULT
},
modules => {
if (compilation.moduleMemCaches) {
throw new Error(
"optimization.usedExports can't be used with cacheUnaffected as export usage is a global effect"
);
}

const logger = compilation.getLogger(
"webpack.FlagDependencyUsagePlugin"
);

/** @type {Map<ExportsInfo, Module>} */
const exportInfoToModuleMap = new Map();

Expand Down

0 comments on commit 91b6972

Please sign in to comment.