Merge pull request #12894 from webpack/bugfix/memory-leak

lib/ChunkGraph.js

@@ -348,6 +348,7 @@ class ChunkGraph {
 		}
 		cgc.modules.clear();
 		chunk.disconnectFromGroups();
+		ChunkGraph.clearChunkGraphForChunk(chunk);
 	}
 
 	/**
@@ -895,6 +896,7 @@ class ChunkGraph {
 			chunkA.addGroup(chunkGroup);
 			chunkB.removeGroup(chunkGroup);
 		}
+		ChunkGraph.clearChunkGraphForChunk(chunkB);
 	}
 
 	/**
@@ -1500,6 +1502,15 @@ Caller might not support runtime-dependent code generation (opt-out via optimiza
 		chunkGraphForModuleMap.set(module, chunkGraph);
 	}
 
+	// TODO remove in webpack 6
+	/**
+	 * @param {Module} module the module
+	 * @returns {void}
+	 */
+	static clearChunkGraphForModule(module) {
+		chunkGraphForModuleMap.delete(module);
+	}
+
 	// TODO remove in webpack 6
 	/**
 	 * @param {Chunk} chunk the chunk
@@ -1540,6 +1551,15 @@ Caller might not support runtime-dependent code generation (opt-out via optimiza
 	static setChunkGraphForChunk(chunk, chunkGraph) {
 		chunkGraphForChunkMap.set(chunk, chunkGraph);
 	}
+
+	// TODO remove in webpack 6
+	/**
+	 * @param {Chunk} chunk the chunk
+	 * @returns {void}
+	 */
+	static clearChunkGraphForChunk(chunk) {
+		chunkGraphForChunkMap.delete(chunk);
+	}
 }
 
 // TODO remove in webpack 6

lib/Compiler.js

@@ -17,9 +17,11 @@ const { SizeOnlySource } = require("webpack-sources");
 const webpack = require("./");
 const Cache = require("./Cache");
 const CacheFacade = require("./CacheFacade");
+const ChunkGraph = require("./ChunkGraph");
 const Compilation = require("./Compilation");
 const ConcurrentCompilationError = require("./ConcurrentCompilationError");
 const ContextModuleFactory = require("./ContextModuleFactory");
+const ModuleGraph = require("./ModuleGraph");
 const NormalModuleFactory = require("./NormalModuleFactory");
 const RequestShortener = require("./RequestShortener");
 const ResolverFactory = require("./ResolverFactory");
@@ -254,6 +256,9 @@ class Compiler {
 		/** @type {boolean} */
 		this.watchMode = false;
 
+		/** @type {Compilation} */
+		this._lastCompilation = undefined;
+
 		/** @private @type {WeakMap<Source, { sizeOnlySource: SizeOnlySource, writtenTo: Map<string, number> }>} */
 		this._assetEmittingSourceCache = new WeakMap();
 		/** @private @type {Map<string, number>} */
@@ -350,6 +355,22 @@ class Compiler {
 		);
 	}
 
+	// TODO webpack 6: solve this in a better way
+	// e.g. move compilation specific info from Modules into ModuleGraph
+	_cleanupLastCompilation() {
+		if (this._lastCompilation !== undefined) {
+			for (const module of this._lastCompilation.modules) {
+				ChunkGraph.clearChunkGraphForModule(module);
+				ModuleGraph.clearModuleGraphForModule(module);
+				module.cleanupForCache();
+			}
+			for (const chunk of this._lastCompilation.chunks) {
+				ChunkGraph.clearChunkGraphForChunk(chunk);
+			}
+			this._lastCompilation = undefined;
+		}
+	}
+
 	/**
 	 * @param {WatchOptions} watchOptions the watcher's options
 	 * @param {Callback<Stats>} handler signals when the call finishes
@@ -978,7 +999,8 @@ ${other}`);
 	}
 
 	createCompilation() {
-		return new Compilation(this);
+		this._cleanupLastCompilation();
+		return (this._lastCompilation = new Compilation(this));
 	}
 
 	/**

lib/ContextModule.js

@@ -138,7 +138,14 @@ class ContextModule extends Module {
 		const m = /** @type {ContextModule} */ (module);
 		this.resolveDependencies = m.resolveDependencies;
 		this.options = m.options;
-		this.resolveOptions = m.resolveOptions;
+	}
+
+	/**
+	 * Assuming this module is in the cache. Remove internal references to allow freeing some memory.
+	 */
+	cleanupForCache() {
+		super.cleanupForCache();
+		this.resolveDependencies = undefined;
 	}
 
 	prettyRegExp(regexString) {

lib/DelegatedModule.js

@@ -224,6 +224,14 @@ class DelegatedModule extends Module {
 		this.originalRequest = m.originalRequest;
 		this.delegateData = m.delegateData;
 	}
+
+	/**
+	 * Assuming this module is in the cache. Remove internal references to allow freeing some memory.
+	 */
+	cleanupForCache() {
+		super.cleanupForCache();
+		this.delegateData = undefined;
+	}
 }
 
 makeSerializable(DelegatedModule, "webpack/lib/DelegatedModule");

lib/DllModule.js

@@ -143,6 +143,14 @@ class DllModule extends Module {
 		super.updateCacheModule(module);
 		this.dependencies = module.dependencies;
 	}
+
+	/**
+	 * Assuming this module is in the cache. Remove internal references to allow freeing some memory.
+	 */
+	cleanupForCache() {
+		super.cleanupForCache();
+		this.dependencies = undefined;
+	}
 }
 
 makeSerializable(DllModule, "webpack/lib/DllModule");

lib/Module.js

@@ -27,6 +27,7 @@ const makeSerializable = require("./util/makeSerializable");
 /** @typedef {import("./ExportsInfo").UsageStateType} UsageStateType */
 /** @typedef {import("./FileSystemInfo")} FileSystemInfo */
 /** @typedef {import("./ModuleGraphConnection").ConnectionState} ConnectionState */
+/** @typedef {import("./NormalModuleFactory")} NormalModuleFactory */
 /** @typedef {import("./RequestShortener")} RequestShortener */
 /** @typedef {import("./ResolverFactory").ResolverWithOptions} ResolverWithOptions */
 /** @typedef {import("./RuntimeTemplate")} RuntimeTemplate */
@@ -897,6 +898,36 @@ class Module extends DependenciesBlock {
 		this.resolveOptions = module.resolveOptions;
 	}
 
+	/**
+	 * Module should be unsafe cached. Get data that's needed for that.
+	 * This data will be passed to restoreFromUnsafeCache later.
+	 * @returns {object} cached data
+	 */
+	getUnsafeCacheData() {
+		return {
+			factoryMeta: this.factoryMeta,
+			resolveOptions: this.resolveOptions
+		};
+	}
+
+	/**
+	 * restore unsafe cache data
+	 * @param {object} unsafeCacheData data from getUnsafeCacheData
+	 * @param {NormalModuleFactory} normalModuleFactory the normal module factory handling the unsafe caching
+	 */
+	_restoreFromUnsafeCache(unsafeCacheData, normalModuleFactory) {
+		this.factoryMeta = unsafeCacheData.factoryMeta;
+		this.resolveOptions = unsafeCacheData.resolveOptions;
+	}
+
+	/**
+	 * Assuming this module is in the cache. Remove internal references to allow freeing some memory.
+	 */
+	cleanupForCache() {
+		this.factoryMeta = undefined;
+		this.resolveOptions = undefined;
+	}
+
 	/**
 	 * @returns {Source | null} the original source for the module before webpack transformation
 	 */

lib/ModuleGraph.js

@@ -739,6 +739,15 @@ class ModuleGraph {
 	static setModuleGraphForModule(module, moduleGraph) {
 		moduleGraphForModuleMap.set(module, moduleGraph);
 	}
+
+	// TODO remove in webpack 6
+	/**
+	 * @param {Module} module the module
+	 * @returns {void}
+	 */
+	static clearModuleGraphForModule(module) {
+		moduleGraphForModuleMap.delete(module);
+	}
 }
 
 // TODO remove in webpack 6

lib/NormalModule.js

@@ -55,6 +55,7 @@ const memoize = require("./util/memoize");
 /** @typedef {import("./Module").NeedBuildContext} NeedBuildContext */
 /** @typedef {import("./ModuleGraph")} ModuleGraph */
 /** @typedef {import("./ModuleGraphConnection").ConnectionState} ConnectionState */
+/** @typedef {import("./NormalModuleFactory")} NormalModuleFactory */
 /** @typedef {import("./Parser")} Parser */
 /** @typedef {import("./RequestShortener")} RequestShortener */
 /** @typedef {import("./ResolverFactory").ResolverWithOptions} ResolverWithOptions */
@@ -205,7 +206,9 @@ class NormalModule extends Module {
 	 * @param {string} options.resource path + query of the real resource
 	 * @param {string | undefined} options.matchResource path + query of the matched resource (virtual)
 	 * @param {Parser} options.parser the parser used
+	 * @param {object} options.parserOptions the options of the parser used
 	 * @param {Generator} options.generator the generator used
+	 * @param {object} options.generatorOptions the options of the generator used
 	 * @param {Object} options.resolveOptions options used for resolving requests from this module
 	 */
 	constructor({
@@ -218,7 +221,9 @@ class NormalModule extends Module {
 		resource,
 		matchResource,
 		parser,
+		parserOptions,
 		generator,
+		generatorOptions,
 		resolveOptions
 	}) {
 		super(type, getContext(resource), layer);
@@ -234,8 +239,10 @@ class NormalModule extends Module {
 		this.binary = /^(asset|webassembly)\b/.test(type);
 		/** @type {Parser} */
 		this.parser = parser;
+		this.parserOptions = parserOptions;
 		/** @type {Generator} */
 		this.generator = generator;
+		this.generatorOptions = generatorOptions;
 		/** @type {string} */
 		this.resource = resource;
 		/** @type {string | undefined} */
@@ -319,12 +326,56 @@ class NormalModule extends Module {
 		this.userRequest = m.userRequest;
 		this.rawRequest = m.rawRequest;
 		this.parser = m.parser;
+		this.parserOptions = m.parserOptions;
 		this.generator = m.generator;
+		this.generatorOptions = m.generatorOptions;
 		this.resource = m.resource;
 		this.matchResource = m.matchResource;
 		this.loaders = m.loaders;
 	}
 
+	/**
+	 * Assuming this module is in the cache. Remove internal references to allow freeing some memory.
+	 */
+	cleanupForCache() {
+		super.cleanupForCache();
+		this.parser = undefined;
+		this.parserOptions = undefined;
+		this.generator = undefined;
+		this.generatorOptions = undefined;
+	}
+
+	/**
+	 * Module should be unsafe cached. Get data that's needed for that.
+	 * This data will be passed to restoreFromUnsafeCache later.
+	 * @returns {object} cached data
+	 */
+	getUnsafeCacheData() {
+		const data = super.getUnsafeCacheData();
+		data.parserOptions = this.parserOptions;
+		data.generatorOptions = this.generatorOptions;
+		return data;
+	}
+
+	restoreFromUnsafeCache(unsafeCacheData, normalModuleFactory) {
+		this._restoreFromUnsafeCache(unsafeCacheData, normalModuleFactory);
+	}
+
+	/**
+	 * restore unsafe cache data
+	 * @param {object} unsafeCacheData data from getUnsafeCacheData
+	 * @param {NormalModuleFactory} normalModuleFactory the normal module factory handling the unsafe caching
+	 */
+	_restoreFromUnsafeCache(unsafeCacheData, normalModuleFactory) {
+		this.parserOptions = unsafeCacheData.parserOptions;
+		this.parser = normalModuleFactory.getParser(this.type, this.parserOptions);
+		this.generatorOptions = unsafeCacheData.generatorOptions;
+		this.generator = normalModuleFactory.getGenerator(
+			this.type,
+			this.generatorOptions
+		);
+	}
+
 	/**
 	 * @param {string} context the compilation context
 	 * @param {string} name the asset name
@@ -668,7 +719,7 @@ class NormalModule extends Module {
 			},
 			(err, result) => {
 				if (!result) {
-					processResult(
+					return processResult(
 						err || new Error("No result from loader-runner processing"),
 						null
 					);
@@ -1158,7 +1209,9 @@ class NormalModule extends Module {
 			loaders: null,
 			matchResource: null,
 			parser: null,
+			parserOptions: null,
 			generator: null,
+			generatorOptions: null,
 			resolveOptions: null
 		});
 		obj.deserialize(context);