add experimental lazy compilation

declarations/WebpackOptions.d.ts

@@ -1041,6 +1041,30 @@ export interface Experiments {
 	 * Enable module and chunk layers.
 	 */
 	layers?: boolean;
+	/**
+	 * Compile import()s only when they are accessed.
+	 */
+	lazyCompilation?:
+		| boolean
+		| {
+				/**
+				 * A custom backend.
+				 */
+				backend?:
+					| ((
+							compiler: import("../lib/Compiler"),
+							client: string,
+							callback: (err?: Error, api?: any) => void
+					  ) => void)
+					| ((
+							compiler: import("../lib/Compiler"),
+							client: string
+					  ) => Promise<any>);
+				/**
+				 * A custom client.
+				 */
+				client?: string;
+		  };
 	/**
 	 * Allow output javascript files as module source type.
 	 */

examples/lazy-compilation/README.md

@@ -0,0 +1,67 @@
+To run this example you need to install `webpack-dev-server` and run `webpack serve`.
+
+# example.js
+
+```javascript
+const libraries = {
+	react: () => import("react"),
+	acorn: () => import("acorn"),
+	"core-js": () => import("core-js"),
+	lodash: () => import("lodash"),
+	xxhashjs: () => import("xxhashjs"),
+	"all of them": () => import("./all")
+};
+
+window.onload = () => {
+	document.body.style = "font-size: 16pt;";
+	const pre = document.createElement("pre");
+	pre.style = "height: 200px; overflow-y: auto";
+	pre.innerText =
+		"Click on a button to load the library with import(). The first click triggers a lazy compilation of the module.";
+	for (const key of Object.keys(libraries)) {
+		const button = document.createElement("button");
+		const loadFn = libraries[key];
+		button.innerText = key;
+		button.onclick = async () => {
+			pre.innerText = "Loading " + key + "...";
+			const result = await loadFn();
+			pre.innerText = `${key} = {\n  ${Object.keys(result).join(",\n  ")}\n}`;
+		};
+		document.body.appendChild(button);
+	}
+	const button = document.createElement("button");
+	button.innerText = "Load more...";
+	button.onclick = async () => {
+		pre.innerText = "Loading more...";
+		await import("./more");
+		pre.innerText = "More libraries available.";
+	};
+	document.body.appendChild(button);
+	document.body.appendChild(pre);
+};
+```
+
+# webpack.config.js
+
+```javascript
+const { HotModuleReplacementPlugin } = require("../../");
+
+module.exports = {
+	mode: "development",
+	entry: {
+		main: "./example.js"
+	},
+	cache: {
+		type: "filesystem",
+		idleTimeout: 5000
+	},
+	experiments: {
+		lazyCompilation: true
+	},
+	devServer: {
+		hot: true,
+		publicPath: "/dist/"
+	},
+	plugins: [new HotModuleReplacementPlugin()]
+};
+```

examples/lazy-compilation/all.js

@@ -0,0 +1,8 @@
+export * from "react";
+export * from "react-dom";
+export * from "acorn";
+export * from "core-js";
+export * from "date-fns";
+export * from "lodash";
+export * from "lodash-es";
+export * from "xxhashjs";

examples/lazy-compilation/build.js

@@ -0,0 +1 @@
+require("../build-common");

examples/lazy-compilation/example.js

@@ -0,0 +1,36 @@
+const libraries = {
+	react: () => import("react"),
+	acorn: () => import("acorn"),
+	"core-js": () => import("core-js"),
+	lodash: () => import("lodash"),
+	xxhashjs: () => import("xxhashjs"),
+	"all of them": () => import("./all")
+};
+
+window.onload = () => {
+	document.body.style = "font-size: 16pt;";
+	const pre = document.createElement("pre");
+	pre.style = "height: 200px; overflow-y: auto";
+	pre.innerText =
+		"Click on a button to load the library with import(). The first click triggers a lazy compilation of the module.";
+	for (const key of Object.keys(libraries)) {
+		const button = document.createElement("button");
+		const loadFn = libraries[key];
+		button.innerText = key;
+		button.onclick = async () => {
+			pre.innerText = "Loading " + key + "...";
+			const result = await loadFn();
+			pre.innerText = `${key} = {\n  ${Object.keys(result).join(",\n  ")}\n}`;
+		};
+		document.body.appendChild(button);
+	}
+	const button = document.createElement("button");
+	button.innerText = "Load more...";
+	button.onclick = async () => {
+		pre.innerText = "Loading more...";
+		await import("./more");
+		pre.innerText = "More libraries available.";
+	};
+	document.body.appendChild(button);
+	document.body.appendChild(pre);
+};

examples/lazy-compilation/index.html

@@ -0,0 +1,6 @@
+<!DOCTYPE html>
+<html>
+	<head>
+		<script src="dist/main.js"></script>
+	</head>
+</html>

examples/lazy-compilation/more.js

@@ -0,0 +1,21 @@
+const libraries = {
+	"react-dom": () => import("react-dom"),
+	"date-fns": () => import("date-fns"),
+	xxhashjs: () => import("xxhashjs"),
+	"lodash-es": () => import("lodash-es")
+};
+
+const pre = document.querySelector("pre");
+for (const key of Object.keys(libraries)) {
+	const button = document.createElement("button");
+	const loadFn = libraries[key];
+	button.innerText = key;
+	button.onclick = async () => {
+		pre.innerText = "Loading " + key + "...";
+		const result = await loadFn();
+		pre.innerText = `${key} = {\n  ${Object.keys(result).join(",\n  ")}\n}`;
+	};
+	document.body.appendChild(button);
+}
+
+export {};

examples/lazy-compilation/template.md

@@ -0,0 +1,13 @@
+To run this example you need to install `webpack-dev-server` and run `webpack serve`.
+
+# example.js
+
+```javascript
+_{{example.js}}_
+```
+
+# webpack.config.js
+
+```javascript
+_{{webpack.config.js}}_
+```

examples/lazy-compilation/webpack.config.js

@@ -0,0 +1,20 @@
+const { HotModuleReplacementPlugin } = require("../../");
+
+module.exports = {
+	mode: "development",
+	entry: {
+		main: "./example.js"
+	},
+	cache: {
+		type: "filesystem",
+		idleTimeout: 5000
+	},
+	experiments: {
+		lazyCompilation: true
+	},
+	devServer: {
+		hot: true,
+		publicPath: "/dist/"
+	},
+	plugins: [new HotModuleReplacementPlugin()]
+};

hot/lazy-compilation-node.js

@@ -0,0 +1,29 @@
+/* global __resourceQuery */
+
+"use strict";
+
+if (!module.hot) {
+	throw new Error(
+		"Environment doesn't support lazy compilation (requires Hot Module Replacement enabled)"
+	);
+}
+
+var urlBase = decodeURIComponent(__resourceQuery.slice(1));
+exports.keepAlive = function (key) {
+	var response;
+	require("http")
+		.request(
+			urlBase + key,
+			{
+				agent: false,
+				headers: { accept: "text/event-stream" }
+			},
+			function (res) {
+				response = res;
+			}
+		)
+		.end();
+	return function () {
+		response.destroy();
+	};
+};

hot/lazy-compilation-web.js

@@ -0,0 +1,40 @@
+/* global __resourceQuery */
+
+"use strict";
+
+if (typeof EventSource !== "function" || !module.hot) {
+	throw new Error(
+		"Environment doesn't support lazy compilation (requires EventSource and Hot Module Replacement enabled)"
+	);
+}
+
+var urlBase = decodeURIComponent(__resourceQuery.slice(1));
+var activeEventSource;
+var activeKeys = new Map();
+
+var updateEventSource = function updateEventSource() {
+	if (activeEventSource) activeEventSource.close();
+	activeEventSource = new EventSource(
+		urlBase + Array.from(activeKeys.keys()).join("@")
+	);
+};
+
+exports.keepAlive = function (key) {
+	var value = activeKeys.get(key) || 0;
+	activeKeys.set(key, value + 1);
+	if (value === 0) {
+		updateEventSource();
+	}
+
+	return function () {
+		setTimeout(function () {
+			var value = activeKeys.get(key);
+			if (value === 1) {
+				activeKeys.delete(key);
+				updateEventSource();
+			} else {
+				activeKeys.set(key, value - 1);
+			}
+		}, 1000);
+	};
+};

lib/Compiler.js

@@ -168,6 +168,8 @@ class Compiler {
 			invalid: new SyncHook(["filename", "changeTime"]),
 			/** @type {SyncHook<[]>} */
 			watchClose: new SyncHook([]),
+			/** @type {AsyncSeriesHook<[]>} */
+			shutdown: new AsyncSeriesHook([]),
 
 			/** @type {SyncBailHook<[string, string, any[]], true>} */
 			infrastructureLog: new SyncBailHook(["origin", "type", "args"]),
@@ -1075,7 +1077,10 @@ ${other}`);
 	 * @returns {void}
 	 */
 	close(callback) {
-		this.cache.shutdown(callback);
+		this.hooks.shutdown.callAsync(err => {
+			if (err) return callback(err);
+			this.cache.shutdown(callback);
+		});
 	}
 }
 

lib/WebpackOptionsApply.js

@@ -244,6 +244,22 @@ class WebpackOptionsApply extends OptionsApply {
 			}).apply(compiler);
 		}
 
+		if (options.experiments.lazyCompilation) {
+			const LazyCompilationPlugin = require("./hmr/LazyCompilationPlugin");
+			new LazyCompilationPlugin({
+				backend:
+					(typeof options.experiments.lazyCompilation === "object" &&
+						options.experiments.lazyCompilation.backend) ||
+					require("./hmr/lazyCompilationBackend"),
+				client:
+					(typeof options.experiments.lazyCompilation === "object" &&
+						options.experiments.lazyCompilation.client) ||
+					`webpack/hot/lazy-compilation-${
+						options.externalsPresets.node ? "node" : "web"
+					}.js`
+			}).apply(compiler);
+		}
+
 		new EntryOptionPlugin().apply(compiler);
 		compiler.hooks.entryOption.call(options.context, options.entry);
 

lib/cache/PackFileCacheStrategy.js

@@ -11,7 +11,10 @@ const { formatSize } = require("../SizeFormatHelpers");
 const LazySet = require("../util/LazySet");
 const makeSerializable = require("../util/makeSerializable");
 const memoize = require("../util/memoize");
-const { createFileSerializer } = require("../util/serialization");
+const {
+	createFileSerializer,
+	NOT_SERIALIZABLE
+} = require("../util/serialization");
 
 /** @typedef {import("../../declarations/WebpackOptions").SnapshotOptions} SnapshotOptions */
 /** @typedef {import("../Cache").Etag} Etag */
@@ -525,6 +528,7 @@ class PackContentItems {
 					write(value);
 				} catch (e) {
 					rollback(s);
+					if (e === NOT_SERIALIZABLE) continue;
 					logger.warn(
 						`Skipped not serializable cache item '${key}': ${e.message}`
 					);