Merge pull request #12469 from webpack/feature/async-compilation

azure-pipelines.yml

@@ -130,11 +130,13 @@ jobs:
           key: yarn | $(Agent.OS) | yarn.lock
           path: $(YARN_CACHE_FOLDER)
         displayName: "Cache Yarn packages"
-      - script: |
-          yarn --frozen-lockfile
-          yarn link --frozen-lockfile || true
-          yarn link webpack --frozen-lockfile
+      - script: yarn --frozen-lockfile
         displayName: "Install dependencies"
+      - script: yarn link --frozen-lockfile || true
+        displayName: "Link webpack"
+        continueOnError: true
+      - script: yarn link webpack --frozen-lockfile
+        displayName: "Link webpack into node_modules"
       - script: |
           yarn cover:integration --ci --maxWorkers=2 --reporters=jest-junit
         displayName: "Run tests with coverage"

declarations/WebpackOptions.d.ts

@@ -1041,6 +1041,34 @@ export interface Experiments {
 	 * Enable module and chunk layers.
 	 */
 	layers?: boolean;
+	/**
+	 * Compile entrypoints and import()s only when they are accessed.
+	 */
+	lazyCompilation?:
+		| boolean
+		| {
+				/**
+				 * A custom backend.
+				 */
+				backend?:
+					| ((
+							compiler: import("../lib/Compiler"),
+							client: string,
+							callback: (err?: Error, api?: any) => void
+					  ) => void)
+					| ((
+							compiler: import("../lib/Compiler"),
+							client: string
+					  ) => Promise<any>);
+				/**
+				 * A custom client.
+				 */
+				client?: string;
+				/**
+				 * Enable/disable lazy compilation for entries.
+				 */
+				entries?: boolean;
+		  };
 	/**
 	 * Allow output javascript files as module source type.
 	 */

examples/lazy-compilation/README.md

@@ -0,0 +1,67 @@
+To run this example you need to install `webpack-dev-server` and run `webpack serve`.
+
+# example.js
+
+```javascript
+const libraries = {
+	react: () => import("react"),
+	acorn: () => import("acorn"),
+	"core-js": () => import("core-js"),
+	lodash: () => import("lodash"),
+	xxhashjs: () => import("xxhashjs"),
+	"all of them": () => import("./all")
+};
+
+window.onload = () => {
+	document.body.style = "font-size: 16pt;";
+	const pre = document.createElement("pre");
+	pre.style = "height: 200px; overflow-y: auto";
+	pre.innerText =
+		"Click on a button to load the library with import(). The first click triggers a lazy compilation of the module.";
+	for (const key of Object.keys(libraries)) {
+		const button = document.createElement("button");
+		const loadFn = libraries[key];
+		button.innerText = key;
+		button.onclick = async () => {
+			pre.innerText = "Loading " + key + "...";
+			const result = await loadFn();
+			pre.innerText = `${key} = {\n  ${Object.keys(result).join(",\n  ")}\n}`;
+		};
+		document.body.appendChild(button);
+	}
+	const button = document.createElement("button");
+	button.innerText = "Load more...";
+	button.onclick = async () => {
+		pre.innerText = "Loading more...";
+		await import("./more");
+		pre.innerText = "More libraries available.";
+	};
+	document.body.appendChild(button);
+	document.body.appendChild(pre);
+};
+```
+
+# webpack.config.js
+
+```javascript
+const { HotModuleReplacementPlugin } = require("../../");
+
+module.exports = {
+	mode: "development",
+	entry: {
+		main: "./example.js"
+	},
+	cache: {
+		type: "filesystem",
+		idleTimeout: 5000
+	},
+	experiments: {
+		lazyCompilation: true
+	},
+	devServer: {
+		hot: true,
+		publicPath: "/dist/"
+	},
+	plugins: [new HotModuleReplacementPlugin()]
+};
+```

examples/lazy-compilation/all.js

@@ -0,0 +1,8 @@
+export * from "react";
+export * from "react-dom";
+export * from "acorn";
+export * from "core-js";
+export * from "date-fns";
+export * from "lodash";
+export * from "lodash-es";
+export * from "xxhashjs";

examples/lazy-compilation/build.js

@@ -0,0 +1 @@
+require("../build-common");

examples/lazy-compilation/example.js

@@ -0,0 +1,36 @@
+const libraries = {
+	react: () => import("react"),
+	acorn: () => import("acorn"),
+	"core-js": () => import("core-js"),
+	lodash: () => import("lodash"),
+	xxhashjs: () => import("xxhashjs"),
+	"all of them": () => import("./all")
+};
+
+window.onload = () => {
+	document.body.style = "font-size: 16pt;";
+	const pre = document.createElement("pre");
+	pre.style = "height: 200px; overflow-y: auto";
+	pre.innerText =
+		"Click on a button to load the library with import(). The first click triggers a lazy compilation of the module.";
+	for (const key of Object.keys(libraries)) {
+		const button = document.createElement("button");
+		const loadFn = libraries[key];
+		button.innerText = key;
+		button.onclick = async () => {
+			pre.innerText = "Loading " + key + "...";
+			const result = await loadFn();
+			pre.innerText = `${key} = {\n  ${Object.keys(result).join(",\n  ")}\n}`;
+		};
+		document.body.appendChild(button);
+	}
+	const button = document.createElement("button");
+	button.innerText = "Load more...";
+	button.onclick = async () => {
+		pre.innerText = "Loading more...";
+		await import("./more");
+		pre.innerText = "More libraries available.";
+	};
+	document.body.appendChild(button);
+	document.body.appendChild(pre);
+};

examples/lazy-compilation/index.html

@@ -0,0 +1,6 @@
+<!DOCTYPE html>
+<html>
+	<head>
+		<script src="dist/main.js"></script>
+	</head>
+</html>

examples/lazy-compilation/more.js

@@ -0,0 +1,21 @@
+const libraries = {
+	"react-dom": () => import("react-dom"),
+	"date-fns": () => import("date-fns"),
+	xxhashjs: () => import("xxhashjs"),
+	"lodash-es": () => import("lodash-es")
+};
+
+const pre = document.querySelector("pre");
+for (const key of Object.keys(libraries)) {
+	const button = document.createElement("button");
+	const loadFn = libraries[key];
+	button.innerText = key;
+	button.onclick = async () => {
+		pre.innerText = "Loading " + key + "...";
+		const result = await loadFn();
+		pre.innerText = `${key} = {\n  ${Object.keys(result).join(",\n  ")}\n}`;
+	};
+	document.body.appendChild(button);
+}
+
+export {};

examples/lazy-compilation/template.md

@@ -0,0 +1,13 @@
+To run this example you need to install `webpack-dev-server` and run `webpack serve`.
+
+# example.js
+
+```javascript
+_{{example.js}}_
+```
+
+# webpack.config.js
+
+```javascript
+_{{webpack.config.js}}_
+```

examples/lazy-compilation/webpack.config.js

@@ -0,0 +1,22 @@
+const { HotModuleReplacementPlugin } = require("../../");
+
+module.exports = {
+	mode: "development",
+	entry: {
+		main: "./example.js"
+	},
+	cache: {
+		type: "filesystem",
+		idleTimeout: 5000
+	},
+	experiments: {
+		lazyCompilation: {
+			entries: false
+		}
+	},
+	devServer: {
+		hot: true,
+		publicPath: "/dist/"
+	},
+	plugins: [new HotModuleReplacementPlugin()]
+};

hot/lazy-compilation-node.js

@@ -0,0 +1,38 @@
+/* global __resourceQuery */
+
+"use strict";
+
+var urlBase = decodeURIComponent(__resourceQuery.slice(1));
+exports.keepAlive = function (options) {
+	var data = options.data;
+	var onError = options.onError;
+	var active = options.active;
+	var module = options.module;
+	var response;
+	var request = require("http").request(
+		urlBase + data,
+		{
+			agent: false,
+			headers: { accept: "text/event-stream" }
+		},
+		function (res) {
+			response = res;
+			response.on("error", errorHandler);
+			if (!active && !module.hot) {
+				console.log(
+					"Hot Module Replacement is not enabled. Waiting for process restart..."
+				);
+			}
+		}
+	);
+	function errorHandler(err) {
+		err.message =
+			"Problem communicating active modules to the server: " + err.message;
+		onError(err);
+	}
+	request.on("error", errorHandler);
+	request.end();
+	return function () {
+		response.destroy();
+	};
+};

hot/lazy-compilation-web.js

@@ -0,0 +1,74 @@
+/* global __resourceQuery */
+
+"use strict";
+
+if (typeof EventSource !== "function") {
+	throw new Error(
+		"Environment doesn't support lazy compilation (requires EventSource)"
+	);
+}
+
+var urlBase = decodeURIComponent(__resourceQuery.slice(1));
+var activeEventSource;
+var activeKeys = new Map();
+var errorHandlers = new Set();
+
+var updateEventSource = function updateEventSource() {
+	if (activeEventSource) activeEventSource.close();
+	if (activeKeys.size) {
+		activeEventSource = new EventSource(
+			urlBase + Array.from(activeKeys.keys()).join("@")
+		);
+		activeEventSource.onerror = function (event) {
+			errorHandlers.forEach(function (onError) {
+				onError(
+					new Error(
+						"Problem communicating active modules to the server: " +
+							event.message +
+							" " +
+							event.filename +
+							":" +
+							event.lineno +
+							":" +
+							event.colno +
+							" " +
+							event.error
+					)
+				);
+			});
+		};
+	} else {
+		activeEventSource = undefined;
+	}
+};
+
+exports.keepAlive = function (options) {
+	var data = options.data;
+	var onError = options.onError;
+	var active = options.active;
+	var module = options.module;
+	errorHandlers.add(onError);
+	var value = activeKeys.get(data) || 0;
+	activeKeys.set(data, value + 1);
+	if (value === 0) {
+		updateEventSource();
+	}
+	if (!active && !module.hot) {
+		console.log(
+			"Hot Module Replacement is not enabled. Waiting for process restart..."
+		);
+	}
+
+	return function () {
+		errorHandlers.delete(onError);
+		setTimeout(function () {
+			var value = activeKeys.get(data);
+			if (value === 1) {
+				activeKeys.delete(data);
+				updateEventSource();
+			} else {
+				activeKeys.set(data, value - 1);
+			}
+		}, 1000);
+	};
+};

lib/Compiler.js

@@ -168,6 +168,8 @@ class Compiler {
 			invalid: new SyncHook(["filename", "changeTime"]),
 			/** @type {SyncHook<[]>} */
 			watchClose: new SyncHook([]),
+			/** @type {AsyncSeriesHook<[]>} */
+			shutdown: new AsyncSeriesHook([]),
 
 			/** @type {SyncBailHook<[string, string, any[]], true>} */
 			infrastructureLog: new SyncBailHook(["origin", "type", "args"]),
@@ -1075,7 +1077,10 @@ ${other}`);
 	 * @returns {void}
 	 */
 	close(callback) {
-		this.cache.shutdown(callback);
+		this.hooks.shutdown.callAsync(err => {
+			if (err) return callback(err);
+			this.cache.shutdown(callback);
+		});
 	}
 }
 

lib/WebpackOptionsApply.js

@@ -244,6 +244,27 @@ class WebpackOptionsApply extends OptionsApply {
 			}).apply(compiler);
 		}
 
+		if (options.experiments.lazyCompilation) {
+			const LazyCompilationPlugin = require("./hmr/LazyCompilationPlugin");
+			new LazyCompilationPlugin({
+				backend:
+					(typeof options.experiments.lazyCompilation === "object" &&
+						options.experiments.lazyCompilation.backend) ||
+					require("./hmr/lazyCompilationBackend"),
+				client:
+					(typeof options.experiments.lazyCompilation === "object" &&
+						options.experiments.lazyCompilation.client) ||
+					require.resolve(
+						`../hot/lazy-compilation-${
+							options.externalsPresets.node ? "node" : "web"
+						}.js`
+					),
+				entries:
+					typeof options.experiments.lazyCompilation !== "object" ||
+					options.experiments.lazyCompilation.entries !== false
+			}).apply(compiler);
+		}
+
 		new EntryOptionPlugin().apply(compiler);
 		compiler.hooks.entryOption.call(options.context, options.entry);