store cache after large changes

detect larger changes automatically and use shorter timeout
webpack · Jun 28, 2021 · 142bc56 · 142bc56
1 parent 1b46d7e
commit 142bc56
Show file tree

Hide file tree

Showing 11 changed files with 111 additions and 30 deletions.
diff --git a/declarations/WebpackOptions.d.ts b/declarations/WebpackOptions.d.ts
@@ -916,11 +916,15 @@ export interface FileCacheOptions {
 	 */
 	hashAlgorithm?: string;
 	/**
-	 * Time in ms after which idle period the cache storing should happen (only for store: 'pack').
+	 * Time in ms after which idle period the cache storing should happen.
 	 */
 	idleTimeout?: number;
 	/**
-	 * Time in ms after which idle period the initial cache storing should happen (only for store: 'pack').
+	 * Time in ms after which idle period the cache storing should happen when larger changes has been detected (cumulative build time > 2 x avg cache store time).
+	 */
+	idleTimeoutAfterLargeChanges?: number;
+	/**
+	 * Time in ms after which idle period the initial cache storing should happen.
 	 */
 	idleTimeoutForInitialStore?: number;
 	/**

diff --git a/lib/Watching.js b/lib/Watching.js
@@ -86,7 +86,7 @@ class Watching {
 
 	_go(fileTimeInfoEntries, contextTimeInfoEntries, changedFiles, removedFiles) {
 		this._initial = false;
-		this.startTime = Date.now();
+		if (this.startTime === null) this.startTime = Date.now();
 		this.running = true;
 		if (this.watcher) {
 			this.pausedWatcher = this.watcher;
@@ -252,6 +252,7 @@ class Watching {
 			compilation.endTime = Date.now();
 			stats = new Stats(compilation);
 		}
+		this.startTime = null;
 		if (err) return handleError(err);
 
 		const cbs = this.callbacks;

diff --git a/lib/WebpackOptionsApply.js b/lib/WebpackOptionsApply.js
@@ -573,7 +573,8 @@ class WebpackOptionsApply extends OptionsApply {
 									allowCollectingMemory: cacheOptions.allowCollectingMemory
 								}),
 								cacheOptions.idleTimeout,
-								cacheOptions.idleTimeoutForInitialStore
+								cacheOptions.idleTimeoutForInitialStore,
+								cacheOptions.idleTimeoutAfterLargeChanges
 							).apply(compiler);
 							break;
 						}

diff --git a/lib/cache/IdleFileCachePlugin.js b/lib/cache/IdleFileCachePlugin.js
@@ -17,11 +17,18 @@ class IdleFileCachePlugin {
 	 * @param {TODO} strategy cache strategy
 	 * @param {number} idleTimeout timeout
 	 * @param {number} idleTimeoutForInitialStore initial timeout
+	 * @param {number} idleTimeoutAfterLargeChanges timeout after changes
 	 */
-	constructor(strategy, idleTimeout, idleTimeoutForInitialStore) {
+	constructor(
+		strategy,
+		idleTimeout,
+		idleTimeoutForInitialStore,
+		idleTimeoutAfterLargeChanges
+	) {
 		this.strategy = strategy;
 		this.idleTimeout = idleTimeout;
 		this.idleTimeoutForInitialStore = idleTimeoutForInitialStore;
+		this.idleTimeoutAfterLargeChanges = idleTimeoutAfterLargeChanges;
 	}
 
 	/**
@@ -36,9 +43,13 @@ class IdleFileCachePlugin {
 			idleTimeout,
 			this.idleTimeoutForInitialStore
 		);
-
+		const idleTimeoutAfterLargeChanges = this.idleTimeoutAfterLargeChanges;
 		const resolvedPromise = Promise.resolve();
 
+		let timeSpendInBuild = 0;
+		let timeSpendInStore = 0;
+		let avgTimeSpendInStore = 0;
+
 		/** @type {Map<string | typeof BUILD_DEPENDENCIES_KEY, () => Promise>} */
 		const pendingIdleTasks = new Map();
 
@@ -121,9 +132,10 @@ class IdleFileCachePlugin {
 		let isInitialStore = true;
 		const processIdleTasks = () => {
 			if (isIdle) {
+				const startTime = Date.now();
 				if (pendingIdleTasks.size > 0) {
 					const promises = [currentIdlePromise];
-					const maxTime = Date.now() + 100;
+					const maxTime = startTime + 100;
 					let maxCount = 100;
 					for (const [filename, factory] of pendingIdleTasks) {
 						pendingIdleTasks.delete(filename);
@@ -132,13 +144,23 @@ class IdleFileCachePlugin {
 					}
 					currentIdlePromise = Promise.all(promises);
 					currentIdlePromise.then(() => {
+						timeSpendInStore += Date.now() - startTime;
 						// Allow to exit the process between
-						setTimeout(processIdleTasks, 0).unref();
+						idleTimer = setTimeout(processIdleTasks, 0);
+						idleTimer.unref();
 					});
 					return;
 				}
 				currentIdlePromise = currentIdlePromise
-					.then(() => strategy.afterAllStored())
+					.then(async () => {
+						await strategy.afterAllStored();
+						timeSpendInStore += Date.now() - startTime;
+						avgTimeSpendInStore =
+							Math.max(avgTimeSpendInStore, timeSpendInStore) * 0.9 +
+							timeSpendInStore * 0.1;
+						timeSpendInStore = 0;
+						timeSpendInBuild = 0;
+					})
 					.catch(err => {
 						const logger = compiler.getInfrastructureLogger(
 							"IdleFileCachePlugin"
@@ -153,14 +175,34 @@ class IdleFileCachePlugin {
 		compiler.cache.hooks.beginIdle.tap(
 			{ name: "IdleFileCachePlugin", stage: Cache.STAGE_DISK },
 			() => {
-				idleTimer = setTimeout(
-					() => {
-						idleTimer = undefined;
-						isIdle = true;
-						resolvedPromise.then(processIdleTasks);
-					},
-					isInitialStore ? idleTimeoutForInitialStore : idleTimeout
-				);
+				const isLargeChange = timeSpendInBuild > avgTimeSpendInStore * 2;
+				if (isInitialStore && idleTimeoutForInitialStore < idleTimeout) {
+					compiler
+						.getInfrastructureLogger("IdleFileCachePlugin")
+						.log(
+							`Initial cache was generated and cache will be persisted in ${
+								idleTimeoutForInitialStore / 1000
+							}s.`
+						);
+				} else if (
+					isLargeChange &&
+					idleTimeoutAfterLargeChanges < idleTimeout
+				) {
+					compiler
+						.getInfrastructureLogger("IdleFileCachePlugin")
+						.log(
+							`Spend ${Math.round(timeSpendInBuild) / 1000}s in build and ${
+								Math.round(avgTimeSpendInStore) / 1000
+							}s in average in cache store. This is considered as large change and cache will be persisted in ${
+								idleTimeoutAfterLargeChanges / 1000
+							}s.`
+						);
+				}
+				idleTimer = setTimeout(() => {
+					idleTimer = undefined;
+					isIdle = true;
+					resolvedPromise.then(processIdleTasks);
+				}, Math.min(isInitialStore ? idleTimeoutForInitialStore : Infinity, isLargeChange ? idleTimeoutAfterLargeChanges : Infinity, idleTimeout));
 				idleTimer.unref();
 			}
 		);
@@ -174,6 +216,11 @@ class IdleFileCachePlugin {
 				isIdle = false;
 			}
 		);
+		compiler.hooks.done.tap("IdleFileCachePlugin", stats => {
+			// 10% build overhead is ignored, as it's not cacheable
+			timeSpendInBuild *= 0.9;
+			timeSpendInBuild += stats.endTime - stats.startTime;
+		});
 	}
 }
 

diff --git a/lib/config/defaults.js b/lib/config/defaults.js
@@ -295,7 +295,8 @@ const applyCacheDefaults = (cache, { name, mode, development }) => {
 			D(cache, "store", "pack");
 			D(cache, "profile", false);
 			D(cache, "idleTimeout", 60000);
-			D(cache, "idleTimeoutForInitialStore", 0);
+			D(cache, "idleTimeoutForInitialStore", 5000);
+			D(cache, "idleTimeoutAfterLargeChanges", 1000);
 			D(cache, "maxMemoryGenerations", development ? 5 : Infinity);
 			D(cache, "maxAge", 1000 * 60 * 60 * 24 * 60); // 1 month
 			D(cache, "allowCollectingMemory", development);

diff --git a/lib/config/normalization.js b/lib/config/normalization.js
@@ -139,6 +139,7 @@ const getNormalizedWebpackOptions = config => {
 						hashAlgorithm: cache.hashAlgorithm,
 						idleTimeout: cache.idleTimeout,
 						idleTimeoutForInitialStore: cache.idleTimeoutForInitialStore,
+						idleTimeoutAfterLargeChanges: cache.idleTimeoutAfterLargeChanges,
 						name: cache.name,
 						store: cache.store,
 						version: cache.version

diff --git a/schemas/WebpackOptions.check.js b/schemas/WebpackOptions.check.js
diff --git a/schemas/WebpackOptions.json b/schemas/WebpackOptions.json
@@ -956,12 +956,17 @@
           "type": "string"
         },
         "idleTimeout": {
-          "description": "Time in ms after which idle period the cache storing should happen (only for store: 'pack').",
+          "description": "Time in ms after which idle period the cache storing should happen.",
+          "type": "number",
+          "minimum": 0
+        },
+        "idleTimeoutAfterLargeChanges": {
+          "description": "Time in ms after which idle period the cache storing should happen when larger changes has been detected (cumulative build time > 2 x avg cache store time).",
           "type": "number",
           "minimum": 0
         },
         "idleTimeoutForInitialStore": {
-          "description": "Time in ms after which idle period the initial cache storing should happen (only for store: 'pack').",
+          "description": "Time in ms after which idle period the initial cache storing should happen.",
           "type": "number",
           "minimum": 0
         },

diff --git a/test/Defaults.unittest.js b/test/Defaults.unittest.js
@@ -1502,7 +1502,8 @@ describe("Defaults", () => {
 		+     "cacheLocation": "<cwd>/node_modules/.cache/webpack/default-none",
 		+     "hashAlgorithm": "md4",
 		+     "idleTimeout": 60000,
-		+     "idleTimeoutForInitialStore": 0,
+		+     "idleTimeoutAfterLargeChanges": 1000,
+		+     "idleTimeoutForInitialStore": 5000,
 		+     "maxAge": 5184000000,
 		+     "maxMemoryGenerations": Infinity,
 		+     "name": "default-none",
@@ -1543,7 +1544,8 @@ describe("Defaults", () => {
 			+     "cacheLocation": "<cwd>/node_modules/.cache/webpack/default-development",
 			+     "hashAlgorithm": "md4",
 			+     "idleTimeout": 60000,
-			+     "idleTimeoutForInitialStore": 0,
+			+     "idleTimeoutAfterLargeChanges": 1000,
+			+     "idleTimeoutForInitialStore": 5000,
 			+     "maxAge": 5184000000,
 			+     "maxMemoryGenerations": 5,
 			+     "name": "default-development",
@@ -1789,7 +1791,8 @@ describe("Defaults", () => {
 			+     "cacheLocation": "<cwd>/node_modules/.cache/webpack/default-none",
 			+     "hashAlgorithm": "md4",
 			+     "idleTimeout": 60000,
-			+     "idleTimeoutForInitialStore": 0,
+			+     "idleTimeoutAfterLargeChanges": 1000,
+			+     "idleTimeoutForInitialStore": 5000,
 			+     "maxAge": 5184000000,
 			+     "maxMemoryGenerations": Infinity,
 			+     "name": "default-none",

diff --git a/test/__snapshots__/Cli.test.js.snap b/test/__snapshots__/Cli.test.js.snap
@@ -111,26 +111,39 @@ Object {
   "cache-idle-timeout": Object {
     "configs": Array [
       Object {
-        "description": "Time in ms after which idle period the cache storing should happen (only for store: 'pack').",
+        "description": "Time in ms after which idle period the cache storing should happen.",
         "multiple": false,
         "path": "cache.idleTimeout",
         "type": "number",
       },
     ],
-    "description": "Time in ms after which idle period the cache storing should happen (only for store: 'pack').",
+    "description": "Time in ms after which idle period the cache storing should happen.",
+    "multiple": false,
+    "simpleType": "number",
+  },
+  "cache-idle-timeout-after-large-changes": Object {
+    "configs": Array [
+      Object {
+        "description": "Time in ms after which idle period the cache storing should happen when larger changes has been detected (cumulative build time > 2 x avg cache store time).",
+        "multiple": false,
+        "path": "cache.idleTimeoutAfterLargeChanges",
+        "type": "number",
+      },
+    ],
+    "description": "Time in ms after which idle period the cache storing should happen when larger changes has been detected (cumulative build time > 2 x avg cache store time).",
     "multiple": false,
     "simpleType": "number",
   },
   "cache-idle-timeout-for-initial-store": Object {
     "configs": Array [
       Object {
-        "description": "Time in ms after which idle period the initial cache storing should happen (only for store: 'pack').",
+        "description": "Time in ms after which idle period the initial cache storing should happen.",
         "multiple": false,
         "path": "cache.idleTimeoutForInitialStore",
         "type": "number",
       },
     ],
-    "description": "Time in ms after which idle period the initial cache storing should happen (only for store: 'pack').",
+    "description": "Time in ms after which idle period the initial cache storing should happen.",
     "multiple": false,
     "simpleType": "number",
   },

diff --git a/types.d.ts b/types.d.ts
@@ -3785,12 +3785,17 @@ declare interface FileCacheOptions {
 	hashAlgorithm?: string;
 
 	/**
-	 * Time in ms after which idle period the cache storing should happen (only for store: 'pack').
+	 * Time in ms after which idle period the cache storing should happen.
 	 */
 	idleTimeout?: number;
 
 	/**
-	 * Time in ms after which idle period the initial cache storing should happen (only for store: 'pack').
+	 * Time in ms after which idle period the cache storing should happen when larger changes has been detected (cumulative build time > 2 x avg cache store time).
+	 */
+	idleTimeoutAfterLargeChanges?: number;
+
+	/**
+	 * Time in ms after which idle period the initial cache storing should happen.
 	 */
 	idleTimeoutForInitialStore?: number;