diff --git a/lib/FileSystemInfo.js b/lib/FileSystemInfo.js index 80609c152c2..6467a65c727 100644 --- a/lib/FileSystemInfo.js +++ b/lib/FileSystemInfo.js @@ -10,12 +10,13 @@ const asyncLib = require("neo-async"); const AsyncQueue = require("./util/AsyncQueue"); const StackedCacheMap = require("./util/StackedCacheMap"); const createHash = require("./util/createHash"); -const { join, dirname, relative } = require("./util/fs"); +const { join, dirname, relative, lstatReadlinkAbsolute } = require("./util/fs"); const makeSerializable = require("./util/makeSerializable"); const processAsyncTree = require("./util/processAsyncTree"); /** @typedef {import("./WebpackError")} WebpackError */ /** @typedef {import("./logging/Logger").Logger} Logger */ +/** @typedef {import("./util/fs").IStats} IStats */ /** @typedef {import("./util/fs").InputFileSystem} InputFileSystem */ const supportsEsm = +process.versions.modules >= 83; @@ -41,17 +42,52 @@ const INVALID = Symbol("invalid"); * @typedef {Object} FileSystemInfoEntry * @property {number} safeTime * @property {number=} timestamp + */ + +/** + * @typedef {Object} ResolvedContextFileSystemInfoEntry + * @property {number} safeTime + * @property {string=} timestampHash + */ + +/** + * @typedef {Object} ContextFileSystemInfoEntry + * @property {number} safeTime * @property {string=} timestampHash + * @property {ResolvedContextFileSystemInfoEntry=} resolved + * @property {Set=} symlinks */ /** * @typedef {Object} TimestampAndHash * @property {number} safeTime * @property {number=} timestamp + * @property {string} hash + */ + +/** + * @typedef {Object} ResolvedContextTimestampAndHash + * @property {number} safeTime * @property {string=} timestampHash * @property {string} hash */ +/** + * @typedef {Object} ContextTimestampAndHash + * @property {number} safeTime + * @property {string=} timestampHash + * @property {string} hash + * @property {ResolvedContextTimestampAndHash=} resolved + * @property {Set=} symlinks + */ + +/** + * @typedef {Object} ContextHash + * @property {string} hash + * @property {string=} resolved + * @property {Set=} symlinks + */ + /** * @typedef {Object} SnapshotOptimizationEntry * @property {Snapshot} snapshot @@ -175,11 +211,11 @@ class Snapshot { this.fileHashes = undefined; /** @type {Map | undefined} */ this.fileTshs = undefined; - /** @type {Map | undefined} */ + /** @type {Map | undefined} */ this.contextTimestamps = undefined; /** @type {Map | undefined} */ this.contextHashes = undefined; - /** @type {Map | undefined} */ + /** @type {Map | undefined} */ this.contextTshs = undefined; /** @type {Map | undefined} */ this.missingExistence = undefined; @@ -771,11 +807,27 @@ const getManagedItem = (managedPath, path) => { }; /** - * @param {FileSystemInfoEntry} entry file system info entry - * @returns {boolean} existence flag + * @template {ContextFileSystemInfoEntry | ContextTimestampAndHash} T + * @param {T | "ignore"} entry entry + * @returns {T["resolved"] | undefined} the resolved entry */ -const toExistence = entry => { - return Boolean(entry); +const getResolvedTimestamp = entry => { + if (entry === "ignore") return undefined; + if (entry.resolved !== undefined) return entry.resolved; + return entry.symlinks === undefined ? entry : undefined; +}; + +/** + * @param {ContextHash} entry entry + * @returns {string | undefined} the resolved entry + */ +const getResolvedHash = entry => { + if (entry.resolved !== undefined) return entry.resolved; + return entry.symlinks === undefined ? entry.hash : undefined; +}; + +const addAll = (source, target) => { + for (const key of source) target.add(key); }; /** @@ -860,11 +912,11 @@ class FileSystemInfo { this._fileHashes = new Map(); /** @type {Map} */ this._fileTshs = new Map(); - /** @type {StackedCacheMap} */ + /** @type {StackedCacheMap} */ this._contextTimestamps = new StackedCacheMap(); - /** @type {Map} */ + /** @type {Map} */ this._contextHashes = new Map(); - /** @type {Map} */ + /** @type {Map} */ this._contextTshs = new Map(); /** @type {Map} */ this._managedItems = new Map(); @@ -880,18 +932,24 @@ class FileSystemInfo { parallelism: 10, processor: this._readFileHash.bind(this) }); - /** @type {AsyncQueue} */ + /** @type {AsyncQueue} */ this.contextTimestampQueue = new AsyncQueue({ name: "context timestamp", parallelism: 2, processor: this._readContextTimestamp.bind(this) }); - /** @type {AsyncQueue} */ + /** @type {AsyncQueue} */ this.contextHashQueue = new AsyncQueue({ name: "context hash", parallelism: 2, processor: this._readContextHash.bind(this) }); + /** @type {AsyncQueue} */ + this.contextTshQueue = new AsyncQueue({ + name: "context hash and timestamp", + parallelism: 2, + processor: this._readContextTimestampAndHash.bind(this) + }); /** @type {AsyncQueue} */ this.managedItemQueue = new AsyncQueue({ name: "managed item info", @@ -1093,13 +1151,34 @@ class FileSystemInfo { /** * @param {string} path context path - * @param {function(WebpackError=, (FileSystemInfoEntry | "ignore" | null)=): void} callback callback function + * @param {function(WebpackError=, (ResolvedContextFileSystemInfoEntry | "ignore" | null)=): void} callback callback function * @returns {void} */ getContextTimestamp(path, callback) { + const cache = this._contextTimestamps.get(path); + if (cache !== undefined) { + const resolved = getResolvedTimestamp(cache); + if (resolved !== undefined) return callback(null, resolved); + this._resolveContextTimestamp(cache, callback); + } + this.contextTimestampQueue.add(path, (err, entry) => { + const resolved = getResolvedTimestamp(entry); + if (resolved !== undefined) return callback(null, resolved); + this._resolveContextTimestamp(entry, callback); + }); + } + + /** + * @param {string} path context path + * @param {function(WebpackError=, (ContextFileSystemInfoEntry | "ignore" | null)=): void} callback callback function + * @returns {void} + */ + _getUnresolvedContextTimestamp(path, callback) { const cache = this._contextTimestamps.get(path); if (cache !== undefined) return callback(null, cache); - this.contextTimestampQueue.add(path, callback); + this.contextTimestampQueue.add(path, (err, entry) => { + return callback(null, entry); + }); } /** @@ -1120,8 +1199,61 @@ class FileSystemInfo { */ getContextHash(path, callback) { const cache = this._contextHashes.get(path); + if (cache !== undefined) { + const resolved = getResolvedHash(cache); + if (resolved !== undefined) return callback(null, resolved); + this._resolveContextHash(cache, callback); + } + this.contextHashQueue.add(path, (err, entry) => { + const resolved = getResolvedHash(entry); + if (resolved !== undefined) return callback(null, resolved); + this._resolveContextHash(entry, callback); + }); + } + + /** + * @param {string} path context path + * @param {function(WebpackError=, ContextHash=): void} callback callback function + * @returns {void} + */ + _getUnresolvedContextHash(path, callback) { + const cache = this._contextHashes.get(path); + if (cache !== undefined) return callback(null, cache); + this.contextHashQueue.add(path, (err, entry) => { + return callback(null, entry); + }); + } + + /** + * @param {string} path context path + * @param {function(WebpackError=, ResolvedContextTimestampAndHash=): void} callback callback function + * @returns {void} + */ + getContextTsh(path, callback) { + const cache = this._contextTshs.get(path); + if (cache !== undefined) { + const resolved = getResolvedTimestamp(cache); + if (resolved !== undefined) return callback(null, resolved); + this._resolveContextTsh(cache, callback); + } + this.contextTshQueue.add(path, (err, entry) => { + const resolved = getResolvedTimestamp(entry); + if (resolved !== undefined) return callback(null, resolved); + this._resolveContextTsh(entry, callback); + }); + } + + /** + * @param {string} path context path + * @param {function(WebpackError=, ContextTimestampAndHash=): void} callback callback function + * @returns {void} + */ + _getUnresolvedContextTsh(path, callback) { + const cache = this._contextTshs.get(path); if (cache !== undefined) return callback(null, cache); - this.contextHashQueue.add(path, callback); + this.contextTshQueue.add(path, (err, entry) => { + return callback(null, entry); + }); } _createBuildDependenciesResolvers() { @@ -2007,11 +2139,15 @@ class FileSystemInfo { ); for (const path of capturedDirectories) { const cache = this._contextTshs.get(path); - if (cache !== undefined) { - contextTshs.set(path, cache); + let resolved; + if ( + cache !== undefined && + (resolved = getResolvedTimestamp(cache)) !== undefined + ) { + contextTshs.set(path, resolved); } else { jobs++; - this._getContextTimestampAndHash(path, (err, entry) => { + const callback = (err, entry) => { if (err) { if (this.logger) { this.logger.debug( @@ -2023,7 +2159,12 @@ class FileSystemInfo { contextTshs.set(path, entry); jobDone(); } - }); + }; + if (cache !== undefined) { + this._resolveContextTsh(cache, callback); + } else { + this.getContextTsh(path, callback); + } } } break; @@ -2035,11 +2176,15 @@ class FileSystemInfo { ); for (const path of capturedDirectories) { const cache = this._contextHashes.get(path); - if (cache !== undefined) { - contextHashes.set(path, cache); + let resolved; + if ( + cache !== undefined && + (resolved = getResolvedHash(cache)) !== undefined + ) { + contextHashes.set(path, resolved); } else { jobs++; - this.contextHashQueue.add(path, (err, entry) => { + const callback = (err, entry) => { if (err) { if (this.logger) { this.logger.debug( @@ -2051,7 +2196,12 @@ class FileSystemInfo { contextHashes.set(path, entry); jobDone(); } - }); + }; + if (cache !== undefined) { + this._resolveContextHash(cache, callback); + } else { + this.getContextHash(path, callback); + } } } break; @@ -2064,13 +2214,15 @@ class FileSystemInfo { ); for (const path of capturedDirectories) { const cache = this._contextTimestamps.get(path); - if (cache !== undefined) { - if (cache !== "ignore") { - contextTimestamps.set(path, cache); - } - } else { + let resolved; + if ( + cache !== undefined && + (resolved = getResolvedTimestamp(cache)) !== undefined + ) { + contextTimestamps.set(path, resolved); + } else if (cache !== "ignore") { jobs++; - this.contextTimestampQueue.add(path, (err, entry) => { + const callback = (err, entry) => { if (err) { if (this.logger) { this.logger.debug( @@ -2082,7 +2234,12 @@ class FileSystemInfo { contextTimestamps.set(path, entry); jobDone(); } - }); + }; + if (cache !== undefined) { + this._resolveContextTimestamp(cache, callback); + } else { + this.getContextTimestamp(path, callback); + } } } break; @@ -2099,7 +2256,7 @@ class FileSystemInfo { const cache = this._fileTimestamps.get(path); if (cache !== undefined) { if (cache !== "ignore") { - missingExistence.set(path, toExistence(cache)); + missingExistence.set(path, Boolean(cache)); } } else { jobs++; @@ -2112,7 +2269,7 @@ class FileSystemInfo { } jobError(); } else { - missingExistence.set(path, toExistence(entry)); + missingExistence.set(path, Boolean(entry)); jobDone(); } }); @@ -2321,17 +2478,7 @@ class FileSystemInfo { */ const checkFile = (path, current, snap, log = true) => { if (current === snap) return true; - if (!current !== !snap) { - // If existence of item differs - // it's invalid - if (log && this._remainingLogs > 0) { - this._log( - path, - current ? "it didn't exist before" : "it does no longer exist" - ); - } - return false; - } + if (!checkExistence(path, Boolean(current), Boolean(snap))) return false; if (current) { // For existing items only if (typeof startTime === "number" && current.safeTime > startTime) { @@ -2363,6 +2510,34 @@ class FileSystemInfo { } return false; } + } + return true; + }; + /** + * @param {string} path file path + * @param {ResolvedContextFileSystemInfoEntry} current current entry + * @param {ResolvedContextFileSystemInfoEntry} snap entry from snapshot + * @param {boolean} log log reason + * @returns {boolean} true, if ok + */ + const checkContext = (path, current, snap, log = true) => { + if (current === snap) return true; + if (!checkExistence(path, Boolean(current), Boolean(snap))) return false; + if (current) { + // For existing items only + if (typeof startTime === "number" && current.safeTime > startTime) { + // If a change happened after starting reading the item + // this may no longer be valid + if (log && this._remainingLogs > 0) { + this._log( + path, + `it may have changed (%d) after the start time of the snapshot (%d)`, + current.safeTime, + startTime + ); + } + return false; + } if ( snap.timestampHash !== undefined && current.timestampHash !== snap.timestampHash @@ -2487,41 +2662,59 @@ class FileSystemInfo { this._statTestedEntries += contextTimestamps.size; for (const [path, ts] of contextTimestamps) { const cache = this._contextTimestamps.get(path); - if (cache !== undefined) { - if (cache !== "ignore" && !checkFile(path, cache, ts)) { + let resolved; + if ( + cache !== undefined && + (resolved = getResolvedTimestamp(cache)) !== undefined + ) { + if (!checkContext(path, resolved, ts)) { invalid(); return; } - } else { + } else if (cache !== "ignore") { jobs++; - this.contextTimestampQueue.add(path, (err, entry) => { + const callback = (err, entry) => { if (err) return invalidWithError(path, err); - if (!checkFile(path, entry, ts)) { + if (!checkContext(path, entry, ts)) { invalid(); } else { jobDone(); } - }); + }; + if (cache !== undefined) { + this._resolveContextTimestamp(cache, callback); + } else { + this.getContextTimestamp(path, callback); + } } } } const processContextHashSnapshot = (path, hash) => { const cache = this._contextHashes.get(path); - if (cache !== undefined) { - if (cache !== "ignore" && !checkHash(path, cache, hash)) { + let resolved; + if ( + cache !== undefined && + (resolved = getResolvedHash(cache)) !== undefined + ) { + if (!checkHash(path, resolved, hash)) { invalid(); return; } } else { jobs++; - this.contextHashQueue.add(path, (err, entry) => { + const callback = (err, entry) => { if (err) return invalidWithError(path, err); if (!checkHash(path, entry, hash)) { invalid(); } else { jobDone(); } - }); + }; + if (cache !== undefined) { + this._resolveContextHash(cache, callback); + } else { + this.getContextHash(path, callback); + } } }; if (snapshot.hasContextHashes()) { @@ -2539,19 +2732,28 @@ class FileSystemInfo { processContextHashSnapshot(path, tsh); } else { const cache = this._contextTimestamps.get(path); - if (cache !== undefined) { - if (cache === "ignore" || !checkFile(path, cache, tsh, false)) { + let resolved; + if ( + cache !== undefined && + (resolved = getResolvedTimestamp(cache)) !== undefined + ) { + if (!checkContext(path, resolved, tsh, false)) { processContextHashSnapshot(path, tsh.hash); } - } else { + } else if (cache !== "ignore") { jobs++; - this.contextTimestampQueue.add(path, (err, entry) => { + const callback = (err, entry) => { if (err) return invalidWithError(path, err); - if (!checkFile(path, entry, tsh, false)) { + if (!checkContext(path, entry, tsh, false)) { processContextHashSnapshot(path, tsh.hash); } jobDone(); - }); + }; + if (cache !== undefined) { + this._resolveContextTsh(cache, callback); + } else { + this.getContextTsh(path, callback); + } } } } @@ -2564,7 +2766,7 @@ class FileSystemInfo { if (cache !== undefined) { if ( cache !== "ignore" && - !checkExistence(path, toExistence(cache), existence) + !checkExistence(path, Boolean(cache), Boolean(existence)) ) { invalid(); return; @@ -2573,7 +2775,7 @@ class FileSystemInfo { jobs++; this.fileTimestampQueue.add(path, (err, entry) => { if (err) return invalidWithError(path, err); - if (!checkExistence(path, toExistence(entry), existence)) { + if (!checkExistence(path, Boolean(entry), Boolean(existence))) { invalid(); } else { jobDone(); @@ -2727,12 +2929,34 @@ class FileSystemInfo { } } - _readContextTimestamp(path, callback) { + /** + * @template T + * @template ItemType + * @param {Object} options options + * @param {string} options.path path + * @param {function(string): ItemType} options.fromImmutablePath called when context item is an immutable path + * @param {function(string): ItemType} options.fromManagedItem called when context item is a managed path + * @param {function(string, string, function(Error=, ItemType=): void): void} options.fromSymlink called when context item is a symlink + * @param {function(string, IStats, function(Error=, ItemType=): void): void} options.fromFile called when context item is a file + * @param {function(string, IStats, function(Error=, ItemType=): void): void} options.fromDirectory called when context item is a directory + * @param {function(string[], ItemType[]): T} options.reduce called from all context items + * @param {function(Error=, (T)=): void} callback callback + */ + _readContext( + { + path, + fromImmutablePath, + fromManagedItem, + fromSymlink, + fromFile, + fromDirectory, + reduce + }, + callback + ) { this.fs.readdir(path, (err, _files) => { if (err) { if (err.code === "ENOENT") { - this._contextTimestamps.set(path, null); - this._cachedDeprecatedContextTimestamps = undefined; return callback(null, null); } return callback(err); @@ -2745,47 +2969,94 @@ class FileSystemInfo { files, (file, callback) => { const child = join(this.fs, path, file); - this.fs.stat(child, (err, stat) => { - if (err) return callback(err); - - for (const immutablePath of this.immutablePathsWithSlash) { - if (path.startsWith(immutablePath)) { - // ignore any immutable path for timestamping - return callback(null, null); - } + for (const immutablePath of this.immutablePathsWithSlash) { + if (path.startsWith(immutablePath)) { + // ignore any immutable path for timestamping + return callback(null, fromImmutablePath(immutablePath)); } - for (const managedPath of this.managedPathsWithSlash) { - if (path.startsWith(managedPath)) { - const managedItem = getManagedItem(managedPath, child); - if (managedItem) { - // construct timestampHash from managed info - return this.managedItemQueue.add(managedItem, (err, info) => { - if (err) return callback(err); - return callback(null, { - safeTime: 0, - timestampHash: info - }); - }); - } + } + for (const managedPath of this.managedPathsWithSlash) { + if (path.startsWith(managedPath)) { + const managedItem = getManagedItem(managedPath, child); + if (managedItem) { + // construct timestampHash from managed info + return this.managedItemQueue.add(managedItem, (err, info) => { + if (err) return callback(err); + return callback(null, fromManagedItem(info)); + }); } } + } + + lstatReadlinkAbsolute(this.fs, child, (err, stat) => { + if (err) return callback(err); + + if (typeof stat === "string") { + return fromSymlink(child, stat, callback); + } if (stat.isFile()) { - return this.getFileTimestamp(child, callback); + return fromFile(child, stat, callback); } if (stat.isDirectory()) { - this.contextTimestampQueue.increaseParallelism(); - this.getContextTimestamp(child, (err, tsEntry) => { - this.contextTimestampQueue.decreaseParallelism(); - callback(err, tsEntry); - }); - return; + return fromDirectory(child, stat, callback); } callback(null, null); }); }, - (err, tsEntries) => { + (err, results) => { if (err) return callback(err); + const result = reduce(files, results); + callback(null, result); + } + ); + }); + } + + _readContextTimestamp(path, callback) { + this._readContext( + { + path, + fromImmutablePath: () => null, + fromManagedItem: info => ({ + safeTime: 0, + timestampHash: info + }), + fromSymlink: (file, target, callback) => { + callback(null, { + timestampHash: target, + symlinks: new Set([target]) + }); + }, + fromFile: (file, stat, callback) => { + // Prefer the cached value over our new stat to report consistent results + const cache = this._fileTimestamps.get(file); + if (cache !== undefined) + return callback(null, cache === "ignore" ? null : cache); + + const mtime = +stat.mtime; + + if (mtime) applyMtime(mtime); + + const ts = { + safeTime: mtime ? mtime + FS_ACCURACY : Infinity, + timestamp: mtime + }; + + this._fileTimestamps.set(file, ts); + this._cachedDeprecatedFileTimestamps = undefined; + callback(null, ts); + }, + fromDirectory: (directory, stat, callback) => { + this.contextTimestampQueue.increaseParallelism(); + this._getUnresolvedContextTimestamp(directory, (err, tsEntry) => { + this.contextTimestampQueue.decreaseParallelism(); + callback(err, tsEntry); + }); + }, + reduce: (files, tsEntries) => { + let symlinks = undefined; + const hash = createHash("md4"); for (const file of files) hash.update(file); @@ -2802,6 +3073,10 @@ class FileSystemInfo { hash.update("d"); hash.update(`${entry.timestampHash}`); } + if (entry.symlinks !== undefined) { + if (symlinks === undefined) symlinks = new Set(); + addAll(entry.symlinks, symlinks); + } if (entry.safeTime) { safeTime = Math.max(safeTime, entry.safeTime); } @@ -2813,131 +3088,326 @@ class FileSystemInfo { safeTime, timestampHash: digest }; - - this._contextTimestamps.set(path, result); - this._cachedDeprecatedContextTimestamps = undefined; - - callback(null, result); + if (symlinks) result.symlinks = symlinks; + return result; } - ); - }); - } + }, + (err, result) => { + if (err) return callback(err); + this._contextTimestamps.set(path, result); + this._cachedDeprecatedContextTimestamps = undefined; - _readContextHash(path, callback) { - this.fs.readdir(path, (err, _files) => { - if (err) { - if (err.code === "ENOENT") { - this._contextHashes.set(path, null); - return callback(null, null); - } - return callback(err); + callback(null, result); } - const files = /** @type {string[]} */ (_files) - .map(file => file.normalize("NFC")) - .filter(file => !/^\./.test(file)) - .sort(); - asyncLib.map( - files, - (file, callback) => { - const child = join(this.fs, path, file); - this.fs.stat(child, (err, stat) => { - if (err) return callback(err); + ); + } - for (const immutablePath of this.immutablePathsWithSlash) { - if (path.startsWith(immutablePath)) { - // ignore any immutable path for hashing - return callback(null, ""); - } + _resolveContextTimestamp(entry, callback) { + const hashes = []; + let safeTime = 0; + processAsyncTree( + entry.symlinks, + 10, + (target, push, callback) => { + this._getUnresolvedContextTimestamp(target, (err, entry) => { + if (err) return callback(err); + if (entry && entry !== "ignore") { + hashes.push(entry.timestampHash); + if (entry.safeTime) { + safeTime = Math.max(safeTime, entry.safeTime); } - for (const managedPath of this.managedPathsWithSlash) { - if (path.startsWith(managedPath)) { - const managedItem = getManagedItem(managedPath, child); - if (managedItem) { - // construct hash from managed info - return this.managedItemQueue.add(managedItem, (err, info) => { - if (err) return callback(err); - callback(null, info || ""); - }); - } - } + if (entry.symlinks !== undefined) { + for (const target of entry.symlinks) push(target); } + } + callback(); + }); + }, + err => { + if (err) return callback(err); + const hash = createHash("md4"); + hash.update(entry.timestampHash); + if (entry.safeTime) { + safeTime = Math.max(safeTime, entry.safeTime); + } + hashes.sort(); + for (const h of hashes) { + hash.update(h); + } + callback( + null, + (entry.resolved = { + safeTime, + timestampHash: /** @type {string} */ (hash.digest("hex")) + }) + ); + } + ); + } - if (stat.isFile()) { - return this.getFileHash(child, (err, hash) => { - callback(err, hash || ""); - }); - } - if (stat.isDirectory()) { - this.contextHashQueue.increaseParallelism(); - this.getContextHash(child, (err, hash) => { - this.contextHashQueue.decreaseParallelism(); - callback(err, hash || ""); - }); - return; - } - callback(null, ""); + _readContextHash(path, callback) { + this._readContext( + { + path, + fromImmutablePath: () => "", + fromManagedItem: info => info || "", + fromSymlink: (file, target, callback) => { + callback(null, { + hash: target, + symlinks: new Set([target]) }); }, - (err, fileHashes) => { - if (err) return callback(err); + fromFile: (file, stat, callback) => + this.getFileHash(file, (err, hash) => { + callback(err, hash || ""); + }), + fromDirectory: (directory, stat, callback) => { + this.contextHashQueue.increaseParallelism(); + this._getUnresolvedContextHash(directory, (err, hash) => { + this.contextHashQueue.decreaseParallelism(); + callback(err, hash || ""); + }); + }, + /** + * @param {string[]} files files + * @param {(string | ContextHash)[]} fileHashes hashes + * @returns {ContextHash} reduced hash + */ + reduce: (files, fileHashes) => { + let symlinks = undefined; const hash = createHash("md4"); for (const file of files) hash.update(file); - for (const h of fileHashes) hash.update(h); - - const digest = /** @type {string} */ (hash.digest("hex")); - - this._contextHashes.set(path, digest); + for (const entry of fileHashes) { + if (typeof entry === "string") { + hash.update(entry); + } else { + hash.update(entry.hash); + if (entry.symlinks) { + if (symlinks === undefined) symlinks = new Set(); + addAll(entry.symlinks, symlinks); + } + } + } - callback(null, digest); + const result = { + hash: /** @type {string} */ (hash.digest("hex")) + }; + if (symlinks) result.symlinks = symlinks; + return result; } - ); - }); + }, + (err, result) => { + if (err) return callback(err); + this._contextHashes.set(path, result); + return callback(null, result); + } + ); } - _getContextTimestampAndHash(path, callback) { - const continueWithHash = hash => { - const cache = this._contextTimestamps.get(path); - if (cache !== undefined) { - if (cache !== "ignore") { - const result = { - ...cache, - hash - }; - this._contextTshs.set(path, result); - return callback(null, result); - } else { - this._contextTshs.set(path, hash); - return callback(null, hash); + _resolveContextHash(entry, callback) { + const hashes = []; + processAsyncTree( + entry.symlinks, + 10, + (target, push, callback) => { + this._getUnresolvedContextHash(target, (err, hash) => { + if (err) return callback(err); + if (hash) { + hashes.push(hash.hash); + if (hash.symlinks !== undefined) { + for (const target of hash.symlinks) push(target); + } + } + callback(); + }); + }, + err => { + if (err) return callback(err); + const hash = createHash("md4"); + hash.update(entry.hash); + hashes.sort(); + for (const h of hashes) { + hash.update(h); } + callback( + null, + (entry.resolved = /** @type {string} */ (hash.digest("hex"))) + ); + } + ); + } + + _readContextTimestampAndHash(path, callback) { + const finalize = (timestamp, hash) => { + const result = + timestamp === "ignore" + ? hash + : { + ...timestamp, + ...hash + }; + this._contextTshs.set(path, result); + callback(null, result); + }; + const cachedHash = this._contextHashes.get(path); + const cachedTimestamp = this._contextTimestamps.get(path); + if (cachedHash !== undefined) { + if (cachedTimestamp !== undefined) { + finalize(cachedTimestamp, cachedHash); } else { this.contextTimestampQueue.add(path, (err, entry) => { - if (err) { - return callback(err); - } - const result = { - ...entry, - hash - }; - this._contextTshs.set(path, result); - return callback(null, result); + if (err) return callback(err); + finalize(entry, cachedHash); }); } - }; - - const cache = this._contextHashes.get(path); - if (cache !== undefined) { - continueWithHash(cache); } else { - this.contextHashQueue.add(path, (err, entry) => { - if (err) { - return callback(err); - } - continueWithHash(entry); - }); + if (cachedTimestamp !== undefined) { + this.contextHashQueue.add(path, (err, entry) => { + if (err) return callback(err); + finalize(cachedTimestamp, entry); + }); + } else { + this._readContext( + { + path, + fromImmutablePath: () => null, + fromManagedItem: info => ({ + safeTime: 0, + timestampHash: info, + hash: info || "" + }), + fromSymlink: (fle, target, callback) => { + callback(null, { + timestampHash: target, + hash: target, + symlinks: new Set([target]) + }); + }, + fromFile: (file, stat, callback) => { + this._getFileTimestampAndHash(file, callback); + }, + fromDirectory: (directory, stat, callback) => { + this.contextTshQueue.increaseParallelism(); + this.contextTshQueue.add(directory, (err, result) => { + this.contextTshQueue.decreaseParallelism(); + callback(err, result); + }); + }, + /** + * @param {string[]} files files + * @param {(Partial & Partial | string | null)[]} results results + * @returns {ContextTimestampAndHash} tsh + */ + reduce: (files, results) => { + let symlinks = undefined; + + const tsHash = createHash("md4"); + const hash = createHash("md4"); + + for (const file of files) { + tsHash.update(file); + hash.update(file); + } + let safeTime = 0; + for (const entry of results) { + if (!entry) { + tsHash.update("n"); + continue; + } + if (typeof entry === "string") { + tsHash.update("n"); + hash.update(entry); + continue; + } + if (entry.timestamp) { + tsHash.update("f"); + tsHash.update(`${entry.timestamp}`); + } else if (entry.timestampHash) { + tsHash.update("d"); + tsHash.update(`${entry.timestampHash}`); + } + if (entry.symlinks !== undefined) { + if (symlinks === undefined) symlinks = new Set(); + addAll(entry.symlinks, symlinks); + } + if (entry.safeTime) { + safeTime = Math.max(safeTime, entry.safeTime); + } + hash.update(entry.hash); + } + + const result = { + safeTime, + timestampHash: /** @type {string} */ (tsHash.digest("hex")), + hash: /** @type {string} */ (hash.digest("hex")) + }; + if (symlinks) result.symlinks = symlinks; + return result; + } + }, + (err, result) => { + if (err) return callback(err); + this._contextTshs.set(path, result); + return callback(null, result); + } + ); + } } } + _resolveContextTsh(entry, callback) { + const hashes = []; + const tsHashes = []; + let safeTime = 0; + processAsyncTree( + entry.symlinks, + 10, + (target, push, callback) => { + this._getUnresolvedContextTsh(target, (err, entry) => { + if (err) return callback(err); + if (entry) { + hashes.push(entry.hash); + if (entry.timestampHash) tsHashes.push(entry.timestampHash); + if (entry.safeTime) { + safeTime = Math.max(safeTime, entry.safeTime); + } + if (entry.symlinks !== undefined) { + for (const target of entry.symlinks) push(target); + } + } + callback(); + }); + }, + err => { + if (err) return callback(err); + const hash = createHash("md4"); + const tsHash = createHash("md4"); + hash.update(entry.hash); + if (entry.timestampHash) tsHash.update(entry.timestampHash); + if (entry.safeTime) { + safeTime = Math.max(safeTime, entry.safeTime); + } + hashes.sort(); + for (const h of hashes) { + hash.update(h); + } + tsHashes.sort(); + for (const h of tsHashes) { + tsHash.update(h); + } + callback( + null, + (entry.resolved = { + safeTime, + timestampHash: /** @type {string} */ (tsHash.digest("hex")), + hash: /** @type {string} */ (hash.digest("hex")) + }) + ); + } + ); + } + _getManagedItemDirectoryInfo(path, callback) { this.fs.readdir(path, (err, elements) => { if (err) { diff --git a/lib/util/fs.js b/lib/util/fs.js index 5d02a00ead5..49b2bdac401 100644 --- a/lib/util/fs.js +++ b/lib/util/fs.js @@ -59,6 +59,7 @@ const path = require("path"); /** @typedef {function((NodeJS.ErrnoException | null)=, number=): void} NumberCallback */ /** @typedef {function((NodeJS.ErrnoException | null)=, IStats=): void} StatsCallback */ /** @typedef {function((NodeJS.ErrnoException | Error | null)=, any=): void} ReadJsonCallback */ +/** @typedef {function((NodeJS.ErrnoException | Error | null)=, IStats|string=): void} LstatReadlinkAbsoluteCallback */ /** * @typedef {Object} Watcher @@ -103,6 +104,7 @@ const path = require("path"); * @property {function(string, BufferOrStringCallback): void} readlink * @property {function(string, DirentArrayCallback): void} readdir * @property {function(string, StatsCallback): void} stat + * @property {function(string, StatsCallback): void=} lstat * @property {(function(string, BufferOrStringCallback): void)=} realpath * @property {(function(string=): void)=} purge * @property {(function(string, string): string)=} join @@ -282,3 +284,41 @@ const readJson = (fs, p, callback) => { }); }; exports.readJson = readJson; + +/** + * @param {InputFileSystem} fs a file system + * @param {string} p an absolute path + * @param {ReadJsonCallback} callback callback + * @returns {void} + */ +const lstatReadlinkAbsolute = (fs, p, callback) => { + let i = 3; + const doReadLink = () => { + fs.readlink(p, (err, target) => { + if (err && --i > 0) { + // It might was just changed from symlink to file + // we retry 2 times to catch this case before throwing the error + return doStat(); + } + if (err || !target) return doStat(); + const value = target.toString(); + callback(null, join(fs, dirname(fs, p), value)); + }); + }; + const doStat = () => { + if ("lstat" in fs) { + return fs.lstat(p, (err, stats) => { + if (err) return callback(err); + if (stats.isSymbolicLink()) { + return doReadLink(); + } + callback(null, stats); + }); + } else { + return fs.stat(p, callback); + } + }; + if ("lstat" in fs) return doStat(); + doReadLink(); +}; +exports.lstatReadlinkAbsolute = lstatReadlinkAbsolute; diff --git a/test/FileSystemInfo.unittest.js b/test/FileSystemInfo.unittest.js new file mode 100644 index 00000000000..a198ae131ab --- /dev/null +++ b/test/FileSystemInfo.unittest.js @@ -0,0 +1,361 @@ +"use strict"; + +const { createFsFromVolume, Volume } = require("memfs"); +const util = require("util"); +const FileSystemInfo = require("../lib/FileSystemInfo"); +const { buffersSerializer } = require("../lib/util/serialization"); + +describe("FileSystemInfo", () => { + const files = [ + "/path/file.txt", + "/path/nested/deep/file.txt", + "/path/nested/deep/ignored.txt", + "/path/context+files/file.txt", + "/path/context+files/sub/file.txt", + "/path/context+files/sub/ignored.txt", + "/path/node_modules/package/file.txt", + "/path/cache/package-1234/file.txt", + "/path/circular/circular/file2.txt", + "/path/nested/deep/symlink/file.txt", + "/path/context+files/sub/symlink/file.txt", + "/path/context/sub/symlink/file.txt" + ]; + const directories = [ + "/path/context+files", + "/path/context", + "/path/node_modules/package", + "/path/cache/package-1234" + ]; + const missing = [ + "/path/package.json", + "/path/file2.txt", + "/path/context+files/file2.txt", + "/path/node_modules/package.txt", + "/path/node_modules/package/missing.txt", + "/path/cache/package-2345", + "/path/cache/package-1234/missing.txt", + "/path/ignored.txt" + ]; + const ignored = [ + "/path/nested/deep/ignored.txt", + "/path/context+files/sub/ignored.txt", + "/path/context/sub/ignored.txt", + "/path/ignored.txt", + "/path/node_modules/package/ignored.txt", + "/path/cache/package-1234/ignored.txt" + ]; + const managedPaths = ["/path/node_modules"]; + const immutablePaths = ["/path/cache"]; + const createFs = () => { + const fs = createFsFromVolume(new Volume()); + fs.mkdirpSync("/path/context+files/sub"); + fs.mkdirpSync("/path/context/sub"); + fs.mkdirpSync("/path/nested/deep"); + fs.mkdirpSync("/path/node_modules/package"); + fs.mkdirpSync("/path/cache/package-1234"); + fs.mkdirpSync("/path/folder/context"); + fs.mkdirpSync("/path/folder/context+files"); + fs.mkdirpSync("/path/folder/nested"); + fs.writeFileSync("/path/file.txt", "Hello World"); + fs.writeFileSync("/path/file2.txt", "Hello World2"); + fs.writeFileSync("/path/nested/deep/file.txt", "Hello World"); + fs.writeFileSync("/path/nested/deep/ignored.txt", "Ignored"); + fs.writeFileSync("/path/context+files/file.txt", "Hello World"); + fs.writeFileSync("/path/context+files/file2.txt", "Hello World2"); + fs.writeFileSync("/path/context+files/sub/file.txt", "Hello World"); + fs.writeFileSync("/path/context+files/sub/file2.txt", "Hello World2"); + fs.writeFileSync("/path/context+files/sub/file3.txt", "Hello World3"); + fs.writeFileSync("/path/context+files/sub/ignored.txt", "Ignored"); + fs.writeFileSync("/path/context/file.txt", "Hello World"); + fs.writeFileSync("/path/context/file2.txt", "Hello World2"); + fs.writeFileSync("/path/context/sub/file.txt", "Hello World"); + fs.writeFileSync("/path/context/sub/file2.txt", "Hello World2"); + fs.writeFileSync("/path/context/sub/file3.txt", "Hello World3"); + fs.writeFileSync("/path/context/sub/ignored.txt", "Ignored"); + fs.writeFileSync( + "/path/node_modules/package/package.json", + JSON.stringify({ name: "package", version: "1.0.0" }) + ); + fs.writeFileSync("/path/node_modules/package/file.txt", "Hello World"); + fs.writeFileSync("/path/node_modules/package/ignored.txt", "Ignored"); + fs.writeFileSync( + "/path/cache/package-1234/package.json", + JSON.stringify({ name: "package", version: "1.0.0" }) + ); + fs.writeFileSync("/path/cache/package-1234/file.txt", "Hello World"); + fs.writeFileSync("/path/cache/package-1234/ignored.txt", "Ignored"); + fs.symlinkSync("/path", "/path/circular", "dir"); + fs.writeFileSync("/path/folder/context/file.txt", "Hello World"); + fs.writeFileSync("/path/folder/context+files/file.txt", "Hello World"); + fs.writeFileSync("/path/folder/nested/file.txt", "Hello World"); + fs.symlinkSync("/path/folder/context", "/path/context/sub/symlink", "dir"); + fs.symlinkSync( + "/path/folder/context+files", + "/path/context+files/sub/symlink", + "dir" + ); + fs.symlinkSync("/path/folder/nested", "/path/nested/deep/symlink", "dir"); + return fs; + }; + + const createFsInfo = fs => { + const logger = { + error: (...args) => { + throw new Error(util.format(...args)); + } + }; + const fsInfo = new FileSystemInfo(fs, { + logger, + managedPaths, + immutablePaths + }); + for (const method of ["warn", "info", "log", "debug"]) { + fsInfo.logs = []; + fsInfo[method] = []; + logger[method] = (...args) => { + const msg = util.format(...args); + fsInfo[method].push(msg); + fsInfo.logs.push(`[${method}] ${msg}`); + }; + } + fsInfo.addFileTimestamps(new Map(ignored.map(i => [i, "ignore"]))); + return fsInfo; + }; + + const createSnapshot = (fs, options, callback) => { + const fsInfo = createFsInfo(fs); + fsInfo.createSnapshot( + Date.now() + 10000, + files, + directories, + missing, + options, + (err, snapshot) => { + if (err) return callback(err); + snapshot.name = "initial snapshot"; + // create another one to test the caching + fsInfo.createSnapshot( + Date.now() + 10000, + files, + directories, + missing, + options, + (err, snapshot2) => { + if (err) return callback(err); + snapshot2.name = "cached snapshot"; + callback(null, snapshot, snapshot2); + } + ); + } + ); + }; + + const clone = object => { + const serialized = buffersSerializer.serialize(object, {}); + return buffersSerializer.deserialize(serialized, {}); + }; + + const expectSnapshotsState = ( + fs, + snapshot, + snapshot2, + expected, + callback + ) => { + expectSnapshotState(fs, snapshot, expected, err => { + if (err) return callback(err); + if (!snapshot2) return callback(); + expectSnapshotState(fs, snapshot2, expected, callback); + }); + }; + + const expectSnapshotState = (fs, snapshot, expected, callback) => { + const fsInfo = createFsInfo(fs); + const details = snapshot => `${fsInfo.logs.join("\n")} +${util.inspect(snapshot, false, Infinity, true)}`; + fsInfo.checkSnapshotValid(snapshot, (err, valid) => { + if (err) return callback(err); + if (valid !== expected) { + return callback( + new Error(`Expected snapshot to be ${ + expected ? "valid" : "invalid" + } but it is ${valid ? "valid" : "invalid"}: +${details(snapshot)}`) + ); + } + // Another try to check if direct caching works + fsInfo.checkSnapshotValid(snapshot, (err, valid) => { + if (err) return callback(err); + if (valid !== expected) { + return callback( + new Error(`Expected snapshot lead to the same result when directly cached: +${details(snapshot)}`) + ); + } + // Another try to check if indirect caching works + fsInfo.checkSnapshotValid(clone(snapshot), (err, valid) => { + if (err) return callback(err); + if (valid !== expected) { + return callback( + new Error(`Expected snapshot lead to the same result when indirectly cached: +${details(snapshot)}`) + ); + } + callback(); + }); + }); + }); + }; + + const updateFile = (fs, filename) => { + const oldContent = fs.readFileSync(filename, "utf-8"); + if (filename.endsWith(".json")) { + const data = JSON.parse(oldContent); + fs.writeFileSync( + filename, + + JSON.stringify({ + ...data, + version: data.version + ".1" + }) + ); + } else { + fs.writeFileSync( + filename, + + oldContent + "!" + ); + } + }; + + for (const [name, options] of [ + ["timestamp", { timestamp: true }], + ["hash", { hash: true }], + ["tsh", { timestamp: true, hash: true }] + ]) { + describe(`${name} mode`, () => { + it("should always accept an empty snapshot", done => { + const fs = createFs(); + const fsInfo = createFsInfo(fs); + fsInfo.createSnapshot( + Date.now() + 10000, + [], + [], + [], + options, + (err, snapshot) => { + if (err) return done(err); + const fs = createFs(); + expectSnapshotState(fs, snapshot, true, done); + } + ); + }); + + it("should accept a snapshot when fs is unchanged", done => { + const fs = createFs(); + createSnapshot(fs, options, (err, snapshot, snapshot2) => { + if (err) return done(err); + expectSnapshotsState(fs, snapshot, snapshot2, true, done); + }); + }); + + const ignoredFileChanges = [ + "/path/nested/deep/ignored.txt", + "/path/context+files/sub/ignored.txt" + ]; + + for (const fileChange of [ + "/path/file.txt", + "/path/file2.txt", + "/path/nested/deep/file.txt", + "/path/context+files/file.txt", + "/path/context+files/file2.txt", + "/path/context+files/sub/file.txt", + "/path/context+files/sub/file2.txt", + "/path/context+files/sub/file3.txt", + "/path/context/file.txt", + "/path/context/file2.txt", + "/path/context/sub/file.txt", + "/path/context/sub/file2.txt", + "/path/context/sub/file3.txt", + "/path/node_modules/package/package.json", + "/path/folder/context/file.txt", + "/path/folder/context+files/file.txt", + "/path/folder/nested/file.txt", + ...(name !== "timestamp" ? ignoredFileChanges : []), + ...(name === "hash" ? ["/path/context/sub/ignored.txt"] : []) + ]) { + it(`should invalidate the snapshot when ${fileChange} is changed`, done => { + const fs = createFs(); + createSnapshot(fs, options, (err, snapshot, snapshot2) => { + if (err) return done(err); + updateFile(fs, fileChange); + expectSnapshotsState(fs, snapshot, snapshot2, false, done); + }); + }); + } + + for (const fileChange of [ + "/path/node_modules/package/file.txt", + "/path/node_modules/package/ignored.txt", + "/path/cache/package-1234/package.json", + "/path/cache/package-1234/file.txt", + "/path/cache/package-1234/ignored.txt", + ...(name === "timestamp" ? ignoredFileChanges : []), + ...(name !== "hash" ? ["/path/context/sub/ignored.txt"] : []) + ]) { + it(`should not invalidate the snapshot when ${fileChange} is changed`, done => { + const fs = createFs(); + createSnapshot(fs, options, (err, snapshot, snapshot2) => { + if (err) return done(err); + updateFile(fs, fileChange); + expectSnapshotsState(fs, snapshot, snapshot2, true, done); + }); + }); + } + + for (const newFile of [ + "/path/package.json", + "/path/file2.txt", + "/path/context+files/file2.txt", + "/path/node_modules/package.txt" + ]) { + it(`should invalidate the snapshot when ${newFile} is created`, done => { + const fs = createFs(); + createSnapshot(fs, options, (err, snapshot, snapshot2) => { + if (err) return done(err); + fs.writeFileSync(newFile, "New file"); + expectSnapshotsState(fs, snapshot, snapshot2, false, done); + }); + }); + } + + for (const newFile of [ + "/path/node_modules/package/missing.txt", + "/path/cache/package-1234/missing.txt", + "/path/cache/package-2345", + "/path/ignored.txt" + ]) { + it(`should not invalidate the snapshot when ${newFile} is created`, done => { + const fs = createFs(); + createSnapshot(fs, options, (err, snapshot, snapshot2) => { + if (err) return done(err); + fs.writeFileSync(newFile, "New file"); + expectSnapshotsState(fs, snapshot, snapshot2, true, done); + }); + }); + } + + if (name !== "timestamp") { + it("should not invalidate snapshot when only timestamps have changed", done => { + const fs = createFs(); + createSnapshot(fs, options, (err, snapshot, snapshot2) => { + if (err) return done(err); + const fs = createFs(); + expectSnapshotsState(fs, snapshot, snapshot2, true, done); + }); + }); + } + }); + } +}); diff --git a/types.d.ts b/types.d.ts index 28bb1dadb72..f9e8c4f06e8 100644 --- a/types.d.ts +++ b/types.d.ts @@ -2388,6 +2388,17 @@ declare class ContextExclusionPlugin { */ apply(compiler: Compiler): void; } +declare interface ContextFileSystemInfoEntry { + safeTime: number; + timestampHash?: string; + resolved?: ResolvedContextFileSystemInfoEntry; + symlinks?: Set; +} +declare interface ContextHash { + hash: string; + resolved?: string; + symlinks?: Set; +} type ContextMode = | "sync" | "eager" @@ -2455,6 +2466,13 @@ declare class ContextReplacementPlugin { newContentRegExp: any; apply(compiler?: any): void; } +declare interface ContextTimestampAndHash { + safeTime: number; + timestampHash?: string; + hash: string; + resolved?: ResolvedContextTimestampAndHash; + symlinks?: Set; +} type CreateStatsOptionsContext = KnownCreateStatsOptionsContext & Record; type Declaration = FunctionDeclaration | VariableDeclaration | ClassDeclaration; @@ -3968,8 +3986,13 @@ declare abstract class FileSystemInfo { logger?: WebpackLogger; fileTimestampQueue: AsyncQueue; fileHashQueue: AsyncQueue; - contextTimestampQueue: AsyncQueue; - contextHashQueue: AsyncQueue; + contextTimestampQueue: AsyncQueue< + string, + string, + null | ContextFileSystemInfoEntry + >; + contextHashQueue: AsyncQueue; + contextTshQueue: AsyncQueue; managedItemQueue: AsyncQueue; managedItemDirectoryQueue: AsyncQueue>; managedPaths: string[]; @@ -3997,7 +4020,7 @@ declare abstract class FileSystemInfo { path: string, callback: ( arg0?: WebpackError, - arg1?: null | FileSystemInfoEntry | "ignore" + arg1?: null | "ignore" | ResolvedContextFileSystemInfoEntry ) => void ): void; getFileHash( @@ -4008,6 +4031,13 @@ declare abstract class FileSystemInfo { path: string, callback: (arg0?: WebpackError, arg1?: string) => void ): void; + getContextTsh( + path: string, + callback: ( + arg0?: WebpackError, + arg1?: ResolvedContextTimestampAndHash + ) => void + ): void; resolveBuildDependencies( context: string, deps: Iterable, @@ -4045,7 +4075,6 @@ declare abstract class FileSystemInfo { declare interface FileSystemInfoEntry { safeTime: number; timestamp?: number; - timestampHash?: string; } declare interface FileSystemStats { isDirectory: () => boolean; @@ -4504,6 +4533,10 @@ declare interface InputFileSystem { arg0: string, arg1: (arg0?: null | NodeJS.ErrnoException, arg1?: IStats) => void ) => void; + lstat?: ( + arg0: string, + arg1: (arg0?: null | NodeJS.ErrnoException, arg1?: IStats) => void + ) => void; realpath?: ( arg0: string, arg1: (arg0?: null | NodeJS.ErrnoException, arg1?: string | Buffer) => void @@ -9255,6 +9288,15 @@ declare interface ResolvePluginInstance { apply: (resolver: Resolver) => void; } type ResolveRequest = BaseResolveRequest & Partial; +declare interface ResolvedContextFileSystemInfoEntry { + safeTime: number; + timestampHash?: string; +} +declare interface ResolvedContextTimestampAndHash { + safeTime: number; + timestampHash?: string; + hash: string; +} declare abstract class Resolver { fileSystem: FileSystem; options: ResolveOptionsTypes; @@ -10326,9 +10368,9 @@ declare abstract class Snapshot { fileTimestamps?: Map; fileHashes?: Map; fileTshs?: Map; - contextTimestamps?: Map; + contextTimestamps?: Map; contextHashes?: Map; - contextTshs?: Map; + contextTshs?: Map; missingExistence?: Map; managedItemInfo?: Map; managedFiles?: Set; @@ -11242,7 +11284,6 @@ declare class Template { declare interface TimestampAndHash { safeTime: number; timestamp?: number; - timestampHash?: string; hash: string; }