diff --git a/packages/core/core/src/RequestTracker.js b/packages/core/core/src/RequestTracker.js index e8fb95d735b..68e50a79d62 100644 --- a/packages/core/core/src/RequestTracker.js +++ b/packages/core/core/src/RequestTracker.js @@ -65,6 +65,10 @@ export const requestGraphEdgeTypes = { dirname: 7, }; +class FSBailoutError extends Error { + name: string = 'FSBailoutError'; +} + export type RequestGraphEdgeType = $Values; type RequestGraphOpts = {| @@ -263,6 +267,10 @@ const keyFromEnvContentKey = (contentKey: ContentKey): string => const keyFromOptionContentKey = (contentKey: ContentKey): string => contentKey.slice('option:'.length); +// This constant is chosen by local profiling the time to serialise n nodes and tuning until an average time of ~50 ms per blob. +// The goal is to free up the event loop periodically to allow interruption by the user. +const NODES_PER_BLOB = 2 ** 14; + export class RequestGraph extends ContentGraph< RequestGraphNode, RequestGraphEdgeType, @@ -279,6 +287,7 @@ export class RequestGraph extends ContentGraph< invalidateOnBuildNodeIds: Set = new Set(); cachedRequestChunks: Set = new Set(); configKeyNodes: Map> = new Map(); + nodesPerBlob: number = NODES_PER_BLOB; // $FlowFixMe[prop-missing] static deserialize(opts: RequestGraphOpts): RequestGraph { @@ -328,6 +337,8 @@ export class RequestGraph extends ContentGraph< this.optionNodeIds.add(nodeId); } + this.removeCachedRequestChunkForNode(nodeId); + return nodeId; } @@ -855,7 +866,7 @@ export class RequestGraph extends ContentGraph< predictedTime, }, }); - throw new Error( + throw new FSBailoutError( 'Responding to file system events exceeded threshold, start with empty cache.', ); } @@ -1026,14 +1037,10 @@ export class RequestGraph extends ContentGraph< } removeCachedRequestChunkForNode(nodeId: number): void { - this.cachedRequestChunks.delete(Math.floor(nodeId / NODES_PER_BLOB)); + this.cachedRequestChunks.delete(Math.floor(nodeId / this.nodesPerBlob)); } } -// This constant is chosen by local profiling the time to serialise n nodes and tuning until an average time of ~50 ms per blob. -// The goal is to free up the event loop periodically to allow interruption by the user. -const NODES_PER_BLOB = 2 ** 14; - export default class RequestTracker { graph: RequestGraph; farm: WorkerFarm; @@ -1421,17 +1428,30 @@ export default class RequestTracker { } } - for (let i = 0; i * NODES_PER_BLOB < cacheableNodes.length; i += 1) { + let nodeCountsPerBlob = []; + + for ( + let i = 0; + i * this.graph.nodesPerBlob < cacheableNodes.length; + i += 1 + ) { + let nodesStartIndex = i * this.graph.nodesPerBlob; + let nodesEndIndex = Math.min( + (i + 1) * this.graph.nodesPerBlob, + cacheableNodes.length, + ); + + nodeCountsPerBlob.push(nodesEndIndex - nodesStartIndex); + if (!this.graph.hasCachedRequestChunk(i)) { // We assume the request graph nodes are immutable and won't change + let nodesToCache = cacheableNodes.slice(nodesStartIndex, nodesEndIndex); + queue .add(() => serialiseAndSet( getRequestGraphNodeKey(i, cacheKey), - cacheableNodes.slice( - i * NODES_PER_BLOB, - (i + 1) * NODES_PER_BLOB, - ), + nodesToCache, ).then(() => { // Succeeded in writing to disk, save that we have completed this chunk this.graph.setCachedRequestChunk(i); @@ -1449,6 +1469,7 @@ export default class RequestTracker { // Set the request graph after the queue is flushed to avoid writing an invalid state await serialiseAndSet(requestGraphKey, { ...serialisedGraph, + nodeCountsPerBlob, nodes: undefined, }); @@ -1517,19 +1538,24 @@ export async function readAndDeserializeRequestGraph( return deserialize(buffer); }; - let i = 0; - let nodePromises = []; - while (await cache.hasLargeBlob(getRequestGraphNodeKey(i, cacheKey))) { - nodePromises.push(getAndDeserialize(getRequestGraphNodeKey(i, cacheKey))); - i += 1; - } - let serializedRequestGraph = await getAndDeserialize(requestGraphKey); + let nodePromises = serializedRequestGraph.nodeCountsPerBlob.map( + async (nodesCount, i) => { + let nodes = await getAndDeserialize(getRequestGraphNodeKey(i, cacheKey)); + invariant.equal( + nodes.length, + nodesCount, + 'RequestTracker node chunk: invalid node count', + ); + return nodes; + }, + ); + return { requestGraph: RequestGraph.deserialize({ ...serializedRequestGraph, - nodes: (await Promise.all(nodePromises)).flatMap(nodeChunk => nodeChunk), + nodes: (await Promise.all(nodePromises)).flat(), }), // This is used inside parcel query for `.inspectCache` bufferLength, @@ -1543,48 +1569,48 @@ async function loadRequestGraph(options): Async { let cacheKey = getCacheKey(options); let requestGraphKey = `requestGraph-${cacheKey}`; - + let timeout; + const snapshotKey = `snapshot-${cacheKey}`; + const snapshotPath = path.join(options.cacheDir, snapshotKey + '.txt'); if (await options.cache.hasLargeBlob(requestGraphKey)) { - let {requestGraph} = await readAndDeserializeRequestGraph( - options.cache, - requestGraphKey, - cacheKey, - ); + try { + let {requestGraph} = await readAndDeserializeRequestGraph( + options.cache, + requestGraphKey, + cacheKey, + ); - let opts = getWatcherOptions(options); - let snapshotKey = `snapshot-${cacheKey}`; - let snapshotPath = path.join(options.cacheDir, snapshotKey + '.txt'); + let opts = getWatcherOptions(options); - let timeout = setTimeout(() => { - logger.warn({ + timeout = setTimeout(() => { + logger.warn({ + origin: '@parcel/core', + message: `Retrieving file system events since last build...\nThis can take upto a minute after branch changes or npm/yarn installs.`, + }); + }, 5000); + let startTime = Date.now(); + let events = await options.inputFS.getEventsSince( + options.watchDir, + snapshotPath, + opts, + ); + clearTimeout(timeout); + + logger.verbose({ origin: '@parcel/core', - message: `Retrieving file system events since last build...\nThis can take upto a minute after branch changes or npm/yarn installs.`, + message: `File system event count: ${events.length}`, + meta: { + trackableEvent: 'watcher_events_count', + watcherEventCount: events.length, + duration: Date.now() - startTime, + }, }); - }, 5000); - let startTime = Date.now(); - let events = await options.inputFS.getEventsSince( - options.watchDir, - snapshotPath, - opts, - ); - clearTimeout(timeout); - - logger.verbose({ - origin: '@parcel/core', - message: `File system event count: ${events.length}`, - meta: { - trackableEvent: 'watcher_events_count', - watcherEventCount: events.length, - duration: Date.now() - startTime, - }, - }); - requestGraph.invalidateUnpredictableNodes(); - requestGraph.invalidateOnBuildNodes(); - requestGraph.invalidateEnvNodes(options.env); - requestGraph.invalidateOptionNodes(options); + requestGraph.invalidateUnpredictableNodes(); + requestGraph.invalidateOnBuildNodes(); + requestGraph.invalidateEnvNodes(options.env); + requestGraph.invalidateOptionNodes(options); - try { await requestGraph.respondToFSEvents( options.unstableFileInvalidations || events, options, @@ -1592,6 +1618,9 @@ async function loadRequestGraph(options): Async { ); return requestGraph; } catch (e) { + // Prevent logging fs events took too long warning + clearTimeout(timeout); + logErrorOnBailout(options, snapshotPath, e); // This error means respondToFSEvents timed out handling the invalidation events // In this case we'll return a fresh RequestGraph return new RequestGraph(); @@ -1600,3 +1629,31 @@ async function loadRequestGraph(options): Async { return new RequestGraph(); } +function logErrorOnBailout( + options: ParcelOptions, + snapshotPath: string, + e: Error, +): void { + if (e.message && e.message.includes('invalid clockspec')) { + const snapshotContents = options.inputFS.readFileSync( + snapshotPath, + 'utf-8', + ); + logger.warn({ + origin: '@parcel/core', + message: `Error reading clockspec from snapshot, building with clean cache.`, + meta: { + snapshotContents: snapshotContents, + trackableEvent: 'invalid_clockspec_error', + }, + }); + } else if (!(e instanceof FSBailoutError)) { + logger.warn({ + origin: '@parcel/core', + message: `Unexpected error loading cache from disk, building with clean cache.`, + meta: { + trackableEvent: 'cache_load_error', + }, + }); + } +} diff --git a/packages/core/core/test/RequestTracker.test.js b/packages/core/core/test/RequestTracker.test.js index 8315445b00f..a6bd283b0aa 100644 --- a/packages/core/core/test/RequestTracker.test.js +++ b/packages/core/core/test/RequestTracker.test.js @@ -307,4 +307,135 @@ describe('RequestTracker', () => { assert.strictEqual(cachedResult, 'b'); assert.strictEqual(called, false); }); + + it('should ignore stale node chunks from cache', async () => { + let tracker = new RequestTracker({farm, options}); + + // Set the nodes per blob low so we can ensure multiple files without + // creating 17,000 nodes + tracker.graph.nodesPerBlob = 2; + + tracker.graph.addNode({type: 0, id: 'some-file-node-1'}); + tracker.graph.addNode({type: 0, id: 'some-file-node-2'}); + tracker.graph.addNode({type: 0, id: 'some-file-node-3'}); + tracker.graph.addNode({type: 0, id: 'some-file-node-4'}); + tracker.graph.addNode({type: 0, id: 'some-file-node-5'}); + + await tracker.writeToCache(); + + // Create a new request tracker that shouldn't look at the old cache files + tracker = new RequestTracker({farm, options}); + assert.equal(tracker.graph.nodes.length, 0); + + tracker.graph.addNode({type: 0, id: 'some-file-node-1'}); + await tracker.writeToCache(); + + // Init a request tracker that should only read the relevant cache files + tracker = await RequestTracker.init({farm, options}); + assert.equal(tracker.graph.nodes.length, 1); + }); + + it('should init with multiple node chunks', async () => { + let tracker = new RequestTracker({farm, options}); + + // Set the nodes per blob low so we can ensure multiple files without + // creating 17,000 nodes + tracker.graph.nodesPerBlob = 2; + + tracker.graph.addNode({type: 0, id: 'some-file-node-1'}); + tracker.graph.addNode({type: 0, id: 'some-file-node-2'}); + tracker.graph.addNode({type: 0, id: 'some-file-node-3'}); + tracker.graph.addNode({type: 0, id: 'some-file-node-4'}); + tracker.graph.addNode({type: 0, id: 'some-file-node-5'}); + + await tracker.writeToCache(); + + tracker = await RequestTracker.init({farm, options}); + assert.equal(tracker.graph.nodes.length, 5); + }); + + it('should write new nodes to cache', async () => { + let tracker = new RequestTracker({farm, options}); + + tracker.graph.addNode({ + type: 0, + id: 'test-file', + }); + await tracker.writeToCache(); + assert.equal(tracker.graph.nodes.length, 1); + + tracker.graph.addNode({ + type: 0, + id: 'test-file-2', + }); + await tracker.writeToCache(); + assert.equal(tracker.graph.nodes.length, 2); + + // Create a new tracker from cache + tracker = await RequestTracker.init({farm, options}); + + await tracker.writeToCache(); + assert.equal(tracker.graph.nodes.length, 2); + }); + + it('should write updated nodes to cache', async () => { + let tracker = new RequestTracker({farm, options}); + + let contentKey = 'abc'; + await tracker.runRequest({ + id: contentKey, + type: 7, + run: async ({api}: {api: RunAPI, ...}) => { + let result = await Promise.resolve('a'); + api.storeResult(result); + }, + input: null, + }); + assert.equal(await tracker.getRequestResult(contentKey), 'a'); + await tracker.writeToCache(); + + await tracker.runRequest( + { + id: contentKey, + type: 7, + run: async ({api}: {api: RunAPI, ...}) => { + let result = await Promise.resolve('b'); + api.storeResult(result); + }, + input: null, + }, + {force: true}, + ); + assert.equal(await tracker.getRequestResult(contentKey), 'b'); + await tracker.writeToCache(); + + // Create a new tracker from cache + tracker = await RequestTracker.init({farm, options}); + + assert.equal(await tracker.getRequestResult(contentKey), 'b'); + }); + + it('should write invalidated nodes to cache', async () => { + let tracker = new RequestTracker({farm, options}); + + let contentKey = 'abc'; + await tracker.runRequest({ + id: contentKey, + type: 7, + run: () => {}, + input: null, + }); + let nodeId = tracker.graph.getNodeIdByContentKey(contentKey); + assert.equal(tracker.graph.getNode(nodeId)?.invalidateReason, 0); + await tracker.writeToCache(); + + tracker.graph.invalidateNode(nodeId, 1); + assert.equal(tracker.graph.getNode(nodeId)?.invalidateReason, 1); + await tracker.writeToCache(); + + // Create a new tracker from cache + tracker = await RequestTracker.init({farm, options}); + + assert.equal(tracker.graph.getNode(nodeId)?.invalidateReason, 1); + }); });