From 0064fcd179dab8645539d27311d448494c803592 Mon Sep 17 00:00:00 2001 From: Michael Loughry Date: Fri, 5 Oct 2018 12:13:55 -0700 Subject: [PATCH 1/6] Don't process chunks multiple times --- lib/Compilation.js | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/Compilation.js b/lib/Compilation.js index 5f64119a8bc..e9d49624ec2 100644 --- a/lib/Compilation.js +++ b/lib/Compilation.js @@ -1826,6 +1826,8 @@ class Compilation extends Tapable { /** @type {Map>} */ const minAvailableModulesMap = new Map(); + /** @type {Map} */ + const chunkAlreadyQueued = new Map(); // Iterative traversing of the basic chunk graph while (queue2.length) { @@ -1894,11 +1896,15 @@ class Compilation extends Tapable { // 7. Enqueue further traversal for (const nextChunkGroup of nextChunkGroups) { - queue2.enqueue({ - chunkGroup: nextChunkGroup, - availableModules: newAvailableModules, - needCopy: nextChunkGroup.size !== 1 - }); + // Only need to queue each chunk group once. + if (chunkAlreadyQueued.get(nextChunkGroup)) { + queue2.enqueue({ + chunkGroup: nextChunkGroup, + availableModules: newAvailableModules, + needCopy: nextChunkGroup.size !== 1 + }); + chunkAlreadyQueued.set(nextChunkGroup, true); + } } } From 701f0e0e59c977b14ba21d6a3ba3e7b26dc79ae2 Mon Sep 17 00:00:00 2001 From: Michael Loughry Date: Fri, 5 Oct 2018 12:15:38 -0700 Subject: [PATCH 2/6] Try using a priority queue to optimize processDependenciesBlocksForChunkGroups --- lib/Compilation.js | 7 +-- lib/util/PriorityQueue.js | 107 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+), 3 deletions(-) create mode 100644 lib/util/PriorityQueue.js diff --git a/lib/Compilation.js b/lib/Compilation.js index e9d49624ec2..78515a08c97 100644 --- a/lib/Compilation.js +++ b/lib/Compilation.js @@ -31,7 +31,7 @@ const AsyncDependencyToInitialChunkError = require("./AsyncDependencyToInitialCh const Stats = require("./Stats"); const Semaphore = require("./util/Semaphore"); const createHash = require("./util/createHash"); -const Queue = require("./util/Queue"); +const PriorityQueue = require("./util/PriorityQueue"); const SortableSet = require("./util/SortableSet"); const GraphHelpers = require("./GraphHelpers"); const ModuleDependency = require("./dependencies/ModuleDependency"); @@ -1782,8 +1782,9 @@ class Compilation extends Tapable { let availableModules; /** @type {Set} */ let newAvailableModules; - /** @type {Queue} */ - const queue2 = new Queue( + /** @type {PriorityQueue} */ + const queue2 = new PriorityQueue( + mapping => mapping.availableModules.size, inputChunkGroups.map(chunkGroup => ({ chunkGroup, availableModules: new Set(), diff --git a/lib/util/PriorityQueue.js b/lib/util/PriorityQueue.js new file mode 100644 index 00000000000..e120ebf68a1 --- /dev/null +++ b/lib/util/PriorityQueue.js @@ -0,0 +1,107 @@ +"use strict"; + +/** + * @template T + */ +class PriorityQueue { + /** + * @param {function(T):number} priorityFn The function to calculate the item's priority. Lower values will be pushed to the front of the queue + * @param {Iterable=} items The initial elements. + */ + constructor(priorityFn, items) { + /** @type {{priority: number, item: T}[]} */ + this.heap = []; + /** @type {function(T):number}*/ + this.priorityFn = priorityFn; + for (let item of items) { + this.enqueue(item); + } + } + + /** + * Returns the number of elements in this queue. + * @returns {number} The number of elements in this queue. + */ + get length() { + return this.heap.length; + } + + /** + * Appends the specified element to this queue. + * @param {T} item The element to add. + * @returns {void} + */ + enqueue(item) { + this.heap.push({ priority: this.priorityFn(item), item }); + this._siftUp(); + } + + /** + * Retrieves and removes the head of this queue. + * @returns {T | undefined} The head of the queue of `undefined` if this queue is empty. + */ + dequeue() { + const lastIndex = this.length - 1; + if (lastIndex > 0) { + this._swap(0, lastIndex); + } + const poppedValue = this.heap.pop(); + this._siftDown(); + return poppedValue.item; + } + + _getNodePriority(index) { + return this.heap[index].priority; + } + + _getParentIndex(index) { + return ((index + 1) >>> 1) - 1; + } + + _getLeftIndex(index) { + return (index << 1) + 1; + } + + _getRightIndex(index) { + return (index + 1) << 1; + } + + _swap(i, j) { + [this.heap[i], this.heap[j]] = [this.heap[j], this.heap[i]]; + } + + _siftUp() { + let node = this.length - 1; + while ( + node > 0 && + this._getNodePriority(node) < + this._getNodePriority(this._getParentIndex(node)) + ) { + this._swap(node, this._getParentIndex(node)); + node = this._getParentIndex(node); + } + } + + _siftDown() { + let node = 0; + while ( + (this._getLeftIndex(node) < this.length && + this._getNodePriority(this._getLeftIndex(node)) < + this._getNodePriority(node)) || + (this._getRightIndex(node) < this.length && + this._getNodePriority(this._getRightIndex(node)) < + this._getNodePriority(node)) + ) { + let minChild = + this._getRightIndex(node) < this.length && + this._getNodePriority(this._getRightIndex(node)) < + this._getNodePriority(this._getLeftIndex(node)) + ? this._getRightIndex(node) + : this._getLeftIndex(node); + this._swap(node, minChild); + node = minChild; + } + } +} + +module.exports = PriorityQueue; From 1bd8783070a38d64ce9281a3f37618309dcb37a0 Mon Sep 17 00:00:00 2001 From: Michael Loughry Date: Fri, 5 Oct 2018 12:39:55 -0700 Subject: [PATCH 3/6] Fix accidentally duplicated line --- lib/Compilation.js | 144 ++++++++++++++++----------------- test/fixtures/temp-30/file.js | 1 + test/fixtures/temp-30/file2.js | 1 + 3 files changed, 73 insertions(+), 73 deletions(-) create mode 100644 test/fixtures/temp-30/file.js create mode 100644 test/fixtures/temp-30/file2.js diff --git a/lib/Compilation.js b/lib/Compilation.js index 78515a08c97..b8f5a557b13 100644 --- a/lib/Compilation.js +++ b/lib/Compilation.js @@ -1682,94 +1682,92 @@ class Compilation extends Tapable { // Iterative traversal of the Module graph // Recursive would be simpler to write but could result in Stack Overflows while (queue.length) { - while (queue.length) { - const queueItem = queue.pop(); - module = queueItem.module; - block = queueItem.block; - chunk = queueItem.chunk; - chunkGroup = queueItem.chunkGroup; - - switch (queueItem.action) { - case ADD_AND_ENTER_MODULE: { - // We connect Module and Chunk when not already done - if (chunk.addModule(module)) { - module.addChunk(chunk); - } else { - // already connected, skip it - break; - } + const queueItem = queue.pop(); + module = queueItem.module; + block = queueItem.block; + chunk = queueItem.chunk; + chunkGroup = queueItem.chunkGroup; + + switch (queueItem.action) { + case ADD_AND_ENTER_MODULE: { + // We connect Module and Chunk when not already done + if (chunk.addModule(module)) { + module.addChunk(chunk); + } else { + // already connected, skip it + break; } - // fallthrough - case ENTER_MODULE: { - if (chunkGroup !== undefined) { - const index = chunkGroup.getModuleIndex(module); - if (index === undefined) { - chunkGroup.setModuleIndex( - module, - chunkGroupCounters.get(chunkGroup).index++ - ); - } + } + // fallthrough + case ENTER_MODULE: { + if (chunkGroup !== undefined) { + const index = chunkGroup.getModuleIndex(module); + if (index === undefined) { + chunkGroup.setModuleIndex( + module, + chunkGroupCounters.get(chunkGroup).index++ + ); } + } - if (module.index === null) { - module.index = nextFreeModuleIndex++; - } + if (module.index === null) { + module.index = nextFreeModuleIndex++; + } + queue.push({ + action: LEAVE_MODULE, + block, + module, + chunk, + chunkGroup + }); + } + // fallthrough + case PROCESS_BLOCK: { + // get prepared block info + const blockInfo = blockInfoMap.get(block); + + // Traverse all referenced modules + for (let i = blockInfo.modules.length - 1; i >= 0; i--) { + const refModule = blockInfo.modules[i]; + if (chunk.containsModule(refModule)) { + // skip early if already connected + continue; + } + // enqueue the add and enter to enter in the correct order + // this is relevant with circular dependencies queue.push({ - action: LEAVE_MODULE, - block, - module, + action: ADD_AND_ENTER_MODULE, + block: refModule, + module: refModule, chunk, chunkGroup }); } - // fallthrough - case PROCESS_BLOCK: { - // get prepared block info - const blockInfo = blockInfoMap.get(block); - - // Traverse all referenced modules - for (let i = blockInfo.modules.length - 1; i >= 0; i--) { - const refModule = blockInfo.modules[i]; - if (chunk.containsModule(refModule)) { - // skip early if already connected - continue; - } - // enqueue the add and enter to enter in the correct order - // this is relevant with circular dependencies - queue.push({ - action: ADD_AND_ENTER_MODULE, - block: refModule, - module: refModule, - chunk, - chunkGroup - }); - } - // Traverse all Blocks - iterationOfArrayCallback(blockInfo.blocks, iteratorBlock); + // Traverse all Blocks + iterationOfArrayCallback(blockInfo.blocks, iteratorBlock); - if (blockInfo.blocks.length > 0 && module !== block) { - blocksWithNestedBlocks.add(block); - } - break; + if (blockInfo.blocks.length > 0 && module !== block) { + blocksWithNestedBlocks.add(block); } - case LEAVE_MODULE: { - if (chunkGroup !== undefined) { - const index = chunkGroup.getModuleIndex2(module); - if (index === undefined) { - chunkGroup.setModuleIndex2( - module, - chunkGroupCounters.get(chunkGroup).index2++ - ); - } + break; + } + case LEAVE_MODULE: { + if (chunkGroup !== undefined) { + const index = chunkGroup.getModuleIndex2(module); + if (index === undefined) { + chunkGroup.setModuleIndex2( + module, + chunkGroupCounters.get(chunkGroup).index2++ + ); } + } - if (module.index2 === null) { - module.index2 = nextFreeModuleIndex2++; - } - break; + if (module.index2 === null) { + module.index2 = nextFreeModuleIndex2++; } + break; } } const tempQueue = queue; diff --git a/test/fixtures/temp-30/file.js b/test/fixtures/temp-30/file.js new file mode 100644 index 00000000000..3d31365c5ef --- /dev/null +++ b/test/fixtures/temp-30/file.js @@ -0,0 +1 @@ +require('./file2') \ No newline at end of file diff --git a/test/fixtures/temp-30/file2.js b/test/fixtures/temp-30/file2.js new file mode 100644 index 00000000000..94f3610c085 --- /dev/null +++ b/test/fixtures/temp-30/file2.js @@ -0,0 +1 @@ +original \ No newline at end of file From c4e9d15144062d59d7eb4dac29ccb7b6c2a9bcdb Mon Sep 17 00:00:00 2001 From: Michael Loughry Date: Fri, 5 Oct 2018 12:48:35 -0700 Subject: [PATCH 4/6] Revert change to not re-queue chunks --- lib/Compilation.js | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/lib/Compilation.js b/lib/Compilation.js index b8f5a557b13..92fde1d9249 100644 --- a/lib/Compilation.js +++ b/lib/Compilation.js @@ -1825,8 +1825,6 @@ class Compilation extends Tapable { /** @type {Map>} */ const minAvailableModulesMap = new Map(); - /** @type {Map} */ - const chunkAlreadyQueued = new Map(); // Iterative traversing of the basic chunk graph while (queue2.length) { @@ -1895,15 +1893,11 @@ class Compilation extends Tapable { // 7. Enqueue further traversal for (const nextChunkGroup of nextChunkGroups) { - // Only need to queue each chunk group once. - if (chunkAlreadyQueued.get(nextChunkGroup)) { - queue2.enqueue({ - chunkGroup: nextChunkGroup, - availableModules: newAvailableModules, - needCopy: nextChunkGroup.size !== 1 - }); - chunkAlreadyQueued.set(nextChunkGroup, true); - } + queue2.enqueue({ + chunkGroup: nextChunkGroup, + availableModules: newAvailableModules, + needCopy: nextChunkGroup.size !== 1 + }); } } From 9c2882ed6d0ca19deb20b1d2904bef504ac4139c Mon Sep 17 00:00:00 2001 From: Michael Loughry Date: Fri, 5 Oct 2018 12:58:17 -0700 Subject: [PATCH 5/6] Fix accidental regression --- lib/Compilation.js | 144 +++++++++++++++++++++++---------------------- 1 file changed, 73 insertions(+), 71 deletions(-) diff --git a/lib/Compilation.js b/lib/Compilation.js index 92fde1d9249..90daaecbcb8 100644 --- a/lib/Compilation.js +++ b/lib/Compilation.js @@ -1682,92 +1682,94 @@ class Compilation extends Tapable { // Iterative traversal of the Module graph // Recursive would be simpler to write but could result in Stack Overflows while (queue.length) { - const queueItem = queue.pop(); - module = queueItem.module; - block = queueItem.block; - chunk = queueItem.chunk; - chunkGroup = queueItem.chunkGroup; - - switch (queueItem.action) { - case ADD_AND_ENTER_MODULE: { - // We connect Module and Chunk when not already done - if (chunk.addModule(module)) { - module.addChunk(chunk); - } else { - // already connected, skip it - break; - } - } - // fallthrough - case ENTER_MODULE: { - if (chunkGroup !== undefined) { - const index = chunkGroup.getModuleIndex(module); - if (index === undefined) { - chunkGroup.setModuleIndex( - module, - chunkGroupCounters.get(chunkGroup).index++ - ); + while (queue.length) { + const queueItem = queue.pop(); + module = queueItem.module; + block = queueItem.block; + chunk = queueItem.chunk; + chunkGroup = queueItem.chunkGroup; + + switch (queueItem.action) { + case ADD_AND_ENTER_MODULE: { + // We connect Module and Chunk when not already done + if (chunk.addModule(module)) { + module.addChunk(chunk); + } else { + // already connected, skip it + break; } } + // fallthrough + case ENTER_MODULE: { + if (chunkGroup !== undefined) { + const index = chunkGroup.getModuleIndex(module); + if (index === undefined) { + chunkGroup.setModuleIndex( + module, + chunkGroupCounters.get(chunkGroup).index++ + ); + } + } - if (module.index === null) { - module.index = nextFreeModuleIndex++; - } - - queue.push({ - action: LEAVE_MODULE, - block, - module, - chunk, - chunkGroup - }); - } - // fallthrough - case PROCESS_BLOCK: { - // get prepared block info - const blockInfo = blockInfoMap.get(block); - - // Traverse all referenced modules - for (let i = blockInfo.modules.length - 1; i >= 0; i--) { - const refModule = blockInfo.modules[i]; - if (chunk.containsModule(refModule)) { - // skip early if already connected - continue; + if (module.index === null) { + module.index = nextFreeModuleIndex++; } - // enqueue the add and enter to enter in the correct order - // this is relevant with circular dependencies + queue.push({ - action: ADD_AND_ENTER_MODULE, - block: refModule, - module: refModule, + action: LEAVE_MODULE, + block, + module, chunk, chunkGroup }); } + // fallthrough + case PROCESS_BLOCK: { + // get prepared block info + const blockInfo = blockInfoMap.get(block); + + // Traverse all referenced modules + for (let i = blockInfo.modules.length - 1; i >= 0; i--) { + const refModule = blockInfo.modules[i]; + if (chunk.containsModule(refModule)) { + // skip early if already connected + continue; + } + // enqueue the add and enter to enter in the correct order + // this is relevant with circular dependencies + queue.push({ + action: ADD_AND_ENTER_MODULE, + block: refModule, + module: refModule, + chunk, + chunkGroup + }); + } - // Traverse all Blocks - iterationOfArrayCallback(blockInfo.blocks, iteratorBlock); + // Traverse all Blocks + iterationOfArrayCallback(blockInfo.blocks, iteratorBlock); - if (blockInfo.blocks.length > 0 && module !== block) { - blocksWithNestedBlocks.add(block); - } - break; - } - case LEAVE_MODULE: { - if (chunkGroup !== undefined) { - const index = chunkGroup.getModuleIndex2(module); - if (index === undefined) { - chunkGroup.setModuleIndex2( - module, - chunkGroupCounters.get(chunkGroup).index2++ - ); + if (blockInfo.blocks.length > 0 && module !== block) { + blocksWithNestedBlocks.add(block); } + break; } + case LEAVE_MODULE: { + if (chunkGroup !== undefined) { + const index = chunkGroup.getModuleIndex2(module); + if (index === undefined) { + chunkGroup.setModuleIndex2( + module, + chunkGroupCounters.get(chunkGroup).index2++ + ); + } + } - if (module.index2 === null) { - module.index2 = nextFreeModuleIndex2++; + if (module.index2 === null) { + module.index2 = nextFreeModuleIndex2++; + } + break; } - break; } } const tempQueue = queue; From d9736865ce359c97dc84b5c1dc1349194437ddf3 Mon Sep 17 00:00:00 2001 From: Michael Loughry Date: Fri, 5 Oct 2018 13:14:09 -0700 Subject: [PATCH 6/6] Clear temp files --- test/fixtures/temp-30/file.js | 1 - test/fixtures/temp-30/file2.js | 1 - 2 files changed, 2 deletions(-) delete mode 100644 test/fixtures/temp-30/file.js delete mode 100644 test/fixtures/temp-30/file2.js diff --git a/test/fixtures/temp-30/file.js b/test/fixtures/temp-30/file.js deleted file mode 100644 index 3d31365c5ef..00000000000 --- a/test/fixtures/temp-30/file.js +++ /dev/null @@ -1 +0,0 @@ -require('./file2') \ No newline at end of file diff --git a/test/fixtures/temp-30/file2.js b/test/fixtures/temp-30/file2.js deleted file mode 100644 index 94f3610c085..00000000000 --- a/test/fixtures/temp-30/file2.js +++ /dev/null @@ -1 +0,0 @@ -original \ No newline at end of file