From 5a92cfe0e8a55996366e5e28baaad2d71aeb4f99 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Fri, 14 Dec 2018 04:03:08 +0000 Subject: [PATCH 1/2] fix(NA): add verification to avoid using an undefined worker.stdio due to reach the max open files number. --- src/WorkerPool.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/WorkerPool.js b/src/WorkerPool.js index 0fd4406..36f1ad0 100644 --- a/src/WorkerPool.js +++ b/src/WorkerPool.js @@ -21,6 +21,15 @@ class PoolWorker { this.worker = childProcess.spawn(process.execPath, [].concat(options.nodeArgs || []).concat(workerPath, options.parallelJobs), { stdio: ['ignore', 1, 2, 'pipe', 'pipe'], }); + + // This prevents a problem where the worker stdio can be undefined + // when the kernel hits the limit of open files. + // More info can be found on: https://github.com/webpack-contrib/thread-loader/issues/2 + if (!this.worker.stdio) { + throw new Error(`Failed to create the worker pool with workerId: ${workerId} and \ + configuration: ${options}. Please verify if you hit the OS open files limit.`); + } + const [, , , readPipe, writePipe] = this.worker.stdio; this.readPipe = readPipe; this.writePipe = writePipe; From 82a0a856a2fb68bd8baf0fdfe83f710803973cc6 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Sat, 15 Dec 2018 04:55:51 +0000 Subject: [PATCH 2/2] test(NA): implement base tests. --- package-lock.json | 70 ++++++++++++---------- src/WorkerPool.js | 4 +- test/__snapshots__/workerPool.test.js.snap | 3 + test/workerPool.test.js | 32 ++++++++++ 4 files changed, 75 insertions(+), 34 deletions(-) create mode 100644 test/__snapshots__/workerPool.test.js.snap create mode 100644 test/workerPool.test.js diff --git a/package-lock.json b/package-lock.json index 4aca9b7..92d9f0a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -655,19 +655,19 @@ }, "babel-plugin-syntax-async-functions": { "version": "6.13.0", - "resolved": "https://registry.npmjs.org/babel-plugin-syntax-async-functions/-/babel-plugin-syntax-async-functions-6.13.0.tgz", + "resolved": "http://registry.npmjs.org/babel-plugin-syntax-async-functions/-/babel-plugin-syntax-async-functions-6.13.0.tgz", "integrity": "sha1-ytnK0RkbWtY0vzCuCHI5HgZHvpU=", "dev": true }, "babel-plugin-syntax-exponentiation-operator": { "version": "6.13.0", - "resolved": "https://registry.npmjs.org/babel-plugin-syntax-exponentiation-operator/-/babel-plugin-syntax-exponentiation-operator-6.13.0.tgz", + "resolved": "http://registry.npmjs.org/babel-plugin-syntax-exponentiation-operator/-/babel-plugin-syntax-exponentiation-operator-6.13.0.tgz", "integrity": "sha1-nufoM3KQ2pUoggGmpX9BcDF4MN4=", "dev": true }, "babel-plugin-syntax-object-rest-spread": { "version": "6.13.0", - "resolved": "https://registry.npmjs.org/babel-plugin-syntax-object-rest-spread/-/babel-plugin-syntax-object-rest-spread-6.13.0.tgz", + "resolved": "http://registry.npmjs.org/babel-plugin-syntax-object-rest-spread/-/babel-plugin-syntax-object-rest-spread-6.13.0.tgz", "integrity": "sha1-/WU28rzhODb/o6VFjEkDpZe7O/U=", "dev": true }, @@ -1314,7 +1314,7 @@ }, "browserify-rsa": { "version": "4.0.1", - "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz", + "resolved": "http://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz", "integrity": "sha1-IeCr+vbyApzy+vsTNWenAdQTVSQ=", "dev": true, "requires": { @@ -1367,7 +1367,7 @@ }, "buffer": { "version": "4.9.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", + "resolved": "http://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=", "dev": true, "requires": { @@ -1396,7 +1396,7 @@ }, "cac": { "version": "3.0.4", - "resolved": "https://registry.npmjs.org/cac/-/cac-3.0.4.tgz", + "resolved": "http://registry.npmjs.org/cac/-/cac-3.0.4.tgz", "integrity": "sha1-bSTO7Dcu/lybeYgIvH9JtHJCpO8=", "dev": true, "requires": { @@ -1417,7 +1417,7 @@ }, "camelcase-keys": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-3.0.0.tgz", + "resolved": "http://registry.npmjs.org/camelcase-keys/-/camelcase-keys-3.0.0.tgz", "integrity": "sha1-/AxsNgNj9zd+N5O5oWvM8QcMHKQ=", "dev": true, "requires": { @@ -1433,7 +1433,7 @@ }, "minimist": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=", "dev": true } @@ -1555,7 +1555,7 @@ }, "chalk": { "version": "1.1.3", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "resolved": "http://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", "dev": true, "requires": { @@ -2289,7 +2289,7 @@ "dependencies": { "regexpu-core": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/regexpu-core/-/regexpu-core-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/regexpu-core/-/regexpu-core-1.0.0.tgz", "integrity": "sha1-hqdj9Y7k18L2sQLkdkBQ3n7ZDGs=", "dev": true, "requires": { @@ -3047,7 +3047,7 @@ }, "events": { "version": "1.1.1", - "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "resolved": "http://registry.npmjs.org/events/-/events-1.1.1.tgz", "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=", "dev": true }, @@ -3102,7 +3102,7 @@ }, "expand-range": { "version": "1.8.2", - "resolved": "https://registry.npmjs.org/expand-range/-/expand-range-1.8.2.tgz", + "resolved": "http://registry.npmjs.org/expand-range/-/expand-range-1.8.2.tgz", "integrity": "sha1-opnv/TNf4nIeuujiV+x5ZE/IUzc=", "dev": true, "requires": { @@ -3436,7 +3436,8 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", - "dev": true + "dev": true, + "optional": true }, "concat-map": { "version": "0.0.1", @@ -3579,7 +3580,8 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", - "dev": true + "dev": true, + "optional": true }, "ini": { "version": "1.3.5", @@ -3593,6 +3595,7 @@ "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", "dev": true, + "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -3732,7 +3735,8 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", - "dev": true + "dev": true, + "optional": true }, "object-assign": { "version": "4.1.1", @@ -3746,6 +3750,7 @@ "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", "dev": true, + "optional": true, "requires": { "wrappy": "1" } @@ -3883,6 +3888,7 @@ "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", "dev": true, + "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -5397,7 +5403,7 @@ }, "jsesc": { "version": "1.3.0", - "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-1.3.0.tgz", + "resolved": "http://registry.npmjs.org/jsesc/-/jsesc-1.3.0.tgz", "integrity": "sha1-RsP+yMGJKxKwgz25vHYiF226s0s=", "dev": true }, @@ -5436,7 +5442,7 @@ }, "json5": { "version": "0.5.1", - "resolved": "https://registry.npmjs.org/json5/-/json5-0.5.1.tgz", + "resolved": "http://registry.npmjs.org/json5/-/json5-0.5.1.tgz", "integrity": "sha1-Hq3nrMASA0rYTiOWdn6tn6VJWCE=" }, "jsonify": { @@ -5985,7 +5991,7 @@ }, "minimist": { "version": "0.0.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", "dev": true }, @@ -6030,7 +6036,7 @@ }, "mkdirp": { "version": "0.5.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", "dev": true, "requires": { @@ -7442,7 +7448,7 @@ }, "os-homedir": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", + "resolved": "http://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=", "dev": true }, @@ -7463,7 +7469,7 @@ }, "os-tmpdir": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "resolved": "http://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", "dev": true }, @@ -7587,7 +7593,7 @@ }, "path-browserify": { "version": "0.0.0", - "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-0.0.0.tgz", + "resolved": "http://registry.npmjs.org/path-browserify/-/path-browserify-0.0.0.tgz", "integrity": "sha1-oLhwcpquIUAFt9UDLsLLuw+0RRo=", "dev": true }, @@ -7605,7 +7611,7 @@ }, "path-is-absolute": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "resolved": "http://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", "dev": true }, @@ -8682,7 +8688,7 @@ }, "regexpu-core": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/regexpu-core/-/regexpu-core-2.0.0.tgz", + "resolved": "http://registry.npmjs.org/regexpu-core/-/regexpu-core-2.0.0.tgz", "integrity": "sha1-SdA4g3uNz4v6W5pCE5k45uoq4kA=", "dev": true, "requires": { @@ -8712,13 +8718,13 @@ }, "regjsgen": { "version": "0.2.0", - "resolved": "https://registry.npmjs.org/regjsgen/-/regjsgen-0.2.0.tgz", + "resolved": "http://registry.npmjs.org/regjsgen/-/regjsgen-0.2.0.tgz", "integrity": "sha1-bAFq3qxVT3WCP+N6wFuS1aTtsfc=", "dev": true }, "regjsparser": { "version": "0.1.5", - "resolved": "https://registry.npmjs.org/regjsparser/-/regjsparser-0.1.5.tgz", + "resolved": "http://registry.npmjs.org/regjsparser/-/regjsparser-0.1.5.tgz", "integrity": "sha1-fuj4Tcb6eS0/0K4ijSS9lJ6tIFw=", "dev": true, "requires": { @@ -8727,7 +8733,7 @@ "dependencies": { "jsesc": { "version": "0.5.0", - "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-0.5.0.tgz", + "resolved": "http://registry.npmjs.org/jsesc/-/jsesc-0.5.0.tgz", "integrity": "sha1-597mbjXW/Bb3EP6R1c9p9w8IkR0=", "dev": true } @@ -9581,7 +9587,7 @@ }, "sprintf-js": { "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "resolved": "http://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", "dev": true }, @@ -9815,7 +9821,7 @@ }, "stream-browserify": { "version": "2.0.1", - "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.1.tgz", + "resolved": "http://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.1.tgz", "integrity": "sha1-ZiZu5fm9uZQKTkUUyvtDu3Hlyds=", "dev": true, "requires": { @@ -9879,7 +9885,7 @@ }, "strip-ansi": { "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", "dev": true, "requires": { @@ -10220,7 +10226,7 @@ }, "tty-browserify": { "version": "0.0.0", - "resolved": "https://registry.npmjs.org/tty-browserify/-/tty-browserify-0.0.0.tgz", + "resolved": "http://registry.npmjs.org/tty-browserify/-/tty-browserify-0.0.0.tgz", "integrity": "sha1-oVe6QC2iTpv5V/mqadUk7tQpAaY=", "dev": true }, @@ -10580,7 +10586,7 @@ }, "vm-browserify": { "version": "0.0.4", - "resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-0.0.4.tgz", + "resolved": "http://registry.npmjs.org/vm-browserify/-/vm-browserify-0.0.4.tgz", "integrity": "sha1-XX6kW7755Kb/ZflUOOCofDV9WnM=", "dev": true, "requires": { diff --git a/src/WorkerPool.js b/src/WorkerPool.js index 36f1ad0..a578b95 100644 --- a/src/WorkerPool.js +++ b/src/WorkerPool.js @@ -26,8 +26,8 @@ class PoolWorker { // when the kernel hits the limit of open files. // More info can be found on: https://github.com/webpack-contrib/thread-loader/issues/2 if (!this.worker.stdio) { - throw new Error(`Failed to create the worker pool with workerId: ${workerId} and \ - configuration: ${options}. Please verify if you hit the OS open files limit.`); + throw new Error(`Failed to create the worker pool with workerId: ${workerId} and ${'' + }configuration: ${JSON.stringify(options)}. Please verify if you hit the OS open files limit.`); } const [, , , readPipe, writePipe] = this.worker.stdio; diff --git a/test/__snapshots__/workerPool.test.js.snap b/test/__snapshots__/workerPool.test.js.snap new file mode 100644 index 0000000..616344b --- /dev/null +++ b/test/__snapshots__/workerPool.test.js.snap @@ -0,0 +1,3 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`workerPool should throw an error when worker.stdio is undefined 1`] = `"Failed to create the worker pool with workerId: 1 and configuration: {}. Please verify if you hit the OS open files limit."`; diff --git a/test/workerPool.test.js b/test/workerPool.test.js new file mode 100644 index 0000000..6f3ae2b --- /dev/null +++ b/test/workerPool.test.js @@ -0,0 +1,32 @@ +import childProcess from 'child_process'; +import stream from 'stream'; +import WorkerPool from '../src/WorkerPool'; + +jest.mock('child_process', () => { + return { + spawn: jest.fn(() => { + return {}; + }), + }; +}); + +describe('workerPool', () => { + it('should throw an error when worker.stdio is undefined', () => { + childProcess.spawn.mockImplementationOnce(() => { return {}; }); + + const workerPool = new WorkerPool({}); + expect(() => workerPool.createWorker()).toThrowErrorMatchingSnapshot(); + expect(() => workerPool.createWorker()).toThrowError('Please verify if you hit the OS open files limit'); + }); + + it('should not throw an error when worker.stdio is defined', () => { + childProcess.spawn.mockImplementationOnce(() => { + return { + stdio: new Array(5).fill(new stream.PassThrough()), + }; + }); + + const workerPool = new WorkerPool({}); + expect(() => workerPool.createWorker()).not.toThrow(); + }); +});