From a1e4341a5167f8cfe17cdc80013580dc331f5cce Mon Sep 17 00:00:00 2001 From: Andrew Levine Date: Thu, 22 Aug 2019 17:53:12 -0500 Subject: [PATCH] Add module for background minification in a worker pool --- package-lock.json | 137 +++++++++--------- package.json | 5 +- .../__fixtures__/basic-minify/bundle.js | 4 + .../source-mapped-minify/bundle.js | 6 + src/__tests__/minifyWorker.unit.js | 42 ++++++ src/createMinifier.ts | 13 ++ src/fsPromises.ts | 1 + src/minifyWorker.ts | 77 ++++++++++ 8 files changed, 215 insertions(+), 70 deletions(-) create mode 100644 src/__tests__/__fixtures__/basic-minify/bundle.js create mode 100644 src/__tests__/__fixtures__/source-mapped-minify/bundle.js create mode 100644 src/__tests__/minifyWorker.unit.js create mode 100644 src/createMinifier.ts create mode 100644 src/minifyWorker.ts diff --git a/package-lock.json b/package-lock.json index 2eb4efa..3ae16fa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1078,9 +1078,7 @@ "commander": { "version": "2.20.0", "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.0.tgz", - "integrity": "sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ==", - "dev": true, - "optional": true + "integrity": "sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ==" }, "component-emitter": { "version": "1.3.0", @@ -1133,6 +1131,12 @@ "which": "^1.2.9" } }, + "crypto-random-string": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-1.0.0.tgz", + "integrity": "sha1-ojD2T1aDEOFJgAmUB5DsmVRbyn4=", + "dev": true + }, "cssom": { "version": "0.3.8", "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz", @@ -1783,8 +1787,7 @@ "ansi-regex": { "version": "2.1.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "aproba": { "version": "1.2.0", @@ -1805,14 +1808,12 @@ "balanced-match": { "version": "1.0.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, "dev": true, - "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -1827,20 +1828,17 @@ "code-point-at": { "version": "1.1.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "concat-map": { "version": "0.0.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "console-control-strings": { "version": "1.1.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "core-util-is": { "version": "1.0.2", @@ -1957,8 +1955,7 @@ "inherits": { "version": "2.0.3", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "ini": { "version": "1.3.5", @@ -1970,7 +1967,6 @@ "version": "1.0.0", "bundled": true, "dev": true, - "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -1985,7 +1981,6 @@ "version": "3.0.4", "bundled": true, "dev": true, - "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -1993,14 +1988,12 @@ "minimist": { "version": "0.0.8", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "minipass": { "version": "2.3.5", "bundled": true, "dev": true, - "optional": true, "requires": { "safe-buffer": "^5.1.2", "yallist": "^3.0.0" @@ -2019,7 +2012,6 @@ "version": "0.5.1", "bundled": true, "dev": true, - "optional": true, "requires": { "minimist": "0.0.8" } @@ -2100,8 +2092,7 @@ "number-is-nan": { "version": "1.0.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "object-assign": { "version": "4.1.1", @@ -2113,7 +2104,6 @@ "version": "1.4.0", "bundled": true, "dev": true, - "optional": true, "requires": { "wrappy": "1" } @@ -2199,8 +2189,7 @@ "safe-buffer": { "version": "5.1.2", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "safer-buffer": { "version": "2.1.2", @@ -2236,7 +2225,6 @@ "version": "1.0.2", "bundled": true, "dev": true, - "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -2256,7 +2244,6 @@ "version": "3.0.1", "bundled": true, "dev": true, - "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -2300,14 +2287,12 @@ "wrappy": { "version": "1.0.2", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "yallist": { "version": "3.0.3", "bundled": true, - "dev": true, - "optional": true + "dev": true } } }, @@ -2427,8 +2412,7 @@ "has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", - "dev": true + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=" }, "has-symbols": { "version": "1.0.0", @@ -3278,12 +3262,11 @@ } }, "jest-worker": { - "version": "24.6.0", - "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-24.6.0.tgz", - "integrity": "sha512-jDwgW5W9qGNvpI1tNnvajh0a5IE/PuGLFmHk6aR/BZFz8tSgGw17GsDPXAJ6p91IvYDjOw8GpFbvvZGAK+DPQQ==", - "dev": true, + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-24.9.0.tgz", + "integrity": "sha512-51PE4haMSXcHohnSMdM42anbvZANYTqMrr52tVKPqqsPJMzoP6FYYDVqahX/HrAoKEKz3uUPzSvKs9A3qR4iVw==", "requires": { - "merge-stream": "^1.0.1", + "merge-stream": "^2.0.0", "supports-color": "^6.1.0" }, "dependencies": { @@ -3291,7 +3274,6 @@ "version": "6.1.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-6.1.0.tgz", "integrity": "sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ==", - "dev": true, "requires": { "has-flag": "^3.0.0" } @@ -3565,13 +3547,9 @@ } }, "merge-stream": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-1.0.1.tgz", - "integrity": "sha1-QEEgLVCKNCugAXQAjfDCUbjBNeE=", - "dev": true, - "requires": { - "readable-stream": "^2.0.1" - } + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==" }, "merge2": { "version": "1.2.3", @@ -4128,12 +4106,6 @@ "react-is": "^16.8.4" } }, - "process-nextick-args": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", - "dev": true - }, "prompts": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.1.0.tgz", @@ -4199,21 +4171,6 @@ "read-pkg": "^3.0.0" } }, - "readable-stream": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", - "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", - "dev": true, - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, "realpath-native": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/realpath-native/-/realpath-native-1.1.0.tgz", @@ -4852,6 +4809,33 @@ "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", "dev": true }, + "temp-dir": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/temp-dir/-/temp-dir-1.0.0.tgz", + "integrity": "sha1-CnwOom06Oa+n4OvqnB/AvE2qAR0=", + "dev": true + }, + "tempy": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/tempy/-/tempy-0.3.0.tgz", + "integrity": "sha512-WrH/pui8YCwmeiAoxV+lpRH9HpRtgBhSR2ViBPgpGb/wnYDzp21R4MN45fsCGvLROvY67o3byhJRYRONJyImVQ==", + "dev": true, + "requires": { + "temp-dir": "^1.0.0", + "type-fest": "^0.3.1", + "unique-string": "^1.0.0" + } + }, + "terser": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/terser/-/terser-4.2.0.tgz", + "integrity": "sha512-6lPt7lZdZ/13icQJp8XasFOwZjFJkxFFIb/N1fhYEQNoNI3Ilo3KABZ9OocZvZoB39r6SiIk/0+v/bt8nZoSeA==", + "requires": { + "commander": "^2.20.0", + "source-map": "~0.6.1", + "source-map-support": "~0.5.12" + } + }, "test-exclude": { "version": "5.2.3", "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-5.2.3.tgz", @@ -5007,6 +4991,12 @@ "prelude-ls": "~1.1.2" } }, + "type-fest": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.3.1.tgz", + "integrity": "sha512-cUGJnCdr4STbePCgqNFbpVNCepa+kAVohJs1sLhxzdH+gnEoOd8VhbYa7pD3zZYGiURWM2xzEII3fQcRizDkYQ==", + "dev": true + }, "typescript": { "version": "3.5.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.5.3.tgz", @@ -5036,6 +5026,15 @@ "set-value": "^2.0.1" } }, + "unique-string": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unique-string/-/unique-string-1.0.0.tgz", + "integrity": "sha1-nhBXzKhRq7kzmPizOuGHuZyuwRo=", + "dev": true, + "requires": { + "crypto-random-string": "^1.0.0" + } + }, "unset-value": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unset-value/-/unset-value-1.0.0.tgz", diff --git a/package.json b/package.json index 36fe73c..1f91658 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "@types/node": "^12.6.8", "jest": "^24.8.0", "prettier": "^1.18.2", + "tempy": "^0.3.0", "ts-jest": "^24.0.2", "typescript": "^3.5.3" }, @@ -52,10 +53,12 @@ "fast-xml-parser": "^3.12.19", "fromentries": "^1.1.0", "htmlparser2": "^3.10.1", + "jest-worker": "^24.9.0", "jsesc": "^2.5.2", "magic-string": "^0.25.3", "pretty-bytes": "^5.3.0", "requirejs": "^2.3.6", - "source-map-support": "^0.5.12" + "source-map-support": "^0.5.12", + "terser": "^4.2.0" } } diff --git a/src/__tests__/__fixtures__/basic-minify/bundle.js b/src/__tests__/__fixtures__/basic-minify/bundle.js new file mode 100644 index 0000000..6e48385 --- /dev/null +++ b/src/__tests__/__fixtures__/basic-minify/bundle.js @@ -0,0 +1,4 @@ +define(['a'], function(a) { + var c = console; + c.log(a); +}); diff --git a/src/__tests__/__fixtures__/source-mapped-minify/bundle.js b/src/__tests__/__fixtures__/source-mapped-minify/bundle.js new file mode 100644 index 0000000..c3a189b --- /dev/null +++ b/src/__tests__/__fixtures__/source-mapped-minify/bundle.js @@ -0,0 +1,6 @@ +define(['a'], function(a) { + var log = console.log; + log(a); +}); + +//# sourceMappingURL=bundle.js.map diff --git a/src/__tests__/minifyWorker.unit.js b/src/__tests__/minifyWorker.unit.js new file mode 100644 index 0000000..1636f3f --- /dev/null +++ b/src/__tests__/minifyWorker.unit.js @@ -0,0 +1,42 @@ +const { join, basename } = require('path'); +const { copyFile } = require('../fsPromises'); +const { minify } = require('../minifyWorker'); +const { readFile } = require('../fsPromises'); +const tempy = require('tempy'); + +const moveFileToTmpDir = async path => { + const tmpDir = tempy.directory(); + const filename = basename(path); + const tmpPath = join(tmpDir, filename); + await copyFile(path, tmpPath); + return { tmpPath, tmpDir }; +}; +const getFixturePath = (fixtureName, path) => + join(__dirname, '__fixtures__', fixtureName, path); + +test('Minifies JS file', async () => { + const srcPath = getFixturePath('basic-minify', 'bundle.js'); + const { tmpDir, tmpPath } = await moveFileToTmpDir(srcPath); + + const result = await minify(tmpPath); + const resultFilePath = join(tmpDir, result.minFilename); + const minifiedCode = await readFile(resultFilePath, 'utf8'); + + expect(minifiedCode).toMatchInlineSnapshot( + `"define([\\"a\\"],function(n){console.log(n)});"`, + ); +}); + +test('Minfies JS file and chains sourcemap', async () => { + const srcPath = getFixturePath('source-mapped-minify', 'bundle.js'); + const { tmpDir, tmpPath } = await moveFileToTmpDir(srcPath); + + const result = await minify(tmpPath); + const resultFilePath = join(tmpDir, result.minFilename); + const minifiedCode = await readFile(resultFilePath, 'utf8'); + + expect(minifiedCode).toMatchInlineSnapshot( + `"define([\\"a\\"],function(n){(0,console.log)(n)});"`, + ); + // TODO: Assert that sourcemap was chained properly +}); diff --git a/src/createMinifier.ts b/src/createMinifier.ts new file mode 100644 index 0000000..04489b2 --- /dev/null +++ b/src/createMinifier.ts @@ -0,0 +1,13 @@ +import Worker from 'jest-worker'; +import { minify } from './minifyWorker'; + +export function createMinifier() { + const worker = new Worker(require.resolve('./minifyWorker'), { + forkOptions: { + // surface console.log and friends in worker + stdio: 'inherit', + }, + }); + + return (worker as any).minify as typeof minify; +} diff --git a/src/fsPromises.ts b/src/fsPromises.ts index 39ec111..319f462 100644 --- a/src/fsPromises.ts +++ b/src/fsPromises.ts @@ -12,3 +12,4 @@ export const readFile = promisify(fs.readFile); export const mkdir = promisify(fs.mkdir); export const writeFile = promisify(fs.writeFile); export const readdir = promisify(fs.readdir); +export const copyFile = promisify(fs.copyFile); diff --git a/src/minifyWorker.ts b/src/minifyWorker.ts new file mode 100644 index 0000000..e782f7b --- /dev/null +++ b/src/minifyWorker.ts @@ -0,0 +1,77 @@ +import { join, parse, dirname } from 'path'; +import { readFile, writeFile } from './fsPromises'; +import terser from 'terser'; +import { RawSourceMap } from 'source-map'; +import { wrapP } from './wrapP'; + +export type MinificationResult = { + totalBytes: number; + minFilename: string; +}; + +export async function minify(path: string): Promise { + const source = await readFile(path, 'utf8'); + const { 1: sourceMapName } = + source.match(/\/\/#\ssourceMappingURL=(.+\.js\.map)/) || []; + const parsedPath = parse(path); + const targetFilename = `${parsedPath.name}.min${parsedPath.ext}`; + const targetFilePath = join(parsedPath.dir, targetFilename); + + return sourceMapName + ? minifyWithInputMap( + source, + targetFilename, + targetFilePath, + path, + sourceMapName, + ) + : minifyWithoutInputMap(source, targetFilename, targetFilePath); +} + +async function minifyWithoutInputMap( + source: string, + targetFilename: string, + targetFilePath: string, +) { + const result = terser.minify(source); + if (result.error) throw result.error; + await writeFile(targetFilePath, result.code); + return { + totalBytes: Buffer.from(result.code as string).byteLength, + minFilename: targetFilename, + }; +} + +async function minifyWithInputMap( + source: string, + targetFilename: string, + targetFilePath: string, + sourcePath: string, + sourceMapName: string, +) { + const sourceDir = dirname(sourcePath); + const mapPath = join(sourceDir, sourceMapName); + const [err, mapSrc] = await wrapP(readFile(mapPath, 'utf8')); + + if (err) { + // We don't want to fail a build because of a missing sourcemap, + // so instead we fall back to just not using an input map + return minifyWithoutInputMap(source, targetFilename, targetFilePath); + } + + const map = JSON.parse(mapSrc as string) as RawSourceMap; + const result = terser.minify(source, { + sourceMap: { + content: map, + url: sourceMapName, + }, + }); + + if (result.error) throw result.error; + + await writeFile(targetFilePath, result.code); + return { + totalBytes: Buffer.from(result.code as string).byteLength, + minFilename: targetFilename, + }; +}