From d21daa6e1728cdac2d55627945e0257c99e001c2 Mon Sep 17 00:00:00 2001 From: Charles Lyding <19598772+clydin@users.noreply.github.com> Date: Fri, 25 Jun 2021 10:58:52 -0400 Subject: [PATCH 1/2] refactor(@angular-devkit/build-angular): remove `jest-worker` direct dependency The worker pool for differential loading and i18n processing is now managed by the `piscina` dependency. This dependency is already used within the recently added JavaScript optimizer refactoring and reduces both the number of direct dependencies and amount of code to setup the worker pools. --- package.json | 1 - .../angular_devkit/build_angular/BUILD.bazel | 1 - .../angular_devkit/build_angular/package.json | 1 - .../src/utils/action-executor.ts | 74 ++++--------------- .../build_angular/src/utils/process-bundle.ts | 15 +--- yarn.lock | 2 +- 6 files changed, 20 insertions(+), 74 deletions(-) diff --git a/package.json b/package.json index f4f3f27508ee..808493c3b666 100644 --- a/package.json +++ b/package.json @@ -166,7 +166,6 @@ "jasmine": "^3.3.1", "jasmine-core": "~3.7.0", "jasmine-spec-reporter": "~7.0.0", - "jest-worker": "27.0.2", "jquery": "^3.3.1", "jsonc-parser": "3.0.0", "karma": "~6.3.0", diff --git a/packages/angular_devkit/build_angular/BUILD.bazel b/packages/angular_devkit/build_angular/BUILD.bazel index 70776eeeca1a..5c3c68409104 100644 --- a/packages/angular_devkit/build_angular/BUILD.bazel +++ b/packages/angular_devkit/build_angular/BUILD.bazel @@ -154,7 +154,6 @@ ts_library( "@npm//glob", "@npm//https-proxy-agent", "@npm//inquirer", - "@npm//jest-worker", "@npm//karma", "@npm//karma-source-map-support", "@npm//less", diff --git a/packages/angular_devkit/build_angular/package.json b/packages/angular_devkit/build_angular/package.json index b20b7cd271de..20eb9a84cf7d 100644 --- a/packages/angular_devkit/build_angular/package.json +++ b/packages/angular_devkit/build_angular/package.json @@ -38,7 +38,6 @@ "glob": "7.1.7", "https-proxy-agent": "5.0.0", "inquirer": "8.1.1", - "jest-worker": "27.0.2", "karma-source-map-support": "1.4.0", "less": "4.1.1", "less-loader": "10.0.0", diff --git a/packages/angular_devkit/build_angular/src/utils/action-executor.ts b/packages/angular_devkit/build_angular/src/utils/action-executor.ts index b505a75e3021..d7548192e060 100644 --- a/packages/angular_devkit/build_angular/src/utils/action-executor.ts +++ b/packages/angular_devkit/build_angular/src/utils/action-executor.ts @@ -6,73 +6,40 @@ * found in the LICENSE file at https://angular.io/license */ -import { Worker as JestWorker } from 'jest-worker'; -import * as os from 'os'; -import * as path from 'path'; -import { serialize } from 'v8'; +import Piscina from 'piscina'; import { BundleActionCache } from './action-cache'; import { maxWorkers } from './environment-options'; import { I18nOptions } from './i18n-options'; import { InlineOptions, ProcessBundleOptions, ProcessBundleResult } from './process-bundle'; -let workerFile = require.resolve('./process-bundle'); -workerFile = - path.extname(workerFile) === '.ts' ? require.resolve('./process-bundle-bootstrap') : workerFile; +const workerFile = require.resolve('./process-bundle'); export class BundleActionExecutor { - private largeWorker?: JestWorker; - private smallWorker?: JestWorker; + private workerPool?: Piscina; private cache?: BundleActionCache; constructor( private workerOptions: { cachePath?: string; i18n: I18nOptions }, integrityAlgorithm?: string, - private readonly sizeThreshold = 32 * 1024, ) { if (workerOptions.cachePath) { this.cache = new BundleActionCache(workerOptions.cachePath, integrityAlgorithm); } } - private static executeMethod(worker: JestWorker, method: string, input: unknown): Promise { - return (worker as unknown as Record Promise>)[method](input); - } - - private ensureLarge(): JestWorker { - if (this.largeWorker) { - return this.largeWorker; - } - - // larger files are processed in a separate process to limit memory usage in the main process - return (this.largeWorker = new JestWorker(workerFile, { - exposedMethods: ['process', 'inlineLocales'], - setupArgs: [[...serialize(this.workerOptions)]], - numWorkers: maxWorkers, - })); - } - - private ensureSmall(): JestWorker { - if (this.smallWorker) { - return this.smallWorker; + private ensureWorkerPool(): Piscina { + if (this.workerPool) { + return this.workerPool; } - // small files are processed in a limited number of threads to improve speed - // The limited number also prevents a large increase in memory usage for an otherwise short operation - return (this.smallWorker = new JestWorker(workerFile, { - exposedMethods: ['process', 'inlineLocales'], - setupArgs: [this.workerOptions], - numWorkers: os.cpus().length < 2 ? 1 : 2, - enableWorkerThreads: true, - })); - } + this.workerPool = new Piscina({ + filename: workerFile, + name: 'process', + workerData: this.workerOptions, + maxThreads: maxWorkers, + }); - private executeAction(method: string, action: { code: string }): Promise { - // code.length is not an exact byte count but close enough for this - if (action.code.length > this.sizeThreshold) { - return BundleActionExecutor.executeMethod(this.ensureLarge(), method, action); - } else { - return BundleActionExecutor.executeMethod(this.ensureSmall(), method, action); - } + return this.workerPool; } async process(action: ProcessBundleOptions): Promise { @@ -89,7 +56,7 @@ export class BundleActionExecutor { } catch {} } - return this.executeAction('process', action); + return this.ensureWorkerPool().run(action, { name: 'process' }); } processAll(actions: Iterable): AsyncIterable { @@ -99,7 +66,7 @@ export class BundleActionExecutor { async inline( action: InlineOptions, ): Promise<{ file: string; diagnostics: { type: string; message: string }[]; count: number }> { - return this.executeAction('inlineLocales', action); + return this.ensureWorkerPool().run(action, { name: 'inlineLocales' }); } inlineAll(actions: Iterable) { @@ -129,15 +96,6 @@ export class BundleActionExecutor { } stop(): void { - // Floating promises are intentional here - // https://github.com/facebook/jest/tree/56079a5aceacf32333089cea50c64385885fee26/packages/jest-worker#end - if (this.largeWorker) { - // eslint-disable-next-line @typescript-eslint/no-floating-promises - this.largeWorker.end(); - } - if (this.smallWorker) { - // eslint-disable-next-line @typescript-eslint/no-floating-promises - this.smallWorker.end(); - } + void this.workerPool?.destroy(); } } diff --git a/packages/angular_devkit/build_angular/src/utils/process-bundle.ts b/packages/angular_devkit/build_angular/src/utils/process-bundle.ts index 09a25e1facb6..77987699904c 100644 --- a/packages/angular_devkit/build_angular/src/utils/process-bundle.ts +++ b/packages/angular_devkit/build_angular/src/utils/process-bundle.ts @@ -23,8 +23,8 @@ import * as fs from 'fs'; import * as path from 'path'; import { RawSourceMap, SourceMapConsumer, SourceMapGenerator } from 'source-map'; import { minify } from 'terser'; -import * as v8 from 'v8'; import { sources } from 'webpack'; +import { workerData } from 'worker_threads'; import { allowMangle, allowMinify, shouldBeautify } from './environment-options'; import { I18nOptions } from './i18n-options'; @@ -78,16 +78,7 @@ export const enum CacheKey { DownlevelMap = 3, } -let cachePath: string | undefined; -let i18n: I18nOptions | undefined; - -export function setup(data: number[] | { cachePath: string; i18n: I18nOptions }): void { - const options = Array.isArray(data) - ? (v8.deserialize(Buffer.from(data)) as { cachePath: string; i18n: I18nOptions }) - : data; - cachePath = options.cachePath; - i18n = options.i18n; -} +const { cachePath, i18n } = (workerData || {}) as { cachePath?: string; i18n?: I18nOptions }; async function cachePut( content: string, @@ -413,7 +404,7 @@ async function terserMangle( code, options.map, outputCode, - (minifyOutput.map as unknown) as RawSourceMap, + minifyOutput.map as unknown as RawSourceMap, options.filename || '0', code.length > FAST_SOURCEMAP_THRESHOLD, ); diff --git a/yarn.lock b/yarn.lock index 33e2ffc87216..836215343d7a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6695,7 +6695,7 @@ jasminewd2@^2.1.0: resolved "https://registry.yarnpkg.com/jasminewd2/-/jasminewd2-2.2.0.tgz#e37cf0b17f199cce23bea71b2039395246b4ec4e" integrity sha1-43zwsX8ZnM4jvqcbIDk5Uka07E4= -jest-worker@27.0.2, jest-worker@^27.0.2: +jest-worker@^27.0.2: version "27.0.2" resolved "https://registry.yarnpkg.com/jest-worker/-/jest-worker-27.0.2.tgz#4ebeb56cef48b3e7514552f80d0d80c0129f0b05" integrity sha512-EoBdilOTTyOgmHXtw/cPc+ZrCA0KJMrkXzkrPGNwLmnvvlN1nj7MPrxpT7m+otSv2e1TLaVffzDnE/LB14zJMg== From 8b112c8fe1625205bc3e025b53a20280e9cec34f Mon Sep 17 00:00:00 2001 From: Charles Lyding <19598772+clydin@users.noreply.github.com> Date: Fri, 25 Jun 2021 11:40:52 -0400 Subject: [PATCH 2/2] refactor(@angular-devkit/build-angular): lazy load Webpack in differential loading processor Webpack is a large dependency with a large dependency graph. By only loading Webpack when needed in the differential loading and i18n processors, initial startup time can be improved and memory usage can be reduced. --- .../build_angular/src/utils/process-bundle.ts | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/packages/angular_devkit/build_angular/src/utils/process-bundle.ts b/packages/angular_devkit/build_angular/src/utils/process-bundle.ts index 77987699904c..213a34d0ca23 100644 --- a/packages/angular_devkit/build_angular/src/utils/process-bundle.ts +++ b/packages/angular_devkit/build_angular/src/utils/process-bundle.ts @@ -23,15 +23,16 @@ import * as fs from 'fs'; import * as path from 'path'; import { RawSourceMap, SourceMapConsumer, SourceMapGenerator } from 'source-map'; import { minify } from 'terser'; -import { sources } from 'webpack'; import { workerData } from 'worker_threads'; import { allowMangle, allowMinify, shouldBeautify } from './environment-options'; import { I18nOptions } from './i18n-options'; -const { ConcatSource, OriginalSource, ReplaceSource, SourceMapSource } = sources; - type LocalizeUtilities = typeof import('@angular/localize/src/tools/src/source_file_utils'); +// Lazy loaded webpack-sources object +// Webpack is only imported if needed during the processing +let webpackSources: typeof import('webpack').sources | undefined; + // If code size is larger than 500KB, consider lower fidelity but faster sourcemap merge const FAST_SOURCEMAP_THRESHOLD = 500 * 1024; @@ -215,9 +216,14 @@ async function mergeSourceMaps( return mergeSourceMapsFast(inputSourceMap, resultSourceMap); } + // Load Webpack only when needed + if (webpackSources === undefined) { + webpackSources = (await import('webpack')).sources; + } + // SourceMapSource produces high-quality sourcemaps // Final sourcemap will always be available when providing the input sourcemaps - const finalSourceMap = new SourceMapSource( + const finalSourceMap = new webpackSources.SourceMapSource( resultCode, filename, resultSourceMap, @@ -726,6 +732,12 @@ async function inlineLocalesDirect(ast: ParseResult, options: InlineOptions) { delete inputMap.sourceRoot; } + // Load Webpack only when needed + if (webpackSources === undefined) { + webpackSources = (await import('webpack')).sources; + } + const { ConcatSource, OriginalSource, ReplaceSource, SourceMapSource } = webpackSources; + for (const locale of i18n.inlineLocales) { const content = new ReplaceSource( inputMap @@ -752,12 +764,12 @@ async function inlineLocalesDirect(ast: ParseResult, options: InlineOptions) { content.replace(position.start, position.end - 1, code); } - let outputSource: sources.Source = content; + let outputSource: import('webpack').sources.Source = content; if (options.setLocale) { const setLocaleText = `var $localize=Object.assign(void 0===$localize?{}:$localize,{locale:"${locale}"});\n`; // If locale data is provided, load it and prepend to file - let localeDataSource: sources.Source | null = null; + let localeDataSource; const localeDataPath = i18n.locales[locale] && i18n.locales[locale].dataPath; if (localeDataPath) { const localeDataContent = await loadLocaleData(localeDataPath, true, options.es5);