From c5515563f1adae3fc0c51baed10b809861d11b6f Mon Sep 17 00:00:00 2001 From: Alan Agius Date: Wed, 12 Jan 2022 08:03:14 +0100 Subject: [PATCH] fix(@angular-devkit/build-angular): automatically purge stale build cache entries With every build-angular release, previously created cache entries get stale and are no longer used. This causes the cache to keep growing as older files are not purged. With this change we automatically purge entries that have been created with older version of build-angular and can no longer be used with the current installed version. Closes #22323 --- .../build_angular/src/babel/webpack-loader.ts | 3 +- .../src/builders/browser/index.ts | 4 ++ .../src/builders/dev-server/index.ts | 4 ++ .../src/builders/extract-i18n/index.ts | 4 ++ .../build_angular/src/builders/karma/index.ts | 4 ++ .../src/builders/ng-packagr/index.ts | 4 ++ .../src/builders/server/index.ts | 4 ++ .../src/utils/index-file/inline-fonts.ts | 5 +- .../src/utils/normalize-cache.ts | 12 ++++- .../src/utils/package-version.ts | 9 ++++ .../build_angular/src/utils/purge-cache.ts | 53 +++++++++++++++++++ .../src/webpack/utils/helpers.ts | 5 +- .../e2e/tests/build/disk-cache-purge.ts | 27 ++++++++++ 13 files changed, 129 insertions(+), 9 deletions(-) create mode 100644 packages/angular_devkit/build_angular/src/utils/package-version.ts create mode 100644 packages/angular_devkit/build_angular/src/utils/purge-cache.ts create mode 100644 tests/legacy-cli/e2e/tests/build/disk-cache-purge.ts diff --git a/packages/angular_devkit/build_angular/src/babel/webpack-loader.ts b/packages/angular_devkit/build_angular/src/babel/webpack-loader.ts index 958b0cf7f99b..3653ab20c4fc 100644 --- a/packages/angular_devkit/build_angular/src/babel/webpack-loader.ts +++ b/packages/angular_devkit/build_angular/src/babel/webpack-loader.ts @@ -10,6 +10,7 @@ import remapping from '@ampproject/remapping'; import { custom } from 'babel-loader'; import { ScriptTarget } from 'typescript'; import { loadEsmModule } from '../utils/load-esm'; +import { VERSION } from '../utils/package-version'; import { ApplicationPresetOptions, I18nPluginCreators } from './presets/application'; interface AngularCustomOptions extends Omit { @@ -196,7 +197,7 @@ export default custom(() => { ...baseOptions, ...rawOptions, cacheIdentifier: JSON.stringify({ - buildAngular: require('../../package.json').version, + buildAngular: VERSION, customOptions, baseOptions, rawOptions, diff --git a/packages/angular_devkit/build_angular/src/builders/browser/index.ts b/packages/angular_devkit/build_angular/src/builders/browser/index.ts index 255c07b47f25..4236bb37d166 100644 --- a/packages/angular_devkit/build_angular/src/builders/browser/index.ts +++ b/packages/angular_devkit/build_angular/src/builders/browser/index.ts @@ -39,6 +39,7 @@ import { import { normalizeCacheOptions } from '../../utils/normalize-cache'; import { ensureOutputPaths } from '../../utils/output-paths'; import { generateEntryPoints } from '../../utils/package-chunk-sort'; +import { purgeStaleBuildCache } from '../../utils/purge-cache'; import { augmentAppWithServiceWorker } from '../../utils/service-worker'; import { Spinner } from '../../utils/spinner'; import { getSupportedBrowsers } from '../../utils/supported-browsers'; @@ -157,6 +158,9 @@ export function buildWebpackBrowser( ), ); + // Purge old build disk cache. + await purgeStaleBuildCache(context); + checkInternetExplorerSupport(sysProjectRoot, context.logger); return { diff --git a/packages/angular_devkit/build_angular/src/builders/dev-server/index.ts b/packages/angular_devkit/build_angular/src/builders/dev-server/index.ts index 208c095bac90..5a19d8002ba8 100644 --- a/packages/angular_devkit/build_angular/src/builders/dev-server/index.ts +++ b/packages/angular_devkit/build_angular/src/builders/dev-server/index.ts @@ -28,6 +28,7 @@ import { IndexHtmlTransform } from '../../utils/index-file/index-html-generator' import { createTranslationLoader } from '../../utils/load-translations'; import { NormalizedCachedOptions, normalizeCacheOptions } from '../../utils/normalize-cache'; import { generateEntryPoints } from '../../utils/package-chunk-sort'; +import { purgeStaleBuildCache } from '../../utils/purge-cache'; import { assertCompatibleAngularVersion } from '../../utils/version'; import { generateI18nBrowserWebpackConfigFromContext, @@ -90,6 +91,9 @@ export function serveWebpackBrowser( throw new Error('The builder requires a target.'); } + // Purge old build disk cache. + await purgeStaleBuildCache(context); + options.port = await checkPort(options.port ?? 4200, options.host || 'localhost'); if (options.hmr) { diff --git a/packages/angular_devkit/build_angular/src/builders/extract-i18n/index.ts b/packages/angular_devkit/build_angular/src/builders/extract-i18n/index.ts index 169763a86767..59ca53297c68 100644 --- a/packages/angular_devkit/build_angular/src/builders/extract-i18n/index.ts +++ b/packages/angular_devkit/build_angular/src/builders/extract-i18n/index.ts @@ -17,6 +17,7 @@ import webpack, { Configuration } from 'webpack'; import { ExecutionTransformer } from '../../transforms'; import { createI18nOptions } from '../../utils/i18n-options'; import { loadEsmModule } from '../../utils/load-esm'; +import { purgeStaleBuildCache } from '../../utils/purge-cache'; import { assertCompatibleAngularVersion } from '../../utils/version'; import { generateBrowserWebpackConfigFromContext } from '../../utils/webpack-browser-config'; import { getCommonConfig } from '../../webpack/configs'; @@ -130,6 +131,9 @@ export async function execute( // Check Angular version. assertCompatibleAngularVersion(context.workspaceRoot); + // Purge old build disk cache. + await purgeStaleBuildCache(context); + const browserTarget = targetFromTargetString(options.browserTarget); const browserOptions = await context.validateOptions( await context.getTargetOptions(browserTarget), diff --git a/packages/angular_devkit/build_angular/src/builders/karma/index.ts b/packages/angular_devkit/build_angular/src/builders/karma/index.ts index 1bcb2a348d53..ae5dcfbff171 100644 --- a/packages/angular_devkit/build_angular/src/builders/karma/index.ts +++ b/packages/angular_devkit/build_angular/src/builders/karma/index.ts @@ -14,6 +14,7 @@ import { Observable, from } from 'rxjs'; import { defaultIfEmpty, switchMap } from 'rxjs/operators'; import { Configuration } from 'webpack'; import { ExecutionTransformer } from '../../transforms'; +import { purgeStaleBuildCache } from '../../utils/purge-cache'; import { assertCompatibleAngularVersion } from '../../utils/version'; import { generateBrowserWebpackConfigFromContext } from '../../utils/webpack-browser-config'; import { getCommonConfig, getStylesConfig } from '../../webpack/configs'; @@ -32,6 +33,9 @@ async function initialize( context: BuilderContext, webpackConfigurationTransformer?: ExecutionTransformer, ): Promise<[typeof import('karma'), Configuration]> { + // Purge old build disk cache. + await purgeStaleBuildCache(context); + const { config } = await generateBrowserWebpackConfigFromContext( // only two properties are missing: // * `outputPath` which is fixed for tests diff --git a/packages/angular_devkit/build_angular/src/builders/ng-packagr/index.ts b/packages/angular_devkit/build_angular/src/builders/ng-packagr/index.ts index 27c379a1582b..0122bce230e4 100644 --- a/packages/angular_devkit/build_angular/src/builders/ng-packagr/index.ts +++ b/packages/angular_devkit/build_angular/src/builders/ng-packagr/index.ts @@ -11,6 +11,7 @@ import { join, resolve } from 'path'; import { Observable, from, of } from 'rxjs'; import { catchError, mapTo, switchMap } from 'rxjs/operators'; import { normalizeCacheOptions } from '../../utils/normalize-cache'; +import { purgeStaleBuildCache } from '../../utils/purge-cache'; import { Schema as NgPackagrBuilderOptions } from './schema'; /** @@ -22,6 +23,9 @@ export function execute( ): Observable { return from( (async () => { + // Purge old build disk cache. + await purgeStaleBuildCache(context); + const root = context.workspaceRoot; const packager = (await import('ng-packagr')).ngPackagr(); diff --git a/packages/angular_devkit/build_angular/src/builders/server/index.ts b/packages/angular_devkit/build_angular/src/builders/server/index.ts index 216050d1d5df..848d9ab1a396 100644 --- a/packages/angular_devkit/build_angular/src/builders/server/index.ts +++ b/packages/angular_devkit/build_angular/src/builders/server/index.ts @@ -19,6 +19,7 @@ import { NormalizedBrowserBuilderSchema, deleteOutputDir } from '../../utils'; import { i18nInlineEmittedFiles } from '../../utils/i18n-inlining'; import { I18nOptions } from '../../utils/i18n-options'; import { ensureOutputPaths } from '../../utils/output-paths'; +import { purgeStaleBuildCache } from '../../utils/purge-cache'; import { assertCompatibleAngularVersion } from '../../utils/version'; import { generateI18nBrowserWebpackConfigFromContext } from '../../utils/webpack-browser-config'; import { getCommonConfig, getStylesConfig } from '../../webpack/configs'; @@ -146,6 +147,9 @@ async function initialize( i18n: I18nOptions; target: ScriptTarget; }> { + // Purge old build disk cache. + await purgeStaleBuildCache(context); + const originalOutputPath = options.outputPath; const { config, i18n, target } = await generateI18nBrowserWebpackConfigFromContext( { diff --git a/packages/angular_devkit/build_angular/src/utils/index-file/inline-fonts.ts b/packages/angular_devkit/build_angular/src/utils/index-file/inline-fonts.ts index 6d2a25263f8b..af7719687883 100644 --- a/packages/angular_devkit/build_angular/src/utils/index-file/inline-fonts.ts +++ b/packages/angular_devkit/build_angular/src/utils/index-file/inline-fonts.ts @@ -13,10 +13,9 @@ import proxyAgent from 'https-proxy-agent'; import { join } from 'path'; import { URL } from 'url'; import { NormalizedCachedOptions } from '../normalize-cache'; +import { VERSION } from '../package-version'; import { htmlRewritingStream } from './html-rewriting-stream'; -const packageVersion = require('../../../package.json').version; - interface FontProviderDetails { preconnectUrl: string; } @@ -156,7 +155,7 @@ export class InlineFontsProcessor { } private async getResponse(url: URL): Promise { - const key = `${packageVersion}|${url}`; + const key = `${VERSION}|${url}`; if (this.cachePath) { const entry = await cacache.get.info(this.cachePath, key); diff --git a/packages/angular_devkit/build_angular/src/utils/normalize-cache.ts b/packages/angular_devkit/build_angular/src/utils/normalize-cache.ts index cf3f9fc46f43..5cc6585e8735 100644 --- a/packages/angular_devkit/build_angular/src/utils/normalize-cache.ts +++ b/packages/angular_devkit/build_angular/src/utils/normalize-cache.ts @@ -7,12 +7,17 @@ */ import { json } from '@angular-devkit/core'; -import { resolve } from 'path'; +import { join, resolve } from 'path'; import { cachingDisabled } from './environment-options'; +import { VERSION } from './package-version'; export interface NormalizedCachedOptions { + /** Whether disk cache is enabled. */ enabled: boolean; + /** Disk cache path. Example: `/.angular/cache/v12.0.0`. */ path: string; + /** Disk cache base path. Example: `/.angular/cache`. */ + basePath: string; } interface CacheMetadata { @@ -49,8 +54,11 @@ export function normalizeCacheOptions( } } + const cacheBasePath = resolve(worspaceRoot, path); + return { enabled: cacheEnabled, - path: resolve(worspaceRoot, path), + basePath: cacheBasePath, + path: join(cacheBasePath, VERSION), }; } diff --git a/packages/angular_devkit/build_angular/src/utils/package-version.ts b/packages/angular_devkit/build_angular/src/utils/package-version.ts new file mode 100644 index 000000000000..4634959bd831 --- /dev/null +++ b/packages/angular_devkit/build_angular/src/utils/package-version.ts @@ -0,0 +1,9 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +export const VERSION: string = require('../../package.json').version; diff --git a/packages/angular_devkit/build_angular/src/utils/purge-cache.ts b/packages/angular_devkit/build_angular/src/utils/purge-cache.ts new file mode 100644 index 000000000000..6f11bbd938be --- /dev/null +++ b/packages/angular_devkit/build_angular/src/utils/purge-cache.ts @@ -0,0 +1,53 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import { BuilderContext } from '@angular-devkit/architect'; +import { PathLike, existsSync, promises as fsPromises } from 'fs'; +import { join } from 'path'; +import { normalizeCacheOptions } from './normalize-cache'; + +/** Delete stale cache directories used by previous versions of build-angular. */ +export async function purgeStaleBuildCache(context: BuilderContext): Promise { + const projectName = context.target?.project; + if (!projectName) { + return; + } + + const metadata = await context.getProjectMetadata(projectName); + const { basePath, path, enabled } = normalizeCacheOptions(metadata, context.workspaceRoot); + + if (!enabled || !existsSync(basePath)) { + return; + } + + // The below should be removed and replaced with just `rm` when support for Node.Js 12 is removed. + const { rm, rmdir } = fsPromises as typeof fsPromises & { + rm?: ( + path: PathLike, + options?: { + force?: boolean; + maxRetries?: number; + recursive?: boolean; + retryDelay?: number; + }, + ) => Promise; + }; + + const entriesToDelete = (await fsPromises.readdir(basePath, { withFileTypes: true })) + .filter((d) => join(basePath, d.name) !== path && d.isDirectory()) + .map((d) => { + const subPath = join(basePath, d.name); + try { + return rm + ? rm(subPath, { force: true, recursive: true, maxRetries: 3 }) + : rmdir(subPath, { recursive: true, maxRetries: 3 }); + } catch {} + }); + + await Promise.all(entriesToDelete); +} diff --git a/packages/angular_devkit/build_angular/src/webpack/utils/helpers.ts b/packages/angular_devkit/build_angular/src/webpack/utils/helpers.ts index 4944cb0d0c4d..d692c206d300 100644 --- a/packages/angular_devkit/build_angular/src/webpack/utils/helpers.ts +++ b/packages/angular_devkit/build_angular/src/webpack/utils/helpers.ts @@ -19,6 +19,7 @@ import { ExtraEntryPointClass, } from '../../builders/browser/schema'; import { WebpackConfigOptions } from '../../utils/build-options'; +import { VERSION } from '../../utils/package-version'; export interface HashFormat { chunk: string; @@ -122,8 +123,6 @@ export function getCacheSettings( ): WebpackOptionsNormalized['cache'] { const { enabled, path: cacheDirectory } = wco.buildOptions.cache; if (enabled) { - const packageVersion = require('../../../package.json').version; - return { type: 'filesystem', profile: wco.buildOptions.verbose, @@ -134,7 +133,7 @@ export function getCacheSettings( // None of which are "named". name: createHash('sha1') .update(angularVersion) - .update(packageVersion) + .update(VERSION) .update(wco.projectRoot) .update(JSON.stringify(wco.tsConfig)) .update( diff --git a/tests/legacy-cli/e2e/tests/build/disk-cache-purge.ts b/tests/legacy-cli/e2e/tests/build/disk-cache-purge.ts new file mode 100644 index 000000000000..732117419bed --- /dev/null +++ b/tests/legacy-cli/e2e/tests/build/disk-cache-purge.ts @@ -0,0 +1,27 @@ +import { join } from 'path'; +import { createDir, expectFileNotToExist, expectFileToExist } from '../../utils/fs'; +import { ng } from '../../utils/process'; +import { updateJsonFile } from '../../utils/project'; + +export default async function () { + const cachePath = '.angular/cache'; + const staleCachePath = join(cachePath, 'v1.0.0'); + + // Enable cache for all environments + await updateJsonFile('angular.json', (config) => { + config.cli ??= {}; + config.cli.cache = { + environment: 'all', + enabled: true, + path: cachePath, + }; + }); + + // Create a dummy stale disk cache directory. + await createDir(staleCachePath); + await expectFileToExist(staleCachePath); + + await ng('build'); + await expectFileToExist(cachePath); + await expectFileNotToExist(staleCachePath); +}