From ada14df1e15512cde00fcd43d774011e83eb00ab Mon Sep 17 00:00:00 2001 From: David Michon Date: Mon, 26 Sep 2022 13:29:08 -0700 Subject: [PATCH 1/4] [rush] Move hash to Operation level --- .../src/logic/ProjectChangeAnalyzer.ts | 45 ++--- libraries/rush-lib/src/logic/RushConstants.ts | 2 +- .../src/logic/buildCache/ProjectBuildCache.ts | 57 +----- .../buildCache/test/ProjectBuildCache.test.ts | 13 +- .../ProjectBuildCache.test.ts.snap | 3 + .../src/logic/operations/IOperationRunner.ts | 14 ++ .../src/logic/operations/Operation.ts | 39 ++++ .../operations/OperationExecutionManager.ts | 98 +++++++--- .../operations/OperationExecutionRecord.ts | 16 +- .../logic/operations/ShellOperationRunner.ts | 174 +++++++++--------- 10 files changed, 261 insertions(+), 200 deletions(-) create mode 100644 libraries/rush-lib/src/logic/buildCache/test/__snapshots__/ProjectBuildCache.test.ts.snap diff --git a/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts b/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts index da0f2cd2413..25c5260390f 100644 --- a/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts +++ b/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts @@ -66,7 +66,7 @@ export class ProjectChangeAnalyzer { */ private _data: IRawRepoState | UNINITIALIZED | undefined = UNINITIALIZED; private readonly _filteredData: Map> = new Map(); - private readonly _projectStateCache: Map = new Map(); + private readonly _projectStateCache: Map, string> = new Map(); private readonly _rushConfiguration: RushConfiguration; private readonly _git: Git; @@ -148,30 +148,31 @@ export class ProjectChangeAnalyzer { project: RushConfigurationProject, terminal: ITerminal ): Promise { - let projectState: string | undefined = this._projectStateCache.get(project); - if (!projectState) { - const packageDeps: Map | undefined = await this._tryGetProjectDependenciesAsync( - project, - terminal - ); - - if (!packageDeps) { - return undefined; - } else { - const sortedPackageDepsFiles: string[] = Array.from(packageDeps.keys()).sort(); - const hash: crypto.Hash = crypto.createHash('sha1'); - for (const packageDepsFile of sortedPackageDepsFiles) { - hash.update(packageDepsFile); - hash.update(RushConstants.hashDelimiter); - hash.update(packageDeps.get(packageDepsFile)!); - hash.update(RushConstants.hashDelimiter); - } + const packageDeps: Map | undefined = await this._tryGetProjectDependenciesAsync( + project, + terminal + ); + return packageDeps ? this._hashProjectDependencies(packageDeps) : undefined; + } - projectState = hash.digest('hex'); - this._projectStateCache.set(project, projectState); + /** + * @internal + */ + public _hashProjectDependencies(packageDeps: Map): string { + let projectState: string | undefined = this._projectStateCache.get(packageDeps); + if (!projectState) { + const sortedPackageDepsFiles: string[] = Array.from(packageDeps.keys()).sort(); + const hash: crypto.Hash = crypto.createHash('sha1'); + for (const packageDepsFile of sortedPackageDepsFiles) { + hash.update(packageDepsFile); + hash.update(RushConstants.hashDelimiter); + hash.update(packageDeps.get(packageDepsFile)!); + hash.update(RushConstants.hashDelimiter); } - } + projectState = hash.digest('hex'); + this._projectStateCache.set(packageDeps, projectState); + } return projectState; } diff --git a/libraries/rush-lib/src/logic/RushConstants.ts b/libraries/rush-lib/src/logic/RushConstants.ts index 47f59d0df3a..1cb22bfe4af 100644 --- a/libraries/rush-lib/src/logic/RushConstants.ts +++ b/libraries/rush-lib/src/logic/RushConstants.ts @@ -173,7 +173,7 @@ export class RushConstants { * Build cache version number, incremented when the logic to create cache entries changes. * Changing this ensures that cache entries generated by an old version will no longer register as a cache hit. */ - public static readonly buildCacheVersion: number = 1; + public static readonly buildCacheVersion: number = 2; /** * Per-project configuration filename. diff --git a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts b/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts index 0c6baf791ba..7a9a8f04ab9 100644 --- a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts +++ b/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts @@ -6,7 +6,6 @@ import * as crypto from 'crypto'; import { FileSystem, Path, ITerminal, FolderItem, InternalError, Async } from '@rushstack/node-core-library'; import { RushConfigurationProject } from '../../api/RushConfigurationProject'; -import { ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; import { RushProjectConfiguration } from '../../api/RushProjectConfiguration'; import { RushConstants } from '../RushConstants'; import { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; @@ -21,8 +20,8 @@ export interface IProjectBuildCacheOptions { projectOutputFolderNames: ReadonlyArray; additionalProjectOutputFilePaths?: ReadonlyArray; command: string; - trackedProjectFiles: string[] | undefined; - projectChangeAnalyzer: ProjectChangeAnalyzer; + trackedProjectFiles: Iterable | undefined; + hash: string; terminal: ITerminal; phaseName: string; } @@ -76,8 +75,8 @@ export class ProjectBuildCache { public static async tryGetProjectBuildCache( options: IProjectBuildCacheOptions ): Promise { - const { terminal, projectConfiguration, projectOutputFolderNames, trackedProjectFiles } = options; - if (!trackedProjectFiles) { + const { terminal, projectConfiguration, projectOutputFolderNames, trackedProjectFiles, hash } = options; + if (!trackedProjectFiles || !hash) { return undefined; } @@ -100,7 +99,7 @@ export class ProjectBuildCache { terminal: ITerminal, projectConfiguration: RushProjectConfiguration, projectOutputFolderNames: ReadonlyArray, - trackedProjectFiles: string[] + trackedProjectFiles: Iterable ): boolean { const normalizedProjectRelativeFolder: string = Path.convertToSlashes( projectConfiguration.project.projectRelativeFolder @@ -177,6 +176,7 @@ export class ProjectBuildCache { } terminal.writeLine('Build cache hit.'); + terminal.writeVerboseLine(cacheId); const projectFolderPath: string = this._project.projectFolder; @@ -395,49 +395,12 @@ export class ProjectBuildCache { private static async _getCacheId(options: IProjectBuildCacheOptions): Promise { // The project state hash is calculated in the following method: - // - The current project's hash (see ProjectChangeAnalyzer.getProjectStateHash) is - // calculated and appended to an array - // - The current project's recursive dependency projects' hashes are calculated - // and appended to the array // - A SHA1 hash is created and the following data is fed into it, in order: // 1. The JSON-serialized list of output folder names for this // project (see ProjectBuildCache._projectOutputFolderNames) // 2. The command that will be run in the project - // 3. Each dependency project hash (from the array constructed in previous steps), - // in sorted alphanumerical-sorted order + // 3. The hash of the projet inputs // - A hex digest of the hash is returned - const projectChangeAnalyzer: ProjectChangeAnalyzer = options.projectChangeAnalyzer; - const projectStates: string[] = []; - const projectsThatHaveBeenProcessed: Set = new Set(); - let projectsToProcess: Set = new Set(); - projectsToProcess.add(options.projectConfiguration.project); - - while (projectsToProcess.size > 0) { - const newProjectsToProcess: Set = new Set(); - for (const projectToProcess of projectsToProcess) { - projectsThatHaveBeenProcessed.add(projectToProcess); - - const projectState: string | undefined = await projectChangeAnalyzer._tryGetProjectStateHashAsync( - projectToProcess, - options.terminal - ); - if (!projectState) { - // If we hit any projects with unknown state, return unknown cache ID - return undefined; - } else { - projectStates.push(projectState); - for (const dependency of projectToProcess.dependencyProjects) { - if (!projectsThatHaveBeenProcessed.has(dependency)) { - newProjectsToProcess.add(dependency); - } - } - } - } - - projectsToProcess = newProjectsToProcess; - } - - const sortedProjectStates: string[] = projectStates.sort(); const hash: crypto.Hash = crypto.createHash('sha1'); // This value is used to force cache bust when the build cache algorithm changes hash.update(`${RushConstants.buildCacheVersion}`); @@ -447,10 +410,8 @@ export class ProjectBuildCache { hash.update(RushConstants.hashDelimiter); hash.update(options.command); hash.update(RushConstants.hashDelimiter); - for (const projectHash of sortedProjectStates) { - hash.update(projectHash); - hash.update(RushConstants.hashDelimiter); - } + hash.update(options.hash); + hash.update(RushConstants.hashDelimiter); const projectStateHash: string = hash.digest('hex'); diff --git a/libraries/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts b/libraries/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts index c690607a45a..d0336b584c8 100644 --- a/libraries/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts +++ b/libraries/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts @@ -4,7 +4,6 @@ import { StringBufferTerminalProvider, Terminal } from '@rushstack/node-core-library'; import { BuildCacheConfiguration } from '../../../api/BuildCacheConfiguration'; import { RushProjectConfiguration } from '../../../api/RushProjectConfiguration'; -import { ProjectChangeAnalyzer } from '../../ProjectChangeAnalyzer'; import { IGenerateCacheEntryIdOptions } from '../CacheEntryId'; import { FileSystemBuildCacheProvider } from '../FileSystemBuildCacheProvider'; @@ -19,11 +18,7 @@ interface ITestOptions { describe(ProjectBuildCache.name, () => { async function prepareSubject(options: Partial): Promise { const terminal: Terminal = new Terminal(new StringBufferTerminalProvider()); - const projectChangeAnalyzer = { - [ProjectChangeAnalyzer.prototype._tryGetProjectStateHashAsync.name]: async () => { - return 'state_hash'; - } - } as unknown as ProjectChangeAnalyzer; + const hash: string = 'state_hash'; const subject: ProjectBuildCache | undefined = await ProjectBuildCache.tryGetProjectBuildCache({ buildCacheConfiguration: { @@ -45,7 +40,7 @@ describe(ProjectBuildCache.name, () => { } as unknown as RushProjectConfiguration, command: 'build', trackedProjectFiles: options.hasOwnProperty('trackedProjectFiles') ? options.trackedProjectFiles : [], - projectChangeAnalyzer, + hash, terminal, phaseName: 'build' }); @@ -56,9 +51,7 @@ describe(ProjectBuildCache.name, () => { describe(ProjectBuildCache.tryGetProjectBuildCache.name, () => { it('returns a ProjectBuildCache with a calculated cacheId value', async () => { const subject: ProjectBuildCache = (await prepareSubject({}))!; - expect(subject['_cacheId']).toMatchInlineSnapshot( - `"acme-wizard/1926f30e8ed24cb47be89aea39e7efd70fcda075"` - ); + expect(subject['_cacheId']).toMatchSnapshot(); }); it('returns undefined if the tracked file list is undefined', async () => { diff --git a/libraries/rush-lib/src/logic/buildCache/test/__snapshots__/ProjectBuildCache.test.ts.snap b/libraries/rush-lib/src/logic/buildCache/test/__snapshots__/ProjectBuildCache.test.ts.snap new file mode 100644 index 00000000000..0290e712377 --- /dev/null +++ b/libraries/rush-lib/src/logic/buildCache/test/__snapshots__/ProjectBuildCache.test.ts.snap @@ -0,0 +1,3 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`ProjectBuildCache tryGetProjectBuildCache returns a ProjectBuildCache with a calculated cacheId value 1`] = `"acme-wizard/bb7633fef1e8a70f64714aabfeda78b5cd803e3c"`; diff --git a/libraries/rush-lib/src/logic/operations/IOperationRunner.ts b/libraries/rush-lib/src/logic/operations/IOperationRunner.ts index 82bcae1ef00..930c0e81644 100644 --- a/libraries/rush-lib/src/logic/operations/IOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/IOperationRunner.ts @@ -26,6 +26,10 @@ export interface IOperationRunnerContext { * Defaults to `true`. Will be `false` if Rush was invoked with `--verbose`. */ quietMode: boolean; + /** + * Defaults to `true`. Will be `false` if a dependency is in an unknown state. + */ + isCacheWriteAllowed: boolean; /** * Object used to report a summary at the end of the Rush invocation. */ @@ -40,6 +44,16 @@ export interface IOperationRunnerContext { * Object used to track elapsed time. */ stopwatch: IStopwatchResult; + + /** + * The hashes of all tracked files pertinent to the operation + */ + trackedFileHashes: ReadonlyMap | undefined; + + /** + * The hash of all inputs to the operation + */ + stateHash: string | undefined; } /** diff --git a/libraries/rush-lib/src/logic/operations/Operation.ts b/libraries/rush-lib/src/logic/operations/Operation.ts index d74795cc246..25a83d0c186 100644 --- a/libraries/rush-lib/src/logic/operations/Operation.ts +++ b/libraries/rush-lib/src/logic/operations/Operation.ts @@ -1,9 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. +import * as crypto from 'crypto'; + import { RushConfigurationProject } from '../../api/RushConfigurationProject'; import { IPhase } from '../../api/CommandLineConfiguration'; import { IOperationRunner } from './IOperationRunner'; +import { RushConstants } from '../RushConstants'; /** * Options for constructing a new Operation. @@ -87,6 +90,42 @@ export class Operation { return this.runner?.name; } + /** + * Computes this operation's input state hash, for use by the caching layer. + * @param localHash - The hash of the local file inputs for this operation + * @param dependencyHashes - The state hashes of this operation's dependencies + */ + public getHash(localHash: string, dependencyHashes: string[]): string { + if (!localHash) { + return ''; + } + + for (const dependencyHash of dependencyHashes) { + if (!dependencyHash) { + return ''; + } + } + + const sortedHashes: string[] = dependencyHashes.sort(); + const hash: crypto.Hash = crypto.createHash('sha1'); + hash.update(localHash); + for (const dependencyHash of sortedHashes) { + hash.update(dependencyHash); + hash.update(RushConstants.hashDelimiter); + } + + // CLI parameters that apply to the phase affect the result + const { associatedPhase } = this; + if (associatedPhase) { + const params: string[] = []; + for (const tsCommandLineParameter of associatedPhase.associatedParameters) { + tsCommandLineParameter.appendToArgList(params); + } + hash.update(params.join(' ')); + } + return hash.digest('hex'); + } + /** * Adds the specified operation as a dependency and updates the consumer list. */ diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts index a84a0cb67d6..802b828e747 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts @@ -3,14 +3,17 @@ import colors from 'colors/safe'; import { TerminalWritable, StdioWritable, TextRewriterTransform } from '@rushstack/terminal'; -import { StreamCollator, CollatedTerminal, CollatedWriter } from '@rushstack/stream-collator'; -import { NewlineKind, Async } from '@rushstack/node-core-library'; +import { StreamCollator, CollatedWriter } from '@rushstack/stream-collator'; +import { NewlineKind, Async, Terminal, ITerminal } from '@rushstack/node-core-library'; import { AsyncOperationQueue, IOperationSortFunction } from './AsyncOperationQueue'; import { Operation } from './Operation'; import { OperationStatus } from './OperationStatus'; import { IOperationExecutionRecordContext, OperationExecutionRecord } from './OperationExecutionRecord'; import { IExecutionResult } from './IOperationExecutionResult'; +import { ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; +import { RushConfigurationProject } from '../../api/RushConfigurationProject'; +import { CollatedTerminalProvider } from '../../utilities/CollatedTerminalProvider'; export interface IOperationExecutionManagerOptions { quietMode: boolean; @@ -36,22 +39,23 @@ export class OperationExecutionManager { private readonly _executionRecords: Map; private readonly _quietMode: boolean; private readonly _parallelism: number; - private readonly _totalOperations: number; private readonly _outputWritable: TerminalWritable; private readonly _colorsNewlinesTransform: TextRewriterTransform; private readonly _streamCollator: StreamCollator; - private readonly _terminal: CollatedTerminal; + private readonly _terminal: ITerminal; // Variables for current status private _hasAnyFailures: boolean; private _hasAnyNonAllowedWarnings: boolean; private _completedOperations: number; + private _totalOperations: number; public constructor(operations: Set, options: IOperationExecutionManagerOptions) { const { quietMode, debugMode, parallelism, changedProjectsOnly } = options; this._completedOperations = 0; + this._totalOperations = 0; this._quietMode = quietMode; this._hasAnyFailures = false; this._hasAnyNonAllowedWarnings = false; @@ -72,7 +76,7 @@ export class OperationExecutionManager { destination: this._colorsNewlinesTransform, onWriterActive: this._streamCollator_onWriterActive }); - this._terminal = this._streamCollator.terminal; + this._terminal = new Terminal(new CollatedTerminalProvider(this._streamCollator.terminal)); // Convert the developer graph to the mutable execution graph const executionRecordContext: IOperationExecutionRecordContext = { @@ -81,7 +85,6 @@ export class OperationExecutionManager { quietMode }; - let totalOperations: number = 0; const executionRecords: Map = (this._executionRecords = new Map()); for (const operation of operations) { const executionRecord: OperationExecutionRecord = new OperationExecutionRecord( @@ -90,12 +93,7 @@ export class OperationExecutionManager { ); executionRecords.set(operation, executionRecord); - if (!executionRecord.runner.silent) { - // Only count non-silent operations - totalOperations++; - } } - this._totalOperations = totalOperations; for (const [operation, consumer] of executionRecords) { for (const dependency of operation.dependencies) { @@ -137,10 +135,10 @@ export class OperationExecutionManager { const middlePart: string = colors.gray(']' + '='.repeat(middlePartLengthMinusTwoBrackets) + '['); - this._terminal.writeStdoutLine('\n' + leftPart + middlePart + rightPart); + this._terminal.writeLine('\n' + leftPart + middlePart + rightPart); if (!this._quietMode) { - this._terminal.writeStdoutLine(''); + this._terminal.writeLine(''); } } }; @@ -149,27 +147,31 @@ export class OperationExecutionManager { * Executes all operations which have been registered, returning a promise which is resolved when all the * operations are completed successfully, or rejects when any operation fails. */ - public async executeAsync(): Promise { + public async executeAsync(projectChangeAnalyzer?: ProjectChangeAnalyzer): Promise { this._completedOperations = 0; - const totalOperations: number = this._totalOperations; + if (projectChangeAnalyzer) { + await this._updateHashesAsync(projectChangeAnalyzer); + } + + const nonSilentOperations: string[] = []; + for (const record of this._executionRecords.values()) { + if (!record.silent) { + nonSilentOperations.push(record.name); + } + } + const totalOperations: number = (this._totalOperations = nonSilentOperations.length); if (!this._quietMode) { const plural: string = totalOperations === 1 ? '' : 's'; - this._terminal.writeStdoutLine(`Selected ${totalOperations} operation${plural}:`); - const nonSilentOperations: string[] = []; - for (const record of this._executionRecords.values()) { - if (!record.runner.silent) { - nonSilentOperations.push(record.name); - } - } + this._terminal.writeLine(`Selected ${totalOperations} operation${plural}:`); nonSilentOperations.sort(); for (const name of nonSilentOperations) { - this._terminal.writeStdoutLine(` ${name}`); + this._terminal.writeLine(` ${name}`); } - this._terminal.writeStdoutLine(''); + this._terminal.writeLine(''); } - this._terminal.writeStdoutLine(`Executing a maximum of ${this._parallelism} simultaneous processes...`); + this._terminal.writeLine(`Executing a maximum of ${this._parallelism} simultaneous processes...`); const maxParallelism: number = Math.min(totalOperations, this._parallelism); const prioritySort: IOperationSortFunction = ( @@ -213,6 +215,47 @@ export class OperationExecutionManager { }; } + private async _updateHashesAsync(state: ProjectChangeAnalyzer): Promise { + this._terminal.writeLine(`Updating state hashes`); + const trackedFilesByProject: Map | undefined> = new Map(); + for (const { associatedProject } of this._executionRecords.keys()) { + if (associatedProject) { + trackedFilesByProject.set(associatedProject, undefined); + } + } + + await Async.forEachAsync(trackedFilesByProject.keys(), async (project) => { + const trackedFiles: Map | undefined = await state._tryGetProjectDependenciesAsync( + project, + this._terminal + ); + trackedFilesByProject.set(project, trackedFiles); + }); + + function getOperationHash(record: OperationExecutionRecord): string { + let { stateHash } = record; + if (stateHash === undefined) { + const { operation } = record; + const { associatedProject } = operation; + stateHash = ''; + if (associatedProject) { + const trackedFiles: Map | undefined = trackedFilesByProject.get(associatedProject); + record.trackedFileHashes = trackedFiles; + const localHash: string = trackedFiles ? state._hashProjectDependencies(trackedFiles) : ''; + if (localHash) { + stateHash = operation.getHash(localHash, Array.from(record.dependencies, getOperationHash)); + } + } + record.stateHash = stateHash; + } + return stateHash; + } + + for (const record of this._executionRecords.values()) { + getOperationHash(record); + } + } + /** * Handles the result of the operation and propagates any relevant effects. */ @@ -246,7 +289,7 @@ export class OperationExecutionManager { // Now that we have the concept of architectural no-ops, we could implement this by replacing // {blockedRecord.runner} with a no-op that sets status to Blocked and logs the blocking // operations. However, the existing behavior is a bit simpler, so keeping that for now. - if (!blockedRecord.runner.silent) { + if (!blockedRecord.silent) { terminal.writeStdoutLine(`"${blockedRecord.name}" is blocked by "${name}".`); } blockedRecord.status = OperationStatus.Blocked; @@ -321,10 +364,11 @@ export class OperationExecutionManager { // Apply status changes to direct dependents for (const item of record.consumers) { if (blockCacheWrite) { - item.runner.isCacheWriteAllowed = false; + item.isCacheWriteAllowed = false; } if (blockSkip) { + // Only relevant in legacy non-build cache flow item.runner.isSkipAllowed = false; } diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts index c285d9b6371..8105c372928 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts @@ -67,6 +67,14 @@ export class OperationExecutionRecord implements IOperationRunnerContext { */ public criticalPathLength: number | undefined = undefined; + public silent: boolean = false; + + public isCacheWriteAllowed: boolean = false; + + public trackedFileHashes: Map | undefined = undefined; + + public stateHash: string | undefined = undefined; + /** * The set of operations that must complete before this operation executes. */ @@ -76,10 +84,12 @@ export class OperationExecutionRecord implements IOperationRunnerContext { */ public readonly consumers: Set = new Set(); + public readonly operation: Operation; + public readonly stopwatch: Stopwatch = new Stopwatch(); public readonly stdioSummarizer: StdioSummarizer = new StdioSummarizer(); - public readonly runner: IOperationRunner; + public runner: IOperationRunner; public readonly weight: number; public readonly _operationStateFile: OperationStateFile | undefined; @@ -88,6 +98,7 @@ export class OperationExecutionRecord implements IOperationRunnerContext { private _collatedWriter: CollatedWriter | undefined = undefined; public constructor(operation: Operation, context: IOperationExecutionRecordContext) { + this.operation = operation; const { runner } = operation; if (!runner) { @@ -95,6 +106,8 @@ export class OperationExecutionRecord implements IOperationRunnerContext { `Operation for phase '${operation.associatedPhase?.name}' and project '${operation.associatedProject?.packageName}' has no runner.` ); } + this.silent = runner.silent; + this.isCacheWriteAllowed = runner.isCacheWriteAllowed; this.runner = runner; this.weight = operation.weight; @@ -146,6 +159,7 @@ export class OperationExecutionRecord implements IOperationRunnerContext { // Delegate global state reporting onResult(this); } finally { + this.silent = this.runner.silent; this._collatedWriter?.close(); this.stdioSummarizer.close(); this.stopwatch.stop(); diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts index 5f5b46a6464..ae5390000dc 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts @@ -151,6 +151,8 @@ export class ShellOperationRunner implements IOperationRunner { ); try { + const { trackedFileHashes, stateHash } = context; + const removeColorsTransform: TextRewriterTransform = new TextRewriterTransform({ destination: projectLogWritable, removeColors: true, @@ -208,40 +210,25 @@ export class ShellOperationRunner implements IOperationRunner { } let projectDeps: IProjectDeps | undefined; - let trackedFiles: string[] | undefined; - try { - const fileHashes: Map | undefined = - await this._projectChangeAnalyzer._tryGetProjectDependenciesAsync(this._rushProject, terminal); - - if (fileHashes) { - const files: { [filePath: string]: string } = {}; - trackedFiles = []; - for (const [filePath, fileHash] of fileHashes) { - files[filePath] = fileHash; - trackedFiles.push(filePath); - } - - projectDeps = { - files, - arguments: this._commandToRun - }; - } else if (this.isSkipAllowed) { - // To test this code path: - // Remove the `.git` folder then run "rush build --verbose" - terminal.writeLine({ - text: PrintUtilities.wrapWords( - 'This workspace does not appear to be tracked by Git. ' + - 'Rush will proceed without incremental execution, caching, and change detection.' - ), - foregroundColor: ColorValue.Cyan - }); + const trackedFiles: Iterable | undefined = trackedFileHashes?.keys(); + if (trackedFileHashes) { + const files: { [filePath: string]: string } = {}; + for (const [filePath, fileHash] of trackedFileHashes) { + files[filePath] = fileHash; } - } catch (error) { + + projectDeps = { + files, + arguments: this._commandToRun + }; + } else if (this.isSkipAllowed) { // To test this code path: - // Delete a project's ".rush/temp/shrinkwrap-deps.json" then run "rush build --verbose" - terminal.writeLine('Unable to calculate incremental state: ' + (error as Error).toString()); + // Remove the `.git` folder then run "rush build --verbose" terminal.writeLine({ - text: 'Rush will proceed without incremental execution, caching, and change detection.', + text: PrintUtilities.wrapWords( + 'This workspace does not appear to be tracked by Git. ' + + 'Rush will proceed without incremental execution, caching, and change detection.' + ), foregroundColor: ColorValue.Cyan }); } @@ -263,15 +250,15 @@ export class ShellOperationRunner implements IOperationRunner { // false if a dependency wasn't able to be skipped. // let buildCacheReadAttempted: boolean = false; + let projectBuildCache: ProjectBuildCache | false | undefined; if (this._isCacheReadAllowed) { - const projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync( - terminal, - trackedFiles - ); + projectBuildCache = + (await this._tryGetProjectBuildCacheAsync(terminal, trackedFiles, stateHash)) || false; buildCacheReadAttempted = !!projectBuildCache; - const restoreFromCacheSuccess: boolean | undefined = - await projectBuildCache?.tryRestoreFromCacheAsync(terminal); + const restoreFromCacheSuccess: boolean | undefined = projectBuildCache + ? await projectBuildCache.tryRestoreFromCacheAsync(terminal) + : undefined; if (restoreFromCacheSuccess) { // Restore the original state of the operation without cache @@ -381,11 +368,15 @@ export class ShellOperationRunner implements IOperationRunner { // If the command is successful, we can calculate project hash, and no dependencies were skipped, // write a new cache entry. - const setCacheEntryPromise: Promise | undefined = this.isCacheWriteAllowed - ? (await this._tryGetProjectBuildCacheAsync(terminal, trackedFiles))?.trySetCacheEntryAsync( - terminal - ) - : undefined; + let setCacheEntryPromise: Promise | undefined; + if (context.isCacheWriteAllowed) { + if (projectBuildCache === undefined) { + projectBuildCache = await this._tryGetProjectBuildCacheAsync(terminal, trackedFiles, stateHash); + } + if (projectBuildCache) { + setCacheEntryPromise = projectBuildCache.trySetCacheEntryAsync(terminal); + } + } const [, cacheWriteSuccess] = await Promise.all([writeProjectStatePromise, setCacheEntryPromise]); @@ -412,61 +403,62 @@ export class ShellOperationRunner implements IOperationRunner { private async _tryGetProjectBuildCacheAsync( terminal: ITerminal, - trackedProjectFiles: string[] | undefined + trackedProjectFiles: Iterable | undefined, + stateHash: string | undefined ): Promise { - if (this._projectBuildCache === UNINITIALIZED) { - this._projectBuildCache = undefined; - - if (this._buildCacheConfiguration && this._buildCacheConfiguration.buildCacheEnabled) { - // Disable legacy skip logic if the build cache is in play - this.isSkipAllowed = false; - - const projectConfiguration: RushProjectConfiguration | undefined = - await RushProjectConfiguration.tryLoadForProjectAsync(this._rushProject, terminal); - if (projectConfiguration) { - projectConfiguration.validatePhaseConfiguration(this._selectedPhases, terminal); - if (projectConfiguration.disableBuildCacheForProject) { - terminal.writeVerboseLine('Caching has been disabled for this project.'); + if (!stateHash) { + // To test this code path: + // Delete a project's ".rush/temp/shrinkwrap-deps.json" then run "rush build --verbose" + terminal.writeLine('Unable to calculate incremental state.'); + terminal.writeLine({ + text: 'Rush will proceed without incremental execution, caching, and change detection.', + foregroundColor: ColorValue.Cyan + }); + return; + } + + if (this._buildCacheConfiguration && this._buildCacheConfiguration.buildCacheEnabled) { + // Disable legacy skip logic if the build cache is in play + this.isSkipAllowed = false; + + const projectConfiguration: RushProjectConfiguration | undefined = + await RushProjectConfiguration.tryLoadForProjectAsync(this._rushProject, terminal); + if (projectConfiguration) { + projectConfiguration.validatePhaseConfiguration(this._selectedPhases, terminal); + if (projectConfiguration.disableBuildCacheForProject) { + terminal.writeVerboseLine('Caching has been disabled for this project.'); + } else { + const operationSettings: IOperationSettings | undefined = + projectConfiguration.operationSettingsByOperationName.get(this._commandName); + if (!operationSettings) { + terminal.writeVerboseLine( + `This project does not define the caching behavior of the "${this._commandName}" command, so caching has been disabled.` + ); + } else if (operationSettings.disableBuildCacheForOperation) { + terminal.writeVerboseLine( + `Caching has been disabled for this project's "${this._commandName}" command.` + ); } else { - const operationSettings: IOperationSettings | undefined = - projectConfiguration.operationSettingsByOperationName.get(this._commandName); - if (!operationSettings) { - terminal.writeVerboseLine( - `This project does not define the caching behavior of the "${this._commandName}" command, so caching has been disabled.` - ); - } else if (operationSettings.disableBuildCacheForOperation) { - terminal.writeVerboseLine( - `Caching has been disabled for this project's "${this._commandName}" command.` - ); - } else { - const projectOutputFolderNames: ReadonlyArray = - operationSettings.outputFolderNames || []; - const additionalProjectOutputFilePaths: ReadonlyArray = [ - OperationStateFile.getFilenameRelativeToProjectRoot(this._phase) - ]; - this._projectBuildCache = await ProjectBuildCache.tryGetProjectBuildCache({ - projectConfiguration, - projectOutputFolderNames, - additionalProjectOutputFilePaths, - buildCacheConfiguration: this._buildCacheConfiguration, - terminal, - command: this._commandToRun, - trackedProjectFiles: trackedProjectFiles, - projectChangeAnalyzer: this._projectChangeAnalyzer, - phaseName: this._phase.name - }); - } + const projectOutputFolderNames: ReadonlyArray = operationSettings.outputFolderNames || []; + return await ProjectBuildCache.tryGetProjectBuildCache({ + projectConfiguration, + projectOutputFolderNames, + buildCacheConfiguration: this._buildCacheConfiguration, + terminal, + command: this._commandToRun, + trackedProjectFiles, + hash: stateHash, + phaseName: this._phase.name + }); } - } else { - terminal.writeVerboseLine( - `Project does not have a ${RushConstants.rushProjectConfigFilename} configuration file, ` + - 'or one provided by a rig, so it does not support caching.' - ); } + } else { + terminal.writeVerboseLine( + `Project does not have a ${RushConstants.rushProjectConfigFilename} configuration file, ` + + 'or one provided by a rig, so it does not support caching.' + ); } } - - return this._projectBuildCache; } } From 31ed2fbb3c34a9807a9aefb67c7b7cb87cf2aec5 Mon Sep 17 00:00:00 2001 From: David Michon Date: Mon, 26 Sep 2022 14:43:30 -0700 Subject: [PATCH 2/4] [rush] Make ProjectChangeAnalyzer synchronous --- .../rush-lib/src/cli/actions/ChangeAction.ts | 4 +- libraries/rush-lib/src/logic/LookupByPath.ts | 92 ++++++++- .../src/logic/ProjectChangeAnalyzer.ts | 195 ++++++++---------- .../rush-lib/src/logic/ProjectWatcher.ts | 10 +- .../operations/OperationExecutionManager.ts | 26 ++- .../operations/OperationExecutionRecord.ts | 2 +- .../src/logic/test/LookupByPath.test.ts | 33 +++ .../logic/test/ProjectChangeAnalyzer.test.ts | 66 +++--- .../ProjectChangeAnalyzer.test.ts.snap | 2 +- 9 files changed, 250 insertions(+), 180 deletions(-) diff --git a/libraries/rush-lib/src/cli/actions/ChangeAction.ts b/libraries/rush-lib/src/cli/actions/ChangeAction.ts index 197f5a37d97..0fb9931ea80 100644 --- a/libraries/rush-lib/src/cli/actions/ChangeAction.ts +++ b/libraries/rush-lib/src/cli/actions/ChangeAction.ts @@ -368,9 +368,7 @@ export class ChangeAction extends BaseRushAction { shouldFetch: !this._noFetchParameter.value, // Lockfile evaluation will expand the set of projects that request change files // Not enabling, since this would be a breaking change - includeExternalDependencies: false, - // Since install may not have happened, cannot read rush-project.json - enableFiltering: false + includeExternalDependencies: false }); const projectHostMap: Map = this._generateHostMap(); diff --git a/libraries/rush-lib/src/logic/LookupByPath.ts b/libraries/rush-lib/src/logic/LookupByPath.ts index 0fbaf994108..a1e26ce6f7b 100644 --- a/libraries/rush-lib/src/logic/LookupByPath.ts +++ b/libraries/rush-lib/src/logic/LookupByPath.ts @@ -15,6 +15,16 @@ interface IPathTreeNode { children: Map> | undefined; } +interface IPrefixEntry { + prefix: string; + index: number; +} + +export interface IPrefixMatch { + value: TItem; + index: number; +} + /** * This class is used to associate POSIX relative paths, such as those returned by `git` commands, * with entities that correspond with ancestor folders, such as Rush Projects. @@ -72,21 +82,35 @@ export class LookupByPath { * `LookupByPath.iteratePathSegments('foo\\bar\\baz', '\\')` yields 'foo', 'bar', 'baz' */ public static *iteratePathSegments(serializedPath: string, delimiter: string = '/'): Iterable { - if (!serializedPath) { + for (const prefixMatch of this._iteratePrefixes(serializedPath, delimiter)) { + yield prefixMatch.prefix; + } + } + + private static *_iteratePrefixes(input: string, delimiter: string = '/'): Iterable { + if (!input) { return; } - let nextIndex: number = serializedPath.indexOf(delimiter); let previousIndex: number = 0; - while (nextIndex >= 0) { - yield serializedPath.slice(previousIndex, nextIndex); + let nextIndex: number = input.indexOf(delimiter); + // Leading segments + while (nextIndex >= 0) { + yield { + prefix: input.slice(previousIndex, nextIndex), + index: nextIndex + }; previousIndex = nextIndex + 1; - nextIndex = serializedPath.indexOf(delimiter, previousIndex); + nextIndex = input.indexOf(delimiter, previousIndex); } - if (previousIndex + 1 < serializedPath.length) { - yield serializedPath.slice(previousIndex); + // Last segment + if (previousIndex + 1 < input.length) { + yield { + prefix: input.slice(previousIndex, input.length), + index: input.length + }; } } @@ -146,6 +170,23 @@ export class LookupByPath { return this.findChildPathFromSegments(LookupByPath.iteratePathSegments(childPath, this.delimiter)); } + /** + * Searches for the item associated with `childPath`, or the nearest ancestor of that path that + * has an associated item. + * + * @returns the found item, or `undefined` if no item was found + * + * @example + * ```ts + * const tree = new LookupByPath([['foo', 1], ['foo/bar', 2]]); + * tree.findChildPathAndIndex('foo/baz'); // returns { item: 1, index: 4 } + * tree.findChildPathAndIndex('foo/bar/baz'); // returns { item: 2, index: 8 } + * ``` + */ + public findChildPathAndIndex(childPath: string): IPrefixMatch | undefined { + return this._findChildPathFromPrefixes(LookupByPath._iteratePrefixes(childPath, this.delimiter)); + } + /** * Searches for the item associated with `childPathSegments`, or the nearest ancestor of that path that * has an associated item. @@ -179,4 +220,41 @@ export class LookupByPath { return best; } + + /** + * Searches for the item associated with `childPathSegments`, or the nearest ancestor of that path that + * has an associated item. + * + * @returns the found item, or `undefined` if no item was found + */ + private _findChildPathFromPrefixes(prefixes: Iterable): IPrefixMatch | undefined { + let node: IPathTreeNode = this._root; + let best: IPrefixMatch | undefined = node.value + ? { + value: node.value, + index: 0 + } + : undefined; + // Trivial cases + if (node.children) { + for (const { prefix: hash, index } of prefixes) { + const child: IPathTreeNode | undefined = node.children.get(hash); + if (!child) { + break; + } + node = child; + if (node.value !== undefined) { + best = { + value: node.value, + index + }; + } + if (!node.children) { + break; + } + } + } + + return best; + } } diff --git a/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts b/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts index 25c5260390f..039f058ff10 100644 --- a/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts +++ b/libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts @@ -3,7 +3,6 @@ import * as path from 'path'; import * as crypto from 'crypto'; -import ignore, { Ignore } from 'ignore'; import { getRepoChanges, @@ -12,17 +11,24 @@ import { getGitHashForFiles, IFileDiffStatus } from '@rushstack/package-deps-hash'; -import { Path, InternalError, FileSystem, ITerminal, Async } from '@rushstack/node-core-library'; +import { Path, InternalError, FileSystem, ITerminal } from '@rushstack/node-core-library'; import { RushConfiguration } from '../api/RushConfiguration'; -import { RushProjectConfiguration } from '../api/RushProjectConfiguration'; import { Git } from './Git'; import { BaseProjectShrinkwrapFile } from './base/BaseProjectShrinkwrapFile'; import { RushConfigurationProject } from '../api/RushConfigurationProject'; import { RushConstants } from './RushConstants'; -import { LookupByPath } from './LookupByPath'; +import { IPrefixMatch, LookupByPath } from './LookupByPath'; import { PnpmShrinkwrapFile } from './pnpm/PnpmShrinkwrapFile'; -import { UNINITIALIZED } from '../utilities/Utilities'; + +/** + * @beta + */ +export type IProjectFileFilter = (projectRelativePath: string) => boolean; +/** + * @beta + */ +export type IProjectFileFilterMap = Map; /** * @beta @@ -39,10 +45,9 @@ export interface IGetChangedProjectsOptions { includeExternalDependencies: boolean; /** - * If set to `true` apply the `incrementalBuildIgnoredGlobs` property in a project's `rush-project.json` - * and exclude matched files from change detection. + * If specified, the filter will be applied to project inputs during comparison */ - enableFiltering: boolean; + filters?: IProjectFileFilterMap; } interface IGitState { @@ -56,23 +61,33 @@ interface IRawRepoState { rootDir: string; } +type IFilterCacheMap = Map>; + /** * @beta */ export class ProjectChangeAnalyzer { - /** - * UNINITIALIZED === we haven't looked - * undefined === data isn't available (i.e. - git isn't present) - */ - private _data: IRawRepoState | UNINITIALIZED | undefined = UNINITIALIZED; - private readonly _filteredData: Map> = new Map(); - private readonly _projectStateCache: Map, string> = new Map(); + private _data: IRawRepoState | undefined; private readonly _rushConfiguration: RushConfiguration; + private readonly _cacheByFilter: WeakMap = new WeakMap(); + private readonly _hashCache: Map, string> = new Map(); private readonly _git: Git; public constructor(rushConfiguration: RushConfiguration) { this._rushConfiguration = rushConfiguration; this._git = new Git(this._rushConfiguration); + this._data = undefined; + } + + /** + * @internal + */ + public _ensureInitialized(terminal: ITerminal): IRawRepoState { + if (!this._data) { + this._data = this._getData(terminal); + } + + return this._data; } /** @@ -83,23 +98,12 @@ export class ProjectChangeAnalyzer { * * @internal */ - public async _tryGetProjectDependenciesAsync( + public _tryGetProjectDependencies( project: RushConfigurationProject, - terminal: ITerminal - ): Promise | undefined> { - // Check the cache for any existing data - let filteredProjectData: Map | undefined = this._filteredData.get(project); - if (filteredProjectData) { - return filteredProjectData; - } - - const data: IRawRepoState | undefined = this._ensureInitialized(terminal); - - if (!data) { - return undefined; - } - - const { projectState, rootDir } = data; + terminal: ITerminal, + fileFilter?: IProjectFileFilter + ): ReadonlyMap | undefined { + const { projectState, rootDir } = this._ensureInitialized(terminal); if (projectState === undefined) { return undefined; @@ -110,26 +114,23 @@ export class ProjectChangeAnalyzer { throw new Error(`Project "${project.packageName}" does not exist in the current Rush configuration.`); } - filteredProjectData = await this._filterProjectDataAsync( - project, - unfilteredProjectData, - rootDir, - terminal - ); + if (!fileFilter) { + return unfilteredProjectData; + } - this._filteredData.set(project, filteredProjectData); - return filteredProjectData; - } + let cacheForFilter: IFilterCacheMap | undefined = this._cacheByFilter.get(fileFilter); + if (!cacheForFilter) { + cacheForFilter = new Map(); + this._cacheByFilter.set(fileFilter, cacheForFilter); + } - /** - * @internal - */ - public _ensureInitialized(terminal: ITerminal): IRawRepoState | undefined { - if (this._data === UNINITIALIZED) { - this._data = this._getData(terminal); + let cacheEntry: ReadonlyMap | undefined = cacheForFilter.get(project); + if (!cacheEntry) { + cacheEntry = this._filterProjectData(project, unfilteredProjectData, rootDir, fileFilter); + cacheForFilter.set(project, cacheEntry); } - return this._data; + return cacheEntry; } /** @@ -144,22 +145,25 @@ export class ProjectChangeAnalyzer { * * @internal */ - public async _tryGetProjectStateHashAsync( + public _tryGetProjectStateHash( project: RushConfigurationProject, - terminal: ITerminal - ): Promise { - const packageDeps: Map | undefined = await this._tryGetProjectDependenciesAsync( + terminal: ITerminal, + fileFilter?: IProjectFileFilter + ): string | undefined { + const packageDeps: ReadonlyMap | undefined = this._tryGetProjectDependencies( project, - terminal + terminal, + fileFilter ); + return packageDeps ? this._hashProjectDependencies(packageDeps) : undefined; } /** * @internal */ - public _hashProjectDependencies(packageDeps: Map): string { - let projectState: string | undefined = this._projectStateCache.get(packageDeps); + public _hashProjectDependencies(packageDeps: ReadonlyMap): string { + let projectState: string | undefined = this._hashCache.get(packageDeps); if (!projectState) { const sortedPackageDepsFiles: string[] = Array.from(packageDeps.keys()).sort(); const hash: crypto.Hash = crypto.createHash('sha1'); @@ -171,19 +175,18 @@ export class ProjectChangeAnalyzer { } projectState = hash.digest('hex'); - this._projectStateCache.set(packageDeps, projectState); + this._hashCache.set(packageDeps, projectState); } return projectState; } - public async _filterProjectDataAsync( + public _filterProjectData( project: RushConfigurationProject, unfilteredProjectData: Map, rootDir: string, - terminal: ITerminal - ): Promise> { - const ignoreMatcher: Ignore | undefined = await this._getIgnoreMatcherForProjectAsync(project, terminal); - if (!ignoreMatcher) { + fileFilter?: IProjectFileFilter + ): Map { + if (!fileFilter) { return unfilteredProjectData; } @@ -196,7 +199,7 @@ export class ProjectChangeAnalyzer { const filteredProjectData: Map = new Map(); for (const [filePath, value] of unfilteredProjectData) { const relativePath: string = filePath.slice(projectKeyLength); - if (!ignoreMatcher.ignores(relativePath)) { + if (fileFilter(relativePath)) { // Add the file path to the filtered data if it is not ignored filteredProjectData.set(filePath, value); } @@ -212,9 +215,18 @@ export class ProjectChangeAnalyzer { public async getChangedProjectsAsync( options: IGetChangedProjectsOptions ): Promise> { + return this.getChangedProjects(options); + } + + /** + * Gets a list of projects that have changed in the current state of the repo + * when compared to the specified branch, optionally taking the shrinkwrap and settings in + * the rush-project.json file into consideration. + */ + public getChangedProjects(options: IGetChangedProjectsOptions): Set { const { _rushConfiguration: rushConfiguration } = this; - const { targetBranchName, terminal, includeExternalDependencies, enableFiltering, shouldFetch } = options; + const { targetBranchName, terminal, includeExternalDependencies, filters, shouldFetch } = options; const gitPath: string = this._git.getGitPathOrThrow(); const repoRoot: string = getRepoRoot(rushConfiguration.rushJsonFolder); @@ -278,51 +290,22 @@ export class ProjectChangeAnalyzer { } } - const changesByProject: Map> = new Map(); const lookup: LookupByPath = rushConfiguration.getProjectLookupForRoot(repoRoot); - for (const [file, diffStatus] of repoChanges) { - const project: RushConfigurationProject | undefined = lookup.findChildPath(file); - if (project) { - if (changedProjects.has(project)) { - // Lockfile changes cannot be ignored via rush-project.json - continue; - } - - if (enableFiltering) { - let projectChanges: Map | undefined = changesByProject.get(project); - if (!projectChanges) { - projectChanges = new Map(); - changesByProject.set(project, projectChanges); + for (const file of repoChanges.keys()) { + const match: IPrefixMatch | undefined = lookup.findChildPathAndIndex(file); + if (match) { + const project: RushConfigurationProject = match.value; + if (!changedProjects.has(project)) { + const projectFilter: IProjectFileFilter | undefined = filters?.get(project); + if (!projectFilter || projectFilter(file.slice(match.index + 1))) { + changedProjects.add(project); } - projectChanges.set(file, diffStatus); - } else { - changedProjects.add(project); } } } - if (enableFiltering) { - // Reading rush-project.json may be problematic if, e.g. rush install has not yet occurred and rigs are in use - await Async.forEachAsync( - changesByProject, - async ([project, projectChanges]) => { - const filteredChanges: Map = await this._filterProjectDataAsync( - project, - projectChanges, - repoRoot, - terminal - ); - - if (filteredChanges.size > 0) { - changedProjects.add(project); - } - }, - { concurrency: 10 } - ); - } - return changedProjects; } @@ -421,20 +404,6 @@ export class ProjectChangeAnalyzer { }; } - private async _getIgnoreMatcherForProjectAsync( - project: RushConfigurationProject, - terminal: ITerminal - ): Promise { - const incrementalBuildIgnoredGlobs: ReadonlyArray | undefined = - await RushProjectConfiguration.tryLoadIgnoreGlobsForProjectAsync(project, terminal); - - if (incrementalBuildIgnoredGlobs && incrementalBuildIgnoredGlobs.length) { - const ignoreMatcher: Ignore = ignore(); - ignoreMatcher.add(incrementalBuildIgnoredGlobs as string[]); - return ignoreMatcher; - } - } - private _getRepoDeps(terminal: ITerminal): IGitState | undefined { try { if (this._git.isPathUnderGitWorkingTree()) { diff --git a/libraries/rush-lib/src/logic/ProjectWatcher.ts b/libraries/rush-lib/src/logic/ProjectWatcher.ts index 3b0c4b37f4b..61055b25899 100644 --- a/libraries/rush-lib/src/logic/ProjectWatcher.ts +++ b/libraries/rush-lib/src/logic/ProjectWatcher.ts @@ -93,7 +93,7 @@ export class ProjectWatcher { pathsToWatch.add(this._repoRoot); } else { for (const project of this._projectsToWatch) { - const projectState: Map = (await previousState._tryGetProjectDependenciesAsync( + const projectState: ReadonlyMap = (await previousState._tryGetProjectDependencies( project, this._terminal ))!; @@ -258,8 +258,8 @@ export class ProjectWatcher { const changedProjects: Set = new Set(); for (const project of this._projectsToWatch) { const [previous, current] = await Promise.all([ - previousState._tryGetProjectDependenciesAsync(project, this._terminal), - state._tryGetProjectDependenciesAsync(project, this._terminal) + previousState._tryGetProjectDependencies(project, this._terminal), + state._tryGetProjectDependencies(project, this._terminal) ]); if (ProjectWatcher._haveProjectDepsChanged(previous, current)) { @@ -287,8 +287,8 @@ export class ProjectWatcher { * @returns `true` if the maps are different, `false` otherwise */ private static _haveProjectDepsChanged( - prev: Map | undefined, - next: Map | undefined + prev: ReadonlyMap | undefined, + next: ReadonlyMap | undefined ): boolean { if (!prev && !next) { return false; diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts index 802b828e747..d7859855411 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts @@ -151,7 +151,7 @@ export class OperationExecutionManager { this._completedOperations = 0; if (projectChangeAnalyzer) { - await this._updateHashesAsync(projectChangeAnalyzer); + await this._updateStateAsync(projectChangeAnalyzer); } const nonSilentOperations: string[] = []; @@ -215,23 +215,20 @@ export class OperationExecutionManager { }; } - private async _updateHashesAsync(state: ProjectChangeAnalyzer): Promise { + private async _updateStateAsync(state: ProjectChangeAnalyzer): Promise { this._terminal.writeLine(`Updating state hashes`); - const trackedFilesByProject: Map | undefined> = new Map(); + const trackedFilesByProject: Map | undefined> = + new Map(); for (const { associatedProject } of this._executionRecords.keys()) { - if (associatedProject) { - trackedFilesByProject.set(associatedProject, undefined); + if (associatedProject && !trackedFilesByProject.has(associatedProject)) { + const trackedFiles: ReadonlyMap | undefined = state._tryGetProjectDependencies( + associatedProject, + this._terminal + ); + trackedFilesByProject.set(associatedProject, trackedFiles); } } - await Async.forEachAsync(trackedFilesByProject.keys(), async (project) => { - const trackedFiles: Map | undefined = await state._tryGetProjectDependenciesAsync( - project, - this._terminal - ); - trackedFilesByProject.set(project, trackedFiles); - }); - function getOperationHash(record: OperationExecutionRecord): string { let { stateHash } = record; if (stateHash === undefined) { @@ -239,7 +236,8 @@ export class OperationExecutionManager { const { associatedProject } = operation; stateHash = ''; if (associatedProject) { - const trackedFiles: Map | undefined = trackedFilesByProject.get(associatedProject); + const trackedFiles: ReadonlyMap | undefined = + trackedFilesByProject.get(associatedProject); record.trackedFileHashes = trackedFiles; const localHash: string = trackedFiles ? state._hashProjectDependencies(trackedFiles) : ''; if (localHash) { diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts index 8105c372928..b9630a469ff 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts @@ -71,7 +71,7 @@ export class OperationExecutionRecord implements IOperationRunnerContext { public isCacheWriteAllowed: boolean = false; - public trackedFileHashes: Map | undefined = undefined; + public trackedFileHashes: ReadonlyMap | undefined = undefined; public stateHash: string | undefined = undefined; diff --git a/libraries/rush-lib/src/logic/test/LookupByPath.test.ts b/libraries/rush-lib/src/logic/test/LookupByPath.test.ts index 0329ac2315c..f72b5a2e9ee 100644 --- a/libraries/rush-lib/src/logic/test/LookupByPath.test.ts +++ b/libraries/rush-lib/src/logic/test/LookupByPath.test.ts @@ -101,3 +101,36 @@ describe(LookupByPath.prototype.findChildPath.name, () => { expect(tree.findChildPathFromSegments(['foo', 'bar', 'baz'])).toEqual(1); }); }); + +describe(LookupByPath.prototype.findChildPathAndIndex.name, () => { + it('returns empty for an empty tree', () => { + expect(new LookupByPath().findChildPathAndIndex('foo')).toEqual(undefined); + }); + it('returns the matching node for a trivial tree', () => { + expect(new LookupByPath([['foo', 1]]).findChildPathAndIndex('foo')).toEqual({ value: 1, index: 3 }); + }); + it('returns the matching node for a single-layer tree', () => { + const tree: LookupByPath = new LookupByPath([ + ['foo', 1], + ['bar', 2], + ['baz', 3] + ]); + + expect(tree.findChildPathAndIndex('foo')).toEqual({ value: 1, index: 3 }); + expect(tree.findChildPathAndIndex('bar')).toEqual({ value: 2, index: 3 }); + expect(tree.findChildPathAndIndex('baz')).toEqual({ value: 3, index: 3 }); + expect(tree.findChildPathAndIndex('buzz')).toEqual(undefined); + }); + it('returns the matching parent for multi-layer queries', () => { + const tree: LookupByPath = new LookupByPath([ + ['foo', 1], + ['bar', 2], + ['baz', 3] + ]); + + expect(tree.findChildPathAndIndex('foo/bar')).toEqual({ value: 1, index: 3 }); + expect(tree.findChildPathAndIndex('bar/baz')).toEqual({ value: 2, index: 3 }); + expect(tree.findChildPathAndIndex('baz/foo')).toEqual({ value: 3, index: 3 }); + expect(tree.findChildPathAndIndex('foo/foo')).toEqual({ value: 1, index: 3 }); + }); +}); diff --git a/libraries/rush-lib/src/logic/test/ProjectChangeAnalyzer.test.ts b/libraries/rush-lib/src/logic/test/ProjectChangeAnalyzer.test.ts index 47a5818cd52..bf41a4c32c6 100644 --- a/libraries/rush-lib/src/logic/test/ProjectChangeAnalyzer.test.ts +++ b/libraries/rush-lib/src/logic/test/ProjectChangeAnalyzer.test.ts @@ -3,13 +3,12 @@ import { StringBufferTerminalProvider, Terminal } from '@rushstack/node-core-library'; -import { ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; +import { IProjectFileFilter, ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; import { RushConfiguration } from '../../api/RushConfiguration'; import { EnvironmentConfiguration } from '../../api/EnvironmentConfiguration'; import { RushConfigurationProject } from '../../api/RushConfigurationProject'; import { RushProjectConfiguration } from '../../api/RushProjectConfiguration'; import { LookupByPath } from '../LookupByPath'; -import { UNINITIALIZED } from '../../utilities/Utilities'; describe(ProjectChangeAnalyzer.name, () => { beforeEach(() => { @@ -57,7 +56,7 @@ describe(ProjectChangeAnalyzer.name, () => { return subject; } - describe(ProjectChangeAnalyzer.prototype._tryGetProjectDependenciesAsync.name, () => { + describe(ProjectChangeAnalyzer.prototype._tryGetProjectDependencies.name, () => { it('returns the files for the specified project', async () => { const projects: RushConfigurationProject[] = [ { @@ -78,23 +77,19 @@ describe(ProjectChangeAnalyzer.name, () => { const subject: ProjectChangeAnalyzer = createTestSubject(projects, files); const terminal: Terminal = new Terminal(new StringBufferTerminalProvider()); - expect(await subject._tryGetProjectDependenciesAsync(projects[0], terminal)).toEqual( + expect(subject._tryGetProjectDependencies(projects[0], terminal)).toEqual( new Map([['apps/apple/core.js', 'a101']]) ); - expect(await subject._tryGetProjectDependenciesAsync(projects[1], terminal)).toEqual( + expect(subject._tryGetProjectDependencies(projects[1], terminal)).toEqual( new Map([['apps/banana/peel.js', 'b201']]) ); }); - it('ignores files specified by project configuration files, relative to project folder', async () => { - // rush-project.json configuration for 'apple' - jest - .spyOn(RushProjectConfiguration, 'tryLoadIgnoreGlobsForProjectAsync') - .mockResolvedValueOnce(['assets/*.png', '*.js.map']); - // rush-project.json configuration for 'banana' does not exist - jest - .spyOn(RushProjectConfiguration, 'tryLoadIgnoreGlobsForProjectAsync') - .mockResolvedValueOnce(undefined); + it('ignores files based on filter, relative to project folder', async () => { + const appleRegex: RegExp = /^assets\/[^\/]+\.png$|(?:^|\/)[^\/]+\.js\.map$/; + const appleFilter: IProjectFileFilter = (projectRelativePath: string) => { + return !appleRegex.test(projectRelativePath); + }; const projects: RushConfigurationProject[] = [ { @@ -119,13 +114,13 @@ describe(ProjectChangeAnalyzer.name, () => { const subject: ProjectChangeAnalyzer = createTestSubject(projects, files); const terminal: Terminal = new Terminal(new StringBufferTerminalProvider()); - expect(await subject._tryGetProjectDependenciesAsync(projects[0], terminal)).toEqual( + expect(subject._tryGetProjectDependencies(projects[0], terminal, appleFilter)).toEqual( new Map([ ['apps/apple/core.js', 'a101'], ['apps/apple/assets/one.jpg', 'a103'] ]) ); - expect(await subject._tryGetProjectDependenciesAsync(projects[1], terminal)).toEqual( + expect(subject._tryGetProjectDependencies(projects[1], terminal)).toEqual( new Map([ ['apps/banana/peel.js', 'b201'], ['apps/banana/peel.js.map', 'b202'] @@ -133,11 +128,11 @@ describe(ProjectChangeAnalyzer.name, () => { ); }); - it('interprets ignored globs as a dot-ignore file (not as individually handled globs)', async () => { - // rush-project.json configuration for 'apple' - jest - .spyOn(RushProjectConfiguration, 'tryLoadIgnoreGlobsForProjectAsync') - .mockResolvedValue(['*.png', 'assets/*.psd', '!assets/important/**']); + it('works with more complex filters', async () => { + const appleRegex: RegExp = /^assets\/[^\/]+\.psd$|(?:^|\/)[^\/]+\.png$/; + const appleFilter: IProjectFileFilter = (projectRelativePath: string) => { + return projectRelativePath.startsWith('assets/important/') || !appleRegex.test(projectRelativePath); + }; const projects: RushConfigurationProject[] = [ { @@ -160,7 +155,7 @@ describe(ProjectChangeAnalyzer.name, () => { // In a dot-ignore file, the later rule '!assets/important/**' should override the previous // rule of '*.png'. This unit test verifies that this behavior doesn't change later if // we modify the implementation. - expect(await subject._tryGetProjectDependenciesAsync(projects[0], terminal)).toEqual( + expect(subject._tryGetProjectDependencies(projects[0], terminal, appleFilter)).toEqual( new Map([ ['apps/apple/assets/important/four.png', 'a104'], ['apps/apple/assets/important/five.psd', 'a105'], @@ -191,13 +186,13 @@ describe(ProjectChangeAnalyzer.name, () => { const subject: ProjectChangeAnalyzer = createTestSubject(projects, files); const terminal: Terminal = new Terminal(new StringBufferTerminalProvider()); - expect(await subject._tryGetProjectDependenciesAsync(projects[0], terminal)).toEqual( + expect(subject._tryGetProjectDependencies(projects[0], terminal)).toEqual( new Map([ ['apps/apple/core.js', 'a101'], ['common/config/rush/pnpm-lock.yaml', 'ffff'] ]) ); - expect(await subject._tryGetProjectDependenciesAsync(projects[1], terminal)).toEqual( + expect(subject._tryGetProjectDependencies(projects[1], terminal)).toEqual( new Map([ ['apps/banana/peel.js', 'b201'], ['common/config/rush/pnpm-lock.yaml', 'ffff'] @@ -218,7 +213,7 @@ describe(ProjectChangeAnalyzer.name, () => { const terminal: Terminal = new Terminal(new StringBufferTerminalProvider()); try { - await subject._tryGetProjectDependenciesAsync( + subject._tryGetProjectDependencies( { packageName: 'carrot' } as RushConfigurationProject, @@ -246,20 +241,19 @@ describe(ProjectChangeAnalyzer.name, () => { // ProjectChangeAnalyzer is inert until someone actually requests project data, // this test makes that expectation explicit. - expect(subject['_data']).toEqual(UNINITIALIZED); - expect(await subject._tryGetProjectDependenciesAsync(projects[0], terminal)).toEqual( - new Map([['apps/apple/core.js', 'a101']]) + expect(subject['_data']).toEqual(undefined); + const originalResult: ReadonlyMap | undefined = subject._tryGetProjectDependencies( + projects[0], + terminal ); + expect(originalResult).toEqual(new Map([['apps/apple/core.js', 'a101']])); expect(subject['_data']).toBeDefined(); - expect(subject['_data']).not.toEqual(UNINITIALIZED); - expect(await subject._tryGetProjectDependenciesAsync(projects[0], terminal)).toEqual( - new Map([['apps/apple/core.js', 'a101']]) - ); + expect(subject._tryGetProjectDependencies(projects[0], terminal)).toBe(originalResult); expect(subject['_getRepoDeps']).toHaveBeenCalledTimes(1); }); }); - describe(ProjectChangeAnalyzer.prototype._tryGetProjectStateHashAsync.name, () => { + describe(ProjectChangeAnalyzer.prototype._tryGetProjectStateHash.name, () => { it('returns a fixed hash snapshot for a set of project deps', async () => { const projects: RushConfigurationProject[] = [ { @@ -276,7 +270,7 @@ describe(ProjectChangeAnalyzer.name, () => { const subject: ProjectChangeAnalyzer = createTestSubject(projects, files); const terminal: Terminal = new Terminal(new StringBufferTerminalProvider()); - expect(await subject._tryGetProjectStateHashAsync(projects[0], terminal)).toMatchInlineSnapshot( + expect(subject._tryGetProjectStateHash(projects[0], terminal)).toMatchInlineSnapshot( `"265536e325cdfac3fa806a51873d927a712fc6c9"` ); }); @@ -311,8 +305,8 @@ describe(ProjectChangeAnalyzer.name, () => { const subjectB: ProjectChangeAnalyzer = createTestSubject(projectsB, filesB); const terminal: Terminal = new Terminal(new StringBufferTerminalProvider()); - expect(await subjectA._tryGetProjectStateHashAsync(projectsA[0], terminal)).toEqual( - await subjectB._tryGetProjectStateHashAsync(projectsB[0], terminal) + expect(subjectA._tryGetProjectStateHash(projectsA[0], terminal)).toEqual( + subjectB._tryGetProjectStateHash(projectsB[0], terminal) ); }); }); diff --git a/libraries/rush-lib/src/logic/test/__snapshots__/ProjectChangeAnalyzer.test.ts.snap b/libraries/rush-lib/src/logic/test/__snapshots__/ProjectChangeAnalyzer.test.ts.snap index bc265de44ee..eb25b7bdbd2 100644 --- a/libraries/rush-lib/src/logic/test/__snapshots__/ProjectChangeAnalyzer.test.ts.snap +++ b/libraries/rush-lib/src/logic/test/__snapshots__/ProjectChangeAnalyzer.test.ts.snap @@ -1,3 +1,3 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`ProjectChangeAnalyzer _tryGetProjectDependenciesAsync throws an exception if the specified project does not exist 1`] = `[Error: Project "carrot" does not exist in the current Rush configuration.]`; +exports[`ProjectChangeAnalyzer _tryGetProjectDependencies throws an exception if the specified project does not exist 1`] = `[Error: Project "carrot" does not exist in the current Rush configuration.]`; From 5a81bd4d90b97552184ed4feaa7ffc55a5c6696c Mon Sep 17 00:00:00 2001 From: David Michon Date: Mon, 26 Sep 2022 17:21:45 -0700 Subject: [PATCH 3/4] [rush] Move cache out of ShellOperationRunner --- common/reviews/api/rush-lib.api.md | 63 ++-- .../cli/scriptActions/PhasedScriptAction.ts | 35 +- ...che.ts => BuildCacheOperationProcessor.ts} | 264 +++++---------- .../src/logic/buildCache/CacheEntryId.ts | 6 +- .../buildCache/test/CacheEntryId.test.ts | 2 +- .../buildCache/test/ProjectBuildCache.test.ts | 65 ---- .../ProjectBuildCache.test.ts.snap | 3 - .../operations/IOperationExecutionResult.ts | 5 - .../logic/operations/IOperationProcessor.ts | 20 ++ .../src/logic/operations/IOperationRunner.ts | 46 ++- .../logic/operations/NullOperationRunner.ts | 4 - .../src/logic/operations/Operation.ts | 84 +++-- .../operations/OperationExecutionManager.ts | 89 ++--- .../operations/OperationExecutionRecord.ts | 114 +++++-- .../OperationResultSummarizerPlugin.ts | 32 +- .../logic/operations/PhasedOperationPlugin.ts | 75 ++++- .../logic/operations/ProjectLogWritable.ts | 11 +- .../logic/operations/ShellOperationRunner.ts | 314 +----------------- .../operations/ShellOperationRunnerPlugin.ts | 18 +- .../test/AsyncOperationQueue.test.ts | 16 + .../operations/test/MockOperationRunner.ts | 9 +- .../test/OperationExecutionManager.test.ts | 58 ++-- .../test/PhasedOperationPlugin.test.ts | 99 +++--- .../OperationExecutionManager.test.ts.snap | 36 +- .../src/pluginFramework/PhasedCommandHooks.ts | 6 +- .../src/phasedCommandHandler.ts | 4 +- 26 files changed, 641 insertions(+), 837 deletions(-) rename libraries/rush-lib/src/logic/buildCache/{ProjectBuildCache.ts => BuildCacheOperationProcessor.ts} (50%) delete mode 100644 libraries/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts delete mode 100644 libraries/rush-lib/src/logic/buildCache/test/__snapshots__/ProjectBuildCache.test.ts.snap create mode 100644 libraries/rush-lib/src/logic/operations/IOperationProcessor.ts diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index ede8a31bcaf..2fd8d5c8494 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -9,7 +9,6 @@ import { AsyncParallelHook } from 'tapable'; import { AsyncSeriesHook } from 'tapable'; import { AsyncSeriesWaterfallHook } from 'tapable'; -import type { CollatedWriter } from '@rushstack/stream-collator'; import type { CommandLineParameter } from '@rushstack/ts-command-line'; import { HookMap } from 'tapable'; import { IPackageJson } from '@rushstack/node-core-library'; @@ -17,9 +16,9 @@ import { ITerminal } from '@rushstack/node-core-library'; import { ITerminalProvider } from '@rushstack/node-core-library'; import { JsonObject } from '@rushstack/node-core-library'; import { PackageNameParser } from '@rushstack/node-core-library'; -import type { StdioSummarizer } from '@rushstack/terminal'; import { SyncHook } from 'tapable'; import { Terminal } from '@rushstack/node-core-library'; +import type { TerminalWritable } from '@rushstack/terminal'; // @public export class ApprovedPackagesConfiguration { @@ -264,7 +263,8 @@ export interface ICreateOperationsContext { readonly isWatch: boolean; readonly phaseSelection: ReadonlySet; readonly projectChangeAnalyzer: ProjectChangeAnalyzer; - readonly projectSelection: ReadonlySet; + // Warning: (ae-forgotten-export) The symbol "RushProjectConfiguration" needs to be exported by the entry point index.d.ts + readonly projectSelection: ReadonlyMap; readonly projectsInUnknownState: ReadonlySet; readonly rushConfiguration: RushConfiguration; } @@ -315,12 +315,13 @@ export interface IFileSystemBuildCacheProviderOptions { export interface IGenerateCacheEntryIdOptions { phaseName: string; projectName: string; - projectStateHash: string; + stateHash: string; } // @beta (undocumented) export interface IGetChangedProjectsOptions { - enableFiltering: boolean; + // Warning: (ae-forgotten-export) The symbol "IProjectFileFilterMap" needs to be exported by the entry point index.d.ts + filters?: IProjectFileFilterMap; includeExternalDependencies: boolean; // (undocumented) shouldFetch?: boolean; @@ -374,22 +375,24 @@ export interface IOperationExecutionResult { readonly error: Error | undefined; readonly nonCachedDurationMs: number | undefined; readonly status: OperationStatus; - readonly stdioSummarizer: StdioSummarizer; readonly stopwatch: IStopwatchResult; } // @alpha export interface IOperationOptions { - phase?: IPhase | undefined; - project?: RushConfigurationProject | undefined; + outputFolderNames?: ReadonlyArray | undefined; + phase: IPhase; + // Warning: (ae-forgotten-export) The symbol "IOperationProcessor" needs to be exported by the entry point index.d.ts + processor?: IOperationProcessor | undefined; + project: RushConfigurationProject; + // Warning: (ae-forgotten-export) The symbol "IProjectFileFilter" needs to be exported by the entry point index.d.ts + projectFileFilter?: IProjectFileFilter | undefined; runner?: IOperationRunner | undefined; } // @beta export interface IOperationRunner { executeAsync(context: IOperationRunnerContext): Promise; - isCacheWriteAllowed: boolean; - isSkipAllowed: boolean; readonly name: string; reportTiming: boolean; silent: boolean; @@ -398,13 +401,15 @@ export interface IOperationRunner { // @beta export interface IOperationRunnerContext { - collatedWriter: CollatedWriter; debugMode: boolean; - // @internal - _operationStateFile?: _OperationStateFile; + isCacheReadAllowed: boolean; + isCacheWriteAllowed: boolean; + isSkipAllowed: boolean; quietMode: boolean; - stdioSummarizer: StdioSummarizer; - stopwatch: IStopwatchResult; + stateHash: string | undefined; + terminal: ITerminal; + terminalWritable: TerminalWritable; + trackedFileHashes: ReadonlyMap | undefined; } // @internal (undocumented) @@ -575,6 +580,8 @@ export class LookupByPath { constructor(entries?: Iterable<[string, TItem]>, delimiter?: string); readonly delimiter: string; findChildPath(childPath: string): TItem | undefined; + // Warning: (ae-forgotten-export) The symbol "IPrefixMatch" needs to be exported by the entry point index.d.ts + findChildPathAndIndex(childPath: string): IPrefixMatch | undefined; findChildPathFromSegments(childPathSegments: Iterable): TItem | undefined; static iteratePathSegments(serializedPath: string, delimiter?: string): Iterable; setItem(serializedPath: string, value: TItem): this; @@ -589,14 +596,23 @@ export class NpmOptionsConfiguration extends PackageManagerOptionsConfigurationB // @alpha export class Operation { - constructor(options?: IOperationOptions); + constructor(options: IOperationOptions); addDependency(dependency: Operation): void; - readonly associatedPhase: IPhase | undefined; - readonly associatedProject: RushConfigurationProject | undefined; + readonly associatedPhase: IPhase; + readonly associatedProject: RushConfigurationProject; readonly consumers: ReadonlySet; deleteDependency(dependency: Operation): void; readonly dependencies: ReadonlySet; + getHash(localHash: string, dependencyHashes: string[]): string; + // (undocumented) + logFilePath: string | undefined; get name(): string | undefined; + // (undocumented) + readonly outputFolderNames: ReadonlyArray; + // (undocumented) + processor: IOperationProcessor | undefined; + // (undocumented) + readonly projectFileFilter: IProjectFileFilter | undefined; runner: IOperationRunner | undefined; weight: number; } @@ -724,14 +740,17 @@ export class ProjectChangeAnalyzer { // Warning: (ae-forgotten-export) The symbol "IRawRepoState" needs to be exported by the entry point index.d.ts // // @internal (undocumented) - _ensureInitialized(terminal: ITerminal): IRawRepoState | undefined; + _ensureInitialized(terminal: ITerminal): IRawRepoState; // (undocumented) - _filterProjectDataAsync(project: RushConfigurationProject, unfilteredProjectData: Map, rootDir: string, terminal: ITerminal): Promise>; + _filterProjectData(project: RushConfigurationProject, unfilteredProjectData: Map, rootDir: string, fileFilter?: IProjectFileFilter): Map; + getChangedProjects(options: IGetChangedProjectsOptions): Set; getChangedProjectsAsync(options: IGetChangedProjectsOptions): Promise>; + // @internal (undocumented) + _hashProjectDependencies(packageDeps: ReadonlyMap): string; // @internal - _tryGetProjectDependenciesAsync(project: RushConfigurationProject, terminal: ITerminal): Promise | undefined>; + _tryGetProjectDependencies(project: RushConfigurationProject, terminal: ITerminal, fileFilter?: IProjectFileFilter): ReadonlyMap | undefined; // @internal - _tryGetProjectStateHashAsync(project: RushConfigurationProject, terminal: ITerminal): Promise; + _tryGetProjectStateHash(project: RushConfigurationProject, terminal: ITerminal, fileFilter?: IProjectFileFilter): string | undefined; } // @public diff --git a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts index c5580f0c6cb..18939f3b2ae 100644 --- a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts +++ b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts @@ -5,7 +5,7 @@ import * as os from 'os'; import colors from 'colors/safe'; import type { AsyncSeriesHook } from 'tapable'; -import { AlreadyReportedError, InternalError, ITerminal, Terminal } from '@rushstack/node-core-library'; +import { AlreadyReportedError, Async, InternalError, ITerminal, Terminal } from '@rushstack/node-core-library'; import { CommandLineFlagParameter, CommandLineParameter, @@ -38,6 +38,7 @@ import { IExecutionResult } from '../../logic/operations/IOperationExecutionResu import { OperationResultSummarizerPlugin } from '../../logic/operations/OperationResultSummarizerPlugin'; import type { ITelemetryOperationResult } from '../../logic/Telemetry'; import { parseParallelism } from '../parsing/ParseParallelism'; +import { RushProjectConfiguration } from '../../api/RushProjectConfiguration'; /** * Constructor parameters for PhasedScriptAction. @@ -293,6 +294,14 @@ export class PhasedScriptAction extends BaseScriptAction { const changedProjectsOnly: boolean = !!this._changedProjectsOnly?.value; + const selectedProjects: Set = + await this._selectionParameters.getSelectedProjectsAsync(terminal); + + if (!selectedProjects.size) { + terminal.writeLine(colors.yellow(`The command line selection parameters did not match any projects.`)); + return; + } + let buildCacheConfiguration: BuildCacheConfiguration | undefined; if (!this._disableBuildCache) { buildCacheConfiguration = await BuildCacheConfiguration.tryLoadAsync( @@ -302,13 +311,15 @@ export class PhasedScriptAction extends BaseScriptAction { ); } - const projectSelection: Set = - await this._selectionParameters.getSelectedProjectsAsync(terminal); - - if (!projectSelection.size) { - terminal.writeLine(colors.yellow(`The command line selection parameters did not match any projects.`)); - return; - } + terminal.writeVerbose(`Loading rush-project.json files...`); + const projectSelection: Map = new Map(); + await Async.forEachAsync(selectedProjects, async (project: RushConfigurationProject) => { + const projectConfiguration: RushProjectConfiguration | undefined = + await RushProjectConfiguration.tryLoadForProjectAsync(project, terminal); + projectSelection.set(project, projectConfiguration); + projectConfiguration?.validatePhaseConfiguration(this._initialPhases, terminal); + }); + terminal.writeVerboseLine(`Loaded.`); const isWatch: boolean = this._watchParameter?.value || this._alwaysWatch; @@ -328,7 +339,7 @@ export class PhasedScriptAction extends BaseScriptAction { phaseSelection: new Set(this._initialPhases), projectChangeAnalyzer, projectSelection, - projectsInUnknownState: projectSelection + projectsInUnknownState: selectedProjects }; const executionManagerOptions: IOperationExecutionManagerOptions = { @@ -397,7 +408,7 @@ export class PhasedScriptAction extends BaseScriptAction { const phaseSelection: Set = new Set(this._watchPhases); - const { projectChangeAnalyzer: initialState, projectSelection: projectsToWatch } = + const { projectChangeAnalyzer: initialState, projectsInUnknownState: projectsToWatch } = initialCreateOperationsContext; // Use async import so that we don't pay the cost for sync builds @@ -494,13 +505,13 @@ export class PhasedScriptAction extends BaseScriptAction { executionManagerOptions ); - const { isInitial, isWatch } = options.createOperationsContext; + const { isInitial, isWatch, projectChangeAnalyzer } = options.createOperationsContext; let success: boolean = false; let result: IExecutionResult | undefined; try { - result = await executionManager.executeAsync(); + result = await executionManager.executeAsync(projectChangeAnalyzer); success = result.status === OperationStatus.Success; await this.hooks.afterExecuteOperations.promise(result, options.createOperationsContext); diff --git a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts b/libraries/rush-lib/src/logic/buildCache/BuildCacheOperationProcessor.ts similarity index 50% rename from libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts rename to libraries/rush-lib/src/logic/buildCache/BuildCacheOperationProcessor.ts index 7a9a8f04ab9..c217982ce47 100644 --- a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts +++ b/libraries/rush-lib/src/logic/buildCache/BuildCacheOperationProcessor.ts @@ -2,28 +2,21 @@ // See LICENSE in the project root for license information. import * as path from 'path'; -import * as crypto from 'crypto'; -import { FileSystem, Path, ITerminal, FolderItem, InternalError, Async } from '@rushstack/node-core-library'; +import { FileSystem, ITerminal, FolderItem, InternalError } from '@rushstack/node-core-library'; import { RushConfigurationProject } from '../../api/RushConfigurationProject'; -import { RushProjectConfiguration } from '../../api/RushProjectConfiguration'; -import { RushConstants } from '../RushConstants'; import { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; -import { ICloudBuildCacheProvider } from './ICloudBuildCacheProvider'; -import { FileSystemBuildCacheProvider } from './FileSystemBuildCacheProvider'; import { TarExecutable } from '../../utilities/TarExecutable'; +import { OperationStatus } from '../operations/OperationStatus'; +import { IOperationProcessor } from '../operations/IOperationProcessor'; +import { IOperationRunnerContext } from '../operations/IOperationRunner'; import { EnvironmentVariableNames } from '../../api/EnvironmentConfiguration'; export interface IProjectBuildCacheOptions { buildCacheConfiguration: BuildCacheConfiguration; - projectConfiguration: RushProjectConfiguration; - projectOutputFolderNames: ReadonlyArray; - additionalProjectOutputFilePaths?: ReadonlyArray; - command: string; - trackedProjectFiles: Iterable | undefined; - hash: string; - terminal: ITerminal; + project: RushConfigurationProject; phaseName: string; + outputFolderNames: ReadonlyArray; } interface IPathsToCache { @@ -31,134 +24,74 @@ interface IPathsToCache { outputFilePaths: string[]; } -export class ProjectBuildCache { +export class BuildCacheOperationProcessor implements IOperationProcessor { /** * null === we haven't tried to initialize yet * undefined === unable to initialize */ private static _tarUtilityPromise: Promise | null = null; + private readonly _buildCacheConfiguration: BuildCacheConfiguration; private readonly _project: RushConfigurationProject; - private readonly _localBuildCacheProvider: FileSystemBuildCacheProvider; - private readonly _cloudBuildCacheProvider: ICloudBuildCacheProvider | undefined; - private readonly _buildCacheEnabled: boolean; - private readonly _cacheWriteEnabled: boolean; - private readonly _projectOutputFolderNames: ReadonlyArray; - private readonly _additionalProjectOutputFilePaths: ReadonlyArray; - private _cacheId: string | undefined; + private readonly _phaseName: string; + private readonly _outputFolderNames: ReadonlyArray; - private constructor(cacheId: string | undefined, options: IProjectBuildCacheOptions) { - const { - buildCacheConfiguration, - projectConfiguration, - projectOutputFolderNames, - additionalProjectOutputFilePaths - } = options; - this._project = projectConfiguration.project; - this._localBuildCacheProvider = buildCacheConfiguration.localCacheProvider; - this._cloudBuildCacheProvider = buildCacheConfiguration.cloudCacheProvider; - this._buildCacheEnabled = buildCacheConfiguration.buildCacheEnabled; - this._cacheWriteEnabled = buildCacheConfiguration.cacheWriteEnabled; - this._projectOutputFolderNames = projectOutputFolderNames || []; - this._additionalProjectOutputFilePaths = additionalProjectOutputFilePaths || []; - this._cacheId = cacheId; - } + private _cacheId: string | undefined; - private static _tryGetTarUtility(terminal: ITerminal): Promise { - if (ProjectBuildCache._tarUtilityPromise === null) { - ProjectBuildCache._tarUtilityPromise = TarExecutable.tryInitializeAsync(terminal); - } + public constructor(options: IProjectBuildCacheOptions) { + const { buildCacheConfiguration, project, outputFolderNames, phaseName } = options; + this._buildCacheConfiguration = buildCacheConfiguration; + this._project = project; + this._phaseName = phaseName; + this._outputFolderNames = outputFolderNames; - return ProjectBuildCache._tarUtilityPromise; + this._cacheId = undefined; } - public static async tryGetProjectBuildCache( - options: IProjectBuildCacheOptions - ): Promise { - const { terminal, projectConfiguration, projectOutputFolderNames, trackedProjectFiles, hash } = options; - if (!trackedProjectFiles || !hash) { - return undefined; - } - - if ( - !ProjectBuildCache._validateProject( - terminal, - projectConfiguration, - projectOutputFolderNames, - trackedProjectFiles - ) - ) { - return undefined; + private static _tryGetTarUtility(terminal: ITerminal): Promise { + if (BuildCacheOperationProcessor._tarUtilityPromise === null) { + BuildCacheOperationProcessor._tarUtilityPromise = TarExecutable.tryInitializeAsync(terminal); } - const cacheId: string | undefined = await ProjectBuildCache._getCacheId(options); - return new ProjectBuildCache(cacheId, options); + return BuildCacheOperationProcessor._tarUtilityPromise; } - private static _validateProject( - terminal: ITerminal, - projectConfiguration: RushProjectConfiguration, - projectOutputFolderNames: ReadonlyArray, - trackedProjectFiles: Iterable - ): boolean { - const normalizedProjectRelativeFolder: string = Path.convertToSlashes( - projectConfiguration.project.projectRelativeFolder - ); - const outputFolders: string[] = []; - if (projectOutputFolderNames) { - for (const outputFolderName of projectOutputFolderNames) { - outputFolders.push(`${normalizedProjectRelativeFolder}/${outputFolderName}/`); - } - } - - const inputOutputFiles: string[] = []; - for (const file of trackedProjectFiles) { - for (const outputFolder of outputFolders) { - if (file.startsWith(outputFolder)) { - inputOutputFiles.push(file); - } - } + public async beforeBuildAsync( + context: Pick + ): Promise { + const { stateHash, terminal, isCacheReadAllowed } = context; + if (!stateHash || !isCacheReadAllowed) { + return OperationStatus.Ready; } - if (inputOutputFiles.length > 0) { - terminal.writeWarningLine( - 'Unable to use build cache. The following files are used to calculate project state ' + - `and are considered project output: ${inputOutputFiles.join(', ')}` - ); - return false; - } else { - return true; - } - } + const { _buildCacheConfiguration: buildCacheConfiguration } = this; + const cacheId: string | undefined = (this._cacheId = buildCacheConfiguration.getCacheEntryId({ + phaseName: this._phaseName, + projectName: this._project.packageName, + stateHash + })); - public async tryRestoreFromCacheAsync(terminal: ITerminal): Promise { - const cacheId: string | undefined = this._cacheId; if (!cacheId) { - terminal.writeWarningLine('Unable to get cache ID. Ensure Git is installed.'); - return false; + return OperationStatus.Ready; } - if (!this._buildCacheEnabled) { - // Skip reading local and cloud build caches, without any noise - return false; - } + const { localCacheProvider, cloudCacheProvider } = buildCacheConfiguration; - let localCacheEntryPath: string | undefined = - await this._localBuildCacheProvider.tryGetCacheEntryPathByIdAsync(terminal, cacheId); + let localCacheEntryPath: string | undefined = await localCacheProvider.tryGetCacheEntryPathByIdAsync( + terminal, + cacheId + ); let cacheEntryBuffer: Buffer | undefined; let updateLocalCacheSuccess: boolean | undefined; - if (!localCacheEntryPath && this._cloudBuildCacheProvider) { + if (!localCacheEntryPath && cloudCacheProvider) { terminal.writeVerboseLine( - 'This project was not found in the local build cache. Querying the cloud build cache.' + 'This operation was not found in the local build cache. Querying the cloud build cache.' ); - cacheEntryBuffer = await this._cloudBuildCacheProvider.tryGetCacheEntryBufferByIdAsync( - terminal, - cacheId - ); + cacheEntryBuffer = await cloudCacheProvider.tryGetCacheEntryBufferByIdAsync(terminal, cacheId); if (cacheEntryBuffer) { try { - localCacheEntryPath = await this._localBuildCacheProvider.trySetCacheEntryBufferAsync( + localCacheEntryPath = await localCacheProvider.trySetCacheEntryBufferAsync( terminal, cacheId, cacheEntryBuffer @@ -171,8 +104,8 @@ export class ProjectBuildCache { } if (!localCacheEntryPath && !cacheEntryBuffer) { - terminal.writeVerboseLine('This project was not found in the build cache.'); - return false; + terminal.writeVerboseLine('This operation was not found in the build cache.'); + return OperationStatus.Ready; } terminal.writeLine('Build cache hit.'); @@ -181,14 +114,16 @@ export class ProjectBuildCache { const projectFolderPath: string = this._project.projectFolder; // Purge output folders - terminal.writeVerboseLine(`Clearing cached folders: ${this._projectOutputFolderNames.join(', ')}`); + terminal.writeVerboseLine(`Clearing cached folders: ${this._outputFolderNames.join(', ')}`); await Promise.all( - this._projectOutputFolderNames.map((outputFolderName: string) => + this._outputFolderNames.map((outputFolderName: string) => FileSystem.deleteFolderAsync(`${projectFolderPath}/${outputFolderName}`) ) ); - const tarUtility: TarExecutable | undefined = await ProjectBuildCache._tryGetTarUtility(terminal); + const tarUtility: TarExecutable | undefined = await BuildCacheOperationProcessor._tryGetTarUtility( + terminal + ); let restoreSuccess: boolean = false; if (tarUtility && localCacheEntryPath) { const logFilePath: string = this._getTarLogFilePath(); @@ -209,24 +144,34 @@ export class ProjectBuildCache { terminal.writeWarningLine('Unable to update the local build cache with data from the cloud cache.'); } - return restoreSuccess; + return restoreSuccess ? OperationStatus.FromCache : OperationStatus.Ready; } - public async trySetCacheEntryAsync(terminal: ITerminal): Promise { - if (!this._cacheWriteEnabled) { + public async afterBuildAsync( + context: Pick, + status: OperationStatus + ): Promise { + const { terminal, isCacheWriteAllowed } = context; + const { _buildCacheConfiguration: buildCacheConfiguration } = this; + + if (status !== OperationStatus.Success) { + return status; + } + + if (!buildCacheConfiguration.cacheWriteEnabled || !isCacheWriteAllowed) { // Skip writing local and cloud build caches, without any noise - return true; + return status; } const cacheId: string | undefined = this._cacheId; if (!cacheId) { - terminal.writeWarningLine('Unable to get cache ID. Ensure Git is installed.'); - return false; + return status; } + const { localCacheProvider, cloudCacheProvider } = buildCacheConfiguration; const filesToCache: IPathsToCache | undefined = await this._tryCollectPathsToCacheAsync(terminal); if (!filesToCache) { - return false; + return OperationStatus.SuccessWithWarning; } terminal.writeVerboseLine( @@ -235,9 +180,11 @@ export class ProjectBuildCache { let localCacheEntryPath: string | undefined; - const tarUtility: TarExecutable | undefined = await ProjectBuildCache._tryGetTarUtility(terminal); + const tarUtility: TarExecutable | undefined = await BuildCacheOperationProcessor._tryGetTarUtility( + terminal + ); if (tarUtility) { - const finalLocalCacheEntryPath: string = this._localBuildCacheProvider.getCacheEntryPath(cacheId); + const finalLocalCacheEntryPath: string = localCacheProvider.getCacheEntryPath(cacheId); // Derive the temp file from the destination path to ensure they are on the same volume const tempLocalCacheEntryPath: string = `${finalLocalCacheEntryPath}.temp`; const logFilePath: string = this._getTarLogFilePath(); @@ -261,14 +208,14 @@ export class ProjectBuildCache { `"tar" exited with code ${tarExitCode} while attempting to create the cache entry. ` + `See "${logFilePath}" for logs from the tar process.` ); - return false; + return OperationStatus.SuccessWithWarning; } } else { terminal.writeWarningLine( `Unable to locate "tar". Please ensure that "tar" is on your PATH environment variable, or set the ` + `${EnvironmentVariableNames.RUSH_TAR_BINARY_PATH} environment variable to the full path to the "tar" binary.` ); - return false; + return OperationStatus.SuccessWithWarning; } let cacheEntryBuffer: Buffer | undefined; @@ -279,14 +226,16 @@ export class ProjectBuildCache { // the configured CLOUD cache. If the cache is enabled, rush is always allowed to read from and // write to the local build cache. - if (this._cloudBuildCacheProvider?.isCacheWriteAllowed) { - if (localCacheEntryPath) { - cacheEntryBuffer = await FileSystem.readFileToBufferAsync(localCacheEntryPath); - } else { - throw new InternalError('Expected the local cache entry path to be set.'); + if (cloudCacheProvider?.isCacheWriteAllowed) { + if (!cacheEntryBuffer) { + if (localCacheEntryPath) { + cacheEntryBuffer = await FileSystem.readFileToBufferAsync(localCacheEntryPath); + } else { + throw new InternalError('Expected the local cache entry path to be set.'); + } } - setCloudCacheEntryPromise = this._cloudBuildCacheProvider?.trySetCacheEntryBufferAsync( + setCloudCacheEntryPromise = cloudCacheProvider?.trySetCacheEntryBufferAsync( terminal, cacheId, cacheEntryBuffer @@ -306,7 +255,7 @@ export class ProjectBuildCache { terminal.writeWarningLine('Unable to set both cloud and local cache entries.'); } - return success; + return success ? status : OperationStatus.SuccessWithWarning; } /** @@ -341,7 +290,7 @@ export class ProjectBuildCache { } // Handle declared output folders. - for (const outputFolder of this._projectOutputFolderNames) { + for (const outputFolder of this._outputFolderNames) { const diskPath: string = `${projectFolderPath}/${outputFolder}`; try { const children: FolderItem[] = await FileSystem.readFolderItemsAsync(diskPath); @@ -367,22 +316,6 @@ export class ProjectBuildCache { return undefined; } - // Add additional output file paths - await Async.forEachAsync( - this._additionalProjectOutputFilePaths, - async (additionalProjectOutputFilePath) => { - const fullPath: string = `${projectFolderPath}/${additionalProjectOutputFilePath}`; - const pathExists: boolean = await FileSystem.existsAsync(fullPath); - if (pathExists) { - outputFilePaths.push(additionalProjectOutputFilePath); - } - }, - { concurrency: 10 } - ); - - // Ensure stable output path order. - outputFilePaths.sort(); - return { outputFilePaths, filteredOutputFolderNames @@ -392,33 +325,4 @@ export class ProjectBuildCache { private _getTarLogFilePath(): string { return path.join(this._project.projectRushTempFolder, `${this._cacheId}.log`); } - - private static async _getCacheId(options: IProjectBuildCacheOptions): Promise { - // The project state hash is calculated in the following method: - // - A SHA1 hash is created and the following data is fed into it, in order: - // 1. The JSON-serialized list of output folder names for this - // project (see ProjectBuildCache._projectOutputFolderNames) - // 2. The command that will be run in the project - // 3. The hash of the projet inputs - // - A hex digest of the hash is returned - const hash: crypto.Hash = crypto.createHash('sha1'); - // This value is used to force cache bust when the build cache algorithm changes - hash.update(`${RushConstants.buildCacheVersion}`); - hash.update(RushConstants.hashDelimiter); - const serializedOutputFolders: string = JSON.stringify(options.projectOutputFolderNames); - hash.update(serializedOutputFolders); - hash.update(RushConstants.hashDelimiter); - hash.update(options.command); - hash.update(RushConstants.hashDelimiter); - hash.update(options.hash); - hash.update(RushConstants.hashDelimiter); - - const projectStateHash: string = hash.digest('hex'); - - return options.buildCacheConfiguration.getCacheEntryId({ - projectName: options.projectConfiguration.project.packageName, - projectStateHash, - phaseName: options.phaseName - }); - } } diff --git a/libraries/rush-lib/src/logic/buildCache/CacheEntryId.ts b/libraries/rush-lib/src/logic/buildCache/CacheEntryId.ts index b3f2f906d40..40a3b3d5948 100644 --- a/libraries/rush-lib/src/logic/buildCache/CacheEntryId.ts +++ b/libraries/rush-lib/src/logic/buildCache/CacheEntryId.ts @@ -19,7 +19,7 @@ export interface IGenerateCacheEntryIdOptions { /** * A hash of the input files */ - projectStateHash: string; + stateHash: string; } /** @@ -40,7 +40,7 @@ export class CacheEntryId { public static parsePattern(pattern?: string): GetCacheEntryIdFunction { if (!pattern) { - return ({ projectStateHash }) => projectStateHash; + return ({ stateHash }) => stateHash; } else { pattern = pattern.trim(); @@ -84,7 +84,7 @@ export class CacheEntryId { } foundHashToken = true; - return `\${${OPTIONS_ARGUMENT_NAME}.projectStateHash}`; + return `\${${OPTIONS_ARGUMENT_NAME}.stateHash}`; } case PROJECT_NAME_TOKEN_NAME: { diff --git a/libraries/rush-lib/src/logic/buildCache/test/CacheEntryId.test.ts b/libraries/rush-lib/src/logic/buildCache/test/CacheEntryId.test.ts index c05247f5c56..79f15036299 100644 --- a/libraries/rush-lib/src/logic/buildCache/test/CacheEntryId.test.ts +++ b/libraries/rush-lib/src/logic/buildCache/test/CacheEntryId.test.ts @@ -14,7 +14,7 @@ describe(CacheEntryId.name, () => { expect( getCacheEntryId({ projectName, - projectStateHash: '09d1ecee6d5f888fa6c35ca804b5dac7c3735ce3', + stateHash: '09d1ecee6d5f888fa6c35ca804b5dac7c3735ce3', phaseName: '_phase:compile', ...generateCacheEntryIdOptions }) diff --git a/libraries/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts b/libraries/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts deleted file mode 100644 index d0336b584c8..00000000000 --- a/libraries/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. -// See LICENSE in the project root for license information. - -import { StringBufferTerminalProvider, Terminal } from '@rushstack/node-core-library'; -import { BuildCacheConfiguration } from '../../../api/BuildCacheConfiguration'; -import { RushProjectConfiguration } from '../../../api/RushProjectConfiguration'; -import { IGenerateCacheEntryIdOptions } from '../CacheEntryId'; -import { FileSystemBuildCacheProvider } from '../FileSystemBuildCacheProvider'; - -import { ProjectBuildCache } from '../ProjectBuildCache'; - -interface ITestOptions { - enabled: boolean; - writeAllowed: boolean; - trackedProjectFiles: string[] | undefined; -} - -describe(ProjectBuildCache.name, () => { - async function prepareSubject(options: Partial): Promise { - const terminal: Terminal = new Terminal(new StringBufferTerminalProvider()); - const hash: string = 'state_hash'; - - const subject: ProjectBuildCache | undefined = await ProjectBuildCache.tryGetProjectBuildCache({ - buildCacheConfiguration: { - buildCacheEnabled: options.hasOwnProperty('enabled') ? options.enabled : true, - getCacheEntryId: (options: IGenerateCacheEntryIdOptions) => - `${options.projectName}/${options.projectStateHash}`, - localCacheProvider: undefined as unknown as FileSystemBuildCacheProvider, - cloudCacheProvider: { - isCacheWriteAllowed: options.hasOwnProperty('writeAllowed') ? options.writeAllowed : false - } - } as unknown as BuildCacheConfiguration, - projectOutputFolderNames: ['dist'], - projectConfiguration: { - project: { - packageName: 'acme-wizard', - projectRelativeFolder: 'apps/acme-wizard', - dependencyProjects: [] - } - } as unknown as RushProjectConfiguration, - command: 'build', - trackedProjectFiles: options.hasOwnProperty('trackedProjectFiles') ? options.trackedProjectFiles : [], - hash, - terminal, - phaseName: 'build' - }); - - return subject; - } - - describe(ProjectBuildCache.tryGetProjectBuildCache.name, () => { - it('returns a ProjectBuildCache with a calculated cacheId value', async () => { - const subject: ProjectBuildCache = (await prepareSubject({}))!; - expect(subject['_cacheId']).toMatchSnapshot(); - }); - - it('returns undefined if the tracked file list is undefined', async () => { - expect( - await prepareSubject({ - trackedProjectFiles: undefined - }) - ).toBe(undefined); - }); - }); -}); diff --git a/libraries/rush-lib/src/logic/buildCache/test/__snapshots__/ProjectBuildCache.test.ts.snap b/libraries/rush-lib/src/logic/buildCache/test/__snapshots__/ProjectBuildCache.test.ts.snap deleted file mode 100644 index 0290e712377..00000000000 --- a/libraries/rush-lib/src/logic/buildCache/test/__snapshots__/ProjectBuildCache.test.ts.snap +++ /dev/null @@ -1,3 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`ProjectBuildCache tryGetProjectBuildCache returns a ProjectBuildCache with a calculated cacheId value 1`] = `"acme-wizard/bb7633fef1e8a70f64714aabfeda78b5cd803e3c"`; diff --git a/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts b/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts index ebb3e892cc2..7073ab90638 100644 --- a/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts +++ b/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import type { StdioSummarizer } from '@rushstack/terminal'; import type { OperationStatus } from './OperationStatus'; import type { IStopwatchResult } from '../../utilities/Stopwatch'; import type { Operation } from './Operation'; @@ -27,10 +26,6 @@ export interface IOperationExecutionResult { * Object tracking execution timing. */ readonly stopwatch: IStopwatchResult; - /** - * Object used to report a summary at the end of the Rush invocation. - */ - readonly stdioSummarizer: StdioSummarizer; /** * The value indicates the duration of the same operation without cache hit. */ diff --git a/libraries/rush-lib/src/logic/operations/IOperationProcessor.ts b/libraries/rush-lib/src/logic/operations/IOperationProcessor.ts new file mode 100644 index 00000000000..e29c5082658 --- /dev/null +++ b/libraries/rush-lib/src/logic/operations/IOperationProcessor.ts @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { IOperationRunnerContext } from './IOperationRunner'; +import { OperationStatus } from './OperationStatus'; + +/** + * + * @alpha + */ +export interface IOperationProcessor { + /** + * + */ + beforeBuildAsync(context: IOperationRunnerContext): Promise; + /** + * + */ + afterBuildAsync(context: IOperationRunnerContext, status: OperationStatus): Promise; +} diff --git a/libraries/rush-lib/src/logic/operations/IOperationRunner.ts b/libraries/rush-lib/src/logic/operations/IOperationRunner.ts index 930c0e81644..830d930c2d9 100644 --- a/libraries/rush-lib/src/logic/operations/IOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/IOperationRunner.ts @@ -1,12 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import type { StdioSummarizer } from '@rushstack/terminal'; -import type { CollatedWriter } from '@rushstack/stream-collator'; +import { ITerminal } from '@rushstack/node-core-library'; +import type { TerminalWritable } from '@rushstack/terminal'; import type { OperationStatus } from './OperationStatus'; -import type { OperationStateFile } from './OperationStateFile'; -import type { IStopwatchResult } from '../../utilities/Stopwatch'; /** * Information passed to the executing `IOperationRunner` @@ -14,36 +12,40 @@ import type { IStopwatchResult } from '../../utilities/Stopwatch'; * @beta */ export interface IOperationRunnerContext { - /** - * The writer into which this `IOperationRunner` should write its logs. - */ - collatedWriter: CollatedWriter; /** * If Rush was invoked with `--debug` */ debugMode: boolean; + /** * Defaults to `true`. Will be `false` if Rush was invoked with `--verbose`. */ quietMode: boolean; + /** * Defaults to `true`. Will be `false` if a dependency is in an unknown state. */ - isCacheWriteAllowed: boolean; + isSkipAllowed: boolean; + /** - * Object used to report a summary at the end of the Rush invocation. + * Defaults to `true`. Will be `false` if a dependency is in an unknown state. */ - stdioSummarizer: StdioSummarizer; + isCacheReadAllowed: boolean; + /** - * Object used to record state of the operation. - * - * @internal + * Defaults to `true`. Will be `false` if a dependency is in an unknown state. */ - _operationStateFile?: OperationStateFile; + isCacheWriteAllowed: boolean; + /** - * Object used to track elapsed time. + * Terminal instance for logging messages. */ - stopwatch: IStopwatchResult; + terminal: ITerminal; + + /** + * Raw terminal for forwarding stdout/stderr + */ + terminalWritable: TerminalWritable; /** * The hashes of all tracked files pertinent to the operation @@ -69,11 +71,6 @@ export interface IOperationRunner { */ readonly name: string; - /** - * This flag determines if the operation is allowed to be skipped if up to date. - */ - isSkipAllowed: boolean; - /** * Indicates that this runner's duration has meaning. */ @@ -90,11 +87,6 @@ export interface IOperationRunner { */ warningsAreAllowed: boolean; - /** - * Indicates if the output of this operation may be written to the cache - */ - isCacheWriteAllowed: boolean; - /** * Method to be executed for the operation. */ diff --git a/libraries/rush-lib/src/logic/operations/NullOperationRunner.ts b/libraries/rush-lib/src/logic/operations/NullOperationRunner.ts index 8e3f14fac25..974e50a360a 100644 --- a/libraries/rush-lib/src/logic/operations/NullOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/NullOperationRunner.ts @@ -31,10 +31,6 @@ export class NullOperationRunner implements IOperationRunner { // This operation does nothing, so timing is meaningless public readonly reportTiming: boolean = false; public readonly silent: boolean; - // The operation may be skipped; it doesn't do anything anyway - public isSkipAllowed: boolean = true; - // The operation is a no-op, so is cacheable. - public isCacheWriteAllowed: boolean = true; // Nothing will get logged, no point allowing warnings public readonly warningsAreAllowed: boolean = false; diff --git a/libraries/rush-lib/src/logic/operations/Operation.ts b/libraries/rush-lib/src/logic/operations/Operation.ts index 25a83d0c186..fdb3b083412 100644 --- a/libraries/rush-lib/src/logic/operations/Operation.ts +++ b/libraries/rush-lib/src/logic/operations/Operation.ts @@ -5,7 +5,9 @@ import * as crypto from 'crypto'; import { RushConfigurationProject } from '../../api/RushConfigurationProject'; import { IPhase } from '../../api/CommandLineConfiguration'; +import { IOperationProcessor } from './IOperationProcessor'; import { IOperationRunner } from './IOperationRunner'; +import { IProjectFileFilter } from '../ProjectChangeAnalyzer'; import { RushConstants } from '../RushConstants'; /** @@ -16,16 +18,28 @@ export interface IOperationOptions { /** * The Rush phase associated with this Operation, if any */ - phase?: IPhase | undefined; + phase: IPhase; /** * The Rush project associated with this Operation, if any */ - project?: RushConfigurationProject | undefined; + project: RushConfigurationProject; /** * When the scheduler is ready to process this `Operation`, the `runner` implements the actual work of * running the operation. */ runner?: IOperationRunner | undefined; + /** + * For use by incremental skip and the build cache, the list of output folders + */ + outputFolderNames?: ReadonlyArray | undefined; + /** + * For use by incremental skip and the build cache, a function to filter tracked files + */ + projectFileFilter?: IProjectFileFilter | undefined; + /** + * Operator to do pre/post build operations + */ + processor?: IOperationProcessor | undefined; } /** @@ -41,12 +55,12 @@ export class Operation { /** * The Rush phase associated with this Operation, if any */ - public readonly associatedPhase: IPhase | undefined; + public readonly associatedPhase: IPhase; /** * The Rush project associated with this Operation, if any */ - public readonly associatedProject: RushConfigurationProject | undefined; + public readonly associatedProject: RushConfigurationProject; /** * A set of all operations which depend on this operation. @@ -62,7 +76,7 @@ export class Operation { * When the scheduler is ready to process this `Operation`, the `runner` implements the actual work of * running the operation. */ - public runner: IOperationRunner | undefined = undefined; + public runner: IOperationRunner | undefined; /** * The weight for this operation. This scalar is the contribution of this operation to the @@ -77,10 +91,33 @@ export class Operation { */ public weight: number = 1; - public constructor(options?: IOperationOptions) { - this.associatedPhase = options?.phase; - this.associatedProject = options?.project; - this.runner = options?.runner; + public readonly outputFolderNames: ReadonlyArray; + + public readonly projectFileFilter: IProjectFileFilter | undefined; + + public processor: IOperationProcessor | undefined; + + public logFilePath: string | undefined = undefined; + + public constructor(options: IOperationOptions) { + const { + phase, + outputFolderNames = [] + } = options; + + this.associatedPhase = phase; + this.associatedProject = options.project; + + const uniqueOutputFolderNames: Set = new Set(outputFolderNames); + uniqueOutputFolderNames.add( + `${RushConstants.projectRushFolderName}/${RushConstants.rushTempFolderName}/operation/${phase.logFilenameIdentifier}`); + const sortedOutputFolderNames: string[] = Array.from(uniqueOutputFolderNames).sort(); + this.outputFolderNames = sortedOutputFolderNames; + + this.projectFileFilter = options.projectFileFilter; + + this.runner = options.runner; + this.processor = options.processor; } /** @@ -106,22 +143,29 @@ export class Operation { } } - const sortedHashes: string[] = dependencyHashes.sort(); const hash: crypto.Hash = crypto.createHash('sha1'); - hash.update(localHash); - for (const dependencyHash of sortedHashes) { - hash.update(dependencyHash); + hash.update(`${RushConstants.buildCacheVersion}`); + hash.update(RushConstants.hashDelimiter); + + // Output folder names are part of the configuration, so include in hash + for (const outputFolder of this.outputFolderNames) { + hash.update(outputFolder); hash.update(RushConstants.hashDelimiter); } // CLI parameters that apply to the phase affect the result - const { associatedPhase } = this; - if (associatedPhase) { - const params: string[] = []; - for (const tsCommandLineParameter of associatedPhase.associatedParameters) { - tsCommandLineParameter.appendToArgList(params); - } - hash.update(params.join(' ')); + const params: string[] = []; + for (const tsCommandLineParameter of this.associatedPhase.associatedParameters) { + tsCommandLineParameter.appendToArgList(params); + } + hash.update(params.join(' ')); + + hash.update(localHash); + + const sortedHashes: string[] = dependencyHashes.sort(); + for (const dependencyHash of sortedHashes) { + hash.update(dependencyHash); + hash.update(RushConstants.hashDelimiter); } return hash.digest('hex'); } diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts index d7859855411..fbc94e0287f 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts @@ -4,7 +4,7 @@ import colors from 'colors/safe'; import { TerminalWritable, StdioWritable, TextRewriterTransform } from '@rushstack/terminal'; import { StreamCollator, CollatedWriter } from '@rushstack/stream-collator'; -import { NewlineKind, Async, Terminal, ITerminal } from '@rushstack/node-core-library'; +import { NewlineKind, Async, Terminal, ITerminal, AlreadyReportedError, Colors } from '@rushstack/node-core-library'; import { AsyncOperationQueue, IOperationSortFunction } from './AsyncOperationQueue'; import { Operation } from './Operation'; @@ -12,8 +12,8 @@ import { OperationStatus } from './OperationStatus'; import { IOperationExecutionRecordContext, OperationExecutionRecord } from './OperationExecutionRecord'; import { IExecutionResult } from './IOperationExecutionResult'; import { ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; -import { RushConfigurationProject } from '../../api/RushConfigurationProject'; import { CollatedTerminalProvider } from '../../utilities/CollatedTerminalProvider'; +import { LookupByPath } from '../LookupByPath'; export interface IOperationExecutionManagerOptions { quietMode: boolean; @@ -216,34 +216,43 @@ export class OperationExecutionManager { } private async _updateStateAsync(state: ProjectChangeAnalyzer): Promise { - this._terminal.writeLine(`Updating state hashes`); - const trackedFilesByProject: Map | undefined> = - new Map(); - for (const { associatedProject } of this._executionRecords.keys()) { - if (associatedProject && !trackedFilesByProject.has(associatedProject)) { - const trackedFiles: ReadonlyMap | undefined = state._tryGetProjectDependencies( - associatedProject, - this._terminal - ); - trackedFilesByProject.set(associatedProject, trackedFiles); - } - } + const { _terminal: terminal } = this; + terminal.writeLine(`Updating operation states...`); + let hasIssues: boolean = false; function getOperationHash(record: OperationExecutionRecord): string { let { stateHash } = record; if (stateHash === undefined) { const { operation } = record; - const { associatedProject } = operation; + const { associatedProject, projectFileFilter, outputFolderNames, processor } = operation; stateHash = ''; - if (associatedProject) { - const trackedFiles: ReadonlyMap | undefined = - trackedFilesByProject.get(associatedProject); - record.trackedFileHashes = trackedFiles; - const localHash: string = trackedFiles ? state._hashProjectDependencies(trackedFiles) : ''; - if (localHash) { - stateHash = operation.getHash(localHash, Array.from(record.dependencies, getOperationHash)); + const trackedFiles: ReadonlyMap | undefined = state._tryGetProjectDependencies( + associatedProject, + terminal, + projectFileFilter + ); + + if (trackedFiles && outputFolderNames?.length && processor) { + const projectOutputLookup: LookupByPath = new LookupByPath( + outputFolderNames.map((relativePath) => [relativePath, relativePath]) + ); + for (const trackedFilePath of trackedFiles.keys()) { + const match: string | undefined = projectOutputLookup.findChildPath(trackedFilePath); + if (match) { + hasIssues = true; + terminal.writeErrorLine( + `Project "${associatedProject.packageName}" contains Git tracked file "${trackedFilePath}" in configured ` + + `output folder "${match}". This is invalid. Either remove the file from Git or change the configuration to make it not an output.` + ); + } } } + + record.trackedFileHashes = trackedFiles; + const localHash: string = trackedFiles ? state._hashProjectDependencies(trackedFiles) : ''; + if (localHash) { + stateHash = operation.getHash(localHash, Array.from(record.dependencies, getOperationHash)); + } record.stateHash = stateHash; } return stateHash; @@ -252,6 +261,12 @@ export class OperationExecutionManager { for (const record of this._executionRecords.values()) { getOperationHash(record); } + + if (hasIssues) { + throw new AlreadyReportedError(); + } + + this._terminal.writeLine(`Finished updating operation states.`); } /** @@ -260,8 +275,8 @@ export class OperationExecutionManager { private _onOperationComplete(record: OperationExecutionRecord): void { const { runner, name, status } = record; - let blockCacheWrite: boolean = !runner.isCacheWriteAllowed; - let blockSkip: boolean = !runner.isSkipAllowed; + let blockCacheWrite: boolean = !record.isCacheWriteAllowed; + let blockSkip: boolean = !record.isSkipAllowed; const silent: boolean = runner.silent; @@ -274,11 +289,11 @@ export class OperationExecutionManager { // Generally speaking, silent operations shouldn't be able to fail, so this is a safety measure. const message: string | undefined = record.error?.message; // This creates the writer, so don't do this globally - const { terminal } = record.collatedWriter; + const { terminal } = record; if (message) { - terminal.writeStderrLine(message); + terminal.writeErrorLine(message); } - terminal.writeStderrLine(colors.red(`"${name}" failed to build.`)); + terminal.writeErrorLine(`"${name}" failed to build.`); const blockedQueue: Set = new Set(record.consumers); for (const blockedRecord of blockedQueue) { if (blockedRecord.status === OperationStatus.Ready) { @@ -288,7 +303,7 @@ export class OperationExecutionManager { // {blockedRecord.runner} with a no-op that sets status to Blocked and logs the blocking // operations. However, the existing behavior is a bit simpler, so keeping that for now. if (!blockedRecord.silent) { - terminal.writeStdoutLine(`"${blockedRecord.name}" is blocked by "${name}".`); + terminal.writeErrorLine(`"${blockedRecord.name}" is blocked by "${name}".`); } blockedRecord.status = OperationStatus.Blocked; @@ -306,8 +321,8 @@ export class OperationExecutionManager { */ case OperationStatus.FromCache: { if (!silent) { - record.collatedWriter.terminal.writeStdoutLine( - colors.green(`"${name}" was restored from the build cache.`) + record.terminal.writeLine( + Colors.green(`"${name}" was restored from the build cache.`) ); } break; @@ -318,7 +333,7 @@ export class OperationExecutionManager { */ case OperationStatus.Skipped: { if (!silent) { - record.collatedWriter.terminal.writeStdoutLine(colors.green(`"${name}" was skipped.`)); + record.terminal.writeLine(Colors.green(`"${name}" was skipped.`)); } // Skipping means cannot guarantee integrity, so prevent cache writes in dependents. blockCacheWrite = true; @@ -330,15 +345,15 @@ export class OperationExecutionManager { */ case OperationStatus.NoOp: { if (!silent) { - record.collatedWriter.terminal.writeStdoutLine(colors.gray(`"${name}" did not define any work.`)); + record.terminal.writeLine(Colors.gray(`"${name}" did not define any work.`)); } break; } case OperationStatus.Success: { if (!silent) { - record.collatedWriter.terminal.writeStdoutLine( - colors.green(`"${name}" completed successfully in ${record.stopwatch.toString()}.`) + record.terminal.writeLine( + Colors.green(`"${name}" completed successfully in ${record.stopwatch.toString()}.`) ); } // Legacy incremental build, if asked, prevent skip in dependents if the operation executed. @@ -348,8 +363,8 @@ export class OperationExecutionManager { case OperationStatus.SuccessWithWarning: { if (!silent) { - record.collatedWriter.terminal.writeStderrLine( - colors.yellow(`"${name}" completed with warnings in ${record.stopwatch.toString()}.`) + record.terminal.writeWarningLine( + `"${name}" completed with warnings in ${record.stopwatch.toString()}.` ); } // Legacy incremental build, if asked, prevent skip in dependents if the operation executed. @@ -367,7 +382,7 @@ export class OperationExecutionManager { if (blockSkip) { // Only relevant in legacy non-build cache flow - item.runner.isSkipAllowed = false; + item.isSkipAllowed = false; } // Remove this operation from the dependencies, to unblock the scheduler diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts index b9630a469ff..74eb0d87b82 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts @@ -1,15 +1,17 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { StdioSummarizer } from '@rushstack/terminal'; -import { InternalError } from '@rushstack/node-core-library'; -import { CollatedWriter, StreamCollator } from '@rushstack/stream-collator'; +import { DiscardStdoutTransform, SplitterTransform, StderrLineTransform, StdioSummarizer, TerminalWritable, TextRewriterTransform } from '@rushstack/terminal'; +import { InternalError, ITerminal, NewlineKind, Terminal } from '@rushstack/node-core-library'; +import { CollatedTerminal, CollatedWriter, StreamCollator } from '@rushstack/stream-collator'; import { OperationStatus } from './OperationStatus'; import { IOperationRunner, IOperationRunnerContext } from './IOperationRunner'; import { Operation } from './Operation'; import { Stopwatch } from '../../utilities/Stopwatch'; import { OperationStateFile } from './OperationStateFile'; +import { IOperationProcessor } from './IOperationProcessor'; +import { CollatedTerminalProvider } from '../../utilities/CollatedTerminalProvider'; export interface IOperationExecutionRecordContext { streamCollator: StreamCollator; @@ -69,7 +71,9 @@ export class OperationExecutionRecord implements IOperationRunnerContext { public silent: boolean = false; - public isCacheWriteAllowed: boolean = false; + public isSkipAllowed: boolean = true; + public isCacheReadAllowed: boolean = true; + public isCacheWriteAllowed: boolean = true; public trackedFileHashes: ReadonlyMap | undefined = undefined; @@ -95,7 +99,9 @@ export class OperationExecutionRecord implements IOperationRunnerContext { private readonly _context: IOperationExecutionRecordContext; - private _collatedWriter: CollatedWriter | undefined = undefined; + private _writer: CollatedWriter | undefined = undefined; + private _terminalWritable: TerminalWritable | undefined = undefined; + private _terminal: ITerminal | undefined = undefined; public constructor(operation: Operation, context: IOperationExecutionRecordContext) { this.operation = operation; @@ -103,20 +109,20 @@ export class OperationExecutionRecord implements IOperationRunnerContext { if (!runner) { throw new InternalError( - `Operation for phase '${operation.associatedPhase?.name}' and project '${operation.associatedProject?.packageName}' has no runner.` + `Operation for phase '${operation.associatedPhase.name}' and project '${operation.associatedProject.packageName}' has no runner.` ); } this.silent = runner.silent; - this.isCacheWriteAllowed = runner.isCacheWriteAllowed; this.runner = runner; this.weight = operation.weight; - if (operation.associatedPhase && operation.associatedProject) { - this._operationStateFile = new OperationStateFile({ - phase: operation.associatedPhase, - rushProject: operation.associatedProject - }); - } + + const { associatedPhase, associatedProject } = operation; + + this._operationStateFile = associatedPhase && associatedProject ? new OperationStateFile({ + phase: associatedPhase, + rushProject: associatedProject + }) : undefined; this._context = context; } @@ -132,25 +138,83 @@ export class OperationExecutionRecord implements IOperationRunnerContext { return this._context.quietMode; } - public get collatedWriter(): CollatedWriter { - // Lazy instantiate because the registerTask() call affects display ordering - if (!this._collatedWriter) { - this._collatedWriter = this._context.streamCollator.registerTask(this.name); - } - return this._collatedWriter; - } - public get nonCachedDurationMs(): number | undefined { // Lazy calculated because the state file is created/restored later on return this._operationStateFile?.state?.nonCachedDurationMs; } + public get terminalWritable(): TerminalWritable { + if (!this._terminalWritable) { + const stderrLineTransform: StderrLineTransform = new StderrLineTransform({ + destination: this.stdioSummarizer, + newlineKind: NewlineKind.Lf // for StdioSummarizer + }); + + const discardTransform: DiscardStdoutTransform = new DiscardStdoutTransform({ + destination: this._collatedWriter + }); + + const splitterTransform: SplitterTransform = new SplitterTransform({ + destinations: [this.quietMode ? discardTransform : this._collatedWriter, stderrLineTransform] + }); + + const normalizeNewlineTransform: TextRewriterTransform = new TextRewriterTransform({ + destination: splitterTransform, + normalizeNewlines: NewlineKind.Lf, + ensureNewlineAtEnd: true + }); + + this._terminalWritable = normalizeNewlineTransform; + } + return this._terminalWritable; + } + + public get terminal(): ITerminal { + if (!this._terminal) { + const collatedTerminal: CollatedTerminal = new CollatedTerminal(this.terminalWritable); + this._terminal = new Terminal(new CollatedTerminalProvider(collatedTerminal)); + } + return this._terminal; + } + + private get _collatedWriter(): CollatedWriter { + // Lazy instantiate because the registerTask() call affects display ordering + if (!this._writer) { + this._writer = this._context.streamCollator.registerTask(this.name); + } + return this._writer; + } + public async executeAsync(onResult: (record: OperationExecutionRecord) => void): Promise { this.status = OperationStatus.Executing; - this.stopwatch.start(); try { - this.status = await this.runner.executeAsync(this); + let status: OperationStatus = this.status; + const processor: IOperationProcessor | undefined = this.operation.processor; + if (processor) { + // Handle, e.g. build cache read + status = await processor.beforeBuildAsync(this); + if (status !== OperationStatus.Ready) { + this.status = status; + await this._operationStateFile?.tryRestoreAsync(); + return onResult(this); + } + } + + this.stopwatch.start(); + status = await this.runner.executeAsync(this); + this.stopwatch.stop(); + + await this._operationStateFile?.writeAsync({ + nonCachedDurationMs: this.stopwatch.duration * 1000 + }); + + if (processor) { + // Handle, e.g. build cache write + status = await processor.afterBuildAsync(this, status); + } + + this.status = status; // Delegate global state reporting onResult(this); } catch (error) { @@ -160,8 +224,8 @@ export class OperationExecutionRecord implements IOperationRunnerContext { onResult(this); } finally { this.silent = this.runner.silent; - this._collatedWriter?.close(); - this.stdioSummarizer.close(); + this._terminalWritable?.close(); + this._writer?.close(); this.stopwatch.stop(); } } diff --git a/libraries/rush-lib/src/logic/operations/OperationResultSummarizerPlugin.ts b/libraries/rush-lib/src/logic/operations/OperationResultSummarizerPlugin.ts index e328fe78ef1..b907a3f0630 100644 --- a/libraries/rush-lib/src/logic/operations/OperationResultSummarizerPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/OperationResultSummarizerPlugin.ts @@ -2,7 +2,7 @@ // See LICENSE in the project root for license information. import colors from 'colors/safe'; -import { InternalError, ITerminal } from '@rushstack/node-core-library'; +import { FileSystem, InternalError, ITerminal } from '@rushstack/node-core-library'; import { ICreateOperationsContext, IPhasedCommandPlugin, @@ -33,10 +33,10 @@ export class OperationResultSummarizerPlugin implements IPhasedCommandPlugin { } public apply(hooks: PhasedCommandHooks): void { - hooks.afterExecuteOperations.tap( + hooks.afterExecuteOperations.tapPromise( PLUGIN_NAME, - (result: IExecutionResult, context: ICreateOperationsContext): void => { - _printOperationStatus(this._terminal, result); + async (result: IExecutionResult, context: ICreateOperationsContext): Promise => { + await _printOperationStatus(this._terminal, result); } ); } @@ -46,7 +46,7 @@ export class OperationResultSummarizerPlugin implements IPhasedCommandPlugin { * Prints out a report of the status of each project * @internal */ -export function _printOperationStatus(terminal: ITerminal, result: IExecutionResult): void { +export async function _printOperationStatus(terminal: ITerminal, result: IExecutionResult): Promise { const { operationResults } = result; const operationsByStatus: IOperationsByStatus = new Map(); @@ -116,7 +116,7 @@ export function _printOperationStatus(terminal: ITerminal, result: IExecutionRes 'These operations completed successfully:' ); - writeDetailedSummary( + await writeDetailedSummary( terminal, OperationStatus.SuccessWithWarning, operationsByStatus, @@ -132,7 +132,7 @@ export function _printOperationStatus(terminal: ITerminal, result: IExecutionRes 'These operations were blocked by dependencies that failed:' ); - writeDetailedSummary(terminal, OperationStatus.Failure, operationsByStatus, colors.red); + await writeDetailedSummary(terminal, OperationStatus.Failure, operationsByStatus, colors.red); terminal.writeLine(''); @@ -193,13 +193,13 @@ function writeCondensedSummary( terminal.writeLine(''); } -function writeDetailedSummary( +async function writeDetailedSummary( terminal: ITerminal, status: OperationStatus, operationsByStatus: IOperationsByStatus, headingColor: (text: string) => string, shortStatusName?: string -): void { +): Promise { // Example: // // ==[ SUCCESS WITH WARNINGS: 2 projects ]================================ @@ -246,10 +246,16 @@ function writeDetailedSummary( )} ${colors.white(time)} ${colors.gray(']--')}\n` ); - const details: string = operationResult.stdioSummarizer.getReport(); - if (details) { - // Don't write a newline, because the report will always end with a newline - terminal.write(details); + if (operation.logFilePath) { + try { + const details: string = await FileSystem.readFileAsync(operation.logFilePath); + if (details) { + // Don't write a newline, because the report will always end with a newline + terminal.write(details); + } + } catch (err) { + // Do nothing + } } terminal.writeLine(''); diff --git a/libraries/rush-lib/src/logic/operations/PhasedOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/PhasedOperationPlugin.ts index f07984f2a6e..96374e0c24b 100644 --- a/libraries/rush-lib/src/logic/operations/PhasedOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/PhasedOperationPlugin.ts @@ -12,6 +12,10 @@ import type { IPhasedCommandPlugin, PhasedCommandHooks } from '../../pluginFramework/PhasedCommandHooks'; +import { IOperationSettings, RushProjectConfiguration } from '../../api/RushProjectConfiguration'; +import { IProjectFileFilter } from '../ProjectChangeAnalyzer'; +import ignore, { Ignore } from 'ignore'; +import { BuildCacheOperationProcessor } from '../buildCache/BuildCacheOperationProcessor'; const PLUGIN_NAME: 'PhasedOperationPlugin' = 'PhasedOperationPlugin'; @@ -29,15 +33,24 @@ function createOperations( existingOperations: Set, context: ICreateOperationsContext ): Set { - const { projectsInUnknownState: changedProjects, phaseSelection, projectSelection } = context; + const { + projectsInUnknownState: changedProjects, + phaseSelection, + projectSelection, + buildCacheConfiguration + } = context; const operationsWithWork: Set = new Set(); const operations: Map = new Map(); + const fileFilterCache: WeakMap = new WeakMap(); + + const buildCacheEnabled: boolean = !!context.buildCacheConfiguration?.buildCacheEnabled; + // Create tasks for selected phases and projects for (const phase of phaseSelection) { - for (const project of projectSelection) { - getOrCreateOperation(phase, project); + for (const [project, projectConfiguration] of projectSelection) { + getOrCreateOperation(phase, project, projectConfiguration); } } @@ -61,15 +74,60 @@ function createOperations( } return existingOperations; + function getFileFilter(ignoreGlobs: ReadonlyArray | undefined): IProjectFileFilter | undefined { + if (!ignoreGlobs || ignoreGlobs.length === 0) { + return; + } + + let filter: IProjectFileFilter | undefined = fileFilterCache.get(ignoreGlobs); + if (!filter) { + const ignoreMatcher: Ignore = ignore(); + for (const ignoreGlob of ignoreGlobs) { + ignoreMatcher.add(ignoreGlob); + } + filter = ignoreMatcher.createFilter(); + fileFilterCache.set(ignoreGlobs, filter); + } + + return filter; + } // Binds phaseSelection, projectSelection, operations via closure - function getOrCreateOperation(phase: IPhase, project: RushConfigurationProject): Operation { + function getOrCreateOperation( + phase: IPhase, + project: RushConfigurationProject, + projectConfiguration: RushProjectConfiguration | undefined + ): Operation { const key: string = getOperationKey(phase, project); let operation: Operation | undefined = operations.get(key); if (!operation) { + const optionsForPhase: IOperationSettings | undefined = + projectConfiguration?.operationSettingsByOperationName.get(phase.name); + const outputFolderNames: ReadonlyArray = optionsForPhase?.outputFolderNames || []; + const projectFileFilter: IProjectFileFilter | undefined = getFileFilter( + projectConfiguration?.incrementalBuildIgnoredGlobs + ); + + const enableCache: boolean = + buildCacheEnabled && + !!optionsForPhase && + !projectConfiguration?.disableBuildCacheForProject && + !optionsForPhase?.disableBuildCacheForOperation; + operation = new Operation({ project, - phase + phase, + outputFolderNames, + projectFileFilter, + processor: + enableCache && buildCacheConfiguration + ? new BuildCacheOperationProcessor({ + project, + phaseName: phase.name, + outputFolderNames, + buildCacheConfiguration + }) + : undefined }); if (!phaseSelection.has(phase) || !projectSelection.has(project)) { @@ -79,6 +137,7 @@ function createOperations( result: OperationStatus.Skipped, silent: true }); + operation.processor = undefined; } else if (changedProjects.has(project)) { operationsWithWork.add(operation); } @@ -91,7 +150,7 @@ function createOperations( } = phase; for (const depPhase of self) { - operation.addDependency(getOrCreateOperation(depPhase, project)); + operation.addDependency(getOrCreateOperation(depPhase, project, projectConfiguration)); } if (upstream.size) { @@ -99,7 +158,9 @@ function createOperations( if (dependencyProjects.size) { for (const depPhase of upstream) { for (const dependencyProject of dependencyProjects) { - operation.addDependency(getOrCreateOperation(depPhase, dependencyProject)); + operation.addDependency( + getOrCreateOperation(depPhase, dependencyProject, projectSelection.get(dependencyProject)) + ); } } } diff --git a/libraries/rush-lib/src/logic/operations/ProjectLogWritable.ts b/libraries/rush-lib/src/logic/operations/ProjectLogWritable.ts index 0310f0c6269..89ea6896898 100644 --- a/libraries/rush-lib/src/logic/operations/ProjectLogWritable.ts +++ b/libraries/rush-lib/src/logic/operations/ProjectLogWritable.ts @@ -1,9 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { FileSystem, FileWriter, InternalError } from '@rushstack/node-core-library'; +import { FileSystem, FileWriter, InternalError, ITerminal } from '@rushstack/node-core-library'; import { TerminalChunkKind, TerminalWritable, ITerminalChunk } from '@rushstack/terminal'; -import { CollatedTerminal } from '@rushstack/stream-collator'; import { RushConfigurationProject } from '../../api/RushConfigurationProject'; import { PackageNameParsers } from '../../api/PackageNameParsers'; @@ -11,7 +10,7 @@ import { RushConstants } from '../RushConstants'; export class ProjectLogWritable extends TerminalWritable { private readonly _project: RushConfigurationProject; - private readonly _terminal: CollatedTerminal; + private readonly _terminal: ITerminal; private _logPath: string; private _errorLogPath: string; @@ -21,7 +20,7 @@ export class ProjectLogWritable extends TerminalWritable { public constructor( project: RushConfigurationProject, - terminal: CollatedTerminal, + terminal: ITerminal, logFilenameIdentifier: string ) { super(); @@ -89,7 +88,7 @@ export class ProjectLogWritable extends TerminalWritable { try { this._logWriter.close(); } catch (error) { - this._terminal.writeStderrLine('Failed to close file handle for ' + this._logWriter.filePath); + this._terminal.writeErrorLine('Failed to close file handle for ' + this._logWriter.filePath); } this._logWriter = undefined; } @@ -98,7 +97,7 @@ export class ProjectLogWritable extends TerminalWritable { try { this._errorLogWriter.close(); } catch (error) { - this._terminal.writeStderrLine('Failed to close file handle for ' + this._errorLogWriter.filePath); + this._terminal.writeErrorLine('Failed to close file handle for ' + this._errorLogWriter.filePath); } this._errorLogWriter = undefined; } diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts index ae5390000dc..150701645d8 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts @@ -2,77 +2,33 @@ // See LICENSE in the project root for license information. import * as child_process from 'child_process'; -import * as path from 'path'; -import { - JsonFile, - Text, - FileSystem, - JsonObject, - NewlineKind, - InternalError, - ITerminal, - Terminal, - ColorValue -} from '@rushstack/node-core-library'; +import { Text, NewlineKind, InternalError, Terminal } from '@rushstack/node-core-library'; import { TerminalChunkKind, TextRewriterTransform, - StderrLineTransform, - SplitterTransform, - DiscardStdoutTransform, - PrintUtilities + SplitterTransform } from '@rushstack/terminal'; import { CollatedTerminal } from '@rushstack/stream-collator'; -import { Utilities, UNINITIALIZED } from '../../utilities/Utilities'; + +import { Utilities } from '../../utilities/Utilities'; import { OperationStatus } from './OperationStatus'; import { OperationError } from './OperationError'; import { IOperationRunner, IOperationRunnerContext } from './IOperationRunner'; import { ProjectLogWritable } from './ProjectLogWritable'; -import { ProjectBuildCache } from '../buildCache/ProjectBuildCache'; -import { IOperationSettings, RushProjectConfiguration } from '../../api/RushProjectConfiguration'; import { CollatedTerminalProvider } from '../../utilities/CollatedTerminalProvider'; -import { RushConstants } from '../RushConstants'; import { EnvironmentConfiguration } from '../../api/EnvironmentConfiguration'; -import { OperationStateFile } from './OperationStateFile'; import type { RushConfiguration } from '../../api/RushConfiguration'; import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; -import type { ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; -import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; import type { IPhase } from '../../api/CommandLineConfiguration'; -export interface IProjectDeps { - files: { [filePath: string]: string }; - arguments: string; -} - export interface IOperationRunnerOptions { rushProject: RushConfigurationProject; rushConfiguration: RushConfiguration; - buildCacheConfiguration: BuildCacheConfiguration | undefined; commandToRun: string; isIncrementalBuildAllowed: boolean; - projectChangeAnalyzer: ProjectChangeAnalyzer; displayName: string; phase: IPhase; - /** - * The set of phases being executed in the current command, for validation of rush-project.json - */ - selectedPhases: Iterable; -} - -function _areShallowEqual(object1: JsonObject, object2: JsonObject): boolean { - for (const n in object1) { - if (!(n in object2) || object1[n] !== object2[n]) { - return false; - } - } - for (const n in object2) { - if (!(n in object1)) { - return false; - } - } - return true; } /** @@ -85,47 +41,25 @@ export class ShellOperationRunner implements IOperationRunner { // This runner supports cache writes by default. public isCacheWriteAllowed: boolean = true; - public isSkipAllowed: boolean; public readonly reportTiming: boolean = true; public readonly silent: boolean = false; public readonly warningsAreAllowed: boolean; private readonly _rushProject: RushConfigurationProject; - private readonly _phase: IPhase; private readonly _rushConfiguration: RushConfiguration; - private readonly _buildCacheConfiguration: BuildCacheConfiguration | undefined; - private readonly _commandName: string; private readonly _commandToRun: string; - private readonly _isCacheReadAllowed: boolean; - private readonly _projectChangeAnalyzer: ProjectChangeAnalyzer; - private readonly _packageDepsFilename: string; private readonly _logFilenameIdentifier: string; - private readonly _selectedPhases: Iterable; - - /** - * UNINITIALIZED === we haven't tried to initialize yet - * undefined === we didn't create one because the feature is not enabled - */ - private _projectBuildCache: ProjectBuildCache | undefined | UNINITIALIZED = UNINITIALIZED; public constructor(options: IOperationRunnerOptions) { const { phase } = options; this.name = options.displayName; this._rushProject = options.rushProject; - this._phase = phase; this._rushConfiguration = options.rushConfiguration; - this._buildCacheConfiguration = options.buildCacheConfiguration; - this._commandName = phase.name; this._commandToRun = options.commandToRun; - this._isCacheReadAllowed = options.isIncrementalBuildAllowed; - this.isSkipAllowed = options.isIncrementalBuildAllowed; - this._projectChangeAnalyzer = options.projectChangeAnalyzer; - this._packageDepsFilename = `package-deps_${phase.logFilenameIdentifier}.json`; this.warningsAreAllowed = EnvironmentConfiguration.allowWarningsInSuccessfulBuild || phase.allowWarningsOnSuccess || false; this._logFilenameIdentifier = phase.logFilenameIdentifier; - this._selectedPhases = options.selectedPhases; } public async executeAsync(context: IOperationRunnerContext): Promise { @@ -146,43 +80,22 @@ export class ShellOperationRunner implements IOperationRunner { // +--> stdioSummarizer const projectLogWritable: ProjectLogWritable = new ProjectLogWritable( this._rushProject, - context.collatedWriter.terminal, + context.terminal, this._logFilenameIdentifier ); try { - const { trackedFileHashes, stateHash } = context; - const removeColorsTransform: TextRewriterTransform = new TextRewriterTransform({ destination: projectLogWritable, removeColors: true, normalizeNewlines: NewlineKind.OsDefault }); - const splitterTransform2: SplitterTransform = new SplitterTransform({ - destinations: [removeColorsTransform, context.stdioSummarizer] - }); - - const stderrLineTransform: StderrLineTransform = new StderrLineTransform({ - destination: splitterTransform2, - newlineKind: NewlineKind.Lf // for StdioSummarizer - }); - - const discardTransform: DiscardStdoutTransform = new DiscardStdoutTransform({ - destination: context.collatedWriter - }); - - const splitterTransform1: SplitterTransform = new SplitterTransform({ - destinations: [context.quietMode ? discardTransform : context.collatedWriter, stderrLineTransform] + const splitterTransform: SplitterTransform = new SplitterTransform({ + destinations: [context.terminalWritable, removeColorsTransform] }); - const normalizeNewlineTransform: TextRewriterTransform = new TextRewriterTransform({ - destination: splitterTransform1, - normalizeNewlines: NewlineKind.Lf, - ensureNewlineAtEnd: true - }); - - const collatedTerminal: CollatedTerminal = new CollatedTerminal(normalizeNewlineTransform); + const collatedTerminal: CollatedTerminal = new CollatedTerminal(splitterTransform); const terminalProvider: CollatedTerminalProvider = new CollatedTerminalProvider(collatedTerminal, { debugEnabled: context.debugMode }); @@ -190,113 +103,6 @@ export class ShellOperationRunner implements IOperationRunner { let hasWarningOrError: boolean = false; const projectFolder: string = this._rushProject.projectFolder; - let lastProjectDeps: IProjectDeps | undefined = undefined; - - const currentDepsPath: string = path.join( - this._rushProject.projectRushTempFolder, - this._packageDepsFilename - ); - - if (FileSystem.exists(currentDepsPath)) { - try { - lastProjectDeps = JsonFile.load(currentDepsPath); - } catch (e) { - // Warn and ignore - treat failing to load the file as the project being not built. - terminal.writeWarningLine( - `Warning: error parsing ${this._packageDepsFilename}: ${e}. Ignoring and ` + - `treating the command "${this._commandToRun}" as not run.` - ); - } - } - - let projectDeps: IProjectDeps | undefined; - const trackedFiles: Iterable | undefined = trackedFileHashes?.keys(); - if (trackedFileHashes) { - const files: { [filePath: string]: string } = {}; - for (const [filePath, fileHash] of trackedFileHashes) { - files[filePath] = fileHash; - } - - projectDeps = { - files, - arguments: this._commandToRun - }; - } else if (this.isSkipAllowed) { - // To test this code path: - // Remove the `.git` folder then run "rush build --verbose" - terminal.writeLine({ - text: PrintUtilities.wrapWords( - 'This workspace does not appear to be tracked by Git. ' + - 'Rush will proceed without incremental execution, caching, and change detection.' - ), - foregroundColor: ColorValue.Cyan - }); - } - - // If possible, we want to skip this operation -- either by restoring it from the - // cache, if caching is enabled, or determining that the project - // is unchanged (using the older incremental execution logic). These two approaches, - // "caching" and "skipping", are incompatible, so only one applies. - // - // Note that "caching" and "skipping" take two different approaches - // to tracking dependents: - // - // - For caching, "isCacheReadAllowed" is set if a project supports - // incremental builds, and determining whether this project or a dependent - // has changed happens inside the hashing logic. - // - // - For skipping, "isSkipAllowed" is set to true initially, and during - // the process of running dependents, it will be changed by OperationExecutionManager to - // false if a dependency wasn't able to be skipped. - // - let buildCacheReadAttempted: boolean = false; - let projectBuildCache: ProjectBuildCache | false | undefined; - if (this._isCacheReadAllowed) { - projectBuildCache = - (await this._tryGetProjectBuildCacheAsync(terminal, trackedFiles, stateHash)) || false; - - buildCacheReadAttempted = !!projectBuildCache; - const restoreFromCacheSuccess: boolean | undefined = projectBuildCache - ? await projectBuildCache.tryRestoreFromCacheAsync(terminal) - : undefined; - - if (restoreFromCacheSuccess) { - // Restore the original state of the operation without cache - await context._operationStateFile?.tryRestoreAsync(); - return OperationStatus.FromCache; - } - } - if (this.isSkipAllowed && !buildCacheReadAttempted) { - const isPackageUnchanged: boolean = !!( - lastProjectDeps && - projectDeps && - projectDeps.arguments === lastProjectDeps.arguments && - _areShallowEqual(projectDeps.files, lastProjectDeps.files) - ); - - if (isPackageUnchanged) { - return OperationStatus.Skipped; - } - } - - // If the deps file exists, remove it before starting execution. - FileSystem.deleteFile(currentDepsPath); - - // TODO: Remove legacyDepsPath with the next major release of Rush - const legacyDepsPath: string = path.join(this._rushProject.projectFolder, 'package-deps.json'); - // Delete the legacy package-deps.json - FileSystem.deleteFile(legacyDepsPath); - - if (!this._commandToRun) { - // Write deps on success. - if (projectDeps) { - JsonFile.save(projectDeps, currentDepsPath, { - ensureFolderExists: true - }); - } - - return OperationStatus.Success; - } // Run the operation terminal.writeLine('Invoking: ' + this._commandToRun); @@ -329,7 +135,7 @@ export class ShellOperationRunner implements IOperationRunner { }); } - let status: OperationStatus = await new Promise( + const status: OperationStatus = await new Promise( (resolve: (status: OperationStatus) => void, reject: (error: OperationError) => void) => { subProcess.on('close', (code: number) => { try { @@ -347,47 +153,7 @@ export class ShellOperationRunner implements IOperationRunner { } ); - const taskIsSuccessful: boolean = - status === OperationStatus.Success || - (status === OperationStatus.SuccessWithWarning && - this.warningsAreAllowed && - !!this._rushConfiguration.experimentsConfiguration.configuration - .buildCacheWithAllowWarningsInSuccessfulBuild); - - if (taskIsSuccessful && projectDeps) { - // Write deps on success. - const writeProjectStatePromise: Promise = JsonFile.saveAsync(projectDeps, currentDepsPath, { - ensureFolderExists: true - }); - - // If the operation without cache was successful, we can save the state to disk - const { duration: durationInSeconds } = context.stopwatch; - await context._operationStateFile?.writeAsync({ - nonCachedDurationMs: durationInSeconds * 1000 - }); - - // If the command is successful, we can calculate project hash, and no dependencies were skipped, - // write a new cache entry. - let setCacheEntryPromise: Promise | undefined; - if (context.isCacheWriteAllowed) { - if (projectBuildCache === undefined) { - projectBuildCache = await this._tryGetProjectBuildCacheAsync(terminal, trackedFiles, stateHash); - } - if (projectBuildCache) { - setCacheEntryPromise = projectBuildCache.trySetCacheEntryAsync(terminal); - } - } - - const [, cacheWriteSuccess] = await Promise.all([writeProjectStatePromise, setCacheEntryPromise]); - - if (terminalProvider.hasErrors) { - status = OperationStatus.Failure; - } else if (cacheWriteSuccess === false) { - status = OperationStatus.SuccessWithWarning; - } - } - - normalizeNewlineTransform.close(); + removeColorsTransform.close(); // If the pipeline is wired up correctly, then closing normalizeNewlineTransform should // have closed projectLogWritable. @@ -400,66 +166,6 @@ export class ShellOperationRunner implements IOperationRunner { projectLogWritable.close(); } } - - private async _tryGetProjectBuildCacheAsync( - terminal: ITerminal, - trackedProjectFiles: Iterable | undefined, - stateHash: string | undefined - ): Promise { - if (!stateHash) { - // To test this code path: - // Delete a project's ".rush/temp/shrinkwrap-deps.json" then run "rush build --verbose" - terminal.writeLine('Unable to calculate incremental state.'); - terminal.writeLine({ - text: 'Rush will proceed without incremental execution, caching, and change detection.', - foregroundColor: ColorValue.Cyan - }); - return; - } - - if (this._buildCacheConfiguration && this._buildCacheConfiguration.buildCacheEnabled) { - // Disable legacy skip logic if the build cache is in play - this.isSkipAllowed = false; - - const projectConfiguration: RushProjectConfiguration | undefined = - await RushProjectConfiguration.tryLoadForProjectAsync(this._rushProject, terminal); - if (projectConfiguration) { - projectConfiguration.validatePhaseConfiguration(this._selectedPhases, terminal); - if (projectConfiguration.disableBuildCacheForProject) { - terminal.writeVerboseLine('Caching has been disabled for this project.'); - } else { - const operationSettings: IOperationSettings | undefined = - projectConfiguration.operationSettingsByOperationName.get(this._commandName); - if (!operationSettings) { - terminal.writeVerboseLine( - `This project does not define the caching behavior of the "${this._commandName}" command, so caching has been disabled.` - ); - } else if (operationSettings.disableBuildCacheForOperation) { - terminal.writeVerboseLine( - `Caching has been disabled for this project's "${this._commandName}" command.` - ); - } else { - const projectOutputFolderNames: ReadonlyArray = operationSettings.outputFolderNames || []; - return await ProjectBuildCache.tryGetProjectBuildCache({ - projectConfiguration, - projectOutputFolderNames, - buildCacheConfiguration: this._buildCacheConfiguration, - terminal, - command: this._commandToRun, - trackedProjectFiles, - hash: stateHash, - phaseName: this._phase.name - }); - } - } - } else { - terminal.writeVerboseLine( - `Project does not have a ${RushConstants.rushProjectConfigFilename} configuration file, ` + - 'or one provided by a rig, so it does not support caching.' - ); - } - } - } } /** diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts index 31129619a28..9a4da069320 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts @@ -29,13 +29,7 @@ function createShellOperations( operations: Set, context: ICreateOperationsContext ): Set { - const { - buildCacheConfiguration, - isIncrementalBuildAllowed, - phaseSelection: selectedPhases, - projectChangeAnalyzer, - rushConfiguration - } = context; + const { isIncrementalBuildAllowed, rushConfiguration } = context; const customParametersByPhase: Map = new Map(); @@ -56,7 +50,7 @@ function createShellOperations( for (const operation of operations) { const { associatedPhase: phase, associatedProject: project } = operation; - if (phase && project && !operation.runner) { + if (!operation.runner) { // This is a shell command. In the future, may consider having a property on the initial operation // to specify a runner type requested in rush-project.json const customParameterValues: ReadonlyArray = getCustomParameterValuesForPhase(phase); @@ -73,15 +67,12 @@ function createShellOperations( if (commandToRun) { operation.runner = new ShellOperationRunner({ - buildCacheConfiguration, commandToRun: commandToRun || '', displayName, isIncrementalBuildAllowed, phase, - projectChangeAnalyzer, rushConfiguration, - rushProject: project, - selectedPhases + rushProject: project }); } else { // Empty build script indicates a no-op, so use a no-op runner @@ -90,6 +81,9 @@ function createShellOperations( result: OperationStatus.NoOp, silent: false }); + + // Clear the cache processor + operation.processor = undefined; } } } diff --git a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts index 1270b19c262..289ba95546f 100644 --- a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts @@ -5,6 +5,7 @@ import { Operation } from '../Operation'; import { IOperationExecutionRecordContext, OperationExecutionRecord } from '../OperationExecutionRecord'; import { MockOperationRunner } from './MockOperationRunner'; import { AsyncOperationQueue, IOperationSortFunction } from '../AsyncOperationQueue'; +import { IPhase } from '../../../api/CommandLineConfiguration'; function addDependency(consumer: OperationExecutionRecord, dependency: OperationExecutionRecord): void { consumer.dependencies.add(dependency); @@ -15,9 +16,24 @@ function nullSort(a: OperationExecutionRecord, b: OperationExecutionRecord): num return 0; } +const defaultPhase: IPhase = { + name: '_phase:foo', + isSynthetic: false, + ignoreMissingScript: true, + logFilenameIdentifier: 'foo', + allowWarningsOnSuccess: false, + associatedParameters: new Set(), + dependencies: { + self: new Set(), + upstream: new Set() + } +}; + function createRecord(name: string): OperationExecutionRecord { return new OperationExecutionRecord( new Operation({ + phase: defaultPhase, + project: undefined!, runner: new MockOperationRunner(name) }), {} as unknown as IOperationExecutionRecordContext diff --git a/libraries/rush-lib/src/logic/operations/test/MockOperationRunner.ts b/libraries/rush-lib/src/logic/operations/test/MockOperationRunner.ts index 58a71c26a21..858b09a8981 100644 --- a/libraries/rush-lib/src/logic/operations/test/MockOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/test/MockOperationRunner.ts @@ -1,13 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. - -import type { CollatedTerminal } from '@rushstack/stream-collator'; +import { ITerminal } from '@rushstack/node-core-library'; import { OperationStatus } from '../OperationStatus'; import { IOperationRunner, IOperationRunnerContext } from '../IOperationRunner'; export class MockOperationRunner implements IOperationRunner { - private readonly _action: ((terminal: CollatedTerminal) => Promise) | undefined; + private readonly _action: ((terminal: ITerminal) => Promise) | undefined; public readonly name: string; public readonly reportTiming: boolean = true; public readonly silent: boolean = false; @@ -17,7 +16,7 @@ export class MockOperationRunner implements IOperationRunner { public constructor( name: string, - action?: (terminal: CollatedTerminal) => Promise, + action?: (terminal: ITerminal) => Promise, warningsAreAllowed: boolean = false ) { this.name = name; @@ -28,7 +27,7 @@ export class MockOperationRunner implements IOperationRunner { public async executeAsync(context: IOperationRunnerContext): Promise { let result: OperationStatus | undefined; if (this._action) { - result = await this._action(context.collatedWriter.terminal); + result = await this._action(context.terminal); } return result || OperationStatus.Success; } diff --git a/libraries/rush-lib/src/logic/operations/test/OperationExecutionManager.test.ts b/libraries/rush-lib/src/logic/operations/test/OperationExecutionManager.test.ts index faf75490f2e..d7b5c310838 100644 --- a/libraries/rush-lib/src/logic/operations/test/OperationExecutionManager.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/OperationExecutionManager.test.ts @@ -8,7 +8,7 @@ import { EOL } from 'os'; import colors from 'colors/safe'; -import { Terminal } from '@rushstack/node-core-library'; +import { ITerminal, Terminal } from '@rushstack/node-core-library'; import { CollatedTerminal } from '@rushstack/stream-collator'; import { MockWritable } from '@rushstack/terminal'; @@ -22,6 +22,7 @@ import type { IOperationRunner } from '../IOperationRunner'; import { MockOperationRunner } from './MockOperationRunner'; import type { IExecutionResult, IOperationExecutionResult } from '../IOperationExecutionResult'; import { CollatedTerminalProvider } from '../../../utilities/CollatedTerminalProvider'; +import { IPhase } from '../../../api/CommandLineConfiguration'; const mockGetTimeInMs: jest.Mock = jest.fn(); Utilities.getTimeInMs = mockGetTimeInMs; @@ -36,11 +37,26 @@ mockGetTimeInMs.mockImplementation(() => { const mockWritable: MockWritable = new MockWritable(); const mockTerminal: Terminal = new Terminal(new CollatedTerminalProvider(new CollatedTerminal(mockWritable))); +const defaultPhase: IPhase = { + name: '_phase:foo', + isSynthetic: false, + ignoreMissingScript: true, + logFilenameIdentifier: 'foo', + allowWarningsOnSuccess: false, + associatedParameters: new Set(), + dependencies: { + self: new Set(), + upstream: new Set() + } +}; + function createExecutionManager( executionManagerOptions: IOperationExecutionManagerOptions, operationRunner: IOperationRunner ): OperationExecutionManager { const operation: Operation = new Operation({ + phase: defaultPhase, + project: undefined!, runner: operationRunner }); @@ -82,15 +98,15 @@ describe(OperationExecutionManager.name, () => { it('printedStderrAfterError', async () => { executionManager = createExecutionManager( executionManagerOptions, - new MockOperationRunner('stdout+stderr', async (terminal: CollatedTerminal) => { - terminal.writeStdoutLine('Build step 1' + EOL); - terminal.writeStderrLine('Error: step 1 failed' + EOL); + new MockOperationRunner('stdout+stderr', async (terminal: ITerminal) => { + terminal.writeLine('Build step 1' + EOL); + terminal.writeErrorLine('Error: step 1 failed' + EOL); return OperationStatus.Failure; }) ); const result: IExecutionResult = await executionManager.executeAsync(); - _printOperationStatus(mockTerminal, result); + await _printOperationStatus(mockTerminal, result); expect(result.status).toEqual(OperationStatus.Failure); expect(result.operationResults.size).toEqual(1); const firstResult: IOperationExecutionResult = result.operationResults.values().next().value; @@ -104,15 +120,15 @@ describe(OperationExecutionManager.name, () => { it('printedStdoutAfterErrorWithEmptyStderr', async () => { executionManager = createExecutionManager( executionManagerOptions, - new MockOperationRunner('stdout only', async (terminal: CollatedTerminal) => { - terminal.writeStdoutLine('Build step 1' + EOL); - terminal.writeStdoutLine('Error: step 1 failed' + EOL); + new MockOperationRunner('stdout only', async (terminal: ITerminal) => { + terminal.writeLine('Build step 1' + EOL); + terminal.writeLine('Error: step 1 failed' + EOL); return OperationStatus.Failure; }) ); const result: IExecutionResult = await executionManager.executeAsync(); - _printOperationStatus(mockTerminal, result); + await _printOperationStatus(mockTerminal, result); expect(result.status).toEqual(OperationStatus.Failure); expect(result.operationResults.size).toEqual(1); const firstResult: IOperationExecutionResult = result.operationResults.values().next().value; @@ -140,15 +156,15 @@ describe(OperationExecutionManager.name, () => { it('Logs warnings correctly', async () => { executionManager = createExecutionManager( executionManagerOptions, - new MockOperationRunner('success with warnings (failure)', async (terminal: CollatedTerminal) => { - terminal.writeStdoutLine('Build step 1' + EOL); - terminal.writeStdoutLine('Warning: step 1 succeeded with warnings' + EOL); + new MockOperationRunner('success with warnings (failure)', async (terminal: ITerminal) => { + terminal.writeLine('Build step 1' + EOL); + terminal.writeWarningLine('Warning: step 1 succeeded with warnings' + EOL); return OperationStatus.SuccessWithWarning; }) ); const result: IExecutionResult = await executionManager.executeAsync(); - _printOperationStatus(mockTerminal, result); + await _printOperationStatus(mockTerminal, result); expect(result.status).toEqual(OperationStatus.SuccessWithWarning); expect(result.operationResults.size).toEqual(1); const firstResult: IOperationExecutionResult = result.operationResults.values().next().value; @@ -177,9 +193,9 @@ describe(OperationExecutionManager.name, () => { executionManagerOptions, new MockOperationRunner( 'success with warnings (success)', - async (terminal: CollatedTerminal) => { - terminal.writeStdoutLine('Build step 1' + EOL); - terminal.writeStdoutLine('Warning: step 1 succeeded with warnings' + EOL); + async (terminal: ITerminal) => { + terminal.writeLine('Build step 1' + EOL); + terminal.writeWarningLine('Warning: step 1 succeeded with warnings' + EOL); return OperationStatus.SuccessWithWarning; }, /* warningsAreAllowed */ true @@ -187,7 +203,7 @@ describe(OperationExecutionManager.name, () => { ); const result: IExecutionResult = await executionManager.executeAsync(); - _printOperationStatus(mockTerminal, result); + await _printOperationStatus(mockTerminal, result); expect(result.status).toEqual(OperationStatus.Success); expect(result.operationResults.size).toEqual(1); const firstResult: IOperationExecutionResult = result.operationResults.values().next().value; @@ -203,9 +219,9 @@ describe(OperationExecutionManager.name, () => { executionManagerOptions, new MockOperationRunner( 'success with warnings (success)', - async (terminal: CollatedTerminal) => { - terminal.writeStdoutLine('Build step 1' + EOL); - terminal.writeStdoutLine('Warning: step 1 succeeded with warnings' + EOL); + async (terminal: ITerminal) => { + terminal.writeLine('Build step 1' + EOL); + terminal.writeWarningLine('Warning: step 1 succeeded with warnings' + EOL); return OperationStatus.SuccessWithWarning; }, /* warningsAreAllowed */ true @@ -214,7 +230,7 @@ describe(OperationExecutionManager.name, () => { const result: IExecutionResult = await executionManager.executeAsync(); _printTimeline(mockTerminal, result); - _printOperationStatus(mockTerminal, result); + await _printOperationStatus(mockTerminal, result); const allMessages: string = mockWritable.getAllOutput(); expect(allMessages).toContain('Build step 1'); expect(allMessages).toContain('Warning: step 1 succeeded with warnings'); diff --git a/libraries/rush-lib/src/logic/operations/test/PhasedOperationPlugin.test.ts b/libraries/rush-lib/src/logic/operations/test/PhasedOperationPlugin.test.ts index 62b2a308a18..e8e36c15794 100644 --- a/libraries/rush-lib/src/logic/operations/test/PhasedOperationPlugin.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/PhasedOperationPlugin.test.ts @@ -17,6 +17,7 @@ import { RushConstants } from '../../RushConstants'; import { MockOperationRunner } from './MockOperationRunner'; import { ICreateOperationsContext, PhasedCommandHooks } from '../../../pluginFramework/PhasedCommandHooks'; import { RushConfigurationProject } from '../../..'; +import { RushProjectConfiguration } from '../../../api/RushProjectConfiguration'; interface ISerializedOperation { name: string; @@ -43,7 +44,7 @@ describe(PhasedOperationPlugin.name, () => { for (const operation of operations) { const { associatedPhase, associatedProject } = operation; - if (associatedPhase && associatedProject && !operation.runner) { + if (!operation.runner) { const name: string = `${associatedProject.packageName} (${associatedPhase.name.slice( RushConstants.phaseNamePrefix.length )})`; @@ -55,10 +56,18 @@ describe(PhasedOperationPlugin.name, () => { return operations; } + function* mapProjects( + projects: Iterable + ): Iterable<[RushConfigurationProject, RushProjectConfiguration | undefined]> { + for (const project of projects) { + yield [project, undefined]; + } + } + async function testCreateOperationsAsync( - phaseSelection: Set, - projectSelection: Set, - changedProjects: Set + phaseSelection: Iterable, + projectSelection: Iterable, + changedProjects: Iterable ): Promise> { const hooks: PhasedCommandHooks = new PhasedCommandHooks(); // Apply the plugin being tested @@ -70,9 +79,9 @@ describe(PhasedOperationPlugin.name, () => { ICreateOperationsContext, 'phaseSelection' | 'projectSelection' | 'projectsInUnknownState' > = { - phaseSelection, - projectSelection, - projectsInUnknownState: changedProjects + phaseSelection: new Set(phaseSelection), + projectSelection: new Map(mapProjects(projectSelection)), + projectsInUnknownState: new Set(changedProjects) }; const operations: Set = await hooks.createOperations.promise( new Set(), @@ -99,8 +108,8 @@ describe(PhasedOperationPlugin.name, () => { const operations: Set = await testCreateOperationsAsync( buildCommand.phases, - new Set(rushConfiguration.projects), - new Set(rushConfiguration.projects) + rushConfiguration.projects, + rushConfiguration.projects ); // All projects @@ -114,8 +123,8 @@ describe(PhasedOperationPlugin.name, () => { let operations: Set = await testCreateOperationsAsync( buildCommand.phases, - new Set([rushConfiguration.getProjectByName('g')!]), - new Set([rushConfiguration.getProjectByName('g')!]) + [rushConfiguration.getProjectByName('g')!], + [rushConfiguration.getProjectByName('g')!] ); // Single project @@ -123,16 +132,16 @@ describe(PhasedOperationPlugin.name, () => { operations = await testCreateOperationsAsync( buildCommand.phases, - new Set([ + [ rushConfiguration.getProjectByName('f')!, rushConfiguration.getProjectByName('a')!, rushConfiguration.getProjectByName('c')! - ]), - new Set([ + ], + [ rushConfiguration.getProjectByName('f')!, rushConfiguration.getProjectByName('a')!, rushConfiguration.getProjectByName('c')! - ]) + ] ); // Filtered projects @@ -146,22 +155,18 @@ describe(PhasedOperationPlugin.name, () => { let operations: Set = await testCreateOperationsAsync( buildCommand.phases, - new Set(rushConfiguration.projects), - new Set([rushConfiguration.getProjectByName('g')!]) + rushConfiguration.projects, + [rushConfiguration.getProjectByName('g')!] ); // Single project expect(Array.from(operations, serializeOperation)).toMatchSnapshot(); - operations = await testCreateOperationsAsync( - buildCommand.phases, - new Set(rushConfiguration.projects), - new Set([ - rushConfiguration.getProjectByName('f')!, - rushConfiguration.getProjectByName('a')!, - rushConfiguration.getProjectByName('c')! - ]) - ); + operations = await testCreateOperationsAsync(buildCommand.phases, rushConfiguration.projects, [ + rushConfiguration.getProjectByName('f')!, + rushConfiguration.getProjectByName('a')!, + rushConfiguration.getProjectByName('c')! + ]); // Filtered projects expect(Array.from(operations, serializeOperation)).toMatchSnapshot(); @@ -174,12 +179,12 @@ describe(PhasedOperationPlugin.name, () => { const operations: Set = await testCreateOperationsAsync( buildCommand.phases, - new Set([ + [ rushConfiguration.getProjectByName('f')!, rushConfiguration.getProjectByName('a')!, rushConfiguration.getProjectByName('c')! - ]), - new Set([rushConfiguration.getProjectByName('a')!, rushConfiguration.getProjectByName('c')!]) + ], + [rushConfiguration.getProjectByName('a')!, rushConfiguration.getProjectByName('c')!] ); // Single project @@ -189,22 +194,22 @@ describe(PhasedOperationPlugin.name, () => { it('handles filtered phases', async () => { // Single phase with a missing dependency let operations: Set = await testCreateOperationsAsync( - new Set([commandLineConfiguration.phases.get('_phase:upstream-self')!]), - new Set(rushConfiguration.projects), - new Set(rushConfiguration.projects) + [commandLineConfiguration.phases.get('_phase:upstream-self')!], + rushConfiguration.projects, + rushConfiguration.projects ); expect(Array.from(operations, serializeOperation)).toMatchSnapshot(); // Two phases with a missing link operations = await testCreateOperationsAsync( - new Set([ + [ commandLineConfiguration.phases.get('_phase:complex')!, commandLineConfiguration.phases.get('_phase:upstream-3')!, commandLineConfiguration.phases.get('_phase:upstream-1')!, commandLineConfiguration.phases.get('_phase:no-deps')! - ]), - new Set(rushConfiguration.projects), - new Set(rushConfiguration.projects) + ], + rushConfiguration.projects, + rushConfiguration.projects ); expect(Array.from(operations, serializeOperation)).toMatchSnapshot(); }); @@ -212,38 +217,38 @@ describe(PhasedOperationPlugin.name, () => { it('handles filtered phases on filtered projects', async () => { // Single phase with a missing dependency let operations: Set = await testCreateOperationsAsync( - new Set([commandLineConfiguration.phases.get('_phase:upstream-2')!]), - new Set([ + [commandLineConfiguration.phases.get('_phase:upstream-2')!], + [ rushConfiguration.getProjectByName('f')!, rushConfiguration.getProjectByName('a')!, rushConfiguration.getProjectByName('c')! - ]), - new Set([ + ], + [ rushConfiguration.getProjectByName('f')!, rushConfiguration.getProjectByName('a')!, rushConfiguration.getProjectByName('c')! - ]) + ] ); expect(Array.from(operations, serializeOperation)).toMatchSnapshot(); // Phases with missing links operations = await testCreateOperationsAsync( - new Set([ + [ commandLineConfiguration.phases.get('_phase:complex')!, commandLineConfiguration.phases.get('_phase:upstream-3')!, commandLineConfiguration.phases.get('_phase:upstream-1')!, commandLineConfiguration.phases.get('_phase:no-deps')! - ]), - new Set([ + ], + [ rushConfiguration.getProjectByName('f')!, rushConfiguration.getProjectByName('a')!, rushConfiguration.getProjectByName('c')! - ]), - new Set([ + ], + [ rushConfiguration.getProjectByName('f')!, rushConfiguration.getProjectByName('a')!, rushConfiguration.getProjectByName('c')! - ]) + ] ); expect(Array.from(operations, serializeOperation)).toMatchSnapshot(); }); diff --git a/libraries/rush-lib/src/logic/operations/test/__snapshots__/OperationExecutionManager.test.ts.snap b/libraries/rush-lib/src/logic/operations/test/__snapshots__/OperationExecutionManager.test.ts.snap index 98a7e425be2..7e3a1d1c0c2 100644 --- a/libraries/rush-lib/src/logic/operations/test/__snapshots__/OperationExecutionManager.test.ts.snap +++ b/libraries/rush-lib/src/logic/operations/test/__snapshots__/OperationExecutionManager.test.ts.snap @@ -45,8 +45,8 @@ Array [ }, Object { "kind": "E", - "text": "Error: step 1 failed - + "text": "[red]Error: step 1 failed +[default] ", }, Object { @@ -69,7 +69,7 @@ Array [ }, Object { "kind": "O", - "text": "[gray]--[[default] [red]FAILURE: stdout+stderr[default] [gray]]---------------------------------[[default] [white]0.10 seconds[default] [gray]]--[default] + "text": "[gray]--[[default] [red]FAILURE: stdout+stderr[default] [gray]]---------------------------------[[default] [white]0.20 seconds[default] [gray]]--[default] ", }, @@ -157,7 +157,7 @@ Array [ }, Object { "kind": "O", - "text": "[gray]--[[default] [red]FAILURE: stdout only[default] [gray]]-----------------------------------[[default] [white]0.10 seconds[default] [gray]]--[default] + "text": "[gray]--[[default] [red]FAILURE: stdout only[default] [gray]]-----------------------------------[[default] [white]0.20 seconds[default] [gray]]--[default] ", }, @@ -220,9 +220,9 @@ Array [ ", }, Object { - "kind": "O", - "text": "Warning: step 1 succeeded with warnings - + "kind": "E", + "text": "[yellow]Warning: step 1 succeeded with warnings +[default] ", }, Object { @@ -308,9 +308,9 @@ Array [ ", }, Object { - "kind": "O", - "text": "Warning: step 1 succeeded with warnings - + "kind": "E", + "text": "[yellow]Warning: step 1 succeeded with warnings +[default] ", }, Object { @@ -390,9 +390,9 @@ Array [ ", }, Object { - "kind": "O", - "text": "Warning: step 1 succeeded with warnings - + "kind": "E", + "text": "[yellow]Warning: step 1 succeeded with warnings +[default] ", }, Object { @@ -438,6 +438,16 @@ Array [ Object { "kind": "O", "text": " Avg Parallelism Used: 1.0 +", + }, + Object { + "kind": "O", + "text": "BY PHASE: +", + }, + Object { + "kind": "O", + "text": " [cyan] _phase:foo[default] 0.2s ", }, Object { diff --git a/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts b/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts index 1c2e893b67d..55554512b74 100644 --- a/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts +++ b/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts @@ -12,6 +12,7 @@ import type { RushConfigurationProject } from '../api/RushConfigurationProject'; import type { Operation } from '../logic/operations/Operation'; import type { ProjectChangeAnalyzer } from '../logic/ProjectChangeAnalyzer'; import { IExecutionResult } from '../logic/operations/IOperationExecutionResult'; +import { RushProjectConfiguration } from '../api/RushProjectConfiguration'; /** * A plugin that interacts with a phased commands. @@ -61,9 +62,10 @@ export interface ICreateOperationsContext { */ readonly projectChangeAnalyzer: ProjectChangeAnalyzer; /** - * The set of Rush projects selected for the current command execution. + * The set of Rush projects selected for the current command execution, mapped to their resolved + * rush-project.json configurations (or undefined if no configuration provided). */ - readonly projectSelection: ReadonlySet; + readonly projectSelection: ReadonlyMap; /** * The set of Rush projects that have not been built in the current process since they were last modified. * When `isInitial` is true, this will be an exact match of `projectSelection`. diff --git a/rush-plugins/rush-serve-plugin/src/phasedCommandHandler.ts b/rush-plugins/rush-serve-plugin/src/phasedCommandHandler.ts index 0bd721ee504..ac0cada841c 100644 --- a/rush-plugins/rush-serve-plugin/src/phasedCommandHandler.ts +++ b/rush-plugins/rush-serve-plugin/src/phasedCommandHandler.ts @@ -80,12 +80,10 @@ export async function phasedCommandHandler(options: IPhasedCommandHandlerOptions const app: express.Express = express(); - const selectedProjects: ReadonlySet = context.projectSelection; - const serveConfig: RushServeConfiguration = new RushServeConfiguration(); const routingRules: Iterable = await serveConfig.loadProjectConfigsAsync( - selectedProjects, + context.projectSelection.keys(), logger.terminal ); From 08cda71528b391e617c6ff63160a793b42d16826 Mon Sep 17 00:00:00 2001 From: David Michon Date: Tue, 11 Oct 2022 16:44:51 -0700 Subject: [PATCH 4/4] [rush-lib] Refactor hashing --- common/reviews/api/rush-lib.api.md | 18 ++-- .../BuildCacheOperationProcessor.ts | 5 +- .../src/logic/operations/IOperationRunner.ts | 10 +- .../src/logic/operations/Operation.ts | 71 ++++--------- .../operations/OperationExecutionManager.ts | 34 +++--- .../operations/OperationExecutionRecord.ts | 29 +++-- .../src/logic/operations/OperationHash.ts | 100 ++++++++++++++++++ .../logic/operations/OperationStateFile.ts | 33 ++---- 8 files changed, 183 insertions(+), 117 deletions(-) create mode 100644 libraries/rush-lib/src/logic/operations/OperationHash.ts diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 2fd8d5c8494..343d9a68cd2 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -402,28 +402,30 @@ export interface IOperationRunner { // @beta export interface IOperationRunnerContext { debugMode: boolean; + // Warning: (ae-forgotten-export) The symbol "IOperationHashes" needs to be exported by the entry point index.d.ts + hashes: IOperationHashes | undefined; isCacheReadAllowed: boolean; isCacheWriteAllowed: boolean; isSkipAllowed: boolean; quietMode: boolean; - stateHash: string | undefined; terminal: ITerminal; terminalWritable: TerminalWritable; - trackedFileHashes: ReadonlyMap | undefined; } // @internal (undocumented) export interface _IOperationStateFileOptions { // (undocumented) - phase: IPhase; - // (undocumented) - rushProject: RushConfigurationProject; + filename: string; } // @internal (undocumented) export interface _IOperationStateJson { + // (undocumented) + hashes: IOperationHashes | undefined; // (undocumented) nonCachedDurationMs: number; + // (undocumented) + status: OperationStatus; } // @public @@ -603,15 +605,12 @@ export class Operation { readonly consumers: ReadonlySet; deleteDependency(dependency: Operation): void; readonly dependencies: ReadonlySet; - getHash(localHash: string, dependencyHashes: string[]): string; // (undocumented) logFilePath: string | undefined; + readonly metadataFolderRelativePath: string; get name(): string | undefined; - // (undocumented) readonly outputFolderNames: ReadonlyArray; - // (undocumented) processor: IOperationProcessor | undefined; - // (undocumented) readonly projectFileFilter: IProjectFileFilter | undefined; runner: IOperationRunner | undefined; weight: number; @@ -621,7 +620,6 @@ export class Operation { export class _OperationStateFile { constructor(options: _IOperationStateFileOptions); get filename(): string; - static getFilenameRelativeToProjectRoot(phase: IPhase): string; // (undocumented) get state(): _IOperationStateJson | undefined; // (undocumented) diff --git a/libraries/rush-lib/src/logic/buildCache/BuildCacheOperationProcessor.ts b/libraries/rush-lib/src/logic/buildCache/BuildCacheOperationProcessor.ts index c217982ce47..10b36ebff40 100644 --- a/libraries/rush-lib/src/logic/buildCache/BuildCacheOperationProcessor.ts +++ b/libraries/rush-lib/src/logic/buildCache/BuildCacheOperationProcessor.ts @@ -57,9 +57,10 @@ export class BuildCacheOperationProcessor implements IOperationProcessor { } public async beforeBuildAsync( - context: Pick + context: Pick ): Promise { - const { stateHash, terminal, isCacheReadAllowed } = context; + const { hashes, terminal, isCacheReadAllowed } = context; + const stateHash: string | undefined = hashes?.fullHash; if (!stateHash || !isCacheReadAllowed) { return OperationStatus.Ready; } diff --git a/libraries/rush-lib/src/logic/operations/IOperationRunner.ts b/libraries/rush-lib/src/logic/operations/IOperationRunner.ts index 830d930c2d9..52993c52745 100644 --- a/libraries/rush-lib/src/logic/operations/IOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/IOperationRunner.ts @@ -3,6 +3,7 @@ import { ITerminal } from '@rushstack/node-core-library'; import type { TerminalWritable } from '@rushstack/terminal'; +import { IOperationHashes } from './OperationHash'; import type { OperationStatus } from './OperationStatus'; @@ -48,14 +49,9 @@ export interface IOperationRunnerContext { terminalWritable: TerminalWritable; /** - * The hashes of all tracked files pertinent to the operation + * The hashes of the operation state. */ - trackedFileHashes: ReadonlyMap | undefined; - - /** - * The hash of all inputs to the operation - */ - stateHash: string | undefined; + hashes: IOperationHashes | undefined; } /** diff --git a/libraries/rush-lib/src/logic/operations/Operation.ts b/libraries/rush-lib/src/logic/operations/Operation.ts index fdb3b083412..fd5c01b9219 100644 --- a/libraries/rush-lib/src/logic/operations/Operation.ts +++ b/libraries/rush-lib/src/logic/operations/Operation.ts @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import * as crypto from 'crypto'; - import { RushConfigurationProject } from '../../api/RushConfigurationProject'; import { IPhase } from '../../api/CommandLineConfiguration'; import { IOperationProcessor } from './IOperationProcessor'; @@ -91,26 +89,40 @@ export class Operation { */ public weight: number = 1; + /** + * Names of folders (may use '/' to delineate subfolders) into which outputs are written. + * Folders are specified relative to `associatedProject.projectFolder`. + * Implicitly includes `metadataFolderRelativePath`. + */ public readonly outputFolderNames: ReadonlyArray; + /** + * Filter that will be applied to input file list when computing the local input hash for this project. + */ public readonly projectFileFilter: IProjectFileFilter | undefined; + /** + * Pre/post processor for this operation, to handle cache interactions. + */ public processor: IOperationProcessor | undefined; + /** + * Folder into which operation metadata should be written. + */ + public readonly metadataFolderRelativePath: string; + public logFilePath: string | undefined = undefined; public constructor(options: IOperationOptions) { - const { - phase, - outputFolderNames = [] - } = options; + const { phase, outputFolderNames = [] } = options; this.associatedPhase = phase; this.associatedProject = options.project; const uniqueOutputFolderNames: Set = new Set(outputFolderNames); - uniqueOutputFolderNames.add( - `${RushConstants.projectRushFolderName}/${RushConstants.rushTempFolderName}/operation/${phase.logFilenameIdentifier}`); + const metadataRelativePath: string = `${RushConstants.projectRushFolderName}/${RushConstants.rushTempFolderName}/operation/${phase.logFilenameIdentifier}`; + this.metadataFolderRelativePath = metadataRelativePath; + uniqueOutputFolderNames.add(metadataRelativePath); const sortedOutputFolderNames: string[] = Array.from(uniqueOutputFolderNames).sort(); this.outputFolderNames = sortedOutputFolderNames; @@ -127,49 +139,6 @@ export class Operation { return this.runner?.name; } - /** - * Computes this operation's input state hash, for use by the caching layer. - * @param localHash - The hash of the local file inputs for this operation - * @param dependencyHashes - The state hashes of this operation's dependencies - */ - public getHash(localHash: string, dependencyHashes: string[]): string { - if (!localHash) { - return ''; - } - - for (const dependencyHash of dependencyHashes) { - if (!dependencyHash) { - return ''; - } - } - - const hash: crypto.Hash = crypto.createHash('sha1'); - hash.update(`${RushConstants.buildCacheVersion}`); - hash.update(RushConstants.hashDelimiter); - - // Output folder names are part of the configuration, so include in hash - for (const outputFolder of this.outputFolderNames) { - hash.update(outputFolder); - hash.update(RushConstants.hashDelimiter); - } - - // CLI parameters that apply to the phase affect the result - const params: string[] = []; - for (const tsCommandLineParameter of this.associatedPhase.associatedParameters) { - tsCommandLineParameter.appendToArgList(params); - } - hash.update(params.join(' ')); - - hash.update(localHash); - - const sortedHashes: string[] = dependencyHashes.sort(); - for (const dependencyHash of sortedHashes) { - hash.update(dependencyHash); - hash.update(RushConstants.hashDelimiter); - } - return hash.digest('hex'); - } - /** * Adds the specified operation as a dependency and updates the consumer list. */ diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts index fbc94e0287f..56aa7c085c5 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts @@ -4,7 +4,14 @@ import colors from 'colors/safe'; import { TerminalWritable, StdioWritable, TextRewriterTransform } from '@rushstack/terminal'; import { StreamCollator, CollatedWriter } from '@rushstack/stream-collator'; -import { NewlineKind, Async, Terminal, ITerminal, AlreadyReportedError, Colors } from '@rushstack/node-core-library'; +import { + NewlineKind, + Async, + Terminal, + ITerminal, + AlreadyReportedError, + Colors +} from '@rushstack/node-core-library'; import { AsyncOperationQueue, IOperationSortFunction } from './AsyncOperationQueue'; import { Operation } from './Operation'; @@ -14,6 +21,7 @@ import { IExecutionResult } from './IOperationExecutionResult'; import { ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; import { CollatedTerminalProvider } from '../../utilities/CollatedTerminalProvider'; import { LookupByPath } from '../LookupByPath'; +import { getOperationHashes } from './OperationHash'; export interface IOperationExecutionManagerOptions { quietMode: boolean; @@ -221,11 +229,11 @@ export class OperationExecutionManager { let hasIssues: boolean = false; function getOperationHash(record: OperationExecutionRecord): string { - let { stateHash } = record; - if (stateHash === undefined) { + let { hashes } = record; + if (hashes === undefined) { const { operation } = record; const { associatedProject, projectFileFilter, outputFolderNames, processor } = operation; - stateHash = ''; + const trackedFiles: ReadonlyMap | undefined = state._tryGetProjectDependencies( associatedProject, terminal, @@ -236,6 +244,8 @@ export class OperationExecutionManager { const projectOutputLookup: LookupByPath = new LookupByPath( outputFolderNames.map((relativePath) => [relativePath, relativePath]) ); + + // Validate no input/output files for (const trackedFilePath of trackedFiles.keys()) { const match: string | undefined = projectOutputLookup.findChildPath(trackedFilePath); if (match) { @@ -248,14 +258,14 @@ export class OperationExecutionManager { } } - record.trackedFileHashes = trackedFiles; const localHash: string = trackedFiles ? state._hashProjectDependencies(trackedFiles) : ''; - if (localHash) { - stateHash = operation.getHash(localHash, Array.from(record.dependencies, getOperationHash)); - } - record.stateHash = stateHash; + record.hashes = hashes = getOperationHashes( + operation, + localHash, + Array.from(record.dependencies, getOperationHash) + ); } - return stateHash; + return hashes.fullHash; } for (const record of this._executionRecords.values()) { @@ -321,9 +331,7 @@ export class OperationExecutionManager { */ case OperationStatus.FromCache: { if (!silent) { - record.terminal.writeLine( - Colors.green(`"${name}" was restored from the build cache.`) - ); + record.terminal.writeLine(Colors.green(`"${name}" was restored from the build cache.`)); } break; } diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts index 74eb0d87b82..a2570b421d2 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts @@ -1,7 +1,14 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { DiscardStdoutTransform, SplitterTransform, StderrLineTransform, StdioSummarizer, TerminalWritable, TextRewriterTransform } from '@rushstack/terminal'; +import { + DiscardStdoutTransform, + SplitterTransform, + StderrLineTransform, + StdioSummarizer, + TerminalWritable, + TextRewriterTransform +} from '@rushstack/terminal'; import { InternalError, ITerminal, NewlineKind, Terminal } from '@rushstack/node-core-library'; import { CollatedTerminal, CollatedWriter, StreamCollator } from '@rushstack/stream-collator'; @@ -12,6 +19,7 @@ import { Stopwatch } from '../../utilities/Stopwatch'; import { OperationStateFile } from './OperationStateFile'; import { IOperationProcessor } from './IOperationProcessor'; import { CollatedTerminalProvider } from '../../utilities/CollatedTerminalProvider'; +import { IOperationHashes } from './OperationHash'; export interface IOperationExecutionRecordContext { streamCollator: StreamCollator; @@ -75,9 +83,7 @@ export class OperationExecutionRecord implements IOperationRunnerContext { public isCacheReadAllowed: boolean = true; public isCacheWriteAllowed: boolean = true; - public trackedFileHashes: ReadonlyMap | undefined = undefined; - - public stateHash: string | undefined = undefined; + public hashes: IOperationHashes | undefined = undefined; /** * The set of operations that must complete before this operation executes. @@ -117,12 +123,13 @@ export class OperationExecutionRecord implements IOperationRunnerContext { this.runner = runner; this.weight = operation.weight; - const { associatedPhase, associatedProject } = operation; + const { associatedProject } = operation; - this._operationStateFile = associatedPhase && associatedProject ? new OperationStateFile({ - phase: associatedPhase, - rushProject: associatedProject - }) : undefined; + this._operationStateFile = associatedProject + ? new OperationStateFile({ + filename: `${associatedProject.projectFolder}/${operation.metadataFolderRelativePath}/state.json` + }) + : undefined; this._context = context; } @@ -206,7 +213,9 @@ export class OperationExecutionRecord implements IOperationRunnerContext { this.stopwatch.stop(); await this._operationStateFile?.writeAsync({ - nonCachedDurationMs: this.stopwatch.duration * 1000 + nonCachedDurationMs: this.stopwatch.duration * 1000, + hashes: this.hashes, + status }); if (processor) { diff --git a/libraries/rush-lib/src/logic/operations/OperationHash.ts b/libraries/rush-lib/src/logic/operations/OperationHash.ts new file mode 100644 index 00000000000..b35b7147aa0 --- /dev/null +++ b/libraries/rush-lib/src/logic/operations/OperationHash.ts @@ -0,0 +1,100 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as crypto from 'crypto'; +import { RushConstants } from '../RushConstants'; +import { Operation } from './Operation'; + +/** + * @alpha + */ +export interface IOperationHashes { + /** + * Hash of tracked input files + */ + localHash: string; + + /** + * Sorted `fullHash` values of dependencies. + */ + dependencyHashes: string[]; + + /** + * Hash of operation configuration options + */ + configHash: string; + + /** + * The final state hash used for change detection and caching. + * Computed from all other properties. + */ + fullHash: string; +} + +/** + * Computes an operation's state hashes, for use by the caching layer. + * @param operation - The operation to compute hashes for + * @param localHash - The hash of the local file inputs for the operation + * @param dependencyHashes - The full state hashes of the operation's dependencies + * @alpha + */ +export function getOperationHashes( + operation: Operation, + localHash: string, + dependencyHashes: string[] +): IOperationHashes { + const configHash: string = getConfigHash(operation); + const fullHash: string = getFullOperationHash(configHash, localHash, dependencyHashes); + + return { + configHash, + localHash, + dependencyHashes, + fullHash + }; +} + +function getConfigHash(operation: Operation): string { + const configHasher: crypto.Hash = crypto.createHash('sha1'); + configHasher.update(`${RushConstants.buildCacheVersion}`); + configHasher.update(RushConstants.hashDelimiter); + + // Output folder names are part of the configuration, so include in hash + for (const outputFolder of operation.outputFolderNames) { + configHasher.update(outputFolder); + configHasher.update(RushConstants.hashDelimiter); + } + + // CLI parameters that apply to the phase affect the result + const params: string[] = []; + for (const tsCommandLineParameter of operation.associatedPhase.associatedParameters) { + tsCommandLineParameter.appendToArgList(params); + } + configHasher.update(params.join(' ')); + const configHash: string = configHasher.digest('base64'); + + return configHash; +} + +function getFullOperationHash(configHash: string, localHash: string, dependencyHashes: string[]): string { + if (!localHash) { + return ''; + } + + for (const dependencyHash of dependencyHashes) { + if (!dependencyHash) { + return ''; + } + } + + const hash: crypto.Hash = crypto.createHash('sha1'); + hash.update(configHash); + hash.update(localHash); + + const sortedHashes: string[] = dependencyHashes.sort(); + for (const dependencyHash of sortedHashes) { + hash.update(dependencyHash); + hash.update(RushConstants.hashDelimiter); + } + return hash.digest('hex'); +} diff --git a/libraries/rush-lib/src/logic/operations/OperationStateFile.ts b/libraries/rush-lib/src/logic/operations/OperationStateFile.ts index fc3de3d57bc..ed3611e1c26 100644 --- a/libraries/rush-lib/src/logic/operations/OperationStateFile.ts +++ b/libraries/rush-lib/src/logic/operations/OperationStateFile.ts @@ -2,17 +2,15 @@ // See LICENSE in the project root for license information. import { FileSystem, InternalError, JsonFile } from '@rushstack/node-core-library'; -import { RushConstants } from '../RushConstants'; +import { IOperationHashes } from './OperationHash'; -import type { IPhase } from '../../api/CommandLineConfiguration'; -import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; +import { OperationStatus } from './OperationStatus'; /** * @internal */ export interface IOperationStateFileOptions { - rushProject: RushConfigurationProject; - phase: IPhase; + filename: string; } /** @@ -20,6 +18,10 @@ export interface IOperationStateFileOptions { */ export interface IOperationStateJson { nonCachedDurationMs: number; + + hashes: IOperationHashes | undefined; + + status: OperationStatus; } /** @@ -28,29 +30,12 @@ export interface IOperationStateJson { * @internal */ export class OperationStateFile { - private readonly _rushProject: RushConfigurationProject; private readonly _filename: string; private _state: IOperationStateJson | undefined; public constructor(options: IOperationStateFileOptions) { - const { rushProject, phase } = options; - this._rushProject = rushProject; - this._filename = OperationStateFile._getFilename(phase, rushProject); - } - - private static _getFilename(phase: IPhase, project: RushConfigurationProject): string { - const relativeFilename: string = OperationStateFile.getFilenameRelativeToProjectRoot(phase); - return `${project.projectFolder}/${relativeFilename}`; - } - - /** - * ProjectBuildCache expects the relative path for better logging - * - * @internal - */ - public static getFilenameRelativeToProjectRoot(phase: IPhase): string { - const identifier: string = phase.logFilenameIdentifier; - return `${RushConstants.projectRushFolderName}/${RushConstants.rushTempFolderName}/operation/${identifier}/state.json`; + const { filename } = options; + this._filename = filename; } /**