From fd8739d264c5ddab4a6821fcc9d68ec7db198c6a Mon Sep 17 00:00:00 2001 From: Elliot Nelson Date: Sat, 19 Nov 2022 13:00:54 -0500 Subject: [PATCH 1/5] Configure dependsOnEnvVars setting per operation --- .../src/api/RushProjectConfiguration.ts | 25 ++++++++++++++++--- .../src/logic/buildCache/ProjectBuildCache.ts | 13 ++++++++++ .../logic/operations/ShellOperationRunner.ts | 7 ++++++ .../src/schemas/rush-project.schema.json | 9 +++++++ 4 files changed, 51 insertions(+), 3 deletions(-) diff --git a/libraries/rush-lib/src/api/RushProjectConfiguration.ts b/libraries/rush-lib/src/api/RushProjectConfiguration.ts index 5c5503de0c9..3edfe860bc0 100644 --- a/libraries/rush-lib/src/api/RushProjectConfiguration.ts +++ b/libraries/rush-lib/src/api/RushProjectConfiguration.ts @@ -65,6 +65,16 @@ export interface IOperationSettings { * the build scripts to be compatible with caching. */ disableBuildCacheForOperation?: boolean; + + /** + * An optional list of environment variables that can affect this operation. The values of + * these environment variables will become part of the hash when reading and writing the build cache. + * + * Note: generally speaking, all environment variables available to Rush are also available to any + * operations performed -- Rush assumes that environment variables do not affect build outputs unless + * you list them here. + */ + dependsOnEnvVars?: string[]; } interface IOldRushProjectJson { @@ -89,7 +99,7 @@ const RUSH_PROJECT_CONFIGURATION_FILE: ConfigurationFile = } else if (!parent) { return child; } else { - // Merge the outputFolderNames arrays + // Merge any properties that need to be merged const resultOperationSettingsByOperationName: Map = new Map(); for (const parentOperationSettings of parent) { resultOperationSettingsByOperationName.set( @@ -117,7 +127,7 @@ const RUSH_PROJECT_CONFIGURATION_FILE: ConfigurationFile = let mergedOperationSettings: IOperationSettings | undefined = resultOperationSettingsByOperationName.get(operationName); if (mergedOperationSettings) { - // The parent operation settings object already exists, so append to the outputFolderNames + // The parent operation settings object already exists const outputFolderNames: string[] | undefined = mergedOperationSettings.outputFolderNames && childOperationSettings.outputFolderNames ? [ @@ -126,10 +136,19 @@ const RUSH_PROJECT_CONFIGURATION_FILE: ConfigurationFile = ] : mergedOperationSettings.outputFolderNames || childOperationSettings.outputFolderNames; + const dependsOnEnvVars: string[] | undefined = + mergedOperationSettings.dependsOnEnvVars && childOperationSettings.dependsOnEnvVars + ? [ + ...mergedOperationSettings.dependsOnEnvVars, + ...childOperationSettings.dependsOnEnvVars + ] + : mergedOperationSettings.dependsOnEnvVars || childOperationSettings.dependsOnEnvVars; + mergedOperationSettings = { ...mergedOperationSettings, ...childOperationSettings, - outputFolderNames + outputFolderNames, + dependsOnEnvVars }; resultOperationSettingsByOperationName.set(operationName, mergedOperationSettings); } else { diff --git a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts b/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts index 0c6baf791ba..ab11b442d81 100644 --- a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts +++ b/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts @@ -20,6 +20,7 @@ export interface IProjectBuildCacheOptions { projectConfiguration: RushProjectConfiguration; projectOutputFolderNames: ReadonlyArray; additionalProjectOutputFilePaths?: ReadonlyArray; + additionalContext?: Record; command: string; trackedProjectFiles: string[] | undefined; projectChangeAnalyzer: ProjectChangeAnalyzer; @@ -447,6 +448,18 @@ export class ProjectBuildCache { hash.update(RushConstants.hashDelimiter); hash.update(options.command); hash.update(RushConstants.hashDelimiter); + if (options.additionalContext) { + for (const key of Object.keys(options.additionalContext)) { + // Add additional context keys and values. + // + // This choice (to modiy the hash for every key regardless of whether a value is set) implies + // that just _adding_ an env var to the list of dependsOnEnvVars will modify its hash. This + // seems appropriate, because this behavior is consistent whether or not the env var happens + // to have a value. + hash.update(`${key}=${options.additionalContext[key]}`); + hash.update(RushConstants.hashDelimiter); + } + } for (const projectHash of sortedProjectStates) { hash.update(projectHash); hash.update(RushConstants.hashDelimiter); diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts index 5f5b46a6464..147b5609d5d 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts @@ -444,10 +444,17 @@ export class ShellOperationRunner implements IOperationRunner { const additionalProjectOutputFilePaths: ReadonlyArray = [ OperationStateFile.getFilenameRelativeToProjectRoot(this._phase) ]; + const additionalContext: Record = {}; + if (operationSettings.dependsOnEnvVars) { + for (const varName of operationSettings.dependsOnEnvVars) { + additionalContext['$' + varName] = process.env[varName] || ''; + } + } this._projectBuildCache = await ProjectBuildCache.tryGetProjectBuildCache({ projectConfiguration, projectOutputFolderNames, additionalProjectOutputFilePaths, + additionalContext, buildCacheConfiguration: this._buildCacheConfiguration, terminal, command: this._commandToRun, diff --git a/libraries/rush-lib/src/schemas/rush-project.schema.json b/libraries/rush-lib/src/schemas/rush-project.schema.json index fb9fc596af4..6b163cfd04f 100644 --- a/libraries/rush-lib/src/schemas/rush-project.schema.json +++ b/libraries/rush-lib/src/schemas/rush-project.schema.json @@ -51,6 +51,15 @@ "uniqueItems": true }, + "dependsOnEnvVars": { + "type": "array", + "description": "Specify a list of environment variables that affect the output of this operation. If provided, the values of these variables will become part of the hash when reading and writing from cache.", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "disableBuildCacheForOperation": { "description": "Disable caching for this operation. The operation will never be restored from cache. This may be useful if this operation affects state outside of its folder.", "type": "boolean" From d32d3f08c55b00a2f24e39dbb6b7a31e1bf0993c Mon Sep 17 00:00:00 2001 From: Elliot Nelson Date: Sat, 19 Nov 2022 13:14:38 -0500 Subject: [PATCH 2/5] rush change --- .../rush/enelson-env-vars_2022-11-19-18-14.json | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 common/changes/@microsoft/rush/enelson-env-vars_2022-11-19-18-14.json diff --git a/common/changes/@microsoft/rush/enelson-env-vars_2022-11-19-18-14.json b/common/changes/@microsoft/rush/enelson-env-vars_2022-11-19-18-14.json new file mode 100644 index 00000000000..b0e8e8905c8 --- /dev/null +++ b/common/changes/@microsoft/rush/enelson-env-vars_2022-11-19-18-14.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@microsoft/rush", + "comment": "Add dependsOnEnvVars configuration in rush-project.json", + "type": "none" + } + ], + "packageName": "@microsoft/rush" +} \ No newline at end of file From cc09edc4140f9fad5ef7fb8be202152f483e316f Mon Sep 17 00:00:00 2001 From: Elliot Nelson Date: Sat, 19 Nov 2022 13:19:30 -0500 Subject: [PATCH 3/5] fix tests --- libraries/rush-lib/src/api/RushProjectConfiguration.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/rush-lib/src/api/RushProjectConfiguration.ts b/libraries/rush-lib/src/api/RushProjectConfiguration.ts index 3edfe860bc0..f6a04a37c56 100644 --- a/libraries/rush-lib/src/api/RushProjectConfiguration.ts +++ b/libraries/rush-lib/src/api/RushProjectConfiguration.ts @@ -147,8 +147,8 @@ const RUSH_PROJECT_CONFIGURATION_FILE: ConfigurationFile = mergedOperationSettings = { ...mergedOperationSettings, ...childOperationSettings, - outputFolderNames, - dependsOnEnvVars + ...(outputFolderNames ? { outputFolderNames } : {}), + ...(dependsOnEnvVars ? { dependsOnEnvVars } : {}) }; resultOperationSettingsByOperationName.set(operationName, mergedOperationSettings); } else { From 37304acb69fa66d7668c900effbf14eb15b31178 Mon Sep 17 00:00:00 2001 From: Elliot Nelson Date: Mon, 21 Nov 2022 07:36:39 -0500 Subject: [PATCH 4/5] Update libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts Sort included context keys Co-authored-by: Ian Clanton-Thuon --- libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts b/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts index ab11b442d81..3e4e1e2f855 100644 --- a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts +++ b/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts @@ -449,7 +449,7 @@ export class ProjectBuildCache { hash.update(options.command); hash.update(RushConstants.hashDelimiter); if (options.additionalContext) { - for (const key of Object.keys(options.additionalContext)) { + for (const key of Object.keys(options.additionalContext).sort()) { // Add additional context keys and values. // // This choice (to modiy the hash for every key regardless of whether a value is set) implies From fec1704a05986e5f0eb7c2860a111de119254a74 Mon Sep 17 00:00:00 2001 From: Elliot Nelson Date: Mon, 21 Nov 2022 07:38:49 -0500 Subject: [PATCH 5/5] update comment --- .../@microsoft/rush/enelson-env-vars_2022-11-19-18-14.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/changes/@microsoft/rush/enelson-env-vars_2022-11-19-18-14.json b/common/changes/@microsoft/rush/enelson-env-vars_2022-11-19-18-14.json index b0e8e8905c8..0a005e98426 100644 --- a/common/changes/@microsoft/rush/enelson-env-vars_2022-11-19-18-14.json +++ b/common/changes/@microsoft/rush/enelson-env-vars_2022-11-19-18-14.json @@ -2,9 +2,9 @@ "changes": [ { "packageName": "@microsoft/rush", - "comment": "Add dependsOnEnvVars configuration in rush-project.json", + "comment": "Add a \"dependsOnEnvVars\" configuration option to operations in rush-project.json. The variables specified in this option are included in the cache key hash calculation.", "type": "none" } ], "packageName": "@microsoft/rush" -} \ No newline at end of file +}