From 4836fd2568d973048f5ab263c6270d02c2059201 Mon Sep 17 00:00:00 2001 From: Kirill Ivlev <102740624+kirill-ivlev@users.noreply.github.com> Date: Tue, 18 Apr 2023 14:39:56 +0400 Subject: [PATCH] Dotnet6 agent merge master (#4243) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Catch updating exception (#4082) * Node16 version updated to 16.17.1" (#4084) * Bump ubuntu image version to 20.04 (#4079) * Prevent creating host container network (#3788) * Prevent creating host container network * Remove redundant empty lines * Fix invalid knob value fetching * Update ContainerOperationProvider.cs Co-authored-by: Denis Rumyantsev Co-authored-by: Kirill Ivlev <102740624+kirill-ivlev@users.noreply.github.com> Co-authored-by: Konstantin Tyukalov <52399739+KonstantinTyukalov@users.noreply.github.com> * Allow env.sh to accept additional env var names (#3339) The hard-coded list is not necessarily enough. This allows the caller to specify additional environment variables to write to .env. Co-authored-by: Konstantin Tyukalov <52399739+KonstantinTyukalov@users.noreply.github.com> Co-authored-by: Kirill Ivlev <102740624+kirill-ivlev@users.noreply.github.com> * Revert "Fix execution of scripts in variables (#4016)" (#4088) This reverts commit af5328b2656c9bf5a70117044a707e011de77083. * Add Ubuntu 18.04 (#3943) Co-authored-by: Kirill Ivlev <102740624+kirill-ivlev@users.noreply.github.com> Co-authored-by: Konstantin Tyukalov <52399739+KonstantinTyukalov@users.noreply.github.com> * Make agent update list of systems supporting .NET 6 periodically (#4081) * implemented logic to fetch net6.json from server every hour * added await keyword * Moved logic to set warning from ExecutionContext to JobRunner since InitializeJob method is not async * fixed issue with condition in Equals method of OperatingSystem class * small refactoring * net6.json must be read only if it was not fetched from server * refactored code of method "GetNet6SupportedSystems" * restored field net6SupportedSystems to reduce IO oprations and cover case when agent started but net6.json is not older than 1 hour Co-authored-by: Kirill Ivlev <102740624+kirill-ivlev@users.noreply.github.com> * Retries to lock Services database (#4085) * retries to lock database * Decrease timeout + Add retry message Co-authored-by: Konstantin Tyukalov Co-authored-by: Konstantin Tyukalov <52399739+KonstantinTyukalov@users.noreply.github.com> * Fixed name of DockerAdditionalNetworkOptions Knob (#3888) Co-authored-by: Konstantin Tyukalov <52399739+KonstantinTyukalov@users.noreply.github.com> * Format agent source code (#4101) * Run dotnet format * Remove extra semicolons * Fix SonarQube's "'Any()' should be used to test for emptiness" / Code Smell (#3404) * Add script to predict v2->v3 upgrade compatibility (#4092) * Add script to predict v2->v3 upgrade compatibility * Fix typo * Fixed another typo: frpom Co-authored-by: Kirill Ivlev <102740624+kirill-ivlev@users.noreply.github.com> * Added strict boolean knob values (#4103) * Add ConvertToBooleanStrict * Add AsBooleanStrict for agent knobs * update build-job (#4110) * update build-job - added UseDotNet task for linux os * Switch EsrpCodeSigning from V1 to V2 returned back UseDotNet task condition * Set UseDotNet version to 6 on osx * added dotnet 3.1.x before unit tests on win x86 * fix dotnet host search (#4124) * fix dotnet host search - added performMultiLevelLookup * Added runtime to unitTests step * Changed patch version in semver of runtime * remove sdk and runtime from unitTests script * Fixed problem with .NET Core 3.1 installation * Added UseDotNet task in functionalTests Co-authored-by: Roman Shchukin * Add option to skip rhel6 in pipeline and ci (#4121) * skipRhelRelease parameter added * skipRhelRelease parameter fixed * releaseRhelOnly parameter added * Logic of removing not filling lines from releaseNote * Fix pipeline with test releaseNote job * Fix pipeline with test releaseNote job * Fix pipeline with test releaseNote job * Fix pipeline with test releaseNote job * Fix pipeline with test releaseNote job * Fix pipeline with test releaseNote job * Fix pipeline with test releaseNote job * Fix pipeline with test releaseNote job * Fix pipeline with test releaseNote job * Fix pipeline with test releaseNote job * Removed test logs * Added clear empty lines logic * Commented code for test. Test changes in pipeline * Removed test changes * skipRhelRelease parametr added to ci pipeline * Update dotnet to 3.1.32 (#4112) * DockerLogin retries (#4100) * DockerLogin retries - added changes from PR microsoft/azure-pipelines-agent#3840 - fixed incorrect variables * DockerLogin retries - added step UseDotNet@2 * DockerLogin retries - added linux to UseDotNet task * Docker Login retries Implement retry into DockerLogin method * Implement feature flag for docker login retries * revert build-job back * Added retry for docker start * fix delay * update knob description * Update agent Docker methods - reworked Docker methods with retries behavior - changed InvokeWithRetryIfNonZero on to incapsulated logic method * - renamed local variable as is in ADO * Added docker retry behavior to docker version method Co-authored-by: Your Name Co-authored-by: Kirill Ivlev <102740624+kirill-ivlev@users.noreply.github.com> Co-authored-by: Merlyn Oppenheim * Add the exact operation system version and environment to the telemetry (#4104) * Update TaskRunner.cs * Update build-job.yml * Added new variables to telemetry AgentName, MachineName, IsSelfHosted Co-authored-by: Kirill Ivlev <102740624+kirill-ivlev@users.noreply.github.com> * Mingit version upgraded to 2.39.1 (#4126) * Fix agent lint errors (#4117) * Fix CA2000 error with secrets masker * Fix CA1711 for ServiceBootFlag * Fix CA2000 for JobRunner * Resolve CA2000 in StepHost * Resolve CA2000 in Windows service * Resolve CA2000 for WorkerCommandManager * Resove CA2000 for CodeCoverageCommands * Remove unused namespaces * Add fixture for SecretMasker tests * Change Fixtures -> Disposable * Use dispose pattern to fix errors * Formatting * Revert "Allow env.sh to accept additional env var names (#3339)" (#4143) This reverts commit 27e3d4e177e83aac805bf93c38233fa1a9f06101. * Bump azure pipelines task lib to 4.2.0 (#4155) * Add warning for deprecated runners (#4162) * Add warning for deprecated runners * Update strings.json * fix typo * Update NodeHandler.cs * FEATURE 2019014 - Gather telemetry on Agent Azure & Docker Container usage (#4166) * FEATURE 2019014 - Gather telemetry on Agent Azure & Docker Container usage * Re-write try-catch * Update TaskRunner.cs * cleanup * Distinguish AzureInstanceMetadata detection to a separate class --------- Co-authored-by: Kirill Ivlev <102740624+kirill-ivlev@users.noreply.github.com> * Added legacy lib source for support on Ubuntu 22.04 (#4161) * Moved logic to drain queues after each task under knob "AGENT_DRAIN_QUEUES_AFTER_TASK" (#4176) * Added knob "DrainQueuesAfterTask" with default "false" * move logic to drain queues behind knob DrainQueuesAfterTask * added mocking of GetScopedEnvironment method in step execution context * added mocking of GetScopedEnvironment method in step execution context * changed default value for knob "AGENT_DRAIN_QUEUES_AFTER_TASK" to true * implementd AsBooleanStrict and check in JobRunner * Revert "implementd AsBooleanStrict and check in JobRunner" This reverts commit a1c0c678892a03759543a00b03153adfa0cf5e3b. * Reversed the knob * Add issue templates (#4154) * Add issue templates * Update config.yml * update templates * Update config.yml * Added knob to break pipeline unless user opt-in (#4165) * Added knob to force agent fail if user doesn't opt-in updates on unsupported OS * Resolved PR comments --------- Co-authored-by: Kirill Ivlev <102740624+kirill-ivlev@users.noreply.github.com> * Localization update (#4187) * Added files for onelocbuild * Update localize-pipeline.yml for Azure Pipelines * Fixed typo in resource.resx path * Removed en-US * Update localize-pipeline.yml for Azure Pipelines * LEGO: check in for Localization to temporary branch. (#3246) * LEGO: check in for Localization to temporary branch. (#3268) * LEGO: check in for Localization to temporary branch. (#3265) * LEGO: check in for Localization to temporary branch. (#3279) * Temporary renamed localization folders * Renamed localization folders * Set up schedule and notifications for the localization pipeline (#3269) * [localization] Fixed localization pipeline issue with already localized strings replaced (#3332) * Localized file check-in by OneLocBuild Task: Build definition ID 10944: Build ID 14646614 Localized file check-in by OneLocBuild Task * LEGO: check in for Localization to temporary branch. (#3363) Co-authored-by: csigs * LEGO: check in for Localization to temporary branch. (#3364) Co-authored-by: csigs * Localized file check-in by OneLocBuild Task: Build definition ID 10944: Build ID 14919763 Localized file check-in by OneLocBuild Task * Create PR in OneLocBuild task only on third week of sprint (#3374) * Fix localization pipeline * Add missed change * Added option to disable PR creation * update localize pipeline (#3637) Co-authored-by: Ilya Kuleshov * Localization update (#3612) * Removing Localize folder * RHEL/CentOS 7 Fix for Lets Encrypt Change (#3601) * Revert "RHEL/CentOS 7 Fix for Lets Encrypt Change (#3601)" This reverts commit 629921e0faea9739aeb926a6c67a443cbf936b96. * Revert "Removing Localize folder" This reverts commit 33cb0ae7c170e9cfb273365c2bc594c6b0ac4da3. Co-authored-by: AndreyIvanov42 <93121155+AndreyIvanov42@users.noreply.github.com> Co-authored-by: kuleshovilya <87485027+kuleshovilya@users.noreply.github.com> Co-authored-by: Ilya Kuleshov * Removed OneLocBuild folder * Move notifications about Agent Localization PR from Slack to MS Teams - Part 1 (#3744) * Include strings from Misc/src/layoutbin to LocProject file (#4021) * Update LocProject file * Update CopyOption for LocProject * Localized file check-in by OneLocBuild Task: Build definition ID 10944: Build ID 18665127 (#4022) * Revert "Include strings from Misc/src/layoutbin to LocProject file (#4021)" This reverts commit 8f5c0c05b3c526d3ac53b9e540917f690445c9ef. * Revert "Localized file check-in by OneLocBuild Task: Build definition ID 10944: Build ID 18665127 (#4022)" (#4023) This reverts commit b9c714bdbaff2a28136e0e5093fe9beef1dc4183. * Include layoutbin into locproject * Localized file check-in by OneLocBuild Task: Build definition ID 10944: Build ID 18680899 (#4028) * Juno: check in to lego/hb_5733438c-38c5-41ae-9557-a9d9754c4535_20221104175414554. (#4030) * Juno: check in to lego/hb_5733438c-38c5-41ae-9557-a9d9754c4535_20221105085259861. (#4031) * Juno: check in to lego/hb_5733438c-38c5-41ae-9557-a9d9754c4535_20221106085521421. (#4032) * Localized file check-in by OneLocBuild Task: Build definition ID 10944: Build ID 18697046 (#4033) * Juno: check in to lego/hb_5733438c-38c5-41ae-9557-a9d9754c4535_20221107143113520. (#4039) * Localized file check-in by OneLocBuild Task: Build definition ID 10944: Build ID 18706620 (#4044) * Localized file check-in by OneLocBuild Task: Build definition ID 10944: Build ID 19415018 (#4163) * Juno: check in to lego/hb_5733438c-38c5-41ae-9557-a9d9754c4535_20230223095051399. (#4168) * Juno: check in to lego/hb_5733438c-38c5-41ae-9557-a9d9754c4535_20230224095040576. (#4170) * Localized file check-in by OneLocBuild Task: Build definition ID 10944: Build ID 19476421 (#4179) * Update loc strings * Localized file check-in by OneLocBuild Task: Build definition ID 10944: Build ID 19476736 (#4181) * Juno: check in to lego/hb_5733438c-38c5-41ae-9557-a9d9754c4535_20230305095015478. (#4183) * Juno: check in to lego/hb_5733438c-38c5-41ae-9557-a9d9754c4535_20230306095225210. (#4184) * Removing Localize and OneLocBuild folder --------- Co-authored-by: Anatolii Bolshakov (Akvelon INC) Co-authored-by: csigs Co-authored-by: Egor Bryzgalov Co-authored-by: Anatoly Bolshakov Co-authored-by: csigs Co-authored-by: Nikita Ezzhev Co-authored-by: kuleshovilya <87485027+kuleshovilya@users.noreply.github.com> Co-authored-by: Ilya Kuleshov Co-authored-by: AndreyIvanov42 <93121155+AndreyIvanov42@users.noreply.github.com> Co-authored-by: Denis Tikhomirov <90906678+denis-tikhomirov@users.noreply.github.com> Co-authored-by: KonstantinTyukalov * Add "Mariner" to list of .NET 6 supporting systems (#4197) * Cover unsupported OS failure with FF (#4196) * Cover unsupported OS failure with FF * Update strings.json * Update JobRunner.cs * Update JobRunner.cs * Add knob to enable fetching net6.json file from GitHub (#4200) * Added knob "AGENT_ENABLE_FETCHING_NET6_LIST" * implemented knob EnableFetchingNet6List in function GetNet6SupportedSystems * Removed RuntimeKnobSource for EnableFetchingNet6List * removed logic which fails pipeline if there is some error occurred during checking if system supports .NET 6 * fixed typo * Update autoAssignABTT.yml (#4201) * Fixed typos in strings.json (#3523) * Added trimmed user secret to masker dictionary (#4195) * Add trimmed secret value to masker dictionary * [Refactor] Update MinSecretLength property. (#4137) * [Refactor] Update MinSecretLength property. Removed exception throwing, Added MinSecretLengthLimit public property * Update test MinValue test * Update test * add ability to disable publishing tests metadata (#4209) * moved logic of checking if system supports .NET 6 to JobExtension (#4207) * Increase limit for AZP_IGNORE_SECRETS_SHORTER_THAN knob to 6 (#4214) * Update secret masker length limit * Make property static * Decrease limit to 6 * Update tests * Update tests * Update comment --------- Co-authored-by: Kirill Ivlev <102740624+kirill-ivlev@users.noreply.github.com> * Refactor logic to drain queues after each task (#4213) * move draining queues before calling method "Complete" * Revert "BUG 1972388: vsbuild task in YAML build pipeline hangs forever in ADO even though the task has already logged completion (#3979)" This reverts commit b3cf2c0e3167288f4b60e0ba38e17b53cd9fa9f5. * implemented logic to drain web console and timeline queues after each task * corrected typo Co-authored-by: Konstantin Tyukalov <52399739+KonstantinTyukalov@users.noreply.github.com> * renamed variable "drain" to "shouldDrain" --------- Co-authored-by: Konstantin Tyukalov <52399739+KonstantinTyukalov@users.noreply.github.com> Co-authored-by: Kirill Ivlev <102740624+kirill-ivlev@users.noreply.github.com> * git lfs version upgraded (#4218) * Add noderunner.md (#4202) * Add noderunner.md * Minor fixes * Update noderunner.md * Update noderunner.md * Resolve comments * Minor fixes in noderunner doc * minor fixes * Bug 2030875 - azure-pipelines-task-lib repo logs passwords in plain text (#4223) * added logic to unescape vso command in case there is encoded symbols * Added "TaskCommandHelper" class with method "AddSecret" which checks if decoding percents disabled and adds secret with decoded percents as well * fixed issue with test SetEndpointAuthParameter * moved logic to add decoded secret under check if value is empty * Update AzureInstanceMetadataProvider.cs * Update agentversion --------- Co-authored-by: Roman-Shchukin <111063382+Roman-Shchukin@users.noreply.github.com> Co-authored-by: Konstantin Tyukalov <52399739+KonstantinTyukalov@users.noreply.github.com> Co-authored-by: Steve Co-authored-by: Denis Rumyantsev Co-authored-by: Rick Brown Co-authored-by: Liliia Sabitova <107196662+LiliaSabitova@users.noreply.github.com> Co-authored-by: Eric van Wijk Co-authored-by: Sergey Koryshev Co-authored-by: Konstantin Tyukalov Co-authored-by: Rami Co-authored-by: Martin Co-authored-by: ivanduplenskikh <115665590+ivanduplenskikh@users.noreply.github.com> Co-authored-by: Roman Shchukin Co-authored-by: Your Name Co-authored-by: Merlyn Oppenheim Co-authored-by: İsmayıl İsmayılov <110806089+ismayilov-ismayil@users.noreply.github.com> Co-authored-by: Maksim Petrov <47208721+vmapetr@users.noreply.github.com> Co-authored-by: Anatolii Bolshakov (Akvelon INC) Co-authored-by: csigs Co-authored-by: Egor Bryzgalov Co-authored-by: Anatoly Bolshakov Co-authored-by: csigs Co-authored-by: Nikita Ezzhev Co-authored-by: kuleshovilya <87485027+kuleshovilya@users.noreply.github.com> Co-authored-by: Ilya Kuleshov Co-authored-by: AndreyIvanov42 <93121155+AndreyIvanov42@users.noreply.github.com> Co-authored-by: Denis Tikhomirov <90906678+denis-tikhomirov@users.noreply.github.com> Co-authored-by: Maxim Zaytsev Co-authored-by: Rob Hensley --- docs/noderunner.md | 60 ++++++++++++++ src/Agent.Sdk/Knob/AgentKnobs.cs | 7 ++ src/Agent.Sdk/Util/ILoggedSecretMasker.cs | 2 + src/Agent.Sdk/Util/LoggedSecretMasker.cs | 19 ++--- src/Agent.Sdk/Util/PlatformUtil.cs | 3 +- src/Agent.Worker/ExecutionContext.cs | 29 +++---- src/Agent.Worker/JobExtension.cs | 50 ++++++++++++ src/Agent.Worker/JobRunner.cs | 46 ----------- src/Agent.Worker/StepsRunner.cs | 20 ----- src/Agent.Worker/TaskCommandExtension.cs | 28 +++++-- .../TestResults/Utils/TestResultUtils.cs | 14 ++-- src/Agent.Worker/Worker.cs | 4 + src/Agent.Worker/WorkerCommandManager.cs | 3 +- .../JobServerQueue.cs | 78 +++++++++++-------- src/Misc/externals.sh | 2 +- src/Misc/layoutbin/en-US/strings.json | 13 ++-- .../SecretMaskerTests/LoggedSecretMaskerL0.cs | 38 +++++---- src/Test/L0/Worker/TaskCommandExtensionL0.cs | 2 + src/agentversion | 2 +- 19 files changed, 258 insertions(+), 162 deletions(-) create mode 100644 docs/noderunner.md diff --git a/docs/noderunner.md b/docs/noderunner.md new file mode 100644 index 0000000000..393535ad14 --- /dev/null +++ b/docs/noderunner.md @@ -0,0 +1,60 @@ +# Node 6 support + +Agent tasks can be implemented in PowerShell or Node. The agent currently ships with three versions of Node that tasks can target: 6, 10 & 16. + +Since Node 6 has long passed out of the upstream maintenance window, and all officially supported tasks are migrated from Node 6 to Node 10, Node 6 soon will be removed from the agent package. +It's also highly recommended to third-party task maintainers migrate tasks to Node 10 or Node 16. + +However, to support backward compatibility with the Node 6 tasks we provide self-service methods to install the designated Node runner manually. + +## Install Node 6 runner manually + +To support the execution of Node 6 tasks agent should be provided with the latest Node 6 version - `6.17.1.0`. + +Despite that Node 6 is officially reached the End-of-Life, please, notice that it still can have maintenance updates, so it is required for the agent to get the latest binaries. You can check the currently existing Node versions [here](https://nodejs.org/dist/). + +Please use the following steps to manually install the required runner: + +1. Download the latest available version of Node 6 binaries for your operating system from the official Node [registry](https://nodejs.org/dist/). + +1. Create a folder named `node` under the `agent/externals` directory, and extract downloaded Node binaries into that folder. + +You can also use the following commands to install the Node 6 runner via the Powershell or Bash: + +Windows: +```powershell + $agentFolder = "" // Specify the Azure DevOps Agent folder, e.g. C:\agents\my_agent + $osArch = "" // Specify the OS architecture, e.g. x64 / x86 + + New-Item -Type Directory -Path "${agentFolder}\externals\node" + + Invoke-WebRequest -Uri "https://nodejs.org/dist/v6.17.1/win-${osArch}/node.exe" -OutFile "${agentFolder}\externals\node\node.exe" + Invoke-WebRequest -Uri "https://nodejs.org/dist/v6.17.1/win-${osArch}/node.lib" -OutFile "${agentFolder}\externals\node\node.lib" +``` + +Linux / macOS: +```bash + agent_folder="" // Specify the Azure DevOps Agent folder, e.g. /home/user/agents/my_agent + os_platform="" // Specify the OS platform, e.g. linux / darwin + os_arch="" // Specify the OS architecture, e.g. x64 / x86 + + mkdir "${agent_folder}/externals/node" + + wget -O "/tmp/node-v6.17.1-${os_platform}-${os_arch}.tar.gz" "https://nodejs.org/dist/v6.17.1/node-v6.17.1-${os_platform}-${os_arch}.tar.gz" + + tar -xvf "/tmp/node-v6.17.1-${os_platform}-${os_arch}.tar.gz" -C "${agent_folder}/externals/node/" +``` + +## Install Node runner via NodeTaskRunnerInstaller + +You can also use the Azure DevOps task [NodeTaskRunnerInstaller](https://github.com/microsoft/azure-pipelines-tasks/tree/master/Tasks/NodeTaskRunnerInstallerV0) to install the required runner version via Azure DevOps CI. + +Use the following pipeline task sample to install the latest version of Node 6 runner: + +```yaml + - task: NodeTaskRunnerInstaller@0 + inputs: + runnerVersion: 6 +``` + +Please, check more details in [NodeTaskRunnerInstaller task]() documentation [TODO: FIX LINK]. diff --git a/src/Agent.Sdk/Knob/AgentKnobs.cs b/src/Agent.Sdk/Knob/AgentKnobs.cs index 7f33c9cdb4..35dd015be9 100644 --- a/src/Agent.Sdk/Knob/AgentKnobs.cs +++ b/src/Agent.Sdk/Knob/AgentKnobs.cs @@ -172,6 +172,13 @@ public class AgentKnobs new EnvironmentKnobSource("VSTSAGENT_DUMP_JOB_EVENT_LOGS"), new BuiltInDefaultKnobSource("false")); + public static readonly Knob DisableTestsMetadata = new Knob( + nameof(DisableTestsMetadata), + "If true, publishing tests metadata to evidence store will be disabled.", + new RuntimeKnobSource("AZP_AGENT_DISABLE_TESTS_METADATA"), + new EnvironmentKnobSource("AZP_AGENT_DISABLE_TESTS_METADATA"), + new BuiltInDefaultKnobSource("false")); + // Diag logging public static readonly Knob AgentDiagLogPath = new Knob( nameof(AgentDiagLogPath), diff --git a/src/Agent.Sdk/Util/ILoggedSecretMasker.cs b/src/Agent.Sdk/Util/ILoggedSecretMasker.cs index acd832ae77..ffe3d1c0a4 100644 --- a/src/Agent.Sdk/Util/ILoggedSecretMasker.cs +++ b/src/Agent.Sdk/Util/ILoggedSecretMasker.cs @@ -8,6 +8,8 @@ namespace Agent.Sdk.Util /// public interface ILoggedSecretMasker : ISecretMasker { + static int MinSecretLengthLimit { get; } + void AddRegex(String pattern, string origin); void AddValue(String value, string origin); void AddValueEncoder(ValueEncoder encoder, string origin); diff --git a/src/Agent.Sdk/Util/LoggedSecretMasker.cs b/src/Agent.Sdk/Util/LoggedSecretMasker.cs index 2eede77beb..dff7caad6d 100644 --- a/src/Agent.Sdk/Util/LoggedSecretMasker.cs +++ b/src/Agent.Sdk/Util/LoggedSecretMasker.cs @@ -11,9 +11,6 @@ public class LoggedSecretMasker : ILoggedSecretMasker private ISecretMasker _secretMasker; private ITraceWriter _trace; - // We don't allow to skip secrets longer than 4 characters. - private readonly short _maxMinSecretLength = 4; - private void Trace(string msg) { this._trace?.Info(msg); @@ -73,6 +70,10 @@ public void AddRegex(string pattern, string origin) AddRegex(pattern); } + // We don't allow to skip secrets longer than 5 characters. + // Note: the secret that will be ignored is of length n-1. + public static int MinSecretLengthLimit => 6; + public int MinSecretLength { get @@ -81,14 +82,14 @@ public int MinSecretLength } set { - if (value > _maxMinSecretLength) + if (value > MinSecretLengthLimit) { - _secretMasker.MinSecretLength = _maxMinSecretLength; - - throw new ArgumentException($"Not allowed minimum secret length. Set max value: {_maxMinSecretLength}"); + _secretMasker.MinSecretLength = MinSecretLengthLimit; + } + else + { + _secretMasker.MinSecretLength = value; } - - _secretMasker.MinSecretLength = value; } } diff --git a/src/Agent.Sdk/Util/PlatformUtil.cs b/src/Agent.Sdk/Util/PlatformUtil.cs index 17e3725750..37590a1f89 100644 --- a/src/Agent.Sdk/Util/PlatformUtil.cs +++ b/src/Agent.Sdk/Util/PlatformUtil.cs @@ -4,6 +4,8 @@ using System; using System.IO; using System.Linq; +using System.Net; +using System.Net.Http; using System.Reflection; using System.Runtime.InteropServices; using System.Text; @@ -16,7 +18,6 @@ using Newtonsoft.Json; using System.ServiceProcess; using Agent.Sdk.Util; -using System.Net.Http; namespace Agent.Sdk { diff --git a/src/Agent.Worker/ExecutionContext.cs b/src/Agent.Worker/ExecutionContext.cs index f24b16038e..2f9c3ddfc7 100644 --- a/src/Agent.Worker/ExecutionContext.cs +++ b/src/Agent.Worker/ExecutionContext.cs @@ -295,6 +295,12 @@ public TaskResult Complete(TaskResult? result = null, string currentOperation = this.Warning(StringUtil.Loc("TotalThrottlingDelay", TimeSpan.FromMilliseconds(_totalThrottlingDelayInMilliseconds).TotalSeconds)); } + if (!AgentKnobs.DisableDrainQueuesAfterTask.GetValue(this).AsBoolean()) + { + _jobServerQueue.ForceDrainWebConsoleQueue = true; + _jobServerQueue.ForceDrainTimelineQueue = true; + } + _record.CurrentOperation = currentOperation ?? _record.CurrentOperation; _record.ResultCode = resultCode ?? _record.ResultCode; _record.FinishTime = DateTime.UtcNow; @@ -518,26 +524,23 @@ public void InitializeJob(Pipelines.AgentJobRequestMessage message, Cancellation // Prepend Path PrependPath = new List(); - // Docker (JobContainer) - string imageName = Variables.Get("_PREVIEW_VSTS_DOCKER_IMAGE"); - if (string.IsNullOrEmpty(imageName)) - { - imageName = Environment.GetEnvironmentVariable("_PREVIEW_VSTS_DOCKER_IMAGE"); - } - var minSecretLen = AgentKnobs.MaskedSecretMinLength.GetValue(this).AsInt(); + HostContext.SecretMasker.MinSecretLength = minSecretLen; - try + if (HostContext.SecretMasker.MinSecretLength < minSecretLen) { - this.HostContext.SecretMasker.MinSecretLength = minSecretLen; + warnings.Add(StringUtil.Loc("MinSecretsLengtLimitWarning", HostContext.SecretMasker.MinSecretLength)); } - catch (ArgumentException ex) + + HostContext.SecretMasker.RemoveShortSecretsFromDictionary(); + + // Docker (JobContainer) + string imageName = Variables.Get("_PREVIEW_VSTS_DOCKER_IMAGE"); + if (string.IsNullOrEmpty(imageName)) { - warnings.Add(ex.Message); + imageName = Environment.GetEnvironmentVariable("_PREVIEW_VSTS_DOCKER_IMAGE"); } - this.HostContext.SecretMasker.RemoveShortSecretsFromDictionary(); - Containers = new List(); _defaultStepTarget = null; _currentStepTarget = null; diff --git a/src/Agent.Worker/JobExtension.cs b/src/Agent.Worker/JobExtension.cs index 08e7878006..4f556dc94b 100644 --- a/src/Agent.Worker/JobExtension.cs +++ b/src/Agent.Worker/JobExtension.cs @@ -72,6 +72,52 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel context.Start(); context.Section(StringUtil.Loc("StepStarting", StringUtil.Loc("InitializeJob"))); + // Check if a system supports .NET 6 + PackageVersion agentVersion = new PackageVersion(BuildConstants.AgentPackage.Version); + if (agentVersion.Major < 3) + { + try + { + Trace.Verbose("Checking if your system supports .NET 6"); + + string systemId = PlatformUtil.GetSystemId(); + SystemVersion systemVersion = PlatformUtil.GetSystemVersion(); + string notSupportNet6Message = null; + + if (await PlatformUtil.DoesSystemPersistsInNet6Whitelist()) + { + // Check version of the system + if (!await PlatformUtil.IsNet6Supported()) + { + notSupportNet6Message = $"The operating system the agent is running on is \"{systemId}\" ({systemVersion}), which will not be supported by the .NET 6 based v3 agent. Please upgrade the operating system of this host to ensure compatibility with the v3 agent. See https://aka.ms/azdo-pipeline-agent-version"; + if (AgentKnobs.AgentFailOnIncompatibleOS.GetValue(jobContext).AsBoolean() && + !AgentKnobs.AcknowledgeNoUpdates.GetValue(jobContext).AsBoolean()) + { + throw new UnsupportedOsException(StringUtil.Loc("FailAgentOnUnsupportedOs")); + } + } + } + else + { + notSupportNet6Message = $"The operating system the agent is running on is \"{systemId}\" ({systemVersion}), which has not been tested with the .NET 6 based v3 agent. The v2 agent wil not automatically upgrade to the v3 agent. You can manually download the .NET 6 based v3 agent from https://github.com/microsoft/azure-pipelines-agent/releases. See https://aka.ms/azdo-pipeline-agent-version"; + } + + if (!string.IsNullOrWhiteSpace(notSupportNet6Message)) + { + context.Warning(notSupportNet6Message); + } + } + catch (UnsupportedOsException) + { + throw; + } + catch (Exception ex) + { + Trace.Error($"Error has occurred while checking if system supports .NET 6: {ex}"); + context.Warning(ex.Message); + } + } + // Set agent version variable. context.SetVariable(Constants.Variables.Agent.Version, BuildConstants.AgentPackage.Version); context.Output(StringUtil.Loc("AgentNameLog", context.Variables.Get(Constants.Variables.Agent.Name))); @@ -592,4 +638,8 @@ private void OutputSetupInfo(IExecutionContext context) } } } + + public class UnsupportedOsException : Exception { + public UnsupportedOsException(string message) : base(message) { } + } } \ No newline at end of file diff --git a/src/Agent.Worker/JobRunner.cs b/src/Agent.Worker/JobRunner.cs index 2ef8f90039..2a36027981 100644 --- a/src/Agent.Worker/JobRunner.cs +++ b/src/Agent.Worker/JobRunner.cs @@ -103,52 +103,6 @@ public async Task RunAsync(Pipelines.AgentJobRequestMessage message, jobContext = HostContext.CreateService(); jobContext.InitializeJob(message, jobRequestCancellationToken); - // Check if a system supports .NET 6 - PackageVersion agentVersion = new PackageVersion(BuildConstants.AgentPackage.Version); - if (agentVersion.Major < 3) - { - try - { - Trace.Verbose("Checking if your system supports .NET 6"); - - string systemId = PlatformUtil.GetSystemId(); - SystemVersion systemVersion = PlatformUtil.GetSystemVersion(); - string notSupportNet6Message = null; - - if (await PlatformUtil.DoesSystemPersistsInNet6Whitelist()) - { - // Check version of the system - if (!await PlatformUtil.IsNet6Supported()) - { - notSupportNet6Message = $"The operating system the agent is running on is \"{systemId}\" ({systemVersion}), which will not be supported by the .NET 6 based v3 agent. Please upgrade the operating system of this host to ensure compatibility with the v3 agent. See https://aka.ms/azdo-pipeline-agent-version"; - if (AgentKnobs.AgentFailOnIncompatibleOS.GetValue(jobContext).AsBoolean() && - !AgentKnobs.AcknowledgeNoUpdates.GetValue(jobContext).AsBoolean()) - { - jobContext.Error(StringUtil.Loc("FailAgentOnUnsupportedOs")); - return await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed); - } - } - } - else - { - notSupportNet6Message = $"The operating system the agent is running on is \"{systemId}\" ({systemVersion}), which has not been tested with the .NET 6 based v3 agent. The v2 agent wil not automatically upgrade to the v3 agent. You can manually download the .NET 6 based v3 agent from https://github.com/microsoft/azure-pipelines-agent/releases. See https://aka.ms/azdo-pipeline-agent-version"; - } - - if (!string.IsNullOrWhiteSpace(notSupportNet6Message)) - { - - - jobContext.AddIssue(new Issue() { Type = IssueType.Warning, Message = notSupportNet6Message }); - } - } - catch (Exception ex) - { - jobContext.Warning($"Error has occurred while checking if system supports .NET 6: {ex.Message}"); - Trace.Error($"Error has occurred while checking if system supports .NET 6: {ex}"); - - } - } - Trace.Info("Starting the job execution context."); jobContext.Start(); jobContext.Section(StringUtil.Loc("StepStarting", message.JobDisplayName)); diff --git a/src/Agent.Worker/StepsRunner.cs b/src/Agent.Worker/StepsRunner.cs index 4c3af97ce7..b3c80f411d 100644 --- a/src/Agent.Worker/StepsRunner.cs +++ b/src/Agent.Worker/StepsRunner.cs @@ -10,7 +10,6 @@ using System.Threading.Tasks; using Microsoft.TeamFoundation.DistributedTask.Expressions; using Pipelines = Microsoft.TeamFoundation.DistributedTask.Pipelines; -using Microsoft.VisualStudio.Services.CircuitBreaker; using Agent.Sdk.Knob; namespace Microsoft.VisualStudio.Services.Agent.Worker @@ -35,10 +34,6 @@ public interface IStepsRunner : IAgentService public sealed class StepsRunner : AgentService, IStepsRunner { - private IJobServerQueue _jobServerQueue; - - private IJobServerQueue JobServerQueue => _jobServerQueue ??= HostContext.GetService(); - // StepsRunner should never throw exception to caller public async Task RunAsync(IExecutionContext jobContext, IList steps) { @@ -309,21 +304,6 @@ private async Task RunStepAsync(IStep step, CancellationToken jobCancellationTok // Complete the step context. step.ExecutionContext.Section(StringUtil.Loc("StepFinishing", step.DisplayName)); step.ExecutionContext.Complete(); - - if (!AgentKnobs.DisableDrainQueuesAfterTask.GetValue(step.ExecutionContext).AsBoolean()) - { - try - { - // We need to drain the queues after a task just in case if - // there are a lot of items since it can cause some UI hangs. - await JobServerQueue.DrainQueues(); - } - catch (Exception ex) - { - Trace.Error($"Error has occurred while draining queues, it can cause some UI glitches but it doesn't affect a pipeline execution itself: {ex}"); - step.ExecutionContext.Error(ex); - } - } } private async Task SwitchToUtf8Codepage(IStep step) diff --git a/src/Agent.Worker/TaskCommandExtension.cs b/src/Agent.Worker/TaskCommandExtension.cs index f165f8e2b1..8411b02513 100644 --- a/src/Agent.Worker/TaskCommandExtension.cs +++ b/src/Agent.Worker/TaskCommandExtension.cs @@ -36,6 +36,24 @@ public TaskCommandExtension() } } + public static class TaskCommandHelper + { + public static void AddSecret(IExecutionContext context, string value, string origin) + { + if (!string.IsNullOrEmpty(value)) + { + context.GetHostContext().SecretMasker.AddValue(value, origin); + + // if DECODE_PERCENTS = false then we need to add decoded value as a secret as well to prevent its exposion in logs + var unescapePercents = AgentKnobs.DecodePercents.GetValue(context).AsBoolean(); + if (!unescapePercents) + { + context.GetHostContext().SecretMasker.AddValue(CommandStringConvertor.Unescape(value, true), origin); + } + } + } + } + [CommandRestriction(AllowedInRestrictedMode = true)] public sealed class TaskDetailCommand : IWorkerCommand { @@ -536,11 +554,7 @@ public void Execute(IExecutionContext context, Command command) ArgUtil.NotNull(context, nameof(context)); ArgUtil.NotNull(command, nameof(command)); - var data = command.Data; - if (!string.IsNullOrEmpty(data)) - { - context.GetHostContext().SecretMasker.AddValue(data, WellKnownSecretAliases.TaskSetSecretCommand); - } + TaskCommandHelper.AddSecret(context, command.Data, WellKnownSecretAliases.TaskSetSecretCommand); } } @@ -611,7 +625,7 @@ public void Execute(IExecutionContext context, Command command) var unescapePercents = AgentKnobs.DecodePercents.GetValue(context).AsBoolean(); var commandEscapeData = CommandStringConvertor.Escape(command.Data, unescapePercents); - context.GetHostContext().SecretMasker.AddValue(commandEscapeData, WellKnownSecretAliases.TaskSetVariableCommand); + TaskCommandHelper.AddSecret(context, commandEscapeData, WellKnownSecretAliases.TaskSetVariableCommand); } var checker = context.GetHostContext().GetService(); @@ -727,7 +741,7 @@ public void Execute(IExecutionContext context, Command command) // Mask auth parameter data upfront to avoid accidental secret exposure by invalid endpoint/key/data if (String.Equals(field, "authParameter", StringComparison.OrdinalIgnoreCase)) { - context.GetHostContext().SecretMasker.AddValue(data, WellKnownSecretAliases.TaskSetEndpointCommandAuthParameter); + TaskCommandHelper.AddSecret(context, data, WellKnownSecretAliases.TaskSetEndpointCommandAuthParameter); } String endpointIdInput; diff --git a/src/Agent.Worker/TestResults/Utils/TestResultUtils.cs b/src/Agent.Worker/TestResults/Utils/TestResultUtils.cs index 50c6e4543c..5794409a42 100644 --- a/src/Agent.Worker/TestResults/Utils/TestResultUtils.cs +++ b/src/Agent.Worker/TestResults/Utils/TestResultUtils.cs @@ -1,15 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using Agent.Sdk.Knob; using System; -using Microsoft.TeamFoundation.TestClient.PublishTestResults; -using Microsoft.TeamFoundation.TestManagement.WebApi; -using Microsoft.VisualStudio.Services.WebApi; -using Microsoft.TeamFoundation.Core.WebApi; -using System.Linq; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; using Newtonsoft.Json; using Newtonsoft.Json.Serialization; @@ -19,6 +12,11 @@ internal static class TestResultUtils { public static void StoreTestRunSummaryInEnvVar(IExecutionContext executionContext, TestRunSummary testRunSummary, string testRunner, string name, string description = "") { + if (AgentKnobs.DisableTestsMetadata.GetValue(executionContext).AsBoolean()) + { + return; + } + try { string metadata = GetEvidenceStoreMetadata(executionContext, testRunSummary, testRunner, name, description); diff --git a/src/Agent.Worker/Worker.cs b/src/Agent.Worker/Worker.cs index d6882fe964..520c3dfe95 100644 --- a/src/Agent.Worker/Worker.cs +++ b/src/Agent.Worker/Worker.cs @@ -138,6 +138,10 @@ private void AddUserSuppliedSecret(String secret) HostContext.SecretMasker.AddValue(secret.Trim(quoteChar), WellKnownSecretAliases.UserSuppliedSecret); } } + + // Here we add a trimmed secret value to the dictionary in case of a possible leak through external tools. + var trimChars = new char[] { '\r', '\n', ' ' }; + HostContext.SecretMasker.AddValue(secret.Trim(trimChars), WellKnownSecretAliases.UserSuppliedSecret); } private void InitializeSecretMasker(Pipelines.AgentJobRequestMessage message) diff --git a/src/Agent.Worker/WorkerCommandManager.cs b/src/Agent.Worker/WorkerCommandManager.cs index e88061a0ae..7bd1c341f0 100644 --- a/src/Agent.Worker/WorkerCommandManager.cs +++ b/src/Agent.Worker/WorkerCommandManager.cs @@ -9,6 +9,7 @@ using System.Linq; using System.Net.Sockets; using System.Collections.Generic; +using Agent.Sdk; namespace Microsoft.VisualStudio.Services.Agent.Worker { @@ -128,7 +129,7 @@ public bool TryProcessCommand(IExecutionContext context, string input) if (!(string.Equals(command.Area, "task", StringComparison.OrdinalIgnoreCase) && string.Equals(command.Event, "debug", StringComparison.OrdinalIgnoreCase))) { - context.Debug($"Processed: {input}"); + context.Debug($"Processed: {CommandStringConvertor.Unescape(input, unescapePercents)}"); } } } diff --git a/src/Microsoft.VisualStudio.Services.Agent/JobServerQueue.cs b/src/Microsoft.VisualStudio.Services.Agent/JobServerQueue.cs index 1571f152b1..8720ec4b57 100644 --- a/src/Microsoft.VisualStudio.Services.Agent/JobServerQueue.cs +++ b/src/Microsoft.VisualStudio.Services.Agent/JobServerQueue.cs @@ -18,9 +18,10 @@ namespace Microsoft.VisualStudio.Services.Agent [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Naming", "CA1711: Identifiers should not have incorrect suffix")] public interface IJobServerQueue : IAgentService, IThrottlingReporter { + bool ForceDrainWebConsoleQueue { get; set; } + bool ForceDrainTimelineQueue { get; set; } event EventHandler JobServerQueueThrottling; Task ShutdownAsync(); - Task DrainQueues(); void Start(Pipelines.AgentJobRequestMessage jobRequest); void QueueWebConsoleLine(Guid stepRecordId, string line, long lineNumber); void QueueFileUpload(Guid timelineId, Guid timelineRecordId, string type, string name, string path, bool deleteSource); @@ -86,34 +87,15 @@ public sealed class JobServerQueue : AgentService, IJobServerQueue private bool _writeToBlobStoreAttachments = false; private bool _debugMode = false; + public bool ForceDrainWebConsoleQueue { get; set; } + public bool ForceDrainTimelineQueue { get; set; } + public override void Initialize(IHostContext hostContext) { base.Initialize(hostContext); _jobServer = hostContext.GetService(); } - public async Task DrainQueues() - { - // Drain the queue - // ProcessWebConsoleLinesQueueAsync() will never throw exception, live console update is always best effort. - Trace.Verbose("Draining web console line queue."); - await ProcessWebConsoleLinesQueueAsync(runOnce: true); - Trace.Info("Web console line queue drained."); - - // ProcessFilesUploadQueueAsync() will never throw exception, log file upload is always best effort. - Trace.Verbose("Draining file upload queue."); - await ProcessFilesUploadQueueAsync(runOnce: true); - Trace.Info("File upload queue drained."); - - // ProcessTimelinesUpdateQueueAsync() will throw exception during shutdown - // if there is any timeline records that failed to update contains output variabls. - Trace.Verbose("Draining timeline update queue."); - await ProcessTimelinesUpdateQueueAsync(runOnce: true); - Trace.Info("Timeline update queue drained."); - - Trace.Info("All queues are drained."); - } - public void Start(Pipelines.AgentJobRequestMessage jobRequest) { Trace.Entering(); @@ -192,7 +174,24 @@ public async Task ShutdownAsync() _queueInProcess = false; Trace.Info("All queue process task stopped."); - await DrainQueues(); + // Drain the queue + // ProcessWebConsoleLinesQueueAsync() will never throw exception, live console update is always best effort. + Trace.Verbose("Draining web console line queue."); + await ProcessWebConsoleLinesQueueAsync(runOnce: true); + Trace.Info("Web console line queue drained."); + + // ProcessFilesUploadQueueAsync() will never throw exception, log file upload is always best effort. + Trace.Verbose("Draining file upload queue."); + await ProcessFilesUploadQueueAsync(runOnce: true); + Trace.Info("File upload queue drained."); + + // ProcessTimelinesUpdateQueueAsync() will throw exception during shutdown + // if there is any timeline records that failed to update contains output variables. + Trace.Verbose("Draining timeline update queue."); + await ProcessTimelinesUpdateQueueAsync(runOnce: true); + Trace.Info("Timeline update queue drained."); + + Trace.Info("All queue process tasks have been stopped, and all queues are drained."); } public void QueueWebConsoleLine(Guid stepRecordId, string line, long lineNumber) @@ -248,6 +247,12 @@ private async Task ProcessWebConsoleLinesQueueAsync(bool runOnce = false) { while (!_jobCompletionSource.Task.IsCompleted || runOnce) { + bool shouldDrain = ForceDrainWebConsoleQueue; + if (ForceDrainWebConsoleQueue) + { + ForceDrainWebConsoleQueue = false; + } + if (_webConsoleLineAggressiveDequeue && ++_webConsoleLineAggressiveDequeueCount > _webConsoleLineAggressiveDequeueLimit) { Trace.Info("Stop aggressive process web console line queue."); @@ -279,7 +284,7 @@ private async Task ProcessWebConsoleLinesQueueAsync(bool runOnce = false) // process at most about 500 lines of web console line during regular timer dequeue task. // Send the first line of output to the customer right away // It might take a while to reach 500 line outputs, which would cause delays before customers see the first line - if ((!runOnce && linesCounter > 500) || _firstConsoleOutputs) + if ((!runOnce && !shouldDrain && linesCounter > 500) || _firstConsoleOutputs) { break; } @@ -314,7 +319,7 @@ private async Task ProcessWebConsoleLinesQueueAsync(bool runOnce = false) // We batch and produce 500 lines of web console output every 500ms // If customer's task produce massive of outputs, then the last queue drain run might take forever. // So we will only upload the last 200 lines of each step from all buffered web console lines. - if (runOnce && batchedLines.Count > 2) + if ((runOnce || shouldDrain) && batchedLines.Count > 2) { Trace.Info($"Skip {batchedLines.Count - 2} batches web console lines for last run"); batchedLines = batchedLines.TakeLast(2).ToList(); @@ -430,6 +435,8 @@ private async Task ProcessTimelinesUpdateQueueAsync(bool runOnce = false) { while (!_jobCompletionSource.Task.IsCompleted || runOnce) { + bool shouldDrain = ForceDrainTimelineQueue; + List pendingUpdates = new List(); foreach (var timeline in _allTimelines) { @@ -442,7 +449,7 @@ private async Task ProcessTimelinesUpdateQueueAsync(bool runOnce = false) { records.Add(record); // process at most 25 timeline records update for each timeline. - if (!runOnce && records.Count > 25) + if (!runOnce && !shouldDrain && records.Count > 25) { break; } @@ -514,7 +521,7 @@ private async Task ProcessTimelinesUpdateQueueAsync(bool runOnce = false) } } - if (runOnce) + if (runOnce || shouldDrain) { // continue process timeline records update, // we might have more records need update, @@ -535,14 +542,19 @@ private async Task ProcessTimelinesUpdateQueueAsync(bool runOnce = false) } else { - break; + if (ForceDrainTimelineQueue) + { + ForceDrainTimelineQueue = false; + } + if (runOnce) + { + break; + } } } } - else - { - await Task.Delay(_delayForTimelineUpdateDequeue); - } + + await Task.Delay(_delayForTimelineUpdateDequeue); } } diff --git a/src/Misc/externals.sh b/src/Misc/externals.sh index beb2df95ec..36c6235bc8 100644 --- a/src/Misc/externals.sh +++ b/src/Misc/externals.sh @@ -12,7 +12,7 @@ NODE_VERSION="6.17.1" NODE10_VERSION="10.24.1" NODE16_VERSION="16.17.1" MINGIT_VERSION="2.39.1" -LFS_VERSION="2.13.3" +LFS_VERSION="3.3.0" get_abs_path() { # exploits the fact that pwd will print abs path when no args diff --git a/src/Misc/layoutbin/en-US/strings.json b/src/Misc/layoutbin/en-US/strings.json index abda63db65..9e87a3891e 100644 --- a/src/Misc/layoutbin/en-US/strings.json +++ b/src/Misc/layoutbin/en-US/strings.json @@ -285,10 +285,10 @@ "EnvironmentNotFound": "Environment not found: '{0}'", "EnvironmentVariableExceedsMaximumLength": "Environment variable '{0}' exceeds the maximum supported length. Environment variable length: {1} , Maximum supported length: {2}", "EnvironmentVMResourceTags": "Comma separated list of tags (e.g web, db)", - "ErrorDuringBuildGC": "Unable discover garbage based on '{0}'. Try it next time.", - "ErrorDuringBuildGCDelete": "Unable finish GC based on '{0}'. Try it next time.", - "ErrorDuringReleaseGC": "Unable discover garbage based on '{0}'. Try it next time.", - "ErrorDuringReleaseGCDelete": "Unable finish GC based on '{0}'. Try it next time.", + "ErrorDuringBuildGC": "Unable to discover garbage based on '{0}'. Try it next time.", + "ErrorDuringBuildGCDelete": "Unable to finish GC based on '{0}'. Try it next time.", + "ErrorDuringReleaseGC": "Unable to discover garbage based on '{0}'. Try it next time.", + "ErrorDuringReleaseGCDelete": "Unable to finish GC based on '{0}'. Try it next time.", "ErrorOccurred": "An error occurred: {0}", "ErrorOccurredWhilePublishingCCFiles": "Error occurred while publishing code coverage files. Error: {0}", "EulasSectionHeader": "End User License Agreements", @@ -403,8 +403,9 @@ "MinRequiredDockerServerVersion": "Min required docker engine API server version is '{0}', your docker ('{1}') server version is '{2}'", "MinRequiredGitLfsVersion": "Min required git-lfs version is '{0}', your git-lfs ('{1}') version is '{2}'", "MinRequiredGitVersion": "Min required git version is '{0}', your git ('{1}') version is '{2}'", + "MinSecretsLengtLimitWarning": "The value of the minimum length of the secrets is too high. Maximum value is set: {0}", "MissingAgent": "The agent no longer exists on the server. Please reconfigure the agent.", - "MissingAttachmentFile": "Cannot upload task attachment file, attachment file location is not specified or attachment file not exist on disk", + "MissingAttachmentFile": "Cannot upload task attachment file, attachment file location is not specified or attachment file does not exist on disk.", "MissingAttachmentName": "Can't add task attachment, attachment name is not provided.", "MissingAttachmentType": "Can't add task attachment, attachment type is not provided.", "MissingConfig": "Cannot connect to server, because config files are missing. Skipping removing agent from the server.", @@ -594,7 +595,7 @@ "ShallowLfsFetchFail": "Git lfs fetch failed on shallow repository, this might because of git fetch with depth '{0}' doesn't include the lfs fetch commit '{1}'. Please reference documentation (http://go.microsoft.com/fwlink/?LinkId=829603)", "ShutdownMessage": "Restarting the machine in order to launch agent in interactive mode.", "Skipping": "Does not exist. Skipping ", - "SkipTrackingFileWithoutRepoType": "Skip tracking file '{0}', repository type haven't been updated yet.", + "SkipTrackingFileWithoutRepoType": "Skip tracking file '{0}', repository type hasn't been updated yet.", "SourceArtifactProviderNotFound": "Cannot find source provider for artifact of type {0}", "StartingArtifactDownload": "Starting download {0}", "StartMaintenance": "Start maintenance: {0}", diff --git a/src/Test/L0/SecretMaskerTests/LoggedSecretMaskerL0.cs b/src/Test/L0/SecretMaskerTests/LoggedSecretMaskerL0.cs index 982c93bba2..364660c335 100644 --- a/src/Test/L0/SecretMaskerTests/LoggedSecretMaskerL0.cs +++ b/src/Test/L0/SecretMaskerTests/LoggedSecretMaskerL0.cs @@ -58,41 +58,47 @@ public void LoggedSecretMasker_ShortSecret_Removes_From_Dictionary_BoundaryValue { var lsm = new LoggedSecretMasker(_secretMasker) { - MinSecretLength = 3 + MinSecretLength = LoggedSecretMasker.MinSecretLengthLimit }; - var inputMessage = "123456"; + var inputMessage = "1234567"; - lsm.AddValue("123"); - lsm.RemoveShortSecretsFromDictionary(); + lsm.AddValue("12345"); var resultMessage = lsm.MaskSecrets(inputMessage); - Assert.Equal("***456", resultMessage); + Assert.Equal("1234567", resultMessage); } [Fact] [Trait("Level", "L0")] [Trait("Category", "SecretMasker")] - public void LoggedSecretMasker_Skipping_ShortSecrets() + public void LoggedSecretMasker_ShortSecret_Removes_From_Dictionary_BoundaryValue2() { var lsm = new LoggedSecretMasker(_secretMasker) { - MinSecretLength = 3 + MinSecretLength = LoggedSecretMasker.MinSecretLengthLimit }; + var inputMessage = "1234567"; - lsm.AddValue("1"); - var resultMessage = lsm.MaskSecrets(@"123"); + lsm.AddValue("123456"); + var resultMessage = lsm.MaskSecrets(inputMessage); - Assert.Equal("123", resultMessage); + Assert.Equal("***7", resultMessage); } [Fact] [Trait("Level", "L0")] [Trait("Category", "SecretMasker")] - public void LoggedSecretMasker_Throws_Exception_If_Large_MinSecretLength_Specified() + public void LoggedSecretMasker_Skipping_ShortSecrets() { - var lsm = new LoggedSecretMasker(_secretMasker); + var lsm = new LoggedSecretMasker(_secretMasker) + { + MinSecretLength = 3 + }; - Assert.Throws(() => lsm.MinSecretLength = 5); + lsm.AddValue("1"); + var resultMessage = lsm.MaskSecrets(@"123"); + + Assert.Equal("123", resultMessage); } [Fact] @@ -101,11 +107,11 @@ public void LoggedSecretMasker_Throws_Exception_If_Large_MinSecretLength_Specifi public void LoggedSecretMasker_Sets_MinSecretLength_To_MaxValue() { var lsm = new LoggedSecretMasker(_secretMasker); + var expectedMinSecretsLengthValue = LoggedSecretMasker.MinSecretLengthLimit; - try { lsm.MinSecretLength = 5; } - catch (ArgumentException) { } + lsm.MinSecretLength = LoggedSecretMasker.MinSecretLengthLimit + 1; - Assert.Equal(4, lsm.MinSecretLength); + Assert.Equal(expectedMinSecretsLengthValue, lsm.MinSecretLength); } [Fact] diff --git a/src/Test/L0/Worker/TaskCommandExtensionL0.cs b/src/Test/L0/Worker/TaskCommandExtensionL0.cs index 8b96b8e5e6..195534601e 100644 --- a/src/Test/L0/Worker/TaskCommandExtensionL0.cs +++ b/src/Test/L0/Worker/TaskCommandExtensionL0.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Runtime.CompilerServices; +using Agent.Sdk; using Microsoft.TeamFoundation.DistributedTask.WebApi; using Microsoft.VisualStudio.Services.Agent.Worker; using Moq; @@ -200,6 +201,7 @@ private TestHostContext SetupMocks([CallerMemberName] string name = "") _ec.Setup(x => x.Endpoints).Returns(new List { _endpoint }); _ec.Setup(x => x.GetHostContext()).Returns(_hc); + _ec.Setup(x => x.GetScopedEnvironment()).Returns(new SystemEnvironment()); return _hc; } diff --git a/src/agentversion b/src/agentversion index 8a1b952288..9ca90a026f 100644 --- a/src/agentversion +++ b/src/agentversion @@ -1 +1 @@ -3.212.0 \ No newline at end of file +3.220.0 \ No newline at end of file