diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 16234ce0ffd..449de6a708d 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -157,25 +157,25 @@ - + https://github.com/dotnet/arcade - 9cb7d80460fcd74ce5b31633d2deb32a93fd30de + 38aa367e014493c6f6ebfe94d9029bea01d072c4 - + https://github.com/dotnet/arcade - 9cb7d80460fcd74ce5b31633d2deb32a93fd30de + 38aa367e014493c6f6ebfe94d9029bea01d072c4 - + https://github.com/dotnet/arcade - 9cb7d80460fcd74ce5b31633d2deb32a93fd30de + 38aa367e014493c6f6ebfe94d9029bea01d072c4 - + https://github.com/dotnet/arcade - 9cb7d80460fcd74ce5b31633d2deb32a93fd30de + 38aa367e014493c6f6ebfe94d9029bea01d072c4 - + https://github.com/dotnet/arcade - 9cb7d80460fcd74ce5b31633d2deb32a93fd30de + 38aa367e014493c6f6ebfe94d9029bea01d072c4 https://github.com/dotnet/sourcelink diff --git a/eng/Versions.props b/eng/Versions.props index 7b2d5318690..8ead0814b5d 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -48,8 +48,8 @@ - 9.0.0-beta.23580.7 - 9.0.0-beta.23580.7 + 9.0.0-beta.23601.1 + 9.0.0-beta.23601.1 diff --git a/eng/common/post-build/redact-logs.ps1 b/eng/common/post-build/redact-logs.ps1 new file mode 100644 index 00000000000..82d91f6fd02 --- /dev/null +++ b/eng/common/post-build/redact-logs.ps1 @@ -0,0 +1,81 @@ +[CmdletBinding(PositionalBinding=$False)] +param( + [Parameter(Mandatory=$true, Position=0)][string] $InputPath, + [Parameter(Mandatory=$true)][string] $BinlogToolVersion, + [Parameter(Mandatory=$false)][string] $DotnetPath, + [Parameter(Mandatory=$false)][string] $PackageFeed = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json', + # File with strings to redact - separated by newlines. + # For comments start the line with '# ' - such lines are ignored + [Parameter(Mandatory=$false)][string] $TokensFilePath, + [Parameter(ValueFromRemainingArguments=$true)][String[]]$TokensToRedact +) + +try { + . $PSScriptRoot\post-build-utils.ps1 + + $packageName = 'binlogtool' + + $dotnet = $DotnetPath + + if (!$dotnet) { + $dotnetRoot = InitializeDotNetCli -install:$true + $dotnet = "$dotnetRoot\dotnet.exe" + } + + $toolList = & "$dotnet" tool list -g + + if ($toolList -like "*$packageName*") { + & "$dotnet" tool uninstall $packageName -g + } + + $toolPath = "$PSScriptRoot\..\..\..\.tools" + $verbosity = 'minimal' + + New-Item -ItemType Directory -Force -Path $toolPath + + Push-Location -Path $toolPath + + try { + Write-Host "Installing Binlog redactor CLI..." + Write-Host "'$dotnet' new tool-manifest" + & "$dotnet" new tool-manifest + Write-Host "'$dotnet' tool install $packageName --local --add-source '$PackageFeed' -v $verbosity --version $BinlogToolVersion" + & "$dotnet" tool install $packageName --local --add-source "$PackageFeed" -v $verbosity --version $BinlogToolVersion + + if (Test-Path $TokensFilePath) { + Write-Host "Adding additional sensitive data for redaction from file: " $TokensFilePath + $TokensToRedact += Get-Content -Path $TokensFilePath | Foreach {$_.Trim()} | Where { $_ -notmatch "^# " } + } + + $optionalParams = [System.Collections.ArrayList]::new() + + Foreach ($p in $TokensToRedact) + { + if($p -match '^\$\(.*\)$') + { + Write-Host ("Ignoring token {0} as it is probably unexpanded AzDO variable" -f $p) + } + elseif($p) + { + $optionalParams.Add("-p:" + $p) | Out-Null + } + } + + & $dotnet binlogtool redact --input:$InputPath --recurse --in-place ` + @optionalParams + + if ($LastExitCode -ne 0) { + Write-PipelineTelemetryError -Category 'Redactor' -Type 'warning' -Message "Problems using Redactor tool (exit code: $LastExitCode). But ignoring them now." + } + } + finally { + Pop-Location + } + + Write-Host 'done.' +} +catch { + Write-Host $_ + Write-PipelineTelemetryError -Category 'Redactor' -Message "There was an error while trying to redact logs. Error: $_" + ExitWithExitCode 1 +} diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index bb0d9f8b0f1..3b25fd97923 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -48,8 +48,8 @@ jobs: - group: AzureDevOps-Artifact-Feeds-Pats - name: runCodesignValidationInjection value: false - - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: - - template: /eng/common/templates/post-build/common-variables.yml + # unconditional - needed for logs publishing (redactor tool version) + - template: /eng/common/templates/post-build/common-variables.yml pool: # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index 173914f2364..4ef7bd271fb 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -17,6 +17,8 @@ variables: value: 3.0.0 - name: SymbolToolVersion value: 1.0.1 + - name: BinlogToolVersion + value: 1.0.8 - name: runCodesignValidationInjection value: false diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index ef720f9d781..d64236b28cf 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -187,6 +187,7 @@ stages: parameters: StageLabel: 'Validation' JobLabel: 'Signing' + BinlogToolVersion: $(BinlogToolVersion) - job: displayName: SourceLink Validation diff --git a/eng/common/templates/steps/publish-logs.yml b/eng/common/templates/steps/publish-logs.yml index 88f238f36bf..dadf1c464cd 100644 --- a/eng/common/templates/steps/publish-logs.yml +++ b/eng/common/templates/steps/publish-logs.yml @@ -1,6 +1,9 @@ parameters: StageLabel: '' JobLabel: '' + CustomSensitiveDataList: '' + # A default - in case value from eng/common/templates/post-build/common-variables.yml is not passed + BinlogToolVersion: '1.0.8' steps: - task: Powershell@2 @@ -12,7 +15,30 @@ steps: Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ continueOnError: true condition: always() - + +- task: PowerShell@2 + displayName: Redact Logs + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/redact-logs.ps1 + # For now this needs to have explicit list of all sensitive data. Taken from eng/publishing/v3/publish.yml + # Sensitive data can as well be added to $(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' + # If the file exists - sensitive data for redaction will be sourced from it + # (single entry per line, lines starting with '# ' are considered comments and skipped) + arguments: -InputPath '$(Build.SourcesDirectory)/PostBuildLogs' + -BinlogToolVersion ${{parameters.BinlogToolVersion}} + -TokensFilePath '$(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' + '$(publishing-dnceng-devdiv-code-r-build-re)' + '$(MaestroAccessToken)' + '$(dn-bot-all-orgs-artifact-feeds-rw)' + '$(akams-client-id)' + '$(akams-client-secret)' + '$(microsoft-symbol-server-pat)' + '$(symweb-symbol-server-pat)' + '$(dn-bot-all-orgs-build-rw-code-rw)' + ${{parameters.CustomSensitiveDataList}} + continueOnError: true + condition: always() + - task: PublishBuildArtifacts@1 displayName: Publish Logs inputs: diff --git a/global.json b/global.json index b9166922a52..dda727180c5 100644 --- a/global.json +++ b/global.json @@ -12,8 +12,8 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23580.7", - "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.23580.7" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23601.1", + "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.23601.1" }, "sdk": { "version": "9.0.100-alpha.1.23511.2"