From 7262f0e6016f55e9fcaf7f8f0be9a80240eb9129 Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Tue, 14 May 2024 19:33:42 +0200 Subject: [PATCH] Use storage upload task for non-token auth; add test mode (#1225) "az" doesn't contain a tool that seems to support non-SAS-token-based auth. It will authenticate broadly using the service connection, but actual storage copies are performed by grabbing the access key. So, use AzureFileCopy@6, which explicitly supports what we need to do. This task only runs on Windows, so switch the publish stages over from Linux. Add "publishExistingRunID" parameter for test runs. This has been done previously as a hard-coded temporary change, but publish is complex enough now to warrant a more usable and permanent debug feature. --- eng/pipeline/rolling-internal-pipeline.yml | 6 + eng/pipeline/stages/builders-to-stages.yml | 67 ++++----- .../stages/go-builder-matrix-stages.yml | 4 + eng/pipeline/stages/publish-stage.yml | 130 ++++++++++++------ 4 files changed, 132 insertions(+), 75 deletions(-) diff --git a/eng/pipeline/rolling-internal-pipeline.yml b/eng/pipeline/rolling-internal-pipeline.yml index af12e3f1c0..c1da74c2de 100644 --- a/eng/pipeline/rolling-internal-pipeline.yml +++ b/eng/pipeline/rolling-internal-pipeline.yml @@ -20,6 +20,11 @@ parameters: type: string default: nil + - name: publishExistingRunID + displayName: 'For debugging publish steps: skip building, and instead publish the artifacts from an existing run. Leave "nil" otherwise.' + type: string + default: nil + variables: - template: variables/pool-providers.yml # MicroBuild configuration. @@ -61,4 +66,5 @@ extends: createSourceArchive: true createSymbols: true publish: true + publishExistingRunID: ${{ parameters.publishExistingRunID }} releaseVersion: ${{ parameters.releaseVersion }} diff --git a/eng/pipeline/stages/builders-to-stages.yml b/eng/pipeline/stages/builders-to-stages.yml index c126fc3fa2..d81943cc52 100644 --- a/eng/pipeline/stages/builders-to-stages.yml +++ b/eng/pipeline/stages/builders-to-stages.yml @@ -11,6 +11,8 @@ parameters: sign: false # If true, publish build artifacts to blob storage. publish: false + # If changed to specify an existing pipeline run, skip build/sign and publish the existing run. + publishExistingRunID: 'nil' # If true, the stage will run in 1ES PT Official template compatibility mode. official: false # If true, generate source archive tarballs. @@ -20,36 +22,37 @@ parameters: releaseVersion: 'nil' stages: - - ${{ each builder in parameters.builders }}: - - template: pool.yml - parameters: - inner: - template: run-stage.yml - parameters: - builder: ${{ builder }} - createSourceArchive: ${{ parameters.createSourceArchive }} - releaseVersion: ${{ parameters.releaseVersion }} - official: ${{ parameters.official }} - createSymbols: ${{ parameters.createSymbols }} - # Attempt to retry the build on Windows to mitigate flakiness: - # "Access Denied" during EXE copying and general flakiness during tests. - ${{ if eq(builder.os, 'windows') }}: - retryAttempts: [1, 2, 3, 4, "FINAL"] + - ${{ if eq(parameters.publishExistingRunID, 'nil') }}: + - ${{ each builder in parameters.builders }}: + - template: pool.yml + parameters: + inner: + template: run-stage.yml + parameters: + builder: ${{ builder }} + createSourceArchive: ${{ parameters.createSourceArchive }} + releaseVersion: ${{ parameters.releaseVersion }} + official: ${{ parameters.official }} + createSymbols: ${{ parameters.createSymbols }} + # Attempt to retry the build on Windows to mitigate flakiness: + # "Access Denied" during EXE copying and general flakiness during tests. + ${{ if eq(builder.os, 'windows') }}: + retryAttempts: [1, 2, 3, 4, "FINAL"] - - ${{ if eq(parameters.sign, true) }}: - - template: pool.yml - parameters: - inner: - template: sign-stage.yml - parameters: - # This is not a builder, but provide partial builder info for agent selection. - builder: { os: windows, arch: amd64 } - official: ${{ parameters.official }} - # The list of builders to depend on and grab artifacts from. - builders: - - ${{ each builder in parameters.builders }}: - - ${{ if eq(builder.config, 'buildandpack') }}: - - ${{ builder }} + - ${{ if eq(parameters.sign, true) }}: + - template: pool.yml + parameters: + inner: + template: sign-stage.yml + parameters: + # This is not a builder, but provide partial builder info for agent selection. + builder: { os: windows, arch: amd64 } + official: ${{ parameters.official }} + # The list of builders to depend on and grab artifacts from. + builders: + - ${{ each builder in parameters.builders }}: + - ${{ if eq(builder.config, 'buildandpack') }}: + - ${{ builder }} - ${{ if eq(parameters.publish, true) }}: - ${{ if not(startsWith(variables['Build.SourceBranch'], 'refs/heads/internal/')) }}: @@ -59,16 +62,17 @@ stages: template: publish-stage.yml parameters: # This is not a builder, but provide partial builder info for agent selection. - builder: { os: linux, arch: amd64 } + builder: { os: windows, arch: amd64 } official: true public: true + publishExistingRunID: ${{ parameters.publishExistingRunID }} - template: pool.yml parameters: inner: template: publish-stage.yml parameters: - builder: { os: linux, arch: amd64 } + builder: { os: windows, arch: amd64 } official: true public: false builders: @@ -76,3 +80,4 @@ stages: - ${{ if eq(builder.config, 'buildandpack') }}: - ${{ builder }} publishSymbols: ${{ parameters.createSymbols }} + publishExistingRunID: ${{ parameters.publishExistingRunID }} diff --git a/eng/pipeline/stages/go-builder-matrix-stages.yml b/eng/pipeline/stages/go-builder-matrix-stages.yml index d78feef1ee..e525141369 100644 --- a/eng/pipeline/stages/go-builder-matrix-stages.yml +++ b/eng/pipeline/stages/go-builder-matrix-stages.yml @@ -43,6 +43,9 @@ parameters: - name: releaseVersion type: string default: 'nil' + - name: publishExistingRunID + type: string + default: 'nil' stages: - template: shorthand-builders-to-builders.yml @@ -52,6 +55,7 @@ stages: official: ${{ parameters.official }} sign: ${{ parameters.sign }} publish: ${{ parameters.publish }} + publishExistingRunID: ${{ parameters.publishExistingRunID }} createSourceArchive: ${{ parameters.createSourceArchive }} createSymbols: ${{ parameters.createSymbols }} releaseVersion: ${{ parameters.releaseVersion }} diff --git a/eng/pipeline/stages/publish-stage.yml b/eng/pipeline/stages/publish-stage.yml index d4ba3eb299..1f34051884 100644 --- a/eng/pipeline/stages/publish-stage.yml +++ b/eng/pipeline/stages/publish-stage.yml @@ -11,6 +11,10 @@ parameters: - name: pool type: object + - name: publishExistingRunID + type: string + default: 'nil' + # Unused. Declared so pool selection doesn't fail when trying to pass them. - name: builder type: object @@ -29,25 +33,25 @@ stages: displayName: Publish Public ${{ else }}: displayName: Publish Internal - dependsOn: Sign + ${{ if eq(parameters.publishExistingRunID, 'nil') }}: + dependsOn: Sign + ${{ else }}: + dependsOn: [] jobs: - job: Publish pool: ${{ parameters.pool }} variables: - - ${{ if parameters.public }}: - - name: blobContainer + - name: blobContainer + ${{ if parameters.public }}: value: 'https://dotnetbuildoutput.blob.core.windows.net/golang/microsoft' - - name: blobSASArg - value: --sas-token '$(dotnetbuildoutput-golang-write-sas-query)' - - ${{ else }}: - - name: blobContainer + ${{ else }}: value: 'https://golangartifacts.blob.core.windows.net/microsoft' - - name: blobSASArg - value: '' # golangartifacts is set up with service connection auth. + - name: blobPrefix + value: '$(PublishBranchAlias)/$(Build.BuildNumber)' - name: blobDestinationUrl - value: '$(blobContainer)/$(PublishBranchAlias)/$(Build.BuildNumber)' + value: '$(blobContainer)/$(blobPrefix)' - group: go-storage @@ -75,8 +79,7 @@ stages: artifact: SymbolsInternal steps: - - template: ../steps/checkout-unix-task.yml - - template: ../steps/init-pwsh-task.yml + - template: ../steps/checkout-windows-task.yml - template: ../steps/init-submodule-task.yml - pwsh: | @@ -102,11 +105,25 @@ stages: Write-Host "##vso[task.setvariable variable=PublishBranchAlias;]$branch" displayName: Find publish branch alias - - download: current - artifact: Binaries Signed - # Filter out manifests added by 1ES pipeline template. - patterns: '!_manifest/**' - displayName: 'Download: Binaries Signed' + - ${{ if eq(parameters.publishExistingRunID, 'nil') }}: + - download: current + artifact: Binaries Signed + # Filter out manifests added by 1ES pipeline template. + patterns: '!_manifest/**' + displayName: 'Download: Binaries Signed' + - ${{ else }}: + - task: DownloadPipelineArtifact@2 + displayName: 'Download: Binaries Signed (Specific)' + inputs: + buildType: specific + project: $(System.TeamProject) + definition: $(System.DefinitionId) + runVersion: 'specific' + runId: ${{ parameters.publishExistingRunID }} + artifact: Binaries Signed + # Filter out manifests added by 1ES pipeline template. + patterns: '!_manifest/**' + targetPath: '$(Pipeline.Workspace)/Binaries Signed' - pwsh: | eng/run.ps1 createbuildassetjson ` @@ -117,36 +134,61 @@ stages: -o '$(Pipeline.Workspace)/Binaries Signed/assets.json' displayName: 'Create build asset JSON' - - task: AzureCLI@2 - displayName: Upload to blob storage - inputs: - azureSubscription: GoLang - scriptType: bash - scriptLocation: inlineScript - # Send literal '*' to az: it handles the wildcard itself. Az copy only accepts one - # "from" argument, so we can't use the shell's wildcard expansion. - inlineScript: | - az storage copy -s '*' -d '$(blobDestinationUrl)' $(blobSASArg) - workingDirectory: '$(Pipeline.Workspace)/Binaries Signed/' - - - script: | - echo 'Generated links to artifacts in blob storage:' - echo '' - for f in *; do - echo "$(blobDestinationUrl)/$f" - done - displayName: Show uploaded URLs + - ${{ if parameters.public }}: + - task: AzureCLI@2 + displayName: Upload to blob storage + inputs: + azureSubscription: GoLang + scriptType: bash + scriptLocation: inlineScript + # Send literal '*' to az: it handles the wildcard itself. Az copy only accepts one + # "from" argument, so we can't use the shell's wildcard expansion. + inlineScript: | + az storage copy -s '*' -d '$(blobDestinationUrl)' --sas-token '$(dotnetbuildoutput-golang-write-sas-query)' + workingDirectory: '$(Pipeline.Workspace)/Binaries Signed/' + - ${{ else }}: + - task: AzureFileCopy@6 + displayName: Upload to blob storage + inputs: + Destination: AzureBlob + azureSubscription: GoLang + storage: golangartifacts + ContainerName: microsoft + SourcePath: '$(Pipeline.Workspace)/Binaries Signed/*' + BlobPrefix: $(blobPrefix) + + - pwsh: | + Write-Host 'Generated links to artifacts in blob storage:' + Write-Host '' + Get-ChildItem -File -Path '.' | %{ + Write-Host "$(blobDestinationUrl)/$($_.Name)" + } + displayName: Show expected uploaded URLs workingDirectory: '$(Pipeline.Workspace)/Binaries Signed/' - ${{ if eq(parameters.publishSymbols, true) }}: - ${{ each builder in parameters.builders }}: - - download: current - artifact: Symbols ${{ builder.id }} - # Filter out manifests added by 1ES pipeline template. - patterns: '!_manifest/**' - displayName: 'Download: Symbols ${{ builder.id }}' - - - powershell: | + - ${{ if eq(parameters.publishExistingRunID, 'nil') }}: + - download: current + artifact: Symbols ${{ builder.id }} + # Filter out manifests added by 1ES pipeline template. + patterns: '!_manifest/**' + displayName: 'Download: Symbols ${{ builder.id }}' + - ${{ else }}: + - task: DownloadPipelineArtifact@2 + displayName: 'Download: Symbols ${{ builder.id }} (Specific)' + inputs: + buildType: specific + project: $(System.TeamProject) + definition: $(System.DefinitionId) + runVersion: 'specific' + runId: ${{ parameters.publishExistingRunID }} + artifact: Symbols ${{ builder.id }} + # Filter out manifests added by 1ES pipeline template. + patterns: '!_manifest/**' + targetPath: '$(Pipeline.Workspace)/Symbols ${{ builder.id }}' + + - pwsh: | $flatDir = "$(Pipeline.Workspace)/Symbols" New-Item $flatDir -ItemType Directory -ErrorAction Ignore @@ -158,7 +200,7 @@ stages: } Copy-Item $_.FullName $flatDir } - displayName: 'Copy to flat dir: ${{ builder.id }}' + displayName: 'Flatten: Symbols ${{ builder.id }}' workingDirectory: '$(Pipeline.Workspace)' - task: PublishSymbols@2 inputs: