Use storage upload task for non-token auth; add test mode (#1225)

"az" doesn't contain a tool that seems to support non-SAS-token-based auth. It will authenticate broadly using the service connection, but actual storage copies are performed by grabbing the access key. So, use AzureFileCopy@6, which explicitly supports what we need to do. This task only runs on Windows, so switch the publish stages over from Linux. Add "publishExistingRunID" parameter for test runs. This has been done previously as a hard-coded temporary change, but publish is complex enough now to warrant a more usable and permanent debug feature.
microsoft · May 14, 2024 · 7262f0e · 7262f0e
1 parent 1735d31
commit 7262f0e
Show file tree

Hide file tree

Showing 4 changed files with 132 additions and 75 deletions.
diff --git a/eng/pipeline/rolling-internal-pipeline.yml b/eng/pipeline/rolling-internal-pipeline.yml
@@ -20,6 +20,11 @@ parameters:
     type: string
     default: nil
 
+  - name: publishExistingRunID
+    displayName: 'For debugging publish steps: skip building, and instead publish the artifacts from an existing run. Leave "nil" otherwise.'
+    type: string
+    default: nil
+
 variables:
   - template: variables/pool-providers.yml
   # MicroBuild configuration.
@@ -61,4 +66,5 @@ extends:
           createSourceArchive: true
           createSymbols: true
           publish: true
+          publishExistingRunID: ${{ parameters.publishExistingRunID }}
           releaseVersion: ${{ parameters.releaseVersion }}
diff --git a/eng/pipeline/stages/builders-to-stages.yml b/eng/pipeline/stages/builders-to-stages.yml
@@ -11,6 +11,8 @@ parameters:
   sign: false
   # If true, publish build artifacts to blob storage.
   publish: false
+  # If changed to specify an existing pipeline run, skip build/sign and publish the existing run.
+  publishExistingRunID: 'nil'
   # If true, the stage will run in 1ES PT Official template compatibility mode.
   official: false
   # If true, generate source archive tarballs.
@@ -20,36 +22,37 @@ parameters:
   releaseVersion: 'nil'
 
 stages:
-  - ${{ each builder in parameters.builders }}:
-    - template: pool.yml
-      parameters:
-        inner:
-          template: run-stage.yml
-          parameters:
-            builder: ${{ builder }}
-            createSourceArchive: ${{ parameters.createSourceArchive }}
-            releaseVersion: ${{ parameters.releaseVersion }}
-            official: ${{ parameters.official }}
-            createSymbols: ${{ parameters.createSymbols }}
-            # Attempt to retry the build on Windows to mitigate flakiness:
-            # "Access Denied" during EXE copying and general flakiness during tests.
-            ${{ if eq(builder.os, 'windows') }}:
-              retryAttempts: [1, 2, 3, 4, "FINAL"]
+  - ${{ if eq(parameters.publishExistingRunID, 'nil') }}:
+    - ${{ each builder in parameters.builders }}:
+      - template: pool.yml
+        parameters:
+          inner:
+            template: run-stage.yml
+            parameters:
+              builder: ${{ builder }}
+              createSourceArchive: ${{ parameters.createSourceArchive }}
+              releaseVersion: ${{ parameters.releaseVersion }}
+              official: ${{ parameters.official }}
+              createSymbols: ${{ parameters.createSymbols }}
+              # Attempt to retry the build on Windows to mitigate flakiness:
+              # "Access Denied" during EXE copying and general flakiness during tests.
+              ${{ if eq(builder.os, 'windows') }}:
+                retryAttempts: [1, 2, 3, 4, "FINAL"]
 
-  - ${{ if eq(parameters.sign, true) }}:
-    - template: pool.yml
-      parameters:
-        inner:
-          template: sign-stage.yml
-          parameters:
-            # This is not a builder, but provide partial builder info for agent selection.
-            builder: { os: windows, arch: amd64 }
-            official: ${{ parameters.official }}
-            # The list of builders to depend on and grab artifacts from.
-            builders:
-              - ${{ each builder in parameters.builders }}:
-                - ${{ if eq(builder.config, 'buildandpack') }}:
-                  - ${{ builder }}
+    - ${{ if eq(parameters.sign, true) }}:
+      - template: pool.yml
+        parameters:
+          inner:
+            template: sign-stage.yml
+            parameters:
+              # This is not a builder, but provide partial builder info for agent selection.
+              builder: { os: windows, arch: amd64 }
+              official: ${{ parameters.official }}
+              # The list of builders to depend on and grab artifacts from.
+              builders:
+                - ${{ each builder in parameters.builders }}:
+                  - ${{ if eq(builder.config, 'buildandpack') }}:
+                    - ${{ builder }}
 
   - ${{ if eq(parameters.publish, true) }}:
     - ${{ if not(startsWith(variables['Build.SourceBranch'], 'refs/heads/internal/')) }}:
@@ -59,20 +62,22 @@ stages:
             template: publish-stage.yml
             parameters:
               # This is not a builder, but provide partial builder info for agent selection.
-              builder: { os: linux, arch: amd64 }
+              builder: { os: windows, arch: amd64 }
               official: true
               public: true
+              publishExistingRunID: ${{ parameters.publishExistingRunID }}
 
     - template: pool.yml
       parameters:
         inner:
           template: publish-stage.yml
           parameters:
-            builder: { os: linux, arch: amd64 }
+            builder: { os: windows, arch: amd64 }
             official: true
             public: false
             builders:
               - ${{ each builder in parameters.builders }}:
                 - ${{ if eq(builder.config, 'buildandpack') }}:
                   - ${{ builder }}
             publishSymbols: ${{ parameters.createSymbols }}
+            publishExistingRunID: ${{ parameters.publishExistingRunID }}
diff --git a/eng/pipeline/stages/go-builder-matrix-stages.yml b/eng/pipeline/stages/go-builder-matrix-stages.yml
@@ -43,6 +43,9 @@ parameters:
   - name: releaseVersion
     type: string
     default: 'nil'
+  - name: publishExistingRunID
+    type: string
+    default: 'nil'
 
 stages:
   - template: shorthand-builders-to-builders.yml
@@ -52,6 +55,7 @@ stages:
         official: ${{ parameters.official }}
         sign: ${{ parameters.sign }}
         publish: ${{ parameters.publish }}
+        publishExistingRunID: ${{ parameters.publishExistingRunID }}
         createSourceArchive: ${{ parameters.createSourceArchive }}
         createSymbols: ${{ parameters.createSymbols }}
         releaseVersion: ${{ parameters.releaseVersion }}

diff --git a/eng/pipeline/stages/publish-stage.yml b/eng/pipeline/stages/publish-stage.yml
@@ -11,6 +11,10 @@ parameters:
   - name: pool
     type: object
 
+  - name: publishExistingRunID
+    type: string
+    default: 'nil'
+
   # Unused. Declared so pool selection doesn't fail when trying to pass them.
   - name: builder
     type: object
@@ -29,25 +33,25 @@ stages:
       displayName: Publish Public
     ${{ else }}:
       displayName: Publish Internal
-    dependsOn: Sign
+    ${{ if eq(parameters.publishExistingRunID, 'nil') }}:
+      dependsOn: Sign
+    ${{ else }}:
+      dependsOn: []
     jobs:
       - job: Publish
         pool: ${{ parameters.pool }}
 
         variables:
-          - ${{ if parameters.public }}:
-            - name: blobContainer
+          - name: blobContainer
+            ${{ if parameters.public }}:
               value: 'https://dotnetbuildoutput.blob.core.windows.net/golang/microsoft'
-            - name: blobSASArg
-              value: --sas-token '$(dotnetbuildoutput-golang-write-sas-query)'
-          - ${{ else }}:
-            - name: blobContainer
+            ${{ else }}:
               value: 'https://golangartifacts.blob.core.windows.net/microsoft'
-            - name: blobSASArg
-              value: '' # golangartifacts is set up with service connection auth.
 
+          - name: blobPrefix
+            value: '$(PublishBranchAlias)/$(Build.BuildNumber)'
           - name: blobDestinationUrl
-            value: '$(blobContainer)/$(PublishBranchAlias)/$(Build.BuildNumber)'
+            value: '$(blobContainer)/$(blobPrefix)'
 
           - group: go-storage
 
@@ -75,8 +79,7 @@ stages:
                   artifact: SymbolsInternal
 
         steps:
-          - template: ../steps/checkout-unix-task.yml
-          - template: ../steps/init-pwsh-task.yml
+          - template: ../steps/checkout-windows-task.yml
           - template: ../steps/init-submodule-task.yml
 
           - pwsh: |
@@ -102,11 +105,25 @@ stages:
               Write-Host "##vso[task.setvariable variable=PublishBranchAlias;]$branch"
             displayName: Find publish branch alias
 
-          - download: current
-            artifact: Binaries Signed
-            # Filter out manifests added by 1ES pipeline template.
-            patterns: '!_manifest/**'
-            displayName: 'Download: Binaries Signed'
+          - ${{ if eq(parameters.publishExistingRunID, 'nil') }}:
+            - download: current
+              artifact: Binaries Signed
+              # Filter out manifests added by 1ES pipeline template.
+              patterns: '!_manifest/**'
+              displayName: 'Download: Binaries Signed'
+          - ${{ else }}:
+            - task: DownloadPipelineArtifact@2
+              displayName: 'Download: Binaries Signed (Specific)'
+              inputs:
+                buildType: specific
+                project: $(System.TeamProject)
+                definition: $(System.DefinitionId)
+                runVersion: 'specific'
+                runId: ${{ parameters.publishExistingRunID }}
+                artifact: Binaries Signed
+                # Filter out manifests added by 1ES pipeline template.
+                patterns: '!_manifest/**'
+                targetPath: '$(Pipeline.Workspace)/Binaries Signed'
 
           - pwsh: |
               eng/run.ps1 createbuildassetjson `
@@ -117,36 +134,61 @@ stages:
                 -o '$(Pipeline.Workspace)/Binaries Signed/assets.json'
             displayName: 'Create build asset JSON'
 
-          - task: AzureCLI@2
-            displayName: Upload to blob storage
-            inputs:
-              azureSubscription: GoLang
-              scriptType: bash
-              scriptLocation: inlineScript
-              # Send literal '*' to az: it handles the wildcard itself. Az copy only accepts one
-              # "from" argument, so we can't use the shell's wildcard expansion.
-              inlineScript: |
-                az storage copy -s '*' -d '$(blobDestinationUrl)' $(blobSASArg)
-              workingDirectory: '$(Pipeline.Workspace)/Binaries Signed/'
-
-          - script: |
-              echo 'Generated links to artifacts in blob storage:'
-              echo ''
-              for f in *; do
-                echo "$(blobDestinationUrl)/$f"
-              done
-            displayName: Show uploaded URLs
+          - ${{ if parameters.public }}:
+            - task: AzureCLI@2
+              displayName: Upload to blob storage
+              inputs:
+                azureSubscription: GoLang
+                scriptType: bash
+                scriptLocation: inlineScript
+                # Send literal '*' to az: it handles the wildcard itself. Az copy only accepts one
+                # "from" argument, so we can't use the shell's wildcard expansion.
+                inlineScript: |
+                  az storage copy -s '*' -d '$(blobDestinationUrl)' --sas-token '$(dotnetbuildoutput-golang-write-sas-query)'
+                workingDirectory: '$(Pipeline.Workspace)/Binaries Signed/'
+          - ${{ else }}:
+            - task: AzureFileCopy@6
+              displayName: Upload to blob storage
+              inputs:
+                Destination: AzureBlob
+                azureSubscription: GoLang
+                storage: golangartifacts
+                ContainerName: microsoft
+                SourcePath: '$(Pipeline.Workspace)/Binaries Signed/*'
+                BlobPrefix: $(blobPrefix)
+
+          - pwsh: |
+              Write-Host 'Generated links to artifacts in blob storage:'
+              Write-Host ''
+              Get-ChildItem -File -Path '.' | %{
+                Write-Host "$(blobDestinationUrl)/$($_.Name)"
+              }
+            displayName: Show expected uploaded URLs
             workingDirectory: '$(Pipeline.Workspace)/Binaries Signed/'
 
           - ${{ if eq(parameters.publishSymbols, true) }}:
             - ${{ each builder in parameters.builders }}:
-              - download: current
-                artifact: Symbols ${{ builder.id }}
-                # Filter out manifests added by 1ES pipeline template.
-                patterns: '!_manifest/**'
-                displayName: 'Download: Symbols ${{ builder.id }}'
-
-              - powershell: |
+              - ${{ if eq(parameters.publishExistingRunID, 'nil') }}:
+                - download: current
+                  artifact: Symbols ${{ builder.id }}
+                  # Filter out manifests added by 1ES pipeline template.
+                  patterns: '!_manifest/**'
+                  displayName: 'Download: Symbols ${{ builder.id }}'
+              - ${{ else }}:
+                - task: DownloadPipelineArtifact@2
+                  displayName: 'Download: Symbols ${{ builder.id }} (Specific)'
+                  inputs:
+                    buildType: specific
+                    project: $(System.TeamProject)
+                    definition: $(System.DefinitionId)
+                    runVersion: 'specific'
+                    runId: ${{ parameters.publishExistingRunID }}
+                    artifact: Symbols ${{ builder.id }}
+                    # Filter out manifests added by 1ES pipeline template.
+                    patterns: '!_manifest/**'
+                    targetPath: '$(Pipeline.Workspace)/Symbols ${{ builder.id }}'
+
+              - pwsh: |
                   $flatDir = "$(Pipeline.Workspace)/Symbols"
                   New-Item $flatDir -ItemType Directory -ErrorAction Ignore
 
@@ -158,7 +200,7 @@ stages:
                     }
                     Copy-Item $_.FullName $flatDir
                   }
-                displayName: 'Copy to flat dir: ${{ builder.id }}'
+                displayName: 'Flatten: Symbols ${{ builder.id }}'
                 workingDirectory: '$(Pipeline.Workspace)'
             - task: PublishSymbols@2
               inputs: