[4.0] eng | Secure symbols publishing

eng/pipelines/common/templates/jobs/build-signed-akv-package-job.yml

@@ -17,6 +17,7 @@ parameters:
 
 jobs:
 - job: build_signed_akv_package
+  displayName: 'Build Signed AKV Provider Package'
   pool:
     type: windows
 
@@ -63,20 +64,11 @@ jobs:
       product: AKV
       referenceType: package
 
-  # Publish symbols to private server
+  # Publish symbols to servers
   - template: ../steps/publish-symbols-step.yml@self
     parameters:
-      SymAccount: $(PrivateSymAccount)
-      referenceType: package
-      symbolsVersion: ${{variables.AKVNuGetPackageVersion }}
-      product: AKV
-      publishSymbols: ${{ parameters['PublishSymbols'] }}
-
-  # Publish symbols to public server
-  - template: ../steps/publish-symbols-step.yml@self
-    parameters:
-      SymAccount: $(PublicSymAccount)
       referenceType: package
       symbolsVersion: ${{variables.AKVNuGetPackageVersion }}
       product: AKV
       publishSymbols: ${{ parameters['PublishSymbols'] }}
+      symbolsArtifactName: akv_symbols_$(System.TeamProject)_$(Build.Repository.Name)_$(Build.SourceBranchName)_$(NuGetPackageVersion)_$(System.TimelineId)

eng/pipelines/common/templates/jobs/build-signed-package-job.yml

@@ -17,6 +17,7 @@ parameters:
 
 jobs:
 - job: build_signed_package
+  displayName: 'Build Signed MDS Package'
   pool:
     type: windows
 
@@ -51,14 +52,8 @@ jobs:
     parameters:
       product: MDS
 
-  # Publish symbols to private server
+  # Publish symbols to servers
   - template: ../steps/publish-symbols-step.yml@self
     parameters:
-      SymAccount: $(PrivateSymAccount)
-      publishSymbols: ${{ parameters['PublishSymbols'] }}
-
-  # Publish symbols to public server
-  - template: ../steps/publish-symbols-step.yml@self
-    parameters:
-      SymAccount: $(PublicSymAccount)
       publishSymbols: ${{ parameters['PublishSymbols'] }}
+      symbolsArtifactName: mds_symbols_$(System.TeamProject)_$(Build.Repository.Name)_$(Build.SourceBranchName)_$(NuGetPackageVersion)_$(System.TimelineId)

eng/pipelines/common/templates/jobs/run-tests-package-reference-job.yml

@@ -19,6 +19,7 @@ parameters:
 
 jobs:
 - job: run_tests_package_reference
+  displayName: 'Run tests with package reference'
   ${{ if ne(parameters.dependsOn, 'empty')}}:
     dependsOn: '${{parameters.dependsOn }}'
   pool:

eng/pipelines/common/templates/jobs/validate-signed-package-job.yml

@@ -39,6 +39,7 @@ parameters:
 
 jobs:
 - job: validate_signed_package
+  displayName: 'Verify signed package'
   ${{ if ne(parameters.dependsOn, '')}}:
     dependsOn: '${{parameters.dependsOn }}'
   pool:

eng/pipelines/common/templates/steps/esrp-code-signing-step.yml

@@ -17,19 +17,37 @@ parameters:
     type: string
     default: $(artifactDirectory)
 
+  - name: appRegistrationClientId
+    type: string
+    default: $(appRegistrationClientId)
+
+  - name: appRegistrationTenantId
+    type: string
+    default: $(appRegistrationTenantId)
+
 steps:
 - ${{ if eq(parameters.artifactType, 'dll') }}:
-  - task: SFP.build-tasks.custom-build-task-2.EsrpMalwareScanning@4
+  - task: EsrpMalwareScanning@5
     displayName: 'ESRP MalwareScanning'
     inputs:
-      ConnectedServiceName: 'SqlClient ESRP Malware Scanning'
+      ConnectedServiceName: 'ESRP Workload Identity federation service-ADO.Net'
+      AppRegistrationClientId: '${{parameters.appRegistrationClientId }}'
+      AppRegistrationTenantId: '${{parameters.appRegistrationTenantId }}'
+      AuthAKVName: SqlClientDrivers
+      AuthCertName: 'ESRP-Release-Auth'
       FolderPath: '${{parameters.sourceRoot }}'
       Pattern: '*.dll'
-      Region: US
-  - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@4
+      CleanupTempStorage: 1
+      VerboseLogin: 1
+  - task: EsrpCodeSigning@5
     displayName: 'ESRP CodeSigning'
     inputs:
-      ConnectedServiceName: 'SqlClient ESRP Code Signing'
+      ConnectedServiceName: 'ESRP Workload Identity federation service-ADO.Net'
+      AppRegistrationClientId: '${{parameters.appRegistrationClientId }}'
+      AppRegistrationTenantId: '${{parameters.appRegistrationTenantId }}'
+      AuthAKVName: SqlClientDrivers
+      AuthCertName: 'ESRP-Release-Auth'
+      AuthSignCertName: 'ESRP-Release-Sign2'
       FolderPath: '${{parameters.sourceRoot }}'
       Pattern: '*.dll'
       signConfigType: inlineSignParams
@@ -73,17 +91,28 @@ steps:
           ] 
 
 - ${{ if eq(parameters.artifactType, 'pkg') }}:
-  - task: SFP.build-tasks.custom-build-task-2.EsrpMalwareScanning@4
+  - task: EsrpMalwareScanning@5
     displayName: 'ESRP MalwareScanning Nuget Package'
     inputs:
-      ConnectedServiceName: 'SqlClient ESRP Malware Scanning'
+      ConnectedServiceName: 'ESRP Workload Identity federation service-ADO.Net'
+      AppRegistrationClientId: '${{parameters.appRegistrationClientId }}'
+      AppRegistrationTenantId: '${{parameters.appRegistrationTenantId }}'
+      AuthAKVName: SqlClientDrivers
+      AuthCertName: 'ESRP-Release-Auth'
       FolderPath: '${{parameters.artifactDirectory }}'
       Pattern: '*.nupkg'
-      Region: US
-  - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@4
+      CleanupTempStorage: 1
+      VerboseLogin: 1
+  - task: EsrpCodeSigning@5
     displayName: 'ESRP CodeSigning Nuget Package'
     inputs:
-      ConnectedServiceName: 'SqlClient ESRP Code Signing'
+      inputs:
+      ConnectedServiceName: 'ESRP Workload Identity federation service-ADO.Net'
+      AppRegistrationClientId: '${{parameters.appRegistrationClientId }}'
+      AppRegistrationTenantId: '${{parameters.appRegistrationTenantId }}'
+      AuthAKVName: SqlClientDrivers
+      AuthCertName: 'ESRP-Release-Auth'
+      AuthSignCertName: 'ESRP-Release-Sign2'
       FolderPath: '${{parameters.artifactDirectory }}'
       Pattern: '*.nupkg'
       signConfigType: inlineSignParams

eng/pipelines/common/templates/steps/publish-symbols-step.yml

@@ -1,11 +1,14 @@
-#################################################################################
-# Licensed to the .NET Foundation under one or more agreements.                 #
-# The .NET Foundation licenses this file to you under the MIT license.          #
-# See the LICENSE file in the project root for more information.                #
-#################################################################################
+####################################################################################
+# Licensed to the .NET Foundation under one or more agreements.                    #
+# The .NET Foundation licenses this file to you under the MIT license.             #
+# See the LICENSE file in the project root for more information.                   #
+#                                                                                  #
+# doc: https://www.osgwiki.com/wiki/Symbols_Publishing_Pipeline_to_SymWeb_and_MSDL #
+####################################################################################
 parameters:
   - name: SymAccount
     type: string
+    default: 'SqlClientDrivers'
 
   - name: publishSymbols
     type: string
@@ -15,6 +18,23 @@ parameters:
     type: string
     default: '$(NuGetPackageVersion)'
 
+  - name: symbolServer
+    type: string
+    default: '$(SymbolServer)'
+
+  - name: symbolTokenUri
+    type: string
+    default: '$(SymbolTokenUri)'
+
+  - name: symbolsArtifactName
+    type: string
+
+  - name: publishToServers
+    type: object
+    default: 
+      internal: true
+      public: true
+
   - name: referenceType
     default: project
     values:
@@ -29,12 +49,12 @@ parameters:
 
 steps:
 - powershell: 'Write-Host "##vso[task.setvariable variable=ArtifactServices.Symbol.AccountName;]${{parameters.SymAccount}}"'
-  displayName: 'Update Symbol.AccountName ${{parameters.SymAccount}}'
+  displayName: 'Update Symbol.AccountName with ${{parameters.SymAccount}}'
   condition: and(succeeded(), ${{ eq(parameters.publishSymbols, 'true') }})
 
 - ${{ if eq(parameters.product, 'MDS') }}:
   - task: PublishSymbols@2
-    displayName: 'Publish symbols path'
+    displayName: 'Upload symbols to ${{parameters.SymAccount }} org'
     inputs:
       SymbolsFolder: '$(Build.SourcesDirectory)\artifacts\${{parameters.referenceType }}\bin'
       SearchPattern: |
@@ -43,13 +63,16 @@ steps:
       IndexSources: false
       SymbolServerType: TeamServices
       SymbolsMaximumWaitTime: 60
+      SymbolExpirationInDays: 1825 # 5 years
       SymbolsProduct: Microsoft.Data.SqlClient
-      SymbolsVersion: '{{parameters.symbolsVersion }}'
+      SymbolsVersion: ${{parameters.symbolsVersion }}
+      SymbolsArtifactName: ${{parameters.symbolsArtifactName }}
+      Pat: $(System.AccessToken)
     condition: and(succeeded(), ${{ eq(parameters.publishSymbols, 'true') }})
 
 - ${{ if eq(parameters.product, 'AKV') }}:
   - task: PublishSymbols@2
-    displayName: 'Publish symbols path'
+    displayName: 'Upload symbols to ${{parameters.SymAccount }} org'
     inputs:
       SymbolsFolder: '$(Build.SourcesDirectory)\artifacts\${{parameters.referenceType }}\bin'
       SearchPattern: |
@@ -58,6 +81,69 @@ steps:
       IndexSources: false
       SymbolServerType: TeamServices
       SymbolsMaximumWaitTime: 60
+      SymbolExpirationInDays: 1825 # 5 years
       SymbolsProduct: Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider
-      SymbolsVersion: '{{parameters.symbolsVersion }}'
+      SymbolsVersion: ${{parameters.symbolsVersion }}
+      SymbolsArtifactName: ${{parameters.symbolsArtifactName }}
+      Pat: $(System.AccessToken)
     condition: and(succeeded(), ${{ eq(parameters.publishSymbols, 'true') }})
+
+- task: AzureCLI@2
+  displayName: 'Publish symbols'
+  condition: and(succeeded(), ${{ eq(parameters.publishSymbols, 'true') }})
+  inputs:
+    azureSubscription: 'Symbols publishing Workload Identity federation service-ADO.Net'
+    scriptType: ps
+    scriptLocation: inlineScript
+    inlineScript: |
+      $publishToInternalServer = "${{parameters.publishToServers.internal }}".ToLower()
+      $publishToPublicServer = "${{parameters.publishToServers.public }}".ToLower()
+
+      echo "Publishing request name: ${{parameters.symbolsArtifactName }}"
+      echo "Publish to internal server: $publishToInternalServer"
+      echo "Publish to public server: $publishToPublicServer"      
+
+      $symbolServer = "${{parameters.symbolServer }}"
+      $tokenUri = "${{parameters.symbolTokenUri }}"
+      # Registered project name in the symbol publishing pipeline: https://portal.microsofticm.com/imp/v3/incidents/incident/520844254/summary
+      $projectName = "Microsoft.Data.SqlClient.SNI"
+
+      # Get the access token for the symbol publishing service
+      $symbolPublishingToken = az account get-access-token --resource $tokenUri --query accessToken -o tsv
+
+      echo ">  1.Symbol publishing token acquired."
+
+      echo "Registering the request name ..."
+      $requestName = "${{parameters.symbolsArtifactName }}"
+      $requestNameRegistrationBody = "{'requestName': '$requestName'}"
+      Invoke-RestMethod -Method POST -Uri "https://$symbolServer.trafficmanager.net/projects/$projectName/requests" -Headers @{ Authorization = "Bearer $symbolPublishingToken" } -ContentType "application/json" -Body $requestNameRegistrationBody
+
+      echo ">  2.Registration of request name succeeded."
+
+      echo "Publishing the symbols ..."
+      $publishSymbolsBody = "{'publishToInternalServer': $publishToInternalServer, 'publishToPublicServer': $publishToPublicServer}"
+      echo "Publishing symbols request body: $publishSymbolsBody"
+      Invoke-RestMethod -Method POST -Uri "https://$symbolServer.trafficmanager.net/projects/$projectName/requests/$requestName" -Headers @{ Authorization = "Bearer $symbolPublishingToken" } -ContentType "application/json" -Body $publishSymbolsBody
+
+      echo ">  3.Request to publish symbols succeeded."
+
+      # The following REST calls are used to check publishing status.
+      echo ">  4.Checking the status of the request ..."
+
+      Invoke-RestMethod -Method GET -Uri "https://$symbolServer.trafficmanager.net/projects/$projectName/requests/$requestName" -Headers @{ Authorization = "Bearer $symbolPublishingToken" } -ContentType "application/json"
+
+      echo "Use below tables to interpret the values of xxxServerStatus and xxxServerResult fields from the response."
+
+      echo "PublishingStatus"
+      echo "-----------------"
+      echo "0	NotRequested; The request has not been requested to publish."
+      echo "1	Submitted; The request is submitted to be published"
+      echo "2	Processing; The request is still being processed"
+      echo "3	Completed; The request has been completed processing. It can be failed or successful. Check PublishingResult to get more details"
+
+      echo "PublishingResult"
+      echo "-----------------"
+      echo "0	Pending; The request has not completed or has not been requested."
+      echo "1	Succeeded; The request has published successfully"
+      echo "2	Failed; The request has failed to publish"
+      echo "3	Cancelled; The request was cancelled"