diff --git a/eng/pipelines/common/templates/jobs/build-signed-akv-package-job.yml b/eng/pipelines/common/templates/jobs/build-signed-akv-package-job.yml index c7b37f144c..6102dcc27f 100644 --- a/eng/pipelines/common/templates/jobs/build-signed-akv-package-job.yml +++ b/eng/pipelines/common/templates/jobs/build-signed-akv-package-job.yml @@ -17,6 +17,7 @@ parameters: jobs: - job: build_signed_akv_package + displayName: 'Build Signed AKV Provider Package' pool: type: windows @@ -63,20 +64,11 @@ jobs: product: AKV referenceType: package - # Publish symbols to private server + # Publish symbols to servers - template: ../steps/publish-symbols-step.yml@self parameters: - SymAccount: $(PrivateSymAccount) - referenceType: package - symbolsVersion: ${{variables.AKVNuGetPackageVersion }} - product: AKV - publishSymbols: ${{ parameters['PublishSymbols'] }} - - # Publish symbols to public server - - template: ../steps/publish-symbols-step.yml@self - parameters: - SymAccount: $(PublicSymAccount) referenceType: package symbolsVersion: ${{variables.AKVNuGetPackageVersion }} product: AKV publishSymbols: ${{ parameters['PublishSymbols'] }} + symbolsArtifactName: akv_symbols_$(System.TeamProject)_$(Build.Repository.Name)_$(Build.SourceBranchName)_$(NuGetPackageVersion)_$(System.TimelineId) diff --git a/eng/pipelines/common/templates/jobs/build-signed-package-job.yml b/eng/pipelines/common/templates/jobs/build-signed-package-job.yml index d39db0f63e..9059d0e1f5 100644 --- a/eng/pipelines/common/templates/jobs/build-signed-package-job.yml +++ b/eng/pipelines/common/templates/jobs/build-signed-package-job.yml @@ -17,6 +17,7 @@ parameters: jobs: - job: build_signed_package + displayName: 'Build Signed MDS Package' pool: type: windows @@ -51,14 +52,8 @@ jobs: parameters: product: MDS - # Publish symbols to private server + # Publish symbols to servers - template: ../steps/publish-symbols-step.yml@self parameters: - SymAccount: $(PrivateSymAccount) - publishSymbols: ${{ parameters['PublishSymbols'] }} - - # Publish symbols to public server - - template: ../steps/publish-symbols-step.yml@self - parameters: - SymAccount: $(PublicSymAccount) publishSymbols: ${{ parameters['PublishSymbols'] }} + symbolsArtifactName: mds_symbols_$(System.TeamProject)_$(Build.Repository.Name)_$(Build.SourceBranchName)_$(NuGetPackageVersion)_$(System.TimelineId) diff --git a/eng/pipelines/common/templates/jobs/run-tests-package-reference-job.yml b/eng/pipelines/common/templates/jobs/run-tests-package-reference-job.yml index cd0eaa0c0c..a1a98e4877 100644 --- a/eng/pipelines/common/templates/jobs/run-tests-package-reference-job.yml +++ b/eng/pipelines/common/templates/jobs/run-tests-package-reference-job.yml @@ -19,6 +19,9 @@ parameters: jobs: - job: run_tests_package_reference + displayName: 'Run tests with package reference' + ${{ if ne(parameters.dependsOn, 'empty')}}: + dependsOn: '${{parameters.dependsOn }}' pool: type: windows # read more about custom job pool types at https://aka.ms/obpipelines/yaml/jobs isCustom: true diff --git a/eng/pipelines/common/templates/jobs/validate-signed-package-job.yml b/eng/pipelines/common/templates/jobs/validate-signed-package-job.yml index 693b0e6657..6ba0df6e19 100644 --- a/eng/pipelines/common/templates/jobs/validate-signed-package-job.yml +++ b/eng/pipelines/common/templates/jobs/validate-signed-package-job.yml @@ -39,6 +39,9 @@ parameters: jobs: - job: validate_signed_package + displayName: 'Verify signed package' + ${{ if ne(parameters.dependsOn, '')}}: + dependsOn: '${{parameters.dependsOn }}' pool: type: windows # read more about custom job pool types at https://aka.ms/obpipelines/yaml/jobs isCustom: true diff --git a/eng/pipelines/common/templates/steps/esrp-code-signing-step.yml b/eng/pipelines/common/templates/steps/esrp-code-signing-step.yml index d639eac044..67e21ed7ff 100644 --- a/eng/pipelines/common/templates/steps/esrp-code-signing-step.yml +++ b/eng/pipelines/common/templates/steps/esrp-code-signing-step.yml @@ -17,19 +17,37 @@ parameters: type: string default: $(artifactDirectory) + - name: appRegistrationClientId + type: string + default: $(appRegistrationClientId) + + - name: appRegistrationTenantId + type: string + default: $(appRegistrationTenantId) + steps: - ${{ if eq(parameters.artifactType, 'dll') }}: - - task: SFP.build-tasks.custom-build-task-2.EsrpMalwareScanning@4 + - task: EsrpMalwareScanning@5 displayName: 'ESRP MalwareScanning' inputs: - ConnectedServiceName: 'SqlClient ESRP Malware Scanning' + ConnectedServiceName: 'ESRP Workload Identity federation service-ADO.Net' + AppRegistrationClientId: '${{parameters.appRegistrationClientId }}' + AppRegistrationTenantId: '${{parameters.appRegistrationTenantId }}' + AuthAKVName: SqlClientDrivers + AuthCertName: 'ESRP-Release-Auth' FolderPath: '${{parameters.sourceRoot }}' Pattern: '*.dll' - Region: US - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@4 + CleanupTempStorage: 1 + VerboseLogin: 1 + - task: EsrpCodeSigning@5 displayName: 'ESRP CodeSigning' inputs: - ConnectedServiceName: 'SqlClient ESRP Code Signing' + ConnectedServiceName: 'ESRP Workload Identity federation service-ADO.Net' + AppRegistrationClientId: '${{parameters.appRegistrationClientId }}' + AppRegistrationTenantId: '${{parameters.appRegistrationTenantId }}' + AuthAKVName: SqlClientDrivers + AuthCertName: 'ESRP-Release-Auth' + AuthSignCertName: 'ESRP-Release-Sign2' FolderPath: '${{parameters.sourceRoot }}' Pattern: '*.dll' signConfigType: inlineSignParams @@ -73,17 +91,28 @@ steps: ] - ${{ if eq(parameters.artifactType, 'pkg') }}: - - task: SFP.build-tasks.custom-build-task-2.EsrpMalwareScanning@4 + - task: EsrpMalwareScanning@5 displayName: 'ESRP MalwareScanning Nuget Package' inputs: - ConnectedServiceName: 'SqlClient ESRP Malware Scanning' + ConnectedServiceName: 'ESRP Workload Identity federation service-ADO.Net' + AppRegistrationClientId: '${{parameters.appRegistrationClientId }}' + AppRegistrationTenantId: '${{parameters.appRegistrationTenantId }}' + AuthAKVName: SqlClientDrivers + AuthCertName: 'ESRP-Release-Auth' FolderPath: '${{parameters.artifactDirectory }}' Pattern: '*.nupkg' - Region: US - - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@4 + CleanupTempStorage: 1 + VerboseLogin: 1 + - task: EsrpCodeSigning@5 displayName: 'ESRP CodeSigning Nuget Package' inputs: - ConnectedServiceName: 'SqlClient ESRP Code Signing' + inputs: + ConnectedServiceName: 'ESRP Workload Identity federation service-ADO.Net' + AppRegistrationClientId: '${{parameters.appRegistrationClientId }}' + AppRegistrationTenantId: '${{parameters.appRegistrationTenantId }}' + AuthAKVName: SqlClientDrivers + AuthCertName: 'ESRP-Release-Auth' + AuthSignCertName: 'ESRP-Release-Sign2' FolderPath: '${{parameters.artifactDirectory }}' Pattern: '*.nupkg' signConfigType: inlineSignParams diff --git a/eng/pipelines/common/templates/steps/publish-symbols-step.yml b/eng/pipelines/common/templates/steps/publish-symbols-step.yml index d64be80dec..d0e54d7d72 100644 --- a/eng/pipelines/common/templates/steps/publish-symbols-step.yml +++ b/eng/pipelines/common/templates/steps/publish-symbols-step.yml @@ -1,11 +1,14 @@ -################################################################################# -# Licensed to the .NET Foundation under one or more agreements. # -# The .NET Foundation licenses this file to you under the MIT license. # -# See the LICENSE file in the project root for more information. # -################################################################################# +#################################################################################### +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +# # +# doc: https://www.osgwiki.com/wiki/Symbols_Publishing_Pipeline_to_SymWeb_and_MSDL # +#################################################################################### parameters: - name: SymAccount type: string + default: 'SqlClientDrivers' - name: publishSymbols type: string @@ -15,6 +18,23 @@ parameters: type: string default: '$(NuGetPackageVersion)' + - name: symbolServer + type: string + default: '$(SymbolServer)' + + - name: symbolTokenUri + type: string + default: '$(SymbolTokenUri)' + + - name: symbolsArtifactName + type: string + + - name: publishToServers + type: object + default: + internal: true + public: true + - name: referenceType default: project values: @@ -29,12 +49,12 @@ parameters: steps: - powershell: 'Write-Host "##vso[task.setvariable variable=ArtifactServices.Symbol.AccountName;]${{parameters.SymAccount}}"' - displayName: 'Update Symbol.AccountName ${{parameters.SymAccount}}' + displayName: 'Update Symbol.AccountName with ${{parameters.SymAccount}}' condition: and(succeeded(), ${{ eq(parameters.publishSymbols, 'true') }}) - ${{ if eq(parameters.product, 'MDS') }}: - task: PublishSymbols@2 - displayName: 'Publish symbols path' + displayName: 'Upload symbols to ${{parameters.SymAccount }} org' inputs: SymbolsFolder: '$(Build.SourcesDirectory)\artifacts\${{parameters.referenceType }}\bin' SearchPattern: | @@ -43,13 +63,16 @@ steps: IndexSources: false SymbolServerType: TeamServices SymbolsMaximumWaitTime: 60 + SymbolExpirationInDays: 1825 # 5 years SymbolsProduct: Microsoft.Data.SqlClient - SymbolsVersion: '{{parameters.symbolsVersion }}' + SymbolsVersion: ${{parameters.symbolsVersion }} + SymbolsArtifactName: ${{parameters.symbolsArtifactName }} + Pat: $(System.AccessToken) condition: and(succeeded(), ${{ eq(parameters.publishSymbols, 'true') }}) - ${{ if eq(parameters.product, 'AKV') }}: - task: PublishSymbols@2 - displayName: 'Publish symbols path' + displayName: 'Upload symbols to ${{parameters.SymAccount }} org' inputs: SymbolsFolder: '$(Build.SourcesDirectory)\artifacts\${{parameters.referenceType }}\bin' SearchPattern: | @@ -58,6 +81,69 @@ steps: IndexSources: false SymbolServerType: TeamServices SymbolsMaximumWaitTime: 60 + SymbolExpirationInDays: 1825 # 5 years SymbolsProduct: Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider - SymbolsVersion: '{{parameters.symbolsVersion }}' + SymbolsVersion: ${{parameters.symbolsVersion }} + SymbolsArtifactName: ${{parameters.symbolsArtifactName }} + Pat: $(System.AccessToken) condition: and(succeeded(), ${{ eq(parameters.publishSymbols, 'true') }}) + +- task: AzureCLI@2 + displayName: 'Publish symbols' + condition: and(succeeded(), ${{ eq(parameters.publishSymbols, 'true') }}) + inputs: + azureSubscription: 'Symbols publishing Workload Identity federation service-ADO.Net' + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + $publishToInternalServer = "${{parameters.publishToServers.internal }}".ToLower() + $publishToPublicServer = "${{parameters.publishToServers.public }}".ToLower() + + echo "Publishing request name: ${{parameters.symbolsArtifactName }}" + echo "Publish to internal server: $publishToInternalServer" + echo "Publish to public server: $publishToPublicServer" + + $symbolServer = "${{parameters.symbolServer }}" + $tokenUri = "${{parameters.symbolTokenUri }}" + # Registered project name in the symbol publishing pipeline: https://portal.microsofticm.com/imp/v3/incidents/incident/520844254/summary + $projectName = "Microsoft.Data.SqlClient.SNI" + + # Get the access token for the symbol publishing service + $symbolPublishingToken = az account get-access-token --resource $tokenUri --query accessToken -o tsv + + echo "> 1.Symbol publishing token acquired." + + echo "Registering the request name ..." + $requestName = "${{parameters.symbolsArtifactName }}" + $requestNameRegistrationBody = "{'requestName': '$requestName'}" + Invoke-RestMethod -Method POST -Uri "https://$symbolServer.trafficmanager.net/projects/$projectName/requests" -Headers @{ Authorization = "Bearer $symbolPublishingToken" } -ContentType "application/json" -Body $requestNameRegistrationBody + + echo "> 2.Registration of request name succeeded." + + echo "Publishing the symbols ..." + $publishSymbolsBody = "{'publishToInternalServer': $publishToInternalServer, 'publishToPublicServer': $publishToPublicServer}" + echo "Publishing symbols request body: $publishSymbolsBody" + Invoke-RestMethod -Method POST -Uri "https://$symbolServer.trafficmanager.net/projects/$projectName/requests/$requestName" -Headers @{ Authorization = "Bearer $symbolPublishingToken" } -ContentType "application/json" -Body $publishSymbolsBody + + echo "> 3.Request to publish symbols succeeded." + + # The following REST calls are used to check publishing status. + echo "> 4.Checking the status of the request ..." + + Invoke-RestMethod -Method GET -Uri "https://$symbolServer.trafficmanager.net/projects/$projectName/requests/$requestName" -Headers @{ Authorization = "Bearer $symbolPublishingToken" } -ContentType "application/json" + + echo "Use below tables to interpret the values of xxxServerStatus and xxxServerResult fields from the response." + + echo "PublishingStatus" + echo "-----------------" + echo "0 NotRequested; The request has not been requested to publish." + echo "1 Submitted; The request is submitted to be published" + echo "2 Processing; The request is still being processed" + echo "3 Completed; The request has been completed processing. It can be failed or successful. Check PublishingResult to get more details" + + echo "PublishingResult" + echo "-----------------" + echo "0 Pending; The request has not completed or has not been requested." + echo "1 Succeeded; The request has published successfully" + echo "2 Failed; The request has failed to publish" + echo "3 Cancelled; The request was cancelled" diff --git a/eng/pipelines/dotnet-sqlclient-signing-pipeline.yml b/eng/pipelines/dotnet-sqlclient-signing-pipeline.yml index c1ac56cdc8..681c1d962b 100644 --- a/eng/pipelines/dotnet-sqlclient-signing-pipeline.yml +++ b/eng/pipelines/dotnet-sqlclient-signing-pipeline.yml @@ -38,24 +38,33 @@ parameters: # parameters are shown up in ADO UI in a build queue time displayName: 'Enable debug output' type: boolean default: false -- name: oneBranchType - displayName: 'Select OneBranch template' - default: Official - values: - - NonOfficial - - Official + - name: publishSymbols type: boolean default: false + - name: MDS_PackageRef_Version displayName: 'MDS package version of AKV Provider (build AKV)' type: string - default: 3.1.5 + default: 3.0.0 + - name: CurrentNetFxVersion displayName: 'Lowest supported .NET Framework version (MDS validation)' type: string default: 'net461' +- name: enableAllSdlTools + displayName: 'Enable all SDL tools' + type: boolean + default: true + +- name: oneBranchType + displayName: 'Select OneBranch template' + default: Official + values: + - NonOfficial + - Official + variables: - template: /eng/pipelines/libraries/variables.yml@self - name: packageFolderName @@ -77,49 +86,50 @@ resources: ref: refs/heads/main extends: - template: v2/OneBranch.${{ parameters['oneBranchType'] }}.CrossPlat.yml@templates # https://aka.ms/obpipelines/templates + template: v2/OneBranch.${{parameters.oneBranchType }}.CrossPlat.yml@templates # https://aka.ms/obpipelines/templates parameters: globalSdl: # https://aka.ms/obpipelines/sdl apiscan: - enabled: true + enabled: ${{parameters.enableAllSdlTools }} softwareFolder: $(softwareFolder) symbolsFolder: $(symbolsFolder) softwarename: Microsoft.Data.SqlClient versionNumber: $(AssemblyFileVersion) tsa: - enabled: true # onebranch publish all sdl results to TSA. If TSA is disabled all SDL tools will forced into 'break' build mode. + enabled: ${{parameters.enableAllSdlTools }} # onebranch publish all sdl results to TSA. If TSA is disabled all SDL tools will forced into 'break' build mode. codeql: compiled: enabled: false #[warning]Consider running CodeQL on the default branch only. sbom: - enabled: true + enabled: ${{parameters.enableAllSdlTools }} packageName: Microsoft.Data.SqlClient packageVersion: $(NugetPackageVersion) policheck: - enabled: true + enabled: ${{parameters.enableAllSdlTools }} break: true # always break the build on policheck issues. You can disable it by setting to 'false' exclusionsFile: $(REPOROOT)\.config\PolicheckExclusions.xml asyncSdl: enabled: false credscan: - enabled: true + enabled: ${{parameters.enableAllSdlTools }} suppressionsFile: $(REPOROOT)/.config/CredScanSuppressions.json binskim: - enabled: true + enabled: ${{parameters.enableAllSdlTools }} armory: - enabled: true + enabled: ${{parameters.enableAllSdlTools }} break: true eslint: # TypeScript and JavaScript enabled: false roslyn: - enabled: true + enabled: ${{parameters.enableAllSdlTools }} break: true publishLogs: - enabled: true + enabled: ${{parameters.enableAllSdlTools }} tsaOptionsPath: $(REPOROOT)\.config\tsaoptions.json disableLegacyManifest: true stages: - stage: buildAKV + displayName: 'Build AKV Provider' jobs: - template: eng/pipelines/common/templates/jobs/build-signed-akv-package-job.yml@self parameters: @@ -128,6 +138,7 @@ extends: publishSymbols: ${{ parameters['publishSymbols'] }} - stage: buildMDS + displayName: 'Build MDS' jobs: - template: eng/pipelines/common/templates/jobs/build-signed-package-job.yml@self parameters: @@ -135,7 +146,8 @@ extends: softwareFolder: $(softwareFolder) publishSymbols: ${{ parameters['publishSymbols'] }} - - stage: package_validation + - stage: mds_package_validation + displayName: 'MDS Package Validation' dependsOn: buildMDS jobs: - template: eng/pipelines/common/templates/jobs/validate-signed-package-job.yml@self diff --git a/eng/pipelines/libraries/common-variables.yml b/eng/pipelines/libraries/common-variables.yml index 05b658f153..7b2bc00cb1 100644 --- a/eng/pipelines/libraries/common-variables.yml +++ b/eng/pipelines/libraries/common-variables.yml @@ -16,4 +16,8 @@ variables: - name: symbolsFolder value: $(REPOROOT)/symbols - name: artifactDirectory - value: '$(REPOROOT)/packages' \ No newline at end of file + value: '$(REPOROOT)/packages' + - name: appRegistrationClientId + value: 'a0d18a38-fde1-4ba7-92e1-15be16cb6a8e' + - name: appRegistrationTenantId + value: '72f988bf-86f1-41af-91ab-2d7cd011db47'