From 7145ccc6a4891a644871f03cdc674c37c476919c Mon Sep 17 00:00:00 2001 From: Rishabh Chauhan Date: Wed, 27 Mar 2024 16:51:09 +0530 Subject: [PATCH 1/2] Migrate to 1ESPT --- azure-pipelines-pr.yml | 4 +- azure-pipelines.yml | 129 +++++++++++--------------- eng/pipeline-pr.yml | 203 +++++++++++++++++++++++++++++++++++++++++ eng/pipeline.yml | 40 ++------ 4 files changed, 268 insertions(+), 108 deletions(-) create mode 100644 eng/pipeline-pr.yml diff --git a/azure-pipelines-pr.yml b/azure-pipelines-pr.yml index c7b1e1b04fd..29be0cc8f7d 100644 --- a/azure-pipelines-pr.yml +++ b/azure-pipelines-pr.yml @@ -63,12 +63,12 @@ pr: exclude: - Documentation/* -# Call the pipeline.yml template, which does the real work +# Call the pipeline-pr.yml template, which does the real work stages: - stage: build displayName: Build jobs: - - template: /eng/pipeline.yml + - template: /eng/pipeline-pr.yml parameters: ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: runAsPublic: false diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c7b1e1b04fd..c0f9c37e8f5 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,39 +1,27 @@ # This is a simple wrapper for eng/pipeline.yml to get around the limitation of # user-defined variables not being available in yaml template expressions. -# Parameters ARE available in template expressions, and parameters can have default values, -# so they can be used to control yaml flow. -# - +# Parameters ARE available in template expressions, and parameters can have default values variables: - # clean the local repo on the build agents - - name: Build.Repository.Clean +# clean the local repo on the build agents +- name: Build.Repository.Clean + value: true +- name: _DotNetArtifactsCategory + value: WINDOWSDESKTOP +- name: _DotNetValidationArtifactsCategory + value: WINDOWSDESKTOP +- ${{ if or(startswith(variables['Build.SourceBranch'], 'refs/heads/release/'), startswith(variables['Build.SourceBranch'], 'refs/heads/internal/release/'), eq(variables['Build.Reason'], 'Manual')) }}: + - name: PostBuildSign + value: false +- ${{ else }}: + - name: PostBuildSign value: true - - ${{ if or(startswith(variables['Build.SourceBranch'], 'refs/heads/release/'), startswith(variables['Build.SourceBranch'], 'refs/heads/internal/release/')) }}: - - name: PostBuildSign - value: false - - ${{ else }}: - - name: PostBuildSign - value: true - - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: DotNet-Wpf-SDLValidation-Params - - -# This is set in the pipeline directly -# When set to false, CI tests will not be enabled in builds. -# -# _ContinuousIntegrationTestsEnabled: false - -# Setting batch to true, triggers one build at a time. -# if there is a push while a build in progress, it will wait, -# until the running build finishes, and produce a build with all the changes -# -# only trigger ci builds for the master and release branches +- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: DotNet-Wpf-SDLValidation-Params trigger: - batch: true + batch: true branches: - include: + include: - main - release/3.* - release/5.* @@ -47,50 +35,39 @@ trigger: paths: exclude: - Documentation/* - -pr: - autoCancel: true - branches: - include: - - main - - release/3.* - - internal/release/3.* - - release/5.* - - release/6.* - - release/7.* - - experimental/* - paths: - exclude: - - Documentation/* - -# Call the pipeline.yml template, which does the real work -stages: -- stage: build - displayName: Build - jobs: - - template: /eng/pipeline.yml - parameters: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - runAsPublic: false - -- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: eng\common\templates\post-build\post-build.yml - parameters: - publishingInfraVersion: 3 - enableSymbolValidation: false - enableSigningValidation: false - enableNugetValidation: false - enableSourceLinkValidation: false - # This is to enable SDL runs part of Post-Build Validation Stage - SDLValidationParameters: - enable: false - params: ' -SourceToolsList @("policheck","credscan") - -TsaInstanceURL $(_TsaInstanceURL) - -TsaProjectName $(_TsaProjectName) - -TsaNotificationEmail $(_TsaNotificationEmail) - -TsaCodebaseAdmin $(_TsaCodebaseAdmin) - -TsaBugAreaPath $(_TsaBugAreaPath) - -TsaIterationPath $(_TsaIterationPath) - -TsaRepositoryName "wpf" - -TsaCodebaseName "wpf" - -TsaPublish $True' +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + featureFlags: + autoBaseline: true + pool: + name: NetCore1ESPool-Internal + image: 1es-windows-2022-pt + os: windows + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: build + displayName: Build + jobs: + - template: /eng/pipeline.yml@self + parameters: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + runAsPublic: false + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - template: /eng/common/templates-official/post-build/post-build.yml@self + parameters: + publishingInfraVersion: 3 + enableSymbolValidation: false + enableSigningValidation: false + enableNugetValidation: false + enableSourceLinkValidation: false + SDLValidationParameters: + enable: false + params: ' -SourceToolsList @("policheck","credscan") -TsaInstanceURL $(_TsaInstanceURL) -TsaProjectName $(_TsaProjectName) -TsaNotificationEmail $(_TsaNotificationEmail) -TsaCodebaseAdmin $(_TsaCodebaseAdmin) -TsaBugAreaPath $(_TsaBugAreaPath) -TsaIterationPath $(_TsaIterationPath) -TsaRepositoryName "wpf" -TsaCodebaseName "wpf" -TsaPublish $True' \ No newline at end of file diff --git a/eng/pipeline-pr.yml b/eng/pipeline-pr.yml new file mode 100644 index 00000000000..e2b5cc4a594 --- /dev/null +++ b/eng/pipeline-pr.yml @@ -0,0 +1,203 @@ +# +# This file should be kept in sync across https://www.github.com/dotnet/wpf and dotnet-wpf-int repos. +# +# + +parameters: + # Needed because runAsPublic is used in template expressions, which can't read from user-defined variables + # Defaults to true + runAsPublic: true + repoName: dotnet/wpf + +jobs: +- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }}: + - template: /eng/common/templates/job/onelocbuild.yml + parameters: + MirrorRepo: wpf + LclSource: lclFilesfromPackage + LclPackageId: 'LCL-JUNO-PROD-WPF' +- template: /eng/common/templates/jobs/jobs.yml + parameters: + enableMicrobuild: true + enablePublishBuildArtifacts: true + enablePublishTestResults: false # tests run in helix + enablePublishBuildAssets: true + enablePublishUsingPipelines: true + enableTelemetry: true + enableSourceIndex: true + sourceIndexParams: + condition: eq(variables['Build.SourceBranch'], 'refs/heads/main') + binlogPath: artifacts/log/Debug/x86/Build.binlog + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: NetCore-Svc-Public + demands: ImageOverride -equals windows.vs2022preview.amd64.Open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: NetCore1ESPool-Svc-Internal + demands: ImageOverride -equals windows.vs2022preview.amd64 + helixRepo: $(repoName) + + jobs: + - job: Windows_NT + timeoutInMinutes: 120 # how long to run the job before automatically cancelling; see https://github.com/dotnet/wpf/issues/952 + pool: + # For public jobs, use the hosted pool. For internal jobs use the internal pool. + # Will eventually change this to two BYOC pools. + # agent pool can't be read from a user-defined variable (Azure DevOps limitation) + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: NetCore-Svc-Public + demands: ImageOverride -equals windows.vs2022preview.amd64.Open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: NetCore1ESPool-Svc-Internal + demands: ImageOverride -equals windows.vs2022preview.amd64 + variables: + # needed for signing + - name: _TeamName + value: DotNetCore + - name: _SignType + value: real + - name: _SignArgs + value: '' + - name: _PublishArgs + value: '' + - name: _OfficialBuildIdArgs + value: '' + - name: _Platform + value: x86 + - name: _PlatformArgs + value: /p:Platform=$(_Platform) + - name: _PublicBuildPipeline # We will run Helix tests when building in the open, but do not repeat when building and publishing again using the internal build-pipeline + value: true + - name: _TestHelixAgentPool + value: 'Windows.10.Amd64.ClientRS5.Open' # Preferred:'Windows.10.Amd64.Open%3bWindows.7.Amd64.Open%3bWindows.10.Amd64.ClientRS5.Open'; See https://github.com/dotnet/wpf/issues/952 + - name: _HelixStagingDir + value: $(BUILD.STAGINGDIRECTORY)\helix\functests + - name: _HelixSource + value: ${{ parameters.repoName }}/$(Build.SourceBranch) + - name: _HelixToken + value: '' + - name: _HelixCreator + value: ${{ parameters.repoName }} + - ${{ if ne(variables['System.TeamProject'], 'internal') }}: + - name: _InternalRuntimeDownloadArgs + value: '' + - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - group: DotNetBuilds storage account read tokens + - group: AzureDevOps-Artifact-Feeds-Pats + - name: _InternalRuntimeDownloadArgs + value: >- + /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal + /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) + + + # Override some values if we're building internally + - ${{ if eq(parameters.runAsPublic, 'false') }}: + # note: You have to use list syntax here (- name: value) or you will get errors about declaring the same variable multiple times + - name: _SignType + value: real + - group: DotNet-HelixApi-Access + + # note: Even though they are referenced here, user defined variables (like $(_SignType)) are not resolved + # until the agent is running on the machine. They can be overridden any time before they are resolved, + # like in the job matrix below (see Build_Debug) + - name: _SignArgs + value: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) + - name: _PublishArgs + value: /p:DotNetPublishUsingPipelines=true + - name: _OfficialBuildIdArgs + value: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) + - name: _PublicBuildPipeline + value: false + - name: _HelixSource + value: official/${{ parameters.repoName }}/$(Build.SourceBranch) + - name: _HelixToken + value: '$(HelixApiAccessToken)' # from DotNet-HelixApi-Access group + - name: _HelixCreator + value: '' #if _HelixToken is set, Creator must be empty + - name: _TestHelixAgentPool + value: 'Windows.10.Amd64.ClientRS5' # Preferred: 'Windows.10.Amd64%3bWindows.7.Amd64%3bWindows.10.Amd64.ClientRS5' + + strategy: + matrix: + ${{ if eq(parameters.runAsPublic, 'true') }}: + Build_Debug_x86: + _BuildConfig: Debug + # override some variables for debug + # _SignType has to be real for package publishing to succeed - do not override to test. + Build_Release_x86: + _BuildConfig: Release + ${{ if eq(parameters.runAsPublic, 'true') }}: + Build_Debug_x64: + _BuildConfig: Debug + # override some variables for debug + # _SignType has to be real for package publishing to succeed - do not override to test. + _Platform: x64 + Build_Release_x64: + _BuildConfig: Release + _Platform: x64 + ${{ if eq(parameters.runAsPublic, 'true') }}: + Build_Debug_arm64: + _BuildConfig: Debug + # override some variables for debug + # _SignType has to be real for package publishing to succeed - do not override to test. + _Platform: arm64 + Build_Release_arm64: + _BuildConfig: Release + _Platform: arm64 + steps: + - checkout: self + clean: true + + # Set VSO Variable(s) + - powershell: eng\pre-build.ps1 + displayName: Pre-Build - Set VSO Variables + + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - task: PowerShell@2 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 + arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + + # Use utility script to run script command dependent on agent OS. + - script: eng\common\cibuild.cmd + -configuration $(_BuildConfig) + -prepareMachine + $(_PublishArgs) + $(_SignArgs) + $(_OfficialBuildIdArgs) + $(_PlatformArgs) + $(_InternalRuntimeDownloadArgs) + displayName: Windows Build / Publish + # This condition should be kept in sync with the condition for 'Run DRTs' step + # When building on a regular pipeline (!_HelixPipeline), build as usual + # When building on a Helix pipeline, only build Release configs + # (!_HelixPipeline) || + # (_HelixPipeline && _PublicBuildPipeline && _ContinuousIntegrationTestsEnabled && _BuildConfig == Release) + condition: or(ne(variables['_HelixPipeline'], 'true'), and(eq(variables['_HelixPipeline'], 'true') ,eq(variables['_BuildConfig'], 'Release'), eq(variables['_PublicBuildPipeline'], 'true'), eq(variables['_ContinuousIntegrationTestsEnabled'], 'true'))) + + # Run DRTs + - powershell: eng\common\cibuild.cmd + -configuration $(_BuildConfig) + $(_OfficialBuildIdArgs) + $(_PlatformArgs) + -projects $(Build.SourcesDirectory)\eng\helixpublish.proj + /bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\HelixDrt.binlog + displayName: Run Developer Regression Tests on Helix Machine (Release) + env: + HelixSource: $(_HelixSource) + HelixType: 'tests/drt' + HelixBuild: $(Build.BuildNumber) + HelixTargetQueues: $(_TestHelixAgentPool) + HelixAccessToken: $(_HelixToken) # only defined for internal CI + Creator: $(_HelixCreator) + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + # This condition should be kept in sync with the condition for cibuild.cmd step with displayName: "Windows Build / Publish" + # Only run ... + # ...When building on a Helix pipeline, only build Release configs + # + # (_HelixPipeline && _PublicBuildPipeline && _ContinuousIntegrationTestsEnabled && _BuildConfig == Release) + # + condition: and(succeeded(), eq(variables['_HelixPipeline'], 'true') ,eq(variables['_BuildConfig'], 'Release'), eq(variables['_PublicBuildPipeline'], 'true'), eq(variables['_ContinuousIntegrationTestsEnabled'], 'true')) \ No newline at end of file diff --git a/eng/pipeline.yml b/eng/pipeline.yml index e2b5cc4a594..cc90af98ef6 100644 --- a/eng/pipeline.yml +++ b/eng/pipeline.yml @@ -2,21 +2,19 @@ # This file should be kept in sync across https://www.github.com/dotnet/wpf and dotnet-wpf-int repos. # # - parameters: # Needed because runAsPublic is used in template expressions, which can't read from user-defined variables # Defaults to true runAsPublic: true repoName: dotnet/wpf - jobs: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }}: - - template: /eng/common/templates/job/onelocbuild.yml + - template: /eng/common/templates-official/job/onelocbuild.yml@self parameters: MirrorRepo: wpf LclSource: lclFilesfromPackage LclPackageId: 'LCL-JUNO-PROD-WPF' -- template: /eng/common/templates/jobs/jobs.yml +- template: /eng/common/templates-official/jobs/jobs.yml@self parameters: enableMicrobuild: true enablePublishBuildArtifacts: true @@ -29,27 +27,18 @@ jobs: condition: eq(variables['Build.SourceBranch'], 'refs/heads/main') binlogPath: artifacts/log/Debug/x86/Build.binlog pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: NetCore-Svc-Public - demands: ImageOverride -equals windows.vs2022preview.amd64.Open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals windows.vs2022preview.amd64 + name: NetCore1ESPool-Internal + demands: ImageOverride -equals windows.vs2022preview.amd64 helixRepo: $(repoName) - jobs: - job: Windows_NT - timeoutInMinutes: 120 # how long to run the job before automatically cancelling; see https://github.com/dotnet/wpf/issues/952 + timeoutInMinutes: 120 # how long to run the job before automatically cancelling; see https://github.com/dotnet/wpf/issues/952 pool: - # For public jobs, use the hosted pool. For internal jobs use the internal pool. + # For internal jobs use the internal pool. # Will eventually change this to two BYOC pools. # agent pool can't be read from a user-defined variable (Azure DevOps limitation) - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: NetCore-Svc-Public - demands: ImageOverride -equals windows.vs2022preview.amd64.Open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals windows.vs2022preview.amd64 + name: NetCore1ESPool-Internal + demands: ImageOverride -equals windows.vs2022preview.amd64 variables: # needed for signing - name: _TeamName @@ -151,7 +140,6 @@ jobs: # Set VSO Variable(s) - powershell: eng\pre-build.ps1 displayName: Pre-Build - Set VSO Variables - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - task: PowerShell@2 displayName: Setup Private Feeds Credentials @@ -160,16 +148,8 @@ jobs: arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token env: Token: $(dn-bot-dnceng-artifact-feeds-rw) - - # Use utility script to run script command dependent on agent OS. - - script: eng\common\cibuild.cmd - -configuration $(_BuildConfig) - -prepareMachine - $(_PublishArgs) - $(_SignArgs) - $(_OfficialBuildIdArgs) - $(_PlatformArgs) - $(_InternalRuntimeDownloadArgs) + # Use utility script to run script command dependent on agent OS + - script: eng\common\cibuild.cmd -configuration $(_BuildConfig) -prepareMachine $(_PublishArgs) $(_SignArgs) $(_OfficialBuildIdArgs) $(_PlatformArgs) $(_InternalRuntimeDownloadArgs) displayName: Windows Build / Publish # This condition should be kept in sync with the condition for 'Run DRTs' step # When building on a regular pipeline (!_HelixPipeline), build as usual From c1009b5df52e093ca1bda44a86bf4b2469a1fcc8 Mon Sep 17 00:00:00 2001 From: Rishabh Chauhan Date: Thu, 28 Mar 2024 12:27:09 +0530 Subject: [PATCH 2/2] revert public and internal changes --- eng/pipeline-pr.yml | 8 ++++---- eng/pipeline.yml | 16 ++++++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/eng/pipeline-pr.yml b/eng/pipeline-pr.yml index e2b5cc4a594..19093cb4cd8 100644 --- a/eng/pipeline-pr.yml +++ b/eng/pipeline-pr.yml @@ -30,10 +30,10 @@ jobs: binlogPath: artifacts/log/Debug/x86/Build.binlog pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: NetCore-Svc-Public + name: NetCore-Public demands: ImageOverride -equals windows.vs2022preview.amd64.Open ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: NetCore1ESPool-Svc-Internal + name: NetCore1ESPool-Internal demands: ImageOverride -equals windows.vs2022preview.amd64 helixRepo: $(repoName) @@ -45,10 +45,10 @@ jobs: # Will eventually change this to two BYOC pools. # agent pool can't be read from a user-defined variable (Azure DevOps limitation) ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: NetCore-Svc-Public + name: NetCore-Public demands: ImageOverride -equals windows.vs2022preview.amd64.Open ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: NetCore1ESPool-Svc-Internal + name: NetCore1ESPool-Internal demands: ImageOverride -equals windows.vs2022preview.amd64 variables: # needed for signing diff --git a/eng/pipeline.yml b/eng/pipeline.yml index cc90af98ef6..71a0bd35100 100644 --- a/eng/pipeline.yml +++ b/eng/pipeline.yml @@ -27,8 +27,12 @@ jobs: condition: eq(variables['Build.SourceBranch'], 'refs/heads/main') binlogPath: artifacts/log/Debug/x86/Build.binlog pool: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals windows.vs2022preview.amd64 + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: NetCore-Public + demands: ImageOverride -equals windows.vs2022preview.amd64.Open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: NetCore1ESPool-Internal + demands: ImageOverride -equals windows.vs2022preview.amd64 helixRepo: $(repoName) jobs: - job: Windows_NT @@ -37,8 +41,12 @@ jobs: # For internal jobs use the internal pool. # Will eventually change this to two BYOC pools. # agent pool can't be read from a user-defined variable (Azure DevOps limitation) - name: NetCore1ESPool-Internal - demands: ImageOverride -equals windows.vs2022preview.amd64 + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: NetCore-Public + demands: ImageOverride -equals windows.vs2022preview.amd64.Open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: NetCore1ESPool-Internal + demands: ImageOverride -equals windows.vs2022preview.amd64 variables: # needed for signing - name: _TeamName