Backport #92375 and #93082 to release/8.0-staging (#101744)

* Allow multiple post-build steps and allow templated pre and post-build steps (#92375) * Convert AllConfigurations and NETFX jobs to use the global build templates (#93082) --------- Co-authored-by: Jeremy Koritzinsky <jekoritz@microsoft.com>
dotnet · May 2, 2024 · 37a6d07 · 37a6d07
1 parent a0411df
commit 37a6d07
Show file tree

Hide file tree

Showing 27 changed files with 509 additions and 401 deletions.
diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml
@@ -26,8 +26,7 @@ parameters:
   helixQueues: ''
   enablePublishTestResults: false
   testResultsFormat: ''
-  extraStepsTemplate: ''
-  extraStepsParameters: {}
+  postBuildSteps: []
   extraVariablesTemplates: []
   isManualCodeQLBuild: false
   preBuildSteps: []
@@ -209,7 +208,28 @@ jobs:
 
     - ${{ if ne(parameters.preBuildSteps,'') }}:
       - ${{ each preBuildStep in parameters.preBuildSteps }}:
-        - ${{ preBuildStep }}
+        - ${{ if ne(preBuildStep.template, '') }}:
+          - template: ${{ preBuildStep.template }}
+            parameters:
+              osGroup: ${{ parameters.osGroup }}
+              osSubgroup: ${{ parameters.osSubgroup }}
+              archType: ${{ parameters.archType }}
+              buildConfig: ${{ parameters.buildConfig }}
+              runtimeFlavor: ${{ parameters.runtimeFlavor }}
+              runtimeVariant: ${{ parameters.runtimeVariant }}
+              helixQueues: ${{ parameters.helixQueues }}
+              targetRid: ${{ parameters.targetRid }}
+              nameSuffix: ${{ parameters.nameSuffix }}
+              platform: ${{ parameters.platform }}
+              pgoType: ${{ parameters.pgoType }}
+              shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
+              ${{ if ne(preBuildStep.forwardedParameters, '') }}:
+                ${{ each parameter in preBuildStep.forwardedParameters }}:
+                  ${{ parameter }}: ${{ parameters[parameter] }}
+              ${{ if ne(preBuildStep.parameters, '') }}:
+                ${{ insert }}: ${{ preBuildStep.parameters }}
+        - ${{ else }}:
+          - ${{ preBuildStep }}
 
     # Build
     - ${{ if eq(parameters.isSourceBuild, false) }}:
@@ -235,21 +255,29 @@ jobs:
         condition: always()
 
       # If intended to send extra steps after regular build add them here.
-    - ${{ if ne(parameters.extraStepsTemplate, '') }}:
-      - template: ${{ parameters.extraStepsTemplate }}
-        parameters:
-          osGroup: ${{ parameters.osGroup }}
-          osSubgroup: ${{ parameters.osSubgroup }}
-          archType: ${{ parameters.archType }}
-          buildConfig: ${{ parameters.buildConfig }}
-          runtimeFlavor: ${{ parameters.runtimeFlavor }}
-          runtimeVariant: ${{ parameters.runtimeVariant }}
-          helixQueues: ${{ parameters.helixQueues }}
-          targetRid: ${{ parameters.targetRid }}
-          nameSuffix: ${{ parameters.nameSuffix }}
-          platform: ${{ parameters.platform }}
-          shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
-          ${{ insert }}: ${{ parameters.extraStepsParameters }}
+    - ${{ if ne(parameters.postBuildSteps,'') }}:
+      - ${{ each postBuildStep in parameters.postBuildSteps }}:
+        - ${{ if ne(postBuildStep.template, '') }}:
+          - template: ${{ postBuildStep.template }}
+            parameters:
+              osGroup: ${{ parameters.osGroup }}
+              osSubgroup: ${{ parameters.osSubgroup }}
+              archType: ${{ parameters.archType }}
+              buildConfig: ${{ parameters.buildConfig }}
+              runtimeFlavor: ${{ parameters.runtimeFlavor }}
+              runtimeVariant: ${{ parameters.runtimeVariant }}
+              helixQueues: ${{ parameters.helixQueues }}
+              targetRid: ${{ parameters.targetRid }}
+              nameSuffix: ${{ parameters.nameSuffix }}
+              platform: ${{ parameters.platform }}
+              shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
+              ${{ if ne(postBuildStep.forwardedParameters, '') }}:
+                ${{ each parameter in postBuildStep.forwardedParameters }}:
+                  ${{ parameter }}: ${{ parameters[parameter] }}
+              ${{ if ne(postBuildStep.parameters, '') }}:
+                ${{ insert }}: ${{ postBuildStep.parameters }}
+        - ${{ else }}:
+          - ${{ postBuildStep }}
 
       - ${{ if and(eq(parameters.isOfficialBuild, true), eq(parameters.osGroup, 'windows')) }}:
         - powershell: ./eng/collect_vsinfo.ps1 -ArchiveRunName postbuild_log

diff --git a/eng/pipelines/common/templates/browser-wasm-build-tests.yml b/eng/pipelines/common/templates/browser-wasm-build-tests.yml
@@ -117,10 +117,11 @@ jobs:
             eq(variables['isDefaultPipeline'], variables['shouldRunWasmBuildTestsOnDefaultPipeline']))
 
         # extra steps, run tests
-        extraStepsTemplate: /eng/pipelines/libraries/helix.yml
-        extraStepsParameters:
-          creator: dotnet-bot
-          testRunNamePrefixSuffix: Mono_$(_BuildConfig)_$(_hostedOs)
-          extraHelixArguments: /p:BrowserHost=$(_hostedOs)
-          scenarios:
-          - buildwasmapps
+        postBuildSteps:
+          - template: /eng/pipelines/libraries/helix.yml
+            parameters:
+              creator: dotnet-bot
+              testRunNamePrefixSuffix: Mono_$(_BuildConfig)_$(_hostedOs)
+              extraHelixArguments: /p:BrowserHost=$(_hostedOs)
+              scenarios:
+              - buildwasmapps
diff --git a/eng/pipelines/common/templates/simple-wasm-build-tests.yml b/eng/pipelines/common/templates/simple-wasm-build-tests.yml
@@ -41,11 +41,12 @@ jobs:
           eq(variables['alwaysRunVar'], true),
           eq(variables['isDefaultPipeline'], variables['shouldRunWasmBuildTestsOnDefaultPipeline']))
       # extra steps, run tests
-      extraStepsTemplate: /eng/pipelines/libraries/helix.yml
-      extraStepsParameters:
-        creator: dotnet-bot
-        testRunNamePrefixSuffix: Mono_$(_BuildConfig)_$(_hostedOs)
-        extraHelixArguments: /p:BrowserHost=$(_hostedOs)
-        scenarios:
-        - buildwasmapps
+      postBuildSteps:
+        - template: /eng/pipelines/libraries/helix.yml
+          parameters:
+            creator: dotnet-bot
+            testRunNamePrefixSuffix: Mono_$(_BuildConfig)_$(_hostedOs)
+            extraHelixArguments: /p:BrowserHost=$(_hostedOs)
+            scenarios:
+            - buildwasmapps
 
diff --git a/eng/pipelines/common/templates/wasm-build-only.yml b/eng/pipelines/common/templates/wasm-build-only.yml
@@ -38,7 +38,8 @@ jobs:
       buildArgs: -s mono+libs+packs+libs.tests$(workloadSubsetArg) -c $(_BuildConfig) /p:BrowserHost=$(_hostedOs) ${{ parameters.extraBuildArgs }} /p:TestAssemblies=false $(extraBuildArgs)
       timeoutInMinutes: 120
       condition: ${{ parameters.condition }}
-      extraStepsTemplate: /eng/pipelines/common/wasm-post-build-steps.yml
-      extraStepsParameters:
-        publishArtifactsForWorkload: ${{ parameters.publishArtifactsForWorkload }}
-        publishWBT: ${{ parameters.publishWBT }}
+      postBuildSteps:
+        - template: /eng/pipelines/common/wasm-post-build-steps.yml
+          parameters:
+            publishArtifactsForWorkload: ${{ parameters.publishArtifactsForWorkload }}
+            publishWBT: ${{ parameters.publishWBT }}
diff --git a/eng/pipelines/common/templates/wasm-debugger-tests.yml b/eng/pipelines/common/templates/wasm-debugger-tests.yml
@@ -52,10 +52,11 @@ jobs:
           and(
             eq(variables['isDefaultPipeline'], variables['shouldRunOnDefaultPipelines']),
             eq(${{ parameters.isWasmOnlyBuild }}, ${{ parameters.runOnlyOnWasmOnlyPipelines }})))
-      extraStepsTemplate: /eng/pipelines/libraries/helix.yml
-      extraStepsParameters:
-        creator: dotnet-bot
-        testRunNamePrefixSuffix: Mono_${{ parameters.browser }}_$(_BuildConfig)
-        extraHelixArguments: /p:BrowserHost=$(_hostedOs) /p:_DebuggerHosts=${{ parameters.browser }}
-        scenarios:
-        - wasmdebuggertests
+      postBuildSteps:
+        - template: /eng/pipelines/libraries/helix.yml
+          parameters:
+            creator: dotnet-bot
+            testRunNamePrefixSuffix: Mono_${{ parameters.browser }}_$(_BuildConfig)
+            extraHelixArguments: /p:BrowserHost=$(_hostedOs) /p:_DebuggerHosts=${{ parameters.browser }}
+            scenarios:
+            - wasmdebuggertests
diff --git a/eng/pipelines/common/templates/wasm-library-tests.yml b/eng/pipelines/common/templates/wasm-library-tests.yml
@@ -62,9 +62,10 @@ jobs:
           eq(variables['alwaysRunVar'], true),
           eq(variables['isDefaultPipeline'], variables['shouldRunOnDefaultPipelines']))
       # extra steps, run tests
-      extraStepsTemplate: /eng/pipelines/libraries/helix.yml
-      extraStepsParameters:
-        creator: dotnet-bot
-        testRunNamePrefixSuffix: Mono_$(_BuildConfig)
-        extraHelixArguments: /p:BrowserHost=$(_hostedOs) $(_wasmRunSmokeTestsOnlyArg) ${{ parameters.extraHelixArgs }}
-        scenarios: ${{ parameters.scenarios }}
+      postBuildSteps:
+        - template: /eng/pipelines/libraries/helix.yml
+          parameters:
+            creator: dotnet-bot
+            testRunNamePrefixSuffix: Mono_$(_BuildConfig)
+            extraHelixArguments: /p:BrowserHost=$(_hostedOs) $(_wasmRunSmokeTestsOnlyArg) ${{ parameters.extraHelixArgs }}
+            scenarios: ${{ parameters.scenarios }}
diff --git a/eng/pipelines/common/templates/wasm-runtime-tests.yml b/eng/pipelines/common/templates/wasm-runtime-tests.yml
@@ -42,9 +42,10 @@ jobs:
         or(
           eq(variables['alwaysRunVar'], true),
           eq(variables['isDefaultPipeline'], variables['shouldRunOnDefaultPipelines']))
-      extraStepsTemplate: //eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml
-      extraStepsParameters:
-        creator: dotnet-bot
-        testRunNamePrefixSuffix: Mono_$(_BuildConfig)
+      postBuildSteps:
+        - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml
+          parameters:
+            creator: dotnet-bot
+            testRunNamePrefixSuffix: Mono_$(_BuildConfig)
       extraVariablesTemplates:
         - template: /eng/pipelines/common/templates/runtimes/test-variables.yml
diff --git a/eng/pipelines/coreclr/ci.yml b/eng/pipelines/coreclr/ci.yml
@@ -174,4 +174,5 @@ extends:
           jobParameters:
             buildArgs: -s clr.paltests+clr.paltestlist
             nameSuffix: PALTests
-            extraStepsTemplate: /eng/pipelines/coreclr/templates/run-paltests-step.yml
+            postBuildSteps:
+              - template: /eng/pipelines/coreclr/templates/run-paltests-step.yml
diff --git a/eng/pipelines/coreclr/perf-non-wasm-jobs.yml b/eng/pipelines/coreclr/perf-non-wasm-jobs.yml
@@ -48,15 +48,16 @@ jobs:
         buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:BuildMonoAOTCrossCompiler=true /p:MonoLibClang="/usr/local/lib/libclang.so.16" /p:AotHostArchitecture=x64 /p:AotHostOS=linux
         nameSuffix: AOT
         isOfficialBuild: false
-        extraStepsTemplate: /eng/pipelines/common/upload-artifact-step.yml
-        extraStepsParameters:
-          rootFolder: '$(Build.SourcesDirectory)/artifacts/'
-          includeRootFolder: true
-          displayName: AOT Mono Artifacts
-          artifactName: LinuxMonoAOTx64
-          archiveExtension: '.tar.gz'
-          archiveType: tar
-          tarCompression: gz
+        postBuildSteps:
+          - template: /eng/pipelines/common/upload-artifact-step.yml
+            parameters:
+              rootFolder: '$(Build.SourcesDirectory)/artifacts/'
+              includeRootFolder: true
+              displayName: AOT Mono Artifacts
+              artifactName: LinuxMonoAOTx64
+              archiveExtension: '.tar.gz'
+              archiveType: tar
+              tarCompression: gz
 
   # build mono Android scenarios
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -70,15 +71,16 @@ jobs:
         buildArgs: -s mono+libs+host+packs -c $(_BuildConfig)
         nameSuffix: AndroidMono
         isOfficialBuild: false
-        extraStepsTemplate: /eng/pipelines/coreclr/templates/build-perf-sample-apps.yml
-        extraStepsParameters:
-          rootFolder: '$(Build.SourcesDirectory)/artifacts/'
-          includeRootFolder: true
-          displayName: Android Mono Artifacts
-          artifactName: AndroidMonoarm64
-          archiveExtension: '.tar.gz'
-          archiveType: tar
-          tarCompression: gz
+        postBuildSteps:
+          - template: /eng/pipelines/coreclr/templates/build-perf-sample-apps.yml
+            parameters:
+              rootFolder: '$(Build.SourcesDirectory)/artifacts/'
+              includeRootFolder: true
+              displayName: Android Mono Artifacts
+              artifactName: AndroidMonoarm64
+              archiveExtension: '.tar.gz'
+              archiveType: tar
+              tarCompression: gz
 
   # build mono iOS scenarios
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -92,15 +94,16 @@ jobs:
         buildArgs: -s mono+libs+host+packs -c $(_BuildConfig)
         nameSuffix: iOSMono
         isOfficialBuild: false
-        extraStepsTemplate: /eng/pipelines/coreclr/templates/build-perf-sample-apps.yml
-        extraStepsParameters:
-          rootFolder: '$(Build.SourcesDirectory)/artifacts/'
-          includeRootFolder: true
-          displayName: iOS Mono Artifacts
-          artifactName: iOSMonoarm64
-          archiveExtension: '.tar.gz'
-          archiveType: tar
-          tarCompression: gz
+        postBuildSteps:
+          - template: /eng/pipelines/coreclr/templates/build-perf-sample-apps.yml
+            parameters:
+              rootFolder: '$(Build.SourcesDirectory)/artifacts/'
+              includeRootFolder: true
+              displayName: iOS Mono Artifacts
+              artifactName: iOSMonoarm64
+              archiveExtension: '.tar.gz'
+              archiveType: tar
+              tarCompression: gz
 
   # build NativeAOT iOS scenarios
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -114,15 +117,16 @@ jobs:
         buildArgs: --cross -s clr.alljits+clr.tools+clr.nativeaotruntime+clr.nativeaotlibs+libs -c $(_BuildConfig)
         nameSuffix: iOSNativeAOT
         isOfficialBuild: false
-        extraStepsTemplate: /eng/pipelines/coreclr/templates/build-perf-sample-apps.yml
-        extraStepsParameters:
-          rootFolder: '$(Build.SourcesDirectory)/artifacts/'
-          includeRootFolder: true
-          displayName: iOS NativeAOT Artifacts
-          artifactName: iOSNativeAOTarm64
-          archiveExtension: '.tar.gz'
-          archiveType: tar
-          tarCompression: gz
+        postBuildSteps:
+          - template: /eng/pipelines/coreclr/templates/build-perf-sample-apps.yml
+            parameters:
+              rootFolder: '$(Build.SourcesDirectory)/artifacts/'
+              includeRootFolder: true
+              displayName: iOS NativeAOT Artifacts
+              artifactName: iOSNativeAOTarm64
+              archiveExtension: '.tar.gz'
+              archiveType: tar
+              tarCompression: gz
 
   # build mono
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -409,9 +413,10 @@ jobs:
         buildArgs: -s mono+libs+host+packs -c $(_BuildConfig)
         nameSuffix: Mono_Packs
         isOfficialBuild: false
-        extraStepsTemplate: /eng/pipelines/common/upload-intermediate-artifacts-step.yml
-        extraStepsParameters:
-          name: MonoRuntimePacks
+        postBuildSteps:
+          - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml
+            parameters:
+              name: MonoRuntimePacks
 
   # build PerfBDN app
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -429,12 +434,13 @@ jobs:
         isOfficialBuild: false
         pool:
           vmImage: 'macos-12'
-        extraStepsTemplate: /eng/pipelines/coreclr/templates/build-perf-bdn-app.yml
-        extraStepsParameters:
-          rootFolder: '$(Build.SourcesDirectory)/artifacts/'
-          includeRootFolder: true
-          displayName: Android BDN App Artifacts
-          artifactName: PerfBDNAppArm
-          archiveExtension: '.tar.gz'
-          archiveType: tar
-          tarCompression: gz
+        postBuildSteps:
+          - template: /eng/pipelines/coreclr/templates/build-perf-bdn-app.yml
+            parameters:
+              rootFolder: '$(Build.SourcesDirectory)/artifacts/'
+              includeRootFolder: true
+              displayName: Android BDN App Artifacts
+              artifactName: PerfBDNAppArm
+              archiveExtension: '.tar.gz'
+              archiveType: tar
+              tarCompression: gz
diff --git a/eng/pipelines/coreclr/perf-wasm-jobs.yml b/eng/pipelines/coreclr/perf-wasm-jobs.yml
@@ -25,9 +25,10 @@ jobs:
           buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS)
           nameSuffix: wasm
           isOfficialBuild: false
-          extraStepsTemplate: /eng/pipelines/coreclr/perf-wasm-prepare-artifacts-steps.yml
-          extraStepsParameters:
-            configForBuild: Release
+          postBuildSteps:
+            - template: /eng/pipelines/coreclr/perf-wasm-prepare-artifacts-steps.yml
+              parameters:
+                configForBuild: Release
 
   #run mono wasm microbenchmarks perf job
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -94,9 +95,10 @@ jobs:
           buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS)
           nameSuffix: wasm
           isOfficialBuild: false
-          extraStepsTemplate: /eng/pipelines/coreclr/perf-wasm-prepare-artifacts-steps.yml
-          extraStepsParameters:
-            configForBuild: Release
+          postBuildSteps:
+            - template: /eng/pipelines/coreclr/perf-wasm-prepare-artifacts-steps.yml
+              parameters:
+                configForBuild: Release
 
   # run mono wasm interpreter (default) microbenchmarks perf job
   - template: /eng/pipelines/common/platform-matrix.yml

diff --git a/eng/pipelines/coreclr/perf_slow.yml b/eng/pipelines/coreclr/perf_slow.yml
@@ -143,15 +143,16 @@ extends:
               buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:MonoAOTEnableLLVM=true /p:MonoBundleLLVMOptimizer=true /p:BuildMonoAOTCrossCompiler=true /p:MonoLibClang="/usr/local/lib/libclang.so.16" /p:AotHostArchitecture=arm64 /p:AotHostOS=linux
               nameSuffix: AOT
               isOfficialBuild: false
-              extraStepsTemplate: /eng/pipelines/common/upload-artifact-step.yml
-              extraStepsParameters:
-                rootFolder: '$(Build.SourcesDirectory)/artifacts/'
-                includeRootFolder: true
-                displayName: AOT Mono Artifacts
-                artifactName: LinuxMonoAOTarm64
-                archiveExtension: '.tar.gz'
-                archiveType: tar
-                tarCompression: gz
+              postBuildSteps:
+                - template: /eng/pipelines/common/upload-artifact-step.yml
+                  parameters:
+                    rootFolder: '$(Build.SourcesDirectory)/artifacts/'
+                    includeRootFolder: true
+                    displayName: AOT Mono Artifacts
+                    artifactName: LinuxMonoAOTarm64
+                    archiveExtension: '.tar.gz'
+                    archiveType: tar
+                    tarCompression: gz
 
         # run mono aot microbenchmarks perf job
         - template: /eng/pipelines/common/platform-matrix.yml