From d9e23eee897cd47b7de8ccc2e0154a2ba5adba36 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Thu, 7 Aug 2025 16:37:43 +0000 Subject: [PATCH 01/69] create junit artifacts and param for running bm tests --- .pipelines/templates/e2e-template.yml | 53 +++++++++++-------- .../junit/handle-junit-test-results.yml | 32 +++++++++++ .../e2e-ab-update-stage-finalize-test-run.yml | 13 +++-- .../stages/testing_common/e2e-test-run.yml | 42 +++++++-------- .../testing_functional/functional-testing.yml | 12 +++-- .../trident-platform-cicd-template.yml | 5 ++ .pipelines/trident-cicd.yml | 1 + 7 files changed, 102 insertions(+), 56 deletions(-) create mode 100644 .pipelines/templates/stages/junit/handle-junit-test-results.yml diff --git a/.pipelines/templates/e2e-template.yml b/.pipelines/templates/e2e-template.yml index 212d548c1..50e0d7075 100644 --- a/.pipelines/templates/e2e-template.yml +++ b/.pipelines/templates/e2e-template.yml @@ -66,6 +66,11 @@ parameters: type: string default: "submodule" + - name: runBaremetalTests + displayName: "Specify whether to run BareMetal tests" + type: boolean + default: true + stages: # Trident Build/Download - template: stages/trident_rpms/trident-stage.yml @@ -242,18 +247,19 @@ stages: buildPurpose: "daily" runtimeEnv: "host" - # BM Testing (host, daily) - - template: stages/testing_baremetal/baremetal-testing.yml - parameters: - buildPurpose: "daily" - runtimeEnv: "host" + - ${{ if parameters.runBaremetalTests }}: + # BM Testing (host, daily) + - template: stages/testing_baremetal/baremetal-testing.yml + parameters: + buildPurpose: "daily" + runtimeEnv: "host" - # BM Testing (container, daily) - - template: stages/testing_baremetal/baremetal-testing.yml - parameters: - buildPurpose: "daily" - runtimeEnv: "container" - extraDependsOn: "BaremetalDeploymentTesting_host" + # BM Testing (container, daily) + - template: stages/testing_baremetal/baremetal-testing.yml + parameters: + buildPurpose: "daily" + runtimeEnv: "container" + extraDependsOn: "BaremetalDeploymentTesting_host" # VM Testing (container, daily) - template: stages/testing_vm/netlaunch-testing.yml @@ -278,18 +284,19 @@ stages: buildPurpose: "weekly" runtimeEnv: "container" - # BM Testing (host, weekly) - - template: stages/testing_baremetal/baremetal-testing.yml - parameters: - buildPurpose: "weekly" - runtimeEnv: "host" - - # BM Testing (container, weekly) - - template: stages/testing_baremetal/baremetal-testing.yml - parameters: - buildPurpose: "weekly" - runtimeEnv: "container" - extraDependsOn: "BaremetalDeploymentTesting_host" + - ${{ if parameters.runBaremetalTests }}: + # BM Testing (host, weekly) + - template: stages/testing_baremetal/baremetal-testing.yml + parameters: + buildPurpose: "weekly" + runtimeEnv: "host" + + # BM Testing (container, weekly) + - template: stages/testing_baremetal/baremetal-testing.yml + parameters: + buildPurpose: "weekly" + runtimeEnv: "container" + extraDependsOn: "BaremetalDeploymentTesting_host" # TESTING stages for CI - ${{ if eq(parameters.stageType, 'ci') }}: diff --git a/.pipelines/templates/stages/junit/handle-junit-test-results.yml b/.pipelines/templates/stages/junit/handle-junit-test-results.yml new file mode 100644 index 000000000..3a92a691e --- /dev/null +++ b/.pipelines/templates/stages/junit/handle-junit-test-results.yml @@ -0,0 +1,32 @@ +parameters: + - name: testRunName + type: string + - name: junitTestFile + type: string + - name: displayNameSpecifier + type: string + - name: artifactName + type: string + +steps: + - task: PublishTestResults@2 + condition: and(succeededOrFailed(), ne(variables['abActiveVolume'], 'null')) + inputs: + testResultsFormat: JUnit + testResultsFiles: ${{ parameters.junitTestFile }} + testRunTitle: ${{ parameters.testRunName }} + displayName: "Publish junit test results for ${{ parameters.displayNameSpecifier }}" + + - bash: | + rm -rf /tmp/ONEBRANCH_ARTIFACT + mkdir -p /tmp/ONEBRANCH_ARTIFACT + cp ${{ parameters.junitTestFile }} /tmp/ONEBRANCH_ARTIFACT/ + displayName: "Prepare JUnit XML for ${{ parameters.displayNameSpecifier }}" + condition: and(succeededOrFailed(), ne(variables['abActiveVolume'], 'null')) + + - task: PublishBuildArtifacts@1 + displayName: "Publish JUnit XML for ${{ parameters.displayNameSpecifier }}" + condition: and(succeededOrFailed(), ne(variables['abActiveVolume'], 'null')) + inputs: + pathToPublish: /tmp/ONEBRANCH_ARTIFACT/ + artifactName: ${{ parameters.artifactName }} diff --git a/.pipelines/templates/stages/testing_common/e2e-ab-update-stage-finalize-test-run.yml b/.pipelines/templates/stages/testing_common/e2e-ab-update-stage-finalize-test-run.yml index 898a7088b..779ce3717 100644 --- a/.pipelines/templates/stages/testing_common/e2e-ab-update-stage-finalize-test-run.yml +++ b/.pipelines/templates/stages/testing_common/e2e-ab-update-stage-finalize-test-run.yml @@ -211,10 +211,9 @@ steps: displayName: "🔬 Run Trident E2E tests after A/B update into runtime OS B" condition: and(succeeded(), ne(variables['abActiveVolume'], 'null')) - - task: PublishTestResults@2 - condition: and(succeededOrFailed(), ne(variables['abActiveVolume'], 'null')) - inputs: - testResultsFormat: JUnit - testResultsFiles: ./e2e_tests/${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_stage_finalize_ab_update_B_$(System.JobAttempt).junit.xml - testRunTitle: ${{ parameters.deploymentEnvironment }}_trident_e2e_tests_${{ parameters.tridentConfigurationName }}_stage_finalize_ab_update_B_$(System.JobAttempt) - displayName: "Publish test results for A/B update into runtime OS A that was staged and finalized independently" + - template: ../junit/handle-junit-test-results.yml + parameters: + testRunName: "${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_stage_finalize_ab_update_B_$(System.JobAttempt)" + junitTestFile: "./e2e_tests/${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_stage_finalize_ab_update_B_$(System.JobAttempt).junit.xml" + displayNameSpecifier: "A/B update into runtime OS A that was staged and finalized independently" + artifactName: "ab-update-stage" diff --git a/.pipelines/templates/stages/testing_common/e2e-test-run.yml b/.pipelines/templates/stages/testing_common/e2e-test-run.yml index 0484e4433..7fcd43c4d 100644 --- a/.pipelines/templates/stages/testing_common/e2e-test-run.yml +++ b/.pipelines/templates/stages/testing_common/e2e-test-run.yml @@ -100,13 +100,13 @@ steps: workingDirectory: $(Build.SourcesDirectory)/e2e_tests displayName: "🔬 Run Trident E2E tests after clean install of runtime OS" - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFormat: JUnit - testResultsFiles: ./e2e_tests/${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_clean_install_$(System.JobAttempt).junit.xml - testRunTitle: ${{ parameters.deploymentEnvironment }}_trident_e2e_tests_${{ parameters.tridentConfigurationName }}_clean_install_$(System.JobAttempt) - displayName: "Publish test results for clean install of runtime OS" + + - template: ../junit/handle-junit-test-results.yml + parameters: + testRunName: "${{ parameters.deploymentEnvironment }}_trident_e2e_tests_${{ parameters.tridentConfigurationName }}_clean_install_$(System.JobAttempt)" + junitTestFile: "./e2e_tests/${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_clean_install_$(System.JobAttempt).junit.xml" + displayNameSpecifier: "clean install of runtime OS" + artifactName: "trident_clean_install_test_results" # If current config requires A/B update testing, execute script to ssh into the host, update # images in the custom Trident config, and re-run Trident to both stage and finalize A/B update. @@ -204,13 +204,13 @@ steps: displayName: "🔬 Run Trident E2E tests after A/B update into runtime OS B" condition: and(succeeded(), ne(variables['abActiveVolume'], 'null')) - - task: PublishTestResults@2 - condition: and(succeededOrFailed(), ne(variables['abActiveVolume'], 'null')) - inputs: - testResultsFormat: JUnit - testResultsFiles: ./e2e_tests/${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_ab_update_B_$(System.JobAttempt).junit.xml - testRunTitle: ${{ parameters.deploymentEnvironment }}_trident_e2e_tests_${{ parameters.tridentConfigurationName }}_ab_update_B_$(System.JobAttempt) - displayName: "Publish test results for A/B update into runtime OS B" + + - template: ../junit/handle-junit-test-results.yml + parameters: + testRunName: "${{ parameters.deploymentEnvironment }}_trident_e2e_tests_${{ parameters.tridentConfigurationName }}_ab_update_B_$(System.JobAttempt)" + junitTestFile: "./e2e_tests/${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_ab_update_B_$(System.JobAttempt).junit.xml" + displayNameSpecifier: "A/B update into runtime OS B" + artifactName: "trident_ab_update_B_test_results" - bash: | set -eux @@ -307,13 +307,13 @@ steps: displayName: "🔬 Run Trident E2E tests after A/B update into runtime OS A" condition: and(succeeded(), ne(variables['abActiveVolume'], 'null')) - - task: PublishTestResults@2 - condition: and(succeededOrFailed(), ne(variables['abActiveVolume'], 'null')) - inputs: - testResultsFormat: JUnit - testResultsFiles: ./e2e_tests/${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_ab_update_A_$(System.JobAttempt).junit.xml - testRunTitle: ${{ parameters.deploymentEnvironment }}_trident_e2e_tests_${{ parameters.tridentConfigurationName }}_ab_update_A_$(System.JobAttempt) - displayName: "Publish test results for A/B update into runtime OS A" + + - template: ../junit/handle-junit-test-results.yml + parameters: + testRunName: "${{ parameters.deploymentEnvironment }}_trident_e2e_tests_${{ parameters.tridentConfigurationName }}_ab_update_A_$(System.JobAttempt)" + junitTestFile: "./e2e_tests/${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_ab_update_A_$(System.JobAttempt).junit.xml" + displayNameSpecifier: "A/B update into runtime OS A" + artifactName: "trident_ab_update_A_test_results" # Check the value of 'buildPurpose', to determine if we need to execute the # test scenario where A/B update is staged and finalized independently. This diff --git a/.pipelines/templates/stages/testing_functional/functional-testing.yml b/.pipelines/templates/stages/testing_functional/functional-testing.yml index ed4d0ecef..15e9d2c62 100644 --- a/.pipelines/templates/stages/testing_functional/functional-testing.yml +++ b/.pipelines/templates/stages/testing_functional/functional-testing.yml @@ -115,11 +115,13 @@ stages: installNativeDependencies: false rustup: false - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFormat: "JUnit" - testResultsFiles: "$(Build.SourcesDirectory)/target/trident_functional_tests.xml" + + - template: ../junit/handle-junit-test-results.yml + parameters: + testRunName: "FunctionalTests_$(System.JobAttempt)" + junitTestFile: "$(Build.SourcesDirectory)/target/trident_functional_tests.xml" + displayNameSpecifier: "functional tests" + artifactName: "trident-functional" - bash: | set -eux diff --git a/.pipelines/templates/trident-platform-cicd-template.yml b/.pipelines/templates/trident-platform-cicd-template.yml index 949916fe8..c69327ef4 100644 --- a/.pipelines/templates/trident-platform-cicd-template.yml +++ b/.pipelines/templates/trident-platform-cicd-template.yml @@ -17,6 +17,10 @@ parameters: values: - dev - preview + - name: runBaremetalTests + displayName: "Specify whether to run BareMetal tests" + type: boolean + default: false stages: - template: e2e-template.yml @@ -27,3 +31,4 @@ stages: baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} baseImageArm64PipelineBuildId: ${{ parameters.baseImageArm64PipelineBuildId }} forceTridentRebuild: true + runBaremetalTests: ${{ parameters.runBaremetalTests }} diff --git a/.pipelines/trident-cicd.yml b/.pipelines/trident-cicd.yml index 040b3208f..5f5f85209 100644 --- a/.pipelines/trident-cicd.yml +++ b/.pipelines/trident-cicd.yml @@ -51,3 +51,4 @@ extends: buildType: ${{ parameters.buildType }} baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} baseImageArm64PipelineBuildId: ${{ parameters.baseImageArm64PipelineBuildId }} + runBaremetalTests: true From 4398b56de99c29dc8f296d8682545a5723ec6db5 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Thu, 7 Aug 2025 17:03:20 +0000 Subject: [PATCH 02/69] rename to collocate artifacts --- .../e2e-ab-update-stage-finalize-test-run.yml | 2 +- .pipelines/templates/stages/testing_common/e2e-test-run.yml | 6 +++--- .../stages/testing_functional/functional-testing.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.pipelines/templates/stages/testing_common/e2e-ab-update-stage-finalize-test-run.yml b/.pipelines/templates/stages/testing_common/e2e-ab-update-stage-finalize-test-run.yml index 779ce3717..79bf7ca9f 100644 --- a/.pipelines/templates/stages/testing_common/e2e-ab-update-stage-finalize-test-run.yml +++ b/.pipelines/templates/stages/testing_common/e2e-ab-update-stage-finalize-test-run.yml @@ -216,4 +216,4 @@ steps: testRunName: "${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_stage_finalize_ab_update_B_$(System.JobAttempt)" junitTestFile: "./e2e_tests/${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_stage_finalize_ab_update_B_$(System.JobAttempt).junit.xml" displayNameSpecifier: "A/B update into runtime OS A that was staged and finalized independently" - artifactName: "ab-update-stage" + artifactName: "junit_for_trident_ab_update_stage" diff --git a/.pipelines/templates/stages/testing_common/e2e-test-run.yml b/.pipelines/templates/stages/testing_common/e2e-test-run.yml index 7fcd43c4d..e01ee7c57 100644 --- a/.pipelines/templates/stages/testing_common/e2e-test-run.yml +++ b/.pipelines/templates/stages/testing_common/e2e-test-run.yml @@ -106,7 +106,7 @@ steps: testRunName: "${{ parameters.deploymentEnvironment }}_trident_e2e_tests_${{ parameters.tridentConfigurationName }}_clean_install_$(System.JobAttempt)" junitTestFile: "./e2e_tests/${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_clean_install_$(System.JobAttempt).junit.xml" displayNameSpecifier: "clean install of runtime OS" - artifactName: "trident_clean_install_test_results" + artifactName: "junit_for_trident_clean_install" # If current config requires A/B update testing, execute script to ssh into the host, update # images in the custom Trident config, and re-run Trident to both stage and finalize A/B update. @@ -210,7 +210,7 @@ steps: testRunName: "${{ parameters.deploymentEnvironment }}_trident_e2e_tests_${{ parameters.tridentConfigurationName }}_ab_update_B_$(System.JobAttempt)" junitTestFile: "./e2e_tests/${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_ab_update_B_$(System.JobAttempt).junit.xml" displayNameSpecifier: "A/B update into runtime OS B" - artifactName: "trident_ab_update_B_test_results" + artifactName: "junit_for_trident_ab_update_B" - bash: | set -eux @@ -313,7 +313,7 @@ steps: testRunName: "${{ parameters.deploymentEnvironment }}_trident_e2e_tests_${{ parameters.tridentConfigurationName }}_ab_update_A_$(System.JobAttempt)" junitTestFile: "./e2e_tests/${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_ab_update_A_$(System.JobAttempt).junit.xml" displayNameSpecifier: "A/B update into runtime OS A" - artifactName: "trident_ab_update_A_test_results" + artifactName: "junit_for_trident_ab_update_A" # Check the value of 'buildPurpose', to determine if we need to execute the # test scenario where A/B update is staged and finalized independently. This diff --git a/.pipelines/templates/stages/testing_functional/functional-testing.yml b/.pipelines/templates/stages/testing_functional/functional-testing.yml index 15e9d2c62..627dce523 100644 --- a/.pipelines/templates/stages/testing_functional/functional-testing.yml +++ b/.pipelines/templates/stages/testing_functional/functional-testing.yml @@ -121,7 +121,7 @@ stages: testRunName: "FunctionalTests_$(System.JobAttempt)" junitTestFile: "$(Build.SourcesDirectory)/target/trident_functional_tests.xml" displayNameSpecifier: "functional tests" - artifactName: "trident-functional" + artifactName: "junit_for_trident_functionaltests" - bash: | set -eux From 4ae7a4b374cf04e93c0cf75833152d010c7e14f7 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Thu, 7 Aug 2025 18:27:22 +0000 Subject: [PATCH 03/69] dependsOn, baseimageconfig artifact --- .pipelines/templates/e2e-template.yml | 70 +++++----- .pipelines/templates/scale-template.yml | 11 +- .../build_docker_image/trident-container.yml | 31 +---- .../stages/build_image/build-image.yml | 28 ++-- .../stages/building_tools/building-tools.yml | 6 + .../create-base-image-config-artifact.yml | 63 +++++++++ .../common_tasks/find-base-image-version.yml | 127 ++++++++++-------- .../stages/common_tasks/preview-container.yml | 25 +++- .../read-base-image-config-artifact.yml | 26 ++++ .../download_staged/download-staged.yml | 6 + .../testing_functional/functional-testing.yml | 6 + .../stages/testing_servicing/build-image.yml | 20 +-- .../stages/testing_servicing/vm-testing.yml | 32 ----- .../stages/trident_rpms/build-source.yml | 33 ++--- .../templates/stages/trident_rpms/release.yml | 8 +- .../stages/trident_rpms/trident-stage.yml | 26 +--- .../trident_usb_iso/trident-usb-iso.yml | 8 +- .../stages/validate_makefile/dev-build.yml | 6 + .../trident-platform-cicd-template.yml | 9 +- .pipelines/trident-cicd.yml | 39 +++++- 20 files changed, 328 insertions(+), 252 deletions(-) create mode 100644 .pipelines/templates/stages/common_tasks/create-base-image-config-artifact.yml create mode 100644 .pipelines/templates/stages/common_tasks/read-base-image-config-artifact.yml diff --git a/.pipelines/templates/e2e-template.yml b/.pipelines/templates/e2e-template.yml index 50e0d7075..6e41b5781 100644 --- a/.pipelines/templates/e2e-template.yml +++ b/.pipelines/templates/e2e-template.yml @@ -67,31 +67,48 @@ parameters: default: "submodule" - name: runBaremetalTests - displayName: "Specify whether to run BareMetal tests" type: boolean default: true + - name: createBaseImageConfigArtifact + type: boolean + default: true + + - name: previousStage + type: string + default: '' + stages: + - template: stages/common_tasks/create-base-image-config-artifact.yml + parameters: + stageSuffix: e2e + baseimgBuildType: ${{ parameters.baseimgBuildType }} + baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} + baseImageArm64PipelineBuildId: ${{ parameters.baseImageArm64PipelineBuildId }} + previousStage: ${{ parameters.previousStage }} + createBaseImageConfigArtifact: ${{ parameters.createBaseImageConfigArtifact }} + # Trident Build/Download - template: stages/trident_rpms/trident-stage.yml parameters: stageType: ${{ parameters.stageType }} tridentArtifactName: trident-binaries codeCoverage: false - baseimgBuildType: ${{ parameters.baseimgBuildType }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} - baseImageArm64PipelineBuildId: ${{ parameters.baseImageArm64PipelineBuildId }} forceTridentRebuild: ${{ parameters.forceTridentRebuild }} osModifierBranch: ${{ parameters.osModifierBranch }} + dependsOnStage: HandleBaseImageConfig_e2e # Build tools (Go stuff) - template: stages/building_tools/building-tools.yml + parameters: + dependsOnStage: HandleBaseImageConfig_e2e # Makefile validation, only for CI - ${{ if eq(parameters.stageType, 'ci') }}: - template: stages/validate_makefile/dev-build.yml parameters: osModifierBranch: ${{ parameters.osModifierBranch }} + dependsOnStage: HandleBaseImageConfig_e2e # Build FT base Image, only in CI @@ -101,112 +118,97 @@ stages: parameters: imageName: trident-functest dependsOnTrident: false - baseimgBuildType: ${{ parameters.baseimgBuildType }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} clones: 1 + dependsOnStage: HandleBaseImageConfig_e2e - # Build Trident container image + # Build Trident container image (depends on GetTridentBinaries_rpms) - template: stages/build_docker_image/trident-container.yml - parameters: - baseimgBuildType: ${{ parameters.baseimgBuildType }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} # Build Trident installer ISO (host) - template: stages/build_image/build-image.yml parameters: imageName: trident-installer - baseimgBuildType: ${{ parameters.baseimgBuildType }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} clones: 1 + dependsOnStage: HandleBaseImageConfig_e2e # Build Trident split (stage and finalize separated) installer ISO (host) - template: stages/build_image/build-image.yml parameters: imageName: trident-split-installer - baseimgBuildType: ${{ parameters.baseimgBuildType }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} clones: 1 + dependsOnStage: HandleBaseImageConfig_e2e # Build Trident installer ISO (container) - template: stages/build_image/build-image.yml parameters: imageName: trident-container-installer - baseimgBuildType: ${{ parameters.baseimgBuildType }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} runtimeEnv: "container" micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} clones: 1 + dependsOnStage: HandleBaseImageConfig_e2e # Build Trident test image (regular) - template: stages/build_image/build-image.yml parameters: imageName: trident-testimage - baseimgBuildType: ${{ parameters.baseimgBuildType }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} + dependsOnStage: HandleBaseImageConfig_e2e # Build Trident test image (container) - template: stages/build_image/build-image.yml parameters: imageName: trident-container-testimage - baseimgBuildType: ${{ parameters.baseimgBuildType }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} runtimeEnv: "container" micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} + dependsOnStage: HandleBaseImageConfig_e2e # Build Trident test image for verity (host) - template: stages/build_image/build-image.yml parameters: imageName: trident-verity-testimage - baseimgBuildType: ${{ parameters.baseimgBuildType }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} + dependsOnStage: HandleBaseImageConfig_e2e # Build Trident test image for verity (container) - template: stages/build_image/build-image.yml parameters: imageName: trident-container-verity-testimage - baseimgBuildType: ${{ parameters.baseimgBuildType }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} runtimeEnv: "container" micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} + dependsOnStage: HandleBaseImageConfig_e2e # Build Trident test image for usr-verity (host) - template: stages/build_image/build-image.yml parameters: imageName: trident-usrverity-testimage - baseimgBuildType: ${{ parameters.baseimgBuildType }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} + dependsOnStage: HandleBaseImageConfig_e2e # Build Trident test image for usr-verity (container) - template: stages/build_image/build-image.yml parameters: imageName: trident-container-usrverity-testimage runtimeEnv: "container" - baseimgBuildType: ${{ parameters.baseimgBuildType }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} + dependsOnStage: HandleBaseImageConfig_e2e - ${{ if or(parameters.forceTridentRebuild, eq(parameters.stageType, 'pr-e2e'), eq(parameters.stageType, 'ci'), eq(parameters.stageType, 'pr-e2e-azure')) }}: # Build USB-ISO - template: stages/trident_usb_iso/trident-usb-iso.yml parameters: - baseimgBuildType: ${{ parameters.baseimgBuildType }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} - ${{ else }}: @@ -226,8 +228,6 @@ stages: # Test Servicing on VMs - template: stages/testing_servicing/vm-testing.yml parameters: - baseimgBuildType: ${{ parameters.baseimgBuildType }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} includeQemu: true includeUKI: true includeAzure: ${{ parameters.includeAzure }} @@ -240,6 +240,8 @@ stages: - ${{ if eq(parameters.stageType, 'pre') }}: # Functional Testing - template: stages/testing_functional/functional-testing.yml + parameters: + dependsOnStage: HandleBaseImageConfig_e2e # VM Testing (host, daily) - template: stages/testing_vm/netlaunch-testing.yml @@ -271,6 +273,8 @@ stages: - ${{ if eq(parameters.stageType, 'full-validation') }}: # Functional Testing - template: stages/testing_functional/functional-testing.yml + parameters: + dependsOnStage: HandleBaseImageConfig_e2e # VM Testing (host, weekly) - template: stages/testing_vm/netlaunch-testing.yml @@ -305,6 +309,7 @@ stages: parameters: downloadPrebuiltImage: false osModifierBranch: ${{ parameters.osModifierBranch }} + dependsOnStage: HandleBaseImageConfig_e2e # VM Testing (host, post_merge) @@ -325,6 +330,7 @@ stages: - template: stages/testing_functional/functional-testing.yml parameters: rerunTests: false + dependsOnStage: HandleBaseImageConfig_e2e # VM Testing (host, pullrequest) - template: stages/testing_vm/netlaunch-testing.yml diff --git a/.pipelines/templates/scale-template.yml b/.pipelines/templates/scale-template.yml index a8e4bed98..913978fbd 100644 --- a/.pipelines/templates/scale-template.yml +++ b/.pipelines/templates/scale-template.yml @@ -68,6 +68,12 @@ parameters: default: false stages: + - template: stages/common_tasks/create-base-image-config-artifact.yml + parameters: + baseimgBuildType: ${{ parameters.baseimgBuildType }} + baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} + baseImageArm64PipelineBuildId: ${{ parameters.baseImageArm64PipelineBuildId }} + # Build tools (Go stuff) - template: stages/building_tools/building-tools.yml @@ -77,21 +83,16 @@ stages: stageType: ${{ parameters.stageType }} tridentArtifactName: trident-binaries codeCoverage: false - baseimgBuildType: ${{ parameters.baseimgBuildType }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} - baseImageArm64PipelineBuildId: ${{ parameters.baseImageArm64PipelineBuildId }} forceTridentRebuild: ${{ or(eq(parameters.forceTridentRebuild, true), eq(parameters.baseimgBuildType, 'preview')) }} # Test Servicing on VMs - template: stages/testing_servicing/vm-testing.yml parameters: - baseimgBuildType: ${{ parameters.baseimgBuildType }} rollbackTesting: false updateIterationCount: ${{ parameters.numberOfUpdateIterations }} workers: ${{ parameters.numberOfWorkers }} updateCheckTimeoutInMinutes: 200 # Generally need less than a minute per update, so with buffer, this should be updateIterationCount * 2 verboseLogging: ${{ parameters.verboseLogging }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} baseimgAzlVersion: ${{ parameters.baseimgAzlVersion }} includeQemu: ${{ parameters.includeQemu }} includeAzure: ${{ parameters.includeAzure }} diff --git a/.pipelines/templates/stages/build_docker_image/trident-container.yml b/.pipelines/templates/stages/build_docker_image/trident-container.yml index 40cc7ae13..aea729aae 100644 --- a/.pipelines/templates/stages/build_docker_image/trident-container.yml +++ b/.pipelines/templates/stages/build_docker_image/trident-container.yml @@ -1,21 +1,8 @@ -parameters: - - name: baseimgBuildType - displayName: Base Image build type - type: string - values: - - dev - - preview - - release - default: "release" - - - name: baseImagePipelineBuildId - type: string - default: "latestFromBranch" - stages: - stage: BuildTridentContainerImage displayName: Build Docker Image - dependsOn: GetTridentBinaries_rpms + dependsOn: + - GetTridentBinaries_rpms jobs: - job: BuildImage @@ -44,16 +31,10 @@ stages: patterns: "**/*.rpm" targetPath: "$(Build.SourcesDirectory)/bin/RPMS/x86_64" - - ${{ if eq(parameters.baseimgBuildType, 'preview') }}: - - template: ../common_tasks/preview-container.yml - parameters: - dockerfilePath: $(Build.SourcesDirectory)/Dockerfile.runtime - dockerBuildContext: $(Build.SourcesDirectory) - ${{ if eq(parameters.baseImagePipelineBuildId, 'latestFromBranch') }}: - runVersion: ${{ parameters.baseImagePipelineBuildId }} - ${{ else }}: - runVersion: specific - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} + - template: ../common_tasks/preview-container.yml + parameters: + dockerfilePath: $(Build.SourcesDirectory)/Dockerfile.runtime + dockerBuildContext: $(Build.SourcesDirectory) - script: | set -euxo pipefail diff --git a/.pipelines/templates/stages/build_image/build-image.yml b/.pipelines/templates/stages/build_image/build-image.yml index 205474fa9..1523bb959 100644 --- a/.pipelines/templates/stages/build_image/build-image.yml +++ b/.pipelines/templates/stages/build_image/build-image.yml @@ -2,20 +2,6 @@ parameters: - name: imageName type: string - - name: baseimgBuildType - displayName: Base Image build type - type: string - values: - - dev - - preview - - release - default: "release" - - - name: baseImagePipelineBuildId - displayName: "Build Id of the pipeline run, default will select latest successful run from pipeline 2116 ([AMD64-6-OneBranch]-Prod-BuildImages) with tag 3.0-preview" - type: string - default: "latestFromBranch" - - name: runtimeEnv displayName: "Runtime environment (host vs container)" type: string @@ -47,12 +33,18 @@ parameters: type: boolean default: true + - name: dependsOnStage + type: string + default: '' + stages: - stage: TridentTestImg_${{ replace(parameters.imageName, '-', '_') }} displayName: Build ${{ parameters.imageName }} dependsOn: - ${{ if and(eq(parameters.runtimeEnv, 'host'), parameters.dependsOnTrident) }}: - GetTridentBinaries_rpms + - ${{ elseif ne(parameters.dependsOnStage, '') }}: + - ${{ parameters.dependsOnStage }} - ${{ else }}: [] jobs: @@ -79,16 +71,12 @@ stages: condition: and(eq('${{ parameters.runtimeEnv }}', 'host'), eq('${{ parameters.dependsOnTrident }}', true)) - template: ../common_tasks/find-base-image-version.yml - parameters: - baseimgBuildType: ${{ parameters.baseimgBuildType }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} - - template: .pipelines/templates/build-image.yml@test-images parameters: imageName: ${{ parameters.imageName }} clones: ${{ parameters.clones }} - baseimgBuildType: ${{ parameters.baseimgBuildType }} - baseimgVersion: $(baseimgVersion) + baseimgBuildType: $(BASEIMG_BUILD_TYPE) + baseimgVersion: $(BASEIMG_VERSION) azureLinuxVersion: ${{ variables.BASEIMG_AZURE_LINUX_VERSION }} micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} diff --git a/.pipelines/templates/stages/building_tools/building-tools.yml b/.pipelines/templates/stages/building_tools/building-tools.yml index ff5e6c694..aa027e538 100644 --- a/.pipelines/templates/stages/building_tools/building-tools.yml +++ b/.pipelines/templates/stages/building_tools/building-tools.yml @@ -19,9 +19,15 @@ parameters: type: boolean default: true + - name: dependsOnStage + type: string + default: '' + stages: - stage: BuildingTools displayName: Building Tools + ${{ if ne(parameters.dependsOnStage, '') }}: + dependsOn: ${{ parameters.dependsOnStage }} jobs: - job: BuildGoTools diff --git a/.pipelines/templates/stages/common_tasks/create-base-image-config-artifact.yml b/.pipelines/templates/stages/common_tasks/create-base-image-config-artifact.yml new file mode 100644 index 000000000..1b0d7b442 --- /dev/null +++ b/.pipelines/templates/stages/common_tasks/create-base-image-config-artifact.yml @@ -0,0 +1,63 @@ +parameters: + - name: stageSuffix + type: string + + - name: baseimgBuildType + type: string + values: + - dev + - preview + - release + default: "release" + + - name: baseImagePipelineBuildId + type: string + default: "latestFromBranch" + + - name: baseImageArm64PipelineBuildId + type: string + default: "latestFromBranch" + + - name: createBaseImageConfigArtifact + type: boolean + default: true + + - name: previousStage + type: string + default: '' + +stages: + - stage: HandleBaseImageConfig_${{ parameters.stageSuffix }} + displayName: "Create BaseImage config artifact (${{ parameters.stageSuffix }})" + ${{ if ne(parameters.previousStage, '') }}: + dependsOn: ${{ parameters.previousStage }} + jobs: + - job: HandleBaseImageConfigJob_${{ parameters.stageSuffix }} + displayName: "Create BaseImage config artifact (${{ parameters.stageSuffix }})" + pool: + type: linux + variables: + - name: ob_outputDirectory + value: "$(Build.SourcesDirectory)/out" + - name: ob_artifactBaseName + value: "HandleBaseImageConfig_${{ parameters.stageSuffix }}" + steps: + - ${{ if parameters.createBaseImageConfigArtifact }}: + - bash: | + mkdir -p /tmp/config + cat << EOF > /tmp/config/baseimage.json + { + "baseimgBuildType": "${{ parameters.baseimgBuildType }}", + "baseImagePipelineBuildId": "${{ parameters.baseImagePipelineBuildId }}", + "baseImageArm64PipelineBuildId": "${{ parameters.baseImageArm64PipelineBuildId }}" + } + EOF + displayName: "Create Base Image Config artifact" + - task: PublishBuildArtifacts@1 + displayName: "Publish Base Image Config artifact" + inputs: + pathToPublish: /tmp/config + artifactName: base-image-config + - bash: | + echo "BaseImage config handled" + displayName: "BaseImage config" diff --git a/.pipelines/templates/stages/common_tasks/find-base-image-version.yml b/.pipelines/templates/stages/common_tasks/find-base-image-version.yml index 7a846e70b..35b949860 100644 --- a/.pipelines/templates/stages/common_tasks/find-base-image-version.yml +++ b/.pipelines/templates/stages/common_tasks/find-base-image-version.yml @@ -1,60 +1,69 @@ -parameters: - - name: baseimgBuildType - displayName: Base Image build type - type: string - values: - - dev - - preview - - release - default: "release" - - - name: baseImagePipelineBuildId - displayName: "Build Id of the pipeline run, default will select latest successful run from pipeline 2116 ([AMD64-6-OneBranch]-Prod-BuildImages) with tag 3.0-preview" - type: string - steps: - - ${{ if eq(parameters.baseimgBuildType, 'dev') }}: - - script: | - set -eux - baseimgVersion="*.*.*" - if [[ "${{ parameters.baseImagePipelineBuildId }}" != "latestFromBranch" ]]; then - baseimgVersion="${{ parameters.baseImagePipelineBuildId }}" - fi - - set +x - echo "##vso[task.setvariable variable=baseimgVersion]$baseimgVersion" - name: SetBaseImageVersion - displayName: Set Base Image version variable for dev - - ${{ if eq(parameters.baseimgBuildType, 'release') }}: - - script: | - set -eux - baseimgVersion="*.*.*" - - set +x - echo "##vso[task.setvariable variable=baseimgVersion]$baseimgVersion" - name: SetBaseImageVersion - displayName: Set Base Image version variable for release - - ${{ if eq(parameters.baseimgBuildType, 'preview') }}: - - task: DownloadPipelineArtifact@2 - displayName: "Download version-full.config from Prod-BuildImages" - inputs: - buildType: "specific" - project: "mariner" - pipeline: "[AMD64-6-OneBranch]-Prod-BuildImages" - runVersion: specific - branchName: "refs/heads/master" - runId: ${{ parameters.baseImagePipelineBuildId }} - tags: "3.0-preview" - artifactName: "drop_build_rpms_build" - patterns: "**/build-config/version-full.config" - targetPath: "$(Build.SourcesDirectory)/output" - - - script: | - set -eux - versionFullConfigPath=$(find $(Build.SourcesDirectory) -name version-full.config) - baseimgVersion=$(cat $versionFullConfigPath) - - set +x - echo "##vso[task.setvariable variable=baseimgVersion]$baseimgVersion" - name: SetBaseImageVersion - displayName: Set Base Image version variable for preview + - template: read-base-image-config-artifact.yml + - bash: | + set -eux + + echo "BASEIMG_BUILD_TYPE: $(BASEIMG_BUILD_TYPE)" + echo "BASE_IMAGE_PIPELINE_BUILD_ID: $(BASE_IMAGE_PIPELINE_BUILD_ID)" + printenv + displayName: Show variables before + + # baseimgBuildType == dev + - script: | + set -eux + baseimgVersion="*.*.*" + if [[ "$(BASE_IMAGE_PIPELINE_BUILD_ID)" != "latestFromBranch" ]]; then + baseimgVersion="$(BASE_IMAGE_PIPELINE_BUILD_ID)" + fi + + set +x + echo "##vso[task.setvariable variable=BASEIMG_VERSION]$baseimgVersion" + name: SetBaseImageVersionDev + condition: eq(variables['BASEIMG_BUILD_TYPE'], 'dev') + displayName: Set Base Image version variable for dev + + # baseimgBuildType == release + - script: | + set -eux + baseimgVersion="*.*.*" + + set +x + echo "##vso[task.setvariable variable=BASEIMG_VERSION]$baseimgVersion" + name: SetBaseImageVersionRelease + condition: eq(variables['BASEIMG_BUILD_TYPE'], 'release') + displayName: Set Base Image version variable for release + + # baseimgBuildType == preview + - task: DownloadPipelineArtifact@2 + displayName: "Download version-full.config from Prod-BuildImages" + condition: eq(variables['BASEIMG_BUILD_TYPE'], 'preview') + inputs: + buildType: "specific" + project: "mariner" + pipeline: "[AMD64-6-OneBranch]-Prod-BuildImages" + runVersion: specific + branchName: "refs/heads/master" + runId: $(BASE_IMAGE_PIPELINE_BUILD_ID) + tags: "3.0-preview" + artifactName: "drop_build_rpms_build" + patterns: "**/build-config/version-full.config" + targetPath: "$(Build.SourcesDirectory)/output" + - script: | + set -eux + versionFullConfigPath=$(find $(Build.SourcesDirectory) -name version-full.config) + baseimgVersion=$(cat $versionFullConfigPath) + + set +x + echo "##vso[task.setvariable variable=BASEIMG_VERSION]$baseimgVersion" + name: SetBaseImageVersionPreview + condition: eq(variables['BASEIMG_BUILD_TYPE'], 'preview') + displayName: Set Base Image version variable for preview + + - bash: | + set -eux + + echo "BASEIMG_BUILD_TYPE: $(BASEIMG_BUILD_TYPE)" + echo "BASE_IMAGE_PIPELINE_BUILD_ID: $(BASE_IMAGE_PIPELINE_BUILD_ID)" + echo "BASEIMG_VERSION: $(BASEIMG_VERSION)" + printenv + displayName: Show variables after diff --git a/.pipelines/templates/stages/common_tasks/preview-container.yml b/.pipelines/templates/stages/common_tasks/preview-container.yml index d29bcc436..555404052 100644 --- a/.pipelines/templates/stages/common_tasks/preview-container.yml +++ b/.pipelines/templates/stages/common_tasks/preview-container.yml @@ -25,15 +25,33 @@ parameters: displayName: Path in which the Dockerfile will be run, the base for the container image will be saved there. steps: + - template: ../common_tasks/read-base-image-config-artifact.yml + - bash: | + set -eux + + RUN_VERSION=$(BASE_IMAGE_PIPELINE_BUILD_ID) + RUN_ID='' + if [[ "$RUN_VERSION" != "latestFromBranch" ]]; then + RUN_VERSION=specific + RUN_ID=$(BASE_IMAGE_PIPELINE_BUILD_ID) + fi + + set +x + echo "##vso[task.setvariable variable=RUN_VERSION]$RUN_VERSION" + echo "##vso[task.setvariable variable=RUN_ID]$RUN_ID" + displayName: "Configure RUN_VERSION" + condition: eq(variables['BASEIMG_BUILD_TYPE'], 'preview') + - task: DownloadPipelineArtifact@2 displayName: "Download preview of azl container image" + condition: eq(variables['BASEIMG_BUILD_TYPE'], 'preview') inputs: buildType: "specific" project: "mariner" pipeline: ${{ parameters.previewContainerPipeline }} - runVersion: ${{ parameters.runVersion }} + runVersion: $(RUN_VERSION) branchName: "refs/heads/master" - runId: ${{ parameters.baseImagePipelineBuildId }} + runId: $(RUN_ID) tags: "3.0-preview" artifactName: "drop_build_rpms_build" patterns: "**/build-artifacts/container_base/core-*.tar.gz" @@ -51,4 +69,5 @@ steps: ${{ parameters.dockerfilePath }} cat ${{ parameters.dockerfilePath }} - displayName: Replace azl container base with azl preview \ No newline at end of file + displayName: Replace azl container base with azl preview + condition: eq(variables['BASEIMG_BUILD_TYPE'], 'preview') \ No newline at end of file diff --git a/.pipelines/templates/stages/common_tasks/read-base-image-config-artifact.yml b/.pipelines/templates/stages/common_tasks/read-base-image-config-artifact.yml new file mode 100644 index 000000000..769cc5401 --- /dev/null +++ b/.pipelines/templates/stages/common_tasks/read-base-image-config-artifact.yml @@ -0,0 +1,26 @@ +steps: + - task: DownloadPipelineArtifact@2 + displayName: "Download base-image-config" + inputs: + buildType: current + artifactName: base-image-config + targetPath: /tmp/base-image-config + + - bash: | + set -eux + BASEIMG_BUILD_TYPE=$(jq -r .baseimgBuildType /tmp/base-image-config/baseimage.json) + BASE_IMAGE_PIPELINE_BUILD_ID=$(jq -r .baseImagePipelineBuildId /tmp/base-image-config/baseimage.json) + BASE_IMAGE_ARM64_PIPELINE_BUILD_ID=$(jq -r .baseImageArm64PipelineBuildId /tmp/base-image-config/baseimage.json) + + set +x + echo "##vso[task.setvariable variable=BASEIMG_BUILD_TYPE]$BASEIMG_BUILD_TYPE" + echo "##vso[task.setvariable variable=BASE_IMAGE_PIPELINE_BUILD_ID]$BASE_IMAGE_PIPELINE_BUILD_ID" + echo "##vso[task.setvariable variable=BASE_IMAGE_ARM64_PIPELINE_BUILD_ID]$BASE_IMAGE_ARM64_PIPELINE_BUILD_ID" + displayName: "Read Base Image Config artifact into variables" + + - bash: | + set -eux + echo "BASEIMG_BUILD_TYPE = $(BASEIMG_BUILD_TYPE)" + echo "BASE_IMAGE_PIPELINE_BUILD_ID = $(BASE_IMAGE_PIPELINE_BUILD_ID)" + echo "BASE_IMAGE_ARM64_PIPELINE_BUILD_ID = $(BASE_IMAGE_ARM64_PIPELINE_BUILD_ID)" + displayName: "Show Base Image Config artifact variables" diff --git a/.pipelines/templates/stages/download_staged/download-staged.yml b/.pipelines/templates/stages/download_staged/download-staged.yml index 0a9fc1be5..0a4ff4d5d 100644 --- a/.pipelines/templates/stages/download_staged/download-staged.yml +++ b/.pipelines/templates/stages/download_staged/download-staged.yml @@ -22,9 +22,15 @@ parameters: Specific version of the package to download, leave empty to download latest default: "" + - name: dependsOnStage + type: string + default: '' + stages: - stage: GetTridentBinaries_${{ replace(parameters.packageName, '-', '_') }} displayName: Download staged ${{ parameters.packageName }} for ${{ parameters.stageType }} + ${{ if ne(parameters.dependsOnStage, '') }}: + dependsOn: ${{ parameters.dependsOnStage }} jobs: - job: DownloadTrident diff --git a/.pipelines/templates/stages/testing_functional/functional-testing.yml b/.pipelines/templates/stages/testing_functional/functional-testing.yml index 627dce523..01ae53704 100644 --- a/.pipelines/templates/stages/testing_functional/functional-testing.yml +++ b/.pipelines/templates/stages/testing_functional/functional-testing.yml @@ -22,12 +22,18 @@ parameters: type: string default: "submodule" + - name: dependsOnStage + type: string + default: '' + stages: - stage: FunctionalTesting displayName: Functional Testing dependsOn: - ${{ if eq(parameters.downloadPrebuiltImage, false) }}: - TridentTestImg_trident_functest + - ${{ if ne(parameters.dependsOnStage, '') }}: + - ${{ parameters.dependsOnStage }} - ${{ else }}: [] jobs: diff --git a/.pipelines/templates/stages/testing_servicing/build-image.yml b/.pipelines/templates/stages/testing_servicing/build-image.yml index d9a62072d..52bc5a527 100644 --- a/.pipelines/templates/stages/testing_servicing/build-image.yml +++ b/.pipelines/templates/stages/testing_servicing/build-image.yml @@ -1,13 +1,4 @@ parameters: - - name: baseimgBuildType - displayName: Base Image build type - type: string - values: - - dev - - preview - - release - default: "release" - - name: label type: string displayName: "Label for the image" @@ -22,11 +13,6 @@ parameters: type: string displayName: "Base image type" - - name: baseImagePipelineBuildId - displayName: "Build Id of the pipeline run, default will select latest successful run from pipeline 2116 ([AMD64-6-OneBranch]-Prod-BuildImages) with tag 3.0-preview" - type: string - default: "latestFromBranch" - - name: baseimgAzlVersion displayName: Base Image AZL version type: string @@ -80,10 +66,6 @@ jobs: displayName: Download SSH Keys - template: ../common_tasks/find-base-image-version.yml - parameters: - baseimgBuildType: ${{ parameters.baseimgBuildType }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} - - template: .pipelines/templates/trident-testimg-template.yml@test-images parameters: target: ${{ parameters.makeTarget }} @@ -91,7 +73,7 @@ jobs: testImagesRepo: test-images micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} - baseimgBuildType: ${{ parameters.baseimgBuildType }} + baseimgBuildType: $(BASEIMG_BUILD_TYPE) baseimgVersion: $(baseimgVersion) baseimgAzureLinuxVersion: ${{ parameters.baseimgAzlVersion }} downloadTrident: false diff --git a/.pipelines/templates/stages/testing_servicing/vm-testing.yml b/.pipelines/templates/stages/testing_servicing/vm-testing.yml index 7727ad990..3b13a36cd 100644 --- a/.pipelines/templates/stages/testing_servicing/vm-testing.yml +++ b/.pipelines/templates/stages/testing_servicing/vm-testing.yml @@ -4,20 +4,6 @@ parameters: type: boolean default: false - - name: baseimgBuildType - displayName: Base Image build type - type: string - values: - - dev - - preview - - release - default: "release" - - - name: baseImagePipelineBuildId - displayName: "Build Id of the pipeline run, default will select latest successful run from pipeline 2116 ([AMD64-6-OneBranch]-Prod-BuildImages) with tag 3.0-preview" - type: string - default: "latestFromBranch" - - name: baseimgAzlVersion displayName: Base Image AZL version type: string @@ -94,36 +80,30 @@ stages: jobs: - template: build-image.yml parameters: - baseimgBuildType: ${{ parameters.baseimgBuildType }} label: "qemu-base" makeTarget: "build/trident-vm-grub-verity-testimage.qcow2" baseimgType: qemu_guest baseimgAzlVersion: ${{ parameters.baseimgAzlVersion }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} useStagedSshKeys: true - template: build-image.yml parameters: - baseimgBuildType: ${{ parameters.baseimgBuildType }} label: "qemu-update-a" makeTarget: "build/trident-vm-grub-verity-testimage.cosi" baseimgType: qemu_guest baseimgAzlVersion: ${{ parameters.baseimgAzlVersion }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} useStagedSshKeys: true - template: build-image.yml parameters: - baseimgBuildType: ${{ parameters.baseimgBuildType }} label: "qemu-update-b" makeTarget: "build/trident-vm-grub-verity-testimage.cosi" baseimgType: qemu_guest baseimgAzlVersion: ${{ parameters.baseimgAzlVersion }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} useStagedSshKeys: true @@ -138,34 +118,28 @@ stages: jobs: - template: build-image.yml parameters: - baseimgBuildType: ${{ parameters.baseimgBuildType }} label: "azure-base" makeTarget: "build/trident-vm-grub-verity-azure-testimage.vhd" baseimgType: core_selinux baseimgAzlVersion: ${{ parameters.baseimgAzlVersion }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} - template: build-image.yml parameters: - baseimgBuildType: ${{ parameters.baseimgBuildType }} label: "azure-update-a" makeTarget: "build/trident-vm-grub-verity-azure-testimage.cosi" baseimgType: core_selinux baseimgAzlVersion: ${{ parameters.baseimgAzlVersion }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} - template: build-image.yml parameters: - baseimgBuildType: ${{ parameters.baseimgBuildType }} label: "azure-update-b" makeTarget: "build/trident-vm-grub-verity-azure-testimage.cosi" baseimgType: core_selinux baseimgAzlVersion: ${{ parameters.baseimgAzlVersion }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} @@ -179,36 +153,30 @@ stages: jobs: - template: build-image.yml parameters: - baseimgBuildType: ${{ parameters.baseimgBuildType }} label: "uki-base" makeTarget: "build/trident-vm-usr-verity-testimage.qcow2" baseimgType: qemu_guest baseimgAzlVersion: ${{ parameters.baseimgAzlVersion }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} useStagedSshKeys: true - template: build-image.yml parameters: - baseimgBuildType: ${{ parameters.baseimgBuildType }} label: "uki-update-a" makeTarget: "build/trident-vm-usr-verity-testimage.cosi" baseimgType: qemu_guest baseimgAzlVersion: ${{ parameters.baseimgAzlVersion }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} useStagedSshKeys: true - template: build-image.yml parameters: - baseimgBuildType: ${{ parameters.baseimgBuildType }} label: "uki-update-b" makeTarget: "build/trident-vm-usr-verity-testimage.cosi" baseimgType: qemu_guest baseimgAzlVersion: ${{ parameters.baseimgAzlVersion }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} useStagedSshKeys: true diff --git a/.pipelines/templates/stages/trident_rpms/build-source.yml b/.pipelines/templates/stages/trident_rpms/build-source.yml index 52cf11146..71d0665d0 100644 --- a/.pipelines/templates/stages/trident_rpms/build-source.yml +++ b/.pipelines/templates/stages/trident_rpms/build-source.yml @@ -10,30 +10,19 @@ parameters: type: boolean default: true - - name: baseimgBuildType - displayName: Base Image build type - type: string - values: - - dev - - preview - - release - default: "release" - - - name: baseImagePipelineBuildId - type: string - default: "latestFromBranch" - - - name: baseImageArm64PipelineBuildId - type: string - default: "latestFromBranch" - - name: osModifierBranch type: string default: "submodule" + - name: dependsOnStage + type: string + default: '' + stages: - stage: GetTridentBinaries_rpms displayName: Trident RPMs (Build) + ${{ if ne(parameters.dependsOnStage, '') }}: + dependsOn: ${{ parameters.dependsOnStage }} jobs: - job: CheckTrident @@ -89,10 +78,11 @@ stages: - template: ../common_tasks/build-osmodifier.yml parameters: osModifierBranch: ${{ parameters.osModifierBranch }} + - template: ../common_tasks/read-base-image-config-artifact.yml - template: release.yml parameters: - baseimgBuildType: ${{ parameters.baseimgBuildType }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} + baseimgBuildType: $(BASEIMG_BUILD_TYPE) + baseImagePipelineBuildId: $(BASE_IMAGE_PIPELINE_BUILD_ID) previewContainerPipeline: "[AMD64-6-OneBranch]-Prod-BuildImages" - ${{ if eq(parameters.publishToDevFeed, true) }}: - template: publish-dev.yml @@ -119,8 +109,9 @@ stages: - template: ../common_tasks/build-osmodifier.yml parameters: osModifierBranch: ${{ parameters.osModifierBranch }} + - template: ../common_tasks/read-base-image-config-artifact.yml - template: release.yml parameters: - baseimgBuildType: ${{ parameters.baseimgBuildType }} - baseImagePipelineBuildId: ${{ parameters.baseImageArm64PipelineBuildId }} + baseimgBuildType: $(BASEIMG_BUILD_TYPE) + baseImagePipelineBuildId: $(BASE_IMAGE_ARM64_PIPELINE_BUILD_ID) previewContainerPipeline: "[ARM64-6-OneBranch]-Prod-BuildImages" diff --git a/.pipelines/templates/stages/trident_rpms/release.yml b/.pipelines/templates/stages/trident_rpms/release.yml index 3ab09b255..3b166b687 100644 --- a/.pipelines/templates/stages/trident_rpms/release.yml +++ b/.pipelines/templates/stages/trident_rpms/release.yml @@ -2,11 +2,11 @@ parameters: - name: baseimgBuildType displayName: Base Image build type type: string - values: - - dev - - preview - - release default: "release" + # values: + # - dev + # - preview + # - release - name: baseImagePipelineBuildId type: string diff --git a/.pipelines/templates/stages/trident_rpms/trident-stage.yml b/.pipelines/templates/stages/trident_rpms/trident-stage.yml index 204cddf57..6db1d6254 100644 --- a/.pipelines/templates/stages/trident_rpms/trident-stage.yml +++ b/.pipelines/templates/stages/trident_rpms/trident-stage.yml @@ -18,23 +18,6 @@ parameters: - name: tridentArtifactName type: string - - name: baseimgBuildType - displayName: Base Image build type - type: string - values: - - dev - - preview - - release - default: "release" - - - name: baseImagePipelineBuildId - type: string - default: "latestFromBranch" - - - name: baseImageArm64PipelineBuildId - type: string - default: "latestFromBranch" - - name: forceTridentRebuild type: boolean default: false @@ -43,6 +26,10 @@ parameters: type: string default: "submodule" + - name: dependsOnStage + type: string + default: '' + stages: - ${{ if or(parameters.forceTridentRebuild, eq(parameters.stageType, 'pr'), eq(parameters.stageType, 'pr-e2e'), eq(parameters.stageType, 'ci'), eq(parameters.stageType, 'pr-e2e-azure')) }}: - template: build-source.yml @@ -51,10 +38,8 @@ stages: publishToDevFeed: true tridentArtifactName: ${{ parameters.tridentArtifactName }} codeCoverage: ${{ parameters.codeCoverage }} - baseimgBuildType: ${{ parameters.baseimgBuildType }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} - baseImageArm64PipelineBuildId: ${{ parameters.baseImageArm64PipelineBuildId }} osModifierBranch: ${{ parameters.osModifierBranch }} + dependsOnStage: ${{ parameters.dependsOnStage }} - ${{ else }}: - template: ../download_staged/download-staged.yml @@ -62,3 +47,4 @@ stages: artifactName: ${{ parameters.tridentArtifactName }} stageType: ${{ parameters.stageType }} packageName: rpms + dependsOnStage: ${{ parameters.dependsOnStage }} diff --git a/.pipelines/templates/stages/trident_usb_iso/trident-usb-iso.yml b/.pipelines/templates/stages/trident_usb_iso/trident-usb-iso.yml index d23fbed9b..55f6d817d 100644 --- a/.pipelines/templates/stages/trident_usb_iso/trident-usb-iso.yml +++ b/.pipelines/templates/stages/trident_usb_iso/trident-usb-iso.yml @@ -98,10 +98,6 @@ stages: continueOnError: true - template: ../common_tasks/find-base-image-version.yml - parameters: - baseimgBuildType: ${{ parameters.baseimgBuildType }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} - - template: .pipelines/templates/trident-testimg-template.yml@test-images parameters: target: build/azl-installer.iso @@ -109,8 +105,8 @@ stages: testImagesRepo: test-images micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} - baseimgBuildType: ${{ parameters.baseimgBuildType }} - baseimgVersion: $(baseimgVersion) + baseimgBuildType: $(BASEIMG_BUILD_TYPE) + baseimgVersion: $(BASEIMG_VERSION) baseimgAzureLinuxVersion: ${{ parameters.baseimgAzlVersion }} downloadTrident: false diff --git a/.pipelines/templates/stages/validate_makefile/dev-build.yml b/.pipelines/templates/stages/validate_makefile/dev-build.yml index a614913d9..63adf5cdb 100644 --- a/.pipelines/templates/stages/validate_makefile/dev-build.yml +++ b/.pipelines/templates/stages/validate_makefile/dev-build.yml @@ -3,9 +3,15 @@ parameters: type: string default: "submodule" + - name: dependsOnStage + type: string + default: '' + stages: - stage: MakefileValidation displayName: Validate Makefile targets + ${{ if ne(parameters.dependsOnStage, '') }}: + dependsOn: ${{ parameters.dependsOnStage }} jobs: - job: Make diff --git a/.pipelines/templates/trident-platform-cicd-template.yml b/.pipelines/templates/trident-platform-cicd-template.yml index c69327ef4..2f921ce80 100644 --- a/.pipelines/templates/trident-platform-cicd-template.yml +++ b/.pipelines/templates/trident-platform-cicd-template.yml @@ -18,9 +18,14 @@ parameters: - dev - preview - name: runBaremetalTests - displayName: "Specify whether to run BareMetal tests" type: boolean default: false + - name: previousStage + type: string + default: '' + - name: createBaseImageConfigArtifact + type: boolean + default: true stages: - template: e2e-template.yml @@ -32,3 +37,5 @@ stages: baseImageArm64PipelineBuildId: ${{ parameters.baseImageArm64PipelineBuildId }} forceTridentRebuild: true runBaremetalTests: ${{ parameters.runBaremetalTests }} + previousStage: ${{ parameters.previousStage }} + createBaseImageConfigArtifact: ${{ parameters.createBaseImageConfigArtifact }} diff --git a/.pipelines/trident-cicd.yml b/.pipelines/trident-cicd.yml index 5f5f85209..34fdce284 100644 --- a/.pipelines/trident-cicd.yml +++ b/.pipelines/trident-cicd.yml @@ -30,12 +30,12 @@ resources: - repository: platform-pipelines type: git name: platform-pipelines - ref: refs/heads/main + ref: refs/heads/user/bfjelds/enable-idc-testing-artifact - repository: test-images type: git name: test-images - ref: refs/heads/main + ref: refs/heads/user/bfjelds/enable-idc-testing-artifact - repository: platform-telemetry name: platform-telemetry @@ -46,9 +46,38 @@ extends: template: templates/MockOB.yml parameters: stages: - - template: templates/trident-platform-cicd-template.yml + - template: templates/stages/common_tasks/create-base-image-config-artifact.yml parameters: - buildType: ${{ parameters.buildType }} + stageSuffix: cicd + baseimgBuildType: ${{ parameters.buildType }} baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} baseImageArm64PipelineBuildId: ${{ parameters.baseImageArm64PipelineBuildId }} - runBaremetalTests: true + + - template: templates/trident-platform-cicd-template.yml + parameters: + runBaremetalTests: false + previousStage: HandleBaseImageConfig_cicd + baseImagePipelineBuildId: ignore + baseImageArm64PipelineBuildId: ignore + createBaseImageConfigArtifact: false + + - stage: ProcessJunit_Stage + displayName: Process junit files + dependsOn: + - FunctionalTesting + - DeploymentTesting_host + - DeploymentTesting_container + + jobs: + - job: ProcessJunit_Job + displayName: Process junit files + pool: + type: linux + variables: + ob_outputDirectory: "$(Build.SourcesDirectory)/out" + steps: + - script: | + set -eux + echo "Collect junit and do stuff" + displayName: Process junit files + From 0b300bb837ccef39d1a7d87c0e54afb17e90d1c4 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 8 Aug 2025 15:04:21 +0000 Subject: [PATCH 04/69] cleanup, remove extra stage --- .../templates/e2e-extended-template.yml | 88 +++++++++++++++++++ .pipelines/templates/e2e-template.yml | 47 ++++------ .pipelines/templates/pipeline-selector.yml | 2 +- .../create-base-image-config-artifact.yml | 36 +++----- ...ident-platform-cicd-for-prism-template.yml | 2 +- .../trident-platform-cicd-template.yml | 20 ----- .pipelines/trident-cicd.yml | 5 +- 7 files changed, 122 insertions(+), 78 deletions(-) create mode 100644 .pipelines/templates/e2e-extended-template.yml diff --git a/.pipelines/templates/e2e-extended-template.yml b/.pipelines/templates/e2e-extended-template.yml new file mode 100644 index 000000000..c773bad17 --- /dev/null +++ b/.pipelines/templates/e2e-extended-template.yml @@ -0,0 +1,88 @@ +parameters: + - name: "stageType" + displayName: "Pipeline configuration type" + type: string + default: ci + values: + - pr-e2e + - pr-e2e-azure + - ci + - pre + - rel + - full-validation + + - name: baseimgBuildType + displayName: Base Image build type + type: string + values: + - dev + - preview + - release + default: "release" + + - name: baseImagePipelineBuildId + type: string + default: "latestFromBranch" + + - name: baseImageArm64PipelineBuildId + type: string + default: "latestFromBranch" + + - name: blockPublishing + displayName: "Block publishing to artifact feed" + type: boolean + default: false + + - name: forceTridentRebuild + displayName: "Force trident rebuild rather than downloading previously built." + type: boolean + default: false + + - name: includeAzure + displayName: "Include Azure testing" + type: boolean + default: false + + - name: micBuildType + displayName: MIC Build Type + type: string + values: + - dev + - preview + - release + default: release + + - name: micVersion + displayName: MIC Version + type: string + default: "*.*.*" + + - name: numberOfUpdateIterations + displayName: Number of times each worker invokes A/B Update + type: number + default: 3 + + - name: osModifierBranch + type: string + default: "submodule" + +stages: + - template: stages/common_tasks/create-base-image-config-artifact.yml + parameters: + stageSuffix: e2e + baseimgBuildType: ${{ parameters.baseimgBuildType }} + baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} + baseImageArm64PipelineBuildId: ${{ parameters.baseImageArm64PipelineBuildId }} + previousStage: ${{ parameters.previousStage }} + + - template: e2e-template.yml + parameters: + stageType: ${{ parameters.stageType }} + blockPublishing: ${{ parameters.blockPublishing }} + forceTridentRebuild: ${{ parameters.forceTridentRebuild }} + includeAzure: ${{ parameters.includeAzure }} + micBuildType: ${{ parameters.micBuildType }} + micVersion: ${{ parameters.micVersion }} + numberOfUpdateIterations: ${{ parameters.numberOfUpdateIterations }} + osModifierBranch: ${{ parameters.osModifierBranch }} + previousStage: HandleBaseImageConfig_cicd diff --git a/.pipelines/templates/e2e-template.yml b/.pipelines/templates/e2e-template.yml index 6e41b5781..703480f54 100644 --- a/.pipelines/templates/e2e-template.yml +++ b/.pipelines/templates/e2e-template.yml @@ -70,24 +70,11 @@ parameters: type: boolean default: true - - name: createBaseImageConfigArtifact - type: boolean - default: true - - name: previousStage type: string default: '' stages: - - template: stages/common_tasks/create-base-image-config-artifact.yml - parameters: - stageSuffix: e2e - baseimgBuildType: ${{ parameters.baseimgBuildType }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} - baseImageArm64PipelineBuildId: ${{ parameters.baseImageArm64PipelineBuildId }} - previousStage: ${{ parameters.previousStage }} - createBaseImageConfigArtifact: ${{ parameters.createBaseImageConfigArtifact }} - # Trident Build/Download - template: stages/trident_rpms/trident-stage.yml parameters: @@ -96,19 +83,19 @@ stages: codeCoverage: false forceTridentRebuild: ${{ parameters.forceTridentRebuild }} osModifierBranch: ${{ parameters.osModifierBranch }} - dependsOnStage: HandleBaseImageConfig_e2e + dependsOnStage: ${{ parameters.previousStage }} # Build tools (Go stuff) - template: stages/building_tools/building-tools.yml parameters: - dependsOnStage: HandleBaseImageConfig_e2e + dependsOnStage: ${{ parameters.previousStage }} # Makefile validation, only for CI - ${{ if eq(parameters.stageType, 'ci') }}: - template: stages/validate_makefile/dev-build.yml parameters: osModifierBranch: ${{ parameters.osModifierBranch }} - dependsOnStage: HandleBaseImageConfig_e2e + dependsOnStage: ${{ parameters.previousStage }} # Build FT base Image, only in CI @@ -121,7 +108,7 @@ stages: micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} clones: 1 - dependsOnStage: HandleBaseImageConfig_e2e + dependsOnStage: ${{ parameters.previousStage }} # Build Trident container image (depends on GetTridentBinaries_rpms) - template: stages/build_docker_image/trident-container.yml @@ -133,7 +120,7 @@ stages: micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} clones: 1 - dependsOnStage: HandleBaseImageConfig_e2e + dependsOnStage: ${{ parameters.previousStage }} # Build Trident split (stage and finalize separated) installer ISO (host) - template: stages/build_image/build-image.yml @@ -142,7 +129,7 @@ stages: micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} clones: 1 - dependsOnStage: HandleBaseImageConfig_e2e + dependsOnStage: ${{ parameters.previousStage }} # Build Trident installer ISO (container) - template: stages/build_image/build-image.yml @@ -152,7 +139,7 @@ stages: micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} clones: 1 - dependsOnStage: HandleBaseImageConfig_e2e + dependsOnStage: ${{ parameters.previousStage }} # Build Trident test image (regular) - template: stages/build_image/build-image.yml @@ -160,7 +147,7 @@ stages: imageName: trident-testimage micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} - dependsOnStage: HandleBaseImageConfig_e2e + dependsOnStage: ${{ parameters.previousStage }} # Build Trident test image (container) - template: stages/build_image/build-image.yml @@ -169,7 +156,7 @@ stages: runtimeEnv: "container" micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} - dependsOnStage: HandleBaseImageConfig_e2e + dependsOnStage: ${{ parameters.previousStage }} # Build Trident test image for verity (host) - template: stages/build_image/build-image.yml @@ -177,7 +164,7 @@ stages: imageName: trident-verity-testimage micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} - dependsOnStage: HandleBaseImageConfig_e2e + dependsOnStage: ${{ parameters.previousStage }} # Build Trident test image for verity (container) - template: stages/build_image/build-image.yml @@ -186,7 +173,7 @@ stages: runtimeEnv: "container" micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} - dependsOnStage: HandleBaseImageConfig_e2e + dependsOnStage: ${{ parameters.previousStage }} # Build Trident test image for usr-verity (host) - template: stages/build_image/build-image.yml @@ -194,7 +181,7 @@ stages: imageName: trident-usrverity-testimage micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} - dependsOnStage: HandleBaseImageConfig_e2e + dependsOnStage: ${{ parameters.previousStage }} # Build Trident test image for usr-verity (container) - template: stages/build_image/build-image.yml @@ -203,7 +190,7 @@ stages: runtimeEnv: "container" micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} - dependsOnStage: HandleBaseImageConfig_e2e + dependsOnStage: ${{ parameters.previousStage }} - ${{ if or(parameters.forceTridentRebuild, eq(parameters.stageType, 'pr-e2e'), eq(parameters.stageType, 'ci'), eq(parameters.stageType, 'pr-e2e-azure')) }}: # Build USB-ISO @@ -241,7 +228,7 @@ stages: # Functional Testing - template: stages/testing_functional/functional-testing.yml parameters: - dependsOnStage: HandleBaseImageConfig_e2e + dependsOnStage: ${{ parameters.previousStage }} # VM Testing (host, daily) - template: stages/testing_vm/netlaunch-testing.yml @@ -274,7 +261,7 @@ stages: # Functional Testing - template: stages/testing_functional/functional-testing.yml parameters: - dependsOnStage: HandleBaseImageConfig_e2e + dependsOnStage: ${{ parameters.previousStage }} # VM Testing (host, weekly) - template: stages/testing_vm/netlaunch-testing.yml @@ -309,7 +296,7 @@ stages: parameters: downloadPrebuiltImage: false osModifierBranch: ${{ parameters.osModifierBranch }} - dependsOnStage: HandleBaseImageConfig_e2e + dependsOnStage: ${{ parameters.previousStage }} # VM Testing (host, post_merge) @@ -330,7 +317,7 @@ stages: - template: stages/testing_functional/functional-testing.yml parameters: rerunTests: false - dependsOnStage: HandleBaseImageConfig_e2e + dependsOnStage: ${{ parameters.previousStage }} # VM Testing (host, pullrequest) - template: stages/testing_vm/netlaunch-testing.yml diff --git a/.pipelines/templates/pipeline-selector.yml b/.pipelines/templates/pipeline-selector.yml index 78932ee8c..b2420e72d 100644 --- a/.pipelines/templates/pipeline-selector.yml +++ b/.pipelines/templates/pipeline-selector.yml @@ -43,7 +43,7 @@ stages: - ${{ if eq(parameters.targetStage, 'pr') }}: - template: pr-template.yml - ${{ else }}: - - template: e2e-template.yml + - template: e2e-extended-template.yml parameters: stageType: ${{ parameters.targetStage }} blockPublishing: ${{ parameters.blockPublishing }} diff --git a/.pipelines/templates/stages/common_tasks/create-base-image-config-artifact.yml b/.pipelines/templates/stages/common_tasks/create-base-image-config-artifact.yml index 1b0d7b442..816b07fbc 100644 --- a/.pipelines/templates/stages/common_tasks/create-base-image-config-artifact.yml +++ b/.pipelines/templates/stages/common_tasks/create-base-image-config-artifact.yml @@ -18,10 +18,6 @@ parameters: type: string default: "latestFromBranch" - - name: createBaseImageConfigArtifact - type: boolean - default: true - - name: previousStage type: string default: '' @@ -42,22 +38,18 @@ stages: - name: ob_artifactBaseName value: "HandleBaseImageConfig_${{ parameters.stageSuffix }}" steps: - - ${{ if parameters.createBaseImageConfigArtifact }}: - - bash: | - mkdir -p /tmp/config - cat << EOF > /tmp/config/baseimage.json - { - "baseimgBuildType": "${{ parameters.baseimgBuildType }}", - "baseImagePipelineBuildId": "${{ parameters.baseImagePipelineBuildId }}", - "baseImageArm64PipelineBuildId": "${{ parameters.baseImageArm64PipelineBuildId }}" - } - EOF - displayName: "Create Base Image Config artifact" - - task: PublishBuildArtifacts@1 - displayName: "Publish Base Image Config artifact" - inputs: - pathToPublish: /tmp/config - artifactName: base-image-config - bash: | - echo "BaseImage config handled" - displayName: "BaseImage config" + mkdir -p /tmp/config + cat << EOF > /tmp/config/baseimage.json + { + "baseimgBuildType": "${{ parameters.baseimgBuildType }}", + "baseImagePipelineBuildId": "${{ parameters.baseImagePipelineBuildId }}", + "baseImageArm64PipelineBuildId": "${{ parameters.baseImageArm64PipelineBuildId }}" + } + EOF + displayName: "Create Base Image Config artifact" + - task: PublishBuildArtifacts@1 + displayName: "Publish Base Image Config artifact" + inputs: + pathToPublish: /tmp/config + artifactName: base-image-config diff --git a/.pipelines/templates/trident-platform-cicd-for-prism-template.yml b/.pipelines/templates/trident-platform-cicd-for-prism-template.yml index a6dea4a91..ba7640064 100644 --- a/.pipelines/templates/trident-platform-cicd-for-prism-template.yml +++ b/.pipelines/templates/trident-platform-cicd-for-prism-template.yml @@ -24,7 +24,7 @@ parameters: default: "main" stages: - - template: e2e-template.yml + - template: e2e-extended-template.yml parameters: stageType: ci blockPublishing: true diff --git a/.pipelines/templates/trident-platform-cicd-template.yml b/.pipelines/templates/trident-platform-cicd-template.yml index 2f921ce80..df57f26c3 100644 --- a/.pipelines/templates/trident-platform-cicd-template.yml +++ b/.pipelines/templates/trident-platform-cicd-template.yml @@ -4,38 +4,18 @@ # preview build. parameters: - - name: baseImagePipelineBuildId - displayName: "Build Id of the pipeline run, default will select latest successful run from pipeline 2116 ([AMD64-6-OneBranch]-Prod-BuildImages) with tag 3.0-preview" - type: string - - name: baseImageArm64PipelineBuildId - displayName: "Build Id of the pipeline run, default will select latest successful run from pipeline 2117 ([ARM64-6-OneBranch]-Prod-BuildImages) with tag 3.0-preview" - type: string - - name: buildType - displayName: "Pick build type, select 'dev' to pull images and rpms from specified build, select preview to find preview version from specified build and pull images and rpms from feeds and packages.microsoft.com" - type: string - default: dev - values: - - dev - - preview - name: runBaremetalTests type: boolean default: false - name: previousStage type: string default: '' - - name: createBaseImageConfigArtifact - type: boolean - default: true stages: - template: e2e-template.yml parameters: stageType: pre blockPublishing: true - baseimgBuildType: ${{ parameters.buildType }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} - baseImageArm64PipelineBuildId: ${{ parameters.baseImageArm64PipelineBuildId }} forceTridentRebuild: true runBaremetalTests: ${{ parameters.runBaremetalTests }} previousStage: ${{ parameters.previousStage }} - createBaseImageConfigArtifact: ${{ parameters.createBaseImageConfigArtifact }} diff --git a/.pipelines/trident-cicd.yml b/.pipelines/trident-cicd.yml index 34fdce284..75dc857d6 100644 --- a/.pipelines/trident-cicd.yml +++ b/.pipelines/trident-cicd.yml @@ -55,11 +55,8 @@ extends: - template: templates/trident-platform-cicd-template.yml parameters: - runBaremetalTests: false previousStage: HandleBaseImageConfig_cicd - baseImagePipelineBuildId: ignore - baseImageArm64PipelineBuildId: ignore - createBaseImageConfigArtifact: false + # runBaremetalTests: false - stage: ProcessJunit_Stage displayName: Process junit files From 542756aee02600f502a2e7f70a42ac7acd0f2dcf Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 8 Aug 2025 15:15:31 +0000 Subject: [PATCH 05/69] rename param, add comments --- .../templates/e2e-extended-template.yml | 6 ++- .pipelines/templates/e2e-template.yml | 50 ++++++++++++------- .pipelines/trident-cicd.yml | 44 ++++++++-------- 3 files changed, 60 insertions(+), 40 deletions(-) diff --git a/.pipelines/templates/e2e-extended-template.yml b/.pipelines/templates/e2e-extended-template.yml index c773bad17..334899871 100644 --- a/.pipelines/templates/e2e-extended-template.yml +++ b/.pipelines/templates/e2e-extended-template.yml @@ -1,3 +1,7 @@ +# e2e-extended-template will create the base image artifact and +# call e2e-template to run all trident tests for each of our scenarios. +# This includes unit tests, integration tests, and end-to-end tests. + parameters: - name: "stageType" displayName: "Pipeline configuration type" @@ -85,4 +89,4 @@ stages: micVersion: ${{ parameters.micVersion }} numberOfUpdateIterations: ${{ parameters.numberOfUpdateIterations }} osModifierBranch: ${{ parameters.osModifierBranch }} - previousStage: HandleBaseImageConfig_cicd + baseImageArtifactStage: HandleBaseImageConfig_cicd diff --git a/.pipelines/templates/e2e-template.yml b/.pipelines/templates/e2e-template.yml index 703480f54..48e15079b 100644 --- a/.pipelines/templates/e2e-template.yml +++ b/.pipelines/templates/e2e-template.yml @@ -1,3 +1,16 @@ +# e2e-template will run all trident tests for each of our scenarios. +# This includes unit tests, integration tests, and end-to-end tests. +# +# Prior to invoking this template, an artifact must be created that +# contains information about the base image being used. This artifact +# should be created by this template: +# +# .pipelines/templates/stages/common_tasks/create-base-image-config-artifact.yml +# +# The stage that creates the artifact should be passed into e2e-template +# as baseImageArtifactStage. This will ensure that e2e-template waits for +# the base image artifact to be created. +# parameters: - name: "stageType" displayName: "Pipeline configuration type" @@ -70,9 +83,8 @@ parameters: type: boolean default: true - - name: previousStage + - name: baseImageArtifactStage type: string - default: '' stages: # Trident Build/Download @@ -83,19 +95,19 @@ stages: codeCoverage: false forceTridentRebuild: ${{ parameters.forceTridentRebuild }} osModifierBranch: ${{ parameters.osModifierBranch }} - dependsOnStage: ${{ parameters.previousStage }} + dependsOnStage: ${{ parameters.baseImageArtifactStage }} # Build tools (Go stuff) - template: stages/building_tools/building-tools.yml parameters: - dependsOnStage: ${{ parameters.previousStage }} + dependsOnStage: ${{ parameters.baseImageArtifactStage }} # Makefile validation, only for CI - ${{ if eq(parameters.stageType, 'ci') }}: - template: stages/validate_makefile/dev-build.yml parameters: osModifierBranch: ${{ parameters.osModifierBranch }} - dependsOnStage: ${{ parameters.previousStage }} + dependsOnStage: ${{ parameters.baseImageArtifactStage }} # Build FT base Image, only in CI @@ -108,7 +120,7 @@ stages: micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} clones: 1 - dependsOnStage: ${{ parameters.previousStage }} + dependsOnStage: ${{ parameters.baseImageArtifactStage }} # Build Trident container image (depends on GetTridentBinaries_rpms) - template: stages/build_docker_image/trident-container.yml @@ -120,7 +132,7 @@ stages: micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} clones: 1 - dependsOnStage: ${{ parameters.previousStage }} + dependsOnStage: ${{ parameters.baseImageArtifactStage }} # Build Trident split (stage and finalize separated) installer ISO (host) - template: stages/build_image/build-image.yml @@ -129,7 +141,7 @@ stages: micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} clones: 1 - dependsOnStage: ${{ parameters.previousStage }} + dependsOnStage: ${{ parameters.baseImageArtifactStage }} # Build Trident installer ISO (container) - template: stages/build_image/build-image.yml @@ -139,7 +151,7 @@ stages: micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} clones: 1 - dependsOnStage: ${{ parameters.previousStage }} + dependsOnStage: ${{ parameters.baseImageArtifactStage }} # Build Trident test image (regular) - template: stages/build_image/build-image.yml @@ -147,7 +159,7 @@ stages: imageName: trident-testimage micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} - dependsOnStage: ${{ parameters.previousStage }} + dependsOnStage: ${{ parameters.baseImageArtifactStage }} # Build Trident test image (container) - template: stages/build_image/build-image.yml @@ -156,7 +168,7 @@ stages: runtimeEnv: "container" micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} - dependsOnStage: ${{ parameters.previousStage }} + dependsOnStage: ${{ parameters.baseImageArtifactStage }} # Build Trident test image for verity (host) - template: stages/build_image/build-image.yml @@ -164,7 +176,7 @@ stages: imageName: trident-verity-testimage micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} - dependsOnStage: ${{ parameters.previousStage }} + dependsOnStage: ${{ parameters.baseImageArtifactStage }} # Build Trident test image for verity (container) - template: stages/build_image/build-image.yml @@ -173,7 +185,7 @@ stages: runtimeEnv: "container" micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} - dependsOnStage: ${{ parameters.previousStage }} + dependsOnStage: ${{ parameters.baseImageArtifactStage }} # Build Trident test image for usr-verity (host) - template: stages/build_image/build-image.yml @@ -181,7 +193,7 @@ stages: imageName: trident-usrverity-testimage micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} - dependsOnStage: ${{ parameters.previousStage }} + dependsOnStage: ${{ parameters.baseImageArtifactStage }} # Build Trident test image for usr-verity (container) - template: stages/build_image/build-image.yml @@ -190,7 +202,7 @@ stages: runtimeEnv: "container" micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} - dependsOnStage: ${{ parameters.previousStage }} + dependsOnStage: ${{ parameters.baseImageArtifactStage }} - ${{ if or(parameters.forceTridentRebuild, eq(parameters.stageType, 'pr-e2e'), eq(parameters.stageType, 'ci'), eq(parameters.stageType, 'pr-e2e-azure')) }}: # Build USB-ISO @@ -228,7 +240,7 @@ stages: # Functional Testing - template: stages/testing_functional/functional-testing.yml parameters: - dependsOnStage: ${{ parameters.previousStage }} + dependsOnStage: ${{ parameters.baseImageArtifactStage }} # VM Testing (host, daily) - template: stages/testing_vm/netlaunch-testing.yml @@ -261,7 +273,7 @@ stages: # Functional Testing - template: stages/testing_functional/functional-testing.yml parameters: - dependsOnStage: ${{ parameters.previousStage }} + dependsOnStage: ${{ parameters.baseImageArtifactStage }} # VM Testing (host, weekly) - template: stages/testing_vm/netlaunch-testing.yml @@ -296,7 +308,7 @@ stages: parameters: downloadPrebuiltImage: false osModifierBranch: ${{ parameters.osModifierBranch }} - dependsOnStage: ${{ parameters.previousStage }} + dependsOnStage: ${{ parameters.baseImageArtifactStage }} # VM Testing (host, post_merge) @@ -317,7 +329,7 @@ stages: - template: stages/testing_functional/functional-testing.yml parameters: rerunTests: false - dependsOnStage: ${{ parameters.previousStage }} + dependsOnStage: ${{ parameters.baseImageArtifactStage }} # VM Testing (host, pullrequest) - template: stages/testing_vm/netlaunch-testing.yml diff --git a/.pipelines/trident-cicd.yml b/.pipelines/trident-cicd.yml index 75dc857d6..d506b1df5 100644 --- a/.pipelines/trident-cicd.yml +++ b/.pipelines/trident-cicd.yml @@ -1,3 +1,8 @@ +# trident-cicd will create the base image artifact for a given azurelinux +# set of builds and call e2e-template to run all trident tests for each of +# our scenarios. +# +# This is a manual placeholder until an triggered solution is created. parameters: - name: baseImagePipelineBuildId displayName: "Build Id of the pipeline run, default will select latest successful run from pipeline 2116 ([AMD64-6-OneBranch]-Prod-BuildImages) with tag 3.0-preview" @@ -55,26 +60,25 @@ extends: - template: templates/trident-platform-cicd-template.yml parameters: - previousStage: HandleBaseImageConfig_cicd - # runBaremetalTests: false + baseImageArtifactStage: HandleBaseImageConfig_cicd - - stage: ProcessJunit_Stage - displayName: Process junit files - dependsOn: - - FunctionalTesting - - DeploymentTesting_host - - DeploymentTesting_container + # - stage: ProcessJunit_Stage + # displayName: Process junit files + # dependsOn: + # - FunctionalTesting + # - DeploymentTesting_host + # - DeploymentTesting_container - jobs: - - job: ProcessJunit_Job - displayName: Process junit files - pool: - type: linux - variables: - ob_outputDirectory: "$(Build.SourcesDirectory)/out" - steps: - - script: | - set -eux - echo "Collect junit and do stuff" - displayName: Process junit files + # jobs: + # - job: ProcessJunit_Job + # displayName: Process junit files + # pool: + # type: linux + # variables: + # ob_outputDirectory: "$(Build.SourcesDirectory)/out" + # steps: + # - script: | + # set -eux + # echo "Collect and process junit files" + # displayName: Process junit files From 5a897b9abb005681789a0fa388368e79a230513f Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 8 Aug 2025 15:58:39 +0000 Subject: [PATCH 06/69] try --- .../common_tasks/find-base-image-version.yml | 31 ++++++------------- .../read-base-image-config-artifact.yml | 13 +++----- 2 files changed, 14 insertions(+), 30 deletions(-) diff --git a/.pipelines/templates/stages/common_tasks/find-base-image-version.yml b/.pipelines/templates/stages/common_tasks/find-base-image-version.yml index 35b949860..8d2979755 100644 --- a/.pipelines/templates/stages/common_tasks/find-base-image-version.yml +++ b/.pipelines/templates/stages/common_tasks/find-base-image-version.yml @@ -1,8 +1,6 @@ -steps: + steps: - template: read-base-image-config-artifact.yml - bash: | - set -eux - echo "BASEIMG_BUILD_TYPE: $(BASEIMG_BUILD_TYPE)" echo "BASE_IMAGE_PIPELINE_BUILD_ID: $(BASE_IMAGE_PIPELINE_BUILD_ID)" printenv @@ -10,25 +8,21 @@ steps: # baseimgBuildType == dev - script: | - set -eux - baseimgVersion="*.*.*" + BASEIMG_VERSION="*.*.*" if [[ "$(BASE_IMAGE_PIPELINE_BUILD_ID)" != "latestFromBranch" ]]; then - baseimgVersion="$(BASE_IMAGE_PIPELINE_BUILD_ID)" + BASEIMG_VERSION="$(BASE_IMAGE_PIPELINE_BUILD_ID)" fi - - set +x - echo "##vso[task.setvariable variable=BASEIMG_VERSION]$baseimgVersion" + echo "##vso[task.setvariable variable=BASEIMG_VERSION]$BASEIMG_VERSION" + echo "BASEIMG_VERSION: $BASEIMG_VERSION" name: SetBaseImageVersionDev condition: eq(variables['BASEIMG_BUILD_TYPE'], 'dev') displayName: Set Base Image version variable for dev # baseimgBuildType == release - script: | - set -eux - baseimgVersion="*.*.*" - - set +x + BASEIMG_VERSION="*.*.* echo "##vso[task.setvariable variable=BASEIMG_VERSION]$baseimgVersion" + echo "BASEIMG_VERSION: $BASEIMG_VERSION" name: SetBaseImageVersionRelease condition: eq(variables['BASEIMG_BUILD_TYPE'], 'release') displayName: Set Base Image version variable for release @@ -49,21 +43,16 @@ steps: patterns: "**/build-config/version-full.config" targetPath: "$(Build.SourcesDirectory)/output" - script: | - set -eux versionFullConfigPath=$(find $(Build.SourcesDirectory) -name version-full.config) - baseimgVersion=$(cat $versionFullConfigPath) - - set +x - echo "##vso[task.setvariable variable=BASEIMG_VERSION]$baseimgVersion" + BASEIMG_VERSION=$(cat $versionFullConfigPath) + echo "##vso[task.setvariable variable=BASEIMG_VERSION]$BASEIMG_VERSION" + echo "BASEIMG_VERSION: $BASEIMG_VERSION" name: SetBaseImageVersionPreview condition: eq(variables['BASEIMG_BUILD_TYPE'], 'preview') displayName: Set Base Image version variable for preview - bash: | - set -eux - echo "BASEIMG_BUILD_TYPE: $(BASEIMG_BUILD_TYPE)" echo "BASE_IMAGE_PIPELINE_BUILD_ID: $(BASE_IMAGE_PIPELINE_BUILD_ID)" echo "BASEIMG_VERSION: $(BASEIMG_VERSION)" - printenv displayName: Show variables after diff --git a/.pipelines/templates/stages/common_tasks/read-base-image-config-artifact.yml b/.pipelines/templates/stages/common_tasks/read-base-image-config-artifact.yml index 769cc5401..30053189d 100644 --- a/.pipelines/templates/stages/common_tasks/read-base-image-config-artifact.yml +++ b/.pipelines/templates/stages/common_tasks/read-base-image-config-artifact.yml @@ -7,20 +7,15 @@ steps: targetPath: /tmp/base-image-config - bash: | - set -eux BASEIMG_BUILD_TYPE=$(jq -r .baseimgBuildType /tmp/base-image-config/baseimage.json) BASE_IMAGE_PIPELINE_BUILD_ID=$(jq -r .baseImagePipelineBuildId /tmp/base-image-config/baseimage.json) BASE_IMAGE_ARM64_PIPELINE_BUILD_ID=$(jq -r .baseImageArm64PipelineBuildId /tmp/base-image-config/baseimage.json) - set +x + echo "BASEIMG_BUILD_TYPE = $(BASEIMG_BUILD_TYPE)" + echo "BASE_IMAGE_PIPELINE_BUILD_ID = $(BASE_IMAGE_PIPELINE_BUILD_ID)" + echo "BASE_IMAGE_ARM64_PIPELINE_BUILD_ID = $(BASE_IMAGE_ARM64_PIPELINE_BUILD_ID)" + echo "##vso[task.setvariable variable=BASEIMG_BUILD_TYPE]$BASEIMG_BUILD_TYPE" echo "##vso[task.setvariable variable=BASE_IMAGE_PIPELINE_BUILD_ID]$BASE_IMAGE_PIPELINE_BUILD_ID" echo "##vso[task.setvariable variable=BASE_IMAGE_ARM64_PIPELINE_BUILD_ID]$BASE_IMAGE_ARM64_PIPELINE_BUILD_ID" displayName: "Read Base Image Config artifact into variables" - - - bash: | - set -eux - echo "BASEIMG_BUILD_TYPE = $(BASEIMG_BUILD_TYPE)" - echo "BASE_IMAGE_PIPELINE_BUILD_ID = $(BASE_IMAGE_PIPELINE_BUILD_ID)" - echo "BASE_IMAGE_ARM64_PIPELINE_BUILD_ID = $(BASE_IMAGE_ARM64_PIPELINE_BUILD_ID)" - displayName: "Show Base Image Config artifact variables" From ba19bd4ae3fcdde459263293921cf3d645ffb004 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 8 Aug 2025 16:02:06 +0000 Subject: [PATCH 07/69] pass baseImageArtifactStage --- .../templates/trident-platform-cicd-for-prism-template.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.pipelines/templates/trident-platform-cicd-for-prism-template.yml b/.pipelines/templates/trident-platform-cicd-for-prism-template.yml index ba7640064..4a4fd2b5a 100644 --- a/.pipelines/templates/trident-platform-cicd-for-prism-template.yml +++ b/.pipelines/templates/trident-platform-cicd-for-prism-template.yml @@ -23,6 +23,9 @@ parameters: type: string default: "main" + - name: baseImageArtifactStage + type: string + stages: - template: e2e-extended-template.yml parameters: @@ -32,3 +35,4 @@ stages: micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} osModifierBranch: ${{ parameters.osModifierBranch }} + baseImageArtifactStage: ${{ parameters.baseImageArtifactStage }} From 042fdf748d8c8fe39d79f5a45ec39d60067d1e45 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 8 Aug 2025 16:06:15 +0000 Subject: [PATCH 08/69] wrong template --- .../templates/trident-platform-cicd-for-prism-template.yml | 4 ---- .pipelines/templates/trident-platform-cicd-template.yml | 6 +++--- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.pipelines/templates/trident-platform-cicd-for-prism-template.yml b/.pipelines/templates/trident-platform-cicd-for-prism-template.yml index 4a4fd2b5a..ba7640064 100644 --- a/.pipelines/templates/trident-platform-cicd-for-prism-template.yml +++ b/.pipelines/templates/trident-platform-cicd-for-prism-template.yml @@ -23,9 +23,6 @@ parameters: type: string default: "main" - - name: baseImageArtifactStage - type: string - stages: - template: e2e-extended-template.yml parameters: @@ -35,4 +32,3 @@ stages: micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} osModifierBranch: ${{ parameters.osModifierBranch }} - baseImageArtifactStage: ${{ parameters.baseImageArtifactStage }} diff --git a/.pipelines/templates/trident-platform-cicd-template.yml b/.pipelines/templates/trident-platform-cicd-template.yml index df57f26c3..a1b5a58e9 100644 --- a/.pipelines/templates/trident-platform-cicd-template.yml +++ b/.pipelines/templates/trident-platform-cicd-template.yml @@ -7,9 +7,9 @@ parameters: - name: runBaremetalTests type: boolean default: false - - name: previousStage + + - name: baseImageArtifactStage type: string - default: '' stages: - template: e2e-template.yml @@ -18,4 +18,4 @@ stages: blockPublishing: true forceTridentRebuild: true runBaremetalTests: ${{ parameters.runBaremetalTests }} - previousStage: ${{ parameters.previousStage }} + baseImageArtifactStage: ${{ parameters.baseImageArtifactStage }} From ecf6fc9ebd0dd88cbb3f89bce26113a6c66f75ad Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 8 Aug 2025 16:15:12 +0000 Subject: [PATCH 09/69] fix ependsOn --- .pipelines/templates/scale-template.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.pipelines/templates/scale-template.yml b/.pipelines/templates/scale-template.yml index 913978fbd..00ac7f9ff 100644 --- a/.pipelines/templates/scale-template.yml +++ b/.pipelines/templates/scale-template.yml @@ -70,12 +70,15 @@ parameters: stages: - template: stages/common_tasks/create-base-image-config-artifact.yml parameters: + stageSuffix: scale baseimgBuildType: ${{ parameters.baseimgBuildType }} baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} baseImageArm64PipelineBuildId: ${{ parameters.baseImageArm64PipelineBuildId }} # Build tools (Go stuff) - template: stages/building_tools/building-tools.yml + parameters: + dependsOnStage: HandleBaseImageConfig_scale # Trident Build/Download - template: stages/trident_rpms/trident-stage.yml @@ -84,6 +87,7 @@ stages: tridentArtifactName: trident-binaries codeCoverage: false forceTridentRebuild: ${{ or(eq(parameters.forceTridentRebuild, true), eq(parameters.baseimgBuildType, 'preview')) }} + dependsOnStage: HandleBaseImageConfig_scale # Test Servicing on VMs - template: stages/testing_servicing/vm-testing.yml From 2c127e2989dfe101477598f7d3909e0a19420248 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 8 Aug 2025 16:39:01 +0000 Subject: [PATCH 10/69] merged --- .pipelines/trident-cicd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pipelines/trident-cicd.yml b/.pipelines/trident-cicd.yml index d506b1df5..cedf963a8 100644 --- a/.pipelines/trident-cicd.yml +++ b/.pipelines/trident-cicd.yml @@ -35,12 +35,12 @@ resources: - repository: platform-pipelines type: git name: platform-pipelines - ref: refs/heads/user/bfjelds/enable-idc-testing-artifact + ref: refs/heads/main - repository: test-images type: git name: test-images - ref: refs/heads/user/bfjelds/enable-idc-testing-artifact + ref: refs/heads/main - repository: platform-telemetry name: platform-telemetry From b941c1b23d8cfc2d3a801008033106edd959efed Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 8 Aug 2025 16:39:12 +0000 Subject: [PATCH 11/69] missed one --- .../templates/stages/common_tasks/find-base-image-version.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/stages/common_tasks/find-base-image-version.yml b/.pipelines/templates/stages/common_tasks/find-base-image-version.yml index 8d2979755..e852a333e 100644 --- a/.pipelines/templates/stages/common_tasks/find-base-image-version.yml +++ b/.pipelines/templates/stages/common_tasks/find-base-image-version.yml @@ -21,7 +21,7 @@ # baseimgBuildType == release - script: | BASEIMG_VERSION="*.*.* - echo "##vso[task.setvariable variable=BASEIMG_VERSION]$baseimgVersion" + echo "##vso[task.setvariable variable=BASEIMG_VERSION]$BASEIMG_VERSION" echo "BASEIMG_VERSION: $BASEIMG_VERSION" name: SetBaseImageVersionRelease condition: eq(variables['BASEIMG_BUILD_TYPE'], 'release') From d6362807420052b56fe54c958790fd5dc01a3dfa Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 8 Aug 2025 16:41:03 +0000 Subject: [PATCH 12/69] rename --- .pipelines/templates/e2e-template.yml | 6 +++--- .pipelines/templates/trident-platform-cicd-template.yml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.pipelines/templates/e2e-template.yml b/.pipelines/templates/e2e-template.yml index 48e15079b..cb6f43906 100644 --- a/.pipelines/templates/e2e-template.yml +++ b/.pipelines/templates/e2e-template.yml @@ -79,7 +79,7 @@ parameters: type: string default: "submodule" - - name: runBaremetalTests + - name: baremetalTestsEnabled type: boolean default: true @@ -248,7 +248,7 @@ stages: buildPurpose: "daily" runtimeEnv: "host" - - ${{ if parameters.runBaremetalTests }}: + - ${{ if parameters.baremetalTestsEnabled }}: # BM Testing (host, daily) - template: stages/testing_baremetal/baremetal-testing.yml parameters: @@ -287,7 +287,7 @@ stages: buildPurpose: "weekly" runtimeEnv: "container" - - ${{ if parameters.runBaremetalTests }}: + - ${{ if parameters.baremetalTestsEnabled }}: # BM Testing (host, weekly) - template: stages/testing_baremetal/baremetal-testing.yml parameters: diff --git a/.pipelines/templates/trident-platform-cicd-template.yml b/.pipelines/templates/trident-platform-cicd-template.yml index a1b5a58e9..07850c4b2 100644 --- a/.pipelines/templates/trident-platform-cicd-template.yml +++ b/.pipelines/templates/trident-platform-cicd-template.yml @@ -4,7 +4,7 @@ # preview build. parameters: - - name: runBaremetalTests + - name: baremetalTestsEnabled type: boolean default: false @@ -17,5 +17,5 @@ stages: stageType: pre blockPublishing: true forceTridentRebuild: true - runBaremetalTests: ${{ parameters.runBaremetalTests }} + baremetalTestsEnabled: ${{ parameters.baremetalTestsEnabled }} baseImageArtifactStage: ${{ parameters.baseImageArtifactStage }} From 9c67e9b42bdfe8e05db0fbad62d9d563aaad84db Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 8 Aug 2025 16:46:08 +0000 Subject: [PATCH 13/69] cleanup depends --- .../templates/stages/build_image/build-image.yml | 12 ++++++------ .../stages/testing_functional/functional-testing.yml | 11 ++++++----- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.pipelines/templates/stages/build_image/build-image.yml b/.pipelines/templates/stages/build_image/build-image.yml index 1523bb959..4b52298b5 100644 --- a/.pipelines/templates/stages/build_image/build-image.yml +++ b/.pipelines/templates/stages/build_image/build-image.yml @@ -40,12 +40,12 @@ parameters: stages: - stage: TridentTestImg_${{ replace(parameters.imageName, '-', '_') }} displayName: Build ${{ parameters.imageName }} - dependsOn: - - ${{ if and(eq(parameters.runtimeEnv, 'host'), parameters.dependsOnTrident) }}: - - GetTridentBinaries_rpms - - ${{ elseif ne(parameters.dependsOnStage, '') }}: - - ${{ parameters.dependsOnStage }} - - ${{ else }}: [] + ${{ if and(eq(parameters.runtimeEnv, 'host'), parameters.dependsOnTrident) }}: + dependsOn: + - GetTridentBinaries_rpms + ${{ elseif ne(parameters.dependsOnStage, '') }}: + dependsOn: + - ${{ parameters.dependsOnStage }} jobs: - job: BuildTridentTestImg diff --git a/.pipelines/templates/stages/testing_functional/functional-testing.yml b/.pipelines/templates/stages/testing_functional/functional-testing.yml index 01ae53704..6f5aaeef6 100644 --- a/.pipelines/templates/stages/testing_functional/functional-testing.yml +++ b/.pipelines/templates/stages/testing_functional/functional-testing.yml @@ -29,12 +29,13 @@ parameters: stages: - stage: FunctionalTesting displayName: Functional Testing - dependsOn: - - ${{ if eq(parameters.downloadPrebuiltImage, false) }}: - - TridentTestImg_trident_functest - - ${{ if ne(parameters.dependsOnStage, '') }}: + ${{ if eq(parameters.downloadPrebuiltImage, false) }}: + dependsOn: + - TridentTestImg_trident_functest + - ${{ if ne(parameters.dependsOnStage, '') }}: - ${{ parameters.dependsOnStage }} - - ${{ else }}: [] + ${{ elseif ne(parameters.dependsOnStage, '') }}: + dependsOn: ${{ parameters.dependsOnStage }} jobs: - job: FunctionalTests From 5ec37eb9cc60099fc7671718c8a938420205e810 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 8 Aug 2025 17:00:17 +0000 Subject: [PATCH 14/69] missed one --- .pipelines/templates/stages/testing_servicing/build-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/stages/testing_servicing/build-image.yml b/.pipelines/templates/stages/testing_servicing/build-image.yml index 52bc5a527..b0568f578 100644 --- a/.pipelines/templates/stages/testing_servicing/build-image.yml +++ b/.pipelines/templates/stages/testing_servicing/build-image.yml @@ -74,7 +74,7 @@ jobs: micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} baseimgBuildType: $(BASEIMG_BUILD_TYPE) - baseimgVersion: $(baseimgVersion) + baseimgVersion: $(BASEIMG_VERSION) baseimgAzureLinuxVersion: ${{ parameters.baseimgAzlVersion }} downloadTrident: false baseimgType: ${{ parameters.baseimgType }} From 1b36e903c604434adb868e73f8cd1730ba6c07b0 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 8 Aug 2025 17:06:05 +0000 Subject: [PATCH 15/69] remove --- .pipelines/templates/e2e-extended-template.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pipelines/templates/e2e-extended-template.yml b/.pipelines/templates/e2e-extended-template.yml index 334899871..dbb1b0627 100644 --- a/.pipelines/templates/e2e-extended-template.yml +++ b/.pipelines/templates/e2e-extended-template.yml @@ -77,7 +77,6 @@ stages: baseimgBuildType: ${{ parameters.baseimgBuildType }} baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} baseImageArm64PipelineBuildId: ${{ parameters.baseImageArm64PipelineBuildId }} - previousStage: ${{ parameters.previousStage }} - template: e2e-template.yml parameters: From c5dea0be0117a36caf47251f45504bfd6aa9878e Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 8 Aug 2025 17:08:14 +0000 Subject: [PATCH 16/69] fix dependency --- .pipelines/templates/e2e-extended-template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/e2e-extended-template.yml b/.pipelines/templates/e2e-extended-template.yml index dbb1b0627..a0f73b587 100644 --- a/.pipelines/templates/e2e-extended-template.yml +++ b/.pipelines/templates/e2e-extended-template.yml @@ -88,4 +88,4 @@ stages: micVersion: ${{ parameters.micVersion }} numberOfUpdateIterations: ${{ parameters.numberOfUpdateIterations }} osModifierBranch: ${{ parameters.osModifierBranch }} - baseImageArtifactStage: HandleBaseImageConfig_cicd + baseImageArtifactStage: HandleBaseImageConfig_e2e From 65e8ca26fb3d9a1f441b6dfc2a30d46141f40a94 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 8 Aug 2025 17:29:11 +0000 Subject: [PATCH 17/69] fix --- .../templates/stages/common_tasks/find-base-image-version.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/stages/common_tasks/find-base-image-version.yml b/.pipelines/templates/stages/common_tasks/find-base-image-version.yml index e852a333e..7f46a8c52 100644 --- a/.pipelines/templates/stages/common_tasks/find-base-image-version.yml +++ b/.pipelines/templates/stages/common_tasks/find-base-image-version.yml @@ -20,7 +20,7 @@ # baseimgBuildType == release - script: | - BASEIMG_VERSION="*.*.* + BASEIMG_VERSION="*.*.*" echo "##vso[task.setvariable variable=BASEIMG_VERSION]$BASEIMG_VERSION" echo "BASEIMG_VERSION: $BASEIMG_VERSION" name: SetBaseImageVersionRelease From 53ea11718a1b819d8fd63cc6530260226640a309 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 8 Aug 2025 18:21:39 +0000 Subject: [PATCH 18/69] wait for all tests --- .pipelines/trident-cicd.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.pipelines/trident-cicd.yml b/.pipelines/trident-cicd.yml index cedf963a8..c29c428da 100644 --- a/.pipelines/trident-cicd.yml +++ b/.pipelines/trident-cicd.yml @@ -68,6 +68,9 @@ extends: # - FunctionalTesting # - DeploymentTesting_host # - DeploymentTesting_container + # - BaremetalDeploymentTesting_host + # - BaremetalDeploymentTesting_container + # - ServicingTesting # jobs: # - job: ProcessJunit_Job From db519edf403b7efc6edb4cd9d9298cc59b83a53a Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 8 Aug 2025 18:22:25 +0000 Subject: [PATCH 19/69] vm vs bm, host vs container --- .../e2e-ab-update-stage-finalize-test-run.yml | 6 +++--- .../stages/testing_common/e2e-test-run.yml | 18 +++++++++--------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.pipelines/templates/stages/testing_common/e2e-ab-update-stage-finalize-test-run.yml b/.pipelines/templates/stages/testing_common/e2e-ab-update-stage-finalize-test-run.yml index 79bf7ca9f..9319514fb 100644 --- a/.pipelines/templates/stages/testing_common/e2e-ab-update-stage-finalize-test-run.yml +++ b/.pipelines/templates/stages/testing_common/e2e-ab-update-stage-finalize-test-run.yml @@ -201,7 +201,7 @@ steps: python3 -u -m pytest \ -m "${{ parameters.buildPurpose }}" \ --capture=no \ - --junit-xml=${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_stage_finalize_ab_update_B_$(System.JobAttempt).junit.xml \ + --junit-xml=${{ parameters.deploymentEnvironment }}_${{ parameters.runtimeEnv }}_${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_stage_finalize_ab_update_B_$(System.JobAttempt).junit.xml \ --host ${{ parameters.hostIp }} \ --runtime-env ${{ parameters.runtimeEnv }} \ --configuration ${{ parameters.tridentConfigPath }} \ @@ -213,7 +213,7 @@ steps: - template: ../junit/handle-junit-test-results.yml parameters: - testRunName: "${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_stage_finalize_ab_update_B_$(System.JobAttempt)" - junitTestFile: "./e2e_tests/${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_stage_finalize_ab_update_B_$(System.JobAttempt).junit.xml" + testRunName: "${{ parameters.deploymentEnvironment }}_${{ parameters.runtimeEnv }}_${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_stage_finalize_ab_update_B_$(System.JobAttempt)" + junitTestFile: "./e2e_tests/${{ parameters.deploymentEnvironment }}_${{ parameters.runtimeEnv }}_${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_stage_finalize_ab_update_B_$(System.JobAttempt).junit.xml" displayNameSpecifier: "A/B update into runtime OS A that was staged and finalized independently" artifactName: "junit_for_trident_ab_update_stage" diff --git a/.pipelines/templates/stages/testing_common/e2e-test-run.yml b/.pipelines/templates/stages/testing_common/e2e-test-run.yml index e01ee7c57..857dcf72d 100644 --- a/.pipelines/templates/stages/testing_common/e2e-test-run.yml +++ b/.pipelines/templates/stages/testing_common/e2e-test-run.yml @@ -92,7 +92,7 @@ steps: python3 -u -m pytest \ -m "${{ parameters.buildPurpose }}" \ --capture=no \ - --junit-xml=${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_clean_install_$(System.JobAttempt).junit.xml \ + --junit-xml=${{ parameters.deploymentEnvironment }}_${{ parameters.runtimeEnv }}_${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_clean_install_$(System.JobAttempt).junit.xml \ --host ${{ parameters.hostIp }} \ --runtime-env ${{ parameters.runtimeEnv }} \ --configuration ${{ parameters.tridentConfigPath }} @@ -103,8 +103,8 @@ steps: - template: ../junit/handle-junit-test-results.yml parameters: - testRunName: "${{ parameters.deploymentEnvironment }}_trident_e2e_tests_${{ parameters.tridentConfigurationName }}_clean_install_$(System.JobAttempt)" - junitTestFile: "./e2e_tests/${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_clean_install_$(System.JobAttempt).junit.xml" + testRunName: "${{ parameters.deploymentEnvironment }}_${{ parameters.runtimeEnv }}_trident_e2e_tests_${{ parameters.tridentConfigurationName }}_clean_install_$(System.JobAttempt)" + junitTestFile: "./e2e_tests/${{ parameters.deploymentEnvironment }}_${{ parameters.runtimeEnv }}_${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_clean_install_$(System.JobAttempt).junit.xml" displayNameSpecifier: "clean install of runtime OS" artifactName: "junit_for_trident_clean_install" @@ -194,7 +194,7 @@ steps: python3 -u -m pytest \ -m "${{ parameters.buildPurpose }}" \ --capture=no \ - --junit-xml=${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_ab_update_B_$(System.JobAttempt).junit.xml \ + --junit-xml=${{ parameters.deploymentEnvironment }}_${{ parameters.runtimeEnv }}_${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_ab_update_B_$(System.JobAttempt).junit.xml \ --host ${{ parameters.hostIp }} \ --runtime-env ${{ parameters.runtimeEnv }} \ --configuration ${{ parameters.tridentConfigPath }} \ @@ -207,8 +207,8 @@ steps: - template: ../junit/handle-junit-test-results.yml parameters: - testRunName: "${{ parameters.deploymentEnvironment }}_trident_e2e_tests_${{ parameters.tridentConfigurationName }}_ab_update_B_$(System.JobAttempt)" - junitTestFile: "./e2e_tests/${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_ab_update_B_$(System.JobAttempt).junit.xml" + testRunName: "${{ parameters.deploymentEnvironment }}_${{ parameters.runtimeEnv }}_trident_e2e_tests_${{ parameters.tridentConfigurationName }}_ab_update_B_$(System.JobAttempt)" + junitTestFile: "./e2e_tests/${{ parameters.deploymentEnvironment }}_${{ parameters.runtimeEnv }}_${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_ab_update_B_$(System.JobAttempt).junit.xml" displayNameSpecifier: "A/B update into runtime OS B" artifactName: "junit_for_trident_ab_update_B" @@ -297,7 +297,7 @@ steps: python3 -u -m pytest \ -m "${{ parameters.buildPurpose }}" \ --capture=no \ - --junit-xml=${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_ab_update_A_$(System.JobAttempt).junit.xml \ + --junit-xml=${{ parameters.deploymentEnvironment }}_${{ parameters.runtimeEnv }}_${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_ab_update_A_$(System.JobAttempt).junit.xml \ --host ${{ parameters.hostIp }} \ --runtime-env ${{ parameters.runtimeEnv }} \ --configuration ${{ parameters.tridentConfigPath }} \ @@ -310,8 +310,8 @@ steps: - template: ../junit/handle-junit-test-results.yml parameters: - testRunName: "${{ parameters.deploymentEnvironment }}_trident_e2e_tests_${{ parameters.tridentConfigurationName }}_ab_update_A_$(System.JobAttempt)" - junitTestFile: "./e2e_tests/${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_ab_update_A_$(System.JobAttempt).junit.xml" + testRunName: "${{ parameters.deploymentEnvironment }}_${{ parameters.runtimeEnv }}_trident_e2e_tests_${{ parameters.tridentConfigurationName }}_ab_update_A_$(System.JobAttempt)" + junitTestFile: "./e2e_tests/${{ parameters.deploymentEnvironment }}_${{ parameters.runtimeEnv }}_${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_ab_update_A_$(System.JobAttempt).junit.xml" displayNameSpecifier: "A/B update into runtime OS A" artifactName: "junit_for_trident_ab_update_A" From 373b557e1640d9f03af9f76033b05ba80b7228b4 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 8 Aug 2025 20:34:05 +0000 Subject: [PATCH 20/69] fix release.yml --- .../stages/trident_rpms/build-source.yml | 6 ------ .../templates/stages/trident_rpms/release.yml | 15 ++++----------- 2 files changed, 4 insertions(+), 17 deletions(-) diff --git a/.pipelines/templates/stages/trident_rpms/build-source.yml b/.pipelines/templates/stages/trident_rpms/build-source.yml index 71d0665d0..e6799bf0c 100644 --- a/.pipelines/templates/stages/trident_rpms/build-source.yml +++ b/.pipelines/templates/stages/trident_rpms/build-source.yml @@ -78,11 +78,8 @@ stages: - template: ../common_tasks/build-osmodifier.yml parameters: osModifierBranch: ${{ parameters.osModifierBranch }} - - template: ../common_tasks/read-base-image-config-artifact.yml - template: release.yml parameters: - baseimgBuildType: $(BASEIMG_BUILD_TYPE) - baseImagePipelineBuildId: $(BASE_IMAGE_PIPELINE_BUILD_ID) previewContainerPipeline: "[AMD64-6-OneBranch]-Prod-BuildImages" - ${{ if eq(parameters.publishToDevFeed, true) }}: - template: publish-dev.yml @@ -109,9 +106,6 @@ stages: - template: ../common_tasks/build-osmodifier.yml parameters: osModifierBranch: ${{ parameters.osModifierBranch }} - - template: ../common_tasks/read-base-image-config-artifact.yml - template: release.yml parameters: - baseimgBuildType: $(BASEIMG_BUILD_TYPE) - baseImagePipelineBuildId: $(BASE_IMAGE_ARM64_PIPELINE_BUILD_ID) previewContainerPipeline: "[ARM64-6-OneBranch]-Prod-BuildImages" diff --git a/.pipelines/templates/stages/trident_rpms/release.yml b/.pipelines/templates/stages/trident_rpms/release.yml index 3b166b687..8b178195b 100644 --- a/.pipelines/templates/stages/trident_rpms/release.yml +++ b/.pipelines/templates/stages/trident_rpms/release.yml @@ -41,17 +41,10 @@ steps: workingDirectory: $(Build.SourcesDirectory) displayName: Start Docker - - ${{ if eq(parameters.baseimgBuildType, 'preview') }}: - - template: ../common_tasks/preview-container.yml - parameters: - dockerfilePath: $(Build.SourcesDirectory)/Dockerfile.full - dockerBuildContext: $(Build.SourcesDirectory) - previewContainerPipeline: ${{ parameters.previewContainerPipeline }} - ${{ if eq(parameters.baseImagePipelineBuildId, 'latestFromBranch') }}: - runVersion: ${{ parameters.baseImagePipelineBuildId }} - ${{ else }}: - runVersion: specific - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} + - template: ../common_tasks/preview-container.yml + parameters: + dockerfilePath: $(Build.SourcesDirectory)/Dockerfile.runtime + dockerBuildContext: $(Build.SourcesDirectory) - script: | set -eux From 604f939ffa1ca5a192bedbd127d3bfd896e7ac8e Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 8 Aug 2025 20:35:33 +0000 Subject: [PATCH 21/69] missed a save --- .../templates/stages/trident_rpms/release.yml | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/.pipelines/templates/stages/trident_rpms/release.yml b/.pipelines/templates/stages/trident_rpms/release.yml index 8b178195b..91373ebaa 100644 --- a/.pipelines/templates/stages/trident_rpms/release.yml +++ b/.pipelines/templates/stages/trident_rpms/release.yml @@ -1,17 +1,4 @@ parameters: - - name: baseimgBuildType - displayName: Base Image build type - type: string - default: "release" - # values: - # - dev - # - preview - # - release - - - name: baseImagePipelineBuildId - type: string - default: "latestFromBranch" - - name: previewContainerPipeline type: string default: "[AMD64-6-OneBranch]-Prod-BuildImages" @@ -43,8 +30,9 @@ steps: - template: ../common_tasks/preview-container.yml parameters: - dockerfilePath: $(Build.SourcesDirectory)/Dockerfile.runtime + dockerfilePath: $(Build.SourcesDirectory)/Dockerfile.full dockerBuildContext: $(Build.SourcesDirectory) + previewContainerPipeline: ${{ parameters.previewContainerPipeline }} - script: | set -eux From 75d7a2388fa215aea632b62c319f29e6e80d601f Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 8 Aug 2025 21:06:18 +0000 Subject: [PATCH 22/69] fix output issue --- .../read-base-image-config-artifact.yml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.pipelines/templates/stages/common_tasks/read-base-image-config-artifact.yml b/.pipelines/templates/stages/common_tasks/read-base-image-config-artifact.yml index 30053189d..55235f932 100644 --- a/.pipelines/templates/stages/common_tasks/read-base-image-config-artifact.yml +++ b/.pipelines/templates/stages/common_tasks/read-base-image-config-artifact.yml @@ -7,15 +7,26 @@ steps: targetPath: /tmp/base-image-config - bash: | + echo "base image config artifact:" + cat /tmp/base-image-config/baseimage.json + BASEIMG_BUILD_TYPE=$(jq -r .baseimgBuildType /tmp/base-image-config/baseimage.json) BASE_IMAGE_PIPELINE_BUILD_ID=$(jq -r .baseImagePipelineBuildId /tmp/base-image-config/baseimage.json) BASE_IMAGE_ARM64_PIPELINE_BUILD_ID=$(jq -r .baseImageArm64PipelineBuildId /tmp/base-image-config/baseimage.json) - echo "BASEIMG_BUILD_TYPE = $(BASEIMG_BUILD_TYPE)" - echo "BASE_IMAGE_PIPELINE_BUILD_ID = $(BASE_IMAGE_PIPELINE_BUILD_ID)" - echo "BASE_IMAGE_ARM64_PIPELINE_BUILD_ID = $(BASE_IMAGE_ARM64_PIPELINE_BUILD_ID)" + echo "base image config read from artifact:" + echo "BASEIMG_BUILD_TYPE = $BASEIMG_BUILD_TYPE" + echo "BASE_IMAGE_PIPELINE_BUILD_ID = $BASE_IMAGE_PIPELINE_BUILD_ID" + echo "BASE_IMAGE_ARM64_PIPELINE_BUILD_ID = $BASE_IMAGE_ARM64_PIPELINE_BUILD_ID" echo "##vso[task.setvariable variable=BASEIMG_BUILD_TYPE]$BASEIMG_BUILD_TYPE" echo "##vso[task.setvariable variable=BASE_IMAGE_PIPELINE_BUILD_ID]$BASE_IMAGE_PIPELINE_BUILD_ID" echo "##vso[task.setvariable variable=BASE_IMAGE_ARM64_PIPELINE_BUILD_ID]$BASE_IMAGE_ARM64_PIPELINE_BUILD_ID" displayName: "Read Base Image Config artifact into variables" + + - bash: | + echo "base image config runtime variables:" + echo "BASEIMG_BUILD_TYPE = $(BASEIMG_BUILD_TYPE)" + echo "BASE_IMAGE_PIPELINE_BUILD_ID = $(BASE_IMAGE_PIPELINE_BUILD_ID)" + echo "BASE_IMAGE_ARM64_PIPELINE_BUILD_ID = $(BASE_IMAGE_ARM64_PIPELINE_BUILD_ID)" + displayName: "Output Base Image Config runtime variables" From dbcf204c6f6c7c3c15fb066bfed1ab1b2d891612 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 8 Aug 2025 21:15:19 +0000 Subject: [PATCH 23/69] remove unused params --- .pipelines/templates/e2e-template.yml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/.pipelines/templates/e2e-template.yml b/.pipelines/templates/e2e-template.yml index cb6f43906..c860a70be 100644 --- a/.pipelines/templates/e2e-template.yml +++ b/.pipelines/templates/e2e-template.yml @@ -24,23 +24,6 @@ parameters: - rel - full-validation - - name: baseimgBuildType - displayName: Base Image build type - type: string - values: - - dev - - preview - - release - default: "release" - - - name: baseImagePipelineBuildId - type: string - default: "latestFromBranch" - - - name: baseImageArm64PipelineBuildId - type: string - default: "latestFromBranch" - - name: blockPublishing displayName: "Block publishing to artifact feed" type: boolean From f93648e2cb67c20065aced8b5b407854a1c561fd Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 8 Aug 2025 21:56:15 +0000 Subject: [PATCH 24/69] handle dev container --- .../stages/common_tasks/find-base-image-version.yml | 1 - .../stages/common_tasks/preview-container.yml | 11 +++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.pipelines/templates/stages/common_tasks/find-base-image-version.yml b/.pipelines/templates/stages/common_tasks/find-base-image-version.yml index 7f46a8c52..88fd4f72a 100644 --- a/.pipelines/templates/stages/common_tasks/find-base-image-version.yml +++ b/.pipelines/templates/stages/common_tasks/find-base-image-version.yml @@ -3,7 +3,6 @@ - bash: | echo "BASEIMG_BUILD_TYPE: $(BASEIMG_BUILD_TYPE)" echo "BASE_IMAGE_PIPELINE_BUILD_ID: $(BASE_IMAGE_PIPELINE_BUILD_ID)" - printenv displayName: Show variables before # baseimgBuildType == dev diff --git a/.pipelines/templates/stages/common_tasks/preview-container.yml b/.pipelines/templates/stages/common_tasks/preview-container.yml index 555404052..c28bf8e3d 100644 --- a/.pipelines/templates/stages/common_tasks/preview-container.yml +++ b/.pipelines/templates/stages/common_tasks/preview-container.yml @@ -30,21 +30,24 @@ steps: set -eux RUN_VERSION=$(BASE_IMAGE_PIPELINE_BUILD_ID) + RUN_TAGS='3.0-preview' RUN_ID='' if [[ "$RUN_VERSION" != "latestFromBranch" ]]; then RUN_VERSION=specific RUN_ID=$(BASE_IMAGE_PIPELINE_BUILD_ID) + RUN_TAGS='' fi set +x echo "##vso[task.setvariable variable=RUN_VERSION]$RUN_VERSION" echo "##vso[task.setvariable variable=RUN_ID]$RUN_ID" displayName: "Configure RUN_VERSION" - condition: eq(variables['BASEIMG_BUILD_TYPE'], 'preview') + condition: ne(variables['BASEIMG_BUILD_TYPE'], 'release') + # BASEIMG_BUILD_TYPE != release - task: DownloadPipelineArtifact@2 displayName: "Download preview of azl container image" - condition: eq(variables['BASEIMG_BUILD_TYPE'], 'preview') + condition: ne(variables['BASEIMG_BUILD_TYPE'], 'release') inputs: buildType: "specific" project: "mariner" @@ -52,7 +55,7 @@ steps: runVersion: $(RUN_VERSION) branchName: "refs/heads/master" runId: $(RUN_ID) - tags: "3.0-preview" + tags: $(RUN_TAGS) artifactName: "drop_build_rpms_build" patterns: "**/build-artifacts/container_base/core-*.tar.gz" targetPath: "$(Pipeline.Workspace)" @@ -70,4 +73,4 @@ steps: cat ${{ parameters.dockerfilePath }} displayName: Replace azl container base with azl preview - condition: eq(variables['BASEIMG_BUILD_TYPE'], 'preview') \ No newline at end of file + condition: ne(variables['BASEIMG_BUILD_TYPE'], 'release') \ No newline at end of file From a2a58e7c5084483ec2a94b915e3e3259f538eab7 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 8 Aug 2025 22:08:17 +0000 Subject: [PATCH 25/69] use ob_ for base-image artifact; move base-image config to own stage folder --- .../templates/e2e-extended-template.yml | 5 ++-- .pipelines/templates/e2e-template.yml | 2 +- .pipelines/templates/scale-template.yml | 7 +++--- .../create-base-image-config-artifact.yml | 24 +++++++------------ .../read-base-image-config-artifact.yml | 0 .../common_tasks/find-base-image-version.yml | 2 +- .../stages/common_tasks/preview-container.yml | 2 +- .pipelines/trident-cicd.yml | 5 ++-- 8 files changed, 18 insertions(+), 29 deletions(-) rename .pipelines/templates/stages/{common_tasks => base_image_config}/create-base-image-config-artifact.yml (57%) rename .pipelines/templates/stages/{common_tasks => base_image_config}/read-base-image-config-artifact.yml (100%) diff --git a/.pipelines/templates/e2e-extended-template.yml b/.pipelines/templates/e2e-extended-template.yml index a0f73b587..984489b20 100644 --- a/.pipelines/templates/e2e-extended-template.yml +++ b/.pipelines/templates/e2e-extended-template.yml @@ -71,9 +71,8 @@ parameters: default: "submodule" stages: - - template: stages/common_tasks/create-base-image-config-artifact.yml + - template: stages/base_image_config/create-base-image-config-artifact.yml parameters: - stageSuffix: e2e baseimgBuildType: ${{ parameters.baseimgBuildType }} baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} baseImageArm64PipelineBuildId: ${{ parameters.baseImageArm64PipelineBuildId }} @@ -88,4 +87,4 @@ stages: micVersion: ${{ parameters.micVersion }} numberOfUpdateIterations: ${{ parameters.numberOfUpdateIterations }} osModifierBranch: ${{ parameters.osModifierBranch }} - baseImageArtifactStage: HandleBaseImageConfig_e2e + baseImageArtifactStage: CreateBaseImageConfig diff --git a/.pipelines/templates/e2e-template.yml b/.pipelines/templates/e2e-template.yml index c860a70be..a86e3850e 100644 --- a/.pipelines/templates/e2e-template.yml +++ b/.pipelines/templates/e2e-template.yml @@ -5,7 +5,7 @@ # contains information about the base image being used. This artifact # should be created by this template: # -# .pipelines/templates/stages/common_tasks/create-base-image-config-artifact.yml +# .pipelines/templates/stages/base_image_config/create-base-image-config-artifact.yml # # The stage that creates the artifact should be passed into e2e-template # as baseImageArtifactStage. This will ensure that e2e-template waits for diff --git a/.pipelines/templates/scale-template.yml b/.pipelines/templates/scale-template.yml index 00ac7f9ff..14b59d2f8 100644 --- a/.pipelines/templates/scale-template.yml +++ b/.pipelines/templates/scale-template.yml @@ -68,9 +68,8 @@ parameters: default: false stages: - - template: stages/common_tasks/create-base-image-config-artifact.yml + - template: stages/base_image_config/create-base-image-config-artifact.yml parameters: - stageSuffix: scale baseimgBuildType: ${{ parameters.baseimgBuildType }} baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} baseImageArm64PipelineBuildId: ${{ parameters.baseImageArm64PipelineBuildId }} @@ -78,7 +77,7 @@ stages: # Build tools (Go stuff) - template: stages/building_tools/building-tools.yml parameters: - dependsOnStage: HandleBaseImageConfig_scale + dependsOnStage: CreateBaseImageConfig # Trident Build/Download - template: stages/trident_rpms/trident-stage.yml @@ -87,7 +86,7 @@ stages: tridentArtifactName: trident-binaries codeCoverage: false forceTridentRebuild: ${{ or(eq(parameters.forceTridentRebuild, true), eq(parameters.baseimgBuildType, 'preview')) }} - dependsOnStage: HandleBaseImageConfig_scale + dependsOnStage: CreateBaseImageConfig # Test Servicing on VMs - template: stages/testing_servicing/vm-testing.yml diff --git a/.pipelines/templates/stages/common_tasks/create-base-image-config-artifact.yml b/.pipelines/templates/stages/base_image_config/create-base-image-config-artifact.yml similarity index 57% rename from .pipelines/templates/stages/common_tasks/create-base-image-config-artifact.yml rename to .pipelines/templates/stages/base_image_config/create-base-image-config-artifact.yml index 816b07fbc..e6a72499c 100644 --- a/.pipelines/templates/stages/common_tasks/create-base-image-config-artifact.yml +++ b/.pipelines/templates/stages/base_image_config/create-base-image-config-artifact.yml @@ -1,7 +1,4 @@ parameters: - - name: stageSuffix - type: string - - name: baseimgBuildType type: string values: @@ -23,24 +20,24 @@ parameters: default: '' stages: - - stage: HandleBaseImageConfig_${{ parameters.stageSuffix }} - displayName: "Create BaseImage config artifact (${{ parameters.stageSuffix }})" + - stage: CreateBaseImageConfig + displayName: "Create BaseImage config artifact" ${{ if ne(parameters.previousStage, '') }}: dependsOn: ${{ parameters.previousStage }} jobs: - - job: HandleBaseImageConfigJob_${{ parameters.stageSuffix }} - displayName: "Create BaseImage config artifact (${{ parameters.stageSuffix }})" + - job: CreateBaseImageConfigJob + displayName: "Create BaseImage config artifact" pool: type: linux variables: - name: ob_outputDirectory - value: "$(Build.SourcesDirectory)/out" + value: "$(Build.SourcesDirectory)/base-image-config" - name: ob_artifactBaseName - value: "HandleBaseImageConfig_${{ parameters.stageSuffix }}" + value: "base-image-config" steps: - bash: | - mkdir -p /tmp/config - cat << EOF > /tmp/config/baseimage.json + mkdir -p $(ob_outputDirectory) + cat << EOF > $(ob_outputDirectory)/baseimage.json { "baseimgBuildType": "${{ parameters.baseimgBuildType }}", "baseImagePipelineBuildId": "${{ parameters.baseImagePipelineBuildId }}", @@ -48,8 +45,3 @@ stages: } EOF displayName: "Create Base Image Config artifact" - - task: PublishBuildArtifacts@1 - displayName: "Publish Base Image Config artifact" - inputs: - pathToPublish: /tmp/config - artifactName: base-image-config diff --git a/.pipelines/templates/stages/common_tasks/read-base-image-config-artifact.yml b/.pipelines/templates/stages/base_image_config/read-base-image-config-artifact.yml similarity index 100% rename from .pipelines/templates/stages/common_tasks/read-base-image-config-artifact.yml rename to .pipelines/templates/stages/base_image_config/read-base-image-config-artifact.yml diff --git a/.pipelines/templates/stages/common_tasks/find-base-image-version.yml b/.pipelines/templates/stages/common_tasks/find-base-image-version.yml index 88fd4f72a..006ab56ef 100644 --- a/.pipelines/templates/stages/common_tasks/find-base-image-version.yml +++ b/.pipelines/templates/stages/common_tasks/find-base-image-version.yml @@ -1,5 +1,5 @@ steps: - - template: read-base-image-config-artifact.yml + - template: ../base_image_config/read-base-image-config-artifact.yml - bash: | echo "BASEIMG_BUILD_TYPE: $(BASEIMG_BUILD_TYPE)" echo "BASE_IMAGE_PIPELINE_BUILD_ID: $(BASE_IMAGE_PIPELINE_BUILD_ID)" diff --git a/.pipelines/templates/stages/common_tasks/preview-container.yml b/.pipelines/templates/stages/common_tasks/preview-container.yml index c28bf8e3d..7bbba3d3b 100644 --- a/.pipelines/templates/stages/common_tasks/preview-container.yml +++ b/.pipelines/templates/stages/common_tasks/preview-container.yml @@ -25,7 +25,7 @@ parameters: displayName: Path in which the Dockerfile will be run, the base for the container image will be saved there. steps: - - template: ../common_tasks/read-base-image-config-artifact.yml + - template: ../base_image_config/read-base-image-config-artifact.yml - bash: | set -eux diff --git a/.pipelines/trident-cicd.yml b/.pipelines/trident-cicd.yml index c29c428da..bfabd4326 100644 --- a/.pipelines/trident-cicd.yml +++ b/.pipelines/trident-cicd.yml @@ -51,16 +51,15 @@ extends: template: templates/MockOB.yml parameters: stages: - - template: templates/stages/common_tasks/create-base-image-config-artifact.yml + - template: templates/stages/base_image_config/create-base-image-config-artifact.yml parameters: - stageSuffix: cicd baseimgBuildType: ${{ parameters.buildType }} baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} baseImageArm64PipelineBuildId: ${{ parameters.baseImageArm64PipelineBuildId }} - template: templates/trident-platform-cicd-template.yml parameters: - baseImageArtifactStage: HandleBaseImageConfig_cicd + baseImageArtifactStage: CreateBaseImageConfig # - stage: ProcessJunit_Stage # displayName: Process junit files From 2d6fc8fc042d813344c68c8d2af7a37685902094 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 8 Aug 2025 22:10:02 +0000 Subject: [PATCH 26/69] remove dependsOn for dev-build.yml --- .pipelines/templates/e2e-template.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.pipelines/templates/e2e-template.yml b/.pipelines/templates/e2e-template.yml index a86e3850e..2a5a7cf8f 100644 --- a/.pipelines/templates/e2e-template.yml +++ b/.pipelines/templates/e2e-template.yml @@ -90,8 +90,6 @@ stages: - template: stages/validate_makefile/dev-build.yml parameters: osModifierBranch: ${{ parameters.osModifierBranch }} - dependsOnStage: ${{ parameters.baseImageArtifactStage }} - # Build FT base Image, only in CI - ${{ if eq(parameters.stageType, 'ci') }}: From 13104392929171e693a5249c6c1edb98fa376943 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 8 Aug 2025 22:14:51 +0000 Subject: [PATCH 27/69] remove unused --- .../stages/common_tasks/preview-container.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.pipelines/templates/stages/common_tasks/preview-container.yml b/.pipelines/templates/stages/common_tasks/preview-container.yml index 7bbba3d3b..df2ede3a2 100644 --- a/.pipelines/templates/stages/common_tasks/preview-container.yml +++ b/.pipelines/templates/stages/common_tasks/preview-container.yml @@ -1,19 +1,9 @@ parameters: - - name: runVersion - type: string - default: latestFromBranch - displayName: Specific run or latest successful run - - name: previewContainerPipeline type: string default: "[AMD64-6-OneBranch]-Prod-BuildImages" displayName: Name of preview container pipeline - - name: baseImagePipelineBuildId - type: string - default: "" - displayName: Build Id of the pipeline run - - name: dockerfilePath type: string default: "" From 25b473a696af76d343ca65f542fd8771c68cece1 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 8 Aug 2025 22:56:10 +0000 Subject: [PATCH 28/69] remove unused param --- .../stages/trident_usb_iso/trident-usb-iso.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.pipelines/templates/stages/trident_usb_iso/trident-usb-iso.yml b/.pipelines/templates/stages/trident_usb_iso/trident-usb-iso.yml index 55f6d817d..42922df5e 100644 --- a/.pipelines/templates/stages/trident_usb_iso/trident-usb-iso.yml +++ b/.pipelines/templates/stages/trident_usb_iso/trident-usb-iso.yml @@ -1,18 +1,4 @@ parameters: - - name: baseimgBuildType - displayName: Base Image build type - type: string - values: - - dev - - preview - - release - default: "release" - - - name: baseImagePipelineBuildId - displayName: "Build Id of the pipeline run, default will select latest successful run from pipeline 2116 ([AMD64-6-OneBranch]-Prod-BuildImages) with tag 3.0-preview" - type: string - default: "latestFromBranch" - - name: baseimgAzlVersion displayName: Base Image AZL version type: string From 5fe75cd196e9191f98fd33dccfe6adeae58b7dab Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 8 Aug 2025 23:02:04 +0000 Subject: [PATCH 29/69] build dev container --- .../stages/common_tasks/preview-container.yml | 22 ++++++++++++++----- .../stages/trident_rpms/build-source.yml | 4 ++-- .../templates/stages/trident_rpms/release.yml | 10 +++++---- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/.pipelines/templates/stages/common_tasks/preview-container.yml b/.pipelines/templates/stages/common_tasks/preview-container.yml index df2ede3a2..85be3efa2 100644 --- a/.pipelines/templates/stages/common_tasks/preview-container.yml +++ b/.pipelines/templates/stages/common_tasks/preview-container.yml @@ -1,8 +1,10 @@ parameters: - - name: previewContainerPipeline + - name: targetArchitecture type: string - default: "[AMD64-6-OneBranch]-Prod-BuildImages" - displayName: Name of preview container pipeline + default: "AMD64" + values: + - ARM64 + - AMD64 - name: dockerfilePath type: string @@ -20,17 +22,27 @@ steps: set -eux RUN_VERSION=$(BASE_IMAGE_PIPELINE_BUILD_ID) + RUN_BRANCH='refs/heads/master' RUN_TAGS='3.0-preview' RUN_ID='' if [[ "$RUN_VERSION" != "latestFromBranch" ]]; then RUN_VERSION=specific RUN_ID=$(BASE_IMAGE_PIPELINE_BUILD_ID) + RUN_BRANCH='' RUN_TAGS='' fi + PIPELINE_NAME="[AMD64-6-OneBranch]-Prod-BuildImages" + if [[ "${{ parameters.targetArchitecture }}" == "ARM64" ]]; then + PIPELINE_NAME="[ARM64-6-OneBranch]-Prod-BuildImages" + fi + set +x echo "##vso[task.setvariable variable=RUN_VERSION]$RUN_VERSION" + echo "##vso[task.setvariable variable=RUN_BRANCH]$RUN_BRANCH" echo "##vso[task.setvariable variable=RUN_ID]$RUN_ID" + echo "##vso[task.setvariable variable=RUN_TAGS]$RUN_TAGS" + echo "##vso[task.setvariable variable=PIPELINE_NAME]$PIPELINE_NAME" displayName: "Configure RUN_VERSION" condition: ne(variables['BASEIMG_BUILD_TYPE'], 'release') @@ -41,9 +53,9 @@ steps: inputs: buildType: "specific" project: "mariner" - pipeline: ${{ parameters.previewContainerPipeline }} + pipeline: $(PIPELINE_NAME) runVersion: $(RUN_VERSION) - branchName: "refs/heads/master" + branchName: $(RUN_BRANCH) runId: $(RUN_ID) tags: $(RUN_TAGS) artifactName: "drop_build_rpms_build" diff --git a/.pipelines/templates/stages/trident_rpms/build-source.yml b/.pipelines/templates/stages/trident_rpms/build-source.yml index e6799bf0c..b053c441c 100644 --- a/.pipelines/templates/stages/trident_rpms/build-source.yml +++ b/.pipelines/templates/stages/trident_rpms/build-source.yml @@ -80,7 +80,7 @@ stages: osModifierBranch: ${{ parameters.osModifierBranch }} - template: release.yml parameters: - previewContainerPipeline: "[AMD64-6-OneBranch]-Prod-BuildImages" + targetArchitecture: "AMD64" - ${{ if eq(parameters.publishToDevFeed, true) }}: - template: publish-dev.yml @@ -108,4 +108,4 @@ stages: osModifierBranch: ${{ parameters.osModifierBranch }} - template: release.yml parameters: - previewContainerPipeline: "[ARM64-6-OneBranch]-Prod-BuildImages" + targetArchitecture: "ARM64" diff --git a/.pipelines/templates/stages/trident_rpms/release.yml b/.pipelines/templates/stages/trident_rpms/release.yml index 91373ebaa..4c0f94c30 100644 --- a/.pipelines/templates/stages/trident_rpms/release.yml +++ b/.pipelines/templates/stages/trident_rpms/release.yml @@ -1,8 +1,10 @@ parameters: - - name: previewContainerPipeline + - name: targetArchitecture type: string - default: "[AMD64-6-OneBranch]-Prod-BuildImages" - displayName: Name of preview container pipeline + default: "AMD64" + values: + - ARM64 + - AMD64 steps: @@ -32,7 +34,7 @@ steps: parameters: dockerfilePath: $(Build.SourcesDirectory)/Dockerfile.full dockerBuildContext: $(Build.SourcesDirectory) - previewContainerPipeline: ${{ parameters.previewContainerPipeline }} + targetArchitecture: ${{ parameters.targetArchitecture }} - script: | set -eux From bf57afffec34e338b8fc5cfbfa5c511e1263ef6f Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 8 Aug 2025 23:06:35 +0000 Subject: [PATCH 30/69] remove extra dependsOn --- .pipelines/templates/e2e-template.yml | 2 -- .pipelines/templates/stages/trident_rpms/trident-stage.yml | 1 - 2 files changed, 3 deletions(-) diff --git a/.pipelines/templates/e2e-template.yml b/.pipelines/templates/e2e-template.yml index 2a5a7cf8f..afab3ecc7 100644 --- a/.pipelines/templates/e2e-template.yml +++ b/.pipelines/templates/e2e-template.yml @@ -82,8 +82,6 @@ stages: # Build tools (Go stuff) - template: stages/building_tools/building-tools.yml - parameters: - dependsOnStage: ${{ parameters.baseImageArtifactStage }} # Makefile validation, only for CI - ${{ if eq(parameters.stageType, 'ci') }}: diff --git a/.pipelines/templates/stages/trident_rpms/trident-stage.yml b/.pipelines/templates/stages/trident_rpms/trident-stage.yml index 6db1d6254..e0d90ec75 100644 --- a/.pipelines/templates/stages/trident_rpms/trident-stage.yml +++ b/.pipelines/templates/stages/trident_rpms/trident-stage.yml @@ -47,4 +47,3 @@ stages: artifactName: ${{ parameters.tridentArtifactName }} stageType: ${{ parameters.stageType }} packageName: rpms - dependsOnStage: ${{ parameters.dependsOnStage }} From 271af001ef10f11a02ee05a095cc64ccf903db80 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 8 Aug 2025 23:54:17 +0000 Subject: [PATCH 31/69] build-functest is enough depends for functests --- .../stages/testing_functional/functional-testing.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.pipelines/templates/stages/testing_functional/functional-testing.yml b/.pipelines/templates/stages/testing_functional/functional-testing.yml index 6f5aaeef6..2c06561ae 100644 --- a/.pipelines/templates/stages/testing_functional/functional-testing.yml +++ b/.pipelines/templates/stages/testing_functional/functional-testing.yml @@ -32,10 +32,6 @@ stages: ${{ if eq(parameters.downloadPrebuiltImage, false) }}: dependsOn: - TridentTestImg_trident_functest - - ${{ if ne(parameters.dependsOnStage, '') }}: - - ${{ parameters.dependsOnStage }} - ${{ elseif ne(parameters.dependsOnStage, '') }}: - dependsOn: ${{ parameters.dependsOnStage }} jobs: - job: FunctionalTests From ab49abd0146c406b94c0f9d312c0e9a23f46c901 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Fri, 8 Aug 2025 23:54:32 +0000 Subject: [PATCH 32/69] fix arch --- .../templates/stages/common_tasks/preview-container.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.pipelines/templates/stages/common_tasks/preview-container.yml b/.pipelines/templates/stages/common_tasks/preview-container.yml index 85be3efa2..ef24a771b 100644 --- a/.pipelines/templates/stages/common_tasks/preview-container.yml +++ b/.pipelines/templates/stages/common_tasks/preview-container.yml @@ -21,7 +21,12 @@ steps: - bash: | set -eux - RUN_VERSION=$(BASE_IMAGE_PIPELINE_BUILD_ID) + pipelineId=$(BASE_IMAGE_PIPELINE_BUILD_ID) + if [[ "${{ parameters.targetArchitecture }}" == "ARM64" ]]; then + pipelineId=$(BASE_IMAGE_ARM64_PIPELINE_BUILD_ID) + fi + + RUN_VERSION=$pipelineId RUN_BRANCH='refs/heads/master' RUN_TAGS='3.0-preview' RUN_ID='' From a9824572ff9f7f015a19580c326c5ac2979487d7 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Sat, 9 Aug 2025 00:04:55 +0000 Subject: [PATCH 33/69] save the files --- .pipelines/templates/stages/common_tasks/preview-container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/stages/common_tasks/preview-container.yml b/.pipelines/templates/stages/common_tasks/preview-container.yml index ef24a771b..4c631fcb4 100644 --- a/.pipelines/templates/stages/common_tasks/preview-container.yml +++ b/.pipelines/templates/stages/common_tasks/preview-container.yml @@ -32,7 +32,7 @@ steps: RUN_ID='' if [[ "$RUN_VERSION" != "latestFromBranch" ]]; then RUN_VERSION=specific - RUN_ID=$(BASE_IMAGE_PIPELINE_BUILD_ID) + RUN_ID=$(pipelineId) RUN_BRANCH='' RUN_TAGS='' fi From 84826dc3de7cc8d982063c7ef81e8fe6325ebed3 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Sat, 9 Aug 2025 00:10:57 +0000 Subject: [PATCH 34/69] shell variable --- .pipelines/templates/stages/common_tasks/preview-container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/stages/common_tasks/preview-container.yml b/.pipelines/templates/stages/common_tasks/preview-container.yml index 4c631fcb4..9e5179a4a 100644 --- a/.pipelines/templates/stages/common_tasks/preview-container.yml +++ b/.pipelines/templates/stages/common_tasks/preview-container.yml @@ -32,7 +32,7 @@ steps: RUN_ID='' if [[ "$RUN_VERSION" != "latestFromBranch" ]]; then RUN_VERSION=specific - RUN_ID=$(pipelineId) + RUN_ID=$pipelineId RUN_BRANCH='' RUN_TAGS='' fi From bd6ff73487e01de111a49f4835742d1ceb51443a Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Sat, 9 Aug 2025 00:26:46 +0000 Subject: [PATCH 35/69] override download functest --- .pipelines/templates/e2e-template.yml | 15 +++++++++++++-- .../templates/trident-platform-cicd-template.yml | 1 + 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/e2e-template.yml b/.pipelines/templates/e2e-template.yml index afab3ecc7..00e1087a7 100644 --- a/.pipelines/templates/e2e-template.yml +++ b/.pipelines/templates/e2e-template.yml @@ -66,6 +66,10 @@ parameters: type: boolean default: true + - name: forceFunctionalTestImageRebuild + type: boolean + default: false + - name: baseImageArtifactStage type: string @@ -89,8 +93,8 @@ stages: parameters: osModifierBranch: ${{ parameters.osModifierBranch }} - # Build FT base Image, only in CI - - ${{ if eq(parameters.stageType, 'ci') }}: + # Build FT base Image, only for CI or if forceFunctionalTestImageRebuild is true + - ${{ if or(eq(parameters.stageType, 'ci'), eq(parameters.forceFunctionalTestImageRebuild, true)) }}: # Build Trident installer ISO (host) - template: stages/build_image/build-image.yml parameters: @@ -220,6 +224,8 @@ stages: - template: stages/testing_functional/functional-testing.yml parameters: dependsOnStage: ${{ parameters.baseImageArtifactStage }} + ${{ if parameters.forceFunctionalTestImageRebuild }}: + downloadPrebuiltImage: false # VM Testing (host, daily) - template: stages/testing_vm/netlaunch-testing.yml @@ -253,6 +259,9 @@ stages: - template: stages/testing_functional/functional-testing.yml parameters: dependsOnStage: ${{ parameters.baseImageArtifactStage }} + ${{ if parameters.forceFunctionalTestImageRebuild }}: + downloadPrebuiltImage: false + # VM Testing (host, weekly) - template: stages/testing_vm/netlaunch-testing.yml @@ -309,6 +318,8 @@ stages: parameters: rerunTests: false dependsOnStage: ${{ parameters.baseImageArtifactStage }} + ${{ if parameters.forceFunctionalTestImageRebuild }}: + downloadPrebuiltImage: false # VM Testing (host, pullrequest) - template: stages/testing_vm/netlaunch-testing.yml diff --git a/.pipelines/templates/trident-platform-cicd-template.yml b/.pipelines/templates/trident-platform-cicd-template.yml index 07850c4b2..7c0160eed 100644 --- a/.pipelines/templates/trident-platform-cicd-template.yml +++ b/.pipelines/templates/trident-platform-cicd-template.yml @@ -17,5 +17,6 @@ stages: stageType: pre blockPublishing: true forceTridentRebuild: true + forceFunctionalTestImageRebuild: true baremetalTestsEnabled: ${{ parameters.baremetalTestsEnabled }} baseImageArtifactStage: ${{ parameters.baseImageArtifactStage }} From fadf5c6ee1a71477c44c45d378df8940093767a1 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Sat, 9 Aug 2025 00:47:30 +0000 Subject: [PATCH 36/69] go tools do not need base image --- .pipelines/templates/scale-template.yml | 2 -- .../templates/stages/building_tools/building-tools.yml | 6 ------ 2 files changed, 8 deletions(-) diff --git a/.pipelines/templates/scale-template.yml b/.pipelines/templates/scale-template.yml index 14b59d2f8..5732c89fc 100644 --- a/.pipelines/templates/scale-template.yml +++ b/.pipelines/templates/scale-template.yml @@ -76,8 +76,6 @@ stages: # Build tools (Go stuff) - template: stages/building_tools/building-tools.yml - parameters: - dependsOnStage: CreateBaseImageConfig # Trident Build/Download - template: stages/trident_rpms/trident-stage.yml diff --git a/.pipelines/templates/stages/building_tools/building-tools.yml b/.pipelines/templates/stages/building_tools/building-tools.yml index aa027e538..ff5e6c694 100644 --- a/.pipelines/templates/stages/building_tools/building-tools.yml +++ b/.pipelines/templates/stages/building_tools/building-tools.yml @@ -19,15 +19,9 @@ parameters: type: boolean default: true - - name: dependsOnStage - type: string - default: '' - stages: - stage: BuildingTools displayName: Building Tools - ${{ if ne(parameters.dependsOnStage, '') }}: - dependsOn: ${{ parameters.dependsOnStage }} jobs: - job: BuildGoTools From c5c10b587635fd85ce57bf7100682d6448df4ffb Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Sat, 9 Aug 2025 01:09:05 +0000 Subject: [PATCH 37/69] generalize junit skipping --- .../templates/stages/junit/handle-junit-test-results.yml | 6 +++--- .pipelines/templates/stages/testing_common/e2e-test-run.yml | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.pipelines/templates/stages/junit/handle-junit-test-results.yml b/.pipelines/templates/stages/junit/handle-junit-test-results.yml index 3a92a691e..c53a93a0d 100644 --- a/.pipelines/templates/stages/junit/handle-junit-test-results.yml +++ b/.pipelines/templates/stages/junit/handle-junit-test-results.yml @@ -10,7 +10,7 @@ parameters: steps: - task: PublishTestResults@2 - condition: and(succeededOrFailed(), ne(variables['abActiveVolume'], 'null')) + condition: and(succeededOrFailed(), ne(variables['skipJunitHandling'], 'true')) inputs: testResultsFormat: JUnit testResultsFiles: ${{ parameters.junitTestFile }} @@ -22,11 +22,11 @@ steps: mkdir -p /tmp/ONEBRANCH_ARTIFACT cp ${{ parameters.junitTestFile }} /tmp/ONEBRANCH_ARTIFACT/ displayName: "Prepare JUnit XML for ${{ parameters.displayNameSpecifier }}" - condition: and(succeededOrFailed(), ne(variables['abActiveVolume'], 'null')) + condition: and(succeededOrFailed(), ne(variables['skipJunitHandling'], 'true')) - task: PublishBuildArtifacts@1 displayName: "Publish JUnit XML for ${{ parameters.displayNameSpecifier }}" - condition: and(succeededOrFailed(), ne(variables['abActiveVolume'], 'null')) + condition: and(succeededOrFailed(), ne(variables['skipJunitHandling'], 'true')) inputs: pathToPublish: /tmp/ONEBRANCH_ARTIFACT/ artifactName: ${{ parameters.artifactName }} diff --git a/.pipelines/templates/stages/testing_common/e2e-test-run.yml b/.pipelines/templates/stages/testing_common/e2e-test-run.yml index 857dcf72d..b5fbec940 100644 --- a/.pipelines/templates/stages/testing_common/e2e-test-run.yml +++ b/.pipelines/templates/stages/testing_common/e2e-test-run.yml @@ -75,6 +75,7 @@ steps: else echo "Trident config does not require A/B update testing" echo "##vso[task.setvariable variable=abActiveVolume]null" + echo "##vso[task.setvariable variable=skipJunitHandling]true" fi workingDirectory: $(Build.SourcesDirectory)/e2e_tests displayName: "Check if Trident config requires A/B update testing" From 4eff1184cbc8b416a5dc8a9a434c2b2a68f93737 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Sun, 10 Aug 2025 19:01:23 +0000 Subject: [PATCH 38/69] fix pr --- .pipelines/templates/pr-template.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.pipelines/templates/pr-template.yml b/.pipelines/templates/pr-template.yml index 0a46f681d..19096bdda 100644 --- a/.pipelines/templates/pr-template.yml +++ b/.pipelines/templates/pr-template.yml @@ -1,7 +1,13 @@ stages: + - template: stages/base_image_config/create-base-image-config-artifact.yml + parameters: + baseimgBuildType: release + baseImagePipelineBuildId: latestFromBranch + baseImageArm64PipelineBuildId: latestFromBranch - template: stages/trident_rpms/trident-stage.yml parameters: stageType: pr tridentArtifactName: trident-binaries + dependsOnStage: CreateBaseImageConfig - template: stages/validate_makefile/dev-build.yml - template: stages/building_tools/building-tools.yml From ca61af08a2984814eb9180cd1814b5d7d4082bf6 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 12 Aug 2025 15:44:18 +0000 Subject: [PATCH 39/69] split architectures --- ...ed-template.yml => e2e-arm64-template.yml} | 62 +++++++++---------- .pipelines/templates/e2e-template.yml | 3 +- .pipelines/templates/pipeline-selector.yml | 11 +++- .../stages/trident_rpms/build-source.yml | 8 +++ .../stages/trident_rpms/trident-stage.yml | 6 ++ ...ident-platform-cicd-for-prism-template.yml | 13 +++- .../trident-platform-cicd-template.yml | 15 +++++ .pipelines/trident-cicd.yml | 11 ++++ 8 files changed, 91 insertions(+), 38 deletions(-) rename .pipelines/templates/{e2e-extended-template.yml => e2e-arm64-template.yml} (53%) diff --git a/.pipelines/templates/e2e-extended-template.yml b/.pipelines/templates/e2e-arm64-template.yml similarity index 53% rename from .pipelines/templates/e2e-extended-template.yml rename to .pipelines/templates/e2e-arm64-template.yml index 984489b20..07ab7200c 100644 --- a/.pipelines/templates/e2e-extended-template.yml +++ b/.pipelines/templates/e2e-arm64-template.yml @@ -1,7 +1,16 @@ -# e2e-extended-template will create the base image artifact and -# call e2e-template to run all trident tests for each of our scenarios. +# e2e-ARM64-template will run ARM64 trident tests for each of our scenarios. # This includes unit tests, integration tests, and end-to-end tests. - +# +# Prior to invoking this template, an artifact must be created that +# contains information about the base image being used. This artifact +# should be created by this template: +# +# .pipelines/templates/stages/base_image_config/create-base-image-config-artifact.yml +# +# The stage that creates the artifact should be passed into e2e-arm64-template +# as baseImageArtifactStage. This will ensure that e2e-template waits for +# the base image artifact to be created. +# parameters: - name: "stageType" displayName: "Pipeline configuration type" @@ -15,23 +24,6 @@ parameters: - rel - full-validation - - name: baseimgBuildType - displayName: Base Image build type - type: string - values: - - dev - - preview - - release - default: "release" - - - name: baseImagePipelineBuildId - type: string - default: "latestFromBranch" - - - name: baseImageArm64PipelineBuildId - type: string - default: "latestFromBranch" - - name: blockPublishing displayName: "Block publishing to artifact feed" type: boolean @@ -69,22 +61,26 @@ parameters: - name: osModifierBranch type: string default: "submodule" + + - name: baremetalTestsEnabled + type: boolean + default: true + + - name: forceFunctionalTestImageRebuild + type: boolean + default: false + + - name: baseImageArtifactStage + type: string stages: - - template: stages/base_image_config/create-base-image-config-artifact.yml - parameters: - baseimgBuildType: ${{ parameters.baseimgBuildType }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} - baseImageArm64PipelineBuildId: ${{ parameters.baseImageArm64PipelineBuildId }} - - - template: e2e-template.yml + # Trident Build/Download + - template: stages/trident_rpms/trident-stage.yml parameters: stageType: ${{ parameters.stageType }} - blockPublishing: ${{ parameters.blockPublishing }} + tridentArtifactName: trident-binaries + codeCoverage: false forceTridentRebuild: ${{ parameters.forceTridentRebuild }} - includeAzure: ${{ parameters.includeAzure }} - micBuildType: ${{ parameters.micBuildType }} - micVersion: ${{ parameters.micVersion }} - numberOfUpdateIterations: ${{ parameters.numberOfUpdateIterations }} osModifierBranch: ${{ parameters.osModifierBranch }} - baseImageArtifactStage: CreateBaseImageConfig + dependsOnStage: ${{ parameters.baseImageArtifactStage }} + targetArchitecture: arm64 diff --git a/.pipelines/templates/e2e-template.yml b/.pipelines/templates/e2e-template.yml index 00e1087a7..d6d1286cc 100644 --- a/.pipelines/templates/e2e-template.yml +++ b/.pipelines/templates/e2e-template.yml @@ -1,4 +1,4 @@ -# e2e-template will run all trident tests for each of our scenarios. +# e2e-template will run AMD64 trident tests for each of our scenarios. # This includes unit tests, integration tests, and end-to-end tests. # # Prior to invoking this template, an artifact must be created that @@ -83,6 +83,7 @@ stages: forceTridentRebuild: ${{ parameters.forceTridentRebuild }} osModifierBranch: ${{ parameters.osModifierBranch }} dependsOnStage: ${{ parameters.baseImageArtifactStage }} + targetArchitecture: amd64 # Build tools (Go stuff) - template: stages/building_tools/building-tools.yml diff --git a/.pipelines/templates/pipeline-selector.yml b/.pipelines/templates/pipeline-selector.yml index b2420e72d..3bd72bd83 100644 --- a/.pipelines/templates/pipeline-selector.yml +++ b/.pipelines/templates/pipeline-selector.yml @@ -43,12 +43,17 @@ stages: - ${{ if eq(parameters.targetStage, 'pr') }}: - template: pr-template.yml - ${{ else }}: - - template: e2e-extended-template.yml + - template: stages/base_image_config/create-base-image-config-artifact.yml + - template: e2e-template.yml parameters: stageType: ${{ parameters.targetStage }} blockPublishing: ${{ parameters.blockPublishing }} - baseimgBuildType: ${{ parameters.baseimgBuildType }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} # Only perform Azure testing from official pipelines, as only those have access to the Azure subscription. includeAzure: ${{ or(eq(parameters.targetStage, 'pre'), eq(parameters.targetStage, 'ci'), eq(parameters.targetStage, 'pr-e2e-azure')) }} numberOfUpdateIterations: ${{ parameters.numberOfUpdateIterations }} + baseImageArtifactStage: CreateBaseImageConfig + - template: e2e-arm64-template.yml + parameters: + stageType: ${{ parameters.targetStage }} + blockPublishing: ${{ parameters.blockPublishing }} + baseImageArtifactStage: CreateBaseImageConfig diff --git a/.pipelines/templates/stages/trident_rpms/build-source.yml b/.pipelines/templates/stages/trident_rpms/build-source.yml index b053c441c..6803e2399 100644 --- a/.pipelines/templates/stages/trident_rpms/build-source.yml +++ b/.pipelines/templates/stages/trident_rpms/build-source.yml @@ -18,6 +18,12 @@ parameters: type: string default: '' + - name: targetArchitecture + type: string + values: + - amd64 + - arm64 + stages: - stage: GetTridentBinaries_rpms displayName: Trident RPMs (Build) @@ -25,6 +31,7 @@ stages: dependsOn: ${{ parameters.dependsOnStage }} jobs: + - ${{ if eq( parameters.targetArchitecture, 'amd64' ) }}: - job: CheckTrident displayName: Check pool: @@ -84,6 +91,7 @@ stages: - ${{ if eq(parameters.publishToDevFeed, true) }}: - template: publish-dev.yml + - ${{ elseif eq( parameters.targetArchitecture, 'arm64' ) }}: - job: BuildTridentARM64 displayName: Build Trident 3.0 RPMs for ARM64 pool: diff --git a/.pipelines/templates/stages/trident_rpms/trident-stage.yml b/.pipelines/templates/stages/trident_rpms/trident-stage.yml index e0d90ec75..f42f8e26e 100644 --- a/.pipelines/templates/stages/trident_rpms/trident-stage.yml +++ b/.pipelines/templates/stages/trident_rpms/trident-stage.yml @@ -30,6 +30,12 @@ parameters: type: string default: '' + - name: targetArchitecture + type: string + values: + - amd64 + - arm64 + stages: - ${{ if or(parameters.forceTridentRebuild, eq(parameters.stageType, 'pr'), eq(parameters.stageType, 'pr-e2e'), eq(parameters.stageType, 'ci'), eq(parameters.stageType, 'pr-e2e-azure')) }}: - template: build-source.yml diff --git a/.pipelines/templates/trident-platform-cicd-for-prism-template.yml b/.pipelines/templates/trident-platform-cicd-for-prism-template.yml index ba7640064..e690293e5 100644 --- a/.pipelines/templates/trident-platform-cicd-for-prism-template.yml +++ b/.pipelines/templates/trident-platform-cicd-for-prism-template.yml @@ -24,7 +24,8 @@ parameters: default: "main" stages: - - template: e2e-extended-template.yml + - template: stages/base_image_config/create-base-image-config-artifact.yml + - template: e2e-template.yml parameters: stageType: ci blockPublishing: true @@ -32,3 +33,13 @@ stages: micBuildType: ${{ parameters.micBuildType }} micVersion: ${{ parameters.micVersion }} osModifierBranch: ${{ parameters.osModifierBranch }} + baseImageArtifactStage: CreateBaseImageConfig + - template: e2e-arm64-template.yml + parameters: + stageType: ci + blockPublishing: true + forceTridentRebuild: true + micBuildType: ${{ parameters.micBuildType }} + micVersion: ${{ parameters.micVersion }} + osModifierBranch: ${{ parameters.osModifierBranch }} + baseImageArtifactStage: CreateBaseImageConfig diff --git a/.pipelines/templates/trident-platform-cicd-template.yml b/.pipelines/templates/trident-platform-cicd-template.yml index 7c0160eed..ea288566d 100644 --- a/.pipelines/templates/trident-platform-cicd-template.yml +++ b/.pipelines/templates/trident-platform-cicd-template.yml @@ -10,8 +10,15 @@ parameters: - name: baseImageArtifactStage type: string + + - name: targetArchitecture + type: string + values: + - amd64 + - arm64 stages: + - ${{ if eq( parameters.targetArchitecture, 'amd64') }}: - template: e2e-template.yml parameters: stageType: pre @@ -20,3 +27,11 @@ stages: forceFunctionalTestImageRebuild: true baremetalTestsEnabled: ${{ parameters.baremetalTestsEnabled }} baseImageArtifactStage: ${{ parameters.baseImageArtifactStage }} + + - ${{ if eq( parameters.targetArchitecture, 'amd64') }}: + - template: e2e-arm64-template.yml + parameters: + stageType: pre + blockPublishing: true + forceTridentRebuild: true + baseImageArtifactStage: ${{ parameters.baseImageArtifactStage }} diff --git a/.pipelines/trident-cicd.yml b/.pipelines/trident-cicd.yml index bfabd4326..e9c5cc361 100644 --- a/.pipelines/trident-cicd.yml +++ b/.pipelines/trident-cicd.yml @@ -17,6 +17,11 @@ parameters: values: - dev - preview + - name: targetArchitecture + type: string + values: + - amd64 + - arm64 trigger: none @@ -60,6 +65,12 @@ extends: - template: templates/trident-platform-cicd-template.yml parameters: baseImageArtifactStage: CreateBaseImageConfig + targetArchitecture: amd64 + + - template: templates/trident-platform-cicd-template.yml + parameters: + baseImageArtifactStage: CreateBaseImageConfig + targetArchitecture: arm64 # - stage: ProcessJunit_Stage # displayName: Process junit files From be627807eb5bb57076061f4fee138820813d2fd5 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 12 Aug 2025 15:51:51 +0000 Subject: [PATCH 40/69] some fixes --- .../stages/common_tasks/preview-container.yml | 10 +++++----- .../templates/stages/trident_rpms/build-source.yml | 4 ++-- .../templates/stages/trident_rpms/trident-stage.yml | 1 + .pipelines/trident-cicd.yml | 5 ----- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/.pipelines/templates/stages/common_tasks/preview-container.yml b/.pipelines/templates/stages/common_tasks/preview-container.yml index 9e5179a4a..ea9161bc2 100644 --- a/.pipelines/templates/stages/common_tasks/preview-container.yml +++ b/.pipelines/templates/stages/common_tasks/preview-container.yml @@ -1,10 +1,10 @@ parameters: - name: targetArchitecture type: string - default: "AMD64" + default: "amd64" values: - - ARM64 - - AMD64 + - arm64 + - amd64 - name: dockerfilePath type: string @@ -22,7 +22,7 @@ steps: set -eux pipelineId=$(BASE_IMAGE_PIPELINE_BUILD_ID) - if [[ "${{ parameters.targetArchitecture }}" == "ARM64" ]]; then + if [[ "${{ parameters.targetArchitecture }}" == "arm64" ]]; then pipelineId=$(BASE_IMAGE_ARM64_PIPELINE_BUILD_ID) fi @@ -38,7 +38,7 @@ steps: fi PIPELINE_NAME="[AMD64-6-OneBranch]-Prod-BuildImages" - if [[ "${{ parameters.targetArchitecture }}" == "ARM64" ]]; then + if [[ "${{ parameters.targetArchitecture }}" == "arm64" ]]; then PIPELINE_NAME="[ARM64-6-OneBranch]-Prod-BuildImages" fi diff --git a/.pipelines/templates/stages/trident_rpms/build-source.yml b/.pipelines/templates/stages/trident_rpms/build-source.yml index 6803e2399..f791f2bcb 100644 --- a/.pipelines/templates/stages/trident_rpms/build-source.yml +++ b/.pipelines/templates/stages/trident_rpms/build-source.yml @@ -87,7 +87,7 @@ stages: osModifierBranch: ${{ parameters.osModifierBranch }} - template: release.yml parameters: - targetArchitecture: "AMD64" + targetArchitecture: "amd64" - ${{ if eq(parameters.publishToDevFeed, true) }}: - template: publish-dev.yml @@ -116,4 +116,4 @@ stages: osModifierBranch: ${{ parameters.osModifierBranch }} - template: release.yml parameters: - targetArchitecture: "ARM64" + targetArchitecture: "arm64" diff --git a/.pipelines/templates/stages/trident_rpms/trident-stage.yml b/.pipelines/templates/stages/trident_rpms/trident-stage.yml index f42f8e26e..4ff442a91 100644 --- a/.pipelines/templates/stages/trident_rpms/trident-stage.yml +++ b/.pipelines/templates/stages/trident_rpms/trident-stage.yml @@ -46,6 +46,7 @@ stages: codeCoverage: ${{ parameters.codeCoverage }} osModifierBranch: ${{ parameters.osModifierBranch }} dependsOnStage: ${{ parameters.dependsOnStage }} + targetArchitecture: ${{ parameters.targetArchitecture }} - ${{ else }}: - template: ../download_staged/download-staged.yml diff --git a/.pipelines/trident-cicd.yml b/.pipelines/trident-cicd.yml index e9c5cc361..de701c341 100644 --- a/.pipelines/trident-cicd.yml +++ b/.pipelines/trident-cicd.yml @@ -17,11 +17,6 @@ parameters: values: - dev - preview - - name: targetArchitecture - type: string - values: - - amd64 - - arm64 trigger: none From 9ddb3e7d9f3f8105334550b8d56aa6252fc21744 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 12 Aug 2025 15:53:06 +0000 Subject: [PATCH 41/69] arch --- .pipelines/templates/stages/trident_rpms/release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pipelines/templates/stages/trident_rpms/release.yml b/.pipelines/templates/stages/trident_rpms/release.yml index 4c0f94c30..2ded06eb2 100644 --- a/.pipelines/templates/stages/trident_rpms/release.yml +++ b/.pipelines/templates/stages/trident_rpms/release.yml @@ -1,10 +1,10 @@ parameters: - name: targetArchitecture type: string - default: "AMD64" + default: "amd64" values: - - ARM64 - - AMD64 + - arm64 + - amd64 steps: From c56624a9ba226763f9e772e31f2a80861e28cc06 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 12 Aug 2025 15:59:24 +0000 Subject: [PATCH 42/69] arch --- .pipelines/templates/e2e-template.yml | 2 +- .../stages/build_docker_image/trident-container.yml | 2 +- .pipelines/templates/stages/build_image/build-image.yml | 2 +- .pipelines/templates/stages/publishing/publish.yml | 2 +- .../templates/stages/testing_servicing/vm-testing.yml | 6 +++--- .pipelines/templates/stages/trident_rpms/build-source.yml | 2 +- .../templates/stages/trident_usb_iso/trident-usb-iso.yml | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.pipelines/templates/e2e-template.yml b/.pipelines/templates/e2e-template.yml index d6d1286cc..2014cc6c0 100644 --- a/.pipelines/templates/e2e-template.yml +++ b/.pipelines/templates/e2e-template.yml @@ -106,7 +106,7 @@ stages: clones: 1 dependsOnStage: ${{ parameters.baseImageArtifactStage }} - # Build Trident container image (depends on GetTridentBinaries_rpms) + # Build Trident container image (depends on GetTridentBinaries_rpms_amd64) - template: stages/build_docker_image/trident-container.yml # Build Trident installer ISO (host) diff --git a/.pipelines/templates/stages/build_docker_image/trident-container.yml b/.pipelines/templates/stages/build_docker_image/trident-container.yml index aea729aae..e39f8bef9 100644 --- a/.pipelines/templates/stages/build_docker_image/trident-container.yml +++ b/.pipelines/templates/stages/build_docker_image/trident-container.yml @@ -2,7 +2,7 @@ stages: - stage: BuildTridentContainerImage displayName: Build Docker Image dependsOn: - - GetTridentBinaries_rpms + - GetTridentBinaries_rpms_amd64 jobs: - job: BuildImage diff --git a/.pipelines/templates/stages/build_image/build-image.yml b/.pipelines/templates/stages/build_image/build-image.yml index 4b52298b5..c8c4daf4b 100644 --- a/.pipelines/templates/stages/build_image/build-image.yml +++ b/.pipelines/templates/stages/build_image/build-image.yml @@ -42,7 +42,7 @@ stages: displayName: Build ${{ parameters.imageName }} ${{ if and(eq(parameters.runtimeEnv, 'host'), parameters.dependsOnTrident) }}: dependsOn: - - GetTridentBinaries_rpms + - GetTridentBinaries_rpms_amd64_amd64 ${{ elseif ne(parameters.dependsOnStage, '') }}: dependsOn: - ${{ parameters.dependsOnStage }} diff --git a/.pipelines/templates/stages/publishing/publish.yml b/.pipelines/templates/stages/publishing/publish.yml index 1899d3303..9a5e15775 100644 --- a/.pipelines/templates/stages/publishing/publish.yml +++ b/.pipelines/templates/stages/publishing/publish.yml @@ -16,7 +16,7 @@ parameters: stages: - stage: Publishing dependsOn: - - GetTridentBinaries_rpms + - GetTridentBinaries_rpms_amd64 - ${{ if eq(parameters.stageType, 'ci') }}: - TridentTestImg_azl_installer - ${{ else }}: diff --git a/.pipelines/templates/stages/testing_servicing/vm-testing.yml b/.pipelines/templates/stages/testing_servicing/vm-testing.yml index 3b13a36cd..d42b57471 100644 --- a/.pipelines/templates/stages/testing_servicing/vm-testing.yml +++ b/.pipelines/templates/stages/testing_servicing/vm-testing.yml @@ -75,7 +75,7 @@ stages: displayName: Build Base and Update Images for QEMU dependsOn: - PrepareSSHKeys - - GetTridentBinaries_rpms + - GetTridentBinaries_rpms_amd64 jobs: - template: build-image.yml @@ -113,7 +113,7 @@ stages: displayName: Build Base and Update Images for Azure dependsOn: - PrepareSSHKeys - - GetTridentBinaries_rpms + - GetTridentBinaries_rpms_amd64 jobs: - template: build-image.yml @@ -148,7 +148,7 @@ stages: displayName: Build Base and Update Images for UKI dependsOn: - PrepareSSHKeys - - GetTridentBinaries_rpms + - GetTridentBinaries_rpms_amd64 jobs: - template: build-image.yml diff --git a/.pipelines/templates/stages/trident_rpms/build-source.yml b/.pipelines/templates/stages/trident_rpms/build-source.yml index f791f2bcb..8b766d76a 100644 --- a/.pipelines/templates/stages/trident_rpms/build-source.yml +++ b/.pipelines/templates/stages/trident_rpms/build-source.yml @@ -25,7 +25,7 @@ parameters: - arm64 stages: - - stage: GetTridentBinaries_rpms + - stage: GetTridentBinaries_rpms_${{ parameters.targetArchitecture }} displayName: Trident RPMs (Build) ${{ if ne(parameters.dependsOnStage, '') }}: dependsOn: ${{ parameters.dependsOnStage }} diff --git a/.pipelines/templates/stages/trident_usb_iso/trident-usb-iso.yml b/.pipelines/templates/stages/trident_usb_iso/trident-usb-iso.yml index 42922df5e..fc00bec9e 100644 --- a/.pipelines/templates/stages/trident_usb_iso/trident-usb-iso.yml +++ b/.pipelines/templates/stages/trident_usb_iso/trident-usb-iso.yml @@ -22,7 +22,7 @@ stages: - stage: TridentTestImg_azl_installer displayName: Build azl-installer dependsOn: - - GetTridentBinaries_rpms + - GetTridentBinaries_rpms_amd64 - BuildTridentContainerImage jobs: From b71eaaaf8b4798d797644eca603d9109ad4e51be Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 12 Aug 2025 16:00:25 +0000 Subject: [PATCH 43/69] arch --- .pipelines/templates/stages/build_image/build-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/stages/build_image/build-image.yml b/.pipelines/templates/stages/build_image/build-image.yml index c8c4daf4b..3b4cd4f54 100644 --- a/.pipelines/templates/stages/build_image/build-image.yml +++ b/.pipelines/templates/stages/build_image/build-image.yml @@ -42,7 +42,7 @@ stages: displayName: Build ${{ parameters.imageName }} ${{ if and(eq(parameters.runtimeEnv, 'host'), parameters.dependsOnTrident) }}: dependsOn: - - GetTridentBinaries_rpms_amd64_amd64 + - GetTridentBinaries_rpms_amd64 ${{ elseif ne(parameters.dependsOnStage, '') }}: dependsOn: - ${{ parameters.dependsOnStage }} From c94c539e8c7e4352815534061d8c8ff82d5f4581 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 12 Aug 2025 16:03:08 +0000 Subject: [PATCH 44/69] arch --- .pipelines/templates/stages/trident_rpms/build-source.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/stages/trident_rpms/build-source.yml b/.pipelines/templates/stages/trident_rpms/build-source.yml index 8b766d76a..e16aa050e 100644 --- a/.pipelines/templates/stages/trident_rpms/build-source.yml +++ b/.pipelines/templates/stages/trident_rpms/build-source.yml @@ -26,7 +26,7 @@ parameters: stages: - stage: GetTridentBinaries_rpms_${{ parameters.targetArchitecture }} - displayName: Trident RPMs (Build) + displayName: Trident ${{ parameters.targetArchitecture }} RPMs (Build) ${{ if ne(parameters.dependsOnStage, '') }}: dependsOn: ${{ parameters.dependsOnStage }} From 3f0fe1851b3677e926320fc73f477238f2afe21a Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 12 Aug 2025 16:19:49 +0000 Subject: [PATCH 45/69] scale arch --- .pipelines/templates/scale-template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pipelines/templates/scale-template.yml b/.pipelines/templates/scale-template.yml index 5732c89fc..794be2ae9 100644 --- a/.pipelines/templates/scale-template.yml +++ b/.pipelines/templates/scale-template.yml @@ -85,6 +85,7 @@ stages: codeCoverage: false forceTridentRebuild: ${{ or(eq(parameters.forceTridentRebuild, true), eq(parameters.baseimgBuildType, 'preview')) }} dependsOnStage: CreateBaseImageConfig + targetArchitecture: amd64 # Test Servicing on VMs - template: stages/testing_servicing/vm-testing.yml From 4a0a8e1e3a0e46948820ee6a0526da1311649a72 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 12 Aug 2025 16:26:38 +0000 Subject: [PATCH 46/69] fix staged --- .pipelines/templates/stages/trident_rpms/trident-stage.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pipelines/templates/stages/trident_rpms/trident-stage.yml b/.pipelines/templates/stages/trident_rpms/trident-stage.yml index 4ff442a91..a2ef300f5 100644 --- a/.pipelines/templates/stages/trident_rpms/trident-stage.yml +++ b/.pipelines/templates/stages/trident_rpms/trident-stage.yml @@ -37,6 +37,7 @@ parameters: - arm64 stages: +- ${{ if eq(parameters.targetArchitecture, 'amd64') }}: - ${{ if or(parameters.forceTridentRebuild, eq(parameters.stageType, 'pr'), eq(parameters.stageType, 'pr-e2e'), eq(parameters.stageType, 'ci'), eq(parameters.stageType, 'pr-e2e-azure')) }}: - template: build-source.yml parameters: From f74ff694d4e62fed183f3c100d3c56d51c83f46d Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 12 Aug 2025 16:28:04 +0000 Subject: [PATCH 47/69] stag --- .pipelines/templates/stages/trident_rpms/trident-stage.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.pipelines/templates/stages/trident_rpms/trident-stage.yml b/.pipelines/templates/stages/trident_rpms/trident-stage.yml index a2ef300f5..8c6d3e641 100644 --- a/.pipelines/templates/stages/trident_rpms/trident-stage.yml +++ b/.pipelines/templates/stages/trident_rpms/trident-stage.yml @@ -37,7 +37,6 @@ parameters: - arm64 stages: -- ${{ if eq(parameters.targetArchitecture, 'amd64') }}: - ${{ if or(parameters.forceTridentRebuild, eq(parameters.stageType, 'pr'), eq(parameters.stageType, 'pr-e2e'), eq(parameters.stageType, 'ci'), eq(parameters.stageType, 'pr-e2e-azure')) }}: - template: build-source.yml parameters: @@ -49,7 +48,7 @@ stages: dependsOnStage: ${{ parameters.dependsOnStage }} targetArchitecture: ${{ parameters.targetArchitecture }} - - ${{ else }}: + - ${{ elseif eq(parameters.targetArchitecture, 'amd64') }}: - template: ../download_staged/download-staged.yml parameters: artifactName: ${{ parameters.tridentArtifactName }} From d0af8a11c559e897a091724c0cc63ecc3ac31b6e Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 12 Aug 2025 16:42:14 +0000 Subject: [PATCH 48/69] arch --- .../stages/trident_rpms/build-source.yml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.pipelines/templates/stages/trident_rpms/build-source.yml b/.pipelines/templates/stages/trident_rpms/build-source.yml index e16aa050e..5d3ba6d23 100644 --- a/.pipelines/templates/stages/trident_rpms/build-source.yml +++ b/.pipelines/templates/stages/trident_rpms/build-source.yml @@ -32,8 +32,8 @@ stages: jobs: - ${{ if eq( parameters.targetArchitecture, 'amd64' ) }}: - - job: CheckTrident - displayName: Check + - job: CheckTrident_${{ parameters.targetArchitecture }} + displayName: Check ${{ parameters.targetArchitecture }} pool: type: linux @@ -45,8 +45,8 @@ stages: - template: ../common_tasks/avoid-pypi-usage.yml - template: check.yml - - job: TestTrident - displayName: Unit Test + - job: TestTrident_${{ parameters.targetArchitecture }} + displayName: Unit Test ${{ parameters.targetArchitecture }} pool: type: linux @@ -64,8 +64,8 @@ stages: codeCoverage: ${{ parameters.codeCoverage }} unitTestCoverageBaseline: 70 # Unit tests - - job: BuildTrident - displayName: Build Trident 3.0 RPMs + - job: BuildTrident_${{ parameters.targetArchitecture }} + displayName: Build Trident 3.0 ${{ parameters.targetArchitecture }} RPMs pool: type: linux @@ -87,13 +87,13 @@ stages: osModifierBranch: ${{ parameters.osModifierBranch }} - template: release.yml parameters: - targetArchitecture: "amd64" + targetArchitecture: ${{ parameters.targetArchitecture }} - ${{ if eq(parameters.publishToDevFeed, true) }}: - template: publish-dev.yml - ${{ elseif eq( parameters.targetArchitecture, 'arm64' ) }}: - - job: BuildTridentARM64 - displayName: Build Trident 3.0 RPMs for ARM64 + - job: BuildTrident_${{ parameters.targetArchitecture }} + displayName: Build Trident 3.0 ${{ parameters.targetArchitecture }} RPMs pool: type: linux name: mariner-hci-ARM64-1es-pool-westcentralus @@ -116,4 +116,4 @@ stages: osModifierBranch: ${{ parameters.osModifierBranch }} - template: release.yml parameters: - targetArchitecture: "arm64" + targetArchitecture: ${{ parameters.targetArchitecture }} From f6b8c0f31d2f4bdce3fbc1b19708a625dcbd43bd Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 12 Aug 2025 16:46:57 +0000 Subject: [PATCH 49/69] fix --- .pipelines/templates/stages/publishing/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/stages/publishing/publish.yml b/.pipelines/templates/stages/publishing/publish.yml index 9a5e15775..fafbf64fd 100644 --- a/.pipelines/templates/stages/publishing/publish.yml +++ b/.pipelines/templates/stages/publishing/publish.yml @@ -20,7 +20,7 @@ stages: - ${{ if eq(parameters.stageType, 'ci') }}: - TridentTestImg_azl_installer - ${{ else }}: - - GetTridentBinaries_usb_iso + - GetTridentBinaries_usb_iso_arm64 - ${{ if eq(parameters.stageType, 'pre') }}: - DeploymentTesting_host - DeploymentTesting_container From 952069c9d57e2b88b209a39bfffa0fa945cc2a4a Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 12 Aug 2025 16:47:46 +0000 Subject: [PATCH 50/69] fix --- .pipelines/templates/stages/download_staged/download-staged.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/stages/download_staged/download-staged.yml b/.pipelines/templates/stages/download_staged/download-staged.yml index 0a4ff4d5d..b9be62ec9 100644 --- a/.pipelines/templates/stages/download_staged/download-staged.yml +++ b/.pipelines/templates/stages/download_staged/download-staged.yml @@ -27,7 +27,7 @@ parameters: default: '' stages: - - stage: GetTridentBinaries_${{ replace(parameters.packageName, '-', '_') }} + - stage: GetTridentBinaries_${{ replace(parameters.packageName, '-', '_') }}_arm64 displayName: Download staged ${{ parameters.packageName }} for ${{ parameters.stageType }} ${{ if ne(parameters.dependsOnStage, '') }}: dependsOn: ${{ parameters.dependsOnStage }} From 3a581b100cabdd360aad047312636e605deaadba Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 12 Aug 2025 16:53:06 +0000 Subject: [PATCH 51/69] fix --- .../templates/stages/download_staged/download-staged.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/stages/download_staged/download-staged.yml b/.pipelines/templates/stages/download_staged/download-staged.yml index b9be62ec9..c37ffb01f 100644 --- a/.pipelines/templates/stages/download_staged/download-staged.yml +++ b/.pipelines/templates/stages/download_staged/download-staged.yml @@ -27,13 +27,13 @@ parameters: default: '' stages: - - stage: GetTridentBinaries_${{ replace(parameters.packageName, '-', '_') }}_arm64 + - stage: GetTridentBinaries_${{ replace(parameters.packageName, '-', '_') }}_amd64 displayName: Download staged ${{ parameters.packageName }} for ${{ parameters.stageType }} ${{ if ne(parameters.dependsOnStage, '') }}: dependsOn: ${{ parameters.dependsOnStage }} jobs: - - job: DownloadTrident + - job: DownloadTrident_arm64 displayName: Download Trident pool: type: linux From 79d53b70815e8c217cd40db8ce4b76daf7200a22 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 12 Aug 2025 17:03:07 +0000 Subject: [PATCH 52/69] imitate real scenario --- .pipelines/trident-cicd.yml | 40 +++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/.pipelines/trident-cicd.yml b/.pipelines/trident-cicd.yml index de701c341..f8831858c 100644 --- a/.pipelines/trident-cicd.yml +++ b/.pipelines/trident-cicd.yml @@ -17,6 +17,14 @@ parameters: values: - dev - preview + - name: targetArchitecture + displayName: "Pick architecture to test" + type: string + default: amd64 + values: + - amd64 + - arm64 + - all trigger: none @@ -54,18 +62,30 @@ extends: - template: templates/stages/base_image_config/create-base-image-config-artifact.yml parameters: baseimgBuildType: ${{ parameters.buildType }} - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} - baseImageArm64PipelineBuildId: ${{ parameters.baseImageArm64PipelineBuildId }} + # Only add amd64 build id if targetArchitecture includes amd64 + ${{ if eq(parameters.targetArchitecture, 'arm64') }}: + baseImagePipelineBuildId: will-be-ignored + ${{ else }}: + baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} + # Only add arm64 build id if targetArchitecture includes arm64 + ${{ if eq(parameters.targetArchitecture, 'amd64') }}: + baseImageArm64PipelineBuildId: will-be-ignored + ${{ else }}: + baseImageArm64PipelineBuildId: ${{ parameters.baseImageArm64PipelineBuildId }} - - template: templates/trident-platform-cicd-template.yml - parameters: - baseImageArtifactStage: CreateBaseImageConfig - targetArchitecture: amd64 + # Test amd64 if targetArchitecture is all or amd64 (i.e. !arm64) + - ${{ if ne(parameters.targetArchitecture, 'arm64') }}: + - template: templates/trident-platform-cicd-template.yml + parameters: + baseImageArtifactStage: CreateBaseImageConfig + targetArchitecture: amd64 - - template: templates/trident-platform-cicd-template.yml - parameters: - baseImageArtifactStage: CreateBaseImageConfig - targetArchitecture: arm64 + # Test amd64 if targetArchitecture is all or arm64 (i.e. !amd64) + - ${{ if ne(parameters.targetArchitecture, 'amd64') }}: + - template: templates/trident-platform-cicd-template.yml + parameters: + baseImageArtifactStage: CreateBaseImageConfig + targetArchitecture: arm64 # - stage: ProcessJunit_Stage # displayName: Process junit files From 67e229f29730ee246076eea874e3c0177f62e5fe Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 12 Aug 2025 17:07:13 +0000 Subject: [PATCH 53/69] arm --- .pipelines/templates/trident-platform-cicd-template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/trident-platform-cicd-template.yml b/.pipelines/templates/trident-platform-cicd-template.yml index ea288566d..a16f86b7c 100644 --- a/.pipelines/templates/trident-platform-cicd-template.yml +++ b/.pipelines/templates/trident-platform-cicd-template.yml @@ -27,8 +27,8 @@ stages: forceFunctionalTestImageRebuild: true baremetalTestsEnabled: ${{ parameters.baremetalTestsEnabled }} baseImageArtifactStage: ${{ parameters.baseImageArtifactStage }} - - - ${{ if eq( parameters.targetArchitecture, 'amd64') }}: + + - ${{ if eq( parameters.targetArchitecture, 'arm64') }}: - template: e2e-arm64-template.yml parameters: stageType: pre From 9125bfb4f351d70133e7ebc1b1336acf5a177125 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 12 Aug 2025 17:15:09 +0000 Subject: [PATCH 54/69] pr --- .pipelines/templates/pr-template.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.pipelines/templates/pr-template.yml b/.pipelines/templates/pr-template.yml index 19096bdda..91af81c57 100644 --- a/.pipelines/templates/pr-template.yml +++ b/.pipelines/templates/pr-template.yml @@ -9,5 +9,13 @@ stages: stageType: pr tridentArtifactName: trident-binaries dependsOnStage: CreateBaseImageConfig + targetArchitecture: amd64 + - template: stages/trident_rpms/trident-stage.yml + parameters: + stageType: pr + tridentArtifactName: trident-binaries + codeCoverage: false + dependsOnStage: CreateBaseImageConfig + targetArchitecture: arm64 - template: stages/validate_makefile/dev-build.yml - template: stages/building_tools/building-tools.yml From 22f1c4a77d65d8341beb60279bf9ef7d638df668 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 12 Aug 2025 17:28:29 +0000 Subject: [PATCH 55/69] remove unused --- .pipelines/templates/e2e-arm64-template.yml | 39 +-------------------- 1 file changed, 1 insertion(+), 38 deletions(-) diff --git a/.pipelines/templates/e2e-arm64-template.yml b/.pipelines/templates/e2e-arm64-template.yml index 07ab7200c..fb9b41cb7 100644 --- a/.pipelines/templates/e2e-arm64-template.yml +++ b/.pipelines/templates/e2e-arm64-template.yml @@ -1,4 +1,4 @@ -# e2e-ARM64-template will run ARM64 trident tests for each of our scenarios. +# e2e-arm64-template will run arm64 trident tests for each of our scenarios. # This includes unit tests, integration tests, and end-to-end tests. # # Prior to invoking this template, an artifact must be created that @@ -24,52 +24,15 @@ parameters: - rel - full-validation - - name: blockPublishing - displayName: "Block publishing to artifact feed" - type: boolean - default: false - - name: forceTridentRebuild displayName: "Force trident rebuild rather than downloading previously built." type: boolean default: false - - name: includeAzure - displayName: "Include Azure testing" - type: boolean - default: false - - - name: micBuildType - displayName: MIC Build Type - type: string - values: - - dev - - preview - - release - default: release - - - name: micVersion - displayName: MIC Version - type: string - default: "*.*.*" - - - name: numberOfUpdateIterations - displayName: Number of times each worker invokes A/B Update - type: number - default: 3 - - name: osModifierBranch type: string default: "submodule" - - name: baremetalTestsEnabled - type: boolean - default: true - - - name: forceFunctionalTestImageRebuild - type: boolean - default: false - - name: baseImageArtifactStage type: string From f47a609c669ce264185543a3bc4f7a24ba809f98 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 12 Aug 2025 19:06:25 +0000 Subject: [PATCH 56/69] remove unused --- .../templates/trident-platform-cicd-for-prism-template.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.pipelines/templates/trident-platform-cicd-for-prism-template.yml b/.pipelines/templates/trident-platform-cicd-for-prism-template.yml index e690293e5..9988e5b46 100644 --- a/.pipelines/templates/trident-platform-cicd-for-prism-template.yml +++ b/.pipelines/templates/trident-platform-cicd-for-prism-template.yml @@ -37,9 +37,6 @@ stages: - template: e2e-arm64-template.yml parameters: stageType: ci - blockPublishing: true forceTridentRebuild: true - micBuildType: ${{ parameters.micBuildType }} - micVersion: ${{ parameters.micVersion }} osModifierBranch: ${{ parameters.osModifierBranch }} baseImageArtifactStage: CreateBaseImageConfig From 9e4e10ae813bb0a09b93f31ec5c3c43e175aa278 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 12 Aug 2025 19:08:21 +0000 Subject: [PATCH 57/69] remove unused --- .pipelines/templates/pipeline-selector.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pipelines/templates/pipeline-selector.yml b/.pipelines/templates/pipeline-selector.yml index 3bd72bd83..8bd68f0b7 100644 --- a/.pipelines/templates/pipeline-selector.yml +++ b/.pipelines/templates/pipeline-selector.yml @@ -55,5 +55,4 @@ stages: - template: e2e-arm64-template.yml parameters: stageType: ${{ parameters.targetStage }} - blockPublishing: ${{ parameters.blockPublishing }} baseImageArtifactStage: CreateBaseImageConfig From 5f2c712f58fdb71eda7c04fdc85a2583d6a04a9a Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 12 Aug 2025 19:51:25 +0000 Subject: [PATCH 58/69] amd --- .pipelines/templates/stages/publishing/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/stages/publishing/publish.yml b/.pipelines/templates/stages/publishing/publish.yml index fafbf64fd..6d09e8df6 100644 --- a/.pipelines/templates/stages/publishing/publish.yml +++ b/.pipelines/templates/stages/publishing/publish.yml @@ -20,7 +20,7 @@ stages: - ${{ if eq(parameters.stageType, 'ci') }}: - TridentTestImg_azl_installer - ${{ else }}: - - GetTridentBinaries_usb_iso_arm64 + - GetTridentBinaries_usb_iso_amd64 - ${{ if eq(parameters.stageType, 'pre') }}: - DeploymentTesting_host - DeploymentTesting_container From a881282977f2703324cc5ea9ad9ce542b835758d Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 12 Aug 2025 20:25:42 +0000 Subject: [PATCH 59/69] defaults --- .pipelines/templates/pr-template.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.pipelines/templates/pr-template.yml b/.pipelines/templates/pr-template.yml index 91af81c57..8960ea350 100644 --- a/.pipelines/templates/pr-template.yml +++ b/.pipelines/templates/pr-template.yml @@ -1,9 +1,5 @@ stages: - template: stages/base_image_config/create-base-image-config-artifact.yml - parameters: - baseimgBuildType: release - baseImagePipelineBuildId: latestFromBranch - baseImageArm64PipelineBuildId: latestFromBranch - template: stages/trident_rpms/trident-stage.yml parameters: stageType: pr From 39a6899008f25d249879cbd7634ffa9f703f5162 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Tue, 12 Aug 2025 20:36:32 +0000 Subject: [PATCH 60/69] fix job --- .pipelines/templates/stages/download_staged/download-staged.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/stages/download_staged/download-staged.yml b/.pipelines/templates/stages/download_staged/download-staged.yml index c37ffb01f..76ad186fc 100644 --- a/.pipelines/templates/stages/download_staged/download-staged.yml +++ b/.pipelines/templates/stages/download_staged/download-staged.yml @@ -33,7 +33,7 @@ stages: dependsOn: ${{ parameters.dependsOnStage }} jobs: - - job: DownloadTrident_arm64 + - job: DownloadTrident_amd64 displayName: Download Trident pool: type: linux From b045941ae02c2d4fe8224f385c38ae78e0c8b6a2 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Wed, 13 Aug 2025 00:52:05 +0000 Subject: [PATCH 61/69] validate steps-based prior stage, fix build-image dependsOn for prerelease --- ...reate-base-image-config-artifact-steps.yml | 31 +++++++++++ .../create-base-image-config-artifact.yml | 16 +++--- .../stages/build_image/build-image.yml | 2 + .pipelines/trident-cicd.yml | 54 +++++++++++++------ 4 files changed, 78 insertions(+), 25 deletions(-) create mode 100644 .pipelines/templates/stages/base_image_config/create-base-image-config-artifact-steps.yml diff --git a/.pipelines/templates/stages/base_image_config/create-base-image-config-artifact-steps.yml b/.pipelines/templates/stages/base_image_config/create-base-image-config-artifact-steps.yml new file mode 100644 index 000000000..62b73ef2b --- /dev/null +++ b/.pipelines/templates/stages/base_image_config/create-base-image-config-artifact-steps.yml @@ -0,0 +1,31 @@ +parameters: + - name: baseimgBuildType + type: string + values: + - dev + - preview + - release + default: "release" + + - name: baseImagePipelineBuildId + type: string + default: "latestFromBranch" + + - name: baseImageArm64PipelineBuildId + type: string + default: "latestFromBranch" + + - name: outputDirectory + type: string + +steps: + - bash: | + mkdir -p ${{ parameters.outputDirectory }} + cat << EOF > ${{ parameters.outputDirectory }}/baseimage.json + { + "baseimgBuildType": "${{ parameters.baseimgBuildType }}", + "baseImagePipelineBuildId": "${{ parameters.baseImagePipelineBuildId }}", + "baseImageArm64PipelineBuildId": "${{ parameters.baseImageArm64PipelineBuildId }}" + } + EOF + displayName: "Create Base Image Config artifact" diff --git a/.pipelines/templates/stages/base_image_config/create-base-image-config-artifact.yml b/.pipelines/templates/stages/base_image_config/create-base-image-config-artifact.yml index e6a72499c..350669a98 100644 --- a/.pipelines/templates/stages/base_image_config/create-base-image-config-artifact.yml +++ b/.pipelines/templates/stages/base_image_config/create-base-image-config-artifact.yml @@ -35,13 +35,9 @@ stages: - name: ob_artifactBaseName value: "base-image-config" steps: - - bash: | - mkdir -p $(ob_outputDirectory) - cat << EOF > $(ob_outputDirectory)/baseimage.json - { - "baseimgBuildType": "${{ parameters.baseimgBuildType }}", - "baseImagePipelineBuildId": "${{ parameters.baseImagePipelineBuildId }}", - "baseImageArm64PipelineBuildId": "${{ parameters.baseImageArm64PipelineBuildId }}" - } - EOF - displayName: "Create Base Image Config artifact" + - template: create-base-image-config-artifact-steps.yml + parameters: + baseimgBuildType: ${{ parameters.baseimgBuildType }} + baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} + baseImageArm64PipelineBuildId: ${{ parameters.baseImageArm64PipelineBuildId }} + outputDirectory: ${{ parameters.outputDirectory }} diff --git a/.pipelines/templates/stages/build_image/build-image.yml b/.pipelines/templates/stages/build_image/build-image.yml index 3b4cd4f54..94cbd1f50 100644 --- a/.pipelines/templates/stages/build_image/build-image.yml +++ b/.pipelines/templates/stages/build_image/build-image.yml @@ -43,6 +43,8 @@ stages: ${{ if and(eq(parameters.runtimeEnv, 'host'), parameters.dependsOnTrident) }}: dependsOn: - GetTridentBinaries_rpms_amd64 + - ${{ if ne(parameters.dependsOnStage, '') }}: + - ${{ parameters.dependsOnStage }} ${{ elseif ne(parameters.dependsOnStage, '') }}: dependsOn: - ${{ parameters.dependsOnStage }} diff --git a/.pipelines/trident-cicd.yml b/.pipelines/trident-cicd.yml index f8831858c..63f0ce994 100644 --- a/.pipelines/trident-cicd.yml +++ b/.pipelines/trident-cicd.yml @@ -59,32 +59,56 @@ extends: template: templates/MockOB.yml parameters: stages: - - template: templates/stages/base_image_config/create-base-image-config-artifact.yml - parameters: - baseimgBuildType: ${{ parameters.buildType }} - # Only add amd64 build id if targetArchitecture includes amd64 - ${{ if eq(parameters.targetArchitecture, 'arm64') }}: - baseImagePipelineBuildId: will-be-ignored - ${{ else }}: - baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} - # Only add arm64 build id if targetArchitecture includes arm64 - ${{ if eq(parameters.targetArchitecture, 'amd64') }}: - baseImageArm64PipelineBuildId: will-be-ignored - ${{ else }}: - baseImageArm64PipelineBuildId: ${{ parameters.baseImageArm64PipelineBuildId }} + - stage: GetBaseImageBuildIds + displayName: "Create BaseImage config artifact" + jobs: + - job: GetBaseImageBuildIdsJob + displayName: "Create BaseImage config artifact" + pool: + type: linux + variables: + - name: ob_outputDirectory + value: "$(Build.SourcesDirectory)/base-image-config" + - name: ob_artifactBaseName + value: "base-image-config" + steps: + - bash: | + BASEIMG_BUILD_TYPE="${{ parameters.buildType }}" + # Only add amd64 build id if targetArchitecture includes amd64 + BASE_IMAGE_PIPELINE_BUILD_ID="will-be-ignored" + if [[ "${{ parameters.targetArchitecture }}" == 'arm64' ]]; then + BASE_IMAGE_PIPELINE_BUILD_ID="${{ parameters.baseImagePipelineBuildId }}" + fi + # Only add arm64 build id if targetArchitecture includes arm64 + BASE_IMAGE_ARM64_PIPELINE_BUILD_ID="will-be-ignored" + if [[ "${{ parameters.targetArchitecture }}" == 'amd64' ]]; then + BASE_IMAGE_ARM64_PIPELINE_BUILD_ID="${{ parameters.baseImageArm64PipelineBuildId }}" + fi + + echo "##vso[task.setvariable variable=BASEIMG_BUILD_TYPE]$BASEIMG_BUILD_TYPE" + echo "##vso[task.setvariable variable=BASE_IMAGE_PIPELINE_BUILD_ID]$BASE_IMAGE_PIPELINE_BUILD_ID" + echo "##vso[task.setvariable variable=BASE_IMAGE_ARM64_PIPELINE_BUILD_ID]$BASE_IMAGE_ARM64_PIPELINE_BUILD_ID" + displayName: Validate variables for our templates + + - template: templates/stages/base_image_config/create-base-image-config-artifact-steps.yml + parameters: + baseimgBuildType: $(BASEIMG_BUILD_TYPE) + baseImagePipelineBuildId: $(BASE_IMAGE_PIPELINE_BUILD_ID) + baseImageArm64PipelineBuildId: $(BASE_IMAGE_ARM64_PIPELINE_BUILD_ID) + outputDirectory: $(ob_outputDirectory) # Test amd64 if targetArchitecture is all or amd64 (i.e. !arm64) - ${{ if ne(parameters.targetArchitecture, 'arm64') }}: - template: templates/trident-platform-cicd-template.yml parameters: - baseImageArtifactStage: CreateBaseImageConfig + baseImageArtifactStage: GetBaseImageBuildIds targetArchitecture: amd64 # Test amd64 if targetArchitecture is all or arm64 (i.e. !amd64) - ${{ if ne(parameters.targetArchitecture, 'amd64') }}: - template: templates/trident-platform-cicd-template.yml parameters: - baseImageArtifactStage: CreateBaseImageConfig + baseImageArtifactStage: GetBaseImageBuildIds targetArchitecture: arm64 # - stage: ProcessJunit_Stage From aa406ab5fc34ccc180bd6455f5850533ace9dae0 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Wed, 13 Aug 2025 00:54:27 +0000 Subject: [PATCH 62/69] fix ob --- .../base_image_config/create-base-image-config-artifact.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/stages/base_image_config/create-base-image-config-artifact.yml b/.pipelines/templates/stages/base_image_config/create-base-image-config-artifact.yml index 350669a98..74ed01268 100644 --- a/.pipelines/templates/stages/base_image_config/create-base-image-config-artifact.yml +++ b/.pipelines/templates/stages/base_image_config/create-base-image-config-artifact.yml @@ -40,4 +40,4 @@ stages: baseimgBuildType: ${{ parameters.baseimgBuildType }} baseImagePipelineBuildId: ${{ parameters.baseImagePipelineBuildId }} baseImageArm64PipelineBuildId: ${{ parameters.baseImageArm64PipelineBuildId }} - outputDirectory: ${{ parameters.outputDirectory }} + outputDirectory: $(ob_outputDirectory) From 61e5810d754b724f5d047264272195219211860a Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Wed, 13 Aug 2025 01:04:20 +0000 Subject: [PATCH 63/69] fix --- .pipelines/trident-cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/trident-cicd.yml b/.pipelines/trident-cicd.yml index 63f0ce994..a7cac6a69 100644 --- a/.pipelines/trident-cicd.yml +++ b/.pipelines/trident-cicd.yml @@ -90,7 +90,7 @@ extends: echo "##vso[task.setvariable variable=BASE_IMAGE_ARM64_PIPELINE_BUILD_ID]$BASE_IMAGE_ARM64_PIPELINE_BUILD_ID" displayName: Validate variables for our templates - - template: templates/stages/base_image_config/create-base-image-config-artifact-steps.yml + - template: stages/base_image_config/create-base-image-config-artifact-steps.yml parameters: baseimgBuildType: $(BASEIMG_BUILD_TYPE) baseImagePipelineBuildId: $(BASE_IMAGE_PIPELINE_BUILD_ID) From 8911ff28c371805198363068e7a68c5546ede354 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Wed, 13 Aug 2025 01:05:24 +0000 Subject: [PATCH 64/69] runtime --- .../base_image_config/create-base-image-config-artifact.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.pipelines/templates/stages/base_image_config/create-base-image-config-artifact.yml b/.pipelines/templates/stages/base_image_config/create-base-image-config-artifact.yml index 74ed01268..63d5f7037 100644 --- a/.pipelines/templates/stages/base_image_config/create-base-image-config-artifact.yml +++ b/.pipelines/templates/stages/base_image_config/create-base-image-config-artifact.yml @@ -1,10 +1,6 @@ parameters: - name: baseimgBuildType type: string - values: - - dev - - preview - - release default: "release" - name: baseImagePipelineBuildId From c4e8ade63560119b2a6248f74b79b1e55214cfee Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Wed, 13 Aug 2025 01:07:40 +0000 Subject: [PATCH 65/69] runtime --- .../create-base-image-config-artifact-steps.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.pipelines/templates/stages/base_image_config/create-base-image-config-artifact-steps.yml b/.pipelines/templates/stages/base_image_config/create-base-image-config-artifact-steps.yml index 62b73ef2b..d1bd6c63d 100644 --- a/.pipelines/templates/stages/base_image_config/create-base-image-config-artifact-steps.yml +++ b/.pipelines/templates/stages/base_image_config/create-base-image-config-artifact-steps.yml @@ -1,10 +1,6 @@ parameters: - name: baseimgBuildType type: string - values: - - dev - - preview - - release default: "release" - name: baseImagePipelineBuildId From f5e472a8fc4b623186d46d141860809e1d8eb57b Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Wed, 13 Aug 2025 01:10:19 +0000 Subject: [PATCH 66/69] unused --- .pipelines/templates/trident-platform-cicd-template.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pipelines/templates/trident-platform-cicd-template.yml b/.pipelines/templates/trident-platform-cicd-template.yml index a16f86b7c..98907fa09 100644 --- a/.pipelines/templates/trident-platform-cicd-template.yml +++ b/.pipelines/templates/trident-platform-cicd-template.yml @@ -32,6 +32,5 @@ stages: - template: e2e-arm64-template.yml parameters: stageType: pre - blockPublishing: true forceTridentRebuild: true baseImageArtifactStage: ${{ parameters.baseImageArtifactStage }} From 24716f39df06de9f719de265dff7579d95def692 Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Wed, 13 Aug 2025 01:20:25 +0000 Subject: [PATCH 67/69] fix --- .pipelines/trident-cicd.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pipelines/trident-cicd.yml b/.pipelines/trident-cicd.yml index a7cac6a69..768cdefff 100644 --- a/.pipelines/trident-cicd.yml +++ b/.pipelines/trident-cicd.yml @@ -74,15 +74,15 @@ extends: steps: - bash: | BASEIMG_BUILD_TYPE="${{ parameters.buildType }}" - # Only add amd64 build id if targetArchitecture includes amd64 - BASE_IMAGE_PIPELINE_BUILD_ID="will-be-ignored" + # Only add amd64 build id if targetArchitecture includes amd64 + BASE_IMAGE_PIPELINE_BUILD_ID="${{ parameters.baseImagePipelineBuildId }}" if [[ "${{ parameters.targetArchitecture }}" == 'arm64' ]]; then - BASE_IMAGE_PIPELINE_BUILD_ID="${{ parameters.baseImagePipelineBuildId }}" + BASE_IMAGE_PIPELINE_BUILD_ID="will-be-ignored" fi # Only add arm64 build id if targetArchitecture includes arm64 - BASE_IMAGE_ARM64_PIPELINE_BUILD_ID="will-be-ignored" + BASE_IMAGE_ARM64_PIPELINE_BUILD_ID="${{ parameters.baseImageArm64PipelineBuildId }}" if [[ "${{ parameters.targetArchitecture }}" == 'amd64' ]]; then - BASE_IMAGE_ARM64_PIPELINE_BUILD_ID="${{ parameters.baseImageArm64PipelineBuildId }}" + BASE_IMAGE_ARM64_PIPELINE_BUILD_ID="will-be-ignored" fi echo "##vso[task.setvariable variable=BASEIMG_BUILD_TYPE]$BASEIMG_BUILD_TYPE" From 1a6d2b85457ff9606811ebafb0147e3e2385d01c Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Wed, 13 Aug 2025 15:16:22 +0000 Subject: [PATCH 68/69] process arm/amd --- .pipelines/trident-cicd.yml | 61 ++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/.pipelines/trident-cicd.yml b/.pipelines/trident-cicd.yml index 768cdefff..9020b6726 100644 --- a/.pipelines/trident-cicd.yml +++ b/.pipelines/trident-cicd.yml @@ -103,6 +103,27 @@ extends: parameters: baseImageArtifactStage: GetBaseImageBuildIds targetArchitecture: amd64 + - stage: ProcessJunit_Stage_amd64 + displayName: Process amd64 junit files + dependsOn: + - FunctionalTesting + - DeploymentTesting_host + - DeploymentTesting_container + - BaremetalDeploymentTesting_host + - BaremetalDeploymentTesting_container + - ServicingTesting + jobs: + - job: ProcessJunit_Job + displayName: Process junit files + pool: + type: linux + variables: + ob_outputDirectory: "$(Build.SourcesDirectory)/out" + steps: + - script: | + set -eux + echo "Collect and process junit files" + displayName: Process junit files # Test amd64 if targetArchitecture is all or arm64 (i.e. !amd64) - ${{ if ne(parameters.targetArchitecture, 'amd64') }}: @@ -110,27 +131,19 @@ extends: parameters: baseImageArtifactStage: GetBaseImageBuildIds targetArchitecture: arm64 - - # - stage: ProcessJunit_Stage - # displayName: Process junit files - # dependsOn: - # - FunctionalTesting - # - DeploymentTesting_host - # - DeploymentTesting_container - # - BaremetalDeploymentTesting_host - # - BaremetalDeploymentTesting_container - # - ServicingTesting - - # jobs: - # - job: ProcessJunit_Job - # displayName: Process junit files - # pool: - # type: linux - # variables: - # ob_outputDirectory: "$(Build.SourcesDirectory)/out" - # steps: - # - script: | - # set -eux - # echo "Collect and process junit files" - # displayName: Process junit files - + - stage: ProcessJunit_Stage_arm64 + displayName: Process arm64 junit files + dependsOn: + - GetTridentBinaries_rpms_arm64 + jobs: + - job: ProcessJunit_Job + displayName: Process junit files + pool: + type: linux + variables: + ob_outputDirectory: "$(Build.SourcesDirectory)/out" + steps: + - script: | + set -eux + echo "No junit tests to collect, if pipeline succeeds to this point, all good" + displayName: Process junit files From a4f51cc3d32ffbfd0016375a2afeb41766d76a7f Mon Sep 17 00:00:00 2001 From: Brian Fjeldstad Date: Wed, 13 Aug 2025 15:18:07 +0000 Subject: [PATCH 69/69] fix dependsOn --- .pipelines/trident-cicd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pipelines/trident-cicd.yml b/.pipelines/trident-cicd.yml index 9020b6726..faabbbe95 100644 --- a/.pipelines/trident-cicd.yml +++ b/.pipelines/trident-cicd.yml @@ -109,8 +109,8 @@ extends: - FunctionalTesting - DeploymentTesting_host - DeploymentTesting_container - - BaremetalDeploymentTesting_host - - BaremetalDeploymentTesting_container + # - BaremetalDeploymentTesting_host + # - BaremetalDeploymentTesting_container - ServicingTesting jobs: - job: ProcessJunit_Job