From d8302384760a60a5722e5f77fedbd7f1a4d0306c Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Fri, 2 Oct 2020 13:35:33 -0500 Subject: [PATCH] Use one stage per job in CI (cherry picked from commit f08ac8a5aaffd3936f5b7c64e925118d9190dfc3) # Conflicts: # .vsts.pipelines/builds/matrix.yml # .vsts.pipelines/jobs/ci-linux_bootstrap.yml # .vsts.pipelines/steps/run-bootstrap.yml --- .vsts.pipelines/builds/ci-internal.yml | 2 +- .vsts.pipelines/builds/ci.yml | 2 +- .vsts.pipelines/builds/matrix.yml | 74 ++++++++++------- .vsts.pipelines/jobs/ci-linux.yml | 9 +- .vsts.pipelines/stages/prepare-artifacts.yml | 86 ++++++++++++++++++++ .vsts.pipelines/stages/stage-jobs-matrix.yml | 18 ++++ 6 files changed, 151 insertions(+), 40 deletions(-) create mode 100644 .vsts.pipelines/stages/prepare-artifacts.yml create mode 100644 .vsts.pipelines/stages/stage-jobs-matrix.yml diff --git a/.vsts.pipelines/builds/ci-internal.yml b/.vsts.pipelines/builds/ci-internal.yml index fea2efe038..af2f90b4b0 100644 --- a/.vsts.pipelines/builds/ci-internal.yml +++ b/.vsts.pipelines/builds/ci-internal.yml @@ -1,4 +1,4 @@ -jobs: +stages: - template: matrix.yml parameters: windowsPoolName: NetCoreInternal-Int-Pool diff --git a/.vsts.pipelines/builds/ci.yml b/.vsts.pipelines/builds/ci.yml index d66a9d94eb..416e46b287 100644 --- a/.vsts.pipelines/builds/ci.yml +++ b/.vsts.pipelines/builds/ci.yml @@ -1,2 +1,2 @@ -jobs: +stages: - template: matrix.yml diff --git a/.vsts.pipelines/builds/matrix.yml b/.vsts.pipelines/builds/matrix.yml index 5e5a5c2709..3fca5193b1 100644 --- a/.vsts.pipelines/builds/matrix.yml +++ b/.vsts.pipelines/builds/matrix.yml @@ -9,58 +9,70 @@ parameters: windowsPoolName: NetCorePublic-Int-Pool windowsQueueName: buildpool.windows.10.amd64.vs2017.open -jobs: -- template: ../jobs/ci-linux.yml +stages: +- template: ../stages/stage-jobs-matrix.yml parameters: - job: centos71 - imageName: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-359e48e-20200313130914 - reportPrebuiltLeaks: true + jobTemplateName: ../jobs/ci-linux.yml + name: centos71 + jobParameters: + imageName: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-359e48e-20200313130914 + reportPrebuiltLeaks: true matrix: Production: {} Online: { type: Online } Offline: { type: Offline } Offline Portable: { type: Offline Portable } -- template: ../jobs/ci-linux.yml +- template: ../stages/stage-jobs-matrix.yml parameters: - job: debian9 - imageName: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-stretch-bfcd90a-20200121150012 + jobTemplateName: ../jobs/ci-linux.yml + name: debian9 + jobParameters: + imageName: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-stretch-bfcd90a-20200121150012 matrix: Production: {} Online: { type: Online } -- template: ../jobs/ci-linux.yml +- template: ../stages/stage-jobs-matrix.yml parameters: - job: fedora30 - imageName: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-30-bfcd90a-20200324132732 + jobTemplateName: ../jobs/ci-linux.yml + name: fedora30 + jobParameters: + imageName: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-30-bfcd90a-20200324132732 matrix: Production: {} Online: { type: Online } Offline: { type: Offline } Offline Portable: { type: Offline Portable } -- template: ../jobs/ci-linux.yml +- template: ../stages/stage-jobs-matrix.yml parameters: - job: ubuntu1804 - imageName: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-85814b7-20200324125103 + jobTemplateName: ../jobs/ci-linux.yml + name: ubuntu1804 + jobParameters: + imageName: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-85814b7-20200324125103 -- template: ../jobs/ci-local.yml +- template: ../stages/stage-jobs-matrix.yml parameters: - job: osx - pool: - name: Hosted macOS - scriptPrefix: ./ - scriptSuffix: .sh - setupMac: true + jobTemplateName: ../jobs/ci-local.yml + name: osx + jobParameters: + pool: + name: Hosted macOS + scriptPrefix: ./ + scriptSuffix: .sh + setupMac: true -- template: ../jobs/ci-local.yml +- template: ../stages/stage-jobs-matrix.yml parameters: - job: windows - pool: - name: ${{ parameters.windowsPoolName }} - queue: ${{ parameters.windowsQueueName }} - scriptPrefix: '' - scriptSuffix: .cmd - setupWindows: true - skipSmokeTest: true - failOnPrebuiltBaselineError: false + jobTemplateName: ../jobs/ci-local.yml + name: windows + jobParameters: + pool: + name: ${{ parameters.windowsPoolName }} + queue: ${{ parameters.windowsQueueName }} + scriptPrefix: '' + scriptSuffix: .cmd + setupWindows: true + skipSmokeTest: true + failOnPrebuiltBaselineError: false diff --git a/.vsts.pipelines/jobs/ci-linux.yml b/.vsts.pipelines/jobs/ci-linux.yml index 8f1dff2326..f0b2712fc4 100644 --- a/.vsts.pipelines/jobs/ci-linux.yml +++ b/.vsts.pipelines/jobs/ci-linux.yml @@ -1,7 +1,5 @@ parameters: job: null - matrix: - Production: {} pool: name: Hosted Ubuntu 1604 imageName: null @@ -9,13 +7,10 @@ parameters: jobs: - job: ${{ parameters.job }} - strategy: - matrix: ${{ parameters.matrix }} pool: ${{ parameters.pool }} timeoutInMinutes: 270 variables: - # Prefix to distinguish artifacts from different legs. - artifactName: ${{ format('{0} $(type)', parameters.job) }} + artifactName: ${{ parameters.job }} # Use ":z" to set selinux flag for sharing in build-owned root dir. https://docs.docker.com/storage/bind-mounts/#configure-the-selinux-label docker.agentSrc.map: -v $(Build.SourcesDirectory):/agentSrc:z docker.agentSrc.work: -w /agentSrc @@ -34,7 +29,7 @@ jobs: stagingDirectory: $(rootDirectory)/sb/staging tarballName: tarball_$(Build.BuildId) # Default type, can be overridden by matrix legs. - type: Production + type: ${{ coalesce(parameters.type, 'Production') }} steps: - template: ../steps/cleanup-unneeded-files.yml diff --git a/.vsts.pipelines/stages/prepare-artifacts.yml b/.vsts.pipelines/stages/prepare-artifacts.yml new file mode 100644 index 0000000000..247ff85758 --- /dev/null +++ b/.vsts.pipelines/stages/prepare-artifacts.yml @@ -0,0 +1,86 @@ +parameters: + gatherJobs: [] + gatherPortableJob: '' + + # downloadBuildConfig by default uses the current build, but can be changed to point at an + # existing/previous build by passing properties like these, instead: + # + # buildType: specific + # buildVersionToDownload: specific + # project: 'public' + # definition: 'source-build-CI' + # buildId: 741400 + # + # By defining this in one place, it's easier to set this up if necessary. This may be useful + # because the build takes a long time relative to prepare-artifacts, and we may want to tweak + # prepare-artifacts in case it fails due to a bug in the future. ...It's also nice to have the + # properties listed in this comment even if we don't use them here, because it's hard to figure + # them out from the docs alone--it'll be a nice reference. + downloadBuildConfig: + buildType: current + +stages: +- stage: PrepareArtifacts + dependsOn: + - ${{ each gather in parameters.gatherJobs }}: + - ${{ gather.job }} + jobs: + - job: PrepareArtifacts + pool: + vmImage: 'ubuntu-18.04' + timeoutInMinutes: 300 + workspace: + clean: all + variables: + artifactStageDir: '$(Build.SourcesDirectory)/artifacts/stage' + nonportableSourceBuiltStageDir: '$(artifactStageDir)/nonportableSourceBuilt' + portableSourceBuiltStageDir: '$(artifactStageDir)/portableSourceBuilt' + allSourceBuiltStageDir: '$(artifactStageDir)/allSourceBuilt' + outputTarballFile: '$(artifactStageDir)/Private.SourceBuilt.Artifacts.$(Build.BuildId).tar.gz' + steps: + - ${{ each gather in parameters.gatherJobs }}: + - task: DownloadBuildArtifacts@0 + displayName: 'Download ${{ gather.job }} nonportable' + inputs: + ${{ insert }}: ${{ parameters.downloadBuildConfig }} + downloadType: single + artifactName: ${{ coalesce(gather.artifactName, format('Tarball {0}', gather.job)) }} + downloadPath: $(nonportableSourceBuiltStageDir) + allowPartiallySucceededBuilds: true + + - task: DownloadBuildArtifacts@0 + displayName: 'Download ${{ parameters.gatherPortableJob }} portable' + inputs: + ${{ insert }}: ${{ parameters.downloadBuildConfig }} + downloadType: single + artifactName: 'Tarball ${{ parameters.gatherPortableJob }}' + downloadPath: $(portableSourceBuiltStageDir) + allowPartiallySucceededBuilds: true + + - script: | + find "$(artifactStageDir)" -type f -exec du -h {} \; + displayName: Show downloaded artifacts + + - script: | + set -xeuo pipefail + mkdir -p "$(allSourceBuiltStageDir)" + + # Extract all source-built assets into a single place. Overlap and overwrites are expected. + # What matters is the portable ones are copied last and ultimately win. + find \ + "$(nonportableSourceBuiltStageDir)" \ + "$(portableSourceBuiltStageDir)" \ + -iname 'Private.SourceBuilt.Artifacts.*.tar.gz' \ + -exec tar -xf {} -C "$(allSourceBuiltStageDir)" \; + + # Intentionally don't create a top-level directory. Matches earlier versions of this artifact. + cd "$(allSourceBuiltStageDir)" + tar --numeric-owner -czf "$(outputTarballFile)" * + displayName: Create source-built artifacts tar.gz + + - publish: '$(outputTarballFile)' + artifact: Private.SourceBuilt.Artifacts + + - script: | + tar -tf "$(outputTarballFile)" | sort + displayName: Show tarball contents diff --git a/.vsts.pipelines/stages/stage-jobs-matrix.yml b/.vsts.pipelines/stages/stage-jobs-matrix.yml new file mode 100644 index 0000000000..1301b7231e --- /dev/null +++ b/.vsts.pipelines/stages/stage-jobs-matrix.yml @@ -0,0 +1,18 @@ +parameters: + name: '' + dependsOn: [] + matrix: + Production: {} + jobParameters: {} + +stages: +- ${{ each matrixPair in parameters.matrix }}: + - stage: ${{ parameters.name }}_${{ replace(coalesce(matrixPair.value.type, 'Production'), ' ', '_') }} + dependsOn: ${{ parameters.dependsOn }} + jobs: + - template: ${{ parameters.jobTemplateName }} + parameters: + ${{ insert }}: ${{ parameters.jobParameters }} + job: ${{ parameters.name }}_${{ replace(coalesce(matrixPair.value.type, 'Production'), ' ', '_') }} + buildTypeName: ${{ matrixPair.key }} + type: ${{ matrixPair.value.type }} \ No newline at end of file