From 572115c54b1e5f38deb4a94f2fff0e073d02097f Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Thu, 20 Oct 2022 11:53:17 -0700 Subject: [PATCH 01/16] Add source-build-pre-release pipeline yaml --- eng/source-build-pre-release.yml | 146 +++++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 eng/source-build-pre-release.yml diff --git a/eng/source-build-pre-release.yml b/eng/source-build-pre-release.yml new file mode 100644 index 0000000000..983ada82ac --- /dev/null +++ b/eng/source-build-pre-release.yml @@ -0,0 +1,146 @@ +trigger: none +pr: none + +pool: + name: NetCore1ESPool-Svc-Internal + demands: ImageOverride -equals 1es-ubuntu-2004 + +resources: + pipelines: + - pipeline: dotnet-staging-pipeline-resource + source: Stage-DotNet + +variables: +- group: DotNet-MSRC-Storage +- name: azDoOrg + value: 'https://dev.azure.com/dnceng/' +- name: azDoProject + value: 'internal' +- name: installerOfficialCiPipelineId + value: '286' +- name: installerTarabllBuildCiPipelineId + value: '1011' +- name: storageAccountName + value: 'dotnetclimsrc' +- name: blobContainerName + value: 'source-build' +- name: blobContainerUploadBaseFilePath + value: 'release' + +stages: + +- stage: GetBuildInfoStage + displayName: Get Build Info + jobs: + - job: GetBuildInfoJob + displayName: Get Build Info + steps: + - download: dotnet-staging-pipeline-resource + artifact: manifests + patterns: 'manifest.json' + displayName: Download Release Manifest + - download: dotnet-staging-pipeline-resource + artifact: drop + patterns: 'config.json' + displayName: Download Release Config + - script: | + set -euo pipefail + + manifest_path=$(PIPELINE.WORKSPACE)/dotnet-staging-pipeline-resource/manifests/manifest.json + config_path=$(PIPELINE.WORKSPACE)/dotnet-staging-pipeline-resource/drop/config.json + + runtime_version="$(jq -r '.Runtime' $config_path)" + release_channel="$(jq -r '.Channel' $config_path)" + + # Source-build only supports the lowest avaiable feature band. + # Sort the SDK releases by number and pick the lowest value. + sdk_version="$(jq -r '.Sdks | sort_by(. | split(".") | map(tonumber)) | .[0]' $config_path)" + + # replace the last two characters in sdk_version with xx + branch_version=$(echo $sdk_version | sed 's/..$/xx/') + branch_name="refs/heads/internal/release/${branch_version}" + + commit=$(jq -r --arg BRANCH "${branch_name}" '.builds[] | select(.repo == "https://dev.azure.com/dnceng/internal/_git/dotnet-installer") | select(.branch == $BRANCH) | .commit' $manifest_path) + + if [[ ! $commit ]]; then + echo "Installer commit not set. Exiting..." + exit 1 + fi + + echo "Release channel: ${release_channel}" + echo "Runtime version: ${runtime_version}" + echo "SDK version: ${sdk_version}" + echo "Installer branch: ${branch_name}" + echo "Installer commit: ${commit}" + + echo "##vso[task.setvariable variable=SdkVersion;isOutput=true]${sdk_version}" + echo "##vso[task.setvariable variable=RuntimeVersion;isOutput=true]${runtime_version}" + echo "##vso[task.setvariable variable=ReleaseChannel;isOutput=true]${release_channel}" + echo "##vso[task.setvariable variable=InstallerCommit;isOutput=true]${commit}" + name: ReadReleaseInfoStep + displayName: Read Release Info + +- stage: UploadSourceTarballStage + displayName: Upload Source Tarball + dependsOn: GetBuildInfoStage + jobs: + - job: UploadSourceTarballJob + displayName: Upload Source Tarball + variables: + - name: InstallerCommit + value: $[ stageDependencies.GetBuildInfoStage.GetBuildInfoJob.outputs['ReadReleaseInfoStep.InstallerCommit'] ] + - name: SdkVersion + value: $[ stageDependencies.GetBuildInfoStage.GetBuildInfoJob.outputs['ReadReleaseInfoStep.SdkVersion'] ] + - name: RuntimeVersion + value: $[ stageDependencies.GetBuildInfoStage.GetBuildInfoJob.outputs['ReadReleaseInfoStep.RuntimeVersion'] ] + - name: ReleaseChannel + value: $[ stageDependencies.GetBuildInfoStage.GetBuildInfoJob.outputs['ReadReleaseInfoStep.ReleaseChannel'] ] + steps: + - script: | + set -euxo pipefail + echo $(ReleaseChannel) + echo $(RuntimeVersion) + echo $(SdkVersion) + echo $(InstallerCommit) + name: TestVariablesStep + displayName: Test Variables + - script: | + set -euo pipefail + az config set extension.use_dynamic_install=yes_without_prompt + + installer_official_runs=$(az pipelines runs list --organization $(azDoOrg) --project $(azDoProject) --pipeline-ids $(installerOfficialCiPipelineId)) + installer_official_run_id=$(echo $installer_official_runs | jq -r '.[] | select(.sourceVersion == "$(InstallerCommit)") | .id') + echo "dotnet-installer-official-ci run ID: ${installer_official_run_id}" + + installer_tarball_build_runs=$(az pipelines runs list --organization $(azDoOrg) --project $(azDoProject) --pipeline-ids $(installerTarabllBuildCiPipelineId)) + installer_tarball_build_run_id=$(echo $installer_tarball_build_runs | jq -r '.[] | select(.sourceVersion == "$(InstallerCommit)") | .id') + echo "dotnet-installer-source-build-tarball-build run ID: ${installer_tarball_build_run_id}" + + echo "##vso[task.setvariable variable=InstallerOfficialRunId]${installer_official_run_id}" + echo "##vso[task.setvariable variable=InstallerTarballBuildRunId]${installer_tarball_build_run_id}" + name: GetAssociatedPipelineRunsStep + displayName: Get Associated Pipeline Run IDs + env: + AZURE_DEVOPS_EXT_PAT: $(System.AccessToken) + - task: DownloadPipelineArtifact@2 + name: DownloadSourceTarballStep + displayName: Download Source Tarball + inputs: + source: 'specific' + artifact: 'BlobArtifacts' + patterns: 'BlobArtifacts/dotnet-sdk-source-*.tar.gz' + project: 'internal' + pipeline: $(installerOfficialCiPipelineId) + runVersion: 'specific' + runId: $(InstallerOfficialRunId) + - script: | + set -euxo pipefail + az config set extension.use_dynamic_install=yes_without_prompt + + source_tarball_upload_path="$(blobContainerUploadBaseFilePath)/$(ReleaseChannel)/$(RuntimeVersion)-$(SdkVersion)" + source_tarball_filename="dotnet-sdk-source-$(SdkVersion).tar.gz" + az storage blob upload --account-name "$(storageAccountName)" --container-name "$(blobContainerName)" --file "$(PIPELINE.WORKSPACE)/${source_tarball_filename}" --name "${source_tarball_upload_path}/${source_tarball_filename}" + name: UploadSourceTarballStep + displayName: Upload Source Tarball + env: + AZURE_STORAGE_KEY: $(dotnetclimsrc-access-key) From 859b3b7fda975a350698943da22e219451bfcc24 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Thu, 20 Oct 2022 14:07:34 -0700 Subject: [PATCH 02/16] Update doc to reference new pipeline --- .../ISSUE_TEMPLATE/releases/release_checklist.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/releases/release_checklist.md b/.github/ISSUE_TEMPLATE/releases/release_checklist.md index a09f054d5b..fb6edfb359 100644 --- a/.github/ISSUE_TEMPLATE/releases/release_checklist.md +++ b/.github/ISSUE_TEMPLATE/releases/release_checklist.md @@ -15,13 +15,14 @@ - This is useful to make sure context is available to review later. It may end up blank if the release goes very smoothly. - There are other notes on servicing in this OneNote. It may be useful to review if something goes wrong to see if it's been fixed before. - File issues appropriately as you encounter problems, and link to them from the notes. Provide info in the issue rather than in the notes. -1. - [ ] Retrieve dotnet/installer commit sha from internal communications for the release. +1. - [ ] Retrieve the final run of the [Stage-DotNet](https://dev.azure.com/dnceng/internal/_build?definitionId=792&_a=summary) pipeline from internal release communications. +1. - [ ] Run the [source-build-pre-release](https://dev.azure.com/dnceng/internal/_build?definitionId=1188) pipeline. When staging the pipeline run, click "Resources" and provide select the final run of Stage-DotNet mentioned above. + - [ ] When the pipeline finishes, look in the logs for the dotnet/installer commit sha that represents the final release. + - This pipeline automatically uploads the dotnet source tarball to dotnetclimsrc 1. - [ ] Ensure the official installer and tarball builds have completed for the release's commit sha. - - [ ] [Installer](https://dev.azure.com/dnceng/internal/_build?definitionId=286) (internal link) - - [ ] Retrieve the source tarball artifact - `BlobArtifacts/dotnet-sdk-source-6.0.xxx.tar.gz` - [ ] [Tarball](https://dev.azure.com/dnceng/internal/_build?definitionId=1011) (internal link) - [ ] Ensure the PoisonTests and SdkContentTests are passing. Warnings indicate a baseline diff and should be inspected carefully. -1. - [ ] [Internal] Gather smoke-test prereqs +1. - [ ] [Internal] Gather smoke-test prereqs ([automation tracking issue](https://github.com/dotnet/source-build/issues/3068)) - [ ] Retrieve smoke-test prereqs artifact for each architecture (e.g. x64 and arm64) from [tarball build](https://dev.azure.com/dnceng/internal/_build?definitionId=1011) (internal link) - [ ] x64 - `Build Tarball CentOS7-Offline_Artifacts/dotnet-smoke-test-prereqs.6.0.xxx.tar.gz` - [ ] arm64 - `Build Tarball Debian9-Offline_Artifacts/dotnet-smoke-test-prereqs.6.0.xxx.tar.gz` @@ -32,13 +33,13 @@ - [ ] Extact arm64 tarball - [ ] Copy four `*linux-arm64*` packages to x64 packages - [ ] Create new `dotnet-smoke-test-prereqs.6.0.xxx.tar.gz` tarball -1. - [ ] [Internal] Upload source and smoke-test-prereqs tarball to dotnetclimsrc storage account. +1. - [ ] [Internal] Upload smoke-test-prereqs tarball to dotnetclimsrc storage account. 1. - [ ] Notify partners of release. Include info about how certain we are that this will be the final Microsoft build. - [Internal] Send the dotnetclimsrc tarball links to partners. - Never overwrite a tarball. At least change the blob storage virtual dir to represent a new build. This can help avoid timing issues and make it more obvious if stale links were accidentally re-sent rather than new ones. - [Non-Internal] Send the dotnet/installer commit sha along w/link to publicly built source tarball. Link to the public instructions for building source-build. 1. - [ ] SYNC POINT: Wait for Microsoft build release. -1. - [ ] Upload the source-build artifacts to dotnetcli/source-built-artifacts blob storage. +1. - [ ] Upload the source-build artifacts to dotnetcli/source-built-artifacts blob storage. ([automation tracking issue](https://github.com/dotnet/source-build/issues/3080)) - [ ] Retrieve the source-build artifacts from [Tarball](https://dev.azure.com/dnceng/internal/_build?definitionId=1011) (internal link) - `Build Tarball CentOS7-Offline_Artifacts/Private.SourceBuilt.Artifacts.6.0.xxx.tar.gz` 1. - [ ] Run the [source-build-release pipeline](https://dev.azure.com/dnceng/internal/_build?definitionId=1124) (internal link). - Set the `SDK Version` parameter. From 90c67b91750d68b89252b3cb51cf4676e4e05e33 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Thu, 20 Oct 2022 16:46:20 -0700 Subject: [PATCH 03/16] Remove extra testing step. --- eng/source-build-pre-release.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/eng/source-build-pre-release.yml b/eng/source-build-pre-release.yml index 983ada82ac..c9f5914453 100644 --- a/eng/source-build-pre-release.yml +++ b/eng/source-build-pre-release.yml @@ -96,14 +96,6 @@ stages: - name: ReleaseChannel value: $[ stageDependencies.GetBuildInfoStage.GetBuildInfoJob.outputs['ReadReleaseInfoStep.ReleaseChannel'] ] steps: - - script: | - set -euxo pipefail - echo $(ReleaseChannel) - echo $(RuntimeVersion) - echo $(SdkVersion) - echo $(InstallerCommit) - name: TestVariablesStep - displayName: Test Variables - script: | set -euo pipefail az config set extension.use_dynamic_install=yes_without_prompt From 04a46e6fade0c11c3a2b78981cc6915e4354c805 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Thu, 20 Oct 2022 16:58:55 -0700 Subject: [PATCH 04/16] Use SdkVersion for download step --- eng/source-build-pre-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/source-build-pre-release.yml b/eng/source-build-pre-release.yml index c9f5914453..260ac95d94 100644 --- a/eng/source-build-pre-release.yml +++ b/eng/source-build-pre-release.yml @@ -120,7 +120,7 @@ stages: inputs: source: 'specific' artifact: 'BlobArtifacts' - patterns: 'BlobArtifacts/dotnet-sdk-source-*.tar.gz' + patterns: 'BlobArtifacts/dotnet-sdk-source-$(SdkVersion).tar.gz' project: 'internal' pipeline: $(installerOfficialCiPipelineId) runVersion: 'specific' From 6e4f5edb0d09183be9358ef35a23ef1189b93356 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Thu, 20 Oct 2022 17:02:35 -0700 Subject: [PATCH 05/16] Fix typo in release checklist Co-authored-by: Michael Simons --- .github/ISSUE_TEMPLATE/releases/release_checklist.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/releases/release_checklist.md b/.github/ISSUE_TEMPLATE/releases/release_checklist.md index fb6edfb359..631640d68e 100644 --- a/.github/ISSUE_TEMPLATE/releases/release_checklist.md +++ b/.github/ISSUE_TEMPLATE/releases/release_checklist.md @@ -16,7 +16,7 @@ - There are other notes on servicing in this OneNote. It may be useful to review if something goes wrong to see if it's been fixed before. - File issues appropriately as you encounter problems, and link to them from the notes. Provide info in the issue rather than in the notes. 1. - [ ] Retrieve the final run of the [Stage-DotNet](https://dev.azure.com/dnceng/internal/_build?definitionId=792&_a=summary) pipeline from internal release communications. -1. - [ ] Run the [source-build-pre-release](https://dev.azure.com/dnceng/internal/_build?definitionId=1188) pipeline. When staging the pipeline run, click "Resources" and provide select the final run of Stage-DotNet mentioned above. +1. - [ ] Run the [source-build-pre-release](https://dev.azure.com/dnceng/internal/_build?definitionId=1188) pipeline. When staging the pipeline run, click "Resources" and select the final run of Stage-DotNet mentioned above. - [ ] When the pipeline finishes, look in the logs for the dotnet/installer commit sha that represents the final release. - This pipeline automatically uploads the dotnet source tarball to dotnetclimsrc 1. - [ ] Ensure the official installer and tarball builds have completed for the release's commit sha. From 652e7cd5f0615eab79006693abbcbca5798f02f7 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Fri, 21 Oct 2022 16:43:54 -0700 Subject: [PATCH 06/16] Rewrites based on code review --- eng/source-build-pre-release.yml | 166 ++++++------------ eng/templates/get-build-info.yml | 118 +++++++++++++ .../installer-pipeline-variables.yml | 10 ++ 3 files changed, 178 insertions(+), 116 deletions(-) create mode 100644 eng/templates/get-build-info.yml create mode 100644 eng/templates/installer-pipeline-variables.yml diff --git a/eng/source-build-pre-release.yml b/eng/source-build-pre-release.yml index 260ac95d94..ec37960b44 100644 --- a/eng/source-build-pre-release.yml +++ b/eng/source-build-pre-release.yml @@ -10,129 +10,63 @@ resources: - pipeline: dotnet-staging-pipeline-resource source: Stage-DotNet +parameters: + - name: useSpecificPipelineRunIDs + displayName: Use Specific Pipeline Run IDs + type: boolean + default: false + - name: dotnetInstallerOfficialRunID + displayName: dotnet-installer-official-ci run ID + type: string + default: '200XXXX' + - name: dotnetInstallerTarballBuildRunID + displayName: dotnet-installer-source-build-tarball-build run ID + type: string + default: '200XXXX' + variables: +- template: templates/installer-pipeline-variables.yml - group: DotNet-MSRC-Storage -- name: azDoOrg - value: 'https://dev.azure.com/dnceng/' -- name: azDoProject - value: 'internal' -- name: installerOfficialCiPipelineId - value: '286' -- name: installerTarabllBuildCiPipelineId - value: '1011' - name: storageAccountName value: 'dotnetclimsrc' - name: blobContainerName value: 'source-build' - name: blobContainerUploadBaseFilePath - value: 'release' - -stages: - -- stage: GetBuildInfoStage - displayName: Get Build Info - jobs: - - job: GetBuildInfoJob - displayName: Get Build Info - steps: - - download: dotnet-staging-pipeline-resource - artifact: manifests - patterns: 'manifest.json' - displayName: Download Release Manifest - - download: dotnet-staging-pipeline-resource - artifact: drop - patterns: 'config.json' - displayName: Download Release Config - - script: | - set -euo pipefail - - manifest_path=$(PIPELINE.WORKSPACE)/dotnet-staging-pipeline-resource/manifests/manifest.json - config_path=$(PIPELINE.WORKSPACE)/dotnet-staging-pipeline-resource/drop/config.json - - runtime_version="$(jq -r '.Runtime' $config_path)" - release_channel="$(jq -r '.Channel' $config_path)" - - # Source-build only supports the lowest avaiable feature band. - # Sort the SDK releases by number and pick the lowest value. - sdk_version="$(jq -r '.Sdks | sort_by(. | split(".") | map(tonumber)) | .[0]' $config_path)" - - # replace the last two characters in sdk_version with xx - branch_version=$(echo $sdk_version | sed 's/..$/xx/') - branch_name="refs/heads/internal/release/${branch_version}" - - commit=$(jq -r --arg BRANCH "${branch_name}" '.builds[] | select(.repo == "https://dev.azure.com/dnceng/internal/_git/dotnet-installer") | select(.branch == $BRANCH) | .commit' $manifest_path) + value: 'Dev' - if [[ ! $commit ]]; then - echo "Installer commit not set. Exiting..." - exit 1 - fi - - echo "Release channel: ${release_channel}" - echo "Runtime version: ${runtime_version}" - echo "SDK version: ${sdk_version}" - echo "Installer branch: ${branch_name}" - echo "Installer commit: ${commit}" - - echo "##vso[task.setvariable variable=SdkVersion;isOutput=true]${sdk_version}" - echo "##vso[task.setvariable variable=RuntimeVersion;isOutput=true]${runtime_version}" - echo "##vso[task.setvariable variable=ReleaseChannel;isOutput=true]${release_channel}" - echo "##vso[task.setvariable variable=InstallerCommit;isOutput=true]${commit}" - name: ReadReleaseInfoStep - displayName: Read Release Info - -- stage: UploadSourceTarballStage +jobs: +- job: UploadSourceTarballJob displayName: Upload Source Tarball - dependsOn: GetBuildInfoStage - jobs: - - job: UploadSourceTarballJob - displayName: Upload Source Tarball - variables: - - name: InstallerCommit - value: $[ stageDependencies.GetBuildInfoStage.GetBuildInfoJob.outputs['ReadReleaseInfoStep.InstallerCommit'] ] - - name: SdkVersion - value: $[ stageDependencies.GetBuildInfoStage.GetBuildInfoJob.outputs['ReadReleaseInfoStep.SdkVersion'] ] - - name: RuntimeVersion - value: $[ stageDependencies.GetBuildInfoStage.GetBuildInfoJob.outputs['ReadReleaseInfoStep.RuntimeVersion'] ] - - name: ReleaseChannel - value: $[ stageDependencies.GetBuildInfoStage.GetBuildInfoJob.outputs['ReadReleaseInfoStep.ReleaseChannel'] ] - steps: - - script: | - set -euo pipefail - az config set extension.use_dynamic_install=yes_without_prompt - installer_official_runs=$(az pipelines runs list --organization $(azDoOrg) --project $(azDoProject) --pipeline-ids $(installerOfficialCiPipelineId)) - installer_official_run_id=$(echo $installer_official_runs | jq -r '.[] | select(.sourceVersion == "$(InstallerCommit)") | .id') - echo "dotnet-installer-official-ci run ID: ${installer_official_run_id}" - - installer_tarball_build_runs=$(az pipelines runs list --organization $(azDoOrg) --project $(azDoProject) --pipeline-ids $(installerTarabllBuildCiPipelineId)) - installer_tarball_build_run_id=$(echo $installer_tarball_build_runs | jq -r '.[] | select(.sourceVersion == "$(InstallerCommit)") | .id') - echo "dotnet-installer-source-build-tarball-build run ID: ${installer_tarball_build_run_id}" - - echo "##vso[task.setvariable variable=InstallerOfficialRunId]${installer_official_run_id}" - echo "##vso[task.setvariable variable=InstallerTarballBuildRunId]${installer_tarball_build_run_id}" - name: GetAssociatedPipelineRunsStep - displayName: Get Associated Pipeline Run IDs - env: - AZURE_DEVOPS_EXT_PAT: $(System.AccessToken) - - task: DownloadPipelineArtifact@2 - name: DownloadSourceTarballStep - displayName: Download Source Tarball - inputs: - source: 'specific' - artifact: 'BlobArtifacts' - patterns: 'BlobArtifacts/dotnet-sdk-source-$(SdkVersion).tar.gz' - project: 'internal' - pipeline: $(installerOfficialCiPipelineId) - runVersion: 'specific' - runId: $(InstallerOfficialRunId) - - script: | - set -euxo pipefail - az config set extension.use_dynamic_install=yes_without_prompt - - source_tarball_upload_path="$(blobContainerUploadBaseFilePath)/$(ReleaseChannel)/$(RuntimeVersion)-$(SdkVersion)" - source_tarball_filename="dotnet-sdk-source-$(SdkVersion).tar.gz" - az storage blob upload --account-name "$(storageAccountName)" --container-name "$(blobContainerName)" --file "$(PIPELINE.WORKSPACE)/${source_tarball_filename}" --name "${source_tarball_upload_path}/${source_tarball_filename}" - name: UploadSourceTarballStep - displayName: Upload Source Tarball - env: - AZURE_STORAGE_KEY: $(dotnetclimsrc-access-key) + steps: + - template: templates/get-build-info.yml + parameters: + stageDotNetPipelineResource: dotnet-staging-pipeline-resource + getAssociatedPipelineRuns: true + useSpecificPipelineRunIDs: ${{ parameters.useSpecificPipelineRunIDs }} + dotnetInstallerOfficialRunID: ${{ parameters.dotnetInstallerOfficialRunID }} + dotnetInstallerTarballBuildRunID: ${{ parameters.dotnetInstallerTarballBuildRunID }} + + - task: DownloadPipelineArtifact@2 + name: DownloadSourceTarballStep + displayName: Download Source Tarball + inputs: + source: 'specific' + artifact: 'BlobArtifacts' + patterns: 'BlobArtifacts/dotnet-sdk-source-$(SdkVersion).tar.gz' + project: 'internal' + pipeline: $(installerOfficialCiPipelineId) + runVersion: 'specific' + runId: $(InstallerOfficialRunId) + + - script: | + set -euxo pipefail + az config set extension.use_dynamic_install=yes_without_prompt + + source_tarball_upload_path="$(blobContainerUploadBaseFilePath)/$(ReleaseChannel)/$(RuntimeVersion)-$(SdkVersion)" + source_tarball_filename="dotnet-sdk-source-$(SdkVersion).tar.gz" + az storage blob upload --account-name "$(storageAccountName)" --container-name "$(blobContainerName)" --file "$(PIPELINE.WORKSPACE)/${source_tarball_filename}" --name "${source_tarball_upload_path}/${source_tarball_filename}" + name: UploadSourceTarballStep + displayName: Upload Source Tarball + env: + AZURE_STORAGE_KEY: $(dotnetclimsrc-access-key) diff --git a/eng/templates/get-build-info.yml b/eng/templates/get-build-info.yml new file mode 100644 index 0000000000..85f2daeede --- /dev/null +++ b/eng/templates/get-build-info.yml @@ -0,0 +1,118 @@ +parameters: + - name: stageDotNetPipelineResource + type: string + default: '' + - name: getAssociatedPipelineRuns + type: boolean + default: false + - name: useSpecificPipelineRunIDs + type: boolean + default: false + - name: dotnetInstallerOfficialRunID + type: string + default: '' + - name: dotnetInstallerTarballBuildRunID + type: string + default: '' + +steps: +- download: ${{ parameters.stageDotNetPipelineResource }} + artifact: manifests + patterns: 'manifest.json' + displayName: Download Release Manifest + +- download: ${{ parameters.stageDotNetPipelineResource }} + artifact: drop + patterns: 'config.json' + displayName: Download Release Config + +- script: | + set -euo pipefail + + manifest_path=$(PIPELINE.WORKSPACE)/dotnet-staging-pipeline-resource/manifests/manifest.json + config_path=$(PIPELINE.WORKSPACE)/dotnet-staging-pipeline-resource/drop/config.json + + runtime_version="$(jq -r '.Runtime' $config_path)" + release_channel="$(jq -r '.Channel' $config_path)" + + # Source-build only supports the lowest avaiable feature band. + # Sort the SDK releases by number and pick the lowest value. + sdk_version="$(jq -r '.Sdks | sort_by(. | split(".") | map(tonumber)) | .[0]' $config_path)" + + # replace the last two characters in sdk_version with xx + branch_version=$(echo $sdk_version | sed 's/..$/xx/') + branch_name="refs/heads/internal/release/${branch_version}" + + commit=$(jq -r --arg BRANCH "${branch_name}" '.builds[] | select(.repo == "https://dev.azure.com/dnceng/internal/_git/dotnet-installer") | select(.branch == $BRANCH) | .commit' $manifest_path) + + if [[ ! $commit ]]; then + echo "##vso[task.logissue type=error]Installer commit not set. Exiting..." + exit 1 + fi + + echo "Release channel: ${release_channel}" + echo "Runtime version: ${runtime_version}" + echo "SDK version: ${sdk_version}" + echo "Installer commit: ${commit}" + + echo "##vso[task.setvariable variable=SdkVersion]${sdk_version}" + echo "##vso[task.setvariable variable=RuntimeVersion]${runtime_version}" + echo "##vso[task.setvariable variable=ReleaseChannel]${release_channel}" + echo "##vso[task.setvariable variable=InstallerCommit]${commit}" + name: ReadReleaseInfoStep + displayName: Read Release Info + +- ${{ if eq(parameters.getAssociatedPipelineRuns, true) }}: + - ${{ if eq(parameters.useSpecificPipelineRunIDs, false) }}: + - script: | + set -euo pipefail + az config set extension.use_dynamic_install=yes_without_prompt + + installer_official_runs=$(az pipelines runs list --organization $(azDoOrg) --project $(azDoProject) --pipeline-ids $(installerOfficialCiPipelineId)) + installer_official_run_ids=$(echo $installer_official_runs | jq -r '[.[] | select(.sourceVersion == "$(InstallerCommit)") | .id]') + num_official_runs=$(echo $installer_official_run_ids | jq 'length') + + if [ "$num_official_runs" != "1" ]; then + echo "##vso[task.logissue type=error]There are ${num_official_runs} of dotnet-installer-official-ci for $(InstallerCommit). Please manually specify run IDs to use." + echo "##vso[task.logissue type=error]Run IDs are: ${installer_official_run_ids}" + exit 1 + fi + + installer_official_run_id=$(echo $installer_official_run_ids | jq -r '.[0]') + + installer_tarball_build_runs=$(az pipelines runs list --organization $(azDoOrg) --project $(azDoProject) --pipeline-ids $(installerTarabllBuildCiPipelineId)) + installer_tarball_build_run_ids=$(echo $installer_tarball_build_runs | jq -r '[.[] | select(.sourceVersion == "$(InstallerCommit)") | .id]') + num_tarball_build_runs=$(echo $installer_tarball_build_run_ids | jq 'length') + + if [ "$num_tarball_build_runs" != "1" ]; then + echo "##vso[task.logissue type=error]There are ${num_tarball_build_runs} runs of dotnet-installer-source-build-tarball-build for $(InstallerCommit). Please manually specify run IDs to use." + echo "##vso[task.logissue type=error]Run IDs are: ${installer_tarball_build_run_ids}" + exit 1 + fi + + installer_tarball_build_run_id=$(echo $installer_tarball_build_run_ids | jq -r '.[0]') + + echo "dotnet-installer-official-ci run ID: ${installer_official_run_id}" + echo "dotnet-installer-source-build-tarball-build run ID: ${installer_tarball_build_run_id}" + + echo "##vso[task.setvariable variable=InstallerOfficialRunId]${installer_official_run_id}" + echo "##vso[task.setvariable variable=InstallerTarballBuildRunId]${installer_tarball_build_run_id}" + name: GetAssociatedPipelineRunsStep + displayName: Get Associated Pipeline Run IDs + env: + AZURE_DEVOPS_EXT_PAT: $(System.AccessToken) + + - ${{ if eq(parameters.useSpecificPipelineRunIDs, true) }}: + - script: | + set -euo pipefail + + installer_official_run_id="${{ parameters.dotnetInstallerOfficialRunID }}" + installer_tarball_build_run_id="${{ parameters.dotnetInstallerTarballBuildRunID }}" + + echo "dotnet-installer-official-ci run ID: ${installer_official_run_id}" + echo "dotnet-installer-source-build-tarball-build run ID: ${installer_tarball_build_run_id}" + + echo "##vso[task.setvariable variable=InstallerOfficialRunId]${installer_official_run_id}" + echo "##vso[task.setvariable variable=InstallerTarballBuildRunId]${installer_tarball_build_run_id}" + name: SetAssociatedPipelineRunsStep + displayName: Set Associated Pipeline Run IDs \ No newline at end of file diff --git a/eng/templates/installer-pipeline-variables.yml b/eng/templates/installer-pipeline-variables.yml new file mode 100644 index 0000000000..81a320a6f7 --- /dev/null +++ b/eng/templates/installer-pipeline-variables.yml @@ -0,0 +1,10 @@ +# Information about dotnet/installer pipelines that won't change and is useful in multiple pipelines +variables: +- name: azDoProject + value: 'internal' +- name: azDoOrg + value: 'https://dev.azure.com/dnceng/' +- name: installerOfficialCiPipelineId + value: '286' +- name: installerTarabllBuildCiPipelineId + value: '1011' \ No newline at end of file From eacbc392aa5d13325ad060307fa60119e4cf291d Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Mon, 24 Oct 2022 11:32:28 -0700 Subject: [PATCH 07/16] Change staging pipeline resource name --- eng/source-build-pre-release.yml | 2 +- eng/templates/get-build-info.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/source-build-pre-release.yml b/eng/source-build-pre-release.yml index ec37960b44..b756321dc6 100644 --- a/eng/source-build-pre-release.yml +++ b/eng/source-build-pre-release.yml @@ -41,7 +41,7 @@ jobs: steps: - template: templates/get-build-info.yml parameters: - stageDotNetPipelineResource: dotnet-staging-pipeline-resource + dotnetStagingPipelineResource: dotnet-staging-pipeline-resource getAssociatedPipelineRuns: true useSpecificPipelineRunIDs: ${{ parameters.useSpecificPipelineRunIDs }} dotnetInstallerOfficialRunID: ${{ parameters.dotnetInstallerOfficialRunID }} diff --git a/eng/templates/get-build-info.yml b/eng/templates/get-build-info.yml index 85f2daeede..ea1eefb534 100644 --- a/eng/templates/get-build-info.yml +++ b/eng/templates/get-build-info.yml @@ -1,5 +1,5 @@ parameters: - - name: stageDotNetPipelineResource + - name: dotnetStagingPipelineResource type: string default: '' - name: getAssociatedPipelineRuns @@ -16,12 +16,12 @@ parameters: default: '' steps: -- download: ${{ parameters.stageDotNetPipelineResource }} +- download: ${{ parameters.dotnetStagingPipelineResource }} artifact: manifests patterns: 'manifest.json' displayName: Download Release Manifest -- download: ${{ parameters.stageDotNetPipelineResource }} +- download: ${{ parameters.dotnetStagingPipelineResource }} artifact: drop patterns: 'config.json' displayName: Download Release Config From a8de4cbdc6203937d2981e1417968735dc158a1c Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Mon, 24 Oct 2022 11:37:57 -0700 Subject: [PATCH 08/16] Try az storage fs instead of blob --- eng/source-build-pre-release.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/source-build-pre-release.yml b/eng/source-build-pre-release.yml index b756321dc6..be4ed765fe 100644 --- a/eng/source-build-pre-release.yml +++ b/eng/source-build-pre-release.yml @@ -65,7 +65,8 @@ jobs: source_tarball_upload_path="$(blobContainerUploadBaseFilePath)/$(ReleaseChannel)/$(RuntimeVersion)-$(SdkVersion)" source_tarball_filename="dotnet-sdk-source-$(SdkVersion).tar.gz" - az storage blob upload --account-name "$(storageAccountName)" --container-name "$(blobContainerName)" --file "$(PIPELINE.WORKSPACE)/${source_tarball_filename}" --name "${source_tarball_upload_path}/${source_tarball_filename}" + # az storage blob upload --account-name "$(storageAccountName)" --container-name "$(blobContainerName)" --file "$(PIPELINE.WORKSPACE)/${source_tarball_filename}" --name "${source_tarball_upload_path}/${source_tarball_filename}" + az storage fs file upload --source "$(PIPELINE.WORKSPACE)/${source_tarball_filename}" -p "${source_tarball_upload_path}/${source_tarball_filename}" -f "$(blobContainerName)" --account-name "$(storageAccountName)" name: UploadSourceTarballStep displayName: Upload Source Tarball env: From a627264ff6476d73695a05ce2c7cb6c2e0cd4d69 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Mon, 24 Oct 2022 11:50:23 -0700 Subject: [PATCH 09/16] Revert "Try az storage fs instead of blob" This reverts commit a8de4cbdc6203937d2981e1417968735dc158a1c. --- eng/source-build-pre-release.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/eng/source-build-pre-release.yml b/eng/source-build-pre-release.yml index be4ed765fe..b756321dc6 100644 --- a/eng/source-build-pre-release.yml +++ b/eng/source-build-pre-release.yml @@ -65,8 +65,7 @@ jobs: source_tarball_upload_path="$(blobContainerUploadBaseFilePath)/$(ReleaseChannel)/$(RuntimeVersion)-$(SdkVersion)" source_tarball_filename="dotnet-sdk-source-$(SdkVersion).tar.gz" - # az storage blob upload --account-name "$(storageAccountName)" --container-name "$(blobContainerName)" --file "$(PIPELINE.WORKSPACE)/${source_tarball_filename}" --name "${source_tarball_upload_path}/${source_tarball_filename}" - az storage fs file upload --source "$(PIPELINE.WORKSPACE)/${source_tarball_filename}" -p "${source_tarball_upload_path}/${source_tarball_filename}" -f "$(blobContainerName)" --account-name "$(storageAccountName)" + az storage blob upload --account-name "$(storageAccountName)" --container-name "$(blobContainerName)" --file "$(PIPELINE.WORKSPACE)/${source_tarball_filename}" --name "${source_tarball_upload_path}/${source_tarball_filename}" name: UploadSourceTarballStep displayName: Upload Source Tarball env: From dcace3278b7959fcc013ffe21d6c6d6e946dfba7 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Mon, 24 Oct 2022 12:47:46 -0700 Subject: [PATCH 10/16] List files from storage account --- eng/source-build-pre-release.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/eng/source-build-pre-release.yml b/eng/source-build-pre-release.yml index b756321dc6..0c60ca0362 100644 --- a/eng/source-build-pre-release.yml +++ b/eng/source-build-pre-release.yml @@ -65,7 +65,14 @@ jobs: source_tarball_upload_path="$(blobContainerUploadBaseFilePath)/$(ReleaseChannel)/$(RuntimeVersion)-$(SdkVersion)" source_tarball_filename="dotnet-sdk-source-$(SdkVersion).tar.gz" + + if [ ! -f "$(PIPELINE.WORKSPACE)/${source_tarball_filename}" ]; then + echo "##vso[task.logissue type=error]File ${source_tarball_filename} not found. It might not have been downloaded. Exiting..." + fi + az storage blob upload --account-name "$(storageAccountName)" --container-name "$(blobContainerName)" --file "$(PIPELINE.WORKSPACE)/${source_tarball_filename}" --name "${source_tarball_upload_path}/${source_tarball_filename}" + + az storage blob directory list --account-name "$(storageAccountName)" --container-name "$(blobContainerName)" --directory-path "${source_tarball_upload_path}" name: UploadSourceTarballStep displayName: Upload Source Tarball env: From 3c61a6326b3545208690cc1fcb3b2d6c37881af1 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Mon, 24 Oct 2022 14:32:04 -0700 Subject: [PATCH 11/16] Exit if tarball build failed --- eng/templates/get-build-info.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/eng/templates/get-build-info.yml b/eng/templates/get-build-info.yml index ea1eefb534..ce578081f7 100644 --- a/eng/templates/get-build-info.yml +++ b/eng/templates/get-build-info.yml @@ -81,7 +81,7 @@ steps: installer_official_run_id=$(echo $installer_official_run_ids | jq -r '.[0]') installer_tarball_build_runs=$(az pipelines runs list --organization $(azDoOrg) --project $(azDoProject) --pipeline-ids $(installerTarabllBuildCiPipelineId)) - installer_tarball_build_run_ids=$(echo $installer_tarball_build_runs | jq -r '[.[] | select(.sourceVersion == "$(InstallerCommit)") | .id]') + installer_tarball_build_run_ids=$(echo $installer_tarball_build_runs | jq -r '[.[] | select(.sourceVersion == "$(InstallerCommit)") | { "result": .result, "id": .id }]') num_tarball_build_runs=$(echo $installer_tarball_build_run_ids | jq 'length') if [ "$num_tarball_build_runs" != "1" ]; then @@ -90,7 +90,13 @@ steps: exit 1 fi - installer_tarball_build_run_id=$(echo $installer_tarball_build_run_ids | jq -r '.[0]') + installer_tarball_build_run_id=$(echo $installer_tarball_build_run_ids | jq -r '.[0].id') + installer_tarball_build_run_result=$(echo $installer_tarball_build_run_ids | jq -r '.[0].result') + + if [[ ("$installer_tarball_build_run_result" == "failed") && "$(useSpecificPipelineRunIDs)" ]]; then + echo "##vso[task.logissue type=error]: dotnet-installer-source-build-tarball-build run ID ${installer_tarball_build_run_id} failed. Please manually specify a tarball build ID to use instead. Exiting..." + exit 1 + fi echo "dotnet-installer-official-ci run ID: ${installer_official_run_id}" echo "dotnet-installer-source-build-tarball-build run ID: ${installer_tarball_build_run_id}" From f1af178898e238069afb3e71ed4d58f3f961b858 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Mon, 24 Oct 2022 15:27:15 -0700 Subject: [PATCH 12/16] Use blob list instead of blob directory --- eng/source-build-pre-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/source-build-pre-release.yml b/eng/source-build-pre-release.yml index 0c60ca0362..740980912e 100644 --- a/eng/source-build-pre-release.yml +++ b/eng/source-build-pre-release.yml @@ -72,7 +72,7 @@ jobs: az storage blob upload --account-name "$(storageAccountName)" --container-name "$(blobContainerName)" --file "$(PIPELINE.WORKSPACE)/${source_tarball_filename}" --name "${source_tarball_upload_path}/${source_tarball_filename}" - az storage blob directory list --account-name "$(storageAccountName)" --container-name "$(blobContainerName)" --directory-path "${source_tarball_upload_path}" + az storage blob list --account-name "$(storageAccountName)" --container-name "$(blobContainerName)" --prefix "${source_tarball_upload_path}" name: UploadSourceTarballStep displayName: Upload Source Tarball env: From 2f1c8ce5ab38946d4297291edc163713445399cb Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Mon, 24 Oct 2022 16:32:11 -0700 Subject: [PATCH 13/16] Check that tarball was uploaded --- eng/source-build-pre-release.yml | 13 +++++++++++-- eng/templates/get-build-info.yml | 2 ++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/eng/source-build-pre-release.yml b/eng/source-build-pre-release.yml index 740980912e..f22bc37f72 100644 --- a/eng/source-build-pre-release.yml +++ b/eng/source-build-pre-release.yml @@ -67,12 +67,21 @@ jobs: source_tarball_filename="dotnet-sdk-source-$(SdkVersion).tar.gz" if [ ! -f "$(PIPELINE.WORKSPACE)/${source_tarball_filename}" ]; then - echo "##vso[task.logissue type=error]File ${source_tarball_filename} not found. It might not have been downloaded. Exiting..." + echo "##vso[task.logissue type=error]File ${source_tarball_filename} not found on disk. It might not have been downloaded. Exiting..." fi az storage blob upload --account-name "$(storageAccountName)" --container-name "$(blobContainerName)" --file "$(PIPELINE.WORKSPACE)/${source_tarball_filename}" --name "${source_tarball_upload_path}/${source_tarball_filename}" - az storage blob list --account-name "$(storageAccountName)" --container-name "$(blobContainerName)" --prefix "${source_tarball_upload_path}" + source_tarball_blob_list=$(az storage blob list --account-name "$(storageAccountName)" --container-name "$(blobContainerName)" --prefix "${source_tarball_upload_path}/${source_tarball_filename}") + number_of_blobs=$(echo $source_tarball_blob_list | jq -r 'length') + + if [ $number_of_blobs -eq 0 ]; then + echo "##vso[task.logissue type=error]File ${source_tarball_filename} not found in blob storage. It might not have been downloaded earlier, and might not have been uploaded to $(blobContainerName). Exiting..." + exit 1 + elif [ $number_of_blobs -gt 1 ]; then + echo "##vso[task.logissue type=error]More than one blob named ${source_tarball_filename} found in blob storage. Exiting..." + exit 1 + fi name: UploadSourceTarballStep displayName: Upload Source Tarball env: diff --git a/eng/templates/get-build-info.yml b/eng/templates/get-build-info.yml index ce578081f7..bf486f065e 100644 --- a/eng/templates/get-build-info.yml +++ b/eng/templates/get-build-info.yml @@ -93,6 +93,8 @@ steps: installer_tarball_build_run_id=$(echo $installer_tarball_build_run_ids | jq -r '.[0].id') installer_tarball_build_run_result=$(echo $installer_tarball_build_run_ids | jq -r '.[0].result') + echo "useSpecificPipelineRunIDs: $(useSpecificPipelineRunIDs)" + if [[ ("$installer_tarball_build_run_result" == "failed") && "$(useSpecificPipelineRunIDs)" ]]; then echo "##vso[task.logissue type=error]: dotnet-installer-source-build-tarball-build run ID ${installer_tarball_build_run_id} failed. Please manually specify a tarball build ID to use instead. Exiting..." exit 1 From 298d9c993255a29f8097a2c0af7a763ba5a5b8a5 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Mon, 24 Oct 2022 16:42:13 -0700 Subject: [PATCH 14/16] Remove unused logging --- eng/templates/get-build-info.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/eng/templates/get-build-info.yml b/eng/templates/get-build-info.yml index bf486f065e..ce578081f7 100644 --- a/eng/templates/get-build-info.yml +++ b/eng/templates/get-build-info.yml @@ -93,8 +93,6 @@ steps: installer_tarball_build_run_id=$(echo $installer_tarball_build_run_ids | jq -r '.[0].id') installer_tarball_build_run_result=$(echo $installer_tarball_build_run_ids | jq -r '.[0].result') - echo "useSpecificPipelineRunIDs: $(useSpecificPipelineRunIDs)" - if [[ ("$installer_tarball_build_run_result" == "failed") && "$(useSpecificPipelineRunIDs)" ]]; then echo "##vso[task.logissue type=error]: dotnet-installer-source-build-tarball-build run ID ${installer_tarball_build_run_id} failed. Please manually specify a tarball build ID to use instead. Exiting..." exit 1 From 6e895b1a0df0c069a55e28dcb267693407eb387e Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Mon, 24 Oct 2022 16:43:53 -0700 Subject: [PATCH 15/16] Change staging pipeline resource name --- eng/source-build-pre-release.yml | 2 +- eng/templates/get-build-info.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/source-build-pre-release.yml b/eng/source-build-pre-release.yml index ec37960b44..b756321dc6 100644 --- a/eng/source-build-pre-release.yml +++ b/eng/source-build-pre-release.yml @@ -41,7 +41,7 @@ jobs: steps: - template: templates/get-build-info.yml parameters: - stageDotNetPipelineResource: dotnet-staging-pipeline-resource + dotnetStagingPipelineResource: dotnet-staging-pipeline-resource getAssociatedPipelineRuns: true useSpecificPipelineRunIDs: ${{ parameters.useSpecificPipelineRunIDs }} dotnetInstallerOfficialRunID: ${{ parameters.dotnetInstallerOfficialRunID }} diff --git a/eng/templates/get-build-info.yml b/eng/templates/get-build-info.yml index 85f2daeede..ea1eefb534 100644 --- a/eng/templates/get-build-info.yml +++ b/eng/templates/get-build-info.yml @@ -1,5 +1,5 @@ parameters: - - name: stageDotNetPipelineResource + - name: dotnetStagingPipelineResource type: string default: '' - name: getAssociatedPipelineRuns @@ -16,12 +16,12 @@ parameters: default: '' steps: -- download: ${{ parameters.stageDotNetPipelineResource }} +- download: ${{ parameters.dotnetStagingPipelineResource }} artifact: manifests patterns: 'manifest.json' displayName: Download Release Manifest -- download: ${{ parameters.stageDotNetPipelineResource }} +- download: ${{ parameters.dotnetStagingPipelineResource }} artifact: drop patterns: 'config.json' displayName: Download Release Config From 257328385ba854aa01bb4cdfcdb4cc413ae07643 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Mon, 24 Oct 2022 16:49:52 -0700 Subject: [PATCH 16/16] Switch back to release filepath --- eng/source-build-pre-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/source-build-pre-release.yml b/eng/source-build-pre-release.yml index f22bc37f72..7e00652ab3 100644 --- a/eng/source-build-pre-release.yml +++ b/eng/source-build-pre-release.yml @@ -32,7 +32,7 @@ variables: - name: blobContainerName value: 'source-build' - name: blobContainerUploadBaseFilePath - value: 'Dev' + value: 'release' jobs: - job: UploadSourceTarballJob