diff --git a/.github/ISSUE_TEMPLATE/releases/release_checklist.md b/.github/ISSUE_TEMPLATE/releases/release_checklist.md index a09f054d5b..631640d68e 100644 --- a/.github/ISSUE_TEMPLATE/releases/release_checklist.md +++ b/.github/ISSUE_TEMPLATE/releases/release_checklist.md @@ -15,13 +15,14 @@ - This is useful to make sure context is available to review later. It may end up blank if the release goes very smoothly. - There are other notes on servicing in this OneNote. It may be useful to review if something goes wrong to see if it's been fixed before. - File issues appropriately as you encounter problems, and link to them from the notes. Provide info in the issue rather than in the notes. -1. - [ ] Retrieve dotnet/installer commit sha from internal communications for the release. +1. - [ ] Retrieve the final run of the [Stage-DotNet](https://dev.azure.com/dnceng/internal/_build?definitionId=792&_a=summary) pipeline from internal release communications. +1. - [ ] Run the [source-build-pre-release](https://dev.azure.com/dnceng/internal/_build?definitionId=1188) pipeline. When staging the pipeline run, click "Resources" and select the final run of Stage-DotNet mentioned above. + - [ ] When the pipeline finishes, look in the logs for the dotnet/installer commit sha that represents the final release. + - This pipeline automatically uploads the dotnet source tarball to dotnetclimsrc 1. - [ ] Ensure the official installer and tarball builds have completed for the release's commit sha. - - [ ] [Installer](https://dev.azure.com/dnceng/internal/_build?definitionId=286) (internal link) - - [ ] Retrieve the source tarball artifact - `BlobArtifacts/dotnet-sdk-source-6.0.xxx.tar.gz` - [ ] [Tarball](https://dev.azure.com/dnceng/internal/_build?definitionId=1011) (internal link) - [ ] Ensure the PoisonTests and SdkContentTests are passing. Warnings indicate a baseline diff and should be inspected carefully. -1. - [ ] [Internal] Gather smoke-test prereqs +1. - [ ] [Internal] Gather smoke-test prereqs ([automation tracking issue](https://github.com/dotnet/source-build/issues/3068)) - [ ] Retrieve smoke-test prereqs artifact for each architecture (e.g. x64 and arm64) from [tarball build](https://dev.azure.com/dnceng/internal/_build?definitionId=1011) (internal link) - [ ] x64 - `Build Tarball CentOS7-Offline_Artifacts/dotnet-smoke-test-prereqs.6.0.xxx.tar.gz` - [ ] arm64 - `Build Tarball Debian9-Offline_Artifacts/dotnet-smoke-test-prereqs.6.0.xxx.tar.gz` @@ -32,13 +33,13 @@ - [ ] Extact arm64 tarball - [ ] Copy four `*linux-arm64*` packages to x64 packages - [ ] Create new `dotnet-smoke-test-prereqs.6.0.xxx.tar.gz` tarball -1. - [ ] [Internal] Upload source and smoke-test-prereqs tarball to dotnetclimsrc storage account. +1. - [ ] [Internal] Upload smoke-test-prereqs tarball to dotnetclimsrc storage account. 1. - [ ] Notify partners of release. Include info about how certain we are that this will be the final Microsoft build. - [Internal] Send the dotnetclimsrc tarball links to partners. - Never overwrite a tarball. At least change the blob storage virtual dir to represent a new build. This can help avoid timing issues and make it more obvious if stale links were accidentally re-sent rather than new ones. - [Non-Internal] Send the dotnet/installer commit sha along w/link to publicly built source tarball. Link to the public instructions for building source-build. 1. - [ ] SYNC POINT: Wait for Microsoft build release. -1. - [ ] Upload the source-build artifacts to dotnetcli/source-built-artifacts blob storage. +1. - [ ] Upload the source-build artifacts to dotnetcli/source-built-artifacts blob storage. ([automation tracking issue](https://github.com/dotnet/source-build/issues/3080)) - [ ] Retrieve the source-build artifacts from [Tarball](https://dev.azure.com/dnceng/internal/_build?definitionId=1011) (internal link) - `Build Tarball CentOS7-Offline_Artifacts/Private.SourceBuilt.Artifacts.6.0.xxx.tar.gz` 1. - [ ] Run the [source-build-release pipeline](https://dev.azure.com/dnceng/internal/_build?definitionId=1124) (internal link). - Set the `SDK Version` parameter. diff --git a/eng/source-build-pre-release.yml b/eng/source-build-pre-release.yml new file mode 100644 index 0000000000..7e00652ab3 --- /dev/null +++ b/eng/source-build-pre-release.yml @@ -0,0 +1,88 @@ +trigger: none +pr: none + +pool: + name: NetCore1ESPool-Svc-Internal + demands: ImageOverride -equals 1es-ubuntu-2004 + +resources: + pipelines: + - pipeline: dotnet-staging-pipeline-resource + source: Stage-DotNet + +parameters: + - name: useSpecificPipelineRunIDs + displayName: Use Specific Pipeline Run IDs + type: boolean + default: false + - name: dotnetInstallerOfficialRunID + displayName: dotnet-installer-official-ci run ID + type: string + default: '200XXXX' + - name: dotnetInstallerTarballBuildRunID + displayName: dotnet-installer-source-build-tarball-build run ID + type: string + default: '200XXXX' + +variables: +- template: templates/installer-pipeline-variables.yml +- group: DotNet-MSRC-Storage +- name: storageAccountName + value: 'dotnetclimsrc' +- name: blobContainerName + value: 'source-build' +- name: blobContainerUploadBaseFilePath + value: 'release' + +jobs: +- job: UploadSourceTarballJob + displayName: Upload Source Tarball + + steps: + - template: templates/get-build-info.yml + parameters: + dotnetStagingPipelineResource: dotnet-staging-pipeline-resource + getAssociatedPipelineRuns: true + useSpecificPipelineRunIDs: ${{ parameters.useSpecificPipelineRunIDs }} + dotnetInstallerOfficialRunID: ${{ parameters.dotnetInstallerOfficialRunID }} + dotnetInstallerTarballBuildRunID: ${{ parameters.dotnetInstallerTarballBuildRunID }} + + - task: DownloadPipelineArtifact@2 + name: DownloadSourceTarballStep + displayName: Download Source Tarball + inputs: + source: 'specific' + artifact: 'BlobArtifacts' + patterns: 'BlobArtifacts/dotnet-sdk-source-$(SdkVersion).tar.gz' + project: 'internal' + pipeline: $(installerOfficialCiPipelineId) + runVersion: 'specific' + runId: $(InstallerOfficialRunId) + + - script: | + set -euxo pipefail + az config set extension.use_dynamic_install=yes_without_prompt + + source_tarball_upload_path="$(blobContainerUploadBaseFilePath)/$(ReleaseChannel)/$(RuntimeVersion)-$(SdkVersion)" + source_tarball_filename="dotnet-sdk-source-$(SdkVersion).tar.gz" + + if [ ! -f "$(PIPELINE.WORKSPACE)/${source_tarball_filename}" ]; then + echo "##vso[task.logissue type=error]File ${source_tarball_filename} not found on disk. It might not have been downloaded. Exiting..." + fi + + az storage blob upload --account-name "$(storageAccountName)" --container-name "$(blobContainerName)" --file "$(PIPELINE.WORKSPACE)/${source_tarball_filename}" --name "${source_tarball_upload_path}/${source_tarball_filename}" + + source_tarball_blob_list=$(az storage blob list --account-name "$(storageAccountName)" --container-name "$(blobContainerName)" --prefix "${source_tarball_upload_path}/${source_tarball_filename}") + number_of_blobs=$(echo $source_tarball_blob_list | jq -r 'length') + + if [ $number_of_blobs -eq 0 ]; then + echo "##vso[task.logissue type=error]File ${source_tarball_filename} not found in blob storage. It might not have been downloaded earlier, and might not have been uploaded to $(blobContainerName). Exiting..." + exit 1 + elif [ $number_of_blobs -gt 1 ]; then + echo "##vso[task.logissue type=error]More than one blob named ${source_tarball_filename} found in blob storage. Exiting..." + exit 1 + fi + name: UploadSourceTarballStep + displayName: Upload Source Tarball + env: + AZURE_STORAGE_KEY: $(dotnetclimsrc-access-key) diff --git a/eng/templates/get-build-info.yml b/eng/templates/get-build-info.yml new file mode 100644 index 0000000000..ce578081f7 --- /dev/null +++ b/eng/templates/get-build-info.yml @@ -0,0 +1,124 @@ +parameters: + - name: dotnetStagingPipelineResource + type: string + default: '' + - name: getAssociatedPipelineRuns + type: boolean + default: false + - name: useSpecificPipelineRunIDs + type: boolean + default: false + - name: dotnetInstallerOfficialRunID + type: string + default: '' + - name: dotnetInstallerTarballBuildRunID + type: string + default: '' + +steps: +- download: ${{ parameters.dotnetStagingPipelineResource }} + artifact: manifests + patterns: 'manifest.json' + displayName: Download Release Manifest + +- download: ${{ parameters.dotnetStagingPipelineResource }} + artifact: drop + patterns: 'config.json' + displayName: Download Release Config + +- script: | + set -euo pipefail + + manifest_path=$(PIPELINE.WORKSPACE)/dotnet-staging-pipeline-resource/manifests/manifest.json + config_path=$(PIPELINE.WORKSPACE)/dotnet-staging-pipeline-resource/drop/config.json + + runtime_version="$(jq -r '.Runtime' $config_path)" + release_channel="$(jq -r '.Channel' $config_path)" + + # Source-build only supports the lowest avaiable feature band. + # Sort the SDK releases by number and pick the lowest value. + sdk_version="$(jq -r '.Sdks | sort_by(. | split(".") | map(tonumber)) | .[0]' $config_path)" + + # replace the last two characters in sdk_version with xx + branch_version=$(echo $sdk_version | sed 's/..$/xx/') + branch_name="refs/heads/internal/release/${branch_version}" + + commit=$(jq -r --arg BRANCH "${branch_name}" '.builds[] | select(.repo == "https://dev.azure.com/dnceng/internal/_git/dotnet-installer") | select(.branch == $BRANCH) | .commit' $manifest_path) + + if [[ ! $commit ]]; then + echo "##vso[task.logissue type=error]Installer commit not set. Exiting..." + exit 1 + fi + + echo "Release channel: ${release_channel}" + echo "Runtime version: ${runtime_version}" + echo "SDK version: ${sdk_version}" + echo "Installer commit: ${commit}" + + echo "##vso[task.setvariable variable=SdkVersion]${sdk_version}" + echo "##vso[task.setvariable variable=RuntimeVersion]${runtime_version}" + echo "##vso[task.setvariable variable=ReleaseChannel]${release_channel}" + echo "##vso[task.setvariable variable=InstallerCommit]${commit}" + name: ReadReleaseInfoStep + displayName: Read Release Info + +- ${{ if eq(parameters.getAssociatedPipelineRuns, true) }}: + - ${{ if eq(parameters.useSpecificPipelineRunIDs, false) }}: + - script: | + set -euo pipefail + az config set extension.use_dynamic_install=yes_without_prompt + + installer_official_runs=$(az pipelines runs list --organization $(azDoOrg) --project $(azDoProject) --pipeline-ids $(installerOfficialCiPipelineId)) + installer_official_run_ids=$(echo $installer_official_runs | jq -r '[.[] | select(.sourceVersion == "$(InstallerCommit)") | .id]') + num_official_runs=$(echo $installer_official_run_ids | jq 'length') + + if [ "$num_official_runs" != "1" ]; then + echo "##vso[task.logissue type=error]There are ${num_official_runs} of dotnet-installer-official-ci for $(InstallerCommit). Please manually specify run IDs to use." + echo "##vso[task.logissue type=error]Run IDs are: ${installer_official_run_ids}" + exit 1 + fi + + installer_official_run_id=$(echo $installer_official_run_ids | jq -r '.[0]') + + installer_tarball_build_runs=$(az pipelines runs list --organization $(azDoOrg) --project $(azDoProject) --pipeline-ids $(installerTarabllBuildCiPipelineId)) + installer_tarball_build_run_ids=$(echo $installer_tarball_build_runs | jq -r '[.[] | select(.sourceVersion == "$(InstallerCommit)") | { "result": .result, "id": .id }]') + num_tarball_build_runs=$(echo $installer_tarball_build_run_ids | jq 'length') + + if [ "$num_tarball_build_runs" != "1" ]; then + echo "##vso[task.logissue type=error]There are ${num_tarball_build_runs} runs of dotnet-installer-source-build-tarball-build for $(InstallerCommit). Please manually specify run IDs to use." + echo "##vso[task.logissue type=error]Run IDs are: ${installer_tarball_build_run_ids}" + exit 1 + fi + + installer_tarball_build_run_id=$(echo $installer_tarball_build_run_ids | jq -r '.[0].id') + installer_tarball_build_run_result=$(echo $installer_tarball_build_run_ids | jq -r '.[0].result') + + if [[ ("$installer_tarball_build_run_result" == "failed") && "$(useSpecificPipelineRunIDs)" ]]; then + echo "##vso[task.logissue type=error]: dotnet-installer-source-build-tarball-build run ID ${installer_tarball_build_run_id} failed. Please manually specify a tarball build ID to use instead. Exiting..." + exit 1 + fi + + echo "dotnet-installer-official-ci run ID: ${installer_official_run_id}" + echo "dotnet-installer-source-build-tarball-build run ID: ${installer_tarball_build_run_id}" + + echo "##vso[task.setvariable variable=InstallerOfficialRunId]${installer_official_run_id}" + echo "##vso[task.setvariable variable=InstallerTarballBuildRunId]${installer_tarball_build_run_id}" + name: GetAssociatedPipelineRunsStep + displayName: Get Associated Pipeline Run IDs + env: + AZURE_DEVOPS_EXT_PAT: $(System.AccessToken) + + - ${{ if eq(parameters.useSpecificPipelineRunIDs, true) }}: + - script: | + set -euo pipefail + + installer_official_run_id="${{ parameters.dotnetInstallerOfficialRunID }}" + installer_tarball_build_run_id="${{ parameters.dotnetInstallerTarballBuildRunID }}" + + echo "dotnet-installer-official-ci run ID: ${installer_official_run_id}" + echo "dotnet-installer-source-build-tarball-build run ID: ${installer_tarball_build_run_id}" + + echo "##vso[task.setvariable variable=InstallerOfficialRunId]${installer_official_run_id}" + echo "##vso[task.setvariable variable=InstallerTarballBuildRunId]${installer_tarball_build_run_id}" + name: SetAssociatedPipelineRunsStep + displayName: Set Associated Pipeline Run IDs \ No newline at end of file diff --git a/eng/templates/installer-pipeline-variables.yml b/eng/templates/installer-pipeline-variables.yml new file mode 100644 index 0000000000..81a320a6f7 --- /dev/null +++ b/eng/templates/installer-pipeline-variables.yml @@ -0,0 +1,10 @@ +# Information about dotnet/installer pipelines that won't change and is useful in multiple pipelines +variables: +- name: azDoProject + value: 'internal' +- name: azDoOrg + value: 'https://dev.azure.com/dnceng/' +- name: installerOfficialCiPipelineId + value: '286' +- name: installerTarabllBuildCiPipelineId + value: '1011' \ No newline at end of file