From e6a0a8a05b51093910586396a52e536b8bff6b8e Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Tue, 22 Nov 2022 00:31:36 +0000 Subject: [PATCH 1/3] Add script to push new branch of security-partners-dotnet from pipeline --- eng/source-build-pre-release.yml | 24 +++++++ eng/update-vmr.sh | 103 +++++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+) create mode 100755 eng/update-vmr.sh diff --git a/eng/source-build-pre-release.yml b/eng/source-build-pre-release.yml index 2a7f0b3bda..e03b19b027 100644 --- a/eng/source-build-pre-release.yml +++ b/eng/source-build-pre-release.yml @@ -31,16 +31,20 @@ parameters: variables: - template: templates/variables/installer-pipelines.yml - group: DotNet-MSRC-Storage +- group: DotNet-Source-Build-All-Orgs-Source-Access - name: storageAccountName value: 'dotnetclimsrc' - name: blobContainerName value: 'source-build' +- name: vmrUpstreamUrl + value: 'https://dnceng@dev.azure.com/dnceng/internal/_git/security-partners-dotnet' - name: blobContainerUploadBaseFilePath ${{ if eq( parameters['isDryRun'], true) }}: value: 'Dev' ${{ else }}: value: 'release' + jobs: - job: UploadSourceTarballJob displayName: Upload Source Tarball @@ -73,3 +77,23 @@ jobs: containerName: '$(blobContainerName)' uploadPath: '$(blobContainerUploadBaseFilePath)/$(ReleaseChannel)/$(RuntimeVersion)-$(SdkVersion)' azureStorageKey: '$(dotnetclimsrc-access-key)' + + - script: | + set -euxo pipefail + + upstream_with_pat=$(echo $(vmrUpstreamUrl) | sed "s,https://.*@,https://dn-bot:${AZDO_PAT}@,g") + + $(Build.SourcesDirectory)/eng/update-vmr.sh \ + --releaseChannel $(ReleaseChannel) \ + --sdkVersion $(SdkVersion) \ + --upstream ${upstream_with_pat} \ + --tarball $(Pipeline.Workspace)/dotnet-sdk-source-$(SdkVersion).tar.gz + + month_year=$(date +"%b%Y" -d "+1 month" | sed 's/.*/\L&/') # e.g. aug2022 + new_branch_name="dev/$(SdkVersion)-${month_year}" + + cd dotnet-vmr/ + git push -u upstream "${new_branch_name}" + displayName: Update security-partners-dotnet + env: + AZDO_PAT: $(dn-bot-all-orgs-build-rw-code-rw) \ No newline at end of file diff --git a/eng/update-vmr.sh b/eng/update-vmr.sh new file mode 100755 index 0000000000..0e1097901c --- /dev/null +++ b/eng/update-vmr.sh @@ -0,0 +1,103 @@ +#!/bin/bash +set -euxo pipefail + +time="$(date +%m%d%y-%H%M)" + +print-help () +{ + echo "Updates the VMR" + echo + echo "Syntax: ./update-vmr.sh --releaseChannel 7.0 --sdkVersion 7.0.100 --upstream --tarball dotnet-sdk-source.tar.gz" + echo + echo "options:" + echo "--releaseChannel, -r The .NET SDK release channel (e.g. 6.0)" + echo "--sdkVersion, -v The .NET SDK version (e.g. 6.0.110)" + echo "--upstream, -u A valid git URL to the upstream repo to base the update branch off of" + echo "--tarball, -t The tarball to update the VMR with" + echo "--isDryRun, -d If set, then don't push results to AzDo" + echo "--help, -h (Optional) print this help message and exit" + echo +} + +SHORT=r:v:u:t:h +LONG=releaseChannel:,sdkVersion:,upstream:,tarball:,help + +OPTS=$(getopt --options $SHORT --long $LONG --name "$0" -- "$@") +if [ $? != 0 ] ; then echo "Failed to parse options." >&2 ; exit 1 ; fi +eval set -- "$OPTS" + +is_dry_run='false' + +while true ; do + case "$1" in + -h | --help ) + print-help + exit 0 + ;; + -r | --releaseChannel ) + release_channel="$2" + shift 2 + ;; + -v | --sdkVersion ) + sdk_version="$2" + shift 2 + ;; + -u | --upstream ) + upstream_url="$2" + shift 2 + ;; + -t | --tarball ) + source_tarball="$2" + shift 2 + ;; + -d | --isDryRun ) + is_dry_run='true' + shift + ;; + -- ) + shift + break + ;; + *) + echo "Internal error! Are you missing required arguments?" + exit 1 + ;; + esac +done + +if [ ! -f "${source_tarball}" ]; then + echo "##vso[task.logissue type=error]File ${source_tarball} not found on disk. Exiting..." +fi + +month_year=$(date +"%b%Y" -d "+1 month" | sed 's/.*/\L&/') # e.g. aug2022 + +vmr_path="$(pwd)/dotnet-vmr" + +# replace the last two characters in sdk_version with xx +branch_version=$(echo ${sdk_version} | sed 's/..$/xx/') +target_branch="release/${branch_version}" # e.g. release/6.0.1xx + +rm -rf "${vmr_path}" +git init "${vmr_path}" + +pushd "${vmr_path}" + git remote add upstream "${upstream_url}" + git fetch upstream "${target_branch}" + + git checkout "${target_branch}" + + new_branch_name="dev/${sdk_version}-${month_year}" + git checkout -b "${new_branch_name}" + + # delete all contents except the .git folder + # otherwise we won't catch deleted files in a commit + ls | grep -v ".git" | xargs rm -rf + tar -xzf "${source_tarball}" -C "${vmr_path}" + + git add -f . + + # taken from https://github.com/dotnet/installer/blob/2fd58dc7e984cacf8c455428bf9bfce75f736f32/eng/pipelines/templates/jobs/vmr-synchronization.yml#L97-L98 + git config user.email "dotnet-maestro[bot]@users.noreply.github.com" + git config user.name "dotnet-maestro[bot]" + git commit -m "Update to .NET ${sdk_version}" +popd From a9b33a12b0217cd94311022e1f44fa63d9a77f10 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Tue, 22 Nov 2022 00:43:01 +0000 Subject: [PATCH 2/3] Minor clean up --- eng/source-build-pre-release.yml | 16 ++++++++++++---- eng/update-vmr.sh | 3 +-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/eng/source-build-pre-release.yml b/eng/source-build-pre-release.yml index e03b19b027..2ee6a46f8f 100644 --- a/eng/source-build-pre-release.yml +++ b/eng/source-build-pre-release.yml @@ -46,8 +46,8 @@ variables: jobs: -- job: UploadSourceTarballJob - displayName: Upload Source Tarball +- job: SourceBuildPreRelease + displayName: Source Build Pre-Release steps: - template: templates/steps/get-build-info.yml @@ -71,6 +71,7 @@ jobs: runId: $(InstallerOfficialRunId) - template: templates/steps/upload-to-blob-storage.yml + condition: and(succeeded(), ne('${{ parameters.isDryRun }}', 'true')) parameters: file: '$(PIPELINE.WORKSPACE)/dotnet-sdk-source-$(SdkVersion).tar.gz' accountName: '$(storageAccountName)' @@ -79,7 +80,7 @@ jobs: azureStorageKey: '$(dotnetclimsrc-access-key)' - script: | - set -euxo pipefail + set -euo pipefail upstream_with_pat=$(echo $(vmrUpstreamUrl) | sed "s,https://.*@,https://dn-bot:${AZDO_PAT}@,g") @@ -93,7 +94,14 @@ jobs: new_branch_name="dev/$(SdkVersion)-${month_year}" cd dotnet-vmr/ - git push -u upstream "${new_branch_name}" + + if [ ${{ parameters.isDryRun }} = True ]; then + echo "Doing a dry run, not pushing to $(vmrUpstreamUrl). List of changes:" + git log --name-status HEAD^..HEAD + else + echo "Pushing branch to $(vmrUpstreamUrl)." + git push -u upstream "${new_branch_name}" + fi displayName: Update security-partners-dotnet env: AZDO_PAT: $(dn-bot-all-orgs-build-rw-code-rw) \ No newline at end of file diff --git a/eng/update-vmr.sh b/eng/update-vmr.sh index 0e1097901c..8ac0312272 100755 --- a/eng/update-vmr.sh +++ b/eng/update-vmr.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -euxo pipefail +set -euo pipefail time="$(date +%m%d%y-%H%M)" @@ -96,7 +96,6 @@ pushd "${vmr_path}" git add -f . - # taken from https://github.com/dotnet/installer/blob/2fd58dc7e984cacf8c455428bf9bfce75f736f32/eng/pipelines/templates/jobs/vmr-synchronization.yml#L97-L98 git config user.email "dotnet-maestro[bot]@users.noreply.github.com" git config user.name "dotnet-maestro[bot]" git commit -m "Update to .NET ${sdk_version}" From 550941fba6e02198efcec78a29294c3de0307865 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Tue, 22 Nov 2022 19:05:29 +0000 Subject: [PATCH 3/3] Address review comments --- eng/source-build-pre-release.yml | 24 +++++-------- eng/update-vmr.sh | 61 +++++++++++++++++++------------- 2 files changed, 44 insertions(+), 41 deletions(-) diff --git a/eng/source-build-pre-release.yml b/eng/source-build-pre-release.yml index 2ee6a46f8f..5810dda3e9 100644 --- a/eng/source-build-pre-release.yml +++ b/eng/source-build-pre-release.yml @@ -71,7 +71,6 @@ jobs: runId: $(InstallerOfficialRunId) - template: templates/steps/upload-to-blob-storage.yml - condition: and(succeeded(), ne('${{ parameters.isDryRun }}', 'true')) parameters: file: '$(PIPELINE.WORKSPACE)/dotnet-sdk-source-$(SdkVersion).tar.gz' accountName: '$(storageAccountName)' @@ -84,24 +83,17 @@ jobs: upstream_with_pat=$(echo $(vmrUpstreamUrl) | sed "s,https://.*@,https://dn-bot:${AZDO_PAT}@,g") - $(Build.SourcesDirectory)/eng/update-vmr.sh \ - --releaseChannel $(ReleaseChannel) \ - --sdkVersion $(SdkVersion) \ - --upstream ${upstream_with_pat} \ - --tarball $(Pipeline.Workspace)/dotnet-sdk-source-$(SdkVersion).tar.gz - - month_year=$(date +"%b%Y" -d "+1 month" | sed 's/.*/\L&/') # e.g. aug2022 - new_branch_name="dev/$(SdkVersion)-${month_year}" - - cd dotnet-vmr/ + args=() + args+=(--releaseChannel $(ReleaseChannel)) + args+=(--sdkVersion $(SdkVersion)) + args+=(--upstream ${upstream_with_pat}) + args+=(--tarball $(Pipeline.Workspace)/dotnet-sdk-source-$(SdkVersion).tar.gz) if [ ${{ parameters.isDryRun }} = True ]; then - echo "Doing a dry run, not pushing to $(vmrUpstreamUrl). List of changes:" - git log --name-status HEAD^..HEAD - else - echo "Pushing branch to $(vmrUpstreamUrl)." - git push -u upstream "${new_branch_name}" + args+=(--isDryRun) fi + + $(Build.SourcesDirectory)/eng/update-vmr.sh "${args[@]}" displayName: Update security-partners-dotnet env: AZDO_PAT: $(dn-bot-all-orgs-build-rw-code-rw) \ No newline at end of file diff --git a/eng/update-vmr.sh b/eng/update-vmr.sh index 8ac0312272..7e082e1975 100755 --- a/eng/update-vmr.sh +++ b/eng/update-vmr.sh @@ -1,8 +1,6 @@ #!/bin/bash set -euo pipefail -time="$(date +%m%d%y-%H%M)" - print-help () { echo "Updates the VMR" @@ -12,15 +10,15 @@ print-help () echo "options:" echo "--releaseChannel, -r The .NET SDK release channel (e.g. 6.0)" echo "--sdkVersion, -v The .NET SDK version (e.g. 6.0.110)" - echo "--upstream, -u A valid git URL to the upstream repo to base the update branch off of" + echo "--upstream, -u A valid git URL to the upstream repo to update from and push to" echo "--tarball, -t The tarball to update the VMR with" - echo "--isDryRun, -d If set, then don't push results to AzDo" + echo "--isDryRun, -d (Optional) If set, then don't push results to upstream" echo "--help, -h (Optional) print this help message and exit" echo } -SHORT=r:v:u:t:h -LONG=releaseChannel:,sdkVersion:,upstream:,tarball:,help +SHORT=r:v:u:t:dh +LONG=releaseChannel:,sdkVersion:,upstream:,tarball:,isDryRun,help OPTS=$(getopt --options $SHORT --long $LONG --name "$0" -- "$@") if [ $? != 0 ] ; then echo "Failed to parse options." >&2 ; exit 1 ; fi @@ -51,7 +49,7 @@ while true ; do shift 2 ;; -d | --isDryRun ) - is_dry_run='true' + is_dry_run=true shift ;; -- ) @@ -65,6 +63,11 @@ while true ; do esac done +: ${release_channel:?Missing --releaseChannel} +: ${sdk_version:?Missing --sdkVersion} +: ${upstream_url:?Missing --upstream} +: ${source_tarball:?Missing --tarball} + if [ ! -f "${source_tarball}" ]; then echo "##vso[task.logissue type=error]File ${source_tarball} not found on disk. Exiting..." fi @@ -81,22 +84,30 @@ rm -rf "${vmr_path}" git init "${vmr_path}" pushd "${vmr_path}" - git remote add upstream "${upstream_url}" - git fetch upstream "${target_branch}" - - git checkout "${target_branch}" - - new_branch_name="dev/${sdk_version}-${month_year}" - git checkout -b "${new_branch_name}" - - # delete all contents except the .git folder - # otherwise we won't catch deleted files in a commit - ls | grep -v ".git" | xargs rm -rf - tar -xzf "${source_tarball}" -C "${vmr_path}" - - git add -f . - - git config user.email "dotnet-maestro[bot]@users.noreply.github.com" - git config user.name "dotnet-maestro[bot]" - git commit -m "Update to .NET ${sdk_version}" + git remote add upstream "${upstream_url}" + git fetch upstream "${target_branch}" --depth=1 + + git checkout "${target_branch}" + + new_branch_name="dev/${sdk_version}-${month_year}" + git checkout -b "${new_branch_name}" + + # delete all contents except the .git folder + # otherwise we won't catch deleted files in a commit + ls | grep -v ".git" | xargs rm -rf + tar -xzf "${source_tarball}" -C "${vmr_path}" + + git add -f . + + git config user.email "dotnet-maestro[bot]@users.noreply.github.com" + git config user.name "dotnet-maestro[bot]" + git commit -m "Update to .NET ${sdk_version}" + + if [ "$is_dry_run" = true ]; then + echo "Doing a dry run, not pushing to upstream. List of changes:" + git log --name-status HEAD^..HEAD + else + echo "Pushing branch to upstream." + git push -u upstream "${new_branch_name}" + fi popd