From d61028f451600c6609a44b6c2f71e4251adf3617 Mon Sep 17 00:00:00 2001 From: William Bezuidenhout Date: Wed, 10 Jan 2024 15:31:28 +0200 Subject: [PATCH 1/8] initial manifest --- release.yaml | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 release.yaml diff --git a/release.yaml b/release.yaml new file mode 100644 index 000000000..33bd34a74 --- /dev/null +++ b/release.yaml @@ -0,0 +1,70 @@ +--- +meta: + productName: "deploy-sourcegraph-docker" + owners: + - "@sourcegraph/release" + repository: "github.com/sourcegraph/deploy-sourcegraph-docker" +inputs: + releaseId: server +requirements: + - name: "go" + cmd: "which go" + fixInstructions: "install golang" + - name: "GitHub cli exists" + cmd: "which gh" + fixInstructions: "install GitHub cli" +internal: + create: + steps: + minor: + - name: docker:tags + cmd: | + ./tools/update-docker-tags.sh {{tag}} + - name: "git:branch" + cmd: | + branch="wb/wip_{{version}}" + git switch -c "${branch}" + git commit -am 'release-major: {{version}}' -m '{{config}}' + major: + - name: docker:tags + cmd: | + ./tools/update-docker-tags.sh {{tag}} + - name: "git:branch" + cmd: | + branch="wb/wip_{{version}}" + git switch -c "${branch}" + git commit -am 'release-major: {{version}}' -m '{{config}}' + finalize: + steps: + - name: "git:finalize" + cmd: | + git checkout wb/wip_{{version}} + git push origin wb/wip_{{version}} + + gh pr create -f -t "PRETEND RELEASE - release-major: build {{version}}" + + git switch -c "wb/release-{{version}}" + git push origin wb/release-{{version}} + git checkout - +test: + steps: + - name: "foo" + cmd: | + echo "Test" + +promoteToPublic: + create: + steps: + - name: docker:tags + cmd: | + ./tools/update-docker-tags.sh {{tag}} + - name: "git:branch" + cmd: | + branch="wb/promote_wip_{{version}}" + git switch -c "${branch}" + git commit -am 'promote-release: {{version}}' -m '{{config}}' + - name: "git:push" + cmd: git push origin wb/promote_wip_{{version}} + - name: "GitHub:create PR" + cmd: | + gh pr create -f -t "PRETEND PROMOTE RELEASE - release: build {{version}}" From 25031af84ba31577afefc070e772ce8e87b91e7f Mon Sep 17 00:00:00 2001 From: William Bezuidenhout Date: Tue, 16 Jan 2024 14:41:35 +0200 Subject: [PATCH 2/8] update manifest - similar flow to k8s and helm - update buildkite pipeline --- .buildkite/pipeline.yaml | 28 +++++++++++++++++ release.yaml | 65 ++++++++++++++++++++++++++++++---------- 2 files changed, 77 insertions(+), 16 deletions(-) diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index a72a84250..b69d112b4 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -35,3 +35,31 @@ steps: label: ":lock: security - checkov" agents: { queue: "standard" } soft_fail: true + + - label: "Release: test" + if: "build.branch =~ /^wip_/" + command: | + wget https://storage.googleapis.com/buildkite_public_assets/sg-rfc795.tar.gz + tar zxf sg-rfc795.tar.gz + chmod +x ./sg-rfc795 + + ./sg-rfc795 release run test --workdir=. --config-from-commit + + - wait + + - label: "Release: finalize" + if: "build.branch =~ /^wip_/" + command: | + wget https://storage.googleapis.com/buildkite_public_assets/sg-rfc795.tar.gz + tar zxf sg-rfc795.tar.gz + chmod +x ./sg-rfc795 + + ./sg-rfc795 release run internal finalize --workdir=. --config-from-commit + - label: "Promote to public: finalize" + if: build.message =~ /^promote_release/ && build.branch =~ /^wip-release/ + command: | + wget https://storage.googleapis.com/buildkite_public_assets/sg-rfc795.tar.gz + tar zxf sg-rfc795.tar.gz + chmod +x ./sg-rfc795 + + ./sg-rfc795 release run promote-to-public finalize --workdir=. --config-from-commit diff --git a/release.yaml b/release.yaml index 33bd34a74..a322f9744 100644 --- a/release.yaml +++ b/release.yaml @@ -16,35 +16,56 @@ requirements: internal: create: steps: + patch: + - name: docker:tags + cmd: | + # TODO: use {{tag}} + ./tools/update-docker-tags.sh {{tag}} + - name: "git:branch" + cmd: | + branch="wip_{{version}}" + git switch -c "${branch}" + git commit -am 'release_patch: {{version}}' -m '{{config}}' + git push origin ${branch} + - name: "gh" + cmd: | + gh pr create -f -t "PRETEND RELEASE WIP: release_patch: build {{version}}" minor: - name: docker:tags cmd: | + # TODO: use {{tag}} ./tools/update-docker-tags.sh {{tag}} - name: "git:branch" cmd: | - branch="wb/wip_{{version}}" + branch="wip_{{version}}" git switch -c "${branch}" - git commit -am 'release-major: {{version}}' -m '{{config}}' + git commit -am 'release_minor: {{version}}' -m '{{config}}' + git push origin ${branch} + - name: "gh" + cmd: | + gh pr create -f -t "PRETEND RELEASE WIP: release_minor: build {{version}}" major: - name: docker:tags cmd: | ./tools/update-docker-tags.sh {{tag}} - name: "git:branch" cmd: | - branch="wb/wip_{{version}}" + branch="wip_{{version}}" git switch -c "${branch}" - git commit -am 'release-major: {{version}}' -m '{{config}}' + git commit -am 'release_major: {{version}}' -m '{{config}}' + git push origin ${branch} + - name: "gh" + cmd: | + gh pr create -f -t "PRETEND RELEASE WIP: release_major: build {{version}}" finalize: steps: - name: "git:finalize" cmd: | - git checkout wb/wip_{{version}} - git push origin wb/wip_{{version}} - - gh pr create -f -t "PRETEND RELEASE - release-major: build {{version}}" - - git switch -c "wb/release-{{version}}" - git push origin wb/release-{{version}} + set -e + branch="wip-release-{{version}}" + git switch -c "${branch}" + echo "pushing branch ${branch}" + git push origin "${branch}" git checkout - test: steps: @@ -60,11 +81,23 @@ promoteToPublic: ./tools/update-docker-tags.sh {{tag}} - name: "git:branch" cmd: | - branch="wb/promote_wip_{{version}}" + branch="promote-release_{{version}}" git switch -c "${branch}" git commit -am 'promote-release: {{version}}' -m '{{config}}' - - name: "git:push" - cmd: git push origin wb/promote_wip_{{version}} - - name: "GitHub:create PR" + git push origin "${branch}" + - name: "gh" + cmd: | + set -e + branch="wip-release-{{version}}" + # we need to fetch from origin just in case this branch doesn't exist locally, so that the PR can find the base + git fetch origin "${branch}" + gh pr create -f -t "PRETEND PROMOTE RELEASE - release: build {{version}}" --base "${branch}" + finalize: + steps: + - name: git:tag cmd: | - gh pr create -f -t "PRETEND PROMOTE RELEASE - release: build {{version}}" + set -e + branch="wip-release-{{version}}" + git checkout "${branch}" + git tag wip-{{version}} + git push origin ${branch} --tags From d13598a0131dbbf035d9a6573d468f560b124e29 Mon Sep 17 00:00:00 2001 From: William Bezuidenhout Date: Thu, 18 Jan 2024 12:07:10 +0200 Subject: [PATCH 3/8] update release manifest to use sg ops update-images instead --- release.yaml | 46 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/release.yaml b/release.yaml index a322f9744..556f3ab4a 100644 --- a/release.yaml +++ b/release.yaml @@ -17,10 +17,16 @@ internal: create: steps: patch: - - name: docker:tags + - name: docker(compose):tags cmd: | - # TODO: use {{tag}} - ./tools/update-docker-tags.sh {{tag}} + set -e + registry=us-central1-docker.pkg.dev/sourcegraph-ci/rfc795-internal + sg-rfc ops update-images --registry ${registry} --kind compose --pin-tag {{tag}} docker-compose/ + - name: docker(shell):tags + cmd: | + set -e + registry=us-central1-docker.pkg.dev/sourcegraph-ci/rfc795-internal + sg-rfc ops update-images --registry ${registry} --kind shell --pin-tag {{tag}} pure-docker/ - name: "git:branch" cmd: | branch="wip_{{version}}" @@ -31,10 +37,16 @@ internal: cmd: | gh pr create -f -t "PRETEND RELEASE WIP: release_patch: build {{version}}" minor: - - name: docker:tags + - name: docker(compose):tags + cmd: | + set -e + registry=us-central1-docker.pkg.dev/sourcegraph-ci/rfc795-internal + sg-rfc ops update-images --registry ${registry} --kind compose --pin-tag {{tag}} docker-compose/ + - name: docker(shell):tags cmd: | - # TODO: use {{tag}} - ./tools/update-docker-tags.sh {{tag}} + set -e + registry=us-central1-docker.pkg.dev/sourcegraph-ci/rfc795-internal + sg-rfc ops update-images --registry ${registry} --kind shell --pin-tag {{tag}} pure-docker/ - name: "git:branch" cmd: | branch="wip_{{version}}" @@ -45,9 +57,16 @@ internal: cmd: | gh pr create -f -t "PRETEND RELEASE WIP: release_minor: build {{version}}" major: - - name: docker:tags + - name: docker(compose):tags + cmd: | + set -e + registry=us-central1-docker.pkg.dev/sourcegraph-ci/rfc795-internal + sg-rfc ops update-images --registry ${registry} --kind compose --pin-tag {{tag}} docker-compose/ + - name: docker(shell):tags cmd: | - ./tools/update-docker-tags.sh {{tag}} + set -e + registry=us-central1-docker.pkg.dev/sourcegraph-ci/rfc795-internal + sg-rfc ops update-images --registry ${registry} --kind shell --pin-tag {{tag}} pure-docker/ - name: "git:branch" cmd: | branch="wip_{{version}}" @@ -76,9 +95,16 @@ test: promoteToPublic: create: steps: - - name: docker:tags + - name: docker(compose):tags cmd: | - ./tools/update-docker-tags.sh {{tag}} + set -e + registry=us-central1-docker.pkg.dev/sourcegraph-ci/rfc795-public + sg-rfc ops update-images --registry ${registry} --kind compose --pin-tag {{tag}} docker-compose/ + - name: docker(shell):tags + cmd: | + set -e + registry=us-central1-docker.pkg.dev/sourcegraph-ci/rfc795-public + sg-rfc ops update-images --registry ${registry} --kind shell --pin-tag {{tag}} pure-docker/ - name: "git:branch" cmd: | branch="promote-release_{{version}}" From e720afdff745d937e8ca5b161c6025d14a6714dd Mon Sep 17 00:00:00 2001 From: William Bezuidenhout Date: Fri, 19 Jan 2024 11:11:59 +0200 Subject: [PATCH 4/8] use newer ruby --- .tool-versions | 1 + 1 file changed, 1 insertion(+) diff --git a/.tool-versions b/.tool-versions index 479b5702e..c8eb11f65 100644 --- a/.tool-versions +++ b/.tool-versions @@ -3,3 +3,4 @@ yarn 1.22.4 shellcheck 0.7.1 golang 1.19.8 python system +ruby 3.1.1 From 22105e1116198400023a64a2b1a5ef741dd3c554 Mon Sep 17 00:00:00 2001 From: William Bezuidenhout Date: Fri, 19 Jan 2024 12:24:24 +0200 Subject: [PATCH 5/8] Revert "use newer ruby" This reverts commit d65eef91cd5de274c4ffdd3ffc14e86cc3891c06. --- .tool-versions | 1 - 1 file changed, 1 deletion(-) diff --git a/.tool-versions b/.tool-versions index c8eb11f65..479b5702e 100644 --- a/.tool-versions +++ b/.tool-versions @@ -3,4 +3,3 @@ yarn 1.22.4 shellcheck 0.7.1 golang 1.19.8 python system -ruby 3.1.1 From 30b8f468fa580a7c505e3bce645745940389364c Mon Sep 17 00:00:00 2001 From: William Bezuidenhout Date: Fri, 19 Jan 2024 12:47:27 +0200 Subject: [PATCH 6/8] pre-command: write pid file --- .buildkite/hooks/pre-command | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 7b1701cc5..012acec51 100755 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -1,9 +1,31 @@ #!/usr/bin/env bash - +# this script is executed by multiple buildkite agents running on one node, so we write the pid to a file +# if the pidfile exists we know another process is executing so we don't need to install anything +# +# A more robust approach would be to use a lock file, but that would require some more work set -eu pushd "$(dirname "${BASH_SOURCE[0]}")"/../.. WORKDIR=$(pwd) +PIDFILE=/var/lock/dobackup.pid + +remove_pidfile() +{ + rm -f "$PIDFILE" +} + +another_instance() +{ + echo "There is another instance running, skipping" + exit 0 +} + +if [ -f "$PIDFILE" ]; then + kill -0 "$(cat $PIDFILE)" && another_instance +fi +trap remove_pidfile EXIT +echo $$ > "$PIDFILE" + echo "Installing asdf dependencies as defined in '${WORKDIR}/.tool-versions':" asdf install From a85238c3dd3642866c065aa4965acc636993fc80 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Thu, 1 Feb 2024 11:51:56 +0100 Subject: [PATCH 7/8] Include a test-plan in the automated PRs --- release.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/release.yaml b/release.yaml index 556f3ab4a..600f204d0 100644 --- a/release.yaml +++ b/release.yaml @@ -35,7 +35,7 @@ internal: git push origin ${branch} - name: "gh" cmd: | - gh pr create -f -t "PRETEND RELEASE WIP: release_patch: build {{version}}" + gh pr create -f -t "PRETEND RELEASE WIP: release_patch: build {{version}}" --body "Test plan: automated release PR, CI will perform additional checks" minor: - name: docker(compose):tags cmd: | @@ -55,7 +55,7 @@ internal: git push origin ${branch} - name: "gh" cmd: | - gh pr create -f -t "PRETEND RELEASE WIP: release_minor: build {{version}}" + gh pr create -f -t "PRETEND RELEASE WIP: release_minor: build {{version}}" --body "Test plan: automated release PR, CI will perform additional checks" major: - name: docker(compose):tags cmd: | @@ -75,7 +75,7 @@ internal: git push origin ${branch} - name: "gh" cmd: | - gh pr create -f -t "PRETEND RELEASE WIP: release_major: build {{version}}" + gh pr create -f -t "PRETEND RELEASE WIP: release_major: build {{version}}" --body "Test plan: automated release PR, CI will perform additional checks" finalize: steps: - name: "git:finalize" @@ -117,7 +117,7 @@ promoteToPublic: branch="wip-release-{{version}}" # we need to fetch from origin just in case this branch doesn't exist locally, so that the PR can find the base git fetch origin "${branch}" - gh pr create -f -t "PRETEND PROMOTE RELEASE - release: build {{version}}" --base "${branch}" + gh pr create -f -t "PRETEND PROMOTE RELEASE - release: build {{version}}" --base "${branch}" --body "Test plan: automated release PR, CI will perform additional checks" finalize: steps: - name: git:tag From 2b2417fd6486fee515ebade94ddfeed3b5cf0d8f Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Thu, 1 Feb 2024 12:09:32 +0100 Subject: [PATCH 8/8] Homogeize wip branch prefix --- .buildkite/pipeline.yaml | 2 +- release.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index b69d112b4..59f71bc74 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -56,7 +56,7 @@ steps: ./sg-rfc795 release run internal finalize --workdir=. --config-from-commit - label: "Promote to public: finalize" - if: build.message =~ /^promote_release/ && build.branch =~ /^wip-release/ + if: build.message =~ /^promote_release/ && build.branch =~ /^wip_release/ command: | wget https://storage.googleapis.com/buildkite_public_assets/sg-rfc795.tar.gz tar zxf sg-rfc795.tar.gz diff --git a/release.yaml b/release.yaml index 600f204d0..1602ade71 100644 --- a/release.yaml +++ b/release.yaml @@ -81,7 +81,7 @@ internal: - name: "git:finalize" cmd: | set -e - branch="wip-release-{{version}}" + branch="wip_release-{{version}}" git switch -c "${branch}" echo "pushing branch ${branch}" git push origin "${branch}" @@ -114,7 +114,7 @@ promoteToPublic: - name: "gh" cmd: | set -e - branch="wip-release-{{version}}" + branch="wip_release-{{version}}" # we need to fetch from origin just in case this branch doesn't exist locally, so that the PR can find the base git fetch origin "${branch}" gh pr create -f -t "PRETEND PROMOTE RELEASE - release: build {{version}}" --base "${branch}" --body "Test plan: automated release PR, CI will perform additional checks" @@ -123,7 +123,7 @@ promoteToPublic: - name: git:tag cmd: | set -e - branch="wip-release-{{version}}" + branch="wip_release-{{version}}" git checkout "${branch}" - git tag wip-{{version}} + git tag wip_{{version}} git push origin ${branch} --tags