From 1754737538f197092fbebbace7a4c943d4c589f1 Mon Sep 17 00:00:00 2001 From: Neil MacDougall Date: Fri, 26 Apr 2019 13:55:22 +0100 Subject: [PATCH 01/11] Check release does not exist before creating --- deploy/ci/console-dev-releases.yml | 25 ++++++++++- .../tasks/dev-releases/check-gh-release.yml | 45 +++++++++++++++++++ .../tasks/dev-releases/generate-tag-files.yml | 6 +-- deploy/ci/tasks/dev-releases/make-release.yml | 10 +++++ 4 files changed, 81 insertions(+), 5 deletions(-) create mode 100644 deploy/ci/tasks/dev-releases/check-gh-release.yml diff --git a/deploy/ci/console-dev-releases.yml b/deploy/ci/console-dev-releases.yml index 5feec9e486..ef7e39e757 100644 --- a/deploy/ci/console-dev-releases.yml +++ b/deploy/ci/console-dev-releases.yml @@ -81,13 +81,34 @@ jobs: params: file: image-tag/*.tar acl: public-read -- name: build-images +- name: check-github plan: - get: stratos passed: [generate-tag-files] trigger: true - get: image-tag - passed: [generate-tag-files] + passed: [generate-tag-files] + params: + unpack: true + - do: + - task: build + privileged: true + timeout: 30m + file: stratos/deploy/ci/tasks/dev-releases/check-gh-release.yml + params: + GITHUB_TOKEN: ((github-access-token)) + GITHUB_USER: ((helm-repo-github-organization)) + GITHUB_REPO: ((helm-repo-github-repository)) + GIT_USER: ((concourse-user)) + GIT_EMAIL: ((concourse-email)) + GIT_PRIVATE_KEY: ((github-private-key)) +- name: build-images + plan: + - get: stratos + passed: [check-github] + trigger: true + - get: image-tag + passed: [check-github] params: unpack: true - aggregate: diff --git a/deploy/ci/tasks/dev-releases/check-gh-release.yml b/deploy/ci/tasks/dev-releases/check-gh-release.yml new file mode 100644 index 0000000000..84529c8419 --- /dev/null +++ b/deploy/ci/tasks/dev-releases/check-gh-release.yml @@ -0,0 +1,45 @@ +--- +platform: linux +inputs: +- name: stratos +- name: image-tag +image_resource: + type: docker-image + source: + # Generated using scripts/Dockerfile.stratos-ci + repository: splatform/stratos-ci-concourse + tag: "latest" + +run: + path: sh + args: + - -exc + - | + # Check that the Github release DOES NOT exist + ROOT_DIR=${PWD} + VERSION=$(cat image-tag/v2-version) + FULL_VERSION=$(cat image-tag/v2-alpha-tag) + GIT_TAG=$(cat image-tag/v2-tag) + STRATOS=${ROOT_DIR}/stratos + source ${STRATOS}/deploy/ci/tasks/dev-releases/github-helper.sh + + # Check tagged release version is consistent with package.json version + TAG_RELEASE_VERSION=$(echo ${FULL_VERSION} | cut -d"-" -f1) + if [ "${TAG_RELEASE_VERSION}" != "${VERSION}" ]; then + echo "Package.json version is not consistent with tag release version! ${TAG_RELEASE_VERSION} != ${VERSION}" + exit 1 + fi + cd ${ROOT_DIR}/stratos + + # Check that the release exists + set +e + github-release info -t ${GIT_TAG} + RETVAL=$? + set -e + + if [ $RETVAL -eq 0 ]; then + echo "Aborting... release already exists" + exit 1 + fi + + echo "Release does not exist.. OK" diff --git a/deploy/ci/tasks/dev-releases/generate-tag-files.yml b/deploy/ci/tasks/dev-releases/generate-tag-files.yml index f79c7ca43a..7e3876cc8e 100644 --- a/deploy/ci/tasks/dev-releases/generate-tag-files.yml +++ b/deploy/ci/tasks/dev-releases/generate-tag-files.yml @@ -63,13 +63,13 @@ run: echo "Created v2-alpha-tag, v2-version and build-args." - echo "Contens of build-args:" + echo "Contents of build-args:" cat build-args - echo "Contens of ui-build-args:" + echo "Contents of ui-build-args:" cat ui-build-args - echo "Contens of v2-alpha-tag:" + echo "Contents of v2-alpha-tag:" cat v2-alpha-tag echo "Creating tag file tar..." diff --git a/deploy/ci/tasks/dev-releases/make-release.yml b/deploy/ci/tasks/dev-releases/make-release.yml index 6568df36fc..3e54629dd5 100644 --- a/deploy/ci/tasks/dev-releases/make-release.yml +++ b/deploy/ci/tasks/dev-releases/make-release.yml @@ -51,6 +51,16 @@ run: echo "Making release" + # Check to see if the release tag already exists - if so, abort - we won't re-release + set +e + github-release info -t ${VERSION} + RETVAL=$? + set -e + if [ $RETVAL -eq 0 ]; then + echo "Release already exists... aborting" + exit 1 + fi + # Delete Stable release on GitHub and recreate it again # This ensures it has the correct date/time and appears in the correct place in the release history From fbfe466864e7f0889085a70309c4b4770209d95e Mon Sep 17 00:00:00 2001 From: Neil MacDougall Date: Fri, 26 Apr 2019 14:56:08 +0100 Subject: [PATCH 02/11] Improvements --- deploy/ci/console-make-release.yml | 2 ++ deploy/ci/tasks/dev-releases/make-release.yml | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/deploy/ci/console-make-release.yml b/deploy/ci/console-make-release.yml index 5aa0a7d62b..e4fe1e8a39 100644 --- a/deploy/ci/console-make-release.yml +++ b/deploy/ci/console-make-release.yml @@ -73,4 +73,6 @@ jobs: DOCKER_REGISTRY: ((docker-registry)) HELM_REPO_PATH: ((helm-repo-path)) HELM_REPO_BRANCH: ((helm-repo-branch)) + SOURCE_REPO_ORG: ((stratos-repository-organization)) + SOURCE_REPO_REPO: ((stratos-repository)) diff --git a/deploy/ci/tasks/dev-releases/make-release.yml b/deploy/ci/tasks/dev-releases/make-release.yml index 3e54629dd5..3d8b807bc0 100644 --- a/deploy/ci/tasks/dev-releases/make-release.yml +++ b/deploy/ci/tasks/dev-releases/make-release.yml @@ -21,6 +21,9 @@ run: STRATOS=${ROOT_DIR}/stratos source ${STRATOS}/deploy/ci/tasks/dev-releases/create-chart-helper.sh source ${STRATOS}/deploy/ci/tasks/dev-releases/github-helper.sh + + SOURCE_REPO=${SOURCE_REPO_ORG}/${SOURCE_REPO_REPO} + ARTIFACT_REPO=${GITHUB_ORG}/${GITHUB_REPO} HELM_REPO=${ROOT_DIR}/helm-repo/${HELM_REPO_PATH} GIT_TAG=$(cat image-tag/v2-alpha-tag) @@ -143,6 +146,13 @@ run: # Upload Helm chart to release github-release upload -t ${VERSION} --file ./tmp/${HELM_FILE} --name "${HELM_FILE}" + # Create a release if the source repo is not the same as the helm repo + if [ "${SOURCE_REPO}" != "${ARTIFACT_REPO}" ]; then + echo "Creating release in GitHub Repository: ${SOURCE_REPO}" + # Create new release in the source repository + github-release release --user ${SOURCE_REPO_ORG} --repo ${SOURCE_REPO_REPO} -t ${VERSION} --name "${RELEASE_TITLE}" ${PRE_RELEASE_ARG} --description "${RELEASE_DESCRIPTION}" + fi + set +e echo "RELEASE COMPLETE" From 4432da1bc7f52e2f33d4b35fdb55e3ab282020ec Mon Sep 17 00:00:00 2001 From: Neil MacDougall Date: Fri, 26 Apr 2019 16:04:57 +0100 Subject: [PATCH 03/11] More tweaks --- deploy/ci/console-make-release.yml | 6 ++++- deploy/ci/tasks/dev-releases/make-release.yml | 23 ++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/deploy/ci/console-make-release.yml b/deploy/ci/console-make-release.yml index e4fe1e8a39..725851cc66 100644 --- a/deploy/ci/console-make-release.yml +++ b/deploy/ci/console-make-release.yml @@ -75,4 +75,8 @@ jobs: HELM_REPO_BRANCH: ((helm-repo-branch)) SOURCE_REPO_ORG: ((stratos-repository-organization)) SOURCE_REPO_REPO: ((stratos-repository)) - + HELM_MIRROR_BASE_URL: ((web-server-base-url)) + HELM_RELEASE_GITCLONE: ((release-helm-repo)) + HELM_RELEASE_REPO_NAME: ((release-helm-repo-name)) + HELM_RELEASE_REPO_BRANCH: ((release-helm-repo-branch)) + HELM_RELEASE_REPO_FOLDER: ((release-helm-stable-folder)) diff --git a/deploy/ci/tasks/dev-releases/make-release.yml b/deploy/ci/tasks/dev-releases/make-release.yml index 3d8b807bc0..31472c79b9 100644 --- a/deploy/ci/tasks/dev-releases/make-release.yml +++ b/deploy/ci/tasks/dev-releases/make-release.yml @@ -4,6 +4,8 @@ inputs: - name: stratos - name: image-tag - name: helm-repo +outputs: +- name: helm-chart image_resource: type: docker-image source: @@ -100,6 +102,9 @@ run: HELM_FILE=console-helm-chart-${VERSION}-${COMMIT}.tgz helm package console cp console*.tgz ${ROOT_DIR}/helm-chart/${HELM_FILE} + + # Temp folder for helm-chart + mkdir -p ${ROOT_DIR}/helm-chart # Add helm chart to the helm repository cd ${ROOT_DIR}/helm-chart/ @@ -154,10 +159,26 @@ run: fi set +e + set +x - echo "RELEASE COMPLETE" + echo "==================================================================================================================" + echo "" + echo "RELEASE COMPLETE =================================================================================================" + echo "" echo "Version : ${VERSION}" echo "Base Version : ${LAST_TAG}" echo "" echo "Helm Chart filename : ${HELM_FILE}" echo "Helm Chart URL : https://github.com/${HELM_REPOSITORY}/releases/download/${VERSION}/${HELM_FILE}" + echo "" + echo "==================================================================================================================" + + # If we have a mirror URL for the Helm Chart then output that at the end + if [ ! -z "${HELM_MIRROR_BASE_URL}" ]; then + echo "" + echo "==================================================================================================================" + echo "Helm Chart will be mirrored to:" + echo "${HELM_MIRROR_BASE_URL}${HELM_FILE}" + echo "==================================================================================================================" + fi + From a5fe56cc86c1abb460ad7507108169355e285433 Mon Sep 17 00:00:00 2001 From: Neil MacDougall Date: Fri, 26 Apr 2019 16:30:30 +0100 Subject: [PATCH 04/11] Fix upload of helm chart --- deploy/ci/tasks/dev-releases/make-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/ci/tasks/dev-releases/make-release.yml b/deploy/ci/tasks/dev-releases/make-release.yml index 31472c79b9..65fc13b6cc 100644 --- a/deploy/ci/tasks/dev-releases/make-release.yml +++ b/deploy/ci/tasks/dev-releases/make-release.yml @@ -149,7 +149,7 @@ run: github-release release -t ${VERSION} --name "${RELEASE_TITLE}" ${PRE_RELEASE_ARG} --description "${RELEASE_DESCRIPTION}" # Upload Helm chart to release - github-release upload -t ${VERSION} --file ./tmp/${HELM_FILE} --name "${HELM_FILE}" + github-release upload -t ${VERSION} --file ${ROOT_DIR}/helm-chart/${HELM_FILE} --name "${HELM_FILE}" # Create a release if the source repo is not the same as the helm repo if [ "${SOURCE_REPO}" != "${ARTIFACT_REPO}" ]; then From 8ac0ca0d7d192e3b9898d48b9d5f9c0a60916daa Mon Sep 17 00:00:00 2001 From: Neil MacDougall Date: Fri, 26 Apr 2019 16:33:44 +0100 Subject: [PATCH 05/11] Don't log key --- deploy/ci/tasks/dev-releases/make-release.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/deploy/ci/tasks/dev-releases/make-release.yml b/deploy/ci/tasks/dev-releases/make-release.yml index 65fc13b6cc..69884e313c 100644 --- a/deploy/ci/tasks/dev-releases/make-release.yml +++ b/deploy/ci/tasks/dev-releases/make-release.yml @@ -37,8 +37,10 @@ run: cd ${ROOT_DIR}/stratos - # Set git username and password so that we can push changes + # Set git username and password so that we can push changes (Don't log) + set +x setupGitConfig + set -x # $VERSION is version number - e.g. 2.3.0 # Find the last tag created for this release tag @@ -117,7 +119,9 @@ run: helm repo index ./ ${MERGE_INDEX} --url https://github.com/${GITHUB_ORG}/${GITHUB_REPO}/releases/download/${RELEASE_VERSION}/ cp index.yaml ${HELM_REPO} cd ${HELM_REPO} + set +x setupAndPushChange + set -x # Tag the repo and create the release on GitHub cd ${ROOT_DIR}/stratos From 94ad6b4f89a8ba55a8f74a461a81cf6b33bd10ae Mon Sep 17 00:00:00 2001 From: Neil MacDougall Date: Fri, 26 Apr 2019 17:14:31 +0100 Subject: [PATCH 06/11] Add helm chart PR creation --- deploy/ci/tasks/dev-releases/make-release.yml | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/deploy/ci/tasks/dev-releases/make-release.yml b/deploy/ci/tasks/dev-releases/make-release.yml index 69884e313c..56ef73b228 100644 --- a/deploy/ci/tasks/dev-releases/make-release.yml +++ b/deploy/ci/tasks/dev-releases/make-release.yml @@ -162,6 +162,32 @@ run: github-release release --user ${SOURCE_REPO_ORG} --repo ${SOURCE_REPO_REPO} -t ${VERSION} --name "${RELEASE_TITLE}" ${PRE_RELEASE_ARG} --description "${RELEASE_DESCRIPTION}" fi + # Create the PR for the Helm Chart changes if configured + if [ ! -z "${HELM_RELEASE_GITCLONE}" ]; then + echo "Creating PR for Helm Chart Changes for this release" + + mkdir -p ${ROOT_DIR}/helm-pr + cd ${ROOT_DIR}/helm-pr + git clone ${HELM_RELEASE_GITCLONE} helm-git-repo + cd helm-git-repo + git checkout ${HELM_RELEASE_REPO_BRANCH} + cd ${HELM_RELEASE_REPO_FOLDER} + cp ${ROOT_DIR}/helm-chart/${HELM_FILE} . + tar -xvf ${HELM_FILE} + rm -rf ./${HELM_FILE} + git add -A + git status + git commit -m "Stratos Release ${VERSION} Helm Chart Changes" + git push -u origin ${HELM_RELEASE_REPO_BRANCH} + + # Now create a PR for the changes + URL=https://api.github.com/repos/${HELM_RELEASE_REPO_NAME}/pulls + TITLE="Stratos Release ${VERSION}" + + curl --fail -H "Content-Type: application/json" -H "Authorization: token ${GITHUB_TOKEN}" \ + -d '{"title":"'"${TITLE}"'", "head": "'"${GIT_BRANCH}"'", "base": "'"${HELM_RELEASE_REPO_BRANCH}"'"}' ${URL} + fi + set +e set +x From 5f37e3863f9beb8c00b758a4e19765de391746ce Mon Sep 17 00:00:00 2001 From: Neil MacDougall Date: Fri, 26 Apr 2019 17:23:10 +0100 Subject: [PATCH 07/11] Last fix --- deploy/ci/tasks/dev-releases/make-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/ci/tasks/dev-releases/make-release.yml b/deploy/ci/tasks/dev-releases/make-release.yml index 56ef73b228..ca89c6fa8b 100644 --- a/deploy/ci/tasks/dev-releases/make-release.yml +++ b/deploy/ci/tasks/dev-releases/make-release.yml @@ -185,7 +185,7 @@ run: TITLE="Stratos Release ${VERSION}" curl --fail -H "Content-Type: application/json" -H "Authorization: token ${GITHUB_TOKEN}" \ - -d '{"title":"'"${TITLE}"'", "head": "'"${GIT_BRANCH}"'", "base": "'"${HELM_RELEASE_REPO_BRANCH}"'"}' ${URL} + -d '{"title":"'"${TITLE}"'", "head": "'"${HELM_RELEASE_REPO_BRANCH}"'", "base": "'"${HELM_RELEASE_REPO_BRANCH}"'"}' ${URL} fi set +e From 87dc3cd5d052a2ac324bd0cfe94b8243263ecefd Mon Sep 17 00:00:00 2001 From: Neil MacDougall Date: Fri, 26 Apr 2019 17:28:16 +0100 Subject: [PATCH 08/11] Ensure PR uses new branch --- deploy/ci/tasks/dev-releases/make-release.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/deploy/ci/tasks/dev-releases/make-release.yml b/deploy/ci/tasks/dev-releases/make-release.yml index ca89c6fa8b..39d48a50fb 100644 --- a/deploy/ci/tasks/dev-releases/make-release.yml +++ b/deploy/ci/tasks/dev-releases/make-release.yml @@ -166,11 +166,14 @@ run: if [ ! -z "${HELM_RELEASE_GITCLONE}" ]; then echo "Creating PR for Helm Chart Changes for this release" + NEW_BRANCH="stratos-${VERSION}" + mkdir -p ${ROOT_DIR}/helm-pr cd ${ROOT_DIR}/helm-pr git clone ${HELM_RELEASE_GITCLONE} helm-git-repo cd helm-git-repo git checkout ${HELM_RELEASE_REPO_BRANCH} + git checkout -b ${NEW_BRANCH} cd ${HELM_RELEASE_REPO_FOLDER} cp ${ROOT_DIR}/helm-chart/${HELM_FILE} . tar -xvf ${HELM_FILE} @@ -178,14 +181,14 @@ run: git add -A git status git commit -m "Stratos Release ${VERSION} Helm Chart Changes" - git push -u origin ${HELM_RELEASE_REPO_BRANCH} + git push -u origin ${NEW_BRANCH} # Now create a PR for the changes URL=https://api.github.com/repos/${HELM_RELEASE_REPO_NAME}/pulls TITLE="Stratos Release ${VERSION}" curl --fail -H "Content-Type: application/json" -H "Authorization: token ${GITHUB_TOKEN}" \ - -d '{"title":"'"${TITLE}"'", "head": "'"${HELM_RELEASE_REPO_BRANCH}"'", "base": "'"${HELM_RELEASE_REPO_BRANCH}"'"}' ${URL} + -d '{"title":"'"${TITLE}"'", "head": "'"${NEW_BRANCH}"'", "base": "'"${HELM_RELEASE_REPO_BRANCH}"'"}' ${URL} fi set +e From 20f1ce35260c81354a1ea5fe0eedff4b1d6a6063 Mon Sep 17 00:00:00 2001 From: Neil MacDougall Date: Fri, 26 Apr 2019 17:40:11 +0100 Subject: [PATCH 09/11] Final change --- deploy/ci/tasks/dev-releases/make-release.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/deploy/ci/tasks/dev-releases/make-release.yml b/deploy/ci/tasks/dev-releases/make-release.yml index 39d48a50fb..6888afba8d 100644 --- a/deploy/ci/tasks/dev-releases/make-release.yml +++ b/deploy/ci/tasks/dev-releases/make-release.yml @@ -177,6 +177,17 @@ run: cd ${HELM_RELEASE_REPO_FOLDER} cp ${ROOT_DIR}/helm-chart/${HELM_FILE} . tar -xvf ${HELM_FILE} + + # Patch the registry hostname and org if needed + if [ ! -z "${HELM_RELEASE_REGISTRY_HOST}" ]; then + echo "Patching Helm Chart Docker Registry and Organisation" + cat ./console/values.yaml + sed -i -e 's/hostname: '"${DOCKER_REG}"'/hostname: '"${HELM_RELEASE_REGISTRY_HOST}"'/g' ./console/values.yaml + sed -i -e 's/organization: '"${DOCKER_ORG}"'/organization: '"${HELM_RELEASE_REGISTRY_ORG}"'/g' ./console/values.yaml + echo "Updated Chart..." + cat ./console/values.yaml + fi + rm -rf ./${HELM_FILE} git add -A git status From 5b36c4c8e2d25c932f32f47d3d83cf4286097fa6 Mon Sep 17 00:00:00 2001 From: Neil MacDougall Date: Fri, 26 Apr 2019 19:08:28 +0100 Subject: [PATCH 10/11] Fix registry change --- deploy/ci/tasks/dev-releases/make-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/ci/tasks/dev-releases/make-release.yml b/deploy/ci/tasks/dev-releases/make-release.yml index 6888afba8d..5038d47954 100644 --- a/deploy/ci/tasks/dev-releases/make-release.yml +++ b/deploy/ci/tasks/dev-releases/make-release.yml @@ -182,7 +182,7 @@ run: if [ ! -z "${HELM_RELEASE_REGISTRY_HOST}" ]; then echo "Patching Helm Chart Docker Registry and Organisation" cat ./console/values.yaml - sed -i -e 's/hostname: '"${DOCKER_REG}"'/hostname: '"${HELM_RELEASE_REGISTRY_HOST}"'/g' ./console/values.yaml + sed -i -e 's/hostname: '"${DOCKER_REGISTRY}"'/hostname: '"${HELM_RELEASE_REGISTRY_HOST}"'/g' ./console/values.yaml sed -i -e 's/organization: '"${DOCKER_ORG}"'/organization: '"${HELM_RELEASE_REGISTRY_ORG}"'/g' ./console/values.yaml echo "Updated Chart..." cat ./console/values.yaml From f941efca980af9a31469be7b28b822bb4d5d687e Mon Sep 17 00:00:00 2001 From: Neil MacDougall Date: Fri, 26 Apr 2019 19:09:57 +0100 Subject: [PATCH 11/11] Ensure we have docker registry variables --- deploy/ci/console-make-release.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/ci/console-make-release.yml b/deploy/ci/console-make-release.yml index 725851cc66..bc86588195 100644 --- a/deploy/ci/console-make-release.yml +++ b/deploy/ci/console-make-release.yml @@ -68,7 +68,6 @@ jobs: GITHUB_REPO: ((helm-repo-github-repository)) GITHUB_TOKEN: ((github-access-token)) GIT_PRIVATE_KEY: ((github-private-key)) - GITHUB_TOKEN: ((github-access-token)) DOCKER_ORG: ((docker-organization)) DOCKER_REGISTRY: ((docker-registry)) HELM_REPO_PATH: ((helm-repo-path)) @@ -80,3 +79,5 @@ jobs: HELM_RELEASE_REPO_NAME: ((release-helm-repo-name)) HELM_RELEASE_REPO_BRANCH: ((release-helm-repo-branch)) HELM_RELEASE_REPO_FOLDER: ((release-helm-stable-folder)) + HELM_RELEASE_REGISTRY_HOST: ((release-repository)) + HELM_RELEASE_REGISTRY_ORG: ((release-repository-organization))