pipeline(precompile): detect blobstore related errors

To avoid corrupted compiled releases use, we need to check errors occurring during precompile. Once a related error is detected, we delete existing deployment and release, without force option. Also: - force tty to display boshrelease download - enable pipefail as we use tee to log and display bosh export-release result
orange-cloudfoundry · Aug 7, 2023 · 46f61cd · 46f61cd
1 parent 80c68ee
commit 46f61cd
Show file tree

Hide file tree

Showing 2 changed files with 51 additions and 3 deletions.
diff --git a/concourse/pipelines/template/bosh-precompile-pipeline.yml.erb b/concourse/pipelines/template/bosh-precompile-pipeline.yml.erb
@@ -526,11 +526,27 @@ jobs:
             args:
               - -ec
               - |
+                set -o pipefail
                 STEMCELL_INFO=$(bosh inspect-local-stemcell --json stemcell/*.tgz | jq '.Tables|.[0]|.Rows|.[0]')
                 export STEMCELL_OS=$(echo ${STEMCELL_INFO} | jq -r '.["os"]')
                 export STEMCELL_VERSION=$(echo ${STEMCELL_INFO} | jq -r '.["version"]')
                 echo "Exporting ${RELEASE_NAME}/${RELEASE_VERSION} - ${STEMCELL_OS}/${STEMCELL_VERSION}"
-                bosh export-release "${RELEASE_NAME}/${RELEASE_VERSION}" "${STEMCELL_OS}/${STEMCELL_VERSION}" --dir=exported-release
+                TIMESTAMP=$(date +'%Y-%m-%d-%H-%M-%S')
+                if ! bosh export-release "${RELEASE_NAME}/${RELEASE_VERSION}" "${STEMCELL_OS}/${STEMCELL_VERSION}" --dir=exported-release --tty|tee exported-release/${RELEASE_NAME}-${RELEASE_VERSION}-${TIMESTAMP}.log;then
+                  BLOBSTORE_ERROR_COUNT=$(grep "blobstore" exported-release/${RELEASE_NAME}-${RELEASE_VERSION}-${TIMESTAMP}.log |wc -l)
+                  if [ $BLOBSTORE_ERROR_COUNT -gt 0 ];then
+                    echo "COA - Blobstore error detected, cleaning deployment and release"
+                    bosh delete-deployment --non-interactive
+                    bosh delete-release ${RELEASE_NAME}/${RELEASE_VERSION} --non-interactive
+                  else
+                   echo "" # new line
+                   echo "COA - No blobstore error detected, retry without cleaning."
+                  fi
+                  exit 1
+                else
+                  echo "" # new line
+                  echo "COA - Bosh export-release successful"
+                fi
           params:
             BOSH_DEPLOYMENT: <%= release %>-deployment
             BOSH_ENVIRONMENT: ((bosh-target))

diff --git a/spec/scripts/generate-depls/fixtures/references/simple-depls-bosh-precompile-ref.yml b/spec/scripts/generate-depls/fixtures/references/simple-depls-bosh-precompile-ref.yml
@@ -339,11 +339,27 @@ jobs:
             args:
               - -ec
               - |
+                set -o pipefail
                 STEMCELL_INFO=$(bosh inspect-local-stemcell --json stemcell/*.tgz | jq '.Tables|.[0]|.Rows|.[0]')
                 export STEMCELL_OS=$(echo ${STEMCELL_INFO} | jq -r '.["os"]')
                 export STEMCELL_VERSION=$(echo ${STEMCELL_INFO} | jq -r '.["version"]')
                 echo "Exporting ${RELEASE_NAME}/${RELEASE_VERSION} - ${STEMCELL_OS}/${STEMCELL_VERSION}"
-                bosh export-release "${RELEASE_NAME}/${RELEASE_VERSION}" "${STEMCELL_OS}/${STEMCELL_VERSION}" --dir=exported-release
+                TIMESTAMP=$(date +'%Y-%m-%d-%H-%M-%S')
+                if ! bosh export-release "${RELEASE_NAME}/${RELEASE_VERSION}" "${STEMCELL_OS}/${STEMCELL_VERSION}" --dir=exported-release --tty|tee exported-release/${RELEASE_NAME}-${RELEASE_VERSION}-${TIMESTAMP}.log;then
+                  BLOBSTORE_ERROR_COUNT=$(grep "blobstore" exported-release/${RELEASE_NAME}-${RELEASE_VERSION}-${TIMESTAMP}.log |wc -l)
+                  if [ $BLOBSTORE_ERROR_COUNT -gt 0 ];then
+                    echo "COA - Blobstore error detected, cleaning deployment and release"
+                    bosh delete-deployment --non-interactive
+                    bosh delete-release ${RELEASE_NAME}/${RELEASE_VERSION} --non-interactive
+                  else
+                   echo "" # new line
+                   echo "COA - No blobstore error detected, retry without cleaning."
+                  fi
+                  exit 1
+                else
+                  echo "" # new line
+                  echo "COA - Bosh export-release successful"
+                fi
           params:
             BOSH_DEPLOYMENT: ntp_boshrelease-deployment
             BOSH_ENVIRONMENT: ((bosh-target))
@@ -454,11 +470,27 @@ jobs:
             args:
               - -ec
               - |
+                set -o pipefail
                 STEMCELL_INFO=$(bosh inspect-local-stemcell --json stemcell/*.tgz | jq '.Tables|.[0]|.Rows|.[0]')
                 export STEMCELL_OS=$(echo ${STEMCELL_INFO} | jq -r '.["os"]')
                 export STEMCELL_VERSION=$(echo ${STEMCELL_INFO} | jq -r '.["version"]')
                 echo "Exporting ${RELEASE_NAME}/${RELEASE_VERSION} - ${STEMCELL_OS}/${STEMCELL_VERSION}"
-                bosh export-release "${RELEASE_NAME}/${RELEASE_VERSION}" "${STEMCELL_OS}/${STEMCELL_VERSION}" --dir=exported-release
+                TIMESTAMP=$(date +'%Y-%m-%d-%H-%M-%S')
+                if ! bosh export-release "${RELEASE_NAME}/${RELEASE_VERSION}" "${STEMCELL_OS}/${STEMCELL_VERSION}" --dir=exported-release --tty|tee exported-release/${RELEASE_NAME}-${RELEASE_VERSION}-${TIMESTAMP}.log;then
+                  BLOBSTORE_ERROR_COUNT=$(grep "blobstore" exported-release/${RELEASE_NAME}-${RELEASE_VERSION}-${TIMESTAMP}.log |wc -l)
+                  if [ $BLOBSTORE_ERROR_COUNT -gt 0 ];then
+                    echo "COA - Blobstore error detected, cleaning deployment and release"
+                    bosh delete-deployment --non-interactive
+                    bosh delete-release ${RELEASE_NAME}/${RELEASE_VERSION} --non-interactive
+                  else
+                   echo "" # new line
+                   echo "COA - No blobstore error detected, retry without cleaning."
+                  fi
+                  exit 1
+                else
+                  echo "" # new line
+                  echo "COA - Bosh export-release successful"
+                fi
           params:
             BOSH_DEPLOYMENT: zookeeper_boshrelease-deployment
             BOSH_ENVIRONMENT: ((bosh-target))