cloudfoundry-community · drnic · Nov 12, 2018 · Mar 30, 2018
diff --git a/boshrelease/pipeline.yml b/boshrelease/pipeline.yml
@@ -70,6 +70,8 @@ groups:
       - testflight-pr
       - rc
       - shipit
+      - compile-release-3541
+      - use-compiled-releases
   - name: versioning
     jobs:
       - major
@@ -340,7 +342,7 @@ jobs:
         put: git
         params:
           rebase: true
-          repository: pushme/git
+          repository: pushme
       - name: tarball
         put: s3-tarball
         params:
@@ -372,6 +374,84 @@ jobs:
           icon_url: (( grab meta.slack.icon ))
           text:    '(( concat meta.slack.fail_url " " meta.pipeline ": shipit job failed" ))'
 
+  - name: compile-release-3541
+    plan:
+      - aggregate:
+        - get: git
+        - get: github
+          trigger: true
+        - get: ubuntu-trusty-stemcell-3541
+          params: {tarball: false}
+      - task: export-release
+        config:
+          platform: linux
+          image_resource:
+            type: docker-image
+            source:
+              repository: (( grab meta.image.name ))
+              tag:        (( grab meta.image.tag ))
+          inputs:
+            - name: git
+            - name: ubuntu-trusty-stemcell-3541
+              path: stemcell
+            - name: github
+              path: release
+          outputs:
+            - name: compiled-release
+          run:
+            path: git/ci/scripts/export-release
+          params:
+            BOSH_ENVIRONMENT:     (( grab meta.bosh-lite.target ))
+            BOSH_CA_CERT:         (( grab meta.bosh-lite.cacert ))
+            BOSH_CLIENT:          (( grab meta.bosh-lite.username ))
+            BOSH_CLIENT_SECRET:   (( grab meta.bosh-lite.password ))
+      - put: compiled-releases
+        params:
+          file: "compiled-release/compiled-releases/*.tgz"
+
+  - name: use-compiled-releases
+    plan:
+      - aggregate:
+        - get: git
+        - get: github
+          passed: [compile-release-3541]
+          trigger: true
+        - get: ubuntu-trusty-stemcell-3541
+          passed: [compile-release-3541]
+          params: {tarball: false}
+        - get: compiled-releases
+          passed: [compile-release-3541]
+      - task: use-compiled-releases
+        config:
+          platform: linux
+          image_resource:
+            type: docker-image
+            source:
+              repository: (( grab meta.image.name ))
+              tag:        (( grab meta.image.tag ))
+          inputs:
+            - name: git
+            - name: ubuntu-trusty-stemcell-3541
+              path: stemcell
+            - name: github
+              path: release
+            - name: compiled-releases
+          outputs:
+            - name: pushme
+          run:
+            path: git/ci/scripts/use-compiled-releases
+          params:
+            REPO_ROOT: git
+            REPO_OUT:  pushme
+            GIT_EMAIL: (( grab meta.git.email ))
+            GIT_NAME:  (( grab meta.git.name ))
+            BRANCH:    (( grab meta.github.branch ))
+      - name: upload-git
+        put: git
+        params:
+          rebase: true
+          repository: pushme
+
 resource_types:
   - name: slack-notification
     type: docker-image
@@ -431,3 +511,18 @@ resources:
       regexp:            (( concat meta.name "-(.*).tgz" ))
       access_key_id:     (( grab meta.aws.access_key ))
       secret_access_key: (( grab meta.aws.secret_key ))
+
+  - name: ubuntu-trusty-stemcell-3541
+    type: bosh-io-stemcell
+    source:
+      name: bosh-warden-boshlite-ubuntu-trusty-go_agent
+      version_family: 3541.latest
+
+  - name: compiled-releases
+    type: s3
+    source:
+      bucket:            (( grab meta.aws.bucket ))
+      region_name:       (( grab meta.aws.region_name ))
+      access_key_id:     (( grab meta.aws.access_key ))
+      secret_access_key: (( grab meta.aws.secret_key ))
+      regexp:            "compiled-releases/.*-(\\d+).tgz"
diff --git a/boshrelease/scripts/bump-release → boshrelease/scripts/bump-dependent-release b/boshrelease/scripts/bump-release → boshrelease/scripts/bump-dependent-release
@@ -41,7 +41,7 @@ version=$(cat ${RELEASE}/version)
 function bump_version {
   manifest_path=$1
   releases_updated=$(spruce merge <<YAML
-releases: $(spruce json $manifest_path | jq --arg name $NAME --arg version $version '.releases | map(
+releases: $(spruce json "$manifest_path" | jq --arg name $NAME --arg version $version '.releases | map(
     if .name == $name
     then . + {"version":$version}
     else .
@@ -50,7 +50,6 @@ releases: $(spruce json $manifest_path | jq --arg name $NAME --arg version $vers
 YAML
 )
   releases_line_number=$(awk '/^releases:/{ print NR; exit }' $manifest_path)
-  manifest_len=$(wc -l $manifest_path | awk '{print $1}')
   manifest_head=$(head -n `expr $releases_line_number - 1` $manifest_path)
   cat > $manifest_path <<YAML
 ${manifest_head}
@@ -59,13 +58,6 @@ ${releases_updated}
 YAML
 }
 
-if [[ -z $(git config --global user.email) ]]; then
-  git config --global user.email "${GIT_EMAIL}"
-fi
-if [[ -z $(git config --global user.name) ]]; then
-  git config --global user.name "${GIT_NAME}"
-fi
-
 cd ${REPO_OUT}
 for manifest_path in $(ls manifests/*.yml); do
   bump_version $manifest_path
@@ -75,6 +67,14 @@ cat <<EOF >>ci/release_notes.md
 * Bumped ${NAME} to v${version}
 EOF
 
+
+if [[ -z $(git config --global user.email) ]]; then
+  git config --global user.email "${GIT_EMAIL}"
+fi
+if [[ -z $(git config --global user.name) ]]; then
+  git config --global user.name "${GIT_NAME}"
+fi
+
 git merge --no-edit ${BRANCH}
 git add -A
 git status

diff --git a/boshrelease/scripts/export-release b/boshrelease/scripts/export-release
@@ -0,0 +1,58 @@
+#!/bin/bash
+
+set -eu
+
+: ${BOSH_ENVIRONMENT:?required}
+: ${BOSH_CA_CERT:?required}
+: ${BOSH_CLIENT:?required}
+: ${BOSH_CLIENT_SECRET:?required}
+
+#
+# stemcell metadata/upload
+#
+
+STEMCELL_OS=${STEMCELL_OS:-ubuntu-trusty}
+STEMCELL_VERSION=$(cat stemcell/version)
+
+bosh -n upload-stemcell "https://bosh.io/d/stemcells/bosh-warden-boshlite-${STEMCELL_OS}-go_agent?v=${STEMCELL_VERSION}"
+
+#
+# release metadata/upload
+#
+
+cd release
+tar -xzf *.tgz $( tar -tzf *.tgz | grep 'release.MF' )
+RELEASE_NAME=$( grep -E '^name: ' release.MF | awk '{print $2}' | tr -d "\"'" )
+RELEASE_VERSION=$( grep -E '^version: ' release.MF | awk '{print $2}' | tr -d "\"'" )
+
+bosh -n upload-release *.tgz
+cd ../
+
+#
+# compilation deployment
+#
+
+cat > manifest.yml <<EOF
+---
+name: compilation
+releases:
+- name: "$RELEASE_NAME"
+  version: "$RELEASE_VERSION"
+stemcells:
+- alias: default
+  os: "$STEMCELL_OS"
+  version: "$STEMCELL_VERSION"
+update:
+  canaries: 1
+  max_in_flight: 1
+  canary_watch_time: 1000 - 90000
+  update_watch_time: 1000 - 90000
+instance_groups: []
+EOF
+
+bosh -n -d compilation deploy manifest.yml
+bosh -d compilation export-release $RELEASE_NAME/$RELEASE_VERSION $STEMCELL_OS/$STEMCELL_VERSION
+
+mkdir -p compiled-release/compiled-releases
+mv *.tgz compiled-release/compiled-releases/$( echo *.tgz | sed "s/\.tgz$/-$( date -u +%Y%m%d%H%M%S ).tgz/" )
+sha1sum  compiled-release/compiled-releases/*.tgz
diff --git a/boshrelease/scripts/shipit b/boshrelease/scripts/shipit
@@ -48,7 +48,9 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 ###############################################################
 
-cd ${REPO_ROOT}
+git clone ${REPO_ROOT} ${REPO_OUT}
+
+pushd ${REPO_OUT}
 RELEASE_NAME=$(bosh2 int config/final.yml --path /final_name)
 
 cat > config/private.yml <<YAML
@@ -67,26 +69,27 @@ header "Create final release..."
 bosh2 -n create-release --final --version "${VERSION}"
 bosh2 -n create-release releases/$RELEASE_NAME/$RELEASE_NAME-$VERSION.yml \
               --tarball releases/$RELEASE_NAME/$RELEASE_NAME-$VERSION.tgz
-cd -
+popd
 
-RELEASE_TGZ=$REPO_ROOT/releases/$RELEASE_NAME/$RELEASE_NAME-$VERSION.tgz
-export SHA1=$(sha1sum $RELEASE_TGZ | head -n1 | awk '{print $1}')
-echo "SHA1=$SHA1"
+RELEASE_TGZ=$REPO_OUT/releases/$RELEASE_NAME/$RELEASE_NAME-$VERSION.tgz
+export RELEASE_SHA1=$(sha1sum $RELEASE_TGZ | head -n1 | awk '{print $1}')
+export RELEASE_URL="https://github.com/${GITHUB_OWNER}/${RELEASE_NAME}-boshrelease/releases/download/v${VERSION}/${RELEASE_NAME}-${VERSION}.tgz"
+echo "RELEASE_SHA1=$RELEASE_SHA1"
 
 mkdir -p ${RELEASE_ROOT}/artifacts
 echo "v${VERSION}"                         > ${RELEASE_ROOT}/tag
-echo "${RELEASE_NAME} v${VERSION}"         > ${RELEASE_ROOT}/name
-mv ${REPO_ROOT}/releases/*/*-${VERSION}.tgz  ${RELEASE_ROOT}/artifacts
-mv ${REPO_ROOT}/ci/release_notes.md          ${RELEASE_ROOT}/notes.md
+echo "v${VERSION}"                         > ${RELEASE_ROOT}/name
+mv ${REPO_OUT}/releases/*/*-${VERSION}.tgz   ${RELEASE_ROOT}/artifacts
+mv ${REPO_OUT}/ci/release_notes.md           ${RELEASE_ROOT}/notes.md
 cat >> ${RELEASE_ROOT}/notes.md <<EOF
 
 ### Deployment
 \`\`\`yaml
 releases:
 - name:    $RELEASE_NAME
   version: $VERSION
-  url:     https://github.com/${GITHUB_OWNER}/${RELEASE_NAME}-boshrelease/releases/download/v${VERSION}/${RELEASE_NAME}-${VERSION}.tgz
-  sha1:    $SHA1
+  url:     $RELEASE_URL
+  sha1:    $RELEASE_SHA1
 \`\`\`
 EOF
 cat > ${RELEASE_ROOT}/notification <<EOF
@@ -102,17 +105,14 @@ if [[ -z $(git config --global user.name) ]]; then
   git config --global user.name "${GIT_NAME}"
 fi
 
-(cd ${REPO_ROOT}
- for MANIFEST_PATH in $(ls manifests/*.yml); do
-   $DIR/update-manifest $GITHUB_OWNER $RELEASE_NAME $VERSION $SHA1 $MANIFEST_PATH
- done
- git merge --no-edit ${BRANCH}
- git add -A
- git status
- git commit -m "release v${VERSION}")
-
-# so that future steps in the pipeline can push our changes
-cp -a ${REPO_ROOT} ${REPO_OUT}
+pushd $REPO_OUT
+$DIR/update-manifest-release $RELEASE_NAME $VERSION $RELEASE_URL $RELEASE_SHA1
+
+git merge --no-edit ${BRANCH}
+git add -A
+git status
+git commit -m "release v${VERSION}"
+popd
 
 cat > ${NOTIFICATION_OUT:-notifications}/message <<EOS
 New ${RELEASE_NAME} v${VERSION} released. <https://github.com/${GITHUB_OWNER}/${RELEASE_NAME}-boshrelease/releases/tag/v${VERSION}|Release notes>.

diff --git a/boshrelease/scripts/update-manifest b/boshrelease/scripts/update-manifest
diff --git a/boshrelease/scripts/update-manifest-release b/boshrelease/scripts/update-manifest-release
@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# The goal of this script is to retain the pretty nature of manifests, whilst updating
+# the "releases:" section.
+#
+# If we use `spruce merge` to do the job, the entire manifest will be reordered -
+# keys will be sorted alphabetically. Comments will be lost.
+#
+# Instead, we assume that "releases:" is the last section of the manifest.
+set -e
+
+NAME=$1
+VERSION=$2
+RELEASE_TGZ_URL=$3
+RELEASE_TGZ_SHA1=$4
+if [[ "${RELEASE_TGZ_SHA1:-X}" == "X" ]]; then
+  echo "USAGE: update-manifest-release NAME VERSION URL SHA1"
+  exit 1
+fi
+: ${REPO_ROOT:?required}
+: ${REPO_OUT:?required}
+
+# So, this is a nested bash/spruce/jq combo.
+# What is happening here is that the "releases:" section of each deployment manifest
+# is being updated with the new version/sha1 for the release.
+#
+# We use "spruce json manifest.yml | jq '.releases'" to extract the existing releases array
+# and the '.releases | map(if .name == $name)' will modify a specific element of the array
+#
+# This gives us a modified "releases: [{...}, {...}]" segment of the final deployment manifest.
+# We now need to merge this back into the original manifest.
+#
+# But, I don't want to just use `spruce merge` for this as it will reorder the manifest and
+# make it ugly. If I didn't care about the manifest's aesthetics then this whole script
+# would be simpler. I want the original layout of the manifest to be retained; and so
+# we will just chomp out the original "releases:" section at the end of the file and
+# paste in the updated releases section.
+function bump_version {
+  manifest_path=$1
+  releases_updated=$(spruce merge <<YAML
+releases: $(spruce json "$manifest_path" | jq --arg name $NAME --arg version $VERSION --arg url $RELEASE_TGZ_URL --arg sha1 $RELEASE_TGZ_SHA1 '.releases | map(
+    if .name == $name
+    then . + {"version":$version,"url":$url,"sha1":$sha1}
+    else .
+    end)'
+)
+YAML
+)
+  releases_line_number=$(awk '/^releases:/{ print NR; exit }' $manifest_path)
+  manifest_head=$(head -n `expr $releases_line_number - 1` $manifest_path)
+  cat > $manifest_path <<YAML
+${manifest_head}
+
+${releases_updated}
+YAML
+}
+
+for manifest_path in $(ls manifests/*.yml); do
+  bump_version $manifest_path
+done