publishing

Author: dpb587

.envrc

@@ -0,0 +1,3 @@
+export GOPATH=$( realpath "$PWD/../../../.." )
+export GOBIN=$GOPATH/bin
+export PATH=$GOBIN:$PATH

Dockerfile

@@ -0,0 +1,7 @@
+FROM alpine:3.6
+RUN apk --no-cache add bash curl git
+RUN \
+  curl -o /usr/local/bin/bosh http://s3.amazonaws.com/bosh-cli-artifacts/bosh-cli-2.0.45-linux-amd64 \
+  && echo 'bf04be72daa7da0c9bbeda16fda7fc7b2b8af51e  /usr/local/bin/bosh' | sha1sum -c \
+  && chmod +x /usr/local/bin/bosh
+ADD bin /opt/metalink-upgrader-pipeline/bin

Gopkg.lock

@@ -0,0 +1,26 @@
+
+# Gopkg.toml example
+#
+# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
+# for detailed Gopkg.toml documentation.
+#
+# required = ["github.com/user/thing/cmd/thing"]
+# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
+#
+# [[constraint]]
+#   name = "github.com/user/project"
+#   version = "1.0.0"
+#
+# [[constraint]]
+#   name = "github.com/user/project2"
+#   branch = "dev"
+#   source = "github.com/myfork/project2"
+#
+# [[override]]
+#  name = "github.com/x/y"
+#  version = "2.4.0"
+
+
+[[constraint]]
+  branch = "v2"
+  name = "gopkg.in/yaml.v2"

Gopkg.toml

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Danny Berger <https://dpb587.me>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

LICENSE

@@ -0,0 +1,48 @@
+# metalink-upgrader-pipeline
+
+For managing a [BOSH](https://bosh.io) release pipeline which upgrades blobs from [metalink repositories](https://github.com/dpb587/metalink/tree/master/repository#metalink-repository).
+
+
+## Usage
+
+A `config/blobs/{blobname}/repository.yml` defines the metalink repository where the blob versions are mirrored.
+
+    uri: git+https://github.com/dpb587/upstream-blob-mirror.git//repository/icu4c
+    version: ^58
+
+A pipeline defines the basic configuration for an upgrader pipeline (`ci/pipelines/upgrader.yml`) which defines, at a minimum, a `repo` resource which references the release repository. After a blob upgrade is triggered, the `config.after_upload_blobs` setting configures what happens after the blobs have been synced.
+
+    config:
+      after_sync_blobs: # run integration tests with a dev release of the new blob
+      - task: create-dev-release
+        file: repo/ci/tasks/create-dev-release/task.yml
+      - task: integration-test
+        file: repo/ci/tasks/integration-test/task.yml
+        privileged: true
+      after_upload_blobs: # push the new blobs to the repo
+      - put: repo
+        params:
+          repository: repo
+    resources:
+    - name: repo # the bosh release repository
+      type: git
+      source:
+        uri: git@github.com:dpb587/openvpn-bosh-release.git
+        branch: master
+        private_key: ((repo_private_key))
+
+Once the blobs and base pipeline have been configured, `metalink-upgrader-pipeline` can be used to generate a pipeline. The generated pipeline will have a new job to handle updates to the upstream blobs. By configuring steps for `after_upload_blobs`, a `bosh upload-blobs` step will first be executed. The job will require several variables to be set: `release_private_yml` (for uploading blobs to the blobstore) and `maintainer_email`, `maintainer_name` (for the `git` commit).
+
+    fly set-pipeline -p openvpn:upgrader \
+      -c <( metalink-upgrader-pipeline ci/pipelines/upgrader.yml ) \
+      -l <( lpass show 'pipeline=openvpn:upgrader' )
+
+The blob jobs automatically trigger whenever a new version is available. When syncing blobs, old blobs in the directory are removed, new blobs are added (not yet uploaded), and a copy of the origin metalink is staged to `config/blobs/{blobname}/metalink.meta4`. When uploading blobs, `config/blobs.yml` is updated with the new blobstore references, and any other already-staged files are committed. After uploading blobs, the repository should be pushed.
+
+
+## Examples
+
+A couple public repositories using this to manage some upstream dependencies.
+
+* [dpb587/openvpn-bosh-release](https://github.com/dpb587/openvpn-bosh-release)
+* [dpb587/ssoca-bosh-release](https://github.com/dpb587/ssoca-bosh-release)

README.md

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+set -eu
+
+cp -rp repo/. updated-repo
+
+cd updated-repo/
+
+
+#
+# replace existing blobs
+#
+
+for file in $( bosh blobs --column=path | grep "^$blob/" | sed "s#$blob/##g" ); do
+  if [[ ! -e "../blob/$file" ]]; then
+    bosh remove-blob "$blob/$file"
+
+    continue
+  fi
+
+  new_digest=$( sha1sum "../blob/$file" | awk '{ print $1 }' )
+  existing_digest=$( bosh blobs --column=path --column=digest | grep "^$blob/$file\s" | awk '{ print $2 }' )
+
+  if [[ "$new_digest" == "$existing_digest" ]]; then
+    echo "$file unchanged"
+
+    continue
+  fi
+
+  bosh remove-blob "$blob/$file"
+
+  bosh add-blob "../blob/$file" "$blob/$file"
+done
+
+
+#
+# add new blobs
+#
+
+for file in $( cd ../blob ; find . -type f | grep -v '^./.resource' | cut -c3- ); do
+  if bosh blobs --column=path | grep -q "^$blob/$file\s" ; then
+    continue
+  fi
+
+  bosh add-blob "../blob/$file" "$blob/$file"
+done
+
+
+#
+# import metalink
+#
+
+cp -p ../blob/.resource/metalink.meta4 "config/blobs/$blob/metalink.meta4"
+
+git add "config/blobs/$blob/metalink.meta4"
+
+echo "blobs: Update $blob to $( cat ../blob/.resource/version )" > .git/COMMIT_MSG

bin/sync-blobs

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+set -eu
+
+task_dir=$PWD
+
+cp -rp repo/. updated-repo
+
+cd updated-repo/
+
+
+#
+# env
+#
+
+export GIT_COMMITTER_NAME="Concourse"
+export GIT_COMMITTER_EMAIL="concourse.ci@localhost"
+
+git config --global user.email "${git_user_email:-ci@localhost}"
+git config --global user.name "${git_user_name:-CI Bot}"
+
+
+#
+# upload-blobs
+#
+
+cat > config/private.yml <<EOF
+${release_private_yml:-}
+EOF
+
+bosh upload-blobs
+
+git add config/blobs.yml
+
+
+#
+# commit
+#
+
+if git diff --staged --exit-code --quiet ; then
+  # no changes pending
+  exit
+elif [ -e .git/COMMIT_MSG ] ; then
+  # use prepared commit message
+  git commit -F .git/COMMIT_MSG
+else
+  git commit -m "${git_message:-Update blobs}"
+fi