Squashed 'tools/' changes from b783528b..35679ee5 (#521)

35679ee5 Merge pull request #110 from weaveworks/parallel-push-errors
3ae41b6f Remove unneeded if block
51ff31a5 Exit on first error
0faad9f7 Check for errors when pushing images in parallel
74dc626b Merge pull request #108 from weaveworks/disable-apt-daily
b4f1d918 Merge pull request #107 from weaveworks/docker-17-update
7436aa14 Override apt daily job to not run immediately on boot
7980f152 Merge pull request #106 from weaveworks/document-docker-install-role
f741e533 Bump to Docker 17.06 from CE repo
61796a1b Update Docker CE Debian repo details
0d86f5e1 Allow for Docker package to be named docker-ce
065c68d4 Document selection of Docker installation role.
38090539 Just --porcelain; it defaults to v1
11400eaa Merge pull request #105 from weaveworks/remove-weaveplugin-remnants
b8b4d64c remove weaveplugin remnants
35099c93 Merge pull request #104 from weaveworks/pull-docker-py
cdd48fc3 Pull docker-py to speed tests/builds up.
e1c6c24b Merge pull request #103 from weaveworks/test-build-tags
d5d71e06 Add -tags option so callers can pass in build tags
8949b2b5 Merge pull request #98 from weaveworks/git-status-tag
ac30687f Merge pull request #100 from weaveworks/python_linting
4b125b55 Pin yapf & flake8 versions
7efb4853 Lint python linting function
444755b7 Swap diff direction to reflect changes required
c5b24346 Install flake8 & yapf
5600eac5 Lint python in build-tools repo
0b02ca93 Add python linting
c011c0dc Merge pull request #79 from kinvolk/schu/python-shebang
6577d078 Merge pull request #99 from weaveworks/shfmt-version
00ce0dcd Use git status instead of diff to add 'WIP' tag
411fd13a Use shfmt v1.3.0 instead of latest from master.
0d6d4da9 Run shfmt 1.3 on the code.
5cdba320 Add sudo
c322ca83 circle.yml: Install shfmt binary.
e59c2251 Install shfmt 1.3 binary.
30706e6b Install pyhcl in the build container.
960d2228 Merge pull request #97 from kinvolk/alban/update-shfmt-3
1d535c7b shellcheck: fix escaping issue
55424986 Merge pull request #96 from kinvolk/alban/update-shfmt-2
32f7cc51 shfmt: fix coding style
09f72af0 lint: print the diff in case of error
571c7d71 Merge pull request #95 from kinvolk/alban/update-shfmt
bead6edd Update for latest shfmt
b08dc4d6 Update for latest shfmt (#94)
2ed8aaa8 Add no-race argument to test script (#92)
80dd78e6 Merge pull request #91 from weaveworks/upgrade-go-1.8.1
08dcd0df Please ./lint as shfmt changed its rules between 1.0.0 and 1.3.0.
a8bc9ab0 Upgrade default Go version to 1.8.1.
41c56221 Merge pull request #90 from weaveworks/build-golang-service-conf
e8ebdd5e broaden imagetag regex to fix haskell build image
ba3fbfa6 Merge pull request #89 from weaveworks/build-golang-service-conf
e506f1b9 Fix up test script for updated shfmt
9216db86 Add stuff for service-conf build to build-goland image
66a9a93c Merge pull request #88 from weaveworks/haskell-image
cb3e3a25 shfmt
74a5239e Haskell build image
4ccd42b9 Trying circle quay login
b2c295fc Merge branch 'common-build'
0ac746fa Trim quay prefix in circle script
c405b311 Merge pull request #87 from weaveworks/common-build
9672d7cb Push build images to quay as they have sane robot accounts
a2bf1123 Review feedback
fef9b7dd Add protobuf tools
10a77ead Update readme
254f2660 Don't need the image name in
ffb59fcb Adding a weaveworks/build-golang image with tags
b8173683 Update min Weave Net docker version
cf87ca33 Merge pull request #86 from weaveworks/lock-kubeadm-version
3ae69196 Add example of custom SSH private key to tf_ssh's usage.
cf8bd8af Add example of custom SSH private key to tf_ansi's usage.
c7d33700 Lock kubeadm's Kubernetes version.
faaaa6f2 Merge pull request #84 from weaveworks/centos-rhel
ef552e7d Select weave-kube YAML URL based on K8S version.
b4c11982 Upgrade default kubernetes_version to 1.6.1.
b82805ef Use a fixed version of kubeadm.
f33888bc Factorise and make kubeconfig option optional.
f7b8b897 Install EPEL repo for CentOS.
615917a6 Fix error in decrypting AWS access key and secret.
86f97b42 Add CentOS 7 AMI and username for AWS via Terraform.
eafd810e Add tf_ansi example with Ansible variables.
2b05787f Skip setup of Docker over TCP for CentOS/RHEL.
84c420b6 Add docker-ce role for CentOS/RHEL.
00a820cd Add setup_weave-net_debug.yml playbook for user issues' debugging.
3eae480b Upgrade default kubernetes_version to 1.5.4.
753921cf Allow injection of Docker installation role.
e1ff90d2 Fix kubectl taint command for 1.5.
b989e97a Fix typo in kubectl taint for single node K8S cluster.
541f58d2 Remove 'install_recommends: no' for ethtool.
c3f97115 Make Ansible role docker-from-get.docker.com work on RHEL/CentOS.
038c0ae8 Add frequently used OS images, for convenience.
d30649f0 Add --insecure-registry to docker.conf
1dd92188 shfmt -i 4 -w push-images
6de96ac7 Add option to not push docker hub images
310f53dc Add push-images script from cortex
8641381b Add port 6443 to kubeadm join commands for K8S 1.6+.
50bf0bc6 Force type of K8S token to string.
08ab1c0c Remove trailing whitespaces.
ae9efb83 Enable testing against K8S release candidates.
9e32194c Secure GCP servers for Scope: open port 80.
a22536ac Secure GCP servers for Scope.
89c3a29b Merge pull request #78 from weaveworks/lint-merge-rebase-issue-in-docs
73ad56d7 Add linter function to avoid bad merge/rebase artefact
31d069d6 Change Python shebang to `#!/usr/bin/env python`
52d695cc Merge pull request #77 from kinvolk/schu/fix-relative-weave-path
77aed016 Merge pull request #73 from weaveworks/mike/sched/fix-unicode-issue
7c080f45 integration/sanity_check: disable SC1090
d6d360a1 integration/gce.sh: update gcloud command
e8def2c4 provisioning/setup: fix shellcheck SC2140
cc022241 integration/config: fix weave path
9c0d6a55 Fix config_management/README.md
334708ca Merge pull request #75 from kinvolk/alban/external-build-1
da2505d6 gce.sh: template: print creation date
e6768547 integration tests: fix user account
85308369 host nameing: add repo name
b556c0ad gce.sh: fix deletion of gce instances
2ecd1c2c integration: fix GCE --zones/--zone parameter
3e863df9 sched: Fix unicode encoding issues
51785b5f Use rm -f and set current dir using BASH_SOURCE.
f5c6d68c Merge pull request #71 from kinvolk/schu/fix-linter-warnings
0269628e Document requirement for `lint_sh`
9a3f09e6 Fix linter warnings
efcf9d21 Merge pull request #53 from weaveworks/2647-testing-mvp
d31ea574 Weave Kube playbook now works with multiple nodes.
27868dd5 Add GCP firewall rule for FastDP crypto.
edc8bb3a Differentiated name of dev and test playbooks, to avoid confusion.
efa3df7c Moved utility Ansible Yaml to library directory.
fcd2769e Add shorthands to run Ansible playbooks against Terraform-provisioned virtual machines.
f7946fbb Add shorthands to SSH into Terraform-provisioned virtual machines.
aad5c6f1 Mention Terraform and Ansible in README.md.
dddabf0e Add Terraform output required for templates' creation.
dcc7d025 Add Ansible configuration playbooks for development environments.
f86481ca Add Ansible configuration playbooks for Docker, K8S and Weave-Net.
efedd258 Git-ignore Ansible retry files.
765c4ca1 Add helper functions to setup Terraform programmatically.
801dd1d1 Add Terraform cloud provisioning scripts.
b8017e19 Install hclfmt on CircleCI.
4815e19b Git-ignore Terraform state files.
0aaebc7d Add script to generate cartesian product of dependencies of cross-version testing.
007d90ab Add script to list OS images from GCP, AWS and DO.
ca65cc02 Add script to list relevant versions of Go, Docker and Kubernetes.
aa66f447 Scripts now source dependencies using absolute path (previously breaking make depending on current directory).
7865e86d Add -p option to parallelise lint.
36c1835b Merge pull request #69 from weaveworks/mflag
98575686 Use mflag and mflagext package from weaveworks/common.
97991128 Quote bash variable.
10a36b33 Merge pull request #67 from weaveworks/shfmt-ignore
a59884f3 Add support for .lintignore.
03cc5989 Don't lint generated protobuf code.
2b55c2df Merge pull request #66 from weaveworks/reduce-test-timeout
d4e163cc Make timeout a flag
49a86091 Reduce test timeout
8fa15cba Merge pull request #63 from weaveworks/test-defaults

git-subtree-dir: tools
git-subtree-split: 35679ee5ff17c4edf864b7c43dc70a40337fcd80

tools/.gitignore

@@ -4,3 +4,7 @@ socks/image.tar
 runner/runner
 *.pyc
 *~
+terraform.tfstate
+terraform.tfstate.backup
+*.retry
+build/**/.uptodate

tools/README.md

@@ -2,12 +2,17 @@
 
 Included in this repo are tools shared by weave.git and scope.git.  They include
 
+- ```build```: a set of docker base-images for building weave
+  projects. These should be used instead of giving each project its
+  own build image.
+- ```provisioning```: a set of Terraform scripts to provision virtual machines in GCP, AWS or Digital Ocean.
+- ```config_management```: a set of Ansible playbooks to configure virtual machines for development, testing, etc.
 - ```cover```: a tool which merges overlapping coverage reports generated by go
   test
 - ```files-with-type```: a tool to search directories for files of a given
   MIME type
-- ```lint```: a script to lint Go project; runs various tools like golint, go
-  vet, errcheck etc
+- ```lint```: a script to lint go, sh and hcl files; runs various tools like
+  golint, go vet, errcheck, shellcheck etc
 - ```rebuild-image```: a script to rebuild docker images when their input files
   change; useful when you using docker images to build your software, but you
   don't want to build the image every time.
@@ -24,6 +29,11 @@ Included in this repo are tools shared by weave.git and scope.git.  They include
 - ```scheduler```: an appengine application that can be used to distribute
   tests across different shards in CircleCI.
 
+## Requirements
+
+- ```lint``` requires shfmt to lint sh files; get shfmt with
+  ```go get -u gopkg.in/mvdan/sh.v1/cmd/shfmt```
+
 ## Using build-tools.git
 
 To allow you to tie your code to a specific version of build-tools.git, such

tools/build/Makefile

@@ -0,0 +1,46 @@
+.PHONY: all clean images
+.DEFAULT_GOAL := all
+
+# Boiler plate for bulding Docker containers.
+# All this must go at top of file I'm afraid.
+IMAGE_PREFIX := quay.io/weaveworks/build-
+IMAGE_TAG := $(shell ../image-tag)
+UPTODATE := .uptodate
+
+# Every directory with a Dockerfile in it builds an image called
+# $(IMAGE_PREFIX)<dirname>. Dependencies (i.e. things that go in the image)
+# still need to be explicitly declared.
+%/$(UPTODATE): %/Dockerfile %/*
+	$(SUDO) docker build -t $(IMAGE_PREFIX)$(shell basename $(@D)) $(@D)/
+	$(SUDO) docker tag $(IMAGE_PREFIX)$(shell basename $(@D)) $(IMAGE_PREFIX)$(shell basename $(@D)):$(IMAGE_TAG)
+	touch $@
+
+# Get a list of directories containing Dockerfiles
+DOCKERFILES := $(shell find . -name tools -prune -o -name vendor -prune -o -type f -name 'Dockerfile' -print)
+UPTODATE_FILES := $(patsubst %/Dockerfile,%/$(UPTODATE),$(DOCKERFILES))
+DOCKER_IMAGE_DIRS := $(patsubst %/Dockerfile,%,$(DOCKERFILES))
+IMAGE_NAMES := $(foreach dir,$(DOCKER_IMAGE_DIRS),$(patsubst %,$(IMAGE_PREFIX)%,$(shell basename $(dir))))
+images:
+	$(info $(IMAGE_NAMES))
+	@echo > /dev/null
+
+# Define imagetag-golang, etc, for each image, which parses the dockerfile and
+# prints an image tag. For example:
+#     FROM golang:1.8.1-stretch
+# in the "foo/Dockerfile" becomes:
+#     $ make imagetag-foo
+#     1.8.1-stretch
+define imagetag_dep
+.PHONY: imagetag-$(1)
+$(patsubst $(IMAGE_PREFIX)%,imagetag-%,$(1)): $(patsubst $(IMAGE_PREFIX)%,%,$(1))/Dockerfile
+	@cat $$< | grep "^FROM " | head -n1 | sed 's/FROM \(.*\):\(.*\)/\2/'
+endef
+$(foreach image, $(IMAGE_NAMES), $(eval $(call imagetag_dep, $(image))))
+
+all: $(UPTODATE_FILES)
+
+clean:
+	$(SUDO) docker rmi $(IMAGE_NAMES) >/dev/null 2>&1 || true
+	rm -rf $(UPTODATE_FILES)
+
+

tools/build/golang/Dockerfile

@@ -0,0 +1,49 @@
+FROM golang:1.8.0-stretch
+RUN apt-get update && \
+    apt-get install -y \
+      curl \
+      file \
+      git \
+      jq \
+      libprotobuf-dev \
+      make \
+      protobuf-compiler \
+      python-pip \
+      python-requests \
+      python-yaml \
+      unzip && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+RUN pip install attrs pyhcl
+RUN curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 && \
+	echo "b1925c2c405458811f0c227266402cf1868b4de529f114722c2e3a5af4ac7bb2  shfmt" | sha256sum -c && \
+	chmod +x shfmt && \
+	mv shfmt /usr/bin
+RUN go clean -i net && \
+	go install -tags netgo std && \
+	go install -race -tags netgo std
+RUN go get -tags netgo \
+		github.com/FiloSottile/gvt \
+		github.com/client9/misspell/cmd/misspell \
+		github.com/fatih/hclfmt \
+		github.com/fzipp/gocyclo \
+		github.com/gogo/protobuf/gogoproto \
+		github.com/gogo/protobuf/protoc-gen-gogoslick \
+		github.com/golang/dep/... \
+		github.com/golang/lint/golint \
+		github.com/golang/protobuf/protoc-gen-go \
+		github.com/kisielk/errcheck \
+		github.com/mjibson/esc \
+		github.com/prometheus/prometheus/cmd/promtool && \
+		rm -rf /go/pkg /go/src
+RUN mkdir protoc && \
+	cd protoc && \
+	curl -O -L https://github.com/google/protobuf/releases/download/v3.1.0/protoc-3.1.0-linux-x86_64.zip && \
+	unzip protoc-3.1.0-linux-x86_64.zip && \
+	cp bin/protoc /usr/bin/ && \
+	chmod o+x /usr/bin/protoc && \
+	cd .. && \
+	rm -rf protoc
+RUN mkdir -p /var/run/secrets/kubernetes.io/serviceaccount && \
+		touch /var/run/secrets/kubernetes.io/serviceaccount/token
+COPY build.sh /
+ENTRYPOINT ["/build.sh"]

tools/build/golang/build.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+set -eu
+
+if [ -n "${SRC_NAME:-}" ]; then
+    SRC_PATH=${SRC_PATH:-$GOPATH/src/$SRC_NAME}
+elif [ -z "${SRC_PATH:-}" ]; then
+    echo "Must set either \$SRC_NAME or \$SRC_PATH."
+    exit 1
+fi
+
+# If we run make directly, any files created on the bind mount
+# will have awkward ownership.  So we switch to a user with the
+# same user and group IDs as source directory.  We have to set a
+# few things up so that sudo works without complaining later on.
+uid=$(stat --format="%u" "$SRC_PATH")
+gid=$(stat --format="%g" "$SRC_PATH")
+echo "weave:x:$uid:$gid::$SRC_PATH:/bin/sh" >>/etc/passwd
+echo "weave:*:::::::" >>/etc/shadow
+echo "weave	ALL=(ALL)	NOPASSWD: ALL" >>/etc/sudoers
+
+su weave -c "PATH=$PATH make -C $SRC_PATH BUILD_IN_CONTAINER=false $*"

tools/build/haskell/Dockerfile

@@ -0,0 +1,4 @@
+FROM fpco/stack-build:lts-8.9
+COPY build.sh /
+COPY copy-libraries /usr/local/bin/
+ENTRYPOINT ["/build.sh"]

tools/build/haskell/build.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+#
+# Build a static Haskell binary using stack.
+
+set -eu
+
+if [ -z "${SRC_PATH:-}" ]; then
+    echo "Must set \$SRC_PATH."
+    exit 1
+fi
+
+make -C "$SRC_PATH" BUILD_IN_CONTAINER=false "$@"

tools/build/haskell/copy-libraries

@@ -0,0 +1,41 @@
+#!/bin/bash
+#
+# Copy dynamically linked libraries for a binary, so we can assemble a Docker
+# image.
+#
+# Run with:
+#   copy-libraries /path/to/binary /output/dir
+#
+# Dependencies:
+# - awk
+# - cp
+# - grep
+# - ldd
+# - mkdir
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# Path to a Linux binary that we're going to run in the container.
+binary_path="${1}"
+# Path to directory to write the output to.
+output_dir="${2}"
+
+exe_name=$(basename "${binary_path}")
+
+# Identify linked libraries.
+libraries=($(ldd "${binary_path}" | awk '{print $(NF-1)}' | grep -v '=>'))
+# Add /bin/sh, which we need for Docker imports.
+libraries+=('/bin/sh')
+
+mkdir -p "${output_dir}"
+
+# Copy executable and all needed libraries into temporary directory.
+cp "${binary_path}" "${output_dir}/${exe_name}"
+for lib in "${libraries[@]}"; do
+    mkdir -p "${output_dir}/$(dirname "$lib")"
+    # Need -L to make sure we get actual libraries & binaries, not symlinks to
+    # them.
+    cp -L "${lib}" "${output_dir}/${lib}"
+done

tools/circle.yml

@@ -13,18 +13,43 @@ dependencies:
     - go install -tags netgo std
     - mkdir -p $(dirname $SRCDIR)
     - cp -r $(pwd)/ $SRCDIR
+    - |
+      curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 && \
+        echo "b1925c2c405458811f0c227266402cf1868b4de529f114722c2e3a5af4ac7bb2  shfmt" | sha256sum -c && \
+        chmod +x shfmt && \
+        sudo mv shfmt /usr/bin
     - |
       cd $SRCDIR;
       go get \
         github.com/fzipp/gocyclo \
         github.com/golang/lint/golint \
         github.com/kisielk/errcheck \
-        gopkg.in/mvdan/sh.v1/cmd/shfmt
+        github.com/fatih/hclfmt
+    - pip install yapf==0.16.2 flake8==3.3.0
 
 test:
   override:
     - cd $SRCDIR; ./lint .
     - cd $SRCDIR/cover; make
     - cd $SRCDIR/socks; make
     - cd $SRCDIR/runner; make
+    - cd $SRCDIR/build; make
+
+deployment:
+  snapshot:
+    branch: master
+    commands:
+      - docker login -e "$DOCKER_REGISTRY_EMAIL" -u "$DOCKER_REGISTRY_USER" -p "$DOCKER_REGISTRY_PASS" "$DOCKER_REGISTRY_URL"
+      - |
+        cd $SRCDIR/build;
+        for image in $(make images); do
+          # Tag the built images with the revision of this repo.
+          docker push "${image}:${GIT_TAG}"
 
+          # Tag the built images with something derived from the base images in
+          # their respective Dockerfiles. So "FROM golang:1.8.0-stretch" as a
+          # base image would lead to a tag of "1.8.0-stretch"
+          IMG_TAG=$(make "imagetag-${image#quay.io/weaveworks/build-}")
+          docker tag "${image}:latest" "${image}:${IMG_TAG}"
+          docker push "${image}:${IMG_TAG}"
+        done