Squashed 'tools/' changes from 9e32194..0d6d4da

0d6d4da Run shfmt 1.3 on the code.
5cdba32 Add sudo
c322ca8 circle.yml: Install shfmt binary.
e59c225 Install shfmt 1.3 binary.
30706e6 Install pyhcl in the build container.
960d222 Merge pull request weaveworks#97 from kinvolk/alban/update-shfmt-3
1d535c7 shellcheck: fix escaping issue
5542498 Merge pull request weaveworks#96 from kinvolk/alban/update-shfmt-2
32f7cc5 shfmt: fix coding style
09f72af lint: print the diff in case of error
571c7d7 Merge pull request weaveworks#95 from kinvolk/alban/update-shfmt
bead6ed Update for latest shfmt
b08dc4d Update for latest shfmt (weaveworks#94)
2ed8aaa Add no-race argument to test script (weaveworks#92)
80dd78e Merge pull request weaveworks#91 from weaveworks/upgrade-go-1.8.1
08dcd0d Please ./lint as shfmt changed its rules between 1.0.0 and 1.3.0.
a8bc9ab Upgrade default Go version to 1.8.1.
41c5622 Merge pull request weaveworks#90 from weaveworks/build-golang-service-conf
e8ebdd5 broaden imagetag regex to fix haskell build image
ba3fbfa Merge pull request weaveworks#89 from weaveworks/build-golang-service-conf
e506f1b Fix up test script for updated shfmt
9216db8 Add stuff for service-conf build to build-goland image
66a9a93 Merge pull request weaveworks#88 from weaveworks/haskell-image
cb3e3a2 shfmt
74a5239 Haskell build image
4ccd42b Trying circle quay login
b2c295f Merge branch 'common-build'
0ac746f Trim quay prefix in circle script
c405b31 Merge pull request weaveworks#87 from weaveworks/common-build
9672d7c Push build images to quay as they have sane robot accounts
a2bf112 Review feedback
fef9b7d Add protobuf tools
10a77ea Update readme
254f266 Don't need the image name in
ffb59fc Adding a weaveworks/build-golang image with tags
b817368 Update min Weave Net docker version
cf87ca3 Merge pull request weaveworks#86 from weaveworks/lock-kubeadm-version
3ae6919 Add example of custom SSH private key to tf_ssh's usage.
cf8bd8a Add example of custom SSH private key to tf_ansi's usage.
c7d3370 Lock kubeadm's Kubernetes version.
faaaa6f Merge pull request weaveworks#84 from weaveworks/centos-rhel
ef552e7 Select weave-kube YAML URL based on K8S version.
b4c1198 Upgrade default kubernetes_version to 1.6.1.
b82805e Use a fixed version of kubeadm.
f33888b Factorise and make kubeconfig option optional.
f7b8b89 Install EPEL repo for CentOS.
615917a Fix error in decrypting AWS access key and secret.
86f97b4 Add CentOS 7 AMI and username for AWS via Terraform.
eafd810 Add tf_ansi example with Ansible variables.
2b05787 Skip setup of Docker over TCP for CentOS/RHEL.
84c420b Add docker-ce role for CentOS/RHEL.
00a820c Add setup_weave-net_debug.yml playbook for user issues' debugging.
3eae480 Upgrade default kubernetes_version to 1.5.4.
753921c Allow injection of Docker installation role.
e1ff90d Fix kubectl taint command for 1.5.
b989e97 Fix typo in kubectl taint for single node K8S cluster.
541f58d Remove 'install_recommends: no' for ethtool.
c3f9711 Make Ansible role docker-from-get.docker.com work on RHEL/CentOS.
038c0ae Add frequently used OS images, for convenience.
d30649f Add --insecure-registry to docker.conf
1dd9218 shfmt -i 4 -w push-images
6de96ac Add option to not push docker hub images
310f53d Add push-images script from cortex
8641381 Add port 6443 to kubeadm join commands for K8S 1.6+.
50bf0bc Force type of K8S token to string.
08ab1c0 Remove trailing whitespaces.
ae9efb8 Enable testing against K8S release candidates.

git-subtree-dir: tools
git-subtree-split: 0d6d4da992a24351e88da92f002a14c65d59f34e

.gitignore

@@ -7,3 +7,4 @@ runner/runner
 terraform.tfstate
 terraform.tfstate.backup
 *.retry
+build/**/.uptodate

README.md

@@ -2,6 +2,9 @@
 
 Included in this repo are tools shared by weave.git and scope.git.  They include
 
+- ```build```: a set of docker base-images for building weave
+  projects. These should be used instead of giving each project its
+  own build image.
 - ```provisioning```: a set of Terraform scripts to provision virtual machines in GCP, AWS or Digital Ocean.
 - ```config_management```: a set of Ansible playbooks to configure virtual machines for development, testing, etc.
 - ```cover```: a tool which merges overlapping coverage reports generated by go

build/Makefile

@@ -0,0 +1,46 @@
+.PHONY: all clean images
+.DEFAULT_GOAL := all
+
+# Boiler plate for bulding Docker containers.
+# All this must go at top of file I'm afraid.
+IMAGE_PREFIX := quay.io/weaveworks/build-
+IMAGE_TAG := $(shell ../image-tag)
+UPTODATE := .uptodate
+
+# Every directory with a Dockerfile in it builds an image called
+# $(IMAGE_PREFIX)<dirname>. Dependencies (i.e. things that go in the image)
+# still need to be explicitly declared.
+%/$(UPTODATE): %/Dockerfile %/*
+	$(SUDO) docker build -t $(IMAGE_PREFIX)$(shell basename $(@D)) $(@D)/
+	$(SUDO) docker tag $(IMAGE_PREFIX)$(shell basename $(@D)) $(IMAGE_PREFIX)$(shell basename $(@D)):$(IMAGE_TAG)
+	touch $@
+
+# Get a list of directories containing Dockerfiles
+DOCKERFILES := $(shell find . -name tools -prune -o -name vendor -prune -o -type f -name 'Dockerfile' -print)
+UPTODATE_FILES := $(patsubst %/Dockerfile,%/$(UPTODATE),$(DOCKERFILES))
+DOCKER_IMAGE_DIRS := $(patsubst %/Dockerfile,%,$(DOCKERFILES))
+IMAGE_NAMES := $(foreach dir,$(DOCKER_IMAGE_DIRS),$(patsubst %,$(IMAGE_PREFIX)%,$(shell basename $(dir))))
+images:
+	$(info $(IMAGE_NAMES))
+	@echo > /dev/null
+
+# Define imagetag-golang, etc, for each image, which parses the dockerfile and
+# prints an image tag. For example:
+#     FROM golang:1.8.1-stretch
+# in the "foo/Dockerfile" becomes:
+#     $ make imagetag-foo
+#     1.8.1-stretch
+define imagetag_dep
+.PHONY: imagetag-$(1)
+$(patsubst $(IMAGE_PREFIX)%,imagetag-%,$(1)): $(patsubst $(IMAGE_PREFIX)%,%,$(1))/Dockerfile
+	@cat $$< | grep "^FROM " | head -n1 | sed 's/FROM \(.*\):\(.*\)/\2/'
+endef
+$(foreach image, $(IMAGE_NAMES), $(eval $(call imagetag_dep, $(image))))
+
+all: $(UPTODATE_FILES)
+
+clean:
+	$(SUDO) docker rmi $(IMAGE_NAMES) >/dev/null 2>&1 || true
+	rm -rf $(UPTODATE_FILES)
+
+

build/golang/Dockerfile

@@ -0,0 +1,48 @@
+FROM golang:1.8.0-stretch
+RUN apt-get update && \
+    apt-get install -y \
+      curl \
+      file \
+      git \
+      jq \
+      libprotobuf-dev \
+      make \
+      protobuf-compiler \
+      python-pip \
+      python-requests \
+      python-yaml \
+      unzip && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+RUN pip install attrs pyhcl
+RUN curl -fsSL -o shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 && \
+	chmod +x shfmt && \
+	mv shfmt /usr/bin
+RUN go clean -i net && \
+	go install -tags netgo std && \
+	go install -race -tags netgo std
+RUN go get -tags netgo \
+		github.com/FiloSottile/gvt \
+		github.com/client9/misspell/cmd/misspell \
+		github.com/fatih/hclfmt \
+		github.com/fzipp/gocyclo \
+		github.com/gogo/protobuf/gogoproto \
+		github.com/gogo/protobuf/protoc-gen-gogoslick \
+		github.com/golang/dep/... \
+		github.com/golang/lint/golint \
+		github.com/golang/protobuf/protoc-gen-go \
+		github.com/kisielk/errcheck \
+		github.com/mjibson/esc \
+		github.com/prometheus/prometheus/cmd/promtool && \
+		rm -rf /go/pkg /go/src
+RUN mkdir protoc && \
+	cd protoc && \
+	curl -O -L https://github.com/google/protobuf/releases/download/v3.1.0/protoc-3.1.0-linux-x86_64.zip && \
+	unzip protoc-3.1.0-linux-x86_64.zip && \
+	cp bin/protoc /usr/bin/ && \
+	chmod o+x /usr/bin/protoc && \
+	cd .. && \
+	rm -rf protoc
+RUN mkdir -p /var/run/secrets/kubernetes.io/serviceaccount && \
+		touch /var/run/secrets/kubernetes.io/serviceaccount/token
+COPY build.sh /
+ENTRYPOINT ["/build.sh"]

build/golang/build.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+set -eu
+
+if [ -n "${SRC_NAME:-}" ]; then
+    SRC_PATH=${SRC_PATH:-$GOPATH/src/$SRC_NAME}
+elif [ -z "${SRC_PATH:-}" ]; then
+    echo "Must set either \$SRC_NAME or \$SRC_PATH."
+    exit 1
+fi
+
+# If we run make directly, any files created on the bind mount
+# will have awkward ownership.  So we switch to a user with the
+# same user and group IDs as source directory.  We have to set a
+# few things up so that sudo works without complaining later on.
+uid=$(stat --format="%u" "$SRC_PATH")
+gid=$(stat --format="%g" "$SRC_PATH")
+echo "weave:x:$uid:$gid::$SRC_PATH:/bin/sh" >>/etc/passwd
+echo "weave:*:::::::" >>/etc/shadow
+echo "weave	ALL=(ALL)	NOPASSWD: ALL" >>/etc/sudoers
+
+su weave -c "PATH=$PATH make -C $SRC_PATH BUILD_IN_CONTAINER=false $*"

build/haskell/Dockerfile

@@ -0,0 +1,4 @@
+FROM fpco/stack-build:lts-8.9
+COPY build.sh /
+COPY copy-libraries /usr/local/bin/
+ENTRYPOINT ["/build.sh"]

build/haskell/build.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+#
+# Build a static Haskell binary using stack.
+
+set -eu
+
+if [ -z "${SRC_PATH:-}" ]; then
+    echo "Must set \$SRC_PATH."
+    exit 1
+fi
+
+make -C "$SRC_PATH" BUILD_IN_CONTAINER=false "$@"

build/haskell/copy-libraries

@@ -0,0 +1,41 @@
+#!/bin/bash
+#
+# Copy dynamically linked libraries for a binary, so we can assemble a Docker
+# image.
+#
+# Run with:
+#   copy-libraries /path/to/binary /output/dir
+#
+# Dependencies:
+# - awk
+# - cp
+# - grep
+# - ldd
+# - mkdir
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# Path to a Linux binary that we're going to run in the container.
+binary_path="${1}"
+# Path to directory to write the output to.
+output_dir="${2}"
+
+exe_name=$(basename "${binary_path}")
+
+# Identify linked libraries.
+libraries=($(ldd "${binary_path}" | awk '{print $(NF-1)}' | grep -v '=>'))
+# Add /bin/sh, which we need for Docker imports.
+libraries+=('/bin/sh')
+
+mkdir -p "${output_dir}"
+
+# Copy executable and all needed libraries into temporary directory.
+cp "${binary_path}" "${output_dir}/${exe_name}"
+for lib in "${libraries[@]}"; do
+    mkdir -p "${output_dir}/$(dirname "$lib")"
+    # Need -L to make sure we get actual libraries & binaries, not symlinks to
+    # them.
+    cp -L "${lib}" "${output_dir}/${lib}"
+done

circle.yml

@@ -13,19 +13,41 @@ dependencies:
     - go install -tags netgo std
     - mkdir -p $(dirname $SRCDIR)
     - cp -r $(pwd)/ $SRCDIR
+    - |
+      curl -fsSL -o shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 && \
+      chmod +x shfmt && \
+      sudo mv shfmt /usr/bin
     - |
       cd $SRCDIR;
       go get \
         github.com/fzipp/gocyclo \
         github.com/golang/lint/golint \
         github.com/kisielk/errcheck \
-        github.com/fatih/hclfmt \
-        gopkg.in/mvdan/sh.v1/cmd/shfmt
+        github.com/fatih/hclfmt
 
 test:
   override:
     - cd $SRCDIR; ./lint .
     - cd $SRCDIR/cover; make
     - cd $SRCDIR/socks; make
     - cd $SRCDIR/runner; make
+    - cd $SRCDIR/build; make
+
+deployment:
+  snapshot:
+    branch: master
+    commands:
+      - docker login -e "$DOCKER_REGISTRY_EMAIL" -u "$DOCKER_REGISTRY_USER" -p "$DOCKER_REGISTRY_PASS" "$DOCKER_REGISTRY_URL"
+      - |
+        cd $SRCDIR/build;
+        for image in $(make images); do
+          # Tag the built images with the revision of this repo.
+          docker push "${image}:${GIT_TAG}"
 
+          # Tag the built images with something derived from the base images in
+          # their respective Dockerfiles. So "FROM golang:1.8.0-stretch" as a
+          # base image would lead to a tag of "1.8.0-stretch"
+          IMG_TAG=$(make "imagetag-${image#quay.io/weaveworks/build-}")
+          docker tag "${image}:latest" "${image}:${IMG_TAG}"
+          docker push "${image}:${IMG_TAG}"
+        done

config_management/group_vars/all

@@ -1,10 +1,11 @@
 ---
-go_version: 1.7.4
+go_version: 1.8.1
 terraform_version: 0.8.5
 docker_version: 1.11.2
-kubernetes_version: 1.5.2
-kubernetes_cni_version: 0.3.0.1
-kubernetes_token: 123456.0123456789123456
+docker_install_role: 'docker-from-get.docker.com'
+kubernetes_version: 1.6.1
+kubernetes_cni_version: 0.5.1
+kubernetes_token: '123456.0123456789123456'
 etcd_container_version: 2.2.5
 kube_discovery_container_version: 1.0
 pause_container_version: 3.0

config_management/roles/dev-tools/tasks/main.yml

@@ -5,7 +5,7 @@
   package:
     name: "{{ item }}"
     state: present
-  with_items: 
+  with_items:
     # weave net dependencies
     - make
     - vagrant

config_management/roles/docker-configuration/files/docker.conf

@@ -0,0 +1,3 @@
+[Service]
+ExecStart=
+ExecStart=/usr/bin/docker daemon -H fd:// -H unix:///var/run/alt-docker.sock -H tcp://0.0.0.0:2375 -s overlay --insecure-registry "weave-ci-registry:5000"

config_management/roles/docker-configuration/tasks/main.yml

@@ -18,17 +18,19 @@
     path: /etc/systemd/system/docker.service.d
     state: directory
     recurse: yes
+  when: ansible_os_family != "RedHat"
 
 - name: enable docker remote api over tcp
   copy:
-    src: "{{ role_path }}/files/docker_over_tcp.conf"
-    dest: /etc/systemd/system/docker.service.d/docker_over_tcp.conf
-  register: docker_over_tcp
+    src: "{{ role_path }}/files/docker.conf"
+    dest: /etc/systemd/system/docker.service.d/docker.conf
+  register: docker_conf
+  when: ansible_os_family != "RedHat"
 
 - name: restart docker service
-  systemd: 
+  systemd:
     name: docker
     state: restarted
-    daemon_reload: yes  # ensure docker_over_tcp.conf is picked up.
+    daemon_reload: yes  # ensure docker.conf is picked up.
     enabled: yes
-  when: docker_over_tcp.changed
+  when: docker_conf.changed or ansible_os_family == "RedHat"

config_management/roles/docker-from-docker-ce-repo/tasks/main.yml

@@ -0,0 +1,29 @@
+# Docker installation from Docker's CentOS Community Edition
+# See also: https://docs.docker.com/engine/installation/linux/centos/
+
+- name: remove all potentially pre existing packages
+  yum:
+    name: '{{ item }}'
+    state: absent
+  with_items:
+    - docker
+    - docker-common
+    - container-selinux
+    - docker-selinux
+    - docker-engine
+
+- name: install yum-utils
+  yum:
+    name: yum-utils
+    state: present
+
+- name: add docker ce repo
+  command: yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
+
+# Note that Docker CE versions do not follow regular Docker versions, but look
+# like, for example: "17.03.0.el7"
+- name: install docker
+  yum:
+    name: 'docker-ce-{{ docker_version }}'
+    update_cache: yes
+    state: present

config_management/roles/docker-from-docker-repo/tasks/debian.yml

@@ -5,12 +5,12 @@
   package:
     name: "{{ item }}"
     state: present
-  with_items: 
+  with_items:
     - apt-transport-https
     - ca-certificates
 
 - name: add apt key for the docker repository
-  apt_key: 
+  apt_key:
     keyserver: hkp://ha.pool.sks-keyservers.net:80
     id: 58118E89F3A912897C070ADBF76221572C52609D
     state: present
@@ -23,13 +23,13 @@
   register: apt_docker_repo
 
 - name: update apt's cache
-  apt: 
+  apt:
     update_cache: yes
   when: apt_key_docker_repo.changed or apt_docker_repo.changed
 
 - name: install docker-engine
   package:
     name: "{{ item }}"
     state: present
-  with_items: 
+  with_items:
     - docker-engine={{ docker_version }}*

config_management/roles/docker-from-docker-repo/tasks/main.yml

@@ -2,15 +2,9 @@
 # Set up Docker
 # See also: https://docs.docker.com/engine/installation/linux/ubuntulinux/#install
 
-- include_role:
-    name: docker-prerequisites
-
 # Distribution-specific tasks:
 - include: debian.yml
   when: ansible_os_family == "Debian"
 
 - include: redhat.yml
   when: ansible_os_family == "RedHat"
-
-- include_role:
-    name: docker-configuration

config_management/roles/docker-from-docker-repo/tasks/redhat.yml

@@ -13,13 +13,13 @@
     state: present
 
 - name: update yum's cache
-  yum: 
+  yum:
     name: "*"
     update_cache: yes
 
 - name: install docker-engine
   package:
     name: "{{ item }}"
     state: present
-  with_items: 
+  with_items:
     - docker-engine-{{ docker_version }}

config_management/roles/docker-from-get.docker.com/tasks/debian.yml

@@ -0,0 +1,8 @@
+---
+# Debian / Ubuntu specific:
+
+- name: apt-import gpg key for the docker repository
+  shell: curl -sSL https://get.docker.com/gpg | sudo apt-key add -
+
+- name: install docker
+  shell: 'curl -sSL https://get.docker.com/ | sed -e s/docker-engine/docker-engine={{ docker_version }}*/ | sh'