From bd8002ed4d3ac311c03bb846610131b07b791849 Mon Sep 17 00:00:00 2001 From: Marcus Cobden Date: Fri, 28 Jul 2017 14:52:24 +0100 Subject: [PATCH] Squashed 'tools/' changes from b783528b..35679ee5 (#521) 35679ee5 Merge pull request #110 from weaveworks/parallel-push-errors 3ae41b6f Remove unneeded if block 51ff31a5 Exit on first error 0faad9f7 Check for errors when pushing images in parallel 74dc626b Merge pull request #108 from weaveworks/disable-apt-daily b4f1d918 Merge pull request #107 from weaveworks/docker-17-update 7436aa14 Override apt daily job to not run immediately on boot 7980f152 Merge pull request #106 from weaveworks/document-docker-install-role f741e533 Bump to Docker 17.06 from CE repo 61796a1b Update Docker CE Debian repo details 0d86f5e1 Allow for Docker package to be named docker-ce 065c68d4 Document selection of Docker installation role. 38090539 Just --porcelain; it defaults to v1 11400eaa Merge pull request #105 from weaveworks/remove-weaveplugin-remnants b8b4d64c remove weaveplugin remnants 35099c93 Merge pull request #104 from weaveworks/pull-docker-py cdd48fc3 Pull docker-py to speed tests/builds up. e1c6c24b Merge pull request #103 from weaveworks/test-build-tags d5d71e06 Add -tags option so callers can pass in build tags 8949b2b5 Merge pull request #98 from weaveworks/git-status-tag ac30687f Merge pull request #100 from weaveworks/python_linting 4b125b55 Pin yapf & flake8 versions 7efb4853 Lint python linting function 444755b7 Swap diff direction to reflect changes required c5b24346 Install flake8 & yapf 5600eac5 Lint python in build-tools repo 0b02ca93 Add python linting c011c0dc Merge pull request #79 from kinvolk/schu/python-shebang 6577d078 Merge pull request #99 from weaveworks/shfmt-version 00ce0dcd Use git status instead of diff to add 'WIP' tag 411fd13a Use shfmt v1.3.0 instead of latest from master. 0d6d4da9 Run shfmt 1.3 on the code. 5cdba320 Add sudo c322ca83 circle.yml: Install shfmt binary. e59c2251 Install shfmt 1.3 binary. 30706e6b Install pyhcl in the build container. 960d2228 Merge pull request #97 from kinvolk/alban/update-shfmt-3 1d535c7b shellcheck: fix escaping issue 55424986 Merge pull request #96 from kinvolk/alban/update-shfmt-2 32f7cc51 shfmt: fix coding style 09f72af0 lint: print the diff in case of error 571c7d71 Merge pull request #95 from kinvolk/alban/update-shfmt bead6edd Update for latest shfmt b08dc4d6 Update for latest shfmt (#94) 2ed8aaa8 Add no-race argument to test script (#92) 80dd78e6 Merge pull request #91 from weaveworks/upgrade-go-1.8.1 08dcd0df Please ./lint as shfmt changed its rules between 1.0.0 and 1.3.0. a8bc9ab0 Upgrade default Go version to 1.8.1. 41c56221 Merge pull request #90 from weaveworks/build-golang-service-conf e8ebdd5e broaden imagetag regex to fix haskell build image ba3fbfa6 Merge pull request #89 from weaveworks/build-golang-service-conf e506f1b9 Fix up test script for updated shfmt 9216db86 Add stuff for service-conf build to build-goland image 66a9a93c Merge pull request #88 from weaveworks/haskell-image cb3e3a25 shfmt 74a5239e Haskell build image 4ccd42b9 Trying circle quay login b2c295fc Merge branch 'common-build' 0ac746fa Trim quay prefix in circle script c405b311 Merge pull request #87 from weaveworks/common-build 9672d7cb Push build images to quay as they have sane robot accounts a2bf1123 Review feedback fef9b7dd Add protobuf tools 10a77ead Update readme 254f2660 Don't need the image name in ffb59fcb Adding a weaveworks/build-golang image with tags b8173683 Update min Weave Net docker version cf87ca33 Merge pull request #86 from weaveworks/lock-kubeadm-version 3ae69196 Add example of custom SSH private key to tf_ssh's usage. cf8bd8af Add example of custom SSH private key to tf_ansi's usage. c7d33700 Lock kubeadm's Kubernetes version. faaaa6f2 Merge pull request #84 from weaveworks/centos-rhel ef552e7d Select weave-kube YAML URL based on K8S version. b4c11982 Upgrade default kubernetes_version to 1.6.1. b82805ef Use a fixed version of kubeadm. f33888bc Factorise and make kubeconfig option optional. f7b8b897 Install EPEL repo for CentOS. 615917a6 Fix error in decrypting AWS access key and secret. 86f97b42 Add CentOS 7 AMI and username for AWS via Terraform. eafd810e Add tf_ansi example with Ansible variables. 2b05787f Skip setup of Docker over TCP for CentOS/RHEL. 84c420b6 Add docker-ce role for CentOS/RHEL. 00a820cd Add setup_weave-net_debug.yml playbook for user issues' debugging. 3eae480b Upgrade default kubernetes_version to 1.5.4. 753921cf Allow injection of Docker installation role. e1ff90d2 Fix kubectl taint command for 1.5. b989e97a Fix typo in kubectl taint for single node K8S cluster. 541f58d2 Remove 'install_recommends: no' for ethtool. c3f97115 Make Ansible role docker-from-get.docker.com work on RHEL/CentOS. 038c0ae8 Add frequently used OS images, for convenience. d30649f0 Add --insecure-registry to docker.conf 1dd92188 shfmt -i 4 -w push-images 6de96ac7 Add option to not push docker hub images 310f53dc Add push-images script from cortex 8641381b Add port 6443 to kubeadm join commands for K8S 1.6+. 50bf0bc6 Force type of K8S token to string. 08ab1c0c Remove trailing whitespaces. ae9efb83 Enable testing against K8S release candidates. 9e32194c Secure GCP servers for Scope: open port 80. a22536ac Secure GCP servers for Scope. 89c3a29b Merge pull request #78 from weaveworks/lint-merge-rebase-issue-in-docs 73ad56d7 Add linter function to avoid bad merge/rebase artefact 31d069d6 Change Python shebang to `#!/usr/bin/env python` 52d695cc Merge pull request #77 from kinvolk/schu/fix-relative-weave-path 77aed016 Merge pull request #73 from weaveworks/mike/sched/fix-unicode-issue 7c080f45 integration/sanity_check: disable SC1090 d6d360a1 integration/gce.sh: update gcloud command e8def2c4 provisioning/setup: fix shellcheck SC2140 cc022241 integration/config: fix weave path 9c0d6a55 Fix config_management/README.md 334708ca Merge pull request #75 from kinvolk/alban/external-build-1 da2505d6 gce.sh: template: print creation date e6768547 integration tests: fix user account 85308369 host nameing: add repo name b556c0ad gce.sh: fix deletion of gce instances 2ecd1c2c integration: fix GCE --zones/--zone parameter 3e863df9 sched: Fix unicode encoding issues 51785b5f Use rm -f and set current dir using BASH_SOURCE. f5c6d68c Merge pull request #71 from kinvolk/schu/fix-linter-warnings 0269628e Document requirement for `lint_sh` 9a3f09e6 Fix linter warnings efcf9d21 Merge pull request #53 from weaveworks/2647-testing-mvp d31ea574 Weave Kube playbook now works with multiple nodes. 27868dd5 Add GCP firewall rule for FastDP crypto. edc8bb3a Differentiated name of dev and test playbooks, to avoid confusion. efa3df7c Moved utility Ansible Yaml to library directory. fcd2769e Add shorthands to run Ansible playbooks against Terraform-provisioned virtual machines. f7946fbb Add shorthands to SSH into Terraform-provisioned virtual machines. aad5c6f1 Mention Terraform and Ansible in README.md. dddabf0e Add Terraform output required for templates' creation. dcc7d025 Add Ansible configuration playbooks for development environments. f86481ca Add Ansible configuration playbooks for Docker, K8S and Weave-Net. efedd258 Git-ignore Ansible retry files. 765c4ca1 Add helper functions to setup Terraform programmatically. 801dd1d1 Add Terraform cloud provisioning scripts. b8017e19 Install hclfmt on CircleCI. 4815e19b Git-ignore Terraform state files. 0aaebc7d Add script to generate cartesian product of dependencies of cross-version testing. 007d90ab Add script to list OS images from GCP, AWS and DO. ca65cc02 Add script to list relevant versions of Go, Docker and Kubernetes. aa66f447 Scripts now source dependencies using absolute path (previously breaking make depending on current directory). 7865e86d Add -p option to parallelise lint. 36c1835b Merge pull request #69 from weaveworks/mflag 98575686 Use mflag and mflagext package from weaveworks/common. 97991128 Quote bash variable. 10a36b33 Merge pull request #67 from weaveworks/shfmt-ignore a59884f3 Add support for .lintignore. 03cc5989 Don't lint generated protobuf code. 2b55c2df Merge pull request #66 from weaveworks/reduce-test-timeout d4e163cc Make timeout a flag 49a86091 Reduce test timeout 8fa15cba Merge pull request #63 from weaveworks/test-defaults git-subtree-dir: tools git-subtree-split: 35679ee5ff17c4edf864b7c43dc70a40337fcd80 --- tools/.gitignore | 4 + tools/README.md | 14 +- tools/build/Makefile | 46 +++ tools/build/golang/Dockerfile | 49 +++ tools/build/golang/build.sh | 22 ++ tools/build/haskell/Dockerfile | 4 + tools/build/haskell/build.sh | 12 + tools/build/haskell/copy-libraries | 41 ++ tools/circle.yml | 27 +- tools/config_management/README.md | 141 +++++++ tools/config_management/group_vars/all | 11 + .../library/setup_ansible_dependencies.yml | 33 ++ .../dev-tools/files/apt-daily.timer.conf | 2 + .../roles/dev-tools/tasks/main.yml | 48 +++ .../docker-configuration/files/docker.conf | 3 + .../roles/docker-configuration/tasks/main.yml | 36 ++ .../tasks/debian.yml | 35 ++ .../docker-from-docker-ce-repo/tasks/main.yml | 10 + .../tasks/redhat.yml | 29 ++ .../docker-from-docker-repo/tasks/debian.yml | 35 ++ .../docker-from-docker-repo/tasks/main.yml | 10 + .../docker-from-docker-repo/tasks/redhat.yml | 25 ++ .../tasks/debian.yml | 8 + .../docker-from-get.docker.com/tasks/main.yml | 10 + .../tasks/redhat.yml | 11 + .../roles/docker-from-tarball/tasks/main.yml | 61 +++ .../roles/docker-from-tarball/vars/main.yml | 5 + .../roles/docker-install/tasks/main.yml | 30 ++ .../docker-prerequisites/tasks/debian.yml | 11 + .../roles/docker-prerequisites/tasks/main.yml | 5 + .../roles/golang-from-tarball/tasks/main.yml | 36 ++ .../roles/kubelet-stop/tasks/main.yml | 14 + .../kubernetes-docker-images/tasks/main.yml | 14 + .../roles/kubernetes-install/tasks/debian.yml | 37 ++ .../roles/kubernetes-install/tasks/main.yml | 16 + .../roles/kubernetes-install/tasks/redhat.yml | 30 ++ .../roles/kubernetes-start/tasks/main.yml | 42 ++ .../roles/setup-ansible/pre_tasks/main.yml | 26 ++ .../roles/sock-shop/tasks/tasks.yml | 34 ++ .../roles/weave-kube/tasks/main.yml | 24 ++ .../roles/weave-net-sources/tasks/main.yml | 24 ++ .../roles/weave-net-utilities/tasks/main.yml | 56 +++ .../roles/weave-net/tasks/main.yml | 26 ++ tools/config_management/setup_weave-kube.yml | 27 ++ .../setup_weave-net_debug.yml | 18 + .../config_management/setup_weave-net_dev.yml | 20 + .../setup_weave-net_test.yml | 20 + tools/dependencies/cross_versions.py | 91 +++++ tools/dependencies/list_os_images.sh | 85 +++++ tools/dependencies/list_versions.py | 343 +++++++++++++++++ tools/image-tag | 2 +- tools/integration/config.sh | 7 +- tools/integration/gce.sh | 33 +- tools/integration/sanity_check.sh | 4 +- tools/lint | 42 +- tools/provisioning/README.md | 55 +++ tools/provisioning/aws/README.md | 90 +++++ tools/provisioning/aws/main.tf | 137 +++++++ tools/provisioning/aws/outputs.tf | 54 +++ tools/provisioning/aws/variables.tf | 68 ++++ tools/provisioning/do/README.md | 98 +++++ tools/provisioning/do/main.tf | 42 ++ tools/provisioning/do/outputs.tf | 57 +++ tools/provisioning/do/variables.tf | 185 +++++++++ tools/provisioning/gcp/README.md | 126 ++++++ tools/provisioning/gcp/main.tf | 79 ++++ tools/provisioning/gcp/outputs.tf | 66 ++++ tools/provisioning/gcp/variables.tf | 77 ++++ tools/provisioning/setup.sh | 361 ++++++++++++++++++ tools/push-images | 53 +++ tools/runner/runner.go | 3 +- tools/sched | 12 +- tools/scheduler/main.py | 269 +++++++------ tools/test | 34 +- 74 files changed, 3556 insertions(+), 159 deletions(-) create mode 100644 tools/build/Makefile create mode 100644 tools/build/golang/Dockerfile create mode 100755 tools/build/golang/build.sh create mode 100644 tools/build/haskell/Dockerfile create mode 100755 tools/build/haskell/build.sh create mode 100755 tools/build/haskell/copy-libraries create mode 100644 tools/config_management/README.md create mode 100644 tools/config_management/group_vars/all create mode 100644 tools/config_management/library/setup_ansible_dependencies.yml create mode 100644 tools/config_management/roles/dev-tools/files/apt-daily.timer.conf create mode 100644 tools/config_management/roles/dev-tools/tasks/main.yml create mode 100644 tools/config_management/roles/docker-configuration/files/docker.conf create mode 100644 tools/config_management/roles/docker-configuration/tasks/main.yml create mode 100644 tools/config_management/roles/docker-from-docker-ce-repo/tasks/debian.yml create mode 100644 tools/config_management/roles/docker-from-docker-ce-repo/tasks/main.yml create mode 100644 tools/config_management/roles/docker-from-docker-ce-repo/tasks/redhat.yml create mode 100644 tools/config_management/roles/docker-from-docker-repo/tasks/debian.yml create mode 100644 tools/config_management/roles/docker-from-docker-repo/tasks/main.yml create mode 100644 tools/config_management/roles/docker-from-docker-repo/tasks/redhat.yml create mode 100644 tools/config_management/roles/docker-from-get.docker.com/tasks/debian.yml create mode 100644 tools/config_management/roles/docker-from-get.docker.com/tasks/main.yml create mode 100644 tools/config_management/roles/docker-from-get.docker.com/tasks/redhat.yml create mode 100644 tools/config_management/roles/docker-from-tarball/tasks/main.yml create mode 100644 tools/config_management/roles/docker-from-tarball/vars/main.yml create mode 100644 tools/config_management/roles/docker-install/tasks/main.yml create mode 100644 tools/config_management/roles/docker-prerequisites/tasks/debian.yml create mode 100644 tools/config_management/roles/docker-prerequisites/tasks/main.yml create mode 100644 tools/config_management/roles/golang-from-tarball/tasks/main.yml create mode 100644 tools/config_management/roles/kubelet-stop/tasks/main.yml create mode 100644 tools/config_management/roles/kubernetes-docker-images/tasks/main.yml create mode 100644 tools/config_management/roles/kubernetes-install/tasks/debian.yml create mode 100644 tools/config_management/roles/kubernetes-install/tasks/main.yml create mode 100644 tools/config_management/roles/kubernetes-install/tasks/redhat.yml create mode 100644 tools/config_management/roles/kubernetes-start/tasks/main.yml create mode 100644 tools/config_management/roles/setup-ansible/pre_tasks/main.yml create mode 100644 tools/config_management/roles/sock-shop/tasks/tasks.yml create mode 100644 tools/config_management/roles/weave-kube/tasks/main.yml create mode 100644 tools/config_management/roles/weave-net-sources/tasks/main.yml create mode 100644 tools/config_management/roles/weave-net-utilities/tasks/main.yml create mode 100644 tools/config_management/roles/weave-net/tasks/main.yml create mode 100644 tools/config_management/setup_weave-kube.yml create mode 100644 tools/config_management/setup_weave-net_debug.yml create mode 100644 tools/config_management/setup_weave-net_dev.yml create mode 100644 tools/config_management/setup_weave-net_test.yml create mode 100755 tools/dependencies/cross_versions.py create mode 100755 tools/dependencies/list_os_images.sh create mode 100755 tools/dependencies/list_versions.py create mode 100755 tools/provisioning/README.md create mode 100644 tools/provisioning/aws/README.md create mode 100755 tools/provisioning/aws/main.tf create mode 100755 tools/provisioning/aws/outputs.tf create mode 100755 tools/provisioning/aws/variables.tf create mode 100755 tools/provisioning/do/README.md create mode 100755 tools/provisioning/do/main.tf create mode 100755 tools/provisioning/do/outputs.tf create mode 100755 tools/provisioning/do/variables.tf create mode 100755 tools/provisioning/gcp/README.md create mode 100755 tools/provisioning/gcp/main.tf create mode 100755 tools/provisioning/gcp/outputs.tf create mode 100755 tools/provisioning/gcp/variables.tf create mode 100755 tools/provisioning/setup.sh create mode 100755 tools/push-images diff --git a/tools/.gitignore b/tools/.gitignore index b6ea60f8fb..308ae9d367 100644 --- a/tools/.gitignore +++ b/tools/.gitignore @@ -4,3 +4,7 @@ socks/image.tar runner/runner *.pyc *~ +terraform.tfstate +terraform.tfstate.backup +*.retry +build/**/.uptodate diff --git a/tools/README.md b/tools/README.md index e570ef7177..9092b8e24d 100644 --- a/tools/README.md +++ b/tools/README.md @@ -2,12 +2,17 @@ Included in this repo are tools shared by weave.git and scope.git. They include +- ```build```: a set of docker base-images for building weave + projects. These should be used instead of giving each project its + own build image. +- ```provisioning```: a set of Terraform scripts to provision virtual machines in GCP, AWS or Digital Ocean. +- ```config_management```: a set of Ansible playbooks to configure virtual machines for development, testing, etc. - ```cover```: a tool which merges overlapping coverage reports generated by go test - ```files-with-type```: a tool to search directories for files of a given MIME type -- ```lint```: a script to lint Go project; runs various tools like golint, go - vet, errcheck etc +- ```lint```: a script to lint go, sh and hcl files; runs various tools like + golint, go vet, errcheck, shellcheck etc - ```rebuild-image```: a script to rebuild docker images when their input files change; useful when you using docker images to build your software, but you don't want to build the image every time. @@ -24,6 +29,11 @@ Included in this repo are tools shared by weave.git and scope.git. They include - ```scheduler```: an appengine application that can be used to distribute tests across different shards in CircleCI. +## Requirements + +- ```lint``` requires shfmt to lint sh files; get shfmt with + ```go get -u gopkg.in/mvdan/sh.v1/cmd/shfmt``` + ## Using build-tools.git To allow you to tie your code to a specific version of build-tools.git, such diff --git a/tools/build/Makefile b/tools/build/Makefile new file mode 100644 index 0000000000..cea049be56 --- /dev/null +++ b/tools/build/Makefile @@ -0,0 +1,46 @@ +.PHONY: all clean images +.DEFAULT_GOAL := all + +# Boiler plate for bulding Docker containers. +# All this must go at top of file I'm afraid. +IMAGE_PREFIX := quay.io/weaveworks/build- +IMAGE_TAG := $(shell ../image-tag) +UPTODATE := .uptodate + +# Every directory with a Dockerfile in it builds an image called +# $(IMAGE_PREFIX). Dependencies (i.e. things that go in the image) +# still need to be explicitly declared. +%/$(UPTODATE): %/Dockerfile %/* + $(SUDO) docker build -t $(IMAGE_PREFIX)$(shell basename $(@D)) $(@D)/ + $(SUDO) docker tag $(IMAGE_PREFIX)$(shell basename $(@D)) $(IMAGE_PREFIX)$(shell basename $(@D)):$(IMAGE_TAG) + touch $@ + +# Get a list of directories containing Dockerfiles +DOCKERFILES := $(shell find . -name tools -prune -o -name vendor -prune -o -type f -name 'Dockerfile' -print) +UPTODATE_FILES := $(patsubst %/Dockerfile,%/$(UPTODATE),$(DOCKERFILES)) +DOCKER_IMAGE_DIRS := $(patsubst %/Dockerfile,%,$(DOCKERFILES)) +IMAGE_NAMES := $(foreach dir,$(DOCKER_IMAGE_DIRS),$(patsubst %,$(IMAGE_PREFIX)%,$(shell basename $(dir)))) +images: + $(info $(IMAGE_NAMES)) + @echo > /dev/null + +# Define imagetag-golang, etc, for each image, which parses the dockerfile and +# prints an image tag. For example: +# FROM golang:1.8.1-stretch +# in the "foo/Dockerfile" becomes: +# $ make imagetag-foo +# 1.8.1-stretch +define imagetag_dep +.PHONY: imagetag-$(1) +$(patsubst $(IMAGE_PREFIX)%,imagetag-%,$(1)): $(patsubst $(IMAGE_PREFIX)%,%,$(1))/Dockerfile + @cat $$< | grep "^FROM " | head -n1 | sed 's/FROM \(.*\):\(.*\)/\2/' +endef +$(foreach image, $(IMAGE_NAMES), $(eval $(call imagetag_dep, $(image)))) + +all: $(UPTODATE_FILES) + +clean: + $(SUDO) docker rmi $(IMAGE_NAMES) >/dev/null 2>&1 || true + rm -rf $(UPTODATE_FILES) + + diff --git a/tools/build/golang/Dockerfile b/tools/build/golang/Dockerfile new file mode 100644 index 0000000000..8ef1d2b04c --- /dev/null +++ b/tools/build/golang/Dockerfile @@ -0,0 +1,49 @@ +FROM golang:1.8.0-stretch +RUN apt-get update && \ + apt-get install -y \ + curl \ + file \ + git \ + jq \ + libprotobuf-dev \ + make \ + protobuf-compiler \ + python-pip \ + python-requests \ + python-yaml \ + unzip && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +RUN pip install attrs pyhcl +RUN curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 && \ + echo "b1925c2c405458811f0c227266402cf1868b4de529f114722c2e3a5af4ac7bb2 shfmt" | sha256sum -c && \ + chmod +x shfmt && \ + mv shfmt /usr/bin +RUN go clean -i net && \ + go install -tags netgo std && \ + go install -race -tags netgo std +RUN go get -tags netgo \ + github.com/FiloSottile/gvt \ + github.com/client9/misspell/cmd/misspell \ + github.com/fatih/hclfmt \ + github.com/fzipp/gocyclo \ + github.com/gogo/protobuf/gogoproto \ + github.com/gogo/protobuf/protoc-gen-gogoslick \ + github.com/golang/dep/... \ + github.com/golang/lint/golint \ + github.com/golang/protobuf/protoc-gen-go \ + github.com/kisielk/errcheck \ + github.com/mjibson/esc \ + github.com/prometheus/prometheus/cmd/promtool && \ + rm -rf /go/pkg /go/src +RUN mkdir protoc && \ + cd protoc && \ + curl -O -L https://github.com/google/protobuf/releases/download/v3.1.0/protoc-3.1.0-linux-x86_64.zip && \ + unzip protoc-3.1.0-linux-x86_64.zip && \ + cp bin/protoc /usr/bin/ && \ + chmod o+x /usr/bin/protoc && \ + cd .. && \ + rm -rf protoc +RUN mkdir -p /var/run/secrets/kubernetes.io/serviceaccount && \ + touch /var/run/secrets/kubernetes.io/serviceaccount/token +COPY build.sh / +ENTRYPOINT ["/build.sh"] diff --git a/tools/build/golang/build.sh b/tools/build/golang/build.sh new file mode 100755 index 0000000000..cf70e1c540 --- /dev/null +++ b/tools/build/golang/build.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +set -eu + +if [ -n "${SRC_NAME:-}" ]; then + SRC_PATH=${SRC_PATH:-$GOPATH/src/$SRC_NAME} +elif [ -z "${SRC_PATH:-}" ]; then + echo "Must set either \$SRC_NAME or \$SRC_PATH." + exit 1 +fi + +# If we run make directly, any files created on the bind mount +# will have awkward ownership. So we switch to a user with the +# same user and group IDs as source directory. We have to set a +# few things up so that sudo works without complaining later on. +uid=$(stat --format="%u" "$SRC_PATH") +gid=$(stat --format="%g" "$SRC_PATH") +echo "weave:x:$uid:$gid::$SRC_PATH:/bin/sh" >>/etc/passwd +echo "weave:*:::::::" >>/etc/shadow +echo "weave ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers + +su weave -c "PATH=$PATH make -C $SRC_PATH BUILD_IN_CONTAINER=false $*" diff --git a/tools/build/haskell/Dockerfile b/tools/build/haskell/Dockerfile new file mode 100644 index 0000000000..8d40c66241 --- /dev/null +++ b/tools/build/haskell/Dockerfile @@ -0,0 +1,4 @@ +FROM fpco/stack-build:lts-8.9 +COPY build.sh / +COPY copy-libraries /usr/local/bin/ +ENTRYPOINT ["/build.sh"] diff --git a/tools/build/haskell/build.sh b/tools/build/haskell/build.sh new file mode 100755 index 0000000000..e80d2abbea --- /dev/null +++ b/tools/build/haskell/build.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# +# Build a static Haskell binary using stack. + +set -eu + +if [ -z "${SRC_PATH:-}" ]; then + echo "Must set \$SRC_PATH." + exit 1 +fi + +make -C "$SRC_PATH" BUILD_IN_CONTAINER=false "$@" diff --git a/tools/build/haskell/copy-libraries b/tools/build/haskell/copy-libraries new file mode 100755 index 0000000000..18cbba6085 --- /dev/null +++ b/tools/build/haskell/copy-libraries @@ -0,0 +1,41 @@ +#!/bin/bash +# +# Copy dynamically linked libraries for a binary, so we can assemble a Docker +# image. +# +# Run with: +# copy-libraries /path/to/binary /output/dir +# +# Dependencies: +# - awk +# - cp +# - grep +# - ldd +# - mkdir + +set -o errexit +set -o nounset +set -o pipefail + +# Path to a Linux binary that we're going to run in the container. +binary_path="${1}" +# Path to directory to write the output to. +output_dir="${2}" + +exe_name=$(basename "${binary_path}") + +# Identify linked libraries. +libraries=($(ldd "${binary_path}" | awk '{print $(NF-1)}' | grep -v '=>')) +# Add /bin/sh, which we need for Docker imports. +libraries+=('/bin/sh') + +mkdir -p "${output_dir}" + +# Copy executable and all needed libraries into temporary directory. +cp "${binary_path}" "${output_dir}/${exe_name}" +for lib in "${libraries[@]}"; do + mkdir -p "${output_dir}/$(dirname "$lib")" + # Need -L to make sure we get actual libraries & binaries, not symlinks to + # them. + cp -L "${lib}" "${output_dir}/${lib}" +done diff --git a/tools/circle.yml b/tools/circle.yml index 6751c66c70..976a68cc9e 100644 --- a/tools/circle.yml +++ b/tools/circle.yml @@ -13,13 +13,19 @@ dependencies: - go install -tags netgo std - mkdir -p $(dirname $SRCDIR) - cp -r $(pwd)/ $SRCDIR + - | + curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 && \ + echo "b1925c2c405458811f0c227266402cf1868b4de529f114722c2e3a5af4ac7bb2 shfmt" | sha256sum -c && \ + chmod +x shfmt && \ + sudo mv shfmt /usr/bin - | cd $SRCDIR; go get \ github.com/fzipp/gocyclo \ github.com/golang/lint/golint \ github.com/kisielk/errcheck \ - gopkg.in/mvdan/sh.v1/cmd/shfmt + github.com/fatih/hclfmt + - pip install yapf==0.16.2 flake8==3.3.0 test: override: @@ -27,4 +33,23 @@ test: - cd $SRCDIR/cover; make - cd $SRCDIR/socks; make - cd $SRCDIR/runner; make + - cd $SRCDIR/build; make + +deployment: + snapshot: + branch: master + commands: + - docker login -e "$DOCKER_REGISTRY_EMAIL" -u "$DOCKER_REGISTRY_USER" -p "$DOCKER_REGISTRY_PASS" "$DOCKER_REGISTRY_URL" + - | + cd $SRCDIR/build; + for image in $(make images); do + # Tag the built images with the revision of this repo. + docker push "${image}:${GIT_TAG}" + # Tag the built images with something derived from the base images in + # their respective Dockerfiles. So "FROM golang:1.8.0-stretch" as a + # base image would lead to a tag of "1.8.0-stretch" + IMG_TAG=$(make "imagetag-${image#quay.io/weaveworks/build-}") + docker tag "${image}:latest" "${image}:${IMG_TAG}" + docker push "${image}:${IMG_TAG}" + done diff --git a/tools/config_management/README.md b/tools/config_management/README.md new file mode 100644 index 0000000000..bf1f6f65f8 --- /dev/null +++ b/tools/config_management/README.md @@ -0,0 +1,141 @@ +# Weaveworks configuration management + +## Introduction + +This project allows you to configure a machine with: + +* Docker and Weave Net for development: `setup_weave-net_dev.yml` +* Docker and Weave Net for testing: `setup_weave-net_test.yml` +* Docker, Kubernetes and Weave Kube (CNI plugin): `setup_weave-kube.yml` + +You can then use these environments for development, testing and debugging. + +## Set up + +You will need [Python](https://www.python.org/downloads/) and [Ansible 2.+](http://docs.ansible.com/ansible/intro_installation.html) installed on your machine and added to your `PATH` in order to be able to configure environments automatically. + +* On any platform, if you have Python installed: `pip install ansible` +* On macOS: `brew install ansible` +* On Linux (via Aptitude): `sudo apt install ansible` +* On Linux (via YUM): `sudo yum install ansible` +* For other platforms or more details, see [here](http://docs.ansible.com/ansible/intro_installation.html) + +Frequent errors during installation are: + +* `fatal error: Python.h: No such file or directory`: install `python-dev` +* `fatal error: ffi.h: No such file or directory`: install `libffi-dev` +* `fatal error: openssl/opensslv.h: No such file or directory`: install `libssl-dev` + +Full steps for a blank Ubuntu/Debian Linux machine: + + sudo apt-get install -qq -y python-pip python-dev libffi-dev libssl-dev + sudo pip install -U cffi + sudo pip install ansible + +## Tags + +These can be used to selectively run (`--tags "tag1,tag2"`) or skip (`--skip-tags "tag1,tag2"`) tasks. + + * `output`: print potentially useful output from hosts (e.g. output of `kubectl get pods --all-namespaces`) + +## Usage + +### Local machine + +``` +ansible-playbook -u -i "localhost", -c local setup_weave-kube.yml +``` + +### Vagrant + +Provision your local VM using Vagrant: + +``` +cd $(mktemp -d -t XXX) +vagrant init ubuntu/xenial64 # or, e.g. centos/7 +vagrant up +``` + +then set the following environment variables by extracting the output of `vagrant ssh-config`: + +``` +eval $(vagrant ssh-config | sed \ +-ne 's/\ *HostName /vagrant_ssh_host=/p' \ +-ne 's/\ *User /vagrant_ssh_user=/p' \ +-ne 's/\ *Port /vagrant_ssh_port=/p' \ +-ne 's/\ *IdentityFile /vagrant_ssh_id_file=/p') +``` + +and finally run: + +``` +ansible-playbook --private-key=$vagrant_ssh_id_file -u $vagrant_ssh_user \ +--ssh-extra-args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \ +-i "$vagrant_ssh_host:$vagrant_ssh_port," setup_weave-kube.yml +``` + +or, for specific versions of Kubernetes and Docker: + +``` +ansible-playbook --private-key=$vagrant_ssh_id_file -u $vagrant_ssh_user \ +--ssh-extra-args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \ +-i "$vagrant_ssh_host:$vagrant_ssh_port," setup_weave-kube.yml \ +--extra-vars "docker_version=1.12.3 kubernetes_version=1.4.4" +``` + +NOTE: Kubernetes APT repo includes only the latest version, so currently +retrieving an older version will fail. + +### Terraform + +Provision your machine using the Terraform scripts from `../provisioning`, then run: + +``` +terraform output ansible_inventory > /tmp/ansible_inventory +``` + +and + +``` +ansible-playbook \ + --private-key="$(terraform output private_key_path)" \ + -u "$(terraform output username)" \ + -i /tmp/ansible_inventory \ + --ssh-extra-args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \ + ../../config_management/setup_weave-kube.yml + +``` + +To specify versions of Kubernetes and Docker see Vagrant examples above. + +N.B.: `--ssh-extra-args` is used to provide: + +* `StrictHostKeyChecking=no`: as VMs come and go, the same IP can be used by a different machine, so checking the host's SSH key may fail. Note that this introduces a risk of a man-in-the-middle attack. +* `UserKnownHostsFile=/dev/null`: if you previously connected a VM with the same IP but a different public key, and added it to `~/.ssh/known_hosts`, SSH may still fail to connect, hence we use `/dev/null` instead of `~/.ssh/known_hosts`. + + +### Docker installation role + +Various ways to install Docker are provided: + +- `docker-from-docker-ce-repo` +- `docker-from-docker-repo` +- `docker-from-get.docker.com` +- `docker-from-tarball` + +each producing a slightly different outcome, which can be useful for testing various setup scenarios. + +The `docker-install` role selects one of the above ways to install Docker based on the `docker_install_role` variable. +The default value for this variable is configured in `group_vars/all`. +You can however override it with whichever role you would want to run by passing the name of the role as a key-value pair in `extra-vars`, e.g.: + +``` +ansible-playbook .yml \ + --extra-vars "docker_install_role=docker-from-docker-ce-repo" +``` + + +## Resources + +* [https://www.vagrantup.com/docs/provisioning/ansible.html](https://www.vagrantup.com/docs/provisioning/ansible.html) +* [http://docs.ansible.com/ansible/guide_vagrant.html](http://docs.ansible.com/ansible/guide_vagrant.html) diff --git a/tools/config_management/group_vars/all b/tools/config_management/group_vars/all new file mode 100644 index 0000000000..d728cce80b --- /dev/null +++ b/tools/config_management/group_vars/all @@ -0,0 +1,11 @@ +--- +go_version: 1.8.1 +terraform_version: 0.8.5 +docker_version: 17.06 +docker_install_role: 'docker-from-docker-ce-repo' +kubernetes_version: 1.6.1 +kubernetes_cni_version: 0.5.1 +kubernetes_token: '123456.0123456789123456' +etcd_container_version: 2.2.5 +kube_discovery_container_version: 1.0 +pause_container_version: 3.0 diff --git a/tools/config_management/library/setup_ansible_dependencies.yml b/tools/config_management/library/setup_ansible_dependencies.yml new file mode 100644 index 0000000000..50263369a1 --- /dev/null +++ b/tools/config_management/library/setup_ansible_dependencies.yml @@ -0,0 +1,33 @@ +--- +################################################################################ +# Install Ansible's dependencies: python and lsb_release, required respectively +# to run Ansible modules and gather Ansible facts. +# +# See also: +# - http://docs.ansible.com/ansible/intro_installation.html#managed-node-requirements +# - http://docs.ansible.com/ansible/setup_module.html +################################################################################ + +- name: check if python is installed (as required by ansible modules) + raw: test -e /usr/bin/python + register: is_python_installed + failed_when: is_python_installed.rc not in [0, 1] + changed_when: false # never mutates state. + +- name: install python if missing (as required by ansible modules) + when: is_python_installed|failed # skip otherwise + raw: (test -e /usr/bin/apt-get && apt-get update && apt-get install -y python-minimal) || (test -e /usr/bin/yum && yum update && yum install -y python) + changed_when: is_python_installed.rc == 1 + +- name: check if lsb_release is installed (as required for ansible facts) + raw: test -e /usr/bin/lsb_release + register: is_lsb_release_installed + failed_when: is_lsb_release_installed.rc not in [0, 1] + changed_when: false # never mutates state. + +- name: install lsb_release if missing (as required for ansible facts) + when: is_lsb_release_installed|failed # skip otherwise + raw: (test -e /usr/bin/apt-get && apt-get install -y lsb_release) || (test -e /usr/bin/yum && yum install -y redhat-lsb-core) + changed_when: is_lsb_release_installed.rc == 1 + +- setup: # gather 'facts', i.e. compensates for 'gather_facts: false' in calling playbook. diff --git a/tools/config_management/roles/dev-tools/files/apt-daily.timer.conf b/tools/config_management/roles/dev-tools/files/apt-daily.timer.conf new file mode 100644 index 0000000000..bd19c61f9e --- /dev/null +++ b/tools/config_management/roles/dev-tools/files/apt-daily.timer.conf @@ -0,0 +1,2 @@ +[Timer] +Persistent=false diff --git a/tools/config_management/roles/dev-tools/tasks/main.yml b/tools/config_management/roles/dev-tools/tasks/main.yml new file mode 100644 index 0000000000..96ac3a2199 --- /dev/null +++ b/tools/config_management/roles/dev-tools/tasks/main.yml @@ -0,0 +1,48 @@ +--- +# Set up Development Environment. + +- name: install development tools + package: + name: "{{ item }}" + state: present + with_items: + # weave net dependencies + - make + - vagrant + # ansible dependencies + - python-pip + - python-dev + - libffi-dev + - libssl-dev + # terraform dependencies + - unzip + # other potentially useful tools: + - aufs-tools + - ethtool + - iputils-arping + - libpcap-dev + - git + - mercurial + - bc + - jq + +- name: install ansible + pip: + name: ansible + state: present + +- name: install terraform + unarchive: + src: 'https://releases.hashicorp.com/terraform/{{ terraform_version }}/terraform_{{ terraform_version }}_linux_{{ {"x86_64": "amd64", "i386": "386"}[ansible_architecture] }}.zip' + remote_src: yes + dest: /usr/bin + mode: 0555 + creates: /usr/bin/terraform + +# Ubuntu runs an apt update process that will run on first boot from image. +# This is of questionable value when the machines are only going to live for a few minutes. +# If you leave them on they will run the process daily. +# Also we have seen the update process create a 'defunct' process which then throws off Weave Net smoke-test checks. +# So, we override the 'persistent' setting so it will still run at the scheduled time but will not try to catch up on first boot. +- name: copy apt daily override + copy: src=apt-daily.timer.conf dest=/etc/systemd/system/apt-daily.timer.d/ diff --git a/tools/config_management/roles/docker-configuration/files/docker.conf b/tools/config_management/roles/docker-configuration/files/docker.conf new file mode 100644 index 0000000000..626d8022b3 --- /dev/null +++ b/tools/config_management/roles/docker-configuration/files/docker.conf @@ -0,0 +1,3 @@ +[Service] +ExecStart= +ExecStart=/usr/bin/dockerd -H fd:// -H unix:///var/run/alt-docker.sock -H tcp://0.0.0.0:2375 -s overlay --insecure-registry "weave-ci-registry:5000" diff --git a/tools/config_management/roles/docker-configuration/tasks/main.yml b/tools/config_management/roles/docker-configuration/tasks/main.yml new file mode 100644 index 0000000000..d67369686a --- /dev/null +++ b/tools/config_management/roles/docker-configuration/tasks/main.yml @@ -0,0 +1,36 @@ +--- +# Configure Docker +# See also: https://docs.docker.com/engine/installation/linux/ubuntulinux/#install + +- name: ensure docker group is present (or create it) + group: + name: docker + state: present + +- name: add user to docker group (avoids sudo-ing) + user: + name: "{{ ansible_user }}" + group: docker + state: present + +- name: ensure docker's systemd directory exists + file: + path: /etc/systemd/system/docker.service.d + state: directory + recurse: yes + when: ansible_os_family != "RedHat" + +- name: enable docker remote api over tcp + copy: + src: "{{ role_path }}/files/docker.conf" + dest: /etc/systemd/system/docker.service.d/docker.conf + register: docker_conf + when: ansible_os_family != "RedHat" + +- name: restart docker service + systemd: + name: docker + state: restarted + daemon_reload: yes # ensure docker.conf is picked up. + enabled: yes + when: docker_conf.changed or ansible_os_family == "RedHat" diff --git a/tools/config_management/roles/docker-from-docker-ce-repo/tasks/debian.yml b/tools/config_management/roles/docker-from-docker-ce-repo/tasks/debian.yml new file mode 100644 index 0000000000..3e2ae1270c --- /dev/null +++ b/tools/config_management/roles/docker-from-docker-ce-repo/tasks/debian.yml @@ -0,0 +1,35 @@ +--- +# Debian / Ubuntu specific: + +- name: install dependencies for docker repository + package: + name: "{{ item }}" + state: present + with_items: + - apt-transport-https + - ca-certificates + +- name: add apt key for the docker repository + apt_key: + keyserver: hkp://ha.pool.sks-keyservers.net:80 + id: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88 + state: present + register: apt_key_docker_repo + +- name: add docker's apt repository ({{ ansible_distribution | lower }}-{{ ansible_distribution_release }}) + apt_repository: + repo: deb https://download.docker.com/linux/ubuntu {{ ansible_lsb.codename|lower }} stable + state: present + register: apt_docker_repo + +- name: update apt's cache + apt: + update_cache: yes + when: apt_key_docker_repo.changed or apt_docker_repo.changed + +- name: install docker-engine + package: + name: "{{ item }}" + state: present + with_items: + - docker-ce={{ docker_version }}* diff --git a/tools/config_management/roles/docker-from-docker-ce-repo/tasks/main.yml b/tools/config_management/roles/docker-from-docker-ce-repo/tasks/main.yml new file mode 100644 index 0000000000..0acb6d8c92 --- /dev/null +++ b/tools/config_management/roles/docker-from-docker-ce-repo/tasks/main.yml @@ -0,0 +1,10 @@ +--- +# Set up Docker +# See also: https://docs.docker.com/engine/installation/linux/ubuntulinux/#install + +# Distribution-specific tasks: +- include: debian.yml + when: ansible_os_family == "Debian" + +- include: redhat.yml + when: ansible_os_family == "RedHat" diff --git a/tools/config_management/roles/docker-from-docker-ce-repo/tasks/redhat.yml b/tools/config_management/roles/docker-from-docker-ce-repo/tasks/redhat.yml new file mode 100644 index 0000000000..ea9a3fa462 --- /dev/null +++ b/tools/config_management/roles/docker-from-docker-ce-repo/tasks/redhat.yml @@ -0,0 +1,29 @@ +# Docker installation from Docker's CentOS Community Edition +# See also: https://docs.docker.com/engine/installation/linux/centos/ + +- name: remove all potentially pre existing packages + yum: + name: '{{ item }}' + state: absent + with_items: + - docker + - docker-common + - container-selinux + - docker-selinux + - docker-engine + +- name: install yum-utils + yum: + name: yum-utils + state: present + +- name: add docker ce repo + command: yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo + +# Note that Docker CE versions do not follow regular Docker versions, but look +# like, for example: "17.03.0.el7" +- name: install docker + yum: + name: 'docker-ce-{{ docker_version }}' + update_cache: yes + state: present diff --git a/tools/config_management/roles/docker-from-docker-repo/tasks/debian.yml b/tools/config_management/roles/docker-from-docker-repo/tasks/debian.yml new file mode 100644 index 0000000000..cc33c2c9fa --- /dev/null +++ b/tools/config_management/roles/docker-from-docker-repo/tasks/debian.yml @@ -0,0 +1,35 @@ +--- +# Debian / Ubuntu specific: + +- name: install dependencies for docker repository + package: + name: "{{ item }}" + state: present + with_items: + - apt-transport-https + - ca-certificates + +- name: add apt key for the docker repository + apt_key: + keyserver: hkp://ha.pool.sks-keyservers.net:80 + id: 58118E89F3A912897C070ADBF76221572C52609D + state: present + register: apt_key_docker_repo + +- name: add docker's apt repository ({{ ansible_distribution | lower }}-{{ ansible_distribution_release }}) + apt_repository: + repo: deb https://apt.dockerproject.org/repo {{ ansible_distribution | lower }}-{{ ansible_distribution_release }} main + state: present + register: apt_docker_repo + +- name: update apt's cache + apt: + update_cache: yes + when: apt_key_docker_repo.changed or apt_docker_repo.changed + +- name: install docker-engine + package: + name: "{{ item }}" + state: present + with_items: + - docker-engine={{ docker_version }}* diff --git a/tools/config_management/roles/docker-from-docker-repo/tasks/main.yml b/tools/config_management/roles/docker-from-docker-repo/tasks/main.yml new file mode 100644 index 0000000000..0acb6d8c92 --- /dev/null +++ b/tools/config_management/roles/docker-from-docker-repo/tasks/main.yml @@ -0,0 +1,10 @@ +--- +# Set up Docker +# See also: https://docs.docker.com/engine/installation/linux/ubuntulinux/#install + +# Distribution-specific tasks: +- include: debian.yml + when: ansible_os_family == "Debian" + +- include: redhat.yml + when: ansible_os_family == "RedHat" diff --git a/tools/config_management/roles/docker-from-docker-repo/tasks/redhat.yml b/tools/config_management/roles/docker-from-docker-repo/tasks/redhat.yml new file mode 100644 index 0000000000..d29964e1b8 --- /dev/null +++ b/tools/config_management/roles/docker-from-docker-repo/tasks/redhat.yml @@ -0,0 +1,25 @@ +--- +# RedHat / CentOS specific: + +- name: add docker' yum repository (centos/{{ ansible_lsb.major_release }}) + yum_repository: + name: docker + description: Docker YUM repo + file: external_repos + baseurl: https://yum.dockerproject.org/repo/main/centos/{{ ansible_lsb.major_release }} + enabled: yes + gpgkey: https://yum.dockerproject.org/gpg + gpgcheck: yes + state: present + +- name: update yum's cache + yum: + name: "*" + update_cache: yes + +- name: install docker-engine + package: + name: "{{ item }}" + state: present + with_items: + - docker-engine-{{ docker_version }} diff --git a/tools/config_management/roles/docker-from-get.docker.com/tasks/debian.yml b/tools/config_management/roles/docker-from-get.docker.com/tasks/debian.yml new file mode 100644 index 0000000000..7444194e57 --- /dev/null +++ b/tools/config_management/roles/docker-from-get.docker.com/tasks/debian.yml @@ -0,0 +1,8 @@ +--- +# Debian / Ubuntu specific: + +- name: apt-import gpg key for the docker repository + shell: curl -sSL https://get.docker.com/gpg | sudo apt-key add - + +- name: install docker + shell: 'curl -sSL https://get.docker.com/ | sed -e s/docker-engine/docker-engine={{ docker_version }}*/ -e s/docker-ce/docker-ce={{ docker_version }}*/ | sh' diff --git a/tools/config_management/roles/docker-from-get.docker.com/tasks/main.yml b/tools/config_management/roles/docker-from-get.docker.com/tasks/main.yml new file mode 100644 index 0000000000..92c497b797 --- /dev/null +++ b/tools/config_management/roles/docker-from-get.docker.com/tasks/main.yml @@ -0,0 +1,10 @@ +--- +# Set up Docker +# See also: legacy gce.sh script + +# Distribution-specific tasks: +- include: debian.yml + when: ansible_os_family == "Debian" + +- include: redhat.yml + when: ansible_os_family == "RedHat" diff --git a/tools/config_management/roles/docker-from-get.docker.com/tasks/redhat.yml b/tools/config_management/roles/docker-from-get.docker.com/tasks/redhat.yml new file mode 100644 index 0000000000..ea7cbfc4f6 --- /dev/null +++ b/tools/config_management/roles/docker-from-get.docker.com/tasks/redhat.yml @@ -0,0 +1,11 @@ +--- +# RedHat / CentOS specific: + +- name: rpm-import gpg key for the docker repository + shell: curl -sSLo /tmp/docker.gpg https://get.docker.com/gpg && sudo rpm --import /tmp/docker.gpg + +- name: install docker + shell: 'curl -sSL https://get.docker.com/ | sed -e s/docker-engine/docker-engine-{{ docker_version }}*/ | sh' + +- name: wait for docker installation to complete + shell: yum install -y yum-utils && yum-complete-transaction diff --git a/tools/config_management/roles/docker-from-tarball/tasks/main.yml b/tools/config_management/roles/docker-from-tarball/tasks/main.yml new file mode 100644 index 0000000000..a233d10a5d --- /dev/null +++ b/tools/config_management/roles/docker-from-tarball/tasks/main.yml @@ -0,0 +1,61 @@ +--- +# Set up Docker +# See also: +# - https://docs.docker.com/engine/installation/linux/ubuntulinux/#install +# - https://github.com/docker/docker/releases + +- include_role: + name: docker-prerequisites + +- name: install daemon + package: + name: daemon + state: present + +- name: 'create directory {{ docker_dir }}/{{ docker_version }}' + file: + path: '{{ docker_dir }}/{{ docker_version }}' + state: directory + mode: 0755 + +- name: download and extract docker + unarchive: + src: 'https://get.docker.com/builds/Linux/x86_64/docker-{{ docker_version }}.tgz' + remote_src: yes + dest: '{{ docker_dir }}/{{ docker_version }}' + extra_opts: '--strip-components=1' + mode: 0555 + creates: '{{ docker_dir }}/{{ docker_version }}/docker' + +- name: create symlink to current version + file: + src: '{{ docker_dir }}/{{ docker_version }}' + dest: '{{ docker_dir }}/current' + state: link + mode: 0555 + +- name: list all files to symlink + find: + paths: '{{ docker_dir }}/current' + file_type: file + register: binaries + changed_when: false + +- name: create symlinks to all binaries + file: + src: '{{ item }}' + dest: /usr/bin/{{ item | basename }} + state: link + with_items: "{{ binaries.files | map(attribute='path') | list }}" + +- name: killall docker + command: killall docker + register: killall + failed_when: false + changed_when: killall.rc == 0 + +- name: start dockerd + command: daemon -- /usr/bin/dockerd + +- include_role: + name: docker-configuration diff --git a/tools/config_management/roles/docker-from-tarball/vars/main.yml b/tools/config_management/roles/docker-from-tarball/vars/main.yml new file mode 100644 index 0000000000..d410668489 --- /dev/null +++ b/tools/config_management/roles/docker-from-tarball/vars/main.yml @@ -0,0 +1,5 @@ +--- +docker_dir: '/opt/docker' +docker_url: '{{ "rc" in {{ docker_version }} | ternary( > + "https://test.docker.com/builds/Linux/x86_64/docker-{{ docker_version }}.tgz", > + "https://get.docker.com/builds/Linux/x86_64/docker-{{ docker_version }}.tgz") }}' diff --git a/tools/config_management/roles/docker-install/tasks/main.yml b/tools/config_management/roles/docker-install/tasks/main.yml new file mode 100644 index 0000000000..cc803cabc9 --- /dev/null +++ b/tools/config_management/roles/docker-install/tasks/main.yml @@ -0,0 +1,30 @@ +--- +# Set up Docker + +- include_role: + name: docker-prerequisites + +# Dynamically include docker installation role using 'when' as Ansible does not +# allow for include_role's name to be set to a variable. Indeed: +# - include_role: +# name: '{{ docker_install_role }}' +# fails with: +# ERROR! 'docker_install_role' is undefined +- include_role: + name: docker-from-docker-repo + when: docker_install_role == 'docker-from-docker-repo' + +- include_role: + name: docker-from-docker-ce-repo + when: docker_install_role == 'docker-from-docker-ce-repo' + +- include_role: + name: docker-from-get.docker.com + when: docker_install_role == 'docker-from-get.docker.com' + +- include_role: + name: docker-from-tarball + when: docker_install_role == 'docker-from-tarball' + +- include_role: + name: docker-configuration diff --git a/tools/config_management/roles/docker-prerequisites/tasks/debian.yml b/tools/config_management/roles/docker-prerequisites/tasks/debian.yml new file mode 100644 index 0000000000..48b0c2e370 --- /dev/null +++ b/tools/config_management/roles/docker-prerequisites/tasks/debian.yml @@ -0,0 +1,11 @@ +--- +# Install Docker's dependencies +# See also: https://docs.docker.com/engine/installation/linux/ubuntulinux/#install + +- name: install linux-image-extra-*/virtual + package: + name: "{{ item }}" + state: present + with_items: + - linux-image-extra-{{ ansible_kernel }} + - linux-image-extra-virtual diff --git a/tools/config_management/roles/docker-prerequisites/tasks/main.yml b/tools/config_management/roles/docker-prerequisites/tasks/main.yml new file mode 100644 index 0000000000..a817737205 --- /dev/null +++ b/tools/config_management/roles/docker-prerequisites/tasks/main.yml @@ -0,0 +1,5 @@ +--- + +# Distribution-specific tasks: +- include: debian.yml + when: ansible_os_family == "Debian" diff --git a/tools/config_management/roles/golang-from-tarball/tasks/main.yml b/tools/config_management/roles/golang-from-tarball/tasks/main.yml new file mode 100644 index 0000000000..55476bf6ae --- /dev/null +++ b/tools/config_management/roles/golang-from-tarball/tasks/main.yml @@ -0,0 +1,36 @@ +--- +# Set up Go. + +- name: install go + unarchive: + src: 'https://storage.googleapis.com/golang/go{{ go_version }}.linux-{{ {"x86_64": "amd64", "i386": "386"}[ansible_architecture] }}.tar.gz' + remote_src: yes + dest: /usr/local + mode: 0777 + creates: /usr/local/go/bin/go + +- name: set go env. vars. and add go to path + blockinfile: + dest: '$HOME/.bashrc' + block: | + export PATH=$PATH:/usr/local/go/bin + export GOPATH=$HOME + state: present + create: yes + mode: 0644 + become: '{{ item }}' + with_items: + - true # Run as root + - false # Run as SSH user + +- name: source ~/.bashrc from ~/.bash_profile + lineinfile: + dest: '$HOME/.bash_profile' + line: '[ -r $HOME/.bashrc ] && source $HOME/.bashrc' + state: present + create: yes + mode: 0644 + become: '{{ item }}' + with_items: + - true # Run as root + - false # Run as SSH user diff --git a/tools/config_management/roles/kubelet-stop/tasks/main.yml b/tools/config_management/roles/kubelet-stop/tasks/main.yml new file mode 100644 index 0000000000..6e5f3148e2 --- /dev/null +++ b/tools/config_management/roles/kubelet-stop/tasks/main.yml @@ -0,0 +1,14 @@ +--- + +- name: check if kubelet service exists + stat: + path: /etc/init.d/kubelet + register: kubelet + +# avoids having weave-net and weave-kube conflict in some test cases (e.g. 130_expose_test.sh) +- name: stop kubelet service + systemd: + name: kubelet + state: stopped + enabled: no + when: kubelet.stat.exists diff --git a/tools/config_management/roles/kubernetes-docker-images/tasks/main.yml b/tools/config_management/roles/kubernetes-docker-images/tasks/main.yml new file mode 100644 index 0000000000..801c46370d --- /dev/null +++ b/tools/config_management/roles/kubernetes-docker-images/tasks/main.yml @@ -0,0 +1,14 @@ +--- + +- name: docker pull images used by k8s tests + docker_image: + name: '{{ item }}' + state: present + with_items: + - gcr.io/google_containers/etcd-amd64:{{ etcd_container_version }} + - gcr.io/google_containers/kube-apiserver-amd64:v{{ kubernetes_version }} + - gcr.io/google_containers/kube-controller-manager-amd64:v{{ kubernetes_version }} + - gcr.io/google_containers/kube-proxy-amd64:v{{ kubernetes_version }} + - gcr.io/google_containers/kube-scheduler-amd64:v{{ kubernetes_version }} + - gcr.io/google_containers/kube-discovery-amd64:{{ kube_discovery_container_version }} + - gcr.io/google_containers/pause-amd64:{{ pause_container_version }} diff --git a/tools/config_management/roles/kubernetes-install/tasks/debian.yml b/tools/config_management/roles/kubernetes-install/tasks/debian.yml new file mode 100644 index 0000000000..9f16edfde6 --- /dev/null +++ b/tools/config_management/roles/kubernetes-install/tasks/debian.yml @@ -0,0 +1,37 @@ +--- +# Debian / Ubuntu specific: + +- name: add apt key for the kubernetes repository + apt_key: + url: https://packages.cloud.google.com/apt/doc/apt-key.gpg + state: present + register: apt_key_k8s_repo + +- name: add kubernetes' apt repository (kubernetes-{{ ansible_distribution_release }}) + apt_repository: + repo: deb http://apt.kubernetes.io/ kubernetes-{{ ansible_distribution_release }} main + state: present + register: apt_k8s_repo + when: '"alpha" not in kubernetes_version and "beta" not in kubernetes_version' + +- name: add kubernetes' apt repository (kubernetes-{{ ansible_distribution_release }}-unstable) + apt_repository: + repo: deb http://apt.kubernetes.io/ kubernetes-{{ ansible_distribution_release }}-unstable main + state: present + register: apt_k8s_repo + when: '"alpha" in kubernetes_version or "beta" in kubernetes_version' + +- name: update apt's cache + apt: + update_cache: yes + when: apt_key_k8s_repo.changed or apt_k8s_repo.changed + +- name: install kubelet and kubectl + package: + name: "{{ item }}" + state: present + with_items: + - kubelet={{ kubernetes_version }}* + - kubectl={{ kubernetes_version }}* + - kubeadm={{ kubernetes_version }}* + - kubernetes-cni={{ kubernetes_cni_version }}* diff --git a/tools/config_management/roles/kubernetes-install/tasks/main.yml b/tools/config_management/roles/kubernetes-install/tasks/main.yml new file mode 100644 index 0000000000..50dcddaf72 --- /dev/null +++ b/tools/config_management/roles/kubernetes-install/tasks/main.yml @@ -0,0 +1,16 @@ +--- +# Install Kubernetes + +# Distribution-specific tasks: +- include: debian.yml + when: ansible_os_family == "Debian" + +- include: redhat.yml + when: ansible_os_family == "RedHat" + +- name: install ebtables + package: + name: "{{ item }}" + state: present + with_items: + - ebtables diff --git a/tools/config_management/roles/kubernetes-install/tasks/redhat.yml b/tools/config_management/roles/kubernetes-install/tasks/redhat.yml new file mode 100644 index 0000000000..293729dc1c --- /dev/null +++ b/tools/config_management/roles/kubernetes-install/tasks/redhat.yml @@ -0,0 +1,30 @@ +--- +# RedHat / CentOS specific: + +- name: add kubernetes' yum repository (kubernetes-el{{ ansible_lsb.major_release }}-x86-64) + yum_repository: + name: kubernetes + description: Kubernetes YUM repo + file: external_repos + baseurl: https://packages.cloud.google.com/yum/repos/kubernetes-el{{ ansible_lsb.major_release }}-x86_64 + enabled: yes + gpgkey: https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg + gpgcheck: yes + state: present + register: yum_k8s_repo + +- name: update yum's cache + yum: + name: "*" + update_cache: yes + when: yum_k8s_repo.changed + +- name: install kubelet and kubectl + package: + name: "{{ item }}" + state: present + with_items: + - kubelet-{{ kubernetes_version }}* + - kubectl-{{ kubernetes_version }}* + - kubeadm-{{ kubernetes_version }}* + - kubernetes-cni-{{ kubernetes_cni_version }}* diff --git a/tools/config_management/roles/kubernetes-start/tasks/main.yml b/tools/config_management/roles/kubernetes-start/tasks/main.yml new file mode 100644 index 0000000000..d343b21c46 --- /dev/null +++ b/tools/config_management/roles/kubernetes-start/tasks/main.yml @@ -0,0 +1,42 @@ +--- +# Start Kubernetes + +- name: kubeadm reset + command: kubeadm reset + +- name: restart kubelet service + systemd: + name: kubelet + state: restarted + enabled: yes + +- name: optionally set kubeconfig option + set_fact: + kubeconfig: '{{ (kubernetes_version >= "1.5.4") | ternary("--kubeconfig /etc/kubernetes/admin.conf", "") }}' + kubernetes_version_option: '{{ (kubernetes_version >= "1.6") | ternary("kubernetes_version", "use-kubernetes-version") }}' + +- name: kubeadm init on the master + command: 'kubeadm init --{{ kubernetes_version_option }}=v{{ kubernetes_version }} --token={{ kubernetes_token }}' + when: ' {{ play_hosts[0] == inventory_hostname }}' + +- name: allow pods to be run on the master (if only node) + command: 'kubectl {{ kubeconfig }} taint nodes --all {{ (kubernetes_version < "1.6") | ternary("dedicated-", "node-role.kubernetes.io/master:NoSchedule-") }}' + when: '{{ play_hosts | length }} == 1' + +- name: kubeadm join on workers + command: 'kubeadm join --token={{ kubernetes_token }} {{ hostvars[play_hosts[0]].private_ip }}{{ (kubernetes_version > "1.6") | ternary(":6443", "") }}' + when: ' {{ play_hosts[0] != inventory_hostname }}' + +- name: list kubernetes' pods + command: kubectl {{ kubeconfig }} get pods --all-namespaces + when: ' {{ play_hosts[0] == inventory_hostname }}' + changed_when: false + register: kubectl_get_pods + tags: + - output + +- name: print outpout of `kubectl get pods --all-namespaces` + debug: msg="{{ kubectl_get_pods.stdout_lines }}" + when: ' {{ play_hosts[0] == inventory_hostname }}' + tags: + - output diff --git a/tools/config_management/roles/setup-ansible/pre_tasks/main.yml b/tools/config_management/roles/setup-ansible/pre_tasks/main.yml new file mode 100644 index 0000000000..efb154917f --- /dev/null +++ b/tools/config_management/roles/setup-ansible/pre_tasks/main.yml @@ -0,0 +1,26 @@ +--- +# Set machine up to be able to run ansible playbooks. + +- name: check if python is installed (as required by ansible modules) + raw: test -e /usr/bin/python + register: is_python_installed + failed_when: is_python_installed.rc not in [0, 1] + changed_when: false # never mutates state. + +- name: install python if missing (as required by ansible modules) + when: is_python_installed|failed # skip otherwise + raw: (test -e /usr/bin/apt-get && apt-get install -y python-minimal) || (test -e /usr/bin/yum && yum install -y python) + changed_when: is_python_installed.rc == 1 + +- name: check if lsb_release is installed (as required for ansible facts) + raw: test -e /usr/bin/lsb_release + register: is_lsb_release_installed + failed_when: is_lsb_release_installed.rc not in [0, 1] + changed_when: false # never mutates state. + +- name: install lsb_release if missing (as required for ansible facts) + when: is_lsb_release_installed|failed # skip otherwise + raw: (test -e /usr/bin/apt-get && apt-get install -y lsb_release) || (test -e /usr/bin/yum && yum install -y lsb_release) + changed_when: is_lsb_release_installed.rc == 1 + +- setup: # gather 'facts', i.e. compensates for the above 'gather_facts: false'. diff --git a/tools/config_management/roles/sock-shop/tasks/tasks.yml b/tools/config_management/roles/sock-shop/tasks/tasks.yml new file mode 100644 index 0000000000..9667ab04b1 --- /dev/null +++ b/tools/config_management/roles/sock-shop/tasks/tasks.yml @@ -0,0 +1,34 @@ +--- +# Set up sock-shop on top of Kubernetes. +# Dependencies on other roles: +# - kubernetes + +- name: create sock-shop namespace in k8s + command: kubectl --kubeconfig /etc/kubernetes/admin.conf create namespace sock-shop + +- name: create sock-shop in k8s + command: kubectl --kubeconfig /etc/kubernetes/admin.conf apply -n sock-shop -f "https://github.com/microservices-demo/microservices-demo/blob/master/deploy/kubernetes/complete-demo.yaml?raw=true" + +- name: describe front-end service + command: kubectl --kubeconfig /etc/kubernetes/admin.conf describe svc front-end -n sock-shop + changed_when: false + register: kubectl_describe_svc_frontend + tags: + - output + +- name: print outpout of `kubectl describe svc front-end -n sock-shop` + debug: msg="{{ kubectl_describe_svc_frontend.stdout_lines }}" + tags: + - output + +- name: list sock-shop k8s' pods + command: kubectl --kubeconfig /etc/kubernetes/admin.conf get pods -n sock-shop + changed_when: false + register: kubectl_get_pods + tags: + - output + +- name: print outpout of `kubectl get pods -n sock-shop` + debug: msg="{{ kubectl_get_pods.stdout_lines }}" + tags: + - output diff --git a/tools/config_management/roles/weave-kube/tasks/main.yml b/tools/config_management/roles/weave-kube/tasks/main.yml new file mode 100644 index 0000000000..e2025eef13 --- /dev/null +++ b/tools/config_management/roles/weave-kube/tasks/main.yml @@ -0,0 +1,24 @@ +--- +# Set up Weave Kube on top of Kubernetes. + +- name: set url for weave-kube daemonset + set_fact: + weave_kube_url: '{{ (kubernetes_version < "1.6") | ternary("https://git.io/weave-kube", "https://git.io/weave-kube-1.6") }}' + +- name: configure weave net's cni plugin + command: 'kubectl {{ kubeconfig }} apply -f {{ weave_kube_url }}' + when: '{{ play_hosts[0] == inventory_hostname }}' + +- name: list kubernetes' pods + command: 'kubectl {{ kubeconfig }} get pods --all-namespaces' + when: '{{ play_hosts[0] == inventory_hostname }}' + changed_when: false + register: kubectl_get_pods + tags: + - output + +- name: print outpout of `kubectl get pods --all-namespaces` + debug: msg="{{ kubectl_get_pods.stdout_lines }}" + when: '{{ play_hosts[0] == inventory_hostname }}' + tags: + - output diff --git a/tools/config_management/roles/weave-net-sources/tasks/main.yml b/tools/config_management/roles/weave-net-sources/tasks/main.yml new file mode 100644 index 0000000000..b0a7815c41 --- /dev/null +++ b/tools/config_management/roles/weave-net-sources/tasks/main.yml @@ -0,0 +1,24 @@ +--- +# Set up Development Environment for Weave Net. + +- name: check if weave net has been checked out + become: false # Run as SSH-user + stat: + path: $HOME/src/github.com/weaveworks/weave + register: weave + failed_when: false + changed_when: false + +- name: git clone weave net + become: false # Run as SSH-user + git: + repo: https://github.com/weaveworks/weave.git + dest: $HOME/src/github.com/weaveworks/weave + when: not weave.stat.exists + +- name: create a convenience symlink to $HOME/src/github.com/weaveworks/weave + become: false # Run as SSH-user + file: + src: $HOME/src/github.com/weaveworks/weave + dest: $HOME/weave + state: link diff --git a/tools/config_management/roles/weave-net-utilities/tasks/main.yml b/tools/config_management/roles/weave-net-utilities/tasks/main.yml new file mode 100644 index 0000000000..6883d23aad --- /dev/null +++ b/tools/config_management/roles/weave-net-utilities/tasks/main.yml @@ -0,0 +1,56 @@ +--- + +- name: install epel-release + package: + name: "{{ item }}" + state: present + with_items: + - epel-release + when: ansible_os_family == "RedHat" + +- name: install jq + package: + name: "{{ item }}" + state: present + with_items: + - jq + +- name: install ethtool (used by the weave script) + package: + name: "{{ item }}" + state: present + with_items: + - ethtool + +- name: install nsenter (used by the weave script) + command: docker run --rm -v /usr/local/bin:/target jpetazzo/nsenter + +- name: install pip (for docker-py) + package: + name: "{{ item }}" + state: present + with_items: + - python-pip + +- name: install docker-py (for docker_image) + pip: + name: docker-py + state: present + +- name: docker pull images used by tests + docker_image: + name: '{{ item }}' + state: present + with_items: + - alpine + - aanand/docker-dnsutils + - weaveworks/hello-world + +- name: docker pull docker-py which is used by tests + docker_image: + name: joffrey/docker-py + tag: '{{ item }}' + state: present + with_items: + - '1.8.1' + - '1.9.0-rc2' diff --git a/tools/config_management/roles/weave-net/tasks/main.yml b/tools/config_management/roles/weave-net/tasks/main.yml new file mode 100644 index 0000000000..0ef5e351d7 --- /dev/null +++ b/tools/config_management/roles/weave-net/tasks/main.yml @@ -0,0 +1,26 @@ +--- +# Set up Weave Net. + +- name: install weave net + get_url: + url: https://git.io/weave + dest: /usr/local/bin/weave + mode: 0555 + +- name: stop weave net + command: /usr/local/bin/weave stop + +- name: start weave net + command: /usr/local/bin/weave launch + +- name: get weave net's status + command: /usr/local/bin/weave status + changed_when: false + register: weave_status + tags: + - output + +- name: print outpout of `weave status` + debug: msg="{{ weave_status.stdout_lines }}" + tags: + - output diff --git a/tools/config_management/setup_weave-kube.yml b/tools/config_management/setup_weave-kube.yml new file mode 100644 index 0000000000..5c68c978d4 --- /dev/null +++ b/tools/config_management/setup_weave-kube.yml @@ -0,0 +1,27 @@ +--- +################################################################################ +# Install Docker and Kubernetes, and configure Kubernetes to +# use Weave Net's CNI plugin (a.k.a. Weave Kube). +# +# See also: +# - http://kubernetes.io/docs/getting-started-guides/kubeadm/ +# - https://github.com/weaveworks/weave-kube +################################################################################ + +- name: install docker, kubernetes and weave-kube + hosts: all + gather_facts: false # required in case Python is not available on the host + become: true + become_user: root + + pre_tasks: + - include: library/setup_ansible_dependencies.yml + + roles: + - docker-install + - weave-net-utilities + - kubernetes-install + - kubernetes-docker-images + - kubelet-stop + - kubernetes-start + - weave-kube diff --git a/tools/config_management/setup_weave-net_debug.yml b/tools/config_management/setup_weave-net_debug.yml new file mode 100644 index 0000000000..ff73a5274d --- /dev/null +++ b/tools/config_management/setup_weave-net_debug.yml @@ -0,0 +1,18 @@ +--- +################################################################################ +# Install Docker from Docker's official repository and Weave Net. +################################################################################ + +- name: install docker and weave net for development + hosts: all + gather_facts: false # required in case Python is not available on the host + become: true + become_user: root + + pre_tasks: + - include: library/setup_ansible_dependencies.yml + + roles: + - docker-install + - weave-net-utilities + - weave-net diff --git a/tools/config_management/setup_weave-net_dev.yml b/tools/config_management/setup_weave-net_dev.yml new file mode 100644 index 0000000000..bdfa08e907 --- /dev/null +++ b/tools/config_management/setup_weave-net_dev.yml @@ -0,0 +1,20 @@ +--- +################################################################################ +# Install Docker from Docker's official repository and Weave Net. +################################################################################ + +- name: install docker and weave net for development + hosts: all + gather_facts: false # required in case Python is not available on the host + become: true + become_user: root + + pre_tasks: + - include: library/setup_ansible_dependencies.yml + + roles: + - dev-tools + - golang-from-tarball + - docker-install + # Do not run this role when building with Vagrant, as sources have been already checked out: + - { role: weave-net-sources, when: "ansible_user != 'vagrant'" } diff --git a/tools/config_management/setup_weave-net_test.yml b/tools/config_management/setup_weave-net_test.yml new file mode 100644 index 0000000000..fbd155df70 --- /dev/null +++ b/tools/config_management/setup_weave-net_test.yml @@ -0,0 +1,20 @@ +--- +################################################################################ +# Install Docker from Docker's official repository and Weave Net. +################################################################################ + +- name: install docker and weave net for testing + hosts: all + gather_facts: false # required in case Python is not available on the host + become: true + become_user: root + + pre_tasks: + - include: library/setup_ansible_dependencies.yml + + roles: + - docker-install + - weave-net-utilities + - kubernetes-install + - kubernetes-docker-images + - kubelet-stop diff --git a/tools/dependencies/cross_versions.py b/tools/dependencies/cross_versions.py new file mode 100755 index 0000000000..dd920f0ef4 --- /dev/null +++ b/tools/dependencies/cross_versions.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python + +# Generate the cross product of latest versions of Weave Net's dependencies: +# - Go +# - Docker +# - Kubernetes +# +# Dependencies: +# - python +# - git +# - list_versions.py +# +# Testing: +# $ python -m doctest -v cross_versions.py + +from os import linesep +from sys import argv, exit, stdout, stderr +from getopt import getopt, GetoptError +from list_versions import DEPS, get_versions_from, filter_versions +from itertools import product + +# See also: /usr/include/sysexits.h +_ERROR_RUNTIME = 1 +_ERROR_ILLEGAL_ARGS = 64 + + +def _usage(error_message=None): + if error_message: + stderr.write('ERROR: ' + error_message + linesep) + stdout.write( + linesep.join([ + 'Usage:', ' cross_versions.py [OPTION]...', 'Examples:', + ' cross_versions.py', ' cross_versions.py -r', + ' cross_versions.py --rc', ' cross_versions.py -l', + ' cross_versions.py --latest', 'Options:', + '-l/--latest Include only the latest version of each major and' + ' minor versions sub-tree.', + '-r/--rc Include release candidate versions.', + '-h/--help Prints this!', '' + ])) + + +def _validate_input(argv): + try: + config = {'rc': False, 'latest': False} + opts, args = getopt(argv, 'hlr', ['help', 'latest', 'rc']) + for opt, value in opts: + if opt in ('-h', '--help'): + _usage() + exit() + if opt in ('-l', '--latest'): + config['latest'] = True + if opt in ('-r', '--rc'): + config['rc'] = True + if len(args) != 0: + raise ValueError('Unsupported argument(s): %s.' % args) + return config + except GetoptError as e: + _usage(str(e)) + exit(_ERROR_ILLEGAL_ARGS) + except ValueError as e: + _usage(str(e)) + exit(_ERROR_ILLEGAL_ARGS) + + +def _versions(dependency, config): + return map(str, + filter_versions( + get_versions_from(DEPS[dependency]['url'], + DEPS[dependency]['re']), + DEPS[dependency]['min'], **config)) + + +def cross_versions(config): + docker_versions = _versions('docker', config) + k8s_versions = _versions('kubernetes', config) + return product(docker_versions, k8s_versions) + + +def main(argv): + try: + config = _validate_input(argv) + print(linesep.join('\t'.join(triple) + for triple in cross_versions(config))) + except Exception as e: + print(str(e)) + exit(_ERROR_RUNTIME) + + +if __name__ == '__main__': + main(argv[1:]) diff --git a/tools/dependencies/list_os_images.sh b/tools/dependencies/list_os_images.sh new file mode 100755 index 0000000000..00db0d0614 --- /dev/null +++ b/tools/dependencies/list_os_images.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +function usage() { + cat <&2 "No AWS owner ID for $1." + exit 1 +} + +if [ -z "$1" ]; then + echo >&2 "No specified provider." + usage + exit 1 +fi + +if [ -z "$2" ]; then + if [ "$1" == "help" ]; then + usage + exit 0 + else + echo >&2 "No specified operating system." + usage + exit 1 + fi +fi + +case "$1" in + 'gcp') + gcloud compute images list --standard-images --regexp=".*?$2.*" \ + --format="csv[no-heading][separator=/](selfLink.map().scope(projects).segment(0),family)" \ + | sort -d + ;; + 'aws') + aws --region "${3:-us-east-1}" ec2 describe-images \ + --owners "$(find_aws_owner_id "$2")" \ + --filters "Name=name,Values=$2*" \ + --query 'Images[*].{name:Name,id:ImageId}' + # Other examples: + # - CentOS: aws --region us-east-1 ec2 describe-images --owners aws-marketplace --filters Name=product-code,Values=aw0evgkw8e5c1q413zgy5pjce + # - Debian: aws --region us-east-1 ec2 describe-images --owners 379101102735 --filters "Name=architecture,Values=x86_64" "Name=name,Values=debian-jessie-*" "Name=root-device-type,Values=ebs" "Name=virtualization-type,Values=hvm" + ;; + 'do') + curl -s -X GET \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $DIGITALOCEAN_TOKEN" \ + "https://api.digitalocean.com/v2/images?page=1&per_page=999999" \ + | jq --raw-output ".images | .[] | .slug" | grep "$2" | sort -d + ;; + *) + echo >&2 "Unknown provider [$1]." + usage + exit 1 + ;; +esac diff --git a/tools/dependencies/list_versions.py b/tools/dependencies/list_versions.py new file mode 100755 index 0000000000..e008ecfef4 --- /dev/null +++ b/tools/dependencies/list_versions.py @@ -0,0 +1,343 @@ +#!/usr/bin/env python + +# List all available versions of Weave Net's dependencies: +# - Go +# - Docker +# - Kubernetes +# +# Depending on the parameters passed, it can gather the equivalent of the below +# bash one-liners: +# git ls-remote --tags https://github.com/golang/go \ +# | grep -oP '(?<=refs/tags/go)[\.\d]+$' \ +# | sort --version-sort +# git ls-remote --tags https://github.com/golang/go \ +# | grep -oP '(?<=refs/tags/go)[\.\d]+rc\d+$' \ +# | sort --version-sort \ +# | tail -n 1 +# git ls-remote --tags https://github.com/docker/docker \ +# | grep -oP '(?<=refs/tags/v)\d+\.\d+\.\d+$' \ +# | sort --version-sort +# git ls-remote --tags https://github.com/docker/docker \ +# | grep -oP '(?<=refs/tags/v)\d+\.\d+\.\d+\-rc\d*$' \ +# | sort --version-sort \ +# | tail -n 1 +# git ls-remote --tags https://github.com/kubernetes/kubernetes \ +# | grep -oP '(?<=refs/tags/v)\d+\.\d+\.\d+$' \ +# | sort --version-sort +# git ls-remote --tags https://github.com/kubernetes/kubernetes \ +# | grep -oP '(?<=refs/tags/v)\d+\.\d+\.\d+\-beta\.\d+$' \ +# | sort --version-sort | tail -n 1 +# +# Dependencies: +# - python +# - git +# +# Testing: +# $ python -m doctest -v list_versions.py + +from os import linesep, path +from sys import argv, exit, stdout, stderr +from getopt import getopt, GetoptError +from subprocess import Popen, PIPE +from pkg_resources import parse_version +from itertools import groupby +from six.moves import filter +import shlex +import re + +# See also: /usr/include/sysexits.h +_ERROR_RUNTIME = 1 +_ERROR_ILLEGAL_ARGS = 64 + +_TAG_REGEX = '^[0-9a-f]{40}\s+refs/tags/%s$' +_VERSION = 'version' +DEPS = { + 'go': { + 'url': 'https://github.com/golang/go', + 're': 'go(?P<%s>[\d\.]+(?:rc\d)*)' % _VERSION, + 'min': None + }, + 'docker': { + 'url': 'https://github.com/docker/docker', + 're': 'v(?P<%s>\d+\.\d+\.\d+(?:\-rc\d)*)' % _VERSION, + # Weave Net only works with Docker from 1.10.0 onwards, so we ignore + # all previous versions: + 'min': '1.10.0', + }, + 'kubernetes': { + 'url': 'https://github.com/kubernetes/kubernetes', + 're': 'v(?P<%s>\d+\.\d+\.\d+(?:\-beta\.\d)*)' % _VERSION, + # Weave Kube requires Kubernetes 1.4.2+, so we ignore all previous + # versions: + 'min': '1.4.2', + } +} + + +class Version(object): + ''' Helper class to parse and manipulate (sort, filter, group) software + versions. ''' + + def __init__(self, version): + self.version = version + self.digits = [ + int(x) if x else 0 + for x in re.match('(\d*)\.?(\d*)\.?(\d*).*?', version).groups() + ] + self.major, self.minor, self.patch = self.digits + self.__parsed = parse_version(version) + self.is_rc = self.__parsed.is_prerelease + + def __lt__(self, other): + return self.__parsed.__lt__(other.__parsed) + + def __gt__(self, other): + return self.__parsed.__gt__(other.__parsed) + + def __le__(self, other): + return self.__parsed.__le__(other.__parsed) + + def __ge__(self, other): + return self.__parsed.__ge__(other.__parsed) + + def __eq__(self, other): + return self.__parsed.__eq__(other.__parsed) + + def __ne__(self, other): + return self.__parsed.__ne__(other.__parsed) + + def __str__(self): + return self.version + + def __repr__(self): + return self.version + + +def _read_go_version_from_dockerfile(): + # Read Go version from weave/build/Dockerfile + dockerfile_path = path.join( + path.dirname(path.dirname(path.dirname(path.realpath(__file__)))), + 'build', 'Dockerfile') + with open(dockerfile_path, 'r') as f: + for line in f: + m = re.match('^FROM golang:(\S*)$', line) + if m: + return m.group(1) + raise RuntimeError( + "Failed to read Go version from weave/build/Dockerfile." + " You may be running this script from somewhere else than weave/tools." + ) + + +def _try_set_min_go_version(): + ''' Set the current version of Go used to build Weave Net's containers as + the minimum version. ''' + try: + DEPS['go']['min'] = _read_go_version_from_dockerfile() + except IOError as e: + stderr.write('WARNING: No minimum Go version set. Root cause: %s%s' % + (e, linesep)) + + +def _sanitize(out): + return out.decode('ascii').strip().split(linesep) + + +def _parse_tag(tag, version_pattern, debug=False): + ''' Parse Git tag output's line using the provided `version_pattern`, e.g.: + >>> _parse_tag( + '915b77eb4efd68916427caf8c7f0b53218c5ea4a refs/tags/v1.4.6', + 'v(?P\d+\.\d+\.\d+(?:\-beta\.\d)*)') + '1.4.6' + ''' + pattern = _TAG_REGEX % version_pattern + m = re.match(pattern, tag) + if m: + return m.group(_VERSION) + elif debug: + stderr.write( + 'ERROR: Failed to parse version out of tag [%s] using [%s].%s' % + (tag, pattern, linesep)) + + +def get_versions_from(git_repo_url, version_pattern): + ''' Get release and release candidates' versions from the provided Git + repository. ''' + git = Popen( + shlex.split('git ls-remote --tags %s' % git_repo_url), stdout=PIPE) + out, err = git.communicate() + status_code = git.returncode + if status_code != 0: + raise RuntimeError('Failed to retrieve git tags from %s. ' + 'Status code: %s. Output: %s. Error: %s' % + (git_repo_url, status_code, out, err)) + return list( + filter(None, (_parse_tag(line, version_pattern) + for line in _sanitize(out)))) + + +def _tree(versions, level=0): + ''' Group versions by major, minor and patch version digits. ''' + if not versions or level >= len(versions[0].digits): + return # Empty versions or no more digits to group by. + versions_tree = [] + for _, versions_group in groupby(versions, lambda v: v.digits[level]): + subtree = _tree(list(versions_group), level + 1) + if subtree: + versions_tree.append(subtree) + # Return the current subtree if non-empty, or the list of "leaf" versions: + return versions_tree if versions_tree else versions + + +def _is_iterable(obj): + ''' + Check if the provided object is an iterable collection, i.e. not a string, + e.g. a list, a generator: + >>> _is_iterable('string') + False + >>> _is_iterable([1, 2, 3]) + True + >>> _is_iterable((x for x in [1, 2, 3])) + True + ''' + return hasattr(obj, '__iter__') and not isinstance(obj, str) + + +def _leaf_versions(tree, rc): + ''' + Recursively traverse the versions tree in a depth-first fashion, + and collect the last node of each branch, i.e. leaf versions. + ''' + versions = [] + if _is_iterable(tree): + for subtree in tree: + versions.extend(_leaf_versions(subtree, rc)) + if not versions: + if rc: + last_rc = next(filter(lambda v: v.is_rc, reversed(tree)), None) + last_prod = next( + filter(lambda v: not v.is_rc, reversed(tree)), None) + if last_rc and last_prod and (last_prod < last_rc): + versions.extend([last_prod, last_rc]) + elif not last_prod: + versions.append(last_rc) + else: + # Either there is no RC, or we ignore the RC as older than + # the latest production version: + versions.append(last_prod) + else: + versions.append(tree[-1]) + return versions + + +def filter_versions(versions, min_version=None, rc=False, latest=False): + ''' Filter provided versions + + >>> filter_versions( + ['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'], + min_version=None, latest=False, rc=False) + [1.0.0, 1.0.1, 1.1.1, 2.0.0] + + >>> filter_versions( + ['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'], + min_version=None, latest=True, rc=False) + [1.0.1, 1.1.1, 2.0.0] + + >>> filter_versions( + ['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'], + min_version=None, latest=False, rc=True) + [1.0.0-beta.1, 1.0.0, 1.0.1, 1.1.1, 1.1.2-rc1, 2.0.0] + + >>> filter_versions( + ['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'], + min_version='1.1.0', latest=False, rc=True) + [1.1.1, 1.1.2-rc1, 2.0.0] + + >>> filter_versions( + ['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'], + min_version=None, latest=True, rc=True) + [1.0.1, 1.1.1, 1.1.2-rc1, 2.0.0] + + >>> filter_versions( + ['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'], + min_version='1.1.0', latest=True, rc=True) + [1.1.1, 1.1.2-rc1, 2.0.0] + ''' + versions = sorted([Version(v) for v in versions]) + if min_version: + min_version = Version(min_version) + versions = [v for v in versions if v >= min_version] + if not rc: + versions = [v for v in versions if not v.is_rc] + if latest: + versions_tree = _tree(versions) + return _leaf_versions(versions_tree, rc) + else: + return versions + + +def _usage(error_message=None): + if error_message: + stderr.write('ERROR: ' + error_message + linesep) + stdout.write( + linesep.join([ + 'Usage:', ' list_versions.py [OPTION]... [DEPENDENCY]', + 'Examples:', ' list_versions.py go', + ' list_versions.py -r docker', + ' list_versions.py --rc docker', + ' list_versions.py -l kubernetes', + ' list_versions.py --latest kubernetes', 'Options:', + '-l/--latest Include only the latest version of each major and' + ' minor versions sub-tree.', + '-r/--rc Include release candidate versions.', + '-h/--help Prints this!', '' + ])) + + +def _validate_input(argv): + try: + config = {'rc': False, 'latest': False} + opts, args = getopt(argv, 'hlr', ['help', 'latest', 'rc']) + for opt, value in opts: + if opt in ('-h', '--help'): + _usage() + exit() + if opt in ('-l', '--latest'): + config['latest'] = True + if opt in ('-r', '--rc'): + config['rc'] = True + if len(args) != 1: + raise ValueError('Please provide a dependency to get versions of.' + ' Expected 1 argument but got %s: %s.' % + (len(args), args)) + dependency = args[0].lower() + if dependency not in DEPS.keys(): + raise ValueError( + 'Please provide a valid dependency.' + ' Supported one dependency among {%s} but got: %s.' % + (', '.join(DEPS.keys()), dependency)) + return dependency, config + except GetoptError as e: + _usage(str(e)) + exit(_ERROR_ILLEGAL_ARGS) + except ValueError as e: + _usage(str(e)) + exit(_ERROR_ILLEGAL_ARGS) + + +def main(argv): + try: + dependency, config = _validate_input(argv) + if dependency == 'go': + _try_set_min_go_version() + versions = get_versions_from(DEPS[dependency]['url'], + DEPS[dependency]['re']) + versions = filter_versions(versions, DEPS[dependency]['min'], **config) + print(linesep.join(map(str, versions))) + except Exception as e: + print(str(e)) + exit(_ERROR_RUNTIME) + + +if __name__ == '__main__': + main(argv[1:]) diff --git a/tools/image-tag b/tools/image-tag index d1fd2f728f..31f023dac0 100755 --- a/tools/image-tag +++ b/tools/image-tag @@ -4,6 +4,6 @@ set -o errexit set -o nounset set -o pipefail -WORKING_SUFFIX=$(if ! git diff --exit-code --quiet HEAD >&2; then echo "-WIP"; else echo ""; fi) +WORKING_SUFFIX=$(if git status --porcelain | grep -qE '^(?:[^?][^ ]|[^ ][^?])\s'; then echo "-WIP"; else echo ""; fi) BRANCH_PREFIX=$(git rev-parse --abbrev-ref HEAD) echo "${BRANCH_PREFIX//\//-}-$(git rev-parse --short HEAD)$WORKING_SUFFIX" diff --git a/tools/integration/config.sh b/tools/integration/config.sh index 3dfa3cac42..5419219203 100644 --- a/tools/integration/config.sh +++ b/tools/integration/config.sh @@ -115,11 +115,8 @@ rm_containers() { start_suite() { for host in $HOSTS; do [ -z "$DEBUG" ] || echo "Cleaning up on $host: removing all containers and resetting weave" - PLUGIN_ID=$(docker_on "$host" ps -aq --filter=name=weaveplugin) - PLUGIN_FILTER="cat" - [ -n "$PLUGIN_ID" ] && PLUGIN_FILTER="grep -v $PLUGIN_ID" # shellcheck disable=SC2046 - rm_containers "$host" $(docker_on "$host" ps -aq 2>/dev/null | $PLUGIN_FILTER) + rm_containers "$host" $(docker_on "$host" ps -aq 2>/dev/null) run_on "$host" "docker network ls | grep -q ' weave ' && docker network rm weave" || true weave_on "$host" reset 2>/dev/null done @@ -130,4 +127,4 @@ end_suite() { whitely assert_end } -WEAVE=$DIR/../weave +WEAVE=$DIR/../../integration/weave diff --git a/tools/integration/gce.sh b/tools/integration/gce.sh index 5129916ede..5c39401846 100755 --- a/tools/integration/gce.sh +++ b/tools/integration/gce.sh @@ -9,7 +9,9 @@ set -e : "${KEY_FILE:=/tmp/gce_private_key.json}" : "${SSH_KEY_FILE:=$HOME/.ssh/gce_ssh_key}" -: "${IMAGE:=ubuntu-14-04}" +: "${IMAGE_FAMILY:=ubuntu-1404-lts}" +: "${IMAGE_PROJECT:=ubuntu-os-cloud}" +: "${USER_ACCOUNT:=ubuntu}" : "${ZONE:=us-central1-a}" : "${PROJECT:=}" : "${TEMPLATE_NAME:=}" @@ -22,7 +24,9 @@ fi SUFFIX="" if [ -n "$CIRCLECI" ]; then - SUFFIX="-${CIRCLE_BUILD_NUM}-$CIRCLE_NODE_INDEX" + SUFFIX="-${CIRCLE_PROJECT_USERNAME}-${CIRCLE_PROJECT_REPONAME}-${CIRCLE_BUILD_NUM}-$CIRCLE_NODE_INDEX" +else + SUFFIX="-${USER}" fi # Setup authentication @@ -40,8 +44,13 @@ function vm_names() { # Delete all vms in this account function destroy() { local names + # shellcheck disable=SC2046 + if [ $(gcloud compute firewall-rules list "test-allow-docker$SUFFIX" 2>/dev/null | wc -l) -gt 0 ]; then + gcloud compute firewall-rules delete "test-allow-docker$SUFFIX" + fi names="$(vm_names)" - if [ "$(gcloud compute instances list --zone "$ZONE" -q "$names" | wc -l)" -le 1 ]; then + # shellcheck disable=SC2086 + if [ "$(gcloud compute instances list --zones "$ZONE" -q $names | wc -l)" -le 1 ]; then return 0 fi for i in {0..10}; do @@ -82,12 +91,16 @@ function try_connect() { function install_docker_on() { name=$1 + echo "Installing Docker on $name for user ${USER_ACCOUNT}" + # shellcheck disable=SC2087 ssh -t "$name" sudo bash -x -s <> /etc/default/docker; service docker restart EOF @@ -107,7 +120,10 @@ function setup() { destroy names=($(vm_names)) - gcloud compute instances create "${names[@]}" --image "$TEMPLATE_NAME" --zone "$ZONE" + gcloud compute instances create "${names[@]}" --image "$TEMPLATE_NAME" --zone "$ZONE" --tags "test$SUFFIX" --network=test + my_ip="$(curl -s http://ipinfo.io/ip)" + gcloud compute firewall-rules create "test-allow-docker$SUFFIX" --network=test --allow tcp:2375,tcp:12375,tcp:4040,tcp:80 --target-tags "test$SUFFIX" --source-ranges "$my_ip" + gcloud compute config-ssh --ssh-key-file "$SSH_KEY_FILE" sed -i '/UserKnownHostsFile=\/dev\/null/d' ~/.ssh/config @@ -136,7 +152,7 @@ function setup() { } function make_template() { - gcloud compute instances create "$TEMPLATE_NAME" --image "$IMAGE" --zone "$ZONE" + gcloud compute instances create "$TEMPLATE_NAME" --image-family "$IMAGE_FAMILY" --image-project "$IMAGE_PROJECT" --zone "$ZONE" gcloud compute config-ssh --ssh-key-file "$SSH_KEY_FILE" name="$TEMPLATE_NAME.$ZONE.$PROJECT" try_connect "$name" @@ -155,7 +171,7 @@ function hosts() { hosts=($hostname "${hosts[@]}") args=("--add-host=$hostname:$(internal_ip "$json" "$name")" "${args[@]}") done - echo export SSH=\"ssh -l vagrant\" + echo export SSH=\"ssh -l "${USER_ACCOUNT}"\" echo "export HOSTS=\"${hosts[*]}\"" echo "export ADD_HOST_ARGS=\"${args[*]}\"" rm "$json" @@ -178,6 +194,9 @@ case "$1" in # see if template exists if ! gcloud compute images list | grep "$PROJECT" | grep "$TEMPLATE_NAME"; then make_template + else + echo "Reusing existing template:" + gcloud compute images describe "$TEMPLATE_NAME" | grep "^creationTimestamp" fi ;; esac diff --git a/tools/integration/sanity_check.sh b/tools/integration/sanity_check.sh index c88337fe80..192112de80 100755 --- a/tools/integration/sanity_check.sh +++ b/tools/integration/sanity_check.sh @@ -1,6 +1,6 @@ #! /bin/bash -# shellcheck disable=SC1091 -. ./config.sh +# shellcheck disable=SC1090,SC1091 +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/config.sh" set -e diff --git a/tools/lint b/tools/lint index c97c2de38d..63c5066185 100755 --- a/tools/lint +++ b/tools/lint @@ -6,7 +6,7 @@ # # For shell files, it runs shfmt. If you don't have that installed, you can get # it with: -# go get -u github.com/mvdan/sh/cmd/shfmt +# go get -u gopkg.in/mvdan/sh.v1/cmd/shfmt # # With no arguments, it lints the current files staged # for git commit. Or you can pass it explicit filenames @@ -113,7 +113,7 @@ lint_sh() { local filename="$1" local lint_result=0 - if ! diff <(shfmt -i 4 "${filename}") "${filename}" >/dev/null; then + if ! diff -u "${filename}" <(shfmt -i 4 "${filename}"); then lint_result=1 echo "${filename}: run shfmt -i 4 -w ${filename}" fi @@ -131,7 +131,7 @@ lint_tf() { local filename="$1" local lint_result=0 - if ! diff <(hclfmt "${filename}") "${filename}" >/dev/null; then + if ! diff -u <(hclfmt "${filename}") "${filename}"; then lint_result=1 echo "${filename}: run hclfmt -w ${filename}" fi @@ -139,6 +139,35 @@ lint_tf() { return $lint_result } +lint_md() { + local filename="$1" + local lint_result=0 + + for i in '=======' '>>>>>>>'; do + if grep -q "${i}" "${filename}"; then + lint_result=1 + echo "${filename}: bad merge/rebase!" + fi + done + + return $lint_result +} + +lint_py() { + local filename="$1" + local lint_result=0 + + if yapf --diff "${filename}" | grep -qE '^[+-]'; then + lint_result=1 + echo "${filename}: run yapf --in-place ${filename}" + else + # Only run flake8 if yapf passes, since they pick up a lot of similar issues + flake8 "${filename}" || lint_result=1 + fi + + return $lint_result +} + lint() { filename="$1" ext="${filename##*\.}" @@ -164,6 +193,8 @@ lint() { go) lint_go "${filename}" || lint_result=1 ;; sh) lint_sh "${filename}" || lint_result=1 ;; tf) lint_tf "${filename}" || lint_result=1 ;; + md) lint_md "${filename}" || lint_result=1 ;; + py) lint_py "${filename}" || lint_result=1 ;; esac spell_check "${filename}" || lint_result=1 @@ -194,7 +225,8 @@ matches_any() { filter_out() { local patterns_file="$1" if [ -n "$patterns_file" ] && [ -r "$patterns_file" ]; then - local patterns=$(sed '/^#.*$/d ; /^\s*$/d' "$patterns_file") # Remove blank lines and comments before we start iterating. + local patterns + patterns=$(sed '/^#.*$/d ; /^\s*$/d' "$patterns_file") # Remove blank lines and comments before we start iterating. [ -n "$DEBUG" ] && echo >&2 "> Filters:" && echo >&2 "$patterns" local filtered_out=() while read -r filename; do @@ -217,7 +249,7 @@ list_files() { if [ $# = 1 ] && [ -f "$1" ]; then lint "$1" elif [ -n "$PARALLEL" ]; then - list_files "$@" | filter_out "$LINT_IGNORE_FILE" | xargs -n1 -P16 $0 + list_files "$@" | filter_out "$LINT_IGNORE_FILE" | xargs -n1 -P16 "$0" else list_files "$@" | filter_out "$LINT_IGNORE_FILE" | lint_files fi diff --git a/tools/provisioning/README.md b/tools/provisioning/README.md new file mode 100755 index 0000000000..627bb42e3e --- /dev/null +++ b/tools/provisioning/README.md @@ -0,0 +1,55 @@ +# Weaveworks provisioning + +## Introduction + +This project allows you to get hold of some machine either locally or on one of the below cloud providers: + +* Amazon Web Services +* Digital Ocean +* Google Cloud Platform + +You can then use these machines as is or run various Ansible playbooks from `../config_management` to set up Weave Net, Kubernetes, etc. + +## Set up + +* You will need [Vagrant](https://www.vagrantup.com) installed on your machine and added to your `PATH` in order to be able to provision local (virtual) machines automatically. + + * On macOS: `brew install vagrant` + * On Linux (via Aptitude): `sudo apt install vagrant` + * If you need a specific version: + + curl -fsS https://releases.hashicorp.com/terraform/x.y.z/terraform_x.y.z_linux_amd64.zip | gunzip > terraform && chmod +x terraform && sudo mv terraform /usr/bin + + * For other platforms or more details, see [here](https://www.vagrantup.com/docs/installation/) + +* You will need [Terraform](https://www.terraform.io) installed on your machine and added to your `PATH` in order to be able to provision cloud-hosted machines automatically. + + * On macOS: `brew install terraform` + * On Linux (via Aptitude): `sudo apt install terraform` + * For other platforms or more details, see [here](https://www.terraform.io/intro/getting-started/install.html) + +* Depending on the cloud provider, you may have to create an account, manually onboard, create and register SSH keys, etc. + Please refer to the `README.md` in each sub-folder for more details. + +## Usage in scripts + +Source `setup.sh`, set the `SECRET_KEY` environment variable, and depending on the cloud provider you want to use, call either: + +* `gcp_on` / `gcp_off` +* `do_on` / `do_off` +* `aws_on` / `aws_off` + +## Usage in shell + +Source `setup.sh`, set the `SECRET_KEY` environment variable, and depending on the cloud provider you want to use, call either: + +* `gcp_on` / `gcp_off` +* `do_on` / `do_off` +* `aws_on` / `aws_off` + +Indeed, the functions defined in `setup.sh` are also exported as aliases, so you can call them from your shell directly. + +Other aliases are also defined, in order to make your life easier: + +* `tf_ssh`: to ease SSH-ing into the virtual machines, reading the username and IP address to use from Terraform, as well as setting default SSH options. +* `tf_ansi`: to ease applying an Ansible playbook to a set of virtual machines, dynamically creating the inventory, as well as setting default SSH options. diff --git a/tools/provisioning/aws/README.md b/tools/provisioning/aws/README.md new file mode 100644 index 0000000000..f4f018f97c --- /dev/null +++ b/tools/provisioning/aws/README.md @@ -0,0 +1,90 @@ +# Amazon Web Services + +## Introduction + +This project allows you to get hold of some machine on Amazon Web Services. +You can then use these machines as is or run various Ansible playbooks from `../config_management` to set up Weave Net, Kubernetes, etc. + +## Setup + +* Log in [weaveworks.signin.aws.amazon.com/console](https://weaveworks.signin.aws.amazon.com/console/) with your account. + +* Go to `Services` > `IAM` > `Users` > Click on your username > `Security credentials` > `Create access key`. + Your access key and secret key will appear on the screen. Set these as environment variables: + +``` +export AWS_ACCESS_KEY_ID= +export AWS_SECRET_ACCESS_KEY= +``` + +* Go to `Services` > `EC2` > Select the availability zone you want to use (see top right corner, e.g. `us-east-1`) > `Import Key Pair`. + Enter your SSH public key and the name for it, and click `Import`. + Set the path to your private key as an environment variable: + +``` +export TF_VAR_aws_public_key_name= +export TF_VAR_aws_private_key_path="$HOME/.ssh/id_rsa" +``` + +* Set your current IP address as an environment variable: + +``` +export TF_VAR_client_ip=$(curl -s -X GET http://checkip.amazonaws.com/) +``` + + or pass it as a Terraform variable: + +``` +$ terraform -var 'client_ip=$(curl -s -X GET http://checkip.amazonaws.com/)' +``` + +### Bash aliases + +You can set the above variables temporarily in your current shell, permanently in your `~/.bashrc` file, or define aliases to activate/deactivate them at will with one single command by adding the below to your `~/.bashrc` file: + +``` +function _aws_on() { + export AWS_ACCESS_KEY_ID="" # Replace with appropriate value. + export AWS_SECRET_ACCESS_KEY="" # Replace with appropriate value. + export TF_VAR_aws_public_key_name="" # Replace with appropriate value. + export TF_VAR_aws_private_key_path="$HOME/.ssh/id_rsa" # Replace with appropriate value. +} +alias _aws_on='_aws_on' +function _aws_off() { + unset AWS_ACCESS_KEY_ID + unset AWS_SECRET_ACCESS_KEY + unset TF_VAR_aws_public_key_name + unset TF_VAR_aws_private_key_path +} +alias _aws_off='_aws_off' +``` + +N.B.: + +* sourcing `../setup.sh` defines aliases called `aws_on` and `aws_off`, similarly to the above (however, notice no `_` in front of the name, as opposed to the ones above); +* `../setup.sh`'s `aws_on` alias needs the `SECRET_KEY` environment variable to be set in order to decrypt sensitive information. + +## Usage + +* Create the machine: `terraform apply` +* Show the machine's status: `terraform show` +* Stop and destroy the machine: `terraform destroy` +* SSH into the newly-created machine: + +``` +$ ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no `terraform output username`@`terraform output public_ips` +# N.B.: the default username will differ depending on the AMI/OS you installed, e.g. ubuntu for Ubuntu, ec2-user for Red Hat, etc. +``` + +or + +``` +source ../setup.sh +tf_ssh 1 # Or the nth machine, if multiple VMs are provisioned. +``` + +## Resources + +* [https://www.terraform.io/docs/providers/aws/](https://www.terraform.io/docs/providers/aws/) +* [https://www.terraform.io/docs/providers/aws/r/instance.html](https://www.terraform.io/docs/providers/aws/r/instance.html) +* [Terraform variables](https://www.terraform.io/intro/getting-started/variables.html) diff --git a/tools/provisioning/aws/main.tf b/tools/provisioning/aws/main.tf new file mode 100755 index 0000000000..f4be8c345d --- /dev/null +++ b/tools/provisioning/aws/main.tf @@ -0,0 +1,137 @@ +# Specify the provider and access details +provider "aws" { + # Access key, secret key and region are sourced from environment variables or input arguments -- see README.md + region = "${var.aws_dc}" +} + +resource "aws_security_group" "allow_ssh" { + name = "${var.name}_allow_ssh" + description = "AWS security group to allow SSH-ing onto AWS EC2 instances (created using Terraform)." + + # Open TCP port for SSH: + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["${var.client_ip}/32"] + } + + tags { + Name = "${var.name}_allow_ssh" + App = "${var.app}" + CreatedBy = "terraform" + } +} + +resource "aws_security_group" "allow_docker" { + name = "${var.name}_allow_docker" + description = "AWS security group to allow communication with Docker on AWS EC2 instances (created using Terraform)." + + # Open TCP port for Docker: + ingress { + from_port = 2375 + to_port = 2375 + protocol = "tcp" + cidr_blocks = ["${var.client_ip}/32"] + } + + tags { + Name = "${var.name}_allow_docker" + App = "${var.app}" + CreatedBy = "terraform" + } +} + +resource "aws_security_group" "allow_weave" { + name = "${var.name}_allow_weave" + description = "AWS security group to allow communication with Weave on AWS EC2 instances (created using Terraform)." + + # Open TCP port for Weave: + ingress { + from_port = 12375 + to_port = 12375 + protocol = "tcp" + cidr_blocks = ["${var.client_ip}/32"] + } + + tags { + Name = "${var.name}_allow_weave" + App = "${var.app}" + CreatedBy = "terraform" + } +} + +resource "aws_security_group" "allow_private_ingress" { + name = "${var.name}_allow_private_ingress" + description = "AWS security group to allow all private ingress traffic on AWS EC2 instances (created using Terraform)." + + # Full inbound local network access on both TCP and UDP + ingress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["${var.aws_vpc_cidr_block}"] + } + + tags { + Name = "${var.name}_allow_private_ingress" + App = "${var.app}" + CreatedBy = "terraform" + } +} + +resource "aws_security_group" "allow_all_egress" { + name = "${var.name}_allow_all_egress" + description = "AWS security group to allow all egress traffic on AWS EC2 instances (created using Terraform)." + + # Full outbound internet access on both TCP and UDP + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags { + Name = "${var.name}_allow_all_egress" + App = "${var.app}" + CreatedBy = "terraform" + } +} + +resource "aws_instance" "tf_test_vm" { + instance_type = "${var.aws_size}" + count = "${var.num_hosts}" + + # Lookup the correct AMI based on the region we specified + ami = "${lookup(var.aws_amis, var.aws_dc)}" + + key_name = "${var.aws_public_key_name}" + + security_groups = [ + "${aws_security_group.allow_ssh.name}", + "${aws_security_group.allow_docker.name}", + "${aws_security_group.allow_weave.name}", + "${aws_security_group.allow_private_ingress.name}", + "${aws_security_group.allow_all_egress.name}", + ] + + # Wait for machine to be SSH-able: + provisioner "remote-exec" { + inline = ["exit"] + + connection { + type = "ssh" + + # Lookup the correct username based on the AMI we specified + user = "${lookup(var.aws_usernames, "${lookup(var.aws_amis, var.aws_dc)}")}" + private_key = "${file("${var.aws_private_key_path}")}" + } + } + + tags { + Name = "${var.name}-${count.index}" + App = "${var.app}" + CreatedBy = "terraform" + } +} diff --git a/tools/provisioning/aws/outputs.tf b/tools/provisioning/aws/outputs.tf new file mode 100755 index 0000000000..587986b8b6 --- /dev/null +++ b/tools/provisioning/aws/outputs.tf @@ -0,0 +1,54 @@ +output "username" { + value = "${lookup(var.aws_usernames, "${lookup(var.aws_amis, var.aws_dc)}")}" +} + +output "public_ips" { + value = ["${aws_instance.tf_test_vm.*.public_ip}"] +} + +output "hostnames" { + value = "${join("\n", + "${formatlist("%v.%v.%v", + aws_instance.tf_test_vm.*.tags.Name, + aws_instance.tf_test_vm.*.availability_zone, + var.app + )}" + )}" +} + +# /etc/hosts file for the Droplets: +output "private_etc_hosts" { + value = "${join("\n", + "${formatlist("%v %v.%v.%v", + aws_instance.tf_test_vm.*.private_ip, + aws_instance.tf_test_vm.*.tags.Name, + aws_instance.tf_test_vm.*.availability_zone, + var.app + )}" + )}" +} + +# /etc/hosts file for the client: +output "public_etc_hosts" { + value = "${join("\n", + "${formatlist("%v %v.%v.%v", + aws_instance.tf_test_vm.*.public_ip, + aws_instance.tf_test_vm.*.tags.Name, + aws_instance.tf_test_vm.*.availability_zone, + var.app + )}" + )}" +} + +output "ansible_inventory" { + value = "${format("[all]\n%s", join("\n", + "${formatlist("%v private_ip=%v", + aws_instance.tf_test_vm.*.public_ip, + aws_instance.tf_test_vm.*.private_ip, + )}" + ))}" +} + +output "private_key_path" { + value = "${var.aws_private_key_path}" +} diff --git a/tools/provisioning/aws/variables.tf b/tools/provisioning/aws/variables.tf new file mode 100755 index 0000000000..5f4b462888 --- /dev/null +++ b/tools/provisioning/aws/variables.tf @@ -0,0 +1,68 @@ +variable "client_ip" { + description = "IP address of the client machine" +} + +variable "app" { + description = "Name of the application using the created EC2 instance(s)." + default = "default" +} + +variable "name" { + description = "Name of the EC2 instance(s)." + default = "test" +} + +variable "num_hosts" { + description = "Number of EC2 instance(s)." + default = 1 +} + +variable "aws_vpc_cidr_block" { + description = "AWS VPC CIDR block to use to attribute private IP addresses." + default = "172.31.0.0/16" +} + +variable "aws_public_key_name" { + description = "Name of the SSH keypair to use in AWS." +} + +variable "aws_private_key_path" { + description = "Path to file containing private key" + default = "~/.ssh/id_rsa" +} + +variable "aws_dc" { + description = "The AWS region to create things in." + default = "us-east-1" +} + +variable "aws_amis" { + default = { + # Ubuntu Server 16.04 LTS (HVM), SSD Volume Type: + "us-east-1" = "ami-40d28157" + "eu-west-2" = "ami-23d0da47" + + # Red Hat Enterprise Linux 7.3 (HVM), SSD Volume Type: + + #"us-east-1" = "ami-b63769a1" + + # CentOS 7 (x86_64) - with Updates HVM + + #"us-east-1" = "ami-6d1c2007" + } +} + +variable "aws_usernames" { + description = "User to SSH as into the AWS instance." + + default = { + "ami-40d28157" = "ubuntu" # Ubuntu Server 16.04 LTS (HVM) + "ami-b63769a1" = "ec2-user" # Red Hat Enterprise Linux 7.3 (HVM) + "ami-6d1c2007" = "centos" # CentOS 7 (x86_64) - with Updates HVM + } +} + +variable "aws_size" { + description = "AWS' selected machine size" + default = "t2.medium" # Instance with 2 cores & 4 GB memory +} diff --git a/tools/provisioning/do/README.md b/tools/provisioning/do/README.md new file mode 100755 index 0000000000..d958f18d8e --- /dev/null +++ b/tools/provisioning/do/README.md @@ -0,0 +1,98 @@ +# Digital Ocean + +## Introduction + +This project allows you to get hold of some machine on Digital Ocean. +You can then use these machines as is or run various Ansible playbooks from `../config_management` to set up Weave Net, Kubernetes, etc. + +## Setup + +* Log in [cloud.digitalocean.com](https://cloud.digitalocean.com) with your account. + +* Go to `Settings` > `Security` > `SSH keys` > `Add SSH Key`. + Enter your SSH public key and the name for it, and click `Add SSH Key`. + Set the path to your private key as an environment variable: + +``` +export DIGITALOCEAN_SSH_KEY_NAME= +export TF_VAR_do_private_key_path="$HOME/.ssh/id_rsa" +``` + +* Go to `API` > `Tokens` > `Personal access tokens` > `Generate New Token` + Enter your token name and click `Generate Token` to get your 64-characters-long API token. + Set these as environment variables: + +``` +export DIGITALOCEAN_TOKEN_NAME="" +export DIGITALOCEAN_TOKEN= +``` + +* Run the following command to get the Digital Ocean ID for your SSH public key (e.g. `1234567`) and set it as an environment variable: + +``` +$ export TF_VAR_do_public_key_id=$(curl -s -X GET -H "Content-Type: application/json" \ +-H "Authorization: Bearer $DIGITALOCEAN_TOKEN" "https://api.digitalocean.com/v2/account/keys" \ +| jq -c --arg key_name "$DIGITALOCEAN_SSH_KEY_NAME" '.ssh_keys | .[] | select(.name==$key_name) | .id') +``` + + or pass it as a Terraform variable: + +``` +$ terraform \ +-var 'do_private_key_path=' \ +-var 'do_public_key_id=' +``` + +### Bash aliases + +You can set the above variables temporarily in your current shell, permanently in your `~/.bashrc` file, or define aliases to activate/deactivate them at will with one single command by adding the below to your `~/.bashrc` file: + +``` +function _do_on() { + export DIGITALOCEAN_TOKEN_NAME="" # Replace with appropriate value. + export DIGITALOCEAN_TOKEN= # Replace with appropriate value. + export DIGITALOCEAN_SSH_KEY_NAME="" # Replace with appropriate value. + export TF_VAR_do_private_key_path="$HOME/.ssh/id_rsa" # Replace with appropriate value. + export TF_VAR_do_public_key_path="$HOME/.ssh/id_rsa.pub" # Replace with appropriate value. + export TF_VAR_do_public_key_id= # Replace with appropriate value. +} +alias _do_on='_do_on' +function _do_off() { + unset DIGITALOCEAN_TOKEN_NAME + unset DIGITALOCEAN_TOKEN + unset DIGITALOCEAN_SSH_KEY_NAME + unset TF_VAR_do_private_key_path + unset TF_VAR_do_public_key_path + unset TF_VAR_do_public_key_id +} +alias _do_off='_do_off' +``` + +N.B.: + +* sourcing `../setup.sh` defines aliases called `do_on` and `do_off`, similarly to the above (however, notice no `_` in front of the name, as opposed to the ones above); +* `../setup.sh`'s `do_on` alias needs the `SECRET_KEY` environment variable to be set in order to decrypt sensitive information. + +## Usage + +* Create the machine: `terraform apply` +* Show the machine's status: `terraform show` +* Stop and destroy the machine: `terraform destroy` +* SSH into the newly-created machine: + +``` +$ ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no `terraform output username`@`terraform output public_ips` +``` + +or + +``` +source ../setup.sh +tf_ssh 1 # Or the nth machine, if multiple VMs are provisioned. +``` + +## Resources + +* [https://www.terraform.io/docs/providers/do/](https://www.terraform.io/docs/providers/do/) +* [https://www.terraform.io/docs/providers/do/r/droplet.html](https://www.terraform.io/docs/providers/do/r/droplet.html) +* [Terraform variables](https://www.terraform.io/intro/getting-started/variables.html) diff --git a/tools/provisioning/do/main.tf b/tools/provisioning/do/main.tf new file mode 100755 index 0000000000..834cdf7d65 --- /dev/null +++ b/tools/provisioning/do/main.tf @@ -0,0 +1,42 @@ +provider "digitalocean" { + # See README.md for setup instructions. +} + +# Tags to label and organize droplets: +resource "digitalocean_tag" "name" { + name = "${var.name}" +} + +resource "digitalocean_tag" "app" { + name = "${var.app}" +} + +resource "digitalocean_tag" "terraform" { + name = "terraform" +} + +resource "digitalocean_droplet" "tf_test_vm" { + ssh_keys = ["${var.do_public_key_id}"] + image = "${var.do_os}" + region = "${var.do_dc}" + size = "${var.do_size}" + name = "${var.name}-${count.index}" + count = "${var.num_hosts}" + + tags = [ + "${var.app}", + "${var.name}", + "terraform", + ] + + # Wait for machine to be SSH-able: + provisioner "remote-exec" { + inline = ["exit"] + + connection { + type = "ssh" + user = "${var.do_username}" + private_key = "${file("${var.do_private_key_path}")}" + } + } +} diff --git a/tools/provisioning/do/outputs.tf b/tools/provisioning/do/outputs.tf new file mode 100755 index 0000000000..5f0ff455be --- /dev/null +++ b/tools/provisioning/do/outputs.tf @@ -0,0 +1,57 @@ +output "username" { + value = "${var.do_username}" +} + +output "public_ips" { + value = ["${digitalocean_droplet.tf_test_vm.*.ipv4_address}"] +} + +output "hostnames" { + value = "${join("\n", + "${formatlist("%v.%v.%v", + digitalocean_droplet.tf_test_vm.*.name, + digitalocean_droplet.tf_test_vm.*.region, + var.app + )}" + )}" +} + +# /etc/hosts file for the Droplets: +# N.B.: by default Digital Ocean droplets only have public IPs, but in order to +# be consistent with other providers' recipes, we provide an output to generate +# an /etc/hosts file on the Droplets, even though it is using public IPs only. +output "private_etc_hosts" { + value = "${join("\n", + "${formatlist("%v %v.%v.%v", + digitalocean_droplet.tf_test_vm.*.ipv4_address, + digitalocean_droplet.tf_test_vm.*.name, + digitalocean_droplet.tf_test_vm.*.region, + var.app + )}" + )}" +} + +# /etc/hosts file for the client: +output "public_etc_hosts" { + value = "${join("\n", + "${formatlist("%v %v.%v.%v", + digitalocean_droplet.tf_test_vm.*.ipv4_address, + digitalocean_droplet.tf_test_vm.*.name, + digitalocean_droplet.tf_test_vm.*.region, + var.app + )}" + )}" +} + +output "ansible_inventory" { + value = "${format("[all]\n%s", join("\n", + "${formatlist("%v private_ip=%v", + digitalocean_droplet.tf_test_vm.*.ipv4_address, + digitalocean_droplet.tf_test_vm.*.ipv4_address + )}" + ))}" +} + +output "private_key_path" { + value = "${var.do_private_key_path}" +} diff --git a/tools/provisioning/do/variables.tf b/tools/provisioning/do/variables.tf new file mode 100755 index 0000000000..6f7f40ed56 --- /dev/null +++ b/tools/provisioning/do/variables.tf @@ -0,0 +1,185 @@ +variable "client_ip" { + description = "IP address of the client machine" +} + +variable "app" { + description = "Name of the application using the created droplet(s)." + default = "default" +} + +variable "name" { + description = "Name of the droplet(s)." + default = "test" +} + +variable "num_hosts" { + description = "Number of droplet(s)." + default = 1 +} + +variable "do_private_key_path" { + description = "Digital Ocean SSH private key path" + default = "~/.ssh/id_rsa" +} + +variable "do_public_key_id" { + description = "Digital Ocean ID for your SSH public key" + + # You can retrieve it and set it as an environment variable this way: + + # $ export TF_VAR_do_public_key_id=$(curl -s -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $DIGITALOCEAN_TOKEN" "https://api.digitalocean.com/v2/account/keys" | jq -c --arg key_name "$DIGITALOCEAN_SSH_KEY_NAME" '.ssh_keys | .[] | select(.name==$key_name) | .id') +} + +variable "do_username" { + description = "Digital Ocean SSH username" + default = "root" +} + +variable "do_os" { + description = "Digital Ocean OS" + default = "ubuntu-16-04-x64" +} + +# curl -s -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $DIGITALOCEAN_TOKEN" "https://api.digitalocean.com/v2/images?page=1&per_page=999999" | jq ".images | .[] | .slug" | grep -P "ubuntu|coreos|centos" | grep -v alpha | grep -v beta +# "ubuntu-16-04-x32" +# "ubuntu-16-04-x64" +# "ubuntu-16-10-x32" +# "ubuntu-16-10-x64" +# "ubuntu-14-04-x32" +# "ubuntu-14-04-x64" +# "ubuntu-12-04-x64" +# "ubuntu-12-04-x32" +# "coreos-stable" +# "centos-6-5-x32" +# "centos-6-5-x64" +# "centos-7-0-x64" +# "centos-7-x64" +# "centos-6-x64" +# "centos-6-x32" +# "centos-5-x64" +# "centos-5-x32" + +# Digital Ocean datacenters +# See also: +# $ curl -s -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $DIGITALOCEAN_TOKEN" "https://api.digitalocean.com/v2/regions" | jq -c ".regions | .[] | .slug" | sort -u + +variable "do_dc_ams2" { + description = "Digital Ocean Amsterdam Datacenter 2" + default = "ams2" +} + +variable "do_dc_ams3" { + description = "Digital Ocean Amsterdam Datacenter 3" + default = "ams3" +} + +variable "do_dc_blr1" { + description = "Digital Ocean Bangalore Datacenter 1" + default = "blr1" +} + +variable "do_dc_fra1" { + description = "Digital Ocean Frankfurt Datacenter 1" + default = "fra1" +} + +variable "do_dc_lon1" { + description = "Digital Ocean London Datacenter 1" + default = "lon1" +} + +variable "do_dc_nyc1" { + description = "Digital Ocean New York Datacenter 1" + default = "nyc1" +} + +variable "do_dc_nyc2" { + description = "Digital Ocean New York Datacenter 2" + default = "nyc2" +} + +variable "do_dc_nyc3" { + description = "Digital Ocean New York Datacenter 3" + default = "nyc3" +} + +variable "do_dc_sfo1" { + description = "Digital Ocean San Francisco Datacenter 1" + default = "sfo1" +} + +variable "do_dc_sfo2" { + description = "Digital Ocean San Francisco Datacenter 2" + default = "sfo2" +} + +variable "do_dc_sgp1" { + description = "Digital Ocean Singapore Datacenter 1" + default = "sgp1" +} + +variable "do_dc_tor1" { + description = "Digital Ocean Toronto Datacenter 1" + default = "tor1" +} + +variable "do_dc" { + description = "Digital Ocean's selected datacenter" + default = "lon1" +} + +variable "do_size" { + description = "Digital Ocean's selected machine size" + default = "4gb" +} + +# Digital Ocean sizes + + +# See also: + + +# $ curl -s -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $DIGITALOCEAN_TOKEN" "https://api.digitalocean.com/v2/sizes" | jq -c ".sizes | .[] | .slug" + + +# "512mb" + + +# "1gb" + + +# "2gb" + + +# "4gb" + + +# "8gb" + + +# "16gb" + + +# "m-16gb" + + +# "32gb" + + +# "m-32gb" + + +# "48gb" + + +# "m-64gb" + + +# "64gb" + + +# "m-128gb" + + +# "m-224gb" + diff --git a/tools/provisioning/gcp/README.md b/tools/provisioning/gcp/README.md new file mode 100755 index 0000000000..b2d6622c81 --- /dev/null +++ b/tools/provisioning/gcp/README.md @@ -0,0 +1,126 @@ +# Google Cloud Platform + +## Introduction + +This project allows you to get hold of some machine on Google Cloud Platform. +You can then use these machines as is or run various Ansible playbooks from `../config_management` to set up Weave Net, Kubernetes, etc. + +## Setup + +* Log in [console.cloud.google.com](https://console.cloud.google.com) with your Google account. + +* Go to `API Manager` > `Credentials` > `Create credentials` > `Service account key`, + in `Service account`, select `Compute Engine default service account`, + in `Key type`, select `JSON`, and then click `Create`. + +* This will download a JSON file to your machine. Place this file wherever you want and then create the following environment variables: + +``` +$ export GOOGLE_CREDENTIALS_FILE="path/to/your.json" +$ export GOOGLE_CREDENTIALS=$(cat "$GOOGLE_CREDENTIALS_FILE") +``` + +* Go to `Compute Engine` > `Metadata` > `SSH keys` and add your username and SSH public key; + or + set it up using `gcloud compute project-info add-metadata --metadata-from-file sshKeys=~/.ssh/id_rsa.pub`. + If you used your default SSH key (i.e. `~/.ssh/id_rsa.pub`), then you do not have anything to do. + Otherwise, you will have to either define the below environment variable: + +``` +$ export TF_VAR_gcp_public_key_path= +$ export TF_VAR_gcp_private_key_path= +``` + + or to pass these as Terraform variables: + +``` +$ terraform \ +-var 'gcp_public_key_path=' \ +-var 'gcp_private_key_path=' +``` + +* Set the username in your public key as an environment variable. + This will be used as the username of the Linux account created on the machine, which you will need to SSH into it later on. + + N.B.: + * GCP already has the username set from the SSH public key you uploaded in the previous step. + * If your username is an email address, e.g. `name@domain.com`, then GCP uses `name` as the username. + +``` +export TF_VAR_gcp_username= +``` + +* Set your current IP address as an environment variable: + +``` +export TF_VAR_client_ip=$(curl -s -X GET http://checkip.amazonaws.com/) +``` + + or pass it as a Terraform variable: + +``` +$ terraform -var 'client_ip=$(curl -s -X GET http://checkip.amazonaws.com/)' +``` + +* Set your project as an environment variable: + +``` +export TF_VAR_gcp_project=weave-net-tests +``` + + or pass it as a Terraform variable: + +``` +$ terraform -var 'gcp_project=weave-net-tests' +``` + +### Bash aliases + +You can set the above variables temporarily in your current shell, permanently in your `~/.bashrc` file, or define aliases to activate/deactivate them at will with one single command by adding the below to your `~/.bashrc` file: + +``` +function _gcp_on() { + export GOOGLE_CREDENTIALS_FILE="&authuser=1 + region = "${var.gcp_region}" + + project = "${var.gcp_project}" +} + +resource "google_compute_instance" "tf_test_vm" { + name = "${var.name}-${count.index}" + machine_type = "${var.gcp_size}" + zone = "${var.gcp_zone}" + count = "${var.num_hosts}" + + disk { + image = "${var.gcp_image}" + } + + tags = [ + "${var.app}", + "${var.name}", + "terraform", + ] + + network_interface { + network = "${var.gcp_network}" + + access_config { + // Ephemeral IP + } + } + + metadata { + ssh-keys = "${var.gcp_username}:${file("${var.gcp_public_key_path}")}" + } + + # Wait for machine to be SSH-able: + provisioner "remote-exec" { + inline = ["exit"] + + connection { + type = "ssh" + user = "${var.gcp_username}" + private_key = "${file("${var.gcp_private_key_path}")}" + } + } +} + +resource "google_compute_firewall" "fw-allow-docker-and-weave" { + name = "${var.name}-allow-docker-and-weave" + network = "${var.gcp_network}" + target_tags = ["${var.name}"] + + allow { + protocol = "tcp" + ports = ["2375", "12375"] + } + + source_ranges = ["${var.client_ip}"] +} + +# Required for FastDP crypto in Weave Net: +resource "google_compute_firewall" "fw-allow-esp" { + name = "${var.name}-allow-esp" + network = "${var.gcp_network}" + target_tags = ["${var.name}"] + + allow { + protocol = "esp" + } + + source_ranges = ["${var.gcp_network_global_cidr}"] +} diff --git a/tools/provisioning/gcp/outputs.tf b/tools/provisioning/gcp/outputs.tf new file mode 100755 index 0000000000..9aa1e33e84 --- /dev/null +++ b/tools/provisioning/gcp/outputs.tf @@ -0,0 +1,66 @@ +output "username" { + value = "${var.gcp_username}" +} + +output "public_ips" { + value = ["${google_compute_instance.tf_test_vm.*.network_interface.0.access_config.0.assigned_nat_ip}"] +} + +output "hostnames" { + value = "${join("\n", + "${formatlist("%v.%v.%v", + google_compute_instance.tf_test_vm.*.name, + google_compute_instance.tf_test_vm.*.zone, + var.app + )}" + )}" +} + +# /etc/hosts file for the Compute Engine instances: +output "private_etc_hosts" { + value = "${join("\n", + "${formatlist("%v %v.%v.%v", + google_compute_instance.tf_test_vm.*.network_interface.0.address, + google_compute_instance.tf_test_vm.*.name, + google_compute_instance.tf_test_vm.*.zone, + var.app + )}" + )}" +} + +# /etc/hosts file for the client: +output "public_etc_hosts" { + value = "${join("\n", + "${formatlist("%v %v.%v.%v", + google_compute_instance.tf_test_vm.*.network_interface.0.access_config.0.assigned_nat_ip, + google_compute_instance.tf_test_vm.*.name, + google_compute_instance.tf_test_vm.*.zone, + var.app + )}" + )}" +} + +output "ansible_inventory" { + value = "${format("[all]\n%s", join("\n", + "${formatlist("%v private_ip=%v", + google_compute_instance.tf_test_vm.*.network_interface.0.access_config.0.assigned_nat_ip, + google_compute_instance.tf_test_vm.*.network_interface.0.address + )}" + ))}" +} + +output "private_key_path" { + value = "${var.gcp_private_key_path}" +} + +output "instances_names" { + value = ["${google_compute_instance.tf_test_vm.*.name}"] +} + +output "image" { + value = "${var.gcp_image}" +} + +output "zone" { + value = "${var.gcp_zone}" +} diff --git a/tools/provisioning/gcp/variables.tf b/tools/provisioning/gcp/variables.tf new file mode 100755 index 0000000000..6b2027b27e --- /dev/null +++ b/tools/provisioning/gcp/variables.tf @@ -0,0 +1,77 @@ +variable "gcp_username" { + description = "Google Cloud Platform SSH username" +} + +variable "app" { + description = "Name of the application using the created Compute Engine instance(s)." + default = "default" +} + +variable "name" { + description = "Name of the Compute Engine instance(s)." + default = "test" +} + +variable "num_hosts" { + description = "Number of Compute Engine instance(s)." + default = 1 +} + +variable "client_ip" { + description = "IP address of the client machine" +} + +variable "gcp_public_key_path" { + description = "Path to file containing public key" + default = "~/.ssh/id_rsa.pub" +} + +variable "gcp_private_key_path" { + description = "Path to file containing private key" + default = "~/.ssh/id_rsa" +} + +variable "gcp_project" { + description = "Google Cloud Platform project" + default = "weave-net-tests" +} + +variable "gcp_image" { + # See also: https://cloud.google.com/compute/docs/images + # For example: + # - "ubuntu-os-cloud/ubuntu-1604-lts" + # - "debian-cloud/debian-8" + # - "centos-cloud/centos-7" + # - "rhel-cloud/rhel7" + description = "Google Cloud Platform OS" + + default = "ubuntu-os-cloud/ubuntu-1604-lts" +} + +variable "gcp_size" { + # See also: + # $ gcloud compute machine-types list + description = "Google Cloud Platform's selected machine size" + + default = "n1-standard-1" +} + +variable "gcp_region" { + description = "Google Cloud Platform's selected region" + default = "us-central1" +} + +variable "gcp_zone" { + description = "Google Cloud Platform's selected zone" + default = "us-central1-a" +} + +variable "gcp_network" { + description = "Google Cloud Platform's selected network" + default = "test" +} + +variable "gcp_network_global_cidr" { + description = "CIDR covering all regions for the selected Google Cloud Platform network" + default = "10.128.0.0/9" +} diff --git a/tools/provisioning/setup.sh b/tools/provisioning/setup.sh new file mode 100755 index 0000000000..456878e0e2 --- /dev/null +++ b/tools/provisioning/setup.sh @@ -0,0 +1,361 @@ +#!/bin/bash +# +# Description: +# Helper functions to programmatically provision (e.g. for CIT). +# Aliases on these functions are also created so that this script can be +# sourced in your shell, in your ~/.bashrc file, etc. and directly called. +# +# Usage: +# Source this file and call the relevant functions. +# + +function ssh_public_key() { + echo -e "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDZBgLQts30PYXEMJnCU21QC+1ZE0Sv/Ry48Au3nYXn1KNoW/7C2qQ3KO2ZnpZRHCstFiU8QIlB9edi0cgcAoDWBkCiFBZEORxMvohWtrRQzf+x59o48lVjA/Fn7G+9hmavhLaDf6Qe7OhH8XUshNtnIQIUvNEWXKE75k32wUbuF8ibhJNpOOYKL4tVXK6IIKg6jR88BwGKPY/NZCl/HbhjnDJY0zCU1pZSprN6o/S953y/XXVozkh1772fCNeu4USfbt0oZOEJ57j6EWwEYIJhoeAEMAoD8ELt/bc/5iex8cuarM4Uib2JHO6WPWbBQ0NlrARIOKLrxkjjfGWarOLWBAgvwQn5zLg1pKb7aI4+jbA+ZSrII5B2HuYE9MDlU8NPL4pHrRfapGLkG/Fe9zNPvScXh+9iSWfD6G5ZoISutjiJO/iVYN0QSuj9QEIj9tl20czFz3Dhnq4sPPl5hoLunyQfajY7C/ipv6ilJyrEc0V6Z9FdPhpEI+HOgJr2vDQTFscQuyfWuzGJDZf6zPdZWo2pBql9E7piARuNAjakylGar/ebkCgfy28XQoDbDT0P0VYp+E8W5EYacx+zc5MuNhRTvbsO12fydT8V61MtA78wM/b0059feph+0zTykEHk670mYVoE3erZX+U1/BVBLSV9QzopO6/Pgx2ryriJfQ== weaveworks-cit" +} + +function decrypt() { + if [ -z "$1" ]; then + echo >&2 "Failed to decode and decrypt $2: no secret key was provided." + return 1 + fi + echo "$3" | openssl base64 -d | openssl enc -d -aes256 -pass "pass:$1" +} + +function ssh_private_key() { + # The private key has been AES256-encrypted and then Base64-encoded using the following command: + # $ openssl enc -in /tmp/weaveworks_cit_id_rsa -e -aes256 -pass stdin | openssl base64 > /tmp/weaveworks_cit_id_rsa.aes.b64 + # The below command does the reverse, i.e. base64-decode and AES-decrypt the file, and prints it to stdout. + # N.B.: Ask the password to Marc, or otherwise re-generate the SSH key using: + # $ ssh-keygen -t rsa -b 4096 -C "weaveworks-cit" + decrypt "$1" "SSH private key" "$( + cat <&2 "Failed to decode and decrypt SSH private key: no secret key was provided." + return 1 + fi + local ssh_private_key_path="$HOME/.ssh/weaveworks_cit_id_rsa" + [ -e "$ssh_private_key_path" ] && rm -f "$ssh_private_key_path" + ssh_private_key "$1" >"$ssh_private_key_path" + chmod 400 "$ssh_private_key_path" + echo "$ssh_private_key_path" +} + +function gcp_credentials() { + # The below GCP service account JSON credentials have been AES256-encrypted and then Base64-encoded using the following command: + # $ openssl enc -in ~/.ssh/weaveworks-cit.json -e -aes256 -pass stdin | openssl base64 > /tmp/weaveworks-cit.json.aes.b64 + # The below command does the reverse, i.e. base64-decode and AES-decrypt the file, and prints it to stdout. + # N.B.: Ask the password to Marc, or otherwise re-generate the credentials for GCP, as per ../tools/provisioning/gcp/README.md. + decrypt "$1" "JSON credentials" "$( + cat <&2 "Failed to configure for Digital Ocean: no value for the SECRET_KEY environment variable." + return 1 + fi + + # SSH public key: + export TF_VAR_do_public_key_path="$HOME/.ssh/weaveworks_cit_id_rsa.pub" + ssh_public_key >"$TF_VAR_do_public_key_path" + export DIGITALOCEAN_SSH_KEY_NAME="weaveworks-cit" + export TF_VAR_do_public_key_id=5228799 + + # SSH private key: + export TF_VAR_do_private_key_path=$(set_up_ssh_private_key "$SECRET_KEY") + + # API token: + # The below Digital Ocean token has been AES256-encrypted and then Base64-encoded using the following command: + # $ openssl enc -in /tmp/digital_ocean_token.txt -e -aes256 -pass stdin | openssl base64 > /tmp/digital_ocean_token.txt.aes.b64 + # The below command does the reverse, i.e. base64-decode and AES-decrypt the file, and prints it to stdout. + # N.B.: Ask the password to Marc, or otherwise re-generate the token for Digital Ocean, as per ../tools/provisioning/do/README.md. + export DIGITALOCEAN_TOKEN=$(decrypt "$SECRET_KEY" "Digital Ocean token" "U2FsdGVkX1/Gq5Rj9dDDraME8xK30JOyJ9dhfQzPBaaePJHqDPIG6of71DdJW0UyFUyRtbRflCPaZ8Um1pDJpU5LoNWQk4uCApC8+xciltT73uQtttLBG8FqgFBvYIHS") + export DIGITALOCEAN_TOKEN_NAME="weaveworks-cit" + export TF_VAR_client_ip=$(curl -s -X GET http://checkip.amazonaws.com/) +} +alias do_on='do_on' + +function do_off() { + unset TF_VAR_do_public_key_path + unset DIGITALOCEAN_SSH_KEY_NAME + unset TF_VAR_do_public_key_id + unset TF_VAR_do_private_key_path + unset DIGITALOCEAN_TOKEN + unset DIGITALOCEAN_TOKEN_NAME + unset TF_VAR_client_ip +} +alias do_off='do_off' + +# shellcheck disable=2155 +function gcp_on() { + # Set up everything required to run tests on GCP. + # Steps from ../tools/provisioning/gcp/README.md have been followed. + # All sensitive files have been encrypted, see respective functions. + if [ -z "$SECRET_KEY" ]; then + echo >&2 "Failed to configure for Google Cloud Platform: no value for the SECRET_KEY environment variable." + return 1 + fi + + # SSH public key and SSH username: + export TF_VAR_gcp_public_key_path="$HOME/.ssh/weaveworks_cit_id_rsa.pub" + ssh_public_key >"$TF_VAR_gcp_public_key_path" + export TF_VAR_gcp_username=$(cut -d' ' -f3 "$TF_VAR_gcp_public_key_path" | cut -d'@' -f1) + + # SSH private key: + export TF_VAR_gcp_private_key_path=$(set_up_ssh_private_key "$SECRET_KEY") + + # JSON credentials: + export GOOGLE_CREDENTIALS_FILE="$HOME/.ssh/weaveworks-cit.json" + [ -e "$GOOGLE_CREDENTIALS_FILE" ] && rm -f "$GOOGLE_CREDENTIALS_FILE" + gcp_credentials "$SECRET_KEY" >"$GOOGLE_CREDENTIALS_FILE" + chmod 400 "$GOOGLE_CREDENTIALS_FILE" + export GOOGLE_CREDENTIALS=$(cat "$GOOGLE_CREDENTIALS_FILE") + + export TF_VAR_client_ip=$(curl -s -X GET http://checkip.amazonaws.com/) + export TF_VAR_gcp_project="${PROJECT:-"weave-net-tests"}" + # shellcheck disable=2015 + [ -z "$PROJECT" ] && echo >&2 "WARNING: no value provided for PROJECT environment variable: defaulted it to $TF_VAR_gcp_project." || true +} +alias gcp_on='gcp_on' + +function gcp_off() { + unset TF_VAR_gcp_public_key_path + unset TF_VAR_gcp_username + unset TF_VAR_gcp_private_key_path + unset GOOGLE_CREDENTIALS_FILE + unset GOOGLE_CREDENTIALS + unset TF_VAR_client_ip + unset TF_VAR_gcp_project +} +alias gcp_off='gcp_off' + +# shellcheck disable=2155 +function aws_on() { + # Set up everything required to run tests on Amazon Web Services. + # Steps from ../tools/provisioning/aws/README.md have been followed. + # All sensitive files have been encrypted, see respective functions. + if [ -z "$SECRET_KEY" ]; then + echo >&2 "Failed to configure for Amazon Web Services: no value for the SECRET_KEY environment variable." + return 1 + fi + + # SSH public key: + export TF_VAR_aws_public_key_name="weaveworks_cit_id_rsa" + + # SSH private key: + export TF_VAR_aws_private_key_path=$(set_up_ssh_private_key "$SECRET_KEY") + + # The below AWS access key ID and secret access key have been AES256-encrypted and then Base64-encoded using the following commands: + # $ openssl enc -in /tmp/aws_access_key_id.txt -e -aes256 -pass stdin | openssl base64 > /tmp/aws_access_key_id.txt.aes.b64 + # $ openssl enc -in /tmp/aws_secret_access_key.txt -e -aes256 -pass stdin | openssl base64 > /tmp/aws_secret_access_key.txt.aes.b64 + # The below commands do the reverse, i.e. base64-decode and AES-decrypt the encrypted and encoded strings, and print it to stdout. + # N.B.: Ask the password to Marc, or otherwise re-generate the AWS access key ID and secret access key, as per ../tools/provisioning/aws/README.md. + export AWS_ACCESS_KEY_ID="$(decrypt "$SECRET_KEY" "AWS access key ID" "U2FsdGVkX18Txjm2PWSlJsToYm1vv4dMTtVLkRNiQbrC6Y6GuIHb1ao5MmGPJ1wf")" + export AWS_SECRET_ACCESS_KEY="$(decrypt "$SECRET_KEY" "AWS secret access key" "$( + cat <&2 <<-EOF +ERROR: $1 + +Usage: + \$ tf_ssh [OPTION]... +Examples: + \$ tf_ssh 1 + \$ tf_ssh 1 -o LogLevel VERBOSE + \$ tf_ssh 1 -i ~/.ssh/custom_private_key_id_rsa +Available machines: +EOF + cat -n >&2 <<<"$(terraform output public_etc_hosts)" +} + +# shellcheck disable=SC2155 +function tf_ssh() { + [ -z "$1" ] && tf_ssh_usage "No host ID provided." && return 1 + local ip="$(sed "$1q;d" <<<"$(terraform output public_etc_hosts)" | cut -d ' ' -f 1)" + shift # Drop the first argument, corresponding to the machine ID, to allow passing other arguments to SSH using "$@" -- see below. + [ -z "$ip" ] && tf_ssh_usage "Invalid host ID provided." && return 1 + # shellcheck disable=SC2029 + ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no "$@" "$(terraform output username)@$ip" +} +alias tf_ssh='tf_ssh' + +function tf_ansi_usage() { + cat >&2 <<-EOF +ERROR: $1 + +Usage: + \$ tf_ansi [OPTION]... +Examples: + \$ tf_ansi setup_weave-net_dev + \$ tf_ansi 1 + \$ tf_ansi 1 -vvv --private-key=~/.ssh/custom_private_key_id_rsa + \$ tf_ansi setup_weave-kube --extra-vars "docker_version=1.12.6 kubernetes_version=1.5.6" +Available playbooks: +EOF + cat -n >&2 <<<"$(for file in "$(dirname "${BASH_SOURCE[0]}")"/../../config_management/*.yml; do basename "$file" | sed 's/.yml//'; done)" +} + +# shellcheck disable=SC2155,SC2064 +function tf_ansi() { + [ -z "$1" ] && tf_ansi_usage "No Ansible playbook provided." && return 1 + local id="$1" + shift # Drop the first argument to allow passing other arguments to Ansible using "$@" -- see below. + if [[ "$id" =~ ^[0-9]+$ ]]; then + local playbooks=(../../config_management/*.yml) + local path="${playbooks[(($id - 1))]}" # Select the ith entry in the list of playbooks (0-based). + else + local path="$(dirname "${BASH_SOURCE[0]}")/../../config_management/$id.yml" + fi + local inventory="$(mktemp /tmp/ansible_inventory_XXX)" + trap 'rm -f $inventory' SIGINT SIGTERM RETURN + echo -e "$(terraform output ansible_inventory)" >"$inventory" + [ ! -r "$path" ] && tf_ansi_usage "Ansible playbook not found: $path" && return 1 + ansible-playbook "$@" -u "$(terraform output username)" -i "$inventory" --ssh-extra-args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" "$path" +} +alias tf_ansi='tf_ansi' diff --git a/tools/push-images b/tools/push-images new file mode 100755 index 0000000000..60edac7c47 --- /dev/null +++ b/tools/push-images @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +QUAY_PREFIX=quay.io/ +IMAGES=$(make images) +IMAGE_TAG=$(./tools/image-tag) + +usage() { + echo "$0 [-no-docker-hub]" +} + +NO_DOCKER_HUB= +while [ $# -gt 0 ]; do + case "$1" in + -no-docker-hub) + NO_DOCKER_HUB=1 + shift 1 + ;; + *) + usage + exit 2 + ;; + esac +done + +pids="" +for image in ${IMAGES}; do + if [[ "$image" == *"build"* ]]; then + continue + fi + echo "Will push ${image}:${IMAGE_TAG}" + docker push "${image}:${IMAGE_TAG}" & + pids="$pids $!" + + if [ -z "$NO_DOCKER_HUB" ]; then + # remove the quey prefix and push to docker hub + docker_hub_image=${image#$QUAY_PREFIX} + docker tag "${image}:${IMAGE_TAG}" "${docker_hub_image}:${IMAGE_TAG}" + echo "Will push ${docker_hub_image}:${IMAGE_TAG}" + docker push "${docker_hub_image}:${IMAGE_TAG}" & + pids="$pids $!" + fi +done + +# Wait individually for tasks so we fail-exit on any non-zero return code +for p in $pids; do + wait $p +done + +wait diff --git a/tools/runner/runner.go b/tools/runner/runner.go index 42f10b1819..38e5a62c98 100644 --- a/tools/runner/runner.go +++ b/tools/runner/runner.go @@ -148,9 +148,10 @@ func updateScheduler(test string, duration float64) { func getSchedule(tests []string) ([]string, error) { var ( + userName = os.Getenv("CIRCLE_PROJECT_USERNAME") project = os.Getenv("CIRCLE_PROJECT_REPONAME") buildNum = os.Getenv("CIRCLE_BUILD_NUM") - testRun = project + "-integration-" + buildNum + testRun = userName + "-" + project + "-integration-" + buildNum shardCount = os.Getenv("CIRCLE_NODE_TOTAL") shardID = os.Getenv("CIRCLE_NODE_INDEX") requestBody = &bytes.Buffer{} diff --git a/tools/sched b/tools/sched index cf47773e5e..a282558f11 100755 --- a/tools/sched +++ b/tools/sched @@ -1,20 +1,20 @@ -#!/usr/bin/python -import sys, string, json, urllib +#!/usr/bin/env python +import sys, string, urllib import requests import optparse def test_time(target, test_name, runtime): r = requests.post(target + "/record/%s/%f" % (urllib.quote(test_name, safe=""), runtime)) - print r.text + print r.text.encode('utf-8') assert r.status_code == 204 def test_sched(target, test_run, shard_count, shard_id): - tests = json.dumps({'tests': string.split(sys.stdin.read())}) - r = requests.post(target + "/schedule/%s/%d/%d" % (test_run, shard_count, shard_id), data=tests) + tests = {'tests': string.split(sys.stdin.read())} + r = requests.post(target + "/schedule/%s/%d/%d" % (test_run, shard_count, shard_id), json=tests) assert r.status_code == 200 result = r.json() for test in sorted(result['tests']): - print test + print test.encode('utf-8') def usage(): print "%s (--target=...) " % sys.argv[0] diff --git a/tools/scheduler/main.py b/tools/scheduler/main.py index 4ed887563a..3b540b54a3 100644 --- a/tools/scheduler/main.py +++ b/tools/scheduler/main.py @@ -19,157 +19,188 @@ # observations faster. alpha = 0.3 + class Test(ndb.Model): - total_run_time = ndb.FloatProperty(default=0.) # Not total, but a EWMA - total_runs = ndb.IntegerProperty(default=0) - - def parallelism(self): - name = self.key.string_id() - m = re.search('(\d+)_test.sh$', name) - if m is None: - return 1 - else: - return int(m.group(1)) - - def cost(self): - p = self.parallelism() - logging.info("Test %s has parallelism %d and avg run time %s", self.key.string_id(), p, self.total_run_time) - return self.parallelism() * self.total_run_time + total_run_time = ndb.FloatProperty(default=0.) # Not total, but a EWMA + total_runs = ndb.IntegerProperty(default=0) + + def parallelism(self): + name = self.key.string_id() + m = re.search('(\d+)_test.sh$', name) + if m is None: + return 1 + else: + return int(m.group(1)) + + def cost(self): + p = self.parallelism() + logging.info("Test %s has parallelism %d and avg run time %s", + self.key.string_id(), p, self.total_run_time) + return self.parallelism() * self.total_run_time + class Schedule(ndb.Model): - shards = ndb.JsonProperty() + shards = ndb.JsonProperty() + @app.route('/record//', methods=['POST']) @ndb.transactional def record(test_name, runtime): - test = Test.get_by_id(test_name) - if test is None: - test = Test(id=test_name) - test.total_run_time = (test.total_run_time * (1-alpha)) + (float(runtime) * alpha) - test.total_runs += 1 - test.put() - return ('', 204) - -@app.route('/schedule///', methods=['POST']) + test = Test.get_by_id(test_name) + if test is None: + test = Test(id=test_name) + test.total_run_time = (test.total_run_time * + (1 - alpha)) + (float(runtime) * alpha) + test.total_runs += 1 + test.put() + return ('', 204) + + +@app.route( + '/schedule///', methods=['POST']) def schedule(test_run, shard_count, shard): - # read tests from body - test_names = flask.request.get_json(force=True)['tests'] - - # first see if we have a scedule already - schedule_id = "%s-%d" % (test_run, shard_count) - schedule = Schedule.get_by_id(schedule_id) - if schedule is not None: + # read tests from body + test_names = flask.request.get_json(force=True)['tests'] + + # first see if we have a scedule already + schedule_id = "%s-%d" % (test_run, shard_count) + schedule = Schedule.get_by_id(schedule_id) + if schedule is not None: + return flask.json.jsonify(tests=schedule.shards[str(shard)]) + + # if not, do simple greedy algorithm + test_times = ndb.get_multi( + ndb.Key(Test, test_name) for test_name in test_names) + + def avg(test): + if test is not None: + return test.cost() + return 1 + + test_times = [(test_name, avg(test)) + for test_name, test in zip(test_names, test_times)] + test_times_dict = dict(test_times) + test_times.sort(key=operator.itemgetter(1)) + + shards = {i: [] for i in xrange(shard_count)} + while test_times: + test_name, time = test_times.pop() + + # find shortest shard and put it in that + s, _ = min( + ((i, sum(test_times_dict[t] for t in shards[i])) + for i in xrange(shard_count)), + key=operator.itemgetter(1)) + + shards[s].append(test_name) + + # atomically insert or retrieve existing schedule + schedule = Schedule.get_or_insert(schedule_id, shards=shards) return flask.json.jsonify(tests=schedule.shards[str(shard)]) - # if not, do simple greedy algorithm - test_times = ndb.get_multi(ndb.Key(Test, test_name) for test_name in test_names) - def avg(test): - if test is not None: - return test.cost() - return 1 - test_times = [(test_name, avg(test)) for test_name, test in zip(test_names, test_times)] - test_times_dict = dict(test_times) - test_times.sort(key=operator.itemgetter(1)) - - shards = {i: [] for i in xrange(shard_count)} - while test_times: - test_name, time = test_times.pop() - - # find shortest shard and put it in that - s, _ = min(((i, sum(test_times_dict[t] for t in shards[i])) - for i in xrange(shard_count)), key=operator.itemgetter(1)) - - shards[s].append(test_name) - - # atomically insert or retrieve existing schedule - schedule = Schedule.get_or_insert(schedule_id, shards=shards) - return flask.json.jsonify(tests=schedule.shards[str(shard)]) FIREWALL_REGEXES = [ - re.compile(r'^(?P\w+)-allow-(?P\w+)-(?P\d+)-(?P\d+)$'), - re.compile(r'^(?P\w+)-(?P\d+)-(?P\d+)-allow-(?P[\w\-]+)$'), + re.compile( + r'^(?P\w+)-allow-(?P\w+)-(?P\d+)-(?P\d+)$' + ), + re.compile(r'^(?P\w+)-(?P\d+)-(?P\d+)-allow-' + r'(?P[\w\-]+)$'), ] NAME_REGEXES = [ - re.compile(r'^host(?P\d+)-(?P\d+)-(?P\d+)$'), - re.compile(r'^test-(?P\d+)-(?P\d+)-(?P\d+)$'), + re.compile(r'^host(?P\d+)-(?P\d+)-(?P\d+)$'), + re.compile(r'^test-(?P\d+)-(?P\d+)-(?P\d+)$'), ] + def _matches_any_regex(name, regexes): - for regex in regexes: - matches = regex.match(name) - if matches: - return matches + for regex in regexes: + matches = regex.match(name) + if matches: + return matches + PROJECTS = [ - ('weaveworks/weave', 'weave-net-tests', 'us-central1-a', True), - ('weaveworks/weave', 'positive-cocoa-90213', 'us-central1-a', True), - ('weaveworks/scope', 'scope-integration-tests', 'us-central1-a', False), + ('weaveworks/weave', 'weave-net-tests', 'us-central1-a', True), + ('weaveworks/weave', 'positive-cocoa-90213', 'us-central1-a', True), + ('weaveworks/scope', 'scope-integration-tests', 'us-central1-a', False), ] + @app.route('/tasks/gc') def gc(): - # Get list of running VMs, pick build id out of VM name - credentials = GoogleCredentials.get_application_default() - compute = discovery.build('compute', 'v1', credentials=credentials) + # Get list of running VMs, pick build id out of VM name + credentials = GoogleCredentials.get_application_default() + compute = discovery.build('compute', 'v1', credentials=credentials) + + for repo, project, zone, gc_fw in PROJECTS: + gc_project(compute, repo, project, zone, gc_fw) - for repo, project, zone, gc_fw in PROJECTS: - gc_project(compute, repo, project, zone, gc_fw) + return "Done" - return "Done" def gc_project(compute, repo, project, zone, gc_fw): - logging.info("GCing %s, %s, %s", repo, project, zone) - # Get list of builds, filter down to running builds: - running = _get_running_builds(repo) - # Stop VMs for builds that aren't running: - _gc_compute_engine_instances(compute, project, zone, running) - # Remove firewall rules for builds that aren't running: - if gc_fw: - _gc_firewall_rules(compute, project, running) + logging.info("GCing %s, %s, %s", repo, project, zone) + # Get list of builds, filter down to running builds: + running = _get_running_builds(repo) + # Stop VMs for builds that aren't running: + _gc_compute_engine_instances(compute, project, zone, running) + # Remove firewall rules for builds that aren't running: + if gc_fw: + _gc_firewall_rules(compute, project, running) + def _get_running_builds(repo): - result = urlfetch.fetch('https://circleci.com/api/v1/project/%s' % repo, - headers={'Accept': 'application/json'}) - assert result.status_code == 200 - builds = json.loads(result.content) - running = {build['build_num'] for build in builds if not build.get('stop_time')} - logging.info("Runnings builds: %r", running) - return running + result = urlfetch.fetch( + 'https://circleci.com/api/v1/project/%s' % repo, + headers={'Accept': 'application/json'}) + assert result.status_code == 200 + builds = json.loads(result.content) + running = { + build['build_num'] + for build in builds if not build.get('stop_time') + } + logging.info("Runnings builds: %r", running) + return running + def _get_hosts_by_build(instances): - host_by_build = collections.defaultdict(list) - for instance in instances['items']: - matches = _matches_any_regex(instance['name'], NAME_REGEXES) - if not matches: - continue - host_by_build[int(matches.group('build'))].append(instance['name']) - logging.info("Running VMs by build: %r", host_by_build) - return host_by_build + host_by_build = collections.defaultdict(list) + for instance in instances['items']: + matches = _matches_any_regex(instance['name'], NAME_REGEXES) + if not matches: + continue + host_by_build[int(matches.group('build'))].append(instance['name']) + logging.info("Running VMs by build: %r", host_by_build) + return host_by_build + def _gc_compute_engine_instances(compute, project, zone, running): - instances = compute.instances().list(project=project, zone=zone).execute() - if 'items' not in instances: - return - host_by_build = _get_hosts_by_build(instances) - stopped = [] - for build, names in host_by_build.iteritems(): - if build in running: - continue - for name in names: - stopped.append(name) - logging.info("Stopping VM %s", name) - compute.instances().delete(project=project, zone=zone, instance=name).execute() - return stopped + instances = compute.instances().list(project=project, zone=zone).execute() + if 'items' not in instances: + return + host_by_build = _get_hosts_by_build(instances) + stopped = [] + for build, names in host_by_build.iteritems(): + if build in running: + continue + for name in names: + stopped.append(name) + logging.info("Stopping VM %s", name) + compute.instances().delete( + project=project, zone=zone, instance=name).execute() + return stopped + def _gc_firewall_rules(compute, project, running): - firewalls = compute.firewalls().list(project=project).execute() - if 'items' not in firewalls: - return - for firewall in firewalls['items']: - matches = _matches_any_regex(firewall['name'], FIREWALL_REGEXES) - if not matches: - continue - if int(matches.group('build')) in running: - continue - logging.info("Deleting firewall rule %s", firewall['name']) - compute.firewalls().delete(project=project, firewall=firewall['name']).execute() + firewalls = compute.firewalls().list(project=project).execute() + if 'items' not in firewalls: + return + for firewall in firewalls['items']: + matches = _matches_any_regex(firewall['name'], FIREWALL_REGEXES) + if not matches: + continue + if int(matches.group('build')) in running: + continue + logging.info("Deleting firewall rule %s", firewall['name']) + compute.firewalls().delete( + project=project, firewall=firewall['name']).execute() diff --git a/tools/test b/tools/test index b55de73388..c87bdd0739 100755 --- a/tools/test +++ b/tools/test @@ -7,6 +7,7 @@ SLOW= NO_GO_GET=true TAGS= PARALLEL= +RACE="-race -covermode=atomic" TIMEOUT=1m usage() { @@ -19,6 +20,10 @@ while [ $# -gt 0 ]; do SLOW=true shift 1 ;; + "-no-race") + RACE= + shift 1 + ;; "-no-go-get") NO_GO_GET=true shift 1 @@ -28,9 +33,13 @@ while [ $# -gt 0 ]; do shift 1 ;; "-netgo") - TAGS="-tags netgo" + TAGS="netgo" shift 1 ;; + "-tags") + TAGS="$2" + shift 2 + ;; "-p") PARALLEL=true shift 1 @@ -46,14 +55,14 @@ while [ $# -gt 0 ]; do esac done -GO_TEST_ARGS=($TAGS -cpu 4 -timeout $TIMEOUT) +GO_TEST_ARGS=(-tags "${TAGS[@]}" -cpu 4 -timeout $TIMEOUT) if [ -n "$SLOW" ] || [ -n "$CIRCLECI" ]; then SLOW=true fi if [ -n "$SLOW" ]; then - GO_TEST_ARGS=("${GO_TEST_ARGS[@]}" -race -covermode=atomic) + GO_TEST_ARGS=("${GO_TEST_ARGS[@]}" ${RACE}) # shellcheck disable=SC2153 if [ -n "$COVERDIR" ]; then @@ -69,7 +78,7 @@ fail=0 if [ -z "$TESTDIRS" ]; then # NB: Relies on paths being prefixed with './'. - TESTDIRS=($(git ls-files -- '*_test.go' | grep -vE '^(vendor|prog|experimental)/' | xargs -n1 dirname | sort -u | sed -e 's|^|./|')) + TESTDIRS=($(git ls-files -- '*_test.go' | grep -vE '^(vendor|experimental)/' | xargs -n1 dirname | sort -u | sed -e 's|^|./|')) else # TESTDIRS on the right side is not really an array variable, it # is just a string with spaces, but it is written like that to @@ -80,7 +89,7 @@ fi # If running on circle, use the scheduler to work out what tests to run on what shard if [ -n "$CIRCLECI" ] && [ -z "$NO_SCHEDULER" ] && [ -x "$DIR/sched" ]; then PREFIX=$(go list -e ./ | sed -e 's/\//-/g') - TESTDIRS=($(echo "${TESTDIRS[@]}" | "$DIR/sched" sched "$PREFIX-$CIRCLE_BUILD_NUM" "$CIRCLE_NODE_TOTAL" "$CIRCLE_NODE_INDEX")) + TESTDIRS=($(echo "${TESTDIRS[@]}" | "$DIR/sched" sched "$PREFIX-$CIRCLE_PROJECT_USERNAME-$CIRCLE_PROJECT_REPONAME-$CIRCLE_BUILD_NUM" "$CIRCLE_NODE_TOTAL" "$CIRCLE_NODE_INDEX")) echo "${TESTDIRS[@]}" fi @@ -92,24 +101,29 @@ go test -i "${GO_TEST_ARGS[@]}" "${TESTDIRS[@]}" run_test() { local dir=$1 if [ -z "$NO_GO_GET" ]; then - go get -t "$TAGS" "$dir" + go get -t -tags "${TAGS[@]}" "$dir" fi local GO_TEST_ARGS_RUN=("${GO_TEST_ARGS[@]}") if [ -n "$SLOW" ]; then - local COVERPKGS=$( ( + local COVERPKGS + COVERPKGS=$( ( go list "$dir" go list -f '{{join .Deps "\n"}}' "$dir" | grep -v "vendor" | grep "^$PACKAGE_BASE/" ) | paste -s -d, -) - local output=$(mktemp "$coverdir/unit.XXXXXXXXXX") + local output + output=$(mktemp "$coverdir/unit.XXXXXXXXXX") local GO_TEST_ARGS_RUN=("${GO_TEST_ARGS[@]}" -coverprofile=$output -coverpkg=$COVERPKGS) fi - local START=$(date +%s) + local START + START=$(date +%s) if ! go test "${GO_TEST_ARGS_RUN[@]}" "$dir"; then fail=1 fi - local RUNTIME=$(($(date +%s) - START)) + local END + END=$(date +%s) + local RUNTIME=$((END - START)) # Report test runtime when running on circle, to help scheduler if [ -n "$CIRCLECI" ] && [ -z "$NO_SCHEDULER" ] && [ -x "$DIR/sched" ]; then