From 031721ca6518ac99dab85bbfa224d2f41f472fd2 Mon Sep 17 00:00:00 2001 From: Jesse Nelson Date: Tue, 19 Jul 2016 12:34:40 -0700 Subject: [PATCH] Squashed 'scripts/make/' content from commit e4c47f6 git-subtree-dir: scripts/make git-subtree-split: e4c47f62ca06ed9d2f4649570b112118c937e522 --- Makefile | 4 +++ README.md | 74 ++++++++++++++++++++++++++++++++++++++++++++ circle.yml | 4 +++ common-docker.mk | 59 +++++++++++++++++++++++++++++++++++ common-go.mk | 65 ++++++++++++++++++++++++++++++++++++++ common-kube.mk | 31 +++++++++++++++++++ common.mk | 11 +++++++ sh/install-gcloud.sh | 51 ++++++++++++++++++++++++++++++ sh/install-go.sh | 70 +++++++++++++++++++++++++++++++++++++++++ 9 files changed, 369 insertions(+) create mode 100644 Makefile create mode 100644 README.md create mode 100644 circle.yml create mode 100644 common-docker.mk create mode 100644 common-go.mk create mode 100644 common-kube.mk create mode 100644 common.mk create mode 100755 sh/install-gcloud.sh create mode 100755 sh/install-go.sh diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..22d5d83 --- /dev/null +++ b/Makefile @@ -0,0 +1,4 @@ +include common.mk +include common-go.mk +include common-docker.mk +include common-kube.mk diff --git a/README.md b/README.md new file mode 100644 index 0000000..098c0f4 --- /dev/null +++ b/README.md @@ -0,0 +1,74 @@ +Common make tasks +================= + +Usage +----- + +### Seting Up the common scripts + +Add these common tasks to your project by using git subtree from the root of your project. + +First add the remote. + +``` +git remote add common_makefiles git@github.com:pantheon-systems/common_makefiles.git --no-tags +``` + +Now add the subtree + +**note:** it is important that you keep the import path set to `scripts/make` as the makefiles assume this structure. + +``` +git subtree add --prefix scripts/make common_makefiles master --squash +``` + +### Using in your Makefile + +you simply need to include the common makefiles you want in your projects root Makefile + +``` +APP=baryon +PROJECT := $$GOOGLE_PROJECT + +include scripts/make/common.mk +include scripts/make/common-kube.mk +include scripts/make/common-go.mk +``` + +### Extending Tasks + +All the common makefile tasks can be extended in your top level Makefile by defining them again. Each common task that can be extended has a `::` target. e.g. `deps::` + +for example if I want to do something after the default build target from common-go.mk I can add to it in my Makefile like so: + +``` +build:: + @echo "this is after the common build" +``` + +Updating the Common tasks +------------------------- + +The `common.mk` file includes a task named `update-makefiles` which you can invoke to pull and squash the latest versions of the common tasks into your project. + +``` +make update-makefiles +``` + +Adding more tasks and common files +---------------------------------- + +make edits here and open a PR against this repo. Please do not push from your subtree on your project. + +### Common Patterns for adding to the repo + +Tasks should follow the form of `--` for example if I have a build task and you want to add windows support you would add a `build-windows` or if you wanted to add a build for onebox you might do `build-onebox-linux` or simply `build-onebox`. + +There is the expectation that if you are doing a context specific task you add the context to your tasks. I.E. `test-circle` + +For now these are the default/expected tasks for any common-LANG file: +* build +* deps +* test + +This isn't written in stone, but I think it is a reasonable expectation that any engineer should be able to checkout any project and run: `make deps && make build && make test` to get things running / testing. diff --git a/circle.yml b/circle.yml new file mode 100644 index 0000000..5385884 --- /dev/null +++ b/circle.yml @@ -0,0 +1,4 @@ +--- +test: + override: + - make diff --git a/common-docker.mk b/common-docker.mk new file mode 100644 index 0000000..1e5219d --- /dev/null +++ b/common-docker.mk @@ -0,0 +1,59 @@ +# Docker common things +# +# INPUT VARIABLES +# - QUAY_USER: The quay.io user to use (usually set in CI) +# - QUAY_PASSWD: The quay passwd to use (usually set in CI) +# - IMAGE: the docker image to use. will be computed if it doesn't exist. +# - REGISTRY: The docker registry to use. defaults to quay. +# +# EXPORT VARIABLES +# - BUILD_NUM: The build number for this build. will use 'dev' if not building +# on circleCI, will use CIRCLE_BUILD_NUM otherwise. +# - IMAGE: The image to use for the build. +# - REGISTRY: The registry to use for the build. +# +#------------------------------------------------------------------------------- +ifeq ($(CIRCLE_BUILD_NUM),) + BUILD_NUM := dev +else + BUILD_NUM := $(CIRCLE_BUILD_NUM) + QUAY := docker login -p "$$QUAY_PASSWD" -u "$$QUAY_USER" -e "unused@unused" quay.io +endif + +# These can be overridden +IMAGE ?= $(REGISTRY)/$(APP):$(BUILD_NUM) +REGISTRY ?= quay.io/getpantheon + +# if there is a docker file then set the docker variable so things can trigger off it +ifneq ("$(wildcard Dockerfile))","") + # file is there + DOCKER:=true +endif + +# determinse the docker tag to build +build-docker:: +ifndef DOCKER + $(error "Docker task called, but no DOCKER variable set. Eitehr Dockerfile is missing or you didn't include common.") +endif +build-docker:: build-linux ## build the docker container + docker build -t $(IMAGE) . + +push:: ## push the container to the registry + docker push $(IMAGE) + +setup-quay:: ## setup docker login for quay.io +ifndef QUAY_PASSWD + $(error "Need to set QUAY_PASSWD environment variable") +endif +ifndef QUAY_USER + $(error "Need to set QUAY_USER environment variable") +endif + @$(QUAY) + +# we call make here to ensure new states are detected +push-circle:: ## build and push the container from circle + make build-docker +push-circle:: setup-quay + make push + +.PHONY:: setup-quay build-docker push push-circle diff --git a/common-go.mk b/common-go.mk new file mode 100644 index 0000000..a4c424b --- /dev/null +++ b/common-go.mk @@ -0,0 +1,65 @@ +# Common Go Tasks +# +# INPUT VARIABLES +# - COVERALLS_TOKEN: Token to use when pushing coverage to coveralls. +# +# - FETCH_CA_CERT: The presence of this variable will add a Pull root ca certs +# to ca-certificats.crt before build. +# +#------------------------------------------------------------------------------- +build:: ## build project for current arch + go build + +build-linux:: _fetch-cert ## build project for linux + GOOS=linux CGO_ENABLED=0 go build -ldflags="-s -w" + +build-circle:: build-linux ## build project for linux. If you need docker you will have to invoke that with an extension + +deps:: _gvt-install ## install dependencies for project assumes you have go binary installed + find ./vendor/* -maxdepth 0 -type d -exec rm -rf "{}" \; + gvt rebuild + +test:: ## run go tests + go test -race -v $$(go list ./... | grep -v /vendor/) + +test-circle:: test test-coveralls ## invoke test tasks for CI + +deps-circle:: ## install Golang and pull dependencies in CI + bash scripts/make/sh/install-go.sh + +deps-coverage:: +ifeq (, $(shell which gotestcover)) + go get github.com/pierrre/gotestcover +endif +ifeq (, $(shell which goveralls)) + go get github.com/mattn/goveralls +endif + +deps-status:: ## check status of deps with gostatus +ifeq (, $(shell which gostatus)) + go get -u github.com/shurcooL/gostatus +endif + go list -f '{{join .Deps "\n"}}' . | gostatus -stdin -v + +test-coveralls:: deps-coverage ## run coverage and report to coveralls +ifdef COVERALLS_TOKEN + gotestcover -v -race -coverprofile=coverage.out $$(go list ./... | grep -v /vendor/) + goveralls -repotoken $$COVERALLS_TOKEN -service=circleci -coverprofile=coverage.out +else + $(error "You asked to use Coveralls, but neglected to set the COVERALLS_TOKEN environment variable") +endif + +test-coverage-html:: deps-coverage ## output html coverage file + go tool cover -html=coverage.html + +_gvt-install:: +ifeq (, $(shell which gvt)) + go get -u github.com/FiloSottile/gvt +endif + +_fetch-cert:: +ifdef FETCH_CA_CERT + curl https://curl.haxx.se/ca/cacert.pem -o ca-certificates.crt +endif + +.PHONY:: _fetch-cert _gvt-install test-coverage-html test-coveralls deps-status deps-coverage deps-circle deps test-circle test build-circle build-linux build diff --git a/common-kube.mk b/common-kube.mk new file mode 100644 index 0000000..df889a9 --- /dev/null +++ b/common-kube.mk @@ -0,0 +1,31 @@ +# Common kube things. This is the simplest set of common kube tasks +# +# INPUT VARIABLES +# - APP: should be defined in your topmost Makefile +# +# EXPORT VARIABLES +# - KUBE_NAMESPACE: represents the kube namespace that has been detected based on +# branch build and circle existence. +#------------------------------------------------------------------------------- + +# If we have no circle branch, use development kube env +# If we are on master branch, use production kube env +# If we are NOT on master use testing kube env +ifeq ($(CIRCLE_BRANCH),) # Dev + KUBE_NAMESPACE := development +else ifeq ($(CIRCLE_BRANCH), master) # prod + KUBE_NAMESPACE := production +else # testing + KUBE_NAMESPACE := testing +endif + +# debatable weather this should be in common or not, but I see it needed enough in dev. +# TODO(jesse): possibly guard this to prevent accidentally nuking production. +force-pod-restart:: ## nuke the pod + kubectl --namespace=$(KUBE_NAMESPACE) get pod -l"app=$(APP)" --no-headers | awk '{print $$1}' | xargs kubectl delete pod + +# extend or define circle deps to install gcloud +deps-circle:: + @bash scripts/make/sh/install-gcloud.sh + +.PHONY:: deps-circle force-pod-restart diff --git a/common.mk b/common.mk new file mode 100644 index 0000000..f5c0eab --- /dev/null +++ b/common.mk @@ -0,0 +1,11 @@ +# common make tasks and variables that should be imported into all projects +# +#------------------------------------------------------------------------------- +help: ## print list of tasks and descriptions + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?##"}; { split($$0,a,":"); printf "\033[36m%-30s\033[0m %s \n", a[2], $$2}' +.DEFAULT_GOAL := help + +update-makefiles: ## update the make subtree, assumes the subtree is in scripts/make + git subtree pull --prefix scripts/make common_makefiles master --squash + +.PHONY:: all help update-makefiles diff --git a/sh/install-gcloud.sh b/sh/install-gcloud.sh new file mode 100755 index 0000000..b94c595 --- /dev/null +++ b/sh/install-gcloud.sh @@ -0,0 +1,51 @@ +#! /bin/bash +# install and configure gcloud on circle-ci +# +# The following ENV vars must be set before calling this script: +# +# CLOUDSDK_CORE_PROJECT # Google Cloud project Id to deploy into +# GCLOUD_EMAIL # user-id for circle to authenticate to google cloud +# GCLOUD_KEY # base64 encoded key +# CLOUDSDK_COMPUTE_ZONE # The compute zone container the GKE container cluster to deploy into +# CLUSTER_ID # ID of the GKE container cluster to deploy into +set -e + +if [ "$CIRCLECI" != "true" ]; then + echo "This script is only intended to run on Circle-CI." + exit 1 +fi + +export CLOUDSDK_CORE_DISABLE_PROMPTS=1 +export CLOUDSDK_PYTHON_SITEPACKAGES=0 + +gcloud="$HOME/google-cloud-sdk/bin/gcloud -q --no-user-output-enabled" +PATH="$gcloud/bin:$PATH" + +if [ ! -d "$HOME/google-cloud-sdk" ]; then + echo "$HOME/gogole-cloud-sdk missing, installing" + pip install pyopenssl + + curl -o "$HOME/google-cloud-sdk.tar.gz" https://dl.google.com/dl/cloudsdk/channels/rapid/google-cloud-sdk.tar.gz + tar -C "$HOME/" -xzvf ~/google-cloud-sdk.tar.gz + bash "$HOME/google-cloud-sdk/install.sh" + + $gcloud components update + $gcloud components update kubectl +fi + +echo "Setting Project" +$gcloud config set project "$CLOUDSDK_CORE_PROJECT" +echo "Setting Zone" +$gcloud config set compute/zone "$CLOUDSDK_COMPUTE_ZONE" +echo "Setting Cluster" +$gcloud config set container/cluster "$CLUSTER_ID" + +echo "$GCLOUD_KEY" | base64 --decode > gcloud.json +$gcloud auth activate-service-account "$GCLOUD_EMAIL" --key-file gcloud.json + +sshkey="$HOME/.ssh/google_compute_engine" +if [ ! -f "$sshkey" ] ; then + ssh-keygen -f "$sshkey" -N "" +fi + +$gcloud container clusters get-credentials "$CLUSTER_ID" --project="$CLOUDSDK_CORE_PROJECT" diff --git a/sh/install-go.sh b/sh/install-go.sh new file mode 100755 index 0000000..7df8822 --- /dev/null +++ b/sh/install-go.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# +# Use this script to help override circle-ci's go inference. +# +# Set env var GOVERSION to the version of go you'd like installed. Then call this script in the +# dependencies/override build phase. Your Go version will be installed to /home/ubuntu/go in the +# container, and your project's source code will be rsync'd into the $GOPATH so that local import +# paths will resolve correctly. +# +# Add `../go` path to your dependencies/cache_directories setting in circle.yml for +# faster builds. +# +# Example circle.yml: +# +# --- +# machine: +# environment: +# GOVERSION: 1.6.1 +# GOPATH: /home/ubuntu/go_workspace +# GOROOT: /home/ubuntu/go +# PATH: /home/ubuntu/go/bin:$GOPATH/bin:$PATH +# +# dependencies: +# cache_directories: +# - ../go_workspace +# - ../go +# +# overide: +# - bash scripts/install-go.sh +# + +set -ex + +if [ "$CIRCLECI" != "true" ]; then + echo "This script meant to only be run on CIRCLECI" + exit 1 +fi + +if [ -z "$GOVERSION" ] ; then + echo "set GOVERSION environment var" + exit 1 +fi + + +function fu_circle { + # convert CIRCLE_REPOSITORY_URL=https://github.com/user/repo -> github.com/user/repo + local IMPORT_PATH + IMPORT_PATH=$(sed -e 's#https://##' <<< "$CIRCLE_REPOSITORY_URL") + sudo rm -rf /usr/local/go + sudo rm -rf /home/ubuntu/.go_workspace || true + sudo ln -s "$HOME/go" /usr/local/go + mkdir -p "$GOPATH/src/$IMPORT_PATH" + rsync -az --delete ./ "$GOPATH/src/$IMPORT_PATH/" + pd=$(pwd) + cd ../ + rm -rf "$pd" + ln -s "$GOPATH/src/$IMPORT_PATH" "$pd" +} + +if "$HOME/go/bin/go" version | grep -q " go$GOVERSION "; then + echo "go $GOVERSION installed preping go import path" + fu_circle + exit 0 +fi + +gotar=go${GOVERSION}.tar.gz +curl -o "$HOME/$gotar" "https://storage.googleapis.com/golang/go${GOVERSION}.linux-amd64.tar.gz" +tar -C "$HOME/" -xzf "$HOME/$gotar" + +fu_circle