From e59f1238c899070c9cabdb71931fc0e863887259 Mon Sep 17 00:00:00 2001 From: Daniel Simmons Date: Thu, 21 Jan 2016 15:13:29 +0100 Subject: [PATCH] Run integration tests on CI Plus a number of improvements to the way tests are run. * Add a script to prettify `go test` output. This means we only see errors in the output, as well as a summary of tests run. * Integration and unit tests are run separately on Travis. This allows them to run in parallel. * Ignore integration test failures, for now. * Godep workspace is now handled transparently in the Makefile by setting the GOPATH. This allows the same environment to work both locally and on Travis. --- .travis.yml | 9 +- Makefile | 22 +++-- test/.gitignore | 1 + test/go-test-prettify | 74 ++++++++++++++++ test/run-integration-tests | 170 +++++++++++++++++++++++++++++++++++++ 5 files changed, 266 insertions(+), 10 deletions(-) create mode 100644 test/.gitignore create mode 100755 test/go-test-prettify create mode 100755 test/run-integration-tests diff --git a/.travis.yml b/.travis.yml index 11eae789..3cfb59e2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,12 +13,15 @@ before_script: - go get github.com/vektra/mockery/.../ env: - - RUN="make test_ci" - - RUN="test/update-coveralls" + - RUN="make test-unit" + - RUN="make test-integration" + - RUN=test/update-coveralls matrix: allow_failures: - - env: RUN="test/update-coveralls" + - env: RUN=test/update-coveralls + # Temporarily allow the integration tests to fail until we fix them + - env: RUN="make test-integration" fast_finish: true script: diff --git a/Makefile b/Makefile index 36334fc3..d5bb867d 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,14 @@ +.PHONY: clean clean-mocks testpop mocks out test test-integration test-unit + +SHELL = /bin/bash + export PATH := $(shell pwd)/scripts/travis/thrift-release/linux-x86_64:$(PATH) export PATH := $(shell pwd)/scripts/travis/thrift-gen-release/linux-x86_64:$(PATH) -.PHONY: clean clean-mocks testpop mocks out test test_ci +# go commands should use the Godeps/_workspace +GODEPS := $(shell pwd)/Godeps/_workspace +OLDGOPATH := $(GOPATH) +export GOPATH = $(GODEPS):$(OLDGOPATH) out: test @@ -15,13 +22,14 @@ clean-mocks: mocks: test/gen-testfiles -test: - godep go generate ./... - godep go test -v ./... +test: test-unit test-integration + +test-integration: + test/run-integration-tests -test_ci: +test-unit: go generate ./... - go test -v ./... + go test -v ./... |test/go-test-prettify testpop: clean - godep go build scripts/testpop/testpop.go + go build scripts/testpop/testpop.go diff --git a/test/.gitignore b/test/.gitignore new file mode 100644 index 00000000..89a152ba --- /dev/null +++ b/test/.gitignore @@ -0,0 +1 @@ +ringpop-common/ diff --git a/test/go-test-prettify b/test/go-test-prettify new file mode 100755 index 00000000..d06728ab --- /dev/null +++ b/test/go-test-prettify @@ -0,0 +1,74 @@ +#!/bin/bash +# +# Parse the output from `go test -v` and prints a summary at the end. Also only +# outputs errors by default unless -v is specified. +# + +declare verbose=false + +if [[ $1 == "-v"* ]]; then + verbose=true +fi + +declare -i run=0 +declare -i passed=0 +declare -i failed=0 +declare -i skipped=0 + +while IFS= read -r line; do + case $line in + "--- PASS:"*) + passed=$(($passed+1)) + ;; + + "--- FAIL:"*) + failed=$(($failed+1)) + + # Output fail lines + if ! $verbose; then + echo "$line" + fi + ;; + + "--- SKIP:"*) + skipped=$(($skipped+1)) + ;; + + # Skip output of "RUN" lines by default + "=== RUN"*) + run=$(($run+1)) + ;; + + # Skip output of mock.go success lines (unicode char is the green tick) + *mock*$(echo -e "\xe2\x9c\x85")*) + ;; + + # Skip and ignore printing junk + PASS|FAIL|\?*) + ;; + + *) + # Output unknown lines + if ! $verbose; then + echo "$line" + fi + ;; + esac + + if $verbose; then + echo "$line" + fi + +done + +echo +echo "# tests: $run" +[ $passed -ne 0 ] && echo "# passed: $passed" +[ $failed -ne 0 ] && echo "# failed: $failed" +[ $skipped -ne 0 ] && echo "# skipped: $skipped" +echo + +# Exit with non-zero code if there were test errors +if [ $failed -ne 0 ]; then + exit 1 +fi diff --git a/test/run-integration-tests b/test/run-integration-tests new file mode 100755 index 00000000..44c712fa --- /dev/null +++ b/test/run-integration-tests @@ -0,0 +1,170 @@ +#!/bin/bash +# +# Run integration tests for ringpop-go. +# +# Integration tests for different cluster sizes are run in parallel. Success +# output is suppressed and only the failures are shown. +# +# 2015-01-14 +# +set -eo pipefail + +declare project_root="${0%/*}/.." +declare ringpop_common_dir="${0%/*}/ringpop-common" +declare tap_filter="${ringpop_common_dir}/test/tap-filter.js" + +declare test_cluster_sizes="1 2 3 4 5 10" +declare test_result= + +declare temp_dir="$(mktemp -d)" + +# Check node is installed +if ! type node &>/dev/null; then + echo "ERROR: missing 'node'" >&2 + exit 1 +fi + +# +# Same as builtin wait, but return code is the number of background processes +# that exited with a non-zero code. +# +wait-all() { + local -i failed=0 + + # We need to explicitly loop through all background jobs and specify the + # pids to `wait`, otherwise `wait` doesn't return the exit code. + for pid in $(jobs -p); do + wait $pid || let "failed+=1" + done + + return $failed +} + +# +# Echos and runs the specified command. +# +run() { + echo "+ $@" >&2 + "$@" +} + +# +# Copy stdin to stdout but prefix each line with the specified string. +# +prefix() { + local _prefix= + + [ -n "$1" ] && _prefix="[$1] " + while IFS= read -r -t 30 line; do + echo "${_prefix}${line}" + done +} + +# +# Clones or updates the ringpop-common repository. +# +fetch-ringpop-common() { + if [ ! -e "$ringpop_common_dir" ]; then + run git clone --depth=1 https://github.com/uber/ringpop-common.git "$ringpop_common_dir" + fi + + run cd "$ringpop_common_dir" + #run git checkout master + run git pull + run cd - >/dev/null + + run cd "${ringpop_common_dir}/test" + run npm install >/dev/null + run cd - >/dev/null + + # Check tap-filter exists in ringpop-common. It is required to filter output + # correctly to stdout/stderr + if ! [ -x "$tap_filter" ]; then + echo "ERROR: missing 'test/tap-filter.js' in ringpop-common" >&2 + exit 1 + fi +} + +# +# Build the testpop binary. +# +build-testpop() { + cd "$project_root" + run make testpop +} + +# +# Run test with specified cluster size. +# +# $1: cluster size +# +run-test-for-cluster-size() { + local cluster_size=$1 + local err=0 + local output_file="${temp_dir}/${cluster_size}.out" + + # Run the tests and buffer the output to a log file. We'll display it later + # if the test fails. This avoids interleaving of output to the terminal + # when tests are running in parallel. + node "${ringpop_common_dir}/test/it-tests.js" \ + -s "[$1]" "${project_root}/testpop" &>$output_file || err=$? + + if [ $PIPESTATUS -gt 0 ]; then + echo "ERROR: Test errored for cluster size $cluster_size" | \ + prefix "test-errors-${cluster_size}" >&2 + return 1 + fi + + if [ $err -ne 0 ]; then + # If the test failed, print a message and display the failures + { + echo "FAIL: Test failed for cluster size $cluster_size" + # Output the test data through tap-filter, which discards success + # info unless -v is specified. + cat "$output_file" |$tap_filter + + } | prefix "test-errors-${cluster_size}" >&2 + + return 1 + fi +} + +# +# Run the integration tests against the testpop binary. +# +run-tests() { + for cluster_size in $test_cluster_sizes; do + echo "Spawning test for cluster size ${cluster_size}..." |prefix "test-runner" + run-test-for-cluster-size $cluster_size & + done + + { + echo + echo "Waiting for tests to complete." + echo + echo "To monitor test output (verbose), run:" + echo " tail -f ${temp_dir}/*.out" + echo + } \ + |prefix "test-runner" + + wait-all +} + +# Fetch and build in parallel +{ fetch-ringpop-common 2>&1|prefix "fetch ringpop-common"; } & +{ build-testpop 2>&1|prefix "build testpop"; } & +wait-all + +# Run integration tests +run-tests +test_result=$? + +if [ $test_result -eq 0 ]; then + echo "Tests passed" + rm -rf "$temp_dir" +else + echo "Tests failed" >&2 +fi + +exit $test_result