Run integration tests on CI

Plus a number of improvements to the way tests are run.

* Add a script to prettify `go test` output. This means we only see
  errors in the output, as well as a summary of tests run.
* Integration and unit tests are run separately on Travis. This allows
  them to run in parallel.
* Ignore integration test failures, for now.
* Godep workspace is now handled transparently in the Makefile by
  setting the GOPATH. This allows the same environment to work both
  locally and on Travis.

.travis.yml

@@ -13,12 +13,15 @@ before_script:
   - go get github.com/vektra/mockery/.../
 
 env:
-  - RUN="make test_ci"
-  - RUN="test/update-coveralls"
+  - RUN="make test-unit"
+  - RUN="make test-integration"
+  - RUN=test/update-coveralls
 
 matrix:
   allow_failures:
-    - env: RUN="test/update-coveralls"
+    - env: RUN=test/update-coveralls
+    # Temporarily allow the integration tests to fail until we fix them
+    - env: RUN="make test-integration"
   fast_finish: true
 
 script:

Makefile

@@ -1,7 +1,14 @@
+.PHONY: clean clean-mocks testpop mocks out test test-integration test-unit
+
+SHELL = /bin/bash
+
 export PATH := $(shell pwd)/scripts/travis/thrift-release/linux-x86_64:$(PATH)
 export PATH := $(shell pwd)/scripts/travis/thrift-gen-release/linux-x86_64:$(PATH)
 
-.PHONY: clean clean-mocks testpop mocks out test test_ci
+# go commands should use the Godeps/_workspace
+GODEPS := $(shell pwd)/Godeps/_workspace
+OLDGOPATH := $(GOPATH)
+export GOPATH = $(GODEPS):$(OLDGOPATH)
 
 out:	test
 
@@ -15,13 +22,14 @@ clean-mocks:
 mocks:
 	test/gen-testfiles
 
-test:
-	godep go generate ./...
-	godep go test -v ./...
+test:	test-unit test-integration
+
+test-integration:
+	test/run-integration-tests
 
-test_ci:
+test-unit:
 	go generate ./...
-	go test -v ./...
+	go test -v ./... |test/go-test-prettify
 
 testpop:	clean
-	godep go build scripts/testpop/testpop.go
+	go build scripts/testpop/testpop.go

test/.gitignore

@@ -0,0 +1 @@
+ringpop-common/

test/go-test-prettify

@@ -0,0 +1,74 @@
+#!/bin/bash
+#
+# Parse the output from `go test -v` and prints a summary at the end. Also only
+# outputs errors by default unless -v is specified.
+#
+
+declare verbose=false
+
+if [[ $1 == "-v"* ]]; then
+	verbose=true
+fi
+
+declare -i run=0
+declare -i passed=0
+declare -i failed=0
+declare -i skipped=0
+
+while IFS= read -r line; do
+	case $line in
+		"--- PASS:"*)
+			passed=$(($passed+1))
+			;;
+
+		"--- FAIL:"*)
+			failed=$(($failed+1))
+
+			# Output fail lines
+			if ! $verbose; then
+				echo "$line"
+			fi
+			;;
+
+		"--- SKIP:"*)
+			skipped=$(($skipped+1))
+			;;
+
+		# Skip output of "RUN" lines by default
+		"=== RUN"*)
+			run=$(($run+1))
+			;;
+
+		# Skip output of mock.go success lines (unicode char is the green tick)
+		*mock*$(echo -e "\xe2\x9c\x85")*)
+			;;
+
+		# Skip and ignore printing junk
+		PASS|FAIL|\?*)
+			;;
+
+		*)
+			# Output unknown lines
+			if ! $verbose; then
+				echo "$line"
+			fi
+			;;
+	esac
+
+	if $verbose; then
+		echo "$line"
+	fi
+
+done
+
+echo
+echo "# tests: $run"
+[ $passed -ne 0 ] && echo "# passed: $passed"
+[ $failed -ne 0 ] && echo "# failed: $failed"
+[ $skipped -ne 0 ] && echo "# skipped: $skipped"
+echo
+
+# Exit with non-zero code if there were test errors
+if [ $failed -ne 0 ]; then
+	exit 1
+fi

test/run-integration-tests

@@ -0,0 +1,170 @@
+#!/bin/bash
+#
+# Run integration tests for ringpop-go.
+#
+# Integration tests for different cluster sizes are run in parallel. Success
+# output is suppressed and only the failures are shown.
+#
+# 2015-01-14
+#
+set -eo pipefail
+
+declare project_root="${0%/*}/.."
+declare ringpop_common_dir="${0%/*}/ringpop-common"
+declare tap_filter="${ringpop_common_dir}/test/tap-filter.js"
+
+declare test_cluster_sizes="1 2 3 4 5 10"
+declare test_result=
+
+declare temp_dir="$(mktemp -d)"
+
+# Check node is installed
+if ! type node &>/dev/null; then
+    echo "ERROR: missing 'node'" >&2
+    exit 1
+fi
+
+#
+# Same as builtin wait, but return code is the number of background processes
+# that exited with a non-zero code.
+#
+wait-all() {
+    local -i failed=0
+
+    # We need to explicitly loop through all background jobs and specify the
+    # pids to `wait`, otherwise `wait` doesn't return the exit code.
+    for pid in $(jobs -p); do
+        wait $pid || let "failed+=1"
+    done
+
+    return $failed
+}
+
+#
+# Echos and runs the specified command.
+#
+run() {
+    echo "+ $@" >&2
+    "$@"
+}
+
+#
+# Copy stdin to stdout but prefix each line with the specified string.
+#
+prefix() {
+    local _prefix=
+
+    [ -n "$1" ] && _prefix="[$1] "
+    while IFS= read -r -t 30 line; do
+        echo "${_prefix}${line}"
+    done
+}
+
+#
+# Clones or updates the ringpop-common repository.
+#
+fetch-ringpop-common() {
+    if [ ! -e "$ringpop_common_dir" ]; then
+        run git clone --depth=1 https://github.com/uber/ringpop-common.git "$ringpop_common_dir"
+    fi
+
+    run cd "$ringpop_common_dir"
+    #run git checkout master
+    run git pull
+    run cd - >/dev/null
+
+    run cd "${ringpop_common_dir}/test"
+    run npm install >/dev/null
+    run cd - >/dev/null
+
+    # Check tap-filter exists in ringpop-common. It is required to filter output
+    # correctly to stdout/stderr
+    if ! [ -x "$tap_filter" ]; then
+        echo "ERROR: missing 'test/tap-filter.js' in ringpop-common" >&2
+        exit 1
+    fi
+}
+
+#
+# Build the testpop binary.
+#
+build-testpop() {
+    cd "$project_root"
+    run make testpop
+}
+
+#
+# Run test with specified cluster size.
+#
+# $1: cluster size
+#
+run-test-for-cluster-size() {
+    local cluster_size=$1
+    local err=0
+    local output_file="${temp_dir}/${cluster_size}.out"
+
+    # Run the tests and buffer the output to a log file. We'll display it later
+    # if the test fails. This avoids interleaving of output to the terminal
+    # when tests are running in parallel.
+    node "${ringpop_common_dir}/test/it-tests.js" \
+        -s "[$1]" "${project_root}/testpop" &>$output_file || err=$?
+
+    if [ $PIPESTATUS -gt 0 ]; then
+        echo "ERROR: Test errored for cluster size $cluster_size" | \
+            prefix "test-errors-${cluster_size}" >&2
+        return 1
+    fi
+
+    if [ $err -ne 0 ]; then
+        # If the test failed, print a message and display the failures
+        {
+            echo "FAIL: Test failed for cluster size $cluster_size"
+            # Output the test data through tap-filter, which discards success
+            # info unless -v is specified.
+            cat "$output_file" |$tap_filter
+
+        } | prefix "test-errors-${cluster_size}" >&2
+
+        return 1
+    fi
+}
+
+#
+# Run the integration tests against the testpop binary.
+#
+run-tests() {
+    for cluster_size in $test_cluster_sizes; do
+        echo "Spawning test for cluster size ${cluster_size}..." |prefix "test-runner"
+        run-test-for-cluster-size $cluster_size &
+    done
+
+    {
+        echo
+        echo "Waiting for tests to complete."
+        echo
+        echo "To monitor test output (verbose), run:"
+        echo "    tail -f ${temp_dir}/*.out"
+        echo
+    } \
+        |prefix "test-runner"
+
+    wait-all
+}
+
+# Fetch and build in parallel
+{ fetch-ringpop-common 2>&1|prefix "fetch ringpop-common"; } &
+{ build-testpop 2>&1|prefix "build testpop"; } &
+wait-all
+
+# Run integration tests
+run-tests
+test_result=$?
+
+if [ $test_result -eq 0 ]; then
+    echo "Tests passed"
+    rm -rf "$temp_dir"
+else
+    echo "Tests failed" >&2
+fi
+
+exit $test_result