Run update tests in parallel

This change makes the update tests from each prior version of TimescaleDB to run in parallel instead of in serial order. This speeds up the testing significantly.
timescale · Oct 29, 2018 · b7a4851 · b7a4851
1 parent 7fb6fba
commit b7a4851
Show file tree

Hide file tree

Showing 6 changed files with 180 additions and 54 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -76,16 +76,24 @@ jobs:
       stage: test
       env: PG_VERSION=9.6.6 PG_GIT_TAG=REL9_6_6
       name: "Update tests (versions w/o constraints support) 9.6"
+      before_install:
+      install:
+      after_failure:
+      after_script:
       script:
-        - PGTEST_TMPDIR=/tmp/ bash -x ./scripts/test_updates_no_constraints.sh
+        - bash -x ./scripts/test_updates_no_constraints.sh
 
     # This tests the ability to upgrade to the latest version from versions with constraint support
     - if: (type = pull_request) OR (type = cron)
       stage: test
       env: PG_VERSION=9.6.6 PG_GIT_TAG=REL9_6_6
       name: "Update tests (versions w/ constraints support) 9.6"
+      before_install:
+      install:
+      after_failure:
+      after_script:
       script:
-        - PGTEST_TMPDIR=/tmp/ bash -x ./scripts/test_updates_with_constraints.sh
+        - bash -x ./scripts/test_updates_with_constraints.sh
 
     # This tests the formatting of a PR.
     - if: (type = pull_request) OR (type = cron) OR NOT (branch = master)

diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh
@@ -5,7 +5,7 @@
 #
 SCRIPT_DIR=$(dirname $0)
 BASE_DIR=${PWD}/${SCRIPT_DIR}/..
-PG_VERSION=${PG_VERSION:-10.4}
+PG_VERSION=${PG_VERSION:-9.6.5}
 PG_IMAGE_TAG=${PG_IMAGE_TAG:-${PG_VERSION}-alpine}
 BUILD_CONTAINER_NAME=${BUILD_CONTAINER_NAME:-pgbuild}
 BUILD_IMAGE_NAME=${BUILD_IMAGE_NAME:-$USER/pgbuild}
@@ -15,7 +15,7 @@ GIT_TAG=$(git -C ${BASE_DIR} rev-parse --short --verify HEAD)
 GIT_ID=$(git -C ${BASE_DIR} describe --dirty | sed -e "s|/|_|g")
 TAG_NAME=${TAG_NAME:-$GIT_ID}
 BUILD_TYPE=${BUILD_TYPE:-Debug}
-USE_OPENSSL=${USE_OPENSSL:-True}
+USE_OPENSSL=${USE_OPENSSL:-true}
 PUSH_PG_IMAGE=${PUSH_PG_IMAGE:-false}
 
 # Full image identifiers
@@ -96,10 +96,10 @@ fi
 
 if ! postgres_build_image_exists; then
     if ! fetch_postgres_build_image; then
-        create_postgres_build_image || exit -1
+        create_postgres_build_image || exit 1
     fi
 fi
 
-build_timescaledb || exit -1
+build_timescaledb || exit 1
 
 message_and_exit
diff --git a/scripts/test_update_from_tag.sh b/scripts/test_update_from_tag.sh
@@ -6,18 +6,28 @@ set -o pipefail
 SCRIPT_DIR=$(dirname $0)
 BASE_DIR=${PWD}/${SCRIPT_DIR}/..
 TEST_VERSION=${TEST_VERSION:-v2}
-PGTEST_TMPDIR=${PGTEST_TMPDIR:-$(mktemp -d 2>/dev/null || mktemp -d -t 'timescaledb_update_test')}
+TEST_TMPDIR=${TEST_TMPDIR:-$(mktemp -d 2>/dev/null || mktemp -d -t 'timescaledb_update_test' || mkdir -p /tmp/${RANDOM})}
 UPDATE_PG_PORT=${UPDATE_PG_PORT:-6432}
 CLEAN_PG_PORT=${CLEAN_PG_PORT:-6433}
 PG_VERSION=${PG_VERSION:-9.6.5} # Need 9.6.x version since we are
                                 # upgrading the extension from
                                 # versions that didn't support PG10.
+GIT_ID=$(git -C ${BASE_DIR} describe --dirty | sed -e "s|/|_|g")
 UPDATE_FROM_IMAGE=${UPDATE_FROM_IMAGE:-timescale/timescaledb}
 UPDATE_FROM_TAG=${UPDATE_FROM_TAG:-0.1.0}
 UPDATE_TO_IMAGE=${UPDATE_TO_IMAGE:-update_test}
-UPDATE_TO_TAG=${UPDATE_TO_TAG:-latest}
+UPDATE_TO_TAG=${UPDATE_TO_TAG:-${GIT_ID}}
 DO_CLEANUP=true
 
+# PID of the current shell
+PID=$$
+
+# Container names. Append shell PID so that we can run this script in parallel
+CONTAINER_ORIG=timescaledb-orig-${PID}
+CONTAINER_CLEAN_RESTORE=timescaledb-clean-restore-${PID}
+CONTAINER_UPDATED=timescaledb-updated-${PID}
+CONTAINER_CLEAN_RERUN=timescaledb-clean-rerun-${PID}
+
 export PG_VERSION
 
 while getopts "d" opt;
@@ -33,28 +43,36 @@ done
 
 shift $((OPTIND-1))
 
-if "$DO_CLEANUP" = "true"; then
-    trap cleanup EXIT
-fi
+trap cleanup EXIT
+
+remove_containers() {
+    docker rm -vf ${CONTAINER_ORIG} 2>/dev/null
+    docker rm -vf ${CONTAINER_CLEAN_RESTORE} 2>/dev/null
+    docker rm -vf ${CONTAINER_UPDATED}  2>/dev/null
+    docker rm -vf ${CONTAINER_CLEAN_RERUN} 2>/dev/null
+    docker volume rm -f ${CLEAN_VOLUME} 2>/dev/null
+    docker volume rm -f ${UPDATE_VOLUME} 2>/dev/null
+}
 
 cleanup() {
     # Save status here so that we can return the status of the last
     # command in the script and not the last command of the cleanup
     # function
-    status="$?"
+    local status="$?"
     set +e # do not exit immediately on failure in cleanup handler
-    if [ $status -eq 0 ]; then
-        rm -rf ${PGTEST_TMPDIR}
-        docker rm -vf timescaledb-orig timescaledb-clean-restore timescaledb-updated 2>/dev/null
+    if [ "$DO_CLEANUP" = "true" ]; then
+        rm -rf ${TEST_TMPDIR}
+        sleep 1
+        remove_containers
     fi
-    echo "Exit status is $status"
-    exit $status
+    echo "Test with pid ${PID} exited with code ${status}"
+    exit ${status}
 }
 
 docker_exec() {
     # Echo to stderr
     >&2 echo -e "\033[1m$1\033[0m: $2"
-    docker exec -it $1 /bin/bash -c "$2"
+    docker exec $1 /bin/bash -c "$2"
 }
 
 docker_pgcmd() {
@@ -67,15 +85,15 @@ docker_pgscript() {
 
 docker_pgtest() {
     >&2 echo -e "\033[1m$1\033[0m: $2"
-    docker exec $1 psql -X -v ECHO=ALL -v ON_ERROR_STOP=1 -h localhost -U postgres -d single -f $2 > ${PGTEST_TMPDIR}/$1.out || exit $?
+    docker exec $1 psql -X -v ECHO=ALL -v ON_ERROR_STOP=1 -h localhost -U postgres -d single -f $2 > ${TEST_TMPDIR}/$1.out || exit $?
 }
 
 docker_pgdiff() {
     >&2 echo -e "\033[1m$1 vs $2\033[0m: $2"
     docker_pgtest $1 $3
     docker_pgtest $2 $3
-    echo "RUNNING:  diff ${PGTEST_TMPDIR}/$1.out ${PGTEST_TMPDIR}/$2.out "
-    diff ${PGTEST_TMPDIR}/$1.out ${PGTEST_TMPDIR}/$2.out | tee ${PGTEST_TMPDIR}/update_test.output
+    echo "RUNNING:  diff ${TEST_TMPDIR}/$1.out ${TEST_TMPDIR}/$2.out "
+    diff ${TEST_TMPDIR}/$1.out ${TEST_TMPDIR}/$2.out | tee ${TEST_TMPDIR}/update_test.output
 }
 
 docker_run() {
@@ -90,8 +108,8 @@ docker_run_vol() {
 
 wait_for_pg() {
     set +e
-    for i in {1..10}; do
-        sleep 2
+    for i in {1..20}; do
+        sleep 0.5
 
         docker_exec $1 "pg_isready -U postgres"
 
@@ -100,7 +118,7 @@ wait_for_pg() {
             # ideal. Apperently, pg_isready is not always a good
             # indication of whether the DB is actually ready to accept
             # queries
-            sleep 2
+            sleep 0.2
             set -e
             return 0
         fi
@@ -110,42 +128,46 @@ wait_for_pg() {
 
 VERSION=`echo ${UPDATE_FROM_TAG} | sed 's/\([0-9]\{0,\}\.[0-9]\{0,\}\.[0-9]\{0,\}\).*/\1/g'`
 echo "Testing from version ${VERSION} (test version ${TEST_VERSION})"
-echo "Using temporary directory $PGTEST_TMPDIR"
+echo "Using temporary directory ${TEST_TMPDIR}"
+
+remove_containers || true
 
-docker rm -f timescaledb-orig timescaledb-updated timescaledb-clean-restore timescaledb-clean-rerun 2>/dev/null || true
-IMAGE_NAME=update_test TAG_NAME=latest bash ${SCRIPT_DIR}/docker-build.sh
+IMAGE_NAME=${UPDATE_TO_IMAGE} TAG_NAME=${UPDATE_TO_TAG} PG_VERSION=${PG_VERSION} bash ${SCRIPT_DIR}/docker-build.sh
 
-docker_run timescaledb-orig ${UPDATE_FROM_IMAGE}:${UPDATE_FROM_TAG}
-docker_run timescaledb-clean-restore ${UPDATE_TO_IMAGE}:${UPDATE_TO_TAG}
-docker_run timescaledb-clean-rerun ${UPDATE_TO_IMAGE}:${UPDATE_TO_TAG}
+docker_run ${CONTAINER_ORIG} ${UPDATE_FROM_IMAGE}:${UPDATE_FROM_TAG}
+docker_run ${CONTAINER_CLEAN_RESTORE} ${UPDATE_TO_IMAGE}:${UPDATE_TO_TAG}
+docker_run ${CONTAINER_CLEAN_RERUN} ${UPDATE_TO_IMAGE}:${UPDATE_TO_TAG}
 
-CLEAN_VOLUME=$(docker inspect timescaledb-clean-restore --format='{{range .Mounts }}{{.Name}}{{end}}')
-UPDATE_VOLUME=$(docker inspect timescaledb-orig --format='{{range .Mounts }}{{.Name}}{{end}}')
+CLEAN_VOLUME=$(docker inspect ${CONTAINER_CLEAN_RESTORE} --format='{{range .Mounts }}{{.Name}}{{end}}')
+UPDATE_VOLUME=$(docker inspect ${CONTAINER_ORIG} --format='{{range .Mounts }}{{.Name}}{{end}}')
 
 echo "Executing setup script on ${VERSION}"
-docker_pgscript timescaledb-orig /src/test/sql/updates/setup.${TEST_VERSION}.sql
-docker rm -f timescaledb-orig
+docker_pgscript ${CONTAINER_ORIG} /src/test/sql/updates/setup.${TEST_VERSION}.sql
+
+# Remove container but keep volume
+docker rm -f ${CONTAINER_ORIG}
 
-docker_run_vol timescaledb-updated ${UPDATE_VOLUME}:/var/lib/postgresql/data ${UPDATE_TO_IMAGE}:${UPDATE_TO_TAG}
+echo "Running update container"
+docker_run_vol ${CONTAINER_UPDATED} ${UPDATE_VOLUME}:/var/lib/postgresql/data ${UPDATE_TO_IMAGE}:${UPDATE_TO_TAG}
 
 echo "Executing ALTER EXTENSION timescaledb UPDATE"
-docker_pgcmd timescaledb-updated "ALTER EXTENSION timescaledb UPDATE"
+docker_pgcmd ${CONTAINER_UPDATED} "ALTER EXTENSION timescaledb UPDATE"
 
-docker_exec timescaledb-updated "pg_dump -h localhost -U postgres -Fc single > /tmp/single.sql"
-docker cp timescaledb-updated:/tmp/single.sql ${PGTEST_TMPDIR}/single.sql
+docker_exec ${CONTAINER_UPDATED} "pg_dump -h localhost -U postgres -Fc single > /tmp/single.sql"
+docker cp ${CONTAINER_UPDATED}:/tmp/single.sql ${TEST_TMPDIR}/single.sql
 
 echo "Executing setup script on clean"
-docker_pgscript timescaledb-clean-rerun /src/test/sql/updates/setup.${TEST_VERSION}.sql
+docker_pgscript ${CONTAINER_CLEAN_RERUN} /src/test/sql/updates/setup.${TEST_VERSION}.sql
 
 echo "Testing updated vs clean"
-docker_pgdiff timescaledb-updated timescaledb-clean-rerun /src/test/sql/updates/test-rerun.sql
+docker_pgdiff ${CONTAINER_UPDATED} ${CONTAINER_CLEAN_RERUN} /src/test/sql/updates/test-rerun.sql
 
 echo "Restoring database on clean version"
-docker cp ${PGTEST_TMPDIR}/single.sql timescaledb-clean-restore:/tmp/single.sql
-docker_exec timescaledb-clean-restore "createdb -h localhost -U postgres single"
-docker_pgcmd timescaledb-clean-restore "ALTER DATABASE single SET timescaledb.restoring='on'"
-docker_exec timescaledb-clean-restore "pg_restore -h localhost -U postgres -d single /tmp/single.sql"
-docker_pgcmd timescaledb-clean-restore "ALTER DATABASE single SET timescaledb.restoring='off'"
+docker cp ${TEST_TMPDIR}/single.sql ${CONTAINER_CLEAN_RESTORE}:/tmp/single.sql
+docker_exec ${CONTAINER_CLEAN_RESTORE} "createdb -h localhost -U postgres single"
+docker_pgcmd ${CONTAINER_CLEAN_RESTORE} "ALTER DATABASE single SET timescaledb.restoring='on'"
+docker_exec ${CONTAINER_CLEAN_RESTORE} "pg_restore -h localhost -U postgres -d single /tmp/single.sql"
+docker_pgcmd ${CONTAINER_CLEAN_RESTORE} "ALTER DATABASE single SET timescaledb.restoring='off'"
 
 echo "Testing restored"
-docker_pgdiff timescaledb-updated timescaledb-clean-restore /src/test/sql/updates/post.${TEST_VERSION}.sql
+docker_pgdiff ${CONTAINER_UPDATED} ${CONTAINER_CLEAN_RESTORE} /src/test/sql/updates/post.${TEST_VERSION}.sql
diff --git a/scripts/test_updates.sh b/scripts/test_updates.sh
@@ -0,0 +1,97 @@
+#!/bin/bash
+
+set -o pipefail
+
+SCRIPT_DIR=$(dirname $0)
+TEST_TMPDIR=${TEST_TMPDIR:-$(mktemp -d 2>/dev/null || mktemp -d -t 'timescaledb_update_test' || mkdir -p /tmp/$RANDOM )}
+BASE_DIR=${PWD}/${SCRIPT_DIR}/..
+TAGS=${TAGS:-}
+TEST_VERSION=${TEST_VERSION:-}
+GIT_ID=$(git -C ${BASE_DIR} describe --dirty | sed -e "s|/|_|g")
+UPDATE_TO_IMAGE=${UPDATE_TO_IMAGE:-update_test}
+UPDATE_TO_TAG=${UPDATE_TO_TAG:-${GIT_ID}}
+PG_VERSION=${PG_VERSION:-9.6.5} # Need 9.6.x version since we are
+                                # upgrading the extension from
+                                # versions that didn't support PG10.
+
+FAILED_TEST=
+
+# Declare a hash table to keep test names keyed by pid
+declare -A tests
+
+while getopts "c" opt;
+do
+    case $opt in
+        c)
+            echo "Forcing cleanup of build image"
+            docker rmi -f ${UPDATE_TO_IMAGE}:${UPDATE_TO_TAG}
+            ;;
+    esac
+done
+
+cleanup() {
+    local exit_code="$?"
+    set +e # do not exit immediately on failure
+    echo "Waiting for remaining tests to finish..."
+    wait
+    if [ -f ${TEST_TMPDIR}/${FAILED_TEST}.log ]; then
+        echo "###### Failed test log below #####"
+        cat ${TEST_TMPDIR}/${FAILED_TEST}.log
+    fi
+    rm -rf ${TEST_TMPDIR}
+    echo "exit code is $exit_code"
+    return $exit_code
+}
+
+kill_all_tests() {
+    local exit_code="$?"
+    set +e # do not exit immediately on failure
+    echo "Killing all tests"
+    kill ${!tests[@]} 2>/dev/null
+    return $exit_code
+}
+
+trap kill_all_tests INT HUP
+trap cleanup EXIT
+
+if [ -z "${TEST_VERSION}" ]; then
+    echo "No TEST_VERSION specified"
+    exit 1
+fi
+
+if [ -z "${TAGS}" ]; then
+    echo "No TAGS specified"
+    exit 1
+fi
+
+# Build the docker image with current source here so that the parallel
+# tests don't all compete in trying to build it first
+IMAGE_NAME=${UPDATE_TO_IMAGE} TAG_NAME=${UPDATE_TO_TAG} PG_VERSION=${PG_VERSION} bash ${SCRIPT_DIR}/docker-build.sh
+
+# Run update tests in parallel
+for tag in ${TAGS};
+do
+    UPDATE_FROM_TAG=${tag} TEST_VERSION=${TEST_VERSION} $(dirname $0)/test_update_from_tag.sh > ${TEST_TMPDIR}/${tag}.log 2>&1 &
+
+    tests[$!]=${tag}
+    echo "Launched test ${tag} with pid $!"
+done
+
+# Need to wait on each pid in a loop to return the exit status of each
+echo "Waiting for tests to finish..."
+
+# Since we are iterating a hash table, the tests are not going to be
+# in order started. But it doesn't matter.
+for pid in ${!tests[@]};
+do
+    wait $pid;
+    exit_code=$?
+    echo "Test ${tests[$pid]} (pid $pid) exited with code $exit_code"
+
+    if [ $exit_code -ne 0 ]; then
+        FAILED_TEST=${tests[$pid]}
+        kill_all_tests
+        exit $exit_code
+    fi
+    $(exit $exit_code)
+done
diff --git a/scripts/test_updates_no_constraints.sh b/scripts/test_updates_no_constraints.sh
@@ -7,9 +7,8 @@
 set -e
 set -o pipefail
 
+SCRIPT_DIR=$(dirname $0)
 TAGS="0.1.0 0.2.0 0.3.0 0.4.0 0.4.1 0.4.2"
+TEST_VERSION="v1"
 
-for tag in ${TAGS};
-do
-    UPDATE_FROM_TAG=${tag} TEST_VERSION="v1" $(dirname $0)/test_update_from_tag.sh
-done
+. ${SCRIPT_DIR}/test_updates.sh
diff --git a/scripts/test_updates_with_constraints.sh b/scripts/test_updates_with_constraints.sh
@@ -10,9 +10,9 @@
 set -e
 set -o pipefail
 
+SCRIPT_DIR=$(dirname $0)
+
 TAGS="0.5.0 0.6.0 0.6.1 0.7.0-pg9.6 0.7.1-pg9.6 0.8.0-pg9.6 0.9.0-pg9.6 0.9.1-pg9.6 0.9.2-pg9.6 0.10.0-pg9.6 0.10.1-pg9.6 0.11.0-pg9.6 0.12.0-pg9.6 1.0.0-rc1-pg9.6 1.0.0-rc2-pg9.6 1.0.0-rc3-pg9.6"
+TEST_VERSION="v2"
 
-for tag in ${TAGS};
-do
-    UPDATE_FROM_TAG=${tag} TEST_VERSION="v2" $(dirname $0)/test_update_from_tag.sh
-done
+. ${SCRIPT_DIR}/test_updates.sh