Separate integration tests

.travis.yml

@@ -1,164 +1,45 @@
-# Travis configuration for gardener.
+# Travis configuration for gardener unit tests.
 language: go
 services:
   - docker
 
 go:
-   - 1.13
+   - 1.15
 
 ###########################################################################
 before_install:
+- echo Branch is ${TRAVIS_BRANCH} and Tag is ${TRAVIS_TAG}
+
 # Coverage tools
 - go get github.com/mattn/goveralls
 - go get github.com/wadey/gocovmerge
 
-- echo Branch is ${TRAVIS_BRANCH} and Tag is $TRAVIS_TAG
-
 # Install gcloud, for integration tests.
-# TODO: maybe just use travis apt: packages: ?
 - $TRAVIS_BUILD_DIR/travis/install_gcloud.sh
 - source "${HOME}/google-cloud-sdk/path.bash.inc"
 
-# Install test credentials.
-# The service account variables are uploaded to travis by running,
-# from root of repo directory:
-#  travis/setup_service_accounts_for_travis.sh
-#
-# All of the gcloud library calls will detect the GOOGLE_APPLICATION_CREDENTIALS
-# environment variable, and use that file for authentication.
-- if [[ -n "$SERVICE_ACCOUNT_mlab_testing" ]] ; then
-  echo "$SERVICE_ACCOUNT_mlab_testing" > $TRAVIS_BUILD_DIR/creds.json ;
-  export GOOGLE_APPLICATION_CREDENTIALS=$TRAVIS_BUILD_DIR/creds.json ;
-  travis/activate_service_account.sh SERVICE_ACCOUNT_mlab_testing ;
-  fi
-
-# Rsync the mlab-testing ndt directory to match expected content.
-- cd $TRAVIS_BUILD_DIR/testfiles
-- ./sync.sh
-- cd $TRAVIS_BUILD_DIR
-
-# These directories will be cached on successful "script" builds, and restored,
-# if available, to save time on future builds.
-cache:
-  directories:
-    - "$HOME/google-cloud-sdk/"
-
 install:
-  # Install kexpand templating tool. Only works from HEAD.
-- go get github.com/kopeio/kexpand
-- $TRAVIS_BUILD_DIR/travis/install_gcloud.sh kubectl
-
-# Install dependencies
+# Install dependencies for all tests.
 - cd $TRAVIS_BUILD_DIR
 - go get -v -t ./...
+- go get -v -tags=integration -t ./...
 
-before_script:
-# Try removing boto config, recommended for datastore emulator.
-- sudo rm -f /etc/boto.cfg
-
+# Install and run datastore emulator.
 - gcloud components install beta
 - gcloud components install cloud-datastore-emulator
-- gcloud beta emulators datastore start --no-store-on-disk &
-- sleep 2 # allow time for emulator to start up.
-- $(gcloud beta emulators datastore env-init)
 
 script:
-# To start, run all the non-integration unit tests.
+# Run all the non-integration unit tests.
 - cd $TRAVIS_BUILD_DIR
-# This is failing since go 1.13.  See https://github.com/golang/go/issues/30374
-# - go test -v -coverpkg=./... -coverprofile=_unit.cov ./...
 - go test -v -coverprofile=_unit.cov ./...
 
-# Rerun modules with integration tests.  This means that some tests are
-# repeated, but otherwise we lose some coverage.
-- if [[ -n "$SERVICE_ACCOUNT_mlab_testing" ]] ; then
-    go test -v -coverprofile=_integration.cov ./... -tags=integration ;
-  fi
+# Run all integration unit tests. Some tests are repeated, but this preserves
+# coverage statistics.
+- ./integration-testing.sh
 
 # Also run some concurrency sensitive tests with -race
 - go test -v ./tracker/... ./ops/... -race
 
-# Combine coverage of unit tests and integration tests and send the results to coveralls.
+# Combine coverage of all unit tests and send the results to coveralls.
 - $HOME/gopath/bin/gocovmerge _*.cov > _merge.cov
 - $HOME/gopath/bin/goveralls -coverprofile=_merge.cov -service=travis-ci || true  # Ignore failure
-
-# Docker build is done in google cloud builer
-
-#################################################################################
-# Deployment Section
-#
-#  Overview:
-#   1.  Test in sandbox during development
-#   2.  Deploy to staging on commit to master
-#   3.  Deploy to prod when a branch is tagged with prod-* or xxx-prod-*
-#
-#  We want to test individual components in sandbox, and avoid stepping on each
-#  other, so we do NOT automate deployment to sandbox.  Each person should
-#  use a branch name to trigger the single deployment that they are working on.
-#
-#  We want to soak all code in staging before deploying to prod.  To avoid
-#  incompatible components, we deploy ALL elements to staging when we merge
-#  to master.
-#
-#  Deployments to prod are done by deliberately tagging a specific commit,
-#  typically in the master branch, with a tag starting with prod-*.
-#  DO NOT just tag the latest version in master, as someone may have
-#  pushed new code that hasn't had a chance to soak in staging.
-#
-#
-# Deploy steps never trigger on a new Pull Request. Deploy steps will trigger
-# on specific branch name patterns, after a merge to master, or on
-# an explicit tag that matches "on:" conditions.
-#################################################################################
-
-
-deploy:
-#########################################
-## Sandbox
-- provider: script
-  script:
-    DATE_SKIP="2" TASK_FILE_SKIP="4"
-    $TRAVIS_BUILD_DIR/travis/kubectl.sh mlab-sandbox data-processing-cluster ./apply-cluster.sh
-  on:
-    repo: m-lab/etl-gardener
-    all_branches: true
-    condition: $TRAVIS_BRANCH == sandbox-* && $TRAVIS_EVENT_TYPE == push
-
-- provider: script
-  script:
-    $TRAVIS_BUILD_DIR/travis/kubectl.sh mlab-sandbox data-processing ./apply-cluster.sh
-  skip_cleanup: true
-  on:
-    repo: m-lab/etl-gardener
-    all_branches: true
-    condition: $TRAVIS_BRANCH == u-sandbox-* && $TRAVIS_EVENT_TYPE == push
-
-#########################################
-## Staging
-- provider: script
-  script:
-    $TRAVIS_BUILD_DIR/travis/kubectl.sh mlab-staging data-processing-cluster ./apply-cluster.sh
-    &&
-    $TRAVIS_BUILD_DIR/travis/kubectl.sh mlab-staging data-processing ./apply-cluster.sh
-  on:
-    repo: m-lab/etl-gardener
-    all_branches: true
-    condition: $TRAVIS_BRANCH == master && $TRAVIS_EVENT_TYPE == push
-
-#########################################
-## Production
-- provider: script
-  script:
-    $TRAVIS_BUILD_DIR/travis/kubectl.sh mlab-oti data-processing-cluster ./apply-cluster.sh
-  on:
-    repo: m-lab/etl-gardener
-    all_branches: true
-    condition: $TRAVIS_TAG == prod-*
-
-- provider: script
-  script:
-    $TRAVIS_BUILD_DIR/travis/kubectl.sh mlab-oti data-processing ./apply-cluster.sh
-  on:
-    repo: m-lab/etl-gardener
-    all_branches: true
-    condition: $TRAVIS_TAG == prod-* || $TRAVIS_TAG == u-prod-*

cloud/bq/ops_test.go

@@ -1,3 +1,5 @@
+// +build integration
+
 package bq_test
 
 import (

cloud/bq/sanity_test.go

@@ -1,3 +1,5 @@
+// +build integration
+
 package bq
 
 import (

cloudbuild.yaml

@@ -0,0 +1,6 @@
+# NOTE: this is a place holder for a complete cloudbuild.yaml.
+steps:
+# Make all git tags available.
+- name: gcr.io/cloud-builders/git
+  id: "Unshallow git clone"
+  args: ["fetch", "--unshallow"]

cmd/gardener/gardener_test.go

@@ -1,3 +1,5 @@
+// +build integration
+
 // Package main defines a service for handling various post-processing
 // and house-keeping tasks associated with the pipelines.
 // Most tasks will be run periodically, but some may be triggered

integration-testing.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+# Exit on error.
+set -e
+
+# Install test credentials for authentication:
+# * gcloud commands will use the activated service account.
+# * Go libraries will use the GOOGLE_APPLICATION_CREDENTIALS.
+if [[ -z "$SERVICE_ACCOUNT_mlab_testing" ]] ; then
+  echo "ERROR: testing service account is unavailable."
+  exit 1
+fi
+
+echo "$SERVICE_ACCOUNT_mlab_testing" > $PWD/creds.json
+# Make credentials available for Go libraries.
+export GOOGLE_APPLICATION_CREDENTIALS=$PWD/creds.json
+if [ -f "/builder/google-cloud-sdk/path.bash.inc" ]; then
+  # Reset home directory for container.
+  HOME=/builder
+fi
+# Make credentials available for gcloud commands.
+travis/activate_service_account.sh SERVICE_ACCOUNT_mlab_testing
+
+# NOTE: do this after setting the service account.
+gcloud config set project mlab-testing
+
+# Rsync the mlab-testing ndt directory to match expected content.
+pushd testfiles
+./sync.sh
+popd
+
+# Remove boto config; recommended for datastore emulator.
+rm -f /etc/boto.cfg || :
+
+# Start datastore emulator.
+gcloud beta emulators datastore start --no-store-on-disk &
+
+t1=$( date +%s )
+until nc -z localhost 8081 ; do
+  sleep 1
+  t2=$( date +%s )
+  # Allow up to a minute for emulator to start up.
+  if [[ $t2 -gt $(( $t1 + 60 )) ]] ; then
+    echo "ERROR: failed to start or detect datastore emulator"
+    break
+  fi
+done
+$(gcloud beta emulators datastore env-init)
+go test -v -tags=integration -coverprofile=_integration.cov ./...
+
+# Shutdown datastore emulator.
+kill %1

job-service/job-service_test.go

@@ -1,3 +1,5 @@
+// +build integration
+
 // Package job provides an http handler to serve up jobs to ETL parsers.
 package job_test
 

ops/actions_test.go

@@ -1,3 +1,5 @@
+// +build integration
+
 package ops_test
 
 import (

persistence/persistence_test.go

@@ -1,3 +1,5 @@
+// +build integration
+
 package persistence_test
 
 import (

rex/rex_test.go

@@ -1,3 +1,5 @@
+// +build integration
+
 package rex_test
 
 import (

state/state_test.go

@@ -1,3 +1,5 @@
+// +build integration
+
 package state_test
 
 import (