Improvements to the test runners

1. Use `go list` to get list of integration dirs to build. This means we
   do not need to have a valid `.go` in every subdirectory and also
   filters out other dirs like "bundles" which may have been created.
2. Add option to specify custom flags for integration and
   integration-cli. This is needed so both suites can be run AND set
   custom flags... since the cli suite does not support standard go
   flags.
3. Add options to skip an entire integration suite.

Signed-off-by: Brian Goff <cpuguy83@gmail.com>

Makefile

@@ -54,9 +54,13 @@ DOCKER_ENVS := \
 	-e DOCKER_USERLANDPROXY \
 	-e DOCKERD_ARGS \
 	-e TEST_INTEGRATION_DIR \
+	-e TEST_SKIP_INTEGRATION \
+	-e TEST_SKIP_INTEGRATION_CLI \
 	-e TESTDEBUG \
 	-e TESTDIRS \
 	-e TESTFLAGS \
+	-e TESTFLAGS_INTEGRATION \
+	-e TESTFLAGS_INTEGRATION_CLI \
 	-e TIMEOUT \
 	-e VALIDATE_REPO \
 	-e VALIDATE_BRANCH \
@@ -182,8 +186,13 @@ test-docker-py: build ## run the docker-py tests
 
 test-integration-cli: test-integration ## (DEPRECATED) use test-integration
 
+ifneq ($(and $(TEST_SKIP_INTEGRATION),$(TEST_SKIP_INTEGRATION_CLI)),)
+test-integration:
+	@echo Both integrations suites skipped per environment variables
+else
 test-integration: build ## run the integration tests
 	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-integration
+endif
 
 test-integration-flaky: build ## run the stress test for all new integration tests
 	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-integration-flaky

hack/make.ps1

@@ -327,19 +327,16 @@ Function Run-UnitTests() {
 # Run the integration tests
 Function Run-IntegrationTests() {
     $env:DOCKER_INTEGRATION_DAEMON_DEST = $root + "\bundles\tmp"
-    $dirs =  Get-ChildItem -Path integration -Directory -Recurse
+    $dirs = go list -test -f '{{- if ne .ForTest `"`" -}}{{- .Dir -}}{{- end -}}' .\integration\...
     $integration_api_dirs = @()
     ForEach($dir in $dirs) {
-        $RelativePath = "." + $dir.FullName -replace "$($PWD.Path -replace "\\","\\")",""
-        If ($RelativePath -notmatch '(^.\\integration($|\\internal)|\\testdata)') {
-            $integration_api_dirs += $dir
-            Write-Host "Building test suite binary $RelativePath"
-            go test -c -o "$RelativePath\test.exe" $RelativePath
-        }
+        $integration_api_dirs += $dir
+        Write-Host "Building test suite binary $dir"
+        go test -c -o "$dir\test.exe" $dir
     }
 
     ForEach($dir in $integration_api_dirs) {
-        Set-Location $dir.FullName
+        Set-Location $dir
         Write-Host "Running $($PWD.Path)"
         $pinfo = New-Object System.Diagnostics.ProcessStartInfo
         $pinfo.FileName = "$($PWD.Path)\test.exe"

hack/make/.integration-test-helpers

@@ -5,6 +5,18 @@
 #
 #     TESTFLAGS='-check.f DockerSuite.TestBuild*' ./hack/make.sh binary test-integration
 #
+
+if [[ "${TESTFLAGS}" = *-check.f* ]]; then
+	echo Skipping integration tests since TESTFLAGS includes integration-cli only flags
+	TEST_SKIP_INTEGRATION=1
+fi
+
+if [[ "${TESTFLAGS}" = *-test.run* ]]; then
+	echo Skipping integration-cli tests since TESTFLAGS includes integration only flags
+	TEST_SKIP_INTEGRATION_CLI=1
+fi
+
+
 if [ -z ${MAKEDIR} ]; then
 	export MAKEDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 fi
@@ -15,36 +27,34 @@ source "$MAKEDIR/.go-autogen"
 : ${TESTFLAGS:=}
 : ${TESTDEBUG:=}
 
-integration_api_dirs=${TEST_INTEGRATION_DIR:-"$(
-	find ./integration -type d |
-	grep -vE '(^./integration($|/internal)|/testdata)')"}
+integration_api_dirs=${TEST_INTEGRATION_DIR:-"$(go list  -test -f '{{- if ne .ForTest "" -}}{{- .Dir -}}{{- end -}}'  ./integration/...)"}
 
 run_test_integration() {
 	set_platform_timeout
-	if [[ "$TESTFLAGS" != *-check.f* ]]; then
+	if [ -z "${TEST_SKIP_INTEGRATION}" ]; then
 		run_test_integration_suites
 	fi
-	if [[ "$TESTFLAGS" != *-test.run* ]]; then
+	if [ -z "${TEST_SKIP_INTEGRATION_CLI}" ]; then
 		run_test_integration_legacy_suites
 	fi
 }
 
 run_test_integration_suites() {
-	local flags="-test.v -test.timeout=${TIMEOUT} $TESTFLAGS"
+	local flags="-test.v -test.timeout=${TIMEOUT} $TESTFLAGS ${TESTFLAGS_INTEGRATION}"
 	for dir in ${integration_api_dirs}; do
 		if ! (
 			cd "$dir"
-			echo "Running $PWD"
+			echo "Running $PWD flags=${flags}"
 			test_env ./test.main ${flags}
 		); then exit 1; fi
 	done
 }
 
 run_test_integration_legacy_suites() {
 	(
-		flags="-check.v -check.timeout=${TIMEOUT} -test.timeout=360m $TESTFLAGS"
+		flags="-check.v -check.timeout=${TIMEOUT} -test.timeout=360m $TESTFLAGS ${TESTFLAGS_INTEGRATION_CLI}"
 		cd integration-cli
-		echo "Running $PWD"
+		echo "Running $PWD flags=${flags}"
 		test_env ./test.main $flags
 	)
 }
@@ -54,10 +64,14 @@ build_test_suite_binaries() {
 		echo "Skipping building test binaries; as DOCKER_INTEGRATION_TESTS_VERIFIED is set"
 		return
 	fi
-	build_test_suite_binary ./integration-cli "test.main"
-	for dir in ${integration_api_dirs}; do
-		build_test_suite_binary "$dir" "test.main"
-	done
+	if [ -z "${TEST_SKIP_INTEGRATION_CLI}" ]; then
+		build_test_suite_binary ./integration-cli "test.main"
+	fi
+	if [ -z "${TEST_SKIP_INTEGRATION}" ]; then
+		for dir in ${integration_api_dirs}; do
+			build_test_suite_binary "$dir" "test.main"
+		done
+	fi
 }
 
 # Build a binary for a test suite package

hack/make/test-integration

@@ -3,6 +3,12 @@ set -e -o pipefail
 
 source hack/make/.integration-test-helpers
 
+if [ ! -z "${TEST_SKIP_INTEGRATION}" ] && [ ! -z "${TEST_SKIP_INTEGRATION_CLI}" ]; then
+	echo integration and integraiton-cli skipped according to env vars
+	exit 0
+fi
+
+
 (
 	build_test_suite_binaries
 	bundle .integration-daemon-start