Limit the number of concurrent tests in integration.go

Integration test often time out because the machine is loaded. Instead of
increasing timeout, this change hopes to address the issue by limiting the
number of tests running simultaneously.

Add a new flag in integration.go to specify the maximum number of concurrent
tests. Set the default in travis and shippable configurations to be 4.

.travis.yml

@@ -24,7 +24,7 @@ install:
 script:
   - KUBE_RACE="-race" KUBE_COVER="y" KUBE_GOVERALLS_BIN="$HOME/gopath/bin/goveralls" KUBE_TIMEOUT='-timeout 300s' KUBE_COVERPROCS=8 KUBE_TEST_API_VERSIONS=$KUBE_TEST_API_VERSIONS ./hack/test-go.sh -- -p=2
   - PATH=$HOME/gopath/bin:./third_party/etcd:$PATH ./hack/test-cmd.sh
-  - PATH=$HOME/gopath/bin:./third_party/etcd:$PATH KUBE_TEST_API_VERSIONS=$KUBE_TEST_API_VERSIONS ./hack/test-integration.sh
+  - PATH=$HOME/gopath/bin:./third_party/etcd:$PATH KUBE_TEST_API_VERSIONS=$KUBE_TEST_API_VERSIONS KUBE_INTEGRATION_TEST_MAX_CONCURRENCY=4 ./hack/test-integration.sh
 
 notifications:
   irc: "chat.freenode.net#google-containers"

cmd/integration/integration.go

@@ -67,6 +67,8 @@ var (
 	fakeDocker1, fakeDocker2 dockertools.FakeDockerClient
 	// API version that should be used by the client to talk to the server.
 	apiVersion string
+	// Limit the number of concurrent tests.
+	maxConcurrency int
 )
 
 type fakeKubeletClient struct{}
@@ -947,6 +949,8 @@ type testFunc func(*client.Client)
 
 func addFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&apiVersion, "apiVersion", latest.Version, "API version that should be used by the client for communicating with the server")
+	fs.IntVar(
+		&maxConcurrency, "maxConcurrency", -1, "Maximum number of tests to be run simultaneously. Unlimited if set to negative.")
 }
 
 func main() {
@@ -994,16 +998,26 @@ func main() {
 		},
 	}
 
-	var wg sync.WaitGroup
-	wg.Add(len(testFuncs))
-	for i := range testFuncs {
-		f := testFuncs[i]
-		go func() {
-			f(kubeClient)
-			wg.Done()
-		}()
+	// Only run at most maxConcurrency tests in parallel.
+	numFinishedTests := 0
+	for numFinishedTests < len(testFuncs) {
+		numTestsToRun := len(testFuncs) - numFinishedTests
+		if maxConcurrency > 0 && numTestsToRun > maxConcurrency {
+			numTestsToRun = maxConcurrency
+		}
+		glog.Infof("Running %d tests in parallel.", numTestsToRun)
+		var wg sync.WaitGroup
+		wg.Add(numTestsToRun)
+		for i := 0; i < numTestsToRun; i++ {
+			f := testFuncs[i+numFinishedTests]
+			go func() {
+				f(kubeClient)
+				wg.Done()
+			}()
+		}
+		wg.Wait()
+		numFinishedTests += numTestsToRun
 	}
-	wg.Wait()
 
 	// Check that kubelet tried to make the containers.
 	// Using a set to list unique creation attempts. Our fake is

hack/test-integration.sh

@@ -26,7 +26,7 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 # Comma separated list of API Versions that should be tested.
 KUBE_TEST_API_VERSIONS=${KUBE_TEST_API_VERSIONS:-"v1beta1,v1beta3"}
-
+KUBE_INTEGRATION_TEST_MAX_CONCURRENCY=${KUBE_INTEGRATION_TEST_MAX_CONCURRENCY:-"-1"}
 
 cleanup() {
   kube::etcd::cleanup
@@ -44,7 +44,8 @@ runTests() {
 
   kube::log::status "Running integration test scenario"
 
-  "${KUBE_OUTPUT_HOSTBIN}/integration" --v=2 --apiVersion="$1"
+  "${KUBE_OUTPUT_HOSTBIN}/integration" --v=2 --apiVersion="$1" \
+  --maxConcurrency="${KUBE_INTEGRATION_TEST_MAX_CONCURRENCY}"
 
   cleanup
 }

shippable.yml

@@ -33,7 +33,7 @@ install:
 script:
   - KUBE_RACE="-race" KUBE_COVER="y" KUBE_GOVERALLS_BIN="$HOME/gopath/bin/goveralls" KUBE_TIMEOUT='-timeout 300s' KUBE_COVERPROCS=8 KUBE_TEST_API_VERSIONS=$KUBE_TEST_API_VERSIONS ./hack/test-go.sh -- -p=2
   - PATH=$HOME/gopath/bin:./third_party/etcd:$PATH ./hack/test-cmd.sh
-  - PATH=$HOME/gopath/bin:./third_party/etcd:$PATH KUBE_TEST_API_VERSIONS=$KUBE_TEST_API_VERSIONS ./hack/test-integration.sh
+  - PATH=$HOME/gopath/bin:./third_party/etcd:$PATH KUBE_TEST_API_VERSIONS=$KUBE_TEST_API_VERSIONS KUBE_INTEGRATION_TEST_MAX_CONCURRENCY=4 ./hack/test-integration.sh
 
 notifications:
   irc: "chat.freenode.net#google-containers"