Merge pull request #405 from xueweiz/test-pr

Rent Boskos project only once per test run.

Makefile

@@ -125,7 +125,7 @@ test: vet fmt
 	GO111MODULE=on go test -mod vendor -timeout=1m -v -race -short -tags "$(BUILD_TAGS)" ./...
 
 e2e-test: vet fmt build-tar
-	GO111MODULE=on ginkgo -nodes=$(PARALLEL) -mod vendor -timeout=10m -v -tags "$(BUILD_TAGS)" \
+	GO111MODULE=on ginkgo -nodes=$(PARALLEL) -mod vendor -timeout=10m -v -tags "$(BUILD_TAGS)" -stream \
 	./test/e2e/metriconly/... -- \
 	-project=$(PROJECT) -zone=$(ZONE) \
 	-image=$(VM_IMAGE) -image-family=$(IMAGE_FAMILY) -image-project=$(IMAGE_PROJECT) \

test/e2e/metriconly/e2e_npd_test.go

@@ -25,6 +25,7 @@ import (
 	"testing"
 	"time"
 
+	"k8s.io/node-problem-detector/pkg/util/tomb"
 	"k8s.io/node-problem-detector/test/e2e/lib/gce"
 	"k8s.io/test-infra/boskos/client"
 
@@ -54,6 +55,16 @@ var boskosWaitDuration = flag.Duration("boskos-wait-duration", 2*time.Minute,
 
 var computeService *compute.Service
 
+// boskosClient helps renting project from Boskos, and is only initialized on Ginkgo node 1.
+var boskosClient *client.Client
+
+// boskosRenewingTomb stops the goroutine keep renewing the Boskos resources.
+var boskosRenewingTomb *tomb.Tomb
+
+// SynchronizedBeforeSuite and SynchronizedAfterSuite help manages singleton resource (a Boskos project) across Ginkgo nodes.
+var _ = ginkgo.SynchronizedBeforeSuite(rentBoskosProjectIfNeededOnNode1, acceptBoskosProjectIfNeededFromNode1)
+var _ = ginkgo.SynchronizedAfterSuite(func() {}, releaseBoskosResourcesOnNode1)
+
 func TestNPD(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping test in short mode.")
@@ -65,13 +76,6 @@ func TestNPD(t *testing.T) {
 		panic(fmt.Sprintf("Unable to create gcloud compute service using defaults. Make sure you are authenticated. %v", err))
 	}
 
-	if *project == "" {
-		boskosClient := client.NewClient(*jobName, *boskosServerURL)
-		*project = acquireProjectOrDie(boskosClient)
-
-		defer releaseProjectOrDie(boskosClient)
-	}
-
 	if *artifactsDir != "" {
 		_, err := os.Stat(*artifactsDir)
 		if err != nil && os.IsNotExist(err) {
@@ -84,30 +88,72 @@ func TestNPD(t *testing.T) {
 	ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "NPD Metric-only Suite", []ginkgo.Reporter{junitReporter})
 }
 
-func acquireProjectOrDie(boskosClient *client.Client) string {
+// rentBoskosProjectIfNeededOnNode1 rents a GCP project from Boskos if no GCP project is specified.
+//
+// rentBoskosProjectIfNeededOnNode1 returns a byte slice containing the project name.
+// rentBoskosProjectIfNeededOnNode1 also initializes boskosClient if necessary.
+// When the tests run in parallel mode in Ginkgo, this rentBoskosProjectIfNeededOnNode1 runs only on
+// Ginkgo node 1. The output should be shared with all other Gingko nodes so that they all use the same
+// GCP project.
+func rentBoskosProjectIfNeededOnNode1() []byte {
+	if *project != "" {
+		return []byte{}
+	}
+
 	fmt.Printf("Renting project from Boskos\n")
+	boskosClient = client.NewClient(*jobName, *boskosServerURL)
+	boskosRenewingTomb = tomb.NewTomb()
+
 	ctx, cancel := context.WithTimeout(context.Background(), *boskosWaitDuration)
 	defer cancel()
 	p, err := boskosClient.AcquireWait(ctx, *boskosProjectType, "free", "busy")
 	Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Unable to rent project from Boskos: %v\n", err))
+	fmt.Printf("Rented project %q from Boskos\n", p.Name)
+
+	go renewBoskosProject(boskosClient, p.Name, boskosRenewingTomb)
 
-	fmt.Printf("Rented project %s from Boskos", p.Name)
+	return []byte(p.Name)
+}
+
+// acceptBoskosProjectIfNeededFromNode1 accepts a GCP project rented from Boskos by Ginkgo node 1.
+//
+// acceptBoskosProjectIfNeededFromNode1 takes the output of rentBoskosProjectIfNeededOnNode1.
+// When the tests run in parallel mode in Ginkgo, this function runs on all Ginkgo nodes.
+func acceptBoskosProjectIfNeededFromNode1(data []byte) {
+	if *project != "" {
+		return
+	}
+
+	boskosProject := string(data)
+	fmt.Printf("Received Boskos project %q from Ginkgo node 1.\n", boskosProject)
+	*project = boskosProject
+}
 
-	go func(boskosClient *client.Client, projectName string) {
-		for range time.Tick(5 * time.Minute) {
+func renewBoskosProject(boskosClient *client.Client, projectName string, boskosRenewingTomb *tomb.Tomb) {
+	defer boskosRenewingTomb.Done()
+	for {
+		select {
+		case <-time.Tick(5 * time.Minute):
+			fmt.Printf("Renewing boskosProject %q\n", projectName)
 			if err := boskosClient.UpdateOne(projectName, "busy", nil); err != nil {
-				fmt.Printf("Failed to update status for project %s with Boskos: %v\n", projectName, err)
+				fmt.Printf("Failed to update status for project %q with Boskos: %v\n", projectName, err)
 			}
+		case <-boskosRenewingTomb.Stopping():
+			return
 		}
-	}(boskosClient, p.Name)
-
-	return p.Name
+	}
 }
 
-func releaseProjectOrDie(boskosClient *client.Client) {
+// releaseBoskosResourcesOnNode1 releases all rented Boskos resources if there is any.
+func releaseBoskosResourcesOnNode1() {
+	if boskosClient == nil {
+		return
+	}
+	boskosRenewingTomb.Stop()
 	if !boskosClient.HasResource() {
 		return
 	}
+	fmt.Printf("Releasing all Boskos resources.\n")
 	err := boskosClient.ReleaseAll("dirty")
 	Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to release project to Boskos: %v", err))
 }