Skip to content

Commit

Permalink
Rent Boskos project only once per test run.
Browse files Browse the repository at this point in the history
The old implementation rents Boskos project for each Ginkgo node.
  • Loading branch information
Xuewei Zhang committed Jan 3, 2020
1 parent 0d0bba9 commit fb8304b
Show file tree
Hide file tree
Showing 2 changed files with 63 additions and 17 deletions.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ test: vet fmt
GO111MODULE=on go test -mod vendor -timeout=1m -v -race -short -tags "$(BUILD_TAGS)" ./...

e2e-test: vet fmt build-tar
GO111MODULE=on ginkgo -nodes=$(PARALLEL) -mod vendor -timeout=10m -v -tags "$(BUILD_TAGS)" \
GO111MODULE=on ginkgo -nodes=$(PARALLEL) -mod vendor -timeout=10m -v -tags "$(BUILD_TAGS)" -stream \
./test/e2e/metriconly/... -- \
-project=$(PROJECT) -zone=$(ZONE) \
-image=$(VM_IMAGE) -image-family=$(IMAGE_FAMILY) -image-project=$(IMAGE_PROJECT) \
Expand Down
78 changes: 62 additions & 16 deletions test/e2e/metriconly/e2e_npd_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"testing"
"time"

"k8s.io/node-problem-detector/pkg/util/tomb"
"k8s.io/node-problem-detector/test/e2e/lib/gce"
"k8s.io/test-infra/boskos/client"

Expand Down Expand Up @@ -54,6 +55,16 @@ var boskosWaitDuration = flag.Duration("boskos-wait-duration", 2*time.Minute,

var computeService *compute.Service

// boskosClient helps renting project from Boskos, and is only initialized on Ginkgo node 1.
var boskosClient *client.Client

// boskosRenewingTomb stops the goroutine keep renewing the Boskos resources.
var boskosRenewingTomb *tomb.Tomb

// SynchronizedBeforeSuite and SynchronizedAfterSuite help manages singleton resource (a Boskos project) across Ginkgo nodes.
var _ = ginkgo.SynchronizedBeforeSuite(rentBoskosProjectIfNeededOnNode1, acceptBoskosProjectIfNeededFromNode1)
var _ = ginkgo.SynchronizedAfterSuite(func() {}, releaseBoskosResourcesOnNode1)

func TestNPD(t *testing.T) {
if testing.Short() {
t.Skip("skipping test in short mode.")
Expand All @@ -65,13 +76,6 @@ func TestNPD(t *testing.T) {
panic(fmt.Sprintf("Unable to create gcloud compute service using defaults. Make sure you are authenticated. %v", err))
}

if *project == "" {
boskosClient := client.NewClient(*jobName, *boskosServerURL)
*project = acquireProjectOrDie(boskosClient)

defer releaseProjectOrDie(boskosClient)
}

if *artifactsDir != "" {
_, err := os.Stat(*artifactsDir)
if err != nil && os.IsNotExist(err) {
Expand All @@ -84,30 +88,72 @@ func TestNPD(t *testing.T) {
ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "NPD Metric-only Suite", []ginkgo.Reporter{junitReporter})
}

func acquireProjectOrDie(boskosClient *client.Client) string {
// rentBoskosProjectIfNeededOnNode1 rents a GCP project from Boskos if no GCP project is specified.
//
// rentBoskosProjectIfNeededOnNode1 returns a byte slice containing the project name.
// rentBoskosProjectIfNeededOnNode1 also initializes boskosClient if necessary.
// When the tests run in parallel mode in Ginkgo, this rentBoskosProjectIfNeededOnNode1 runs only on
// Ginkgo node 1. The output should be shared with all other Gingko nodes so that they all use the same
// GCP project.
func rentBoskosProjectIfNeededOnNode1() []byte {
if *project != "" {
return []byte{}
}

fmt.Printf("Renting project from Boskos\n")
boskosClient = client.NewClient(*jobName, *boskosServerURL)
boskosRenewingTomb = tomb.NewTomb()

ctx, cancel := context.WithTimeout(context.Background(), *boskosWaitDuration)
defer cancel()
p, err := boskosClient.AcquireWait(ctx, *boskosProjectType, "free", "busy")
Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Unable to rent project from Boskos: %v\n", err))
fmt.Printf("Rented project %q from Boskos\n", p.Name)

go renewBoskosProject(boskosClient, p.Name, boskosRenewingTomb)

fmt.Printf("Rented project %s from Boskos", p.Name)
return []byte(p.Name)
}

// acceptBoskosProjectIfNeededFromNode1 accepts a GCP project rented from Boskos by Ginkgo node 1.
//
// acceptBoskosProjectIfNeededFromNode1 takes the output of rentBoskosProjectIfNeededOnNode1.
// When the tests run in parallel mode in Ginkgo, this function runs on all Ginkgo nodes.
func acceptBoskosProjectIfNeededFromNode1(data []byte) {
if *project != "" {
return
}

boskosProject := string(data)
fmt.Printf("Received Boskos project %q from Ginkgo node 1.\n", boskosProject)
*project = boskosProject
}

go func(boskosClient *client.Client, projectName string) {
for range time.Tick(5 * time.Minute) {
func renewBoskosProject(boskosClient *client.Client, projectName string, boskosRenewingTomb *tomb.Tomb) {
defer boskosRenewingTomb.Done()
for {
select {
case <-time.Tick(5 * time.Minute):
fmt.Printf("Renewing boskosProject %q\n", projectName)
if err := boskosClient.UpdateOne(projectName, "busy", nil); err != nil {
fmt.Printf("Failed to update status for project %s with Boskos: %v\n", projectName, err)
fmt.Printf("Failed to update status for project %q with Boskos: %v\n", projectName, err)
}
case <-boskosRenewingTomb.Stopping():
return
}
}(boskosClient, p.Name)

return p.Name
}
}

func releaseProjectOrDie(boskosClient *client.Client) {
// releaseBoskosResourcesOnNode1 releases all rented Boskos resources if there is any.
func releaseBoskosResourcesOnNode1() {
if boskosClient == nil {
return
}
boskosRenewingTomb.Stop()
if !boskosClient.HasResource() {
return
}
fmt.Printf("Releasing all Boskos resources.\n")
err := boskosClient.ReleaseAll("dirty")
Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to release project to Boskos: %v", err))
}
Expand Down

0 comments on commit fb8304b

Please sign in to comment.