Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rent Boskos project only once per test run. #405

Merged
merged 1 commit into from
Jan 7, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ test: vet fmt
GO111MODULE=on go test -mod vendor -timeout=1m -v -race -short -tags "$(BUILD_TAGS)" ./...

e2e-test: vet fmt build-tar
GO111MODULE=on ginkgo -nodes=$(PARALLEL) -mod vendor -timeout=10m -v -tags "$(BUILD_TAGS)" \
GO111MODULE=on ginkgo -nodes=$(PARALLEL) -mod vendor -timeout=10m -v -tags "$(BUILD_TAGS)" -stream \
./test/e2e/metriconly/... -- \
-project=$(PROJECT) -zone=$(ZONE) \
-image=$(VM_IMAGE) -image-family=$(IMAGE_FAMILY) -image-project=$(IMAGE_PROJECT) \
Expand Down
78 changes: 62 additions & 16 deletions test/e2e/metriconly/e2e_npd_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"testing"
"time"

"k8s.io/node-problem-detector/pkg/util/tomb"
"k8s.io/node-problem-detector/test/e2e/lib/gce"
"k8s.io/test-infra/boskos/client"

Expand Down Expand Up @@ -54,6 +55,16 @@ var boskosWaitDuration = flag.Duration("boskos-wait-duration", 2*time.Minute,

var computeService *compute.Service

// boskosClient helps renting project from Boskos, and is only initialized on Ginkgo node 1.
var boskosClient *client.Client

// boskosRenewingTomb stops the goroutine keep renewing the Boskos resources.
var boskosRenewingTomb *tomb.Tomb

// SynchronizedBeforeSuite and SynchronizedAfterSuite help manages singleton resource (a Boskos project) across Ginkgo nodes.
var _ = ginkgo.SynchronizedBeforeSuite(rentBoskosProjectIfNeededOnNode1, acceptBoskosProjectIfNeededFromNode1)
var _ = ginkgo.SynchronizedAfterSuite(func() {}, releaseBoskosResourcesOnNode1)

func TestNPD(t *testing.T) {
if testing.Short() {
t.Skip("skipping test in short mode.")
Expand All @@ -65,13 +76,6 @@ func TestNPD(t *testing.T) {
panic(fmt.Sprintf("Unable to create gcloud compute service using defaults. Make sure you are authenticated. %v", err))
}

if *project == "" {
boskosClient := client.NewClient(*jobName, *boskosServerURL)
*project = acquireProjectOrDie(boskosClient)

defer releaseProjectOrDie(boskosClient)
}

if *artifactsDir != "" {
_, err := os.Stat(*artifactsDir)
if err != nil && os.IsNotExist(err) {
Expand All @@ -84,30 +88,72 @@ func TestNPD(t *testing.T) {
ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "NPD Metric-only Suite", []ginkgo.Reporter{junitReporter})
}

func acquireProjectOrDie(boskosClient *client.Client) string {
// rentBoskosProjectIfNeededOnNode1 rents a GCP project from Boskos if no GCP project is specified.
//
// rentBoskosProjectIfNeededOnNode1 returns a byte slice containing the project name.
// rentBoskosProjectIfNeededOnNode1 also initializes boskosClient if necessary.
// When the tests run in parallel mode in Ginkgo, this rentBoskosProjectIfNeededOnNode1 runs only on
// Ginkgo node 1. The output should be shared with all other Gingko nodes so that they all use the same
// GCP project.
func rentBoskosProjectIfNeededOnNode1() []byte {
if *project != "" {
return []byte{}
}

fmt.Printf("Renting project from Boskos\n")
boskosClient = client.NewClient(*jobName, *boskosServerURL)
boskosRenewingTomb = tomb.NewTomb()

ctx, cancel := context.WithTimeout(context.Background(), *boskosWaitDuration)
defer cancel()
p, err := boskosClient.AcquireWait(ctx, *boskosProjectType, "free", "busy")
Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Unable to rent project from Boskos: %v\n", err))
fmt.Printf("Rented project %q from Boskos\n", p.Name)

go renewBoskosProject(boskosClient, p.Name, boskosRenewingTomb)

fmt.Printf("Rented project %s from Boskos", p.Name)
return []byte(p.Name)
}

// acceptBoskosProjectIfNeededFromNode1 accepts a GCP project rented from Boskos by Ginkgo node 1.
//
// acceptBoskosProjectIfNeededFromNode1 takes the output of rentBoskosProjectIfNeededOnNode1.
// When the tests run in parallel mode in Ginkgo, this function runs on all Ginkgo nodes.
func acceptBoskosProjectIfNeededFromNode1(data []byte) {
if *project != "" {
return
}

boskosProject := string(data)
fmt.Printf("Received Boskos project %q from Ginkgo node 1.\n", boskosProject)
*project = boskosProject
}

go func(boskosClient *client.Client, projectName string) {
for range time.Tick(5 * time.Minute) {
func renewBoskosProject(boskosClient *client.Client, projectName string, boskosRenewingTomb *tomb.Tomb) {
defer boskosRenewingTomb.Done()
for {
select {
case <-time.Tick(5 * time.Minute):
fmt.Printf("Renewing boskosProject %q\n", projectName)
if err := boskosClient.UpdateOne(projectName, "busy", nil); err != nil {
fmt.Printf("Failed to update status for project %s with Boskos: %v\n", projectName, err)
fmt.Printf("Failed to update status for project %q with Boskos: %v\n", projectName, err)
}
case <-boskosRenewingTomb.Stopping():
return
}
}(boskosClient, p.Name)

return p.Name
}
}

func releaseProjectOrDie(boskosClient *client.Client) {
// releaseBoskosResourcesOnNode1 releases all rented Boskos resources if there is any.
func releaseBoskosResourcesOnNode1() {
if boskosClient == nil {
return
}
boskosRenewingTomb.Stop()
if !boskosClient.HasResource() {
return
}
fmt.Printf("Releasing all Boskos resources.\n")
err := boskosClient.ReleaseAll("dirty")
Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to release project to Boskos: %v", err))
}
Expand Down