Merge pull request #10121 from rifelpet/kubetest2-3

Add random AWS zone logic + specify build stage location

tests/e2e/kubetest2-kops/aws/zones.go

@@ -0,0 +1,97 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import (
+	"errors"
+	"math/rand"
+	"sort"
+)
+
+var allZones = []string{
+	"ap-northeast-1a",
+	"ap-northeast-1c",
+	"ap-northeast-1d",
+	"ap-northeast-2a",
+	//"ap-northeast-2b" - AZ does not exist, so we"re breaking the 3 AZs per region target here
+	"ap-northeast-2c",
+	"ap-south-1a",
+	"ap-south-1b",
+	"ap-southeast-1a",
+	"ap-southeast-1b",
+	"ap-southeast-1c",
+	"ap-southeast-2a",
+	"ap-southeast-2b",
+	"ap-southeast-2c",
+	"eu-central-1a",
+	"eu-central-1b",
+	"eu-central-1c",
+	"eu-west-1a",
+	"eu-west-1b",
+	"eu-west-1c",
+	"eu-west-2a",
+	"eu-west-2b",
+	"eu-west-2c",
+	//"eu-west-3a", documented to not support c4 family
+	//"eu-west-3b", documented to not support c4 family
+	//"eu-west-3c", documented to not support c4 family
+	//"us-east-1a", // temporarily removing due to lack of quota test-infra#10043
+	//"us-east-1b", // temporarily removing due to lack of quota test-infra#10043
+	//"us-east-1c", // temporarily removing due to lack of quota test-infra#10043
+	//"us-east-1d", // limiting to 3 zones to not overallocate
+	//"us-east-1e", // limiting to 3 zones to not overallocate
+	//"us-east-1f", // limiting to 3 zones to not overallocate
+	//"us-east-2a", InsufficientInstanceCapacity for c4.large 2018-05-30
+	//"us-east-2b", InsufficientInstanceCapacity for c4.large 2018-05-30
+	//"us-east-2c", InsufficientInstanceCapacity for c4.large 2018-05-30
+	"us-west-1a",
+	"us-west-1b",
+	//"us-west-1c", AZ does not exist, so we"re breaking the 3 AZs per region target here
+	//"us-west-2a", // temporarily removing due to lack of quota test-infra#10043
+	//"us-west-2b", // temporarily removing due to lack of quota test-infra#10043
+	//"us-west-2c", // temporarily removing due to lack of quota test-infra#10043
+}
+
+// ErrNoEligibleRegion indicates the requested number of zones is not available in any region
+var ErrNoEligibleRegion = errors.New("No eligible AWS region found with enough zones")
+
+// RandomZones returns a random set of availability zones within a region
+func RandomZones(count int) ([]string, error) {
+	regions := make(map[string][]string)
+	for _, zone := range allZones {
+		region := zone[:len(zone)-1]
+		regions[region] = append(regions[region], zone)
+	}
+	eligibleRegions := make([][]string, 0)
+	for _, zones := range regions {
+		if len(zones) >= count {
+			eligibleRegions = append(eligibleRegions, zones)
+		}
+	}
+	if len(eligibleRegions) == 0 {
+		return nil, ErrNoEligibleRegion
+	}
+	chosenRegion := eligibleRegions[rand.Int()%len(eligibleRegions)]
+
+	chosenZones := make([]string, 0)
+	randIndexes := rand.Perm(len(chosenRegion))
+	for i := 0; i < count; i++ {
+		chosenZones = append(chosenZones, chosenRegion[randIndexes[i]])
+	}
+	sort.Strings(chosenZones)
+	return chosenZones, nil
+}

tests/e2e/kubetest2-kops/aws/zones_test.go

@@ -0,0 +1,47 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestRandomZones(t *testing.T) {
+	t.Parallel() // marks TLog as capable of running in parallel with other tests
+	tests := []struct {
+		count int
+		err   error
+	}{
+		{1, nil},
+		{2, nil},
+		{3, nil},
+		{4, ErrNoEligibleRegion},
+	}
+	for _, tt := range tests {
+		t.Run(fmt.Sprintf("%v zones", tt.count), func(t *testing.T) {
+			zones, err := RandomZones(tt.count)
+			if err != tt.err {
+				t.Errorf("unexpected error response: %v vs %v. zones: %v", err, tt.err, zones)
+				t.Fail()
+			} else if tt.err == nil && len(zones) != tt.count {
+				t.Errorf("Unexpected number of zones returned: %v vs %v. zones: %v", len(zones), tt.count, zones)
+				t.Fail()
+			}
+		})
+	}
+}

tests/e2e/kubetest2-kops/deployer/build.go

@@ -18,10 +18,17 @@ package deployer
 
 import (
 	"errors"
+	"fmt"
 	"os"
+	"path"
 	"strings"
 )
 
+const (
+	defaultJobName = "pull-kops-e2e-kubernetes-aws"
+	defaultGCSPath = "gs://kops-ci/pulls/%v"
+)
+
 func (d *deployer) Build() error {
 	if err := d.init(); err != nil {
 		return err
@@ -34,10 +41,22 @@ func (d *deployer) Build() error {
 
 func (d *deployer) verifyBuildFlags() error {
 	if d.KopsRoot == "" {
-		return errors.New("required kops-root when building from source")
+		if goPath := os.Getenv("GOPATH"); goPath != "" {
+			d.KopsRoot = path.Join(goPath, "src", "k8s.io", "kops")
+		} else {
+			return errors.New("required kops-root when building from source")
+		}
 	}
-	if !strings.HasPrefix(d.StageLocation, "gs://") {
-		return errors.New("stage-location must be a gs:// path")
+	if d.StageLocation != "" {
+		if !strings.HasPrefix(d.StageLocation, "gs://") {
+			return errors.New("stage-location must be a gs:// path")
+		}
+	} else {
+		jobName := os.Getenv("JOB_NAME")
+		if jobName == "" {
+			jobName = defaultJobName
+		}
+		d.StageLocation = fmt.Sprintf(defaultGCSPath, jobName)
 	}
 	fi, err := os.Stat(d.KopsRoot)
 	if err != nil {

tests/e2e/kubetest2-kops/deployer/common.go

@@ -70,10 +70,6 @@ func (d *deployer) verifyKopsFlags() error {
 			return errors.New("missing required --kops-binary-path")
 		}
 	}
-	_, err := os.Stat(d.KopsBinaryPath)
-	if err != nil {
-		return err
-	}
 
 	switch d.CloudProvider {
 	case "aws":
@@ -113,8 +109,8 @@ func defaultClusterName(cloudProvider string) (string, error) {
 		return "", errors.New("JOB_NAME, and BUILD_ID env vars are required when --cluster-name is not set")
 	}
 
-	buildIDHash := md5.Sum([]byte(buildID))
-	jobHash := md5.Sum([]byte(jobName))
+	buildIDHash := fmt.Sprintf("%x", md5.Sum([]byte(buildID)))
+	jobHash := fmt.Sprintf("%x", md5.Sum([]byte(jobName)))
 
 	var suffix string
 	switch cloudProvider {

tests/e2e/kubetest2-kops/deployer/up.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 
 	"k8s.io/klog/v2"
+	"k8s.io/kops/tests/e2e/kubetest2-kops/aws"
 	"k8s.io/kops/tests/e2e/kubetest2-kops/util"
 	"sigs.k8s.io/kubetest2/pkg/exec"
 )
@@ -35,6 +36,12 @@ func (d *deployer) Up() error {
 	if err != nil {
 		return err
 	}
+
+	zones, err := aws.RandomZones(1)
+	if err != nil {
+		return err
+	}
+
 	args := []string{
 		d.KopsBinaryPath, "create", "cluster",
 		"--name", d.ClusterName,
@@ -47,15 +54,29 @@ func (d *deployer) Up() error {
 		"--node-volume-size", "48",
 		"--override", "cluster.spec.nodePortAccess=0.0.0.0/0",
 		"--ssh-public-key", d.SSHPublicKeyPath,
-		"--zones", "eu-west-2a",
+		"--zones", strings.Join(zones, ","),
 		"--yes",
 	}
 	klog.Info(strings.Join(args, " "))
 	cmd := exec.Command(args[0], args[1:]...)
 	cmd.SetEnv(d.env()...)
 
 	exec.InheritOutput(cmd)
-	return cmd.Run()
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+
+	isUp, err := d.IsUp()
+	if err != nil {
+		return err
+	} else if isUp {
+		klog.V(1).Infof("cluster reported as up")
+	} else {
+		klog.Errorf("cluster reported as down")
+	}
+
+	return nil
 }
 
 func (d *deployer) IsUp() (bool, error) {