diff --git a/tests/e2e/kubetest2-kops/aws/zones.go b/tests/e2e/kubetest2-kops/aws/zones.go new file mode 100644 index 0000000000000..ed740433a9977 --- /dev/null +++ b/tests/e2e/kubetest2-kops/aws/zones.go @@ -0,0 +1,97 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aws + +import ( + "errors" + "math/rand" + "sort" +) + +var allZones = []string{ + "ap-northeast-1a", + "ap-northeast-1c", + "ap-northeast-1d", + "ap-northeast-2a", + //"ap-northeast-2b" - AZ does not exist, so we"re breaking the 3 AZs per region target here + "ap-northeast-2c", + "ap-south-1a", + "ap-south-1b", + "ap-southeast-1a", + "ap-southeast-1b", + "ap-southeast-1c", + "ap-southeast-2a", + "ap-southeast-2b", + "ap-southeast-2c", + "eu-central-1a", + "eu-central-1b", + "eu-central-1c", + "eu-west-1a", + "eu-west-1b", + "eu-west-1c", + "eu-west-2a", + "eu-west-2b", + "eu-west-2c", + //"eu-west-3a", documented to not support c4 family + //"eu-west-3b", documented to not support c4 family + //"eu-west-3c", documented to not support c4 family + //"us-east-1a", // temporarily removing due to lack of quota test-infra#10043 + //"us-east-1b", // temporarily removing due to lack of quota test-infra#10043 + //"us-east-1c", // temporarily removing due to lack of quota test-infra#10043 + //"us-east-1d", // limiting to 3 zones to not overallocate + //"us-east-1e", // limiting to 3 zones to not overallocate + //"us-east-1f", // limiting to 3 zones to not overallocate + //"us-east-2a", InsufficientInstanceCapacity for c4.large 2018-05-30 + //"us-east-2b", InsufficientInstanceCapacity for c4.large 2018-05-30 + //"us-east-2c", InsufficientInstanceCapacity for c4.large 2018-05-30 + "us-west-1a", + "us-west-1b", + //"us-west-1c", AZ does not exist, so we"re breaking the 3 AZs per region target here + //"us-west-2a", // temporarily removing due to lack of quota test-infra#10043 + //"us-west-2b", // temporarily removing due to lack of quota test-infra#10043 + //"us-west-2c", // temporarily removing due to lack of quota test-infra#10043 +} + +// ErrNoEligibleRegion indicates the requested number of zones is not available in any region +var ErrNoEligibleRegion = errors.New("No eligible AWS region found with enough zones") + +// RandomZones returns a random set of availability zones within a region +func RandomZones(count int) ([]string, error) { + regions := make(map[string][]string) + for _, zone := range allZones { + region := zone[:len(zone)-1] + regions[region] = append(regions[region], zone) + } + eligibleRegions := make([][]string, 0) + for _, zones := range regions { + if len(zones) >= count { + eligibleRegions = append(eligibleRegions, zones) + } + } + if len(eligibleRegions) == 0 { + return nil, ErrNoEligibleRegion + } + chosenRegion := eligibleRegions[rand.Int()%len(eligibleRegions)] + + chosenZones := make([]string, 0) + randIndexes := rand.Perm(len(chosenRegion)) + for i := 0; i < count; i++ { + chosenZones = append(chosenZones, chosenRegion[randIndexes[i]]) + } + sort.Strings(chosenZones) + return chosenZones, nil +} diff --git a/tests/e2e/kubetest2-kops/aws/zones_test.go b/tests/e2e/kubetest2-kops/aws/zones_test.go new file mode 100644 index 0000000000000..93e58fcabfd96 --- /dev/null +++ b/tests/e2e/kubetest2-kops/aws/zones_test.go @@ -0,0 +1,47 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aws + +import ( + "fmt" + "testing" +) + +func TestRandomZones(t *testing.T) { + t.Parallel() // marks TLog as capable of running in parallel with other tests + tests := []struct { + count int + err error + }{ + {1, nil}, + {2, nil}, + {3, nil}, + {4, ErrNoEligibleRegion}, + } + for _, tt := range tests { + t.Run(fmt.Sprintf("%v zones", tt.count), func(t *testing.T) { + zones, err := RandomZones(tt.count) + if err != tt.err { + t.Errorf("unexpected error response: %v vs %v. zones: %v", err, tt.err, zones) + t.Fail() + } else if tt.err == nil && len(zones) != tt.count { + t.Errorf("Unexpected number of zones returned: %v vs %v. zones: %v", len(zones), tt.count, zones) + t.Fail() + } + }) + } +} diff --git a/tests/e2e/kubetest2-kops/deployer/build.go b/tests/e2e/kubetest2-kops/deployer/build.go index 7d5bfa5b21258..5e15433167c13 100644 --- a/tests/e2e/kubetest2-kops/deployer/build.go +++ b/tests/e2e/kubetest2-kops/deployer/build.go @@ -18,10 +18,17 @@ package deployer import ( "errors" + "fmt" "os" + "path" "strings" ) +const ( + defaultJobName = "pull-kops-e2e-kubernetes-aws" + defaultGCSPath = "gs://kops-ci/pulls/%v" +) + func (d *deployer) Build() error { if err := d.init(); err != nil { return err @@ -34,10 +41,22 @@ func (d *deployer) Build() error { func (d *deployer) verifyBuildFlags() error { if d.KopsRoot == "" { - return errors.New("required kops-root when building from source") + if goPath := os.Getenv("GOPATH"); goPath != "" { + d.KopsRoot = path.Join(goPath, "src", "k8s.io", "kops") + } else { + return errors.New("required kops-root when building from source") + } } - if !strings.HasPrefix(d.StageLocation, "gs://") { - return errors.New("stage-location must be a gs:// path") + if d.StageLocation != "" { + if !strings.HasPrefix(d.StageLocation, "gs://") { + return errors.New("stage-location must be a gs:// path") + } + } else { + jobName := os.Getenv("JOB_NAME") + if jobName == "" { + jobName = defaultJobName + } + d.StageLocation = fmt.Sprintf(defaultGCSPath, jobName) } fi, err := os.Stat(d.KopsRoot) if err != nil { diff --git a/tests/e2e/kubetest2-kops/deployer/common.go b/tests/e2e/kubetest2-kops/deployer/common.go index 32551f4597c88..c7b7e825bc36d 100644 --- a/tests/e2e/kubetest2-kops/deployer/common.go +++ b/tests/e2e/kubetest2-kops/deployer/common.go @@ -70,10 +70,6 @@ func (d *deployer) verifyKopsFlags() error { return errors.New("missing required --kops-binary-path") } } - _, err := os.Stat(d.KopsBinaryPath) - if err != nil { - return err - } switch d.CloudProvider { case "aws": @@ -113,8 +109,8 @@ func defaultClusterName(cloudProvider string) (string, error) { return "", errors.New("JOB_NAME, and BUILD_ID env vars are required when --cluster-name is not set") } - buildIDHash := md5.Sum([]byte(buildID)) - jobHash := md5.Sum([]byte(jobName)) + buildIDHash := fmt.Sprintf("%x", md5.Sum([]byte(buildID))) + jobHash := fmt.Sprintf("%x", md5.Sum([]byte(jobName))) var suffix string switch cloudProvider { diff --git a/tests/e2e/kubetest2-kops/deployer/up.go b/tests/e2e/kubetest2-kops/deployer/up.go index b36e5ac6fc525..28798ec825275 100644 --- a/tests/e2e/kubetest2-kops/deployer/up.go +++ b/tests/e2e/kubetest2-kops/deployer/up.go @@ -22,6 +22,7 @@ import ( "strings" "k8s.io/klog/v2" + "k8s.io/kops/tests/e2e/kubetest2-kops/aws" "k8s.io/kops/tests/e2e/kubetest2-kops/util" "sigs.k8s.io/kubetest2/pkg/exec" ) @@ -35,6 +36,12 @@ func (d *deployer) Up() error { if err != nil { return err } + + zones, err := aws.RandomZones(1) + if err != nil { + return err + } + args := []string{ d.KopsBinaryPath, "create", "cluster", "--name", d.ClusterName, @@ -47,7 +54,7 @@ func (d *deployer) Up() error { "--node-volume-size", "48", "--override", "cluster.spec.nodePortAccess=0.0.0.0/0", "--ssh-public-key", d.SSHPublicKeyPath, - "--zones", "eu-west-2a", + "--zones", strings.Join(zones, ","), "--yes", } klog.Info(strings.Join(args, " ")) @@ -55,7 +62,21 @@ func (d *deployer) Up() error { cmd.SetEnv(d.env()...) exec.InheritOutput(cmd) - return cmd.Run() + err = cmd.Run() + if err != nil { + return err + } + + isUp, err := d.IsUp() + if err != nil { + return err + } else if isUp { + klog.V(1).Infof("cluster reported as up") + } else { + klog.Errorf("cluster reported as down") + } + + return nil } func (d *deployer) IsUp() (bool, error) {