Allow setting backup regions and zones when creating GKE clusters

kubetest2-gke/deployer/build.go

@@ -57,7 +57,7 @@ func (d *deployer) Build() error {
 	version += "+" + d.commonOptions.RunID()
 	if d.BuildOptions.StageLocation != "" {
 		if err := d.BuildOptions.Stage(version); err != nil {
-			return fmt.Errorf("error staging build: %v", err)
+			return fmt.Errorf("error staging build: %w", err)
 		}
 	}
 	d.Version = version

kubetest2-gke/deployer/commandutils.go

@@ -17,8 +17,11 @@ limitations under the License.
 package deployer
 
 import (
+	"bytes"
 	"fmt"
+	"io"
 	"os"
+	osexec "os/exec"
 	"path/filepath"
 	"strings"
 
@@ -49,7 +52,7 @@ func (d *deployer) prepareGcpIfNeeded(projectID string) error {
 	}
 
 	if err := os.Setenv("CLOUDSDK_CORE_PRINT_UNHANDLED_TRACEBACKS", "1"); err != nil {
-		return fmt.Errorf("could not set CLOUDSDK_CORE_PRINT_UNHANDLED_TRACEBACKS=1: %v", err)
+		return fmt.Errorf("could not set CLOUDSDK_CORE_PRINT_UNHANDLED_TRACEBACKS=1: %w", err)
 	}
 	if err := os.Setenv("CLOUDSDK_API_ENDPOINT_OVERRIDES_CONTAINER", endpoint); err != nil {
 		return err
@@ -112,7 +115,7 @@ func getClusterCredentials(project, loc, cluster string) error {
 	if err := runWithOutput(exec.Command("gcloud",
 		containerArgs("clusters", "get-credentials", cluster, "--project="+project, loc)...),
 	); err != nil {
-		return fmt.Errorf("error executing get-credentials: %v", err)
+		return fmt.Errorf("error executing get-credentials: %w", err)
 	}
 
 	return nil
@@ -150,3 +153,36 @@ func resolveLatestVersionInChannel(loc, channelName string) (string, error) {
 
 	return "", fmt.Errorf("channel %q does not exist in the server config", channelName)
 }
+
+func containerArgs(args ...string) []string {
+	return append(append([]string{}, "container"), args...)
+}
+
+func runWithNoOutput(cmd exec.Cmd) error {
+	exec.NoOutput(cmd)
+	return cmd.Run()
+}
+
+func runWithOutput(cmd exec.Cmd) error {
+	exec.InheritOutput(cmd)
+	return cmd.Run()
+}
+
+func runWithOutputAndBuffer(cmd exec.Cmd) (string, error) {
+	var buf bytes.Buffer
+
+	exec.SetOutput(cmd, io.MultiWriter(os.Stdout, &buf), io.MultiWriter(os.Stderr, &buf))
+	if err := cmd.Run(); err != nil {
+		return "", err
+	}
+	return buf.String(), nil
+}
+
+// execError returns a string format of err including stderr if the
+// err is an ExitError, useful for errors from e.g. exec.Cmd.Output().
+func execError(err error) string {
+	if ee, ok := err.(*osexec.ExitError); ok {
+		return fmt.Sprintf("%v (output: %q)", err, string(ee.Stderr))
+	}
+	return err.Error()
+}

kubetest2-gke/deployer/common.go

@@ -18,15 +18,13 @@ package deployer
 
 import (
 	"fmt"
-	realexec "os/exec"
 	"strconv"
 	"strings"
 	"time"
 
 	"k8s.io/klog"
 
 	"sigs.k8s.io/kubetest2/pkg/boskos"
-	"sigs.k8s.io/kubetest2/pkg/exec"
 )
 
 const (
@@ -71,22 +69,6 @@ func (d *deployer) initialize() error {
 				klog.V(1).Infof("Got project %s from boskos", resource.Name)
 			}
 		}
-
-		// Multi-cluster name adjustment
-		numProjects := len(d.projects)
-		d.projectClustersLayout = make(map[string][]cluster, numProjects)
-		if numProjects > 1 {
-			if err := buildProjectClustersLayout(d.projects, d.clusters, d.projectClustersLayout); err != nil {
-				return fmt.Errorf("failed to build the project clusters layout: %v", err)
-			}
-		} else {
-			// Backwards compatible construction
-			clusters := make([]cluster, len(d.clusters))
-			for i, clusterName := range d.clusters {
-				clusters[i] = cluster{i, clusterName}
-			}
-			d.projectClustersLayout[d.projects[0]] = clusters
-		}
 	}
 
 	if d.commonOptions.ShouldDown() {
@@ -95,6 +77,22 @@ func (d *deployer) initialize() error {
 		}
 	}
 
+	// Multi-cluster name adjustment
+	numProjects := len(d.projects)
+	d.projectClustersLayout = make(map[string][]cluster, numProjects)
+	if numProjects > 1 {
+		if err := buildProjectClustersLayout(d.projects, d.clusters, d.projectClustersLayout); err != nil {
+			return fmt.Errorf("failed to build the project clusters layout: %w", err)
+		}
+	} else {
+		// Backwards compatible construction
+		clusters := make([]cluster, len(d.clusters))
+		for i, clusterName := range d.clusters {
+			clusters[i] = cluster{i, clusterName}
+		}
+		d.projectClustersLayout[d.projects[0]] = clusters
+	}
+
 	return nil
 }
 
@@ -107,7 +105,7 @@ func buildProjectClustersLayout(projects, clusters []string, projectClustersLayo
 		}
 		projectIndex, err := strconv.Atoi(parts[1])
 		if err != nil {
-			return fmt.Errorf("cluster name does not follow contain a valid project index (name:projectIndex. E.g: cluster:0): %v", err)
+			return fmt.Errorf("cluster name does not follow contain a valid project index (name:projectIndex. E.g: cluster:0): %w", err)
 		}
 		if projectIndex >= len(projects) {
 			return fmt.Errorf("project index %d specified in the cluster name should be smaller than the number of projects %d", projectIndex, len(projects))
@@ -117,25 +115,18 @@ func buildProjectClustersLayout(projects, clusters []string, projectClustersLayo
 	return nil
 }
 
-func containerArgs(args ...string) []string {
-	return append(append([]string{}, "container"), args...)
-}
-
-func runWithNoOutput(cmd exec.Cmd) error {
-	exec.NoOutput(cmd)
-	return cmd.Run()
-}
-
-func runWithOutput(cmd exec.Cmd) error {
-	exec.InheritOutput(cmd)
-	return cmd.Run()
+// locationFlag returns the location flags for gcloud commands.
+func locationFlag(region, zone string) string {
+	if zone != "" {
+		return "--zone=" + zone
+	}
+	return "--region=" + region
 }
 
-// execError returns a string format of err including stderr if the
-// err is an ExitError, useful for errors from e.g. exec.Cmd.Output().
-func execError(err error) string {
-	if ee, ok := err.(*realexec.ExitError); ok {
-		return fmt.Sprintf("%v (output: %q)", err, string(ee.Stderr))
+// locationPath returns the location paths in the gcloud service requests.
+func locationPath(region, zone string) string {
+	if zone != "" {
+		return "zones/" + zone
 	}
-	return err.Error()
+	return "regions/" + region
 }

kubetest2-gke/deployer/deployer.go

@@ -99,10 +99,12 @@ type deployer struct {
 	// doInit helps to make sure the initialization is performed only once
 	doInit sync.Once
 	// gke specific details
-	projects []string
-	zone     string
-	region   string
-	clusters []string
+	projects      []string
+	zone          string
+	backupZones   []string
+	region        string
+	backupRegions []string
+	clusters      []string
 	// only used for multi-project multi-cluster profile to save the project-clusters mapping
 	projectClustersLayout map[string][]cluster
 	nodes                 int
@@ -191,15 +193,15 @@ func (d *deployer) verifyLocationFlags() error {
 	} else if d.zone != "" && d.region != "" {
 		return fmt.Errorf("--zone and --region cannot both be set")
 	}
-	return nil
-}
 
-// location returns the location flags for gcloud commands.
-func location(region, zone string) string {
-	if zone != "" {
-		return "--zone=" + zone
+	if d.zone != "" && len(d.backupRegions) != 0 {
+		return fmt.Errorf("--zone and --backup-regions cannot both be set")
+	}
+	if d.region != "" && len(d.backupZones) != 0 {
+		return fmt.Errorf("--region and --backup-zones cannot both be set")
 	}
-	return "--region=" + region
+
+	return nil
 }
 
 func bindFlags(d *deployer) *pflag.FlagSet {
@@ -219,7 +221,9 @@ func bindFlags(d *deployer) *pflag.FlagSet {
 	flags.StringVar(&d.environment, "environment", "prod", "Container API endpoint to use, one of 'test', 'staging', 'prod', or a custom https:// URL. Defaults to prod if not provided")
 	flags.StringSliceVar(&d.projects, "project", []string{}, "Comma separated list of GCP Project(s) to use for creating the cluster.")
 	flags.StringVar(&d.region, "region", "", "For use with gcloud commands to specify the cluster region.")
+	flags.StringSliceVar(&d.backupRegions, "backup-regions", []string{}, "The backup regions to retry the cluster creation when there are unrecoverable errors.")
 	flags.StringVar(&d.zone, "zone", "", "For use with gcloud commands to specify the cluster zone.")
+	flags.StringSliceVar(&d.backupZones, "backup-zones", []string{}, "The backup zones to retry the cluster creation when there are unrecoverable errors.")
 	flags.IntVar(&d.nodes, "num-nodes", defaultNodePool.Nodes, "For use with gcloud commands to specify the number of nodes for the cluster.")
 	flags.StringVar(&d.machineType, "machine-type", defaultNodePool.MachineType, "For use with gcloud commands to specify the machine type for the cluster.")
 	flags.BoolVar(&d.gcpSSHKeyIgnored, "ignore-gcp-ssh-key", false, "Whether the GCP SSH key should be ignored or not for bringing up the cluster.")

kubetest2-gke/deployer/down.go

@@ -39,7 +39,7 @@ func (d *deployer) Down() error {
 			project := d.projects[i]
 			for j := range d.projectClustersLayout[project] {
 				cluster := d.projectClustersLayout[project][j]
-				loc := location(d.region, d.zone)
+				loc := locationFlag(d.region, d.zone)
 
 				wg.Add(1)
 				go func() {

kubetest2-gke/deployer/firewall.go

@@ -73,7 +73,7 @@ func ensureFirewallRulesForSingleProject(project, network string, clusters []clu
 			"--network="+network,
 			"--allow="+e2eAllow,
 			"--target-tags="+tag)); err != nil {
-			return fmt.Errorf("error creating e2e firewall rule: %v", err)
+			return fmt.Errorf("error creating e2e firewall rule: %w", err)
 		}
 	}
 	return nil
@@ -111,7 +111,7 @@ func ensureFirewallRulesForMultiProjects(projects []string, network string, subn
 			"--allow=tcp,udp,icmp",
 			"--direction=INGRESS",
 			"--source-ranges="+sourceRanges)); err != nil {
-			return fmt.Errorf("error creating firewall rule for project %q: %v", curtProject, err)
+			return fmt.Errorf("error creating firewall rule for project %q: %w", curtProject, err)
 		}
 	}
 	return nil
@@ -140,7 +140,7 @@ func (d *deployer) cleanupNetworkFirewalls(hostProject, network string) (int, er
 		commandArgs = append(commandArgs, "--project="+hostProject)
 		errFirewall := runWithOutput(exec.Command("gcloud", commandArgs...))
 		if errFirewall != nil {
-			return 0, fmt.Errorf("error deleting firewall: %v", errFirewall)
+			return 0, fmt.Errorf("error deleting firewall: %w", errFirewall)
 		}
 		// It looks sometimes gcloud exits before the firewall rules are actually deleted,
 		// so sleep 30 seconds to wait for the firewall rules being deleted completely.
@@ -159,7 +159,7 @@ func (d *deployer) getInstanceGroups() error {
 	// Initialize project instance groups structure
 	d.instanceGroups = map[string]map[string][]*ig{}
 
-	location := location(d.region, d.zone)
+	location := locationFlag(d.region, d.zone)
 
 	for _, project := range d.projects {
 		d.instanceGroups[project] = map[string][]*ig{}

kubetest2-gke/deployer/network.go

@@ -113,7 +113,7 @@ func (d *deployer) createNetwork() error {
 		if err := runWithOutput(exec.Command("gcloud", "compute", "networks", "subnets", "create",
 			subnetName,
 			"--project="+hostProject,
-			"--region="+d.region,
+			locationFlag(d.region, d.zone),
 			"--network="+d.network,
 			"--range="+parts[0],
 			"--secondary-range",
@@ -142,7 +142,7 @@ func (d *deployer) deleteNetwork() error {
 			if err := runWithOutput(exec.Command("gcloud", "compute", "networks", "subnets", "delete",
 				subnetName,
 				"--project="+hostProject,
-				"--region="+d.region,
+				locationFlag(d.region, d.zone),
 				"--quiet",
 			)); err != nil {
 				return err
@@ -169,7 +169,7 @@ func transformNetworkName(projects []string, network string) string {
 
 // Returns the sub network args needed for the cluster creation command.
 // Reference: https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-shared-vpc#creating_a_cluster_in_your_first_service_project
-func subNetworkArgs(projects []string, region, network string, projectIndex int) []string {
+func subNetworkArgs(projects []string, region, zone, network string, projectIndex int) []string {
 	// No sub network args need to be added for creating clusters in the host project.
 	if projectIndex == 0 {
 		return []string{}
@@ -181,14 +181,14 @@ func subNetworkArgs(projects []string, region, network string, projectIndex int)
 	subnetName := network + "-" + curtProject
 	return []string{
 		"--enable-ip-alias",
-		fmt.Sprintf("--subnetwork=projects/%s/regions/%s/subnetworks/%s", hostProject, region, subnetName),
+		fmt.Sprintf("--subnetwork=projects/%s/%s/subnetworks/%s", hostProject, locationPath(region, zone), subnetName),
 		fmt.Sprintf("--cluster-secondary-range-name=%s-pods", subnetName),
 		fmt.Sprintf("--services-secondary-range-name=%s-services", subnetName),
 	}
 }
 
 func (d *deployer) setupNetwork() error {
-	if err := enableSharedVPCAndGrantRoles(d.projects, d.region, d.network); err != nil {
+	if err := enableSharedVPCAndGrantRoles(d.projects, d.region, d.zone, d.network); err != nil {
 		return err
 	}
 	if err := grantHostServiceAgentUserRole(d.projects); err != nil {
@@ -199,7 +199,7 @@ func (d *deployer) setupNetwork() error {
 
 // This function implements https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-shared-vpc#enabling_and_granting_roles
 // to enable shared VPC and grant required roles for the multi-project multi-cluster profile.
-func enableSharedVPCAndGrantRoles(projects []string, region, network string) error {
+func enableSharedVPCAndGrantRoles(projects []string, region, zone, network string) error {
 	// Nothing needs to be done for single project.
 	if len(projects) == 1 {
 		return nil
@@ -237,30 +237,30 @@ func enableSharedVPCAndGrantRoles(projects []string, region, network string) err
 		subnetName := network + "-" + serviceProject
 		// Get the subnet etag.
 		subnetETag, err := exec.Output(exec.Command("gcloud", "compute", "networks", "subnets",
-			"get-iam-policy", subnetName, "--project="+networkHostProject, "--region="+region, "--format=value(etag)"))
+			"get-iam-policy", subnetName, "--project="+networkHostProject, locationFlag(region, zone), "--format=value(etag)"))
 		if err != nil {
-			return fmt.Errorf("failed to get the etag for the subnet: %s %s %v", network, region, err)
+			return fmt.Errorf("failed to get the etag for the subnet: %s %w", network, err)
 		}
 		// Get the service project number.
 		serviceProjectNum, err := getProjectNumber(serviceProject)
 		if err != nil {
-			return fmt.Errorf("failed to get the project number for %s: %v", serviceProject, err)
+			return fmt.Errorf("failed to get the project number for %s: %w", serviceProject, err)
 		}
 		gkeServiceAccount := fmt.Sprintf("service-%s@container-engine-robot.iam.gserviceaccount.com", serviceProjectNum)
 		googleAPIServiceAccount := serviceProjectNum + "@cloudservices.gserviceaccount.com"
 
 		// Grant the required IAM roles to service accounts that belong to the service project.
 		tempFile, err := ioutil.TempFile("", "*.yaml")
 		if err != nil {
-			return fmt.Errorf("failed to create a temporary yaml file: %v", err)
+			return fmt.Errorf("failed to create a temporary yaml file: %w", err)
 		}
 		policyStr := fmt.Sprintf(networkUserPolicyTemplate, googleAPIServiceAccount, gkeServiceAccount, strings.TrimSpace(string(subnetETag)))
 		if err = ioutil.WriteFile(tempFile.Name(), []byte(policyStr), os.ModePerm); err != nil {
-			return fmt.Errorf("failed to write the content into %s: %v", tempFile.Name(), err)
+			return fmt.Errorf("failed to write the content into %s: %w", tempFile.Name(), err)
 		}
 		if err = runWithOutput(exec.Command("gcloud", "compute", "networks", "subnets", "set-iam-policy", subnetName,
-			tempFile.Name(), "--project="+networkHostProject, "--region="+region)); err != nil {
-			return fmt.Errorf("failed to set IAM policy: %v", err)
+			tempFile.Name(), "--project="+networkHostProject, locationFlag(region, zone))); err != nil {
+			return fmt.Errorf("failed to set IAM policy: %w", err)
 		}
 	}
 

kubetest2-gke/deployer/options/build.go

@@ -51,7 +51,7 @@ func (bo *BuildOptions) implementationFromStrategy() error {
 			ImageLocation: "",
 		}
 	default:
-		return fmt.Errorf("unknown build strategy: %v", bo.Strategy)
+		return fmt.Errorf("unknown build strategy: %q", bo.Strategy)
 	}
 	return nil
 }

kubetest2-gke/deployer/options/up.go

@@ -25,7 +25,7 @@ type UpOptions struct {
 func (uo *UpOptions) Validate() error {
 	// allow max 99 clusters (should be sufficient for most use cases)
 	if uo.NumClusters < 1 || uo.NumClusters > 99 {
-		return fmt.Errorf("need to specify between 1 and 99 clusters got %q: ", uo.NumClusters)
+		return fmt.Errorf("need to specify between 1 and 99 clusters got %q", uo.NumClusters)
 	}
 	return nil
 }