[Federation] Cleanup e2e framework

test/e2e_federation/BUILD

@@ -49,9 +49,7 @@ go_library(
         "//vendor:k8s.io/apimachinery/pkg/util/rand",
         "//vendor:k8s.io/apimachinery/pkg/util/uuid",
         "//vendor:k8s.io/apimachinery/pkg/util/wait",
-        "//vendor:k8s.io/client-go/rest",
         "//vendor:k8s.io/client-go/tools/clientcmd",
-        "//vendor:k8s.io/client-go/tools/clientcmd/api",
     ],
 )
 

test/e2e_federation/apiserver.go

@@ -18,14 +18,10 @@ package e2e_federation
 
 import (
 	"fmt"
-	"time"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/wait"
-	federationapi "k8s.io/kubernetes/federation/apis/federation/v1beta1"
-	"k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset"
 	"k8s.io/kubernetes/pkg/api/v1"
 	"k8s.io/kubernetes/test/e2e/framework"
 	fedframework "k8s.io/kubernetes/test/e2e_federation/framework"
@@ -61,7 +57,7 @@ var _ = framework.KubeDescribe("Federation apiserver [Feature:Federation]", func
 
 			framework.Logf("Checking that %d clusters are Ready", len(contexts))
 			for _, context := range contexts {
-				clusterIsReadyOrFail(f, &context)
+				fedframework.ClusterIsReadyOrFail(f, &context)
 			}
 			framework.Logf("%d clusters are Ready", len(contexts))
 
@@ -122,19 +118,3 @@ func newService(name, namespace string) *v1.Service {
 		},
 	}
 }
-
-// Verify that the cluster is marked ready.
-func isReady(clusterName string, clientset *federation_clientset.Clientset) error {
-	return wait.PollImmediate(time.Second, 5*time.Minute, func() (bool, error) {
-		c, err := clientset.Federation().Clusters().Get(clusterName, metav1.GetOptions{})
-		if err != nil {
-			return false, err
-		}
-		for _, condition := range c.Status.Conditions {
-			if condition.Type == federationapi.ClusterReady && condition.Status == v1.ConditionTrue {
-				return true, nil
-			}
-		}
-		return false, nil
-	})
-}

test/e2e_federation/daemonset.go

@@ -43,15 +43,15 @@ const (
 
 // Create/delete daemonset api objects
 var _ = framework.KubeDescribe("Federation daemonsets [Feature:Federation]", func() {
-	var clusters map[string]*cluster // All clusters, keyed by cluster name
+	var clusters fedframework.ClusterSlice
 
 	f := fedframework.NewDefaultFederatedFramework("federated-daemonset")
 
 	Describe("DaemonSet objects", func() {
 
 		BeforeEach(func() {
 			fedframework.SkipUnlessFederated(f.ClientSet)
-			clusters, _ = getRegisteredClusters(UserAgentName, f)
+			clusters = f.GetRegisteredClusters()
 		})
 
 		AfterEach(func() {
@@ -114,7 +114,7 @@ func deleteAllDaemonSetsOrFail(clientset *fedclientset.Clientset, nsName string)
 // verifyCascadingDeletionForDS verifies that daemonsets are deleted from
 // underlying clusters when orphan dependents is false and they are not
 // deleted when orphan dependents is true.
-func verifyCascadingDeletionForDS(clientset *fedclientset.Clientset, clusters map[string]*cluster, orphanDependents *bool, nsName string) {
+func verifyCascadingDeletionForDS(clientset *fedclientset.Clientset, clusters fedframework.ClusterSlice, orphanDependents *bool, nsName string) {
 	daemonset := createDaemonSetOrFail(clientset, nsName)
 	daemonsetName := daemonset.Name
 	// Check subclusters if the daemonset was created there.
@@ -140,8 +140,9 @@ func verifyCascadingDeletionForDS(clientset *fedclientset.Clientset, clusters ma
 	errMessages := []string{}
 	// daemon set should be present in underlying clusters unless orphanDependents is false.
 	shouldExist := orphanDependents == nil || *orphanDependents == true
-	for clusterName, clusterClientset := range clusters {
-		_, err := clusterClientset.Extensions().DaemonSets(nsName).Get(daemonsetName, metav1.GetOptions{})
+	for _, cluster := range clusters {
+		clusterName := cluster.Name
+		_, err := cluster.Extensions().DaemonSets(nsName).Get(daemonsetName, metav1.GetOptions{})
 		if shouldExist && errors.IsNotFound(err) {
 			errMessages = append(errMessages, fmt.Sprintf("unexpected NotFound error for daemonset %s in cluster %s, expected daemonset to exist", daemonsetName, clusterName))
 		} else if !shouldExist && !errors.IsNotFound(err) {
@@ -233,7 +234,7 @@ func updateDaemonSetOrFail(clientset *fedclientset.Clientset, namespace string)
 	return newDaemonSet
 }
 
-func waitForDaemonSetShardsOrFail(namespace string, daemonset *v1beta1.DaemonSet, clusters map[string]*cluster) {
+func waitForDaemonSetShardsOrFail(namespace string, daemonset *v1beta1.DaemonSet, clusters fedframework.ClusterSlice) {
 	framework.Logf("Waiting for daemonset %q in %d clusters", daemonset.Name, len(clusters))
 	for _, c := range clusters {
 		waitForDaemonSetOrFail(c.Clientset, namespace, daemonset, true, FederatedDaemonSetTimeout)
@@ -263,7 +264,7 @@ func waitForDaemonSetOrFail(clientset *kubeclientset.Clientset, namespace string
 	}
 }
 
-func waitForDaemonSetShardsUpdatedOrFail(namespace string, daemonset *v1beta1.DaemonSet, clusters map[string]*cluster) {
+func waitForDaemonSetShardsUpdatedOrFail(namespace string, daemonset *v1beta1.DaemonSet, clusters fedframework.ClusterSlice) {
 	framework.Logf("Waiting for daemonset %q in %d clusters", daemonset.Name, len(clusters))
 	for _, c := range clusters {
 		waitForDaemonSetUpdateOrFail(c.Clientset, namespace, daemonset, FederatedDaemonSetTimeout)

test/e2e_federation/deployment.go

@@ -69,11 +69,11 @@ var _ = framework.KubeDescribe("Federation deployments [Feature:Federation]", fu
 	// e2e cases for federated deployment controller
 	Describe("Federated Deployment", func() {
 		var (
-			clusters map[string]*cluster
+			clusters fedframework.ClusterSlice
 		)
 		BeforeEach(func() {
 			fedframework.SkipUnlessFederated(f.ClientSet)
-			clusters, _ = getRegisteredClusters(UserAgentName, f)
+			clusters = f.GetRegisteredClusters()
 		})
 
 		AfterEach(func() {
@@ -140,7 +140,7 @@ func deleteAllDeploymentsOrFail(clientset *fedclientset.Clientset, nsName string
 // verifyCascadingDeletionForDeployment verifies that deployments are deleted
 // from underlying clusters when orphan dependents is false and they are not
 // deleted when orphan dependents is true.
-func verifyCascadingDeletionForDeployment(clientset *fedclientset.Clientset, clusters map[string]*cluster, orphanDependents *bool, nsName string) {
+func verifyCascadingDeletionForDeployment(clientset *fedclientset.Clientset, clusters fedframework.ClusterSlice, orphanDependents *bool, nsName string) {
 	deployment := createDeploymentOrFail(clientset, nsName)
 	deploymentName := deployment.Name
 	// Check subclusters if the deployment was created there.
@@ -166,8 +166,9 @@ func verifyCascadingDeletionForDeployment(clientset *fedclientset.Clientset, clu
 	errMessages := []string{}
 	// deployment should be present in underlying clusters unless orphanDependents is false.
 	shouldExist := orphanDependents == nil || *orphanDependents == true
-	for clusterName, clusterClientset := range clusters {
-		_, err := clusterClientset.Extensions().Deployments(nsName).Get(deploymentName, metav1.GetOptions{})
+	for _, cluster := range clusters {
+		clusterName := cluster.Name
+		_, err := cluster.Extensions().Deployments(nsName).Get(deploymentName, metav1.GetOptions{})
 		if shouldExist && errors.IsNotFound(err) {
 			errMessages = append(errMessages, fmt.Sprintf("unexpected NotFound error for deployment %s in cluster %s, expected deployment to exist", deploymentName, clusterName))
 		} else if !shouldExist && !errors.IsNotFound(err) {
@@ -179,13 +180,13 @@ func verifyCascadingDeletionForDeployment(clientset *fedclientset.Clientset, clu
 	}
 }
 
-func waitForDeploymentOrFail(c *fedclientset.Clientset, namespace string, deploymentName string, clusters map[string]*cluster) {
+func waitForDeploymentOrFail(c *fedclientset.Clientset, namespace string, deploymentName string, clusters fedframework.ClusterSlice) {
 	err := waitForDeployment(c, namespace, deploymentName, clusters)
 	framework.ExpectNoError(err, "Failed to verify deployment %q/%q, err: %v", namespace, deploymentName, err)
 }
 
-func waitForDeployment(c *fedclientset.Clientset, namespace string, deploymentName string, clusters map[string]*cluster) error {
-	err := wait.Poll(10*time.Second, federatedDefaultTestTimeout, func() (bool, error) {
+func waitForDeployment(c *fedclientset.Clientset, namespace string, deploymentName string, clusters fedframework.ClusterSlice) error {
+	err := wait.Poll(10*time.Second, fedframework.FederatedDefaultTestTimeout, func() (bool, error) {
 		fdep, err := c.Deployments(namespace).Get(deploymentName, metav1.GetOptions{})
 		if err != nil {
 			return false, err
@@ -194,12 +195,12 @@ func waitForDeployment(c *fedclientset.Clientset, namespace string, deploymentNa
 		for _, cluster := range clusters {
 			dep, err := cluster.Extensions().Deployments(namespace).Get(deploymentName, metav1.GetOptions{})
 			if err != nil && !errors.IsNotFound(err) {
-				By(fmt.Sprintf("Failed getting deployment: %q/%q/%q, err: %v", cluster.name, namespace, deploymentName, err))
+				By(fmt.Sprintf("Failed getting deployment: %q/%q/%q, err: %v", cluster.Name, namespace, deploymentName, err))
 				return false, err
 			}
 			if err == nil {
 				if !verifyDeployment(fdep, dep) {
-					By(fmt.Sprintf("Deployment meta or spec not match for cluster %q:\n    federation: %v\n    cluster: %v", cluster.name, fdep, dep))
+					By(fmt.Sprintf("Deployment meta or spec not match for cluster %q:\n    federation: %v\n    cluster: %v", cluster.Name, fdep, dep))
 					return false, nil
 				}
 				specReplicas += *dep.Spec.Replicas
@@ -259,7 +260,7 @@ func deleteDeploymentOrFail(clientset *fedclientset.Clientset, nsName string, de
 	}
 
 	// Wait for the deployment to be deleted.
-	err = wait.Poll(10*time.Second, federatedDefaultTestTimeout, func() (bool, error) {
+	err = wait.Poll(10*time.Second, fedframework.FederatedDefaultTestTimeout, func() (bool, error) {
 		_, err := clientset.Extensions().Deployments(nsName).Get(deploymentName, metav1.GetOptions{})
 		if err != nil && errors.IsNotFound(err) {
 			return true, nil

test/e2e_federation/framework/BUILD

@@ -10,6 +10,7 @@ load(
 go_library(
     name = "go_default_library",
     srcs = [
+        "cluster.go",
         "framework.go",
         "util.go",
     ],
@@ -31,6 +32,7 @@ go_library(
         "//vendor:k8s.io/apimachinery/pkg/util/wait",
         "//vendor:k8s.io/client-go/rest",
         "//vendor:k8s.io/client-go/tools/clientcmd",
+        "//vendor:k8s.io/client-go/tools/clientcmd/api",
     ],
 )
 

test/e2e_federation/framework/cluster.go

@@ -0,0 +1,164 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"fmt"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+	federationapi "k8s.io/kubernetes/federation/apis/federation/v1beta1"
+	fedclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset"
+	"k8s.io/kubernetes/pkg/api/v1"
+	kubeclientset "k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+const (
+	KubeAPIQPS   float32 = 20.0
+	KubeAPIBurst         = 30
+
+	userAgentName = "federation-e2e"
+
+	federatedNamespaceTimeout    = 5 * time.Minute
+	federatedClustersWaitTimeout = 1 * time.Minute
+)
+
+// ClusterSlice is a slice of clusters
+type ClusterSlice []*Cluster
+
+// Cluster keeps track of the name and client of a cluster in the federation
+type Cluster struct {
+	Name string
+	*kubeclientset.Clientset
+}
+
+func getRegisteredClusters(f *Framework) ClusterSlice {
+	contexts := f.GetUnderlyingFederatedContexts()
+
+	By("Obtaining a list of all the clusters")
+	clusterList := waitForAllRegisteredClusters(f, len(contexts))
+
+	framework.Logf("Checking that %d clusters are Ready", len(contexts))
+	for _, context := range contexts {
+		ClusterIsReadyOrFail(f, &context)
+	}
+	framework.Logf("%d clusters are Ready", len(contexts))
+
+	clusters := ClusterSlice{}
+	for i, c := range clusterList.Items {
+		framework.Logf("Creating a clientset for the cluster %s", c.Name)
+		Expect(framework.TestContext.KubeConfig).ToNot(Equal(""), "KubeConfig must be specified to load clusters' client config")
+		clusters = append(clusters, &Cluster{
+			Name:      c.Name,
+			Clientset: createClientsetForCluster(c, i, userAgentName),
+		})
+	}
+	waitForNamespaceInFederatedClusters(clusters, f.FederationNamespace.Name, federatedNamespaceTimeout)
+	return clusters
+}
+
+// waitForAllRegisteredClusters waits for all clusters defined in e2e context to be created
+// return ClusterList until the listed cluster items equals clusterCount
+func waitForAllRegisteredClusters(f *Framework, clusterCount int) *federationapi.ClusterList {
+	var clusterList *federationapi.ClusterList
+	if err := wait.PollImmediate(framework.Poll, federatedClustersWaitTimeout, func() (bool, error) {
+		var err error
+		clusterList, err = f.FederationClientset.Federation().Clusters().List(metav1.ListOptions{})
+		if err != nil {
+			return false, err
+		}
+		framework.Logf("%d clusters registered, waiting for %d", len(clusterList.Items), clusterCount)
+		if len(clusterList.Items) == clusterCount {
+			return true, nil
+		}
+		return false, nil
+	}); err != nil {
+		framework.Failf("Failed to list registered clusters: %+v", err)
+	}
+	return clusterList
+}
+
+func createClientsetForCluster(c federationapi.Cluster, i int, userAgentName string) *kubeclientset.Clientset {
+	kubecfg, err := clientcmd.LoadFromFile(framework.TestContext.KubeConfig)
+	framework.ExpectNoError(err, "error loading KubeConfig: %v", err)
+
+	cfgOverride := &clientcmd.ConfigOverrides{
+		ClusterInfo: clientcmdapi.Cluster{
+			Server: c.Spec.ServerAddressByClientCIDRs[0].ServerAddress,
+		},
+	}
+	ccfg := clientcmd.NewNonInteractiveClientConfig(*kubecfg, c.Name, cfgOverride, clientcmd.NewDefaultClientConfigLoadingRules())
+	cfg, err := ccfg.ClientConfig()
+	framework.ExpectNoError(err, "Error creating client config in cluster #%d (%q)", i, c.Name)
+
+	cfg.QPS = KubeAPIQPS
+	cfg.Burst = KubeAPIBurst
+	return kubeclientset.NewForConfigOrDie(restclient.AddUserAgent(cfg, userAgentName))
+}
+
+// waitForNamespaceInFederatedClusters waits for the federated namespace to be created in federated clusters
+func waitForNamespaceInFederatedClusters(clusters ClusterSlice, nsName string, timeout time.Duration) {
+	for _, c := range clusters {
+		name := c.Name
+		err := wait.PollImmediate(framework.Poll, timeout, func() (bool, error) {
+			_, err := c.Clientset.Core().Namespaces().Get(nsName, metav1.GetOptions{})
+			if err != nil {
+				By(fmt.Sprintf("Waiting for namespace %q to be created in cluster %q, err: %v", nsName, name, err))
+				return false, nil
+			}
+			By(fmt.Sprintf("Namespace %q exists in cluster %q", nsName, name))
+			return true, nil
+		})
+		framework.ExpectNoError(err, "Failed to verify federated namespace %q creation in cluster %q", nsName, name)
+	}
+}
+
+// ClusterIsReadyOrFail checks whether the federated cluster of the provided context is ready
+func ClusterIsReadyOrFail(f *Framework, context *E2EContext) {
+	c, err := f.FederationClientset.Federation().Clusters().Get(context.Name, metav1.GetOptions{})
+	framework.ExpectNoError(err, fmt.Sprintf("get cluster: %+v", err))
+	if c.ObjectMeta.Name != context.Name {
+		framework.Failf("cluster name does not match input context: actual=%+v, expected=%+v", c, context)
+	}
+	err = isReady(context.Name, f.FederationClientset)
+	framework.ExpectNoError(err, fmt.Sprintf("unexpected error in verifying if cluster %s is ready: %+v", context.Name, err))
+	framework.Logf("Cluster %s is Ready", context.Name)
+}
+
+// Verify that the cluster is marked ready.
+func isReady(clusterName string, clientset *fedclientset.Clientset) error {
+	return wait.PollImmediate(time.Second, 5*time.Minute, func() (bool, error) {
+		c, err := clientset.Federation().Clusters().Get(clusterName, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		for _, condition := range c.Status.Conditions {
+			if condition.Type == federationapi.ClusterReady && condition.Status == v1.ConditionTrue {
+				return true, nil
+			}
+		}
+		return false, nil
+	})
+}

test/e2e_federation/framework/framework.go

@@ -231,3 +231,7 @@ func (f *Framework) GetUnderlyingFederatedContexts() []E2EContext {
 
 	return e2eContexts
 }
+
+func (f *Framework) GetRegisteredClusters() ClusterSlice {
+	return getRegisteredClusters(f)
+}

test/e2e_federation/framework/util.go

@@ -37,6 +37,8 @@ import (
 	"k8s.io/kubernetes/test/e2e/framework"
 )
 
+const FederatedDefaultTestTimeout = 5 * time.Minute
+
 // Detects whether the federation namespace exists in the underlying cluster
 func SkipUnlessFederated(c clientset.Interface) {
 	federationNS := framework.FederationSystemNamespace()