fix intermittent conflict errors in test suite

cloud/linode/loadbalancers.go

@@ -196,6 +196,7 @@ func (l *loadbalancers) EnsureLoadBalancer(ctx context.Context, clusterName stri
 	sentry.SetTag(ctx, "service", service.Name)
 
 	var nb *linodego.NodeBalancer
+	serviceNn := getServiceNn(service)
 
 	nb, err = l.getNodeBalancerForService(ctx, service)
 	switch err.(type) {
@@ -204,6 +205,7 @@ func (l *loadbalancers) EnsureLoadBalancer(ctx context.Context, clusterName stri
 			sentry.CaptureError(ctx, err)
 			return nil, err
 		}
+		klog.Infof("created new NodeBalancer (%d) for service (%s)", nb.ID, serviceNn)
 
 	case nil:
 		if err = l.updateNodeBalancer(ctx, service, nodes, nb); err != nil {
@@ -216,6 +218,7 @@ func (l *loadbalancers) EnsureLoadBalancer(ctx context.Context, clusterName stri
 		return nil, err
 	}
 
+	klog.Infof("NodeBalancer (%d) has been ensured for service (%s)", nb.ID, serviceNn)
 	lbStatus = makeLoadBalancerStatus(nb)
 
 	if !l.shouldPreserveNodeBalancer(service) {

e2e/test/ccm_e2e_test.go

@@ -128,6 +128,14 @@ var _ = Describe("e2e tests", func() {
 		}
 	}
 
+	var checkNumberOfWorkerNodes = func(numNodes int) {
+		Eventually(func() int {
+			workers, err = f.GetNodeList()
+			Expect(err).NotTo(HaveOccurred())
+			return len(workers)
+		}).Should(Equal(numNodes))
+	}
+
 	var checkNumberOfUpNodes = func(numNodes int) {
 		By("Checking the Number of Up Nodes")
 		Eventually(func() int {
@@ -325,6 +333,11 @@ var _ = Describe("e2e tests", func() {
 
 	FDescribe("Test", func() {
 		Context("LoadBalancer", func() {
+			AfterEach(func() {
+				err := root.Recycle()
+				Expect(err).NotTo(HaveOccurred())
+			})
+
 			Context("With single TLS port", func() {
 				var (
 					pods        []string
@@ -811,10 +824,6 @@ var _ = Describe("e2e tests", func() {
 				It("should use the specified NodeBalancer", func() {
 					By("Creating new NodeBalancer")
 					nbID := createNodeBalancer()
-					defer func() {
-						By("Deleting new NodeBalancer")
-						deleteNodeBalancer(nbID)
-					}()
 
 					By("Annotating service with new NodeBalancer ID")
 					annotations[annLinodeNodeBalancerID] = strconv.Itoa(nbID)
@@ -871,14 +880,14 @@ var _ = Describe("e2e tests", func() {
 				})
 
 				AfterEach(func() {
-					By("Deleting the NodeBalancer")
-					deleteNodeBalancer(nodeBalancerID)
-
 					By("Deleting the Pods")
 					deletePods(pods)
 
 					By("Deleting the Service")
 					deleteService()
+
+					err := root.Recycle()
+					Expect(err).NotTo(HaveOccurred())
 				})
 
 				It("should use the specified NodeBalancer", func() {
@@ -902,8 +911,6 @@ var _ = Describe("e2e tests", func() {
 
 					By("Checking old NodeBalancer was deleted")
 					checkNodeBalancerNotExists(nodeBalancerID)
-
-					nodeBalancerID = nbID
 				})
 			})
 
@@ -954,6 +961,9 @@ var _ = Describe("e2e tests", func() {
 				AfterEach(func() {
 					By("Deleting the NodeBalancer")
 					deleteNodeBalancer(nodeBalancerID)
+
+					err := root.Recycle()
+					Expect(err).NotTo(HaveOccurred())
 				})
 
 				It("should preserve the underlying nodebalancer after service deletion", func() {
@@ -1038,6 +1048,9 @@ var _ = Describe("e2e tests", func() {
 
 					By("Deleting the Newly Created Nodes")
 					deleteNewNode()
+
+					By("Waiting for the Node to be removed")
+					checkNumberOfWorkerNodes(2)
 				})
 
 				It("should reach the same pod every time it requests", func() {

e2e/test/framework/framework.go

@@ -1,6 +1,8 @@
 package framework
 
 import (
+	"fmt"
+
 	"github.com/appscode/go/crypto/rand"
 	"github.com/linode/linodego"
 	"k8s.io/client-go/kubernetes"
@@ -29,6 +31,10 @@ type Framework struct {
 	linodeClient linodego.Client
 }
 
+func generateNamespaceName() string {
+	return rand.WithUniqSuffix("ccm")
+}
+
 func New(
 	restConfig *rest.Config,
 	kubeClient kubernetes.Interface,
@@ -40,7 +46,7 @@ func New(
 		linodeClient: linodeClient,
 
 		name:      "cloud-controller-manager",
-		namespace: rand.WithUniqSuffix("ccm"),
+		namespace: generateNamespaceName(),
 	}
 }
 
@@ -55,6 +61,18 @@ func (f *Framework) Invoke() *Invocation {
 	}
 }
 
+func (f *Framework) Recycle() error {
+	if err := f.DeleteNamespace(); err != nil {
+		return fmt.Errorf("failed to delete namespace (%s)", f.namespace)
+	}
+
+	f.namespace = generateNamespaceName()
+	if err := f.CreateNamespace(); err != nil {
+		return fmt.Errorf("failed to create namespace (%s)", f.namespace)
+	}
+	return nil
+}
+
 type Invocation struct {
 	*rootInvocation
 	LoadBalancer *lbInvocation

e2e/test/framework/service.go

@@ -13,6 +13,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/util/retry"
 )
 
 func (i *lbInvocation) createOrUpdateService(selector, annotations map[string]string, ports []core.ServicePort, isSessionAffinityClientIP bool, isCreate bool) error {
@@ -44,15 +45,17 @@ func (i *lbInvocation) createOrUpdateService(selector, annotations map[string]st
 			return err
 		}
 	} else {
-		options := metav1.GetOptions{}
-		resource, err := service.Get(TestServerResourceName, options)
-		if err != nil {
+		if err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+			options := metav1.GetOptions{}
+			resource, err := service.Get(TestServerResourceName, options)
+			if err != nil {
+				return err
+			}
+			svc.ObjectMeta.ResourceVersion = resource.ResourceVersion
+			svc.Spec.ClusterIP = resource.Spec.ClusterIP
+			_, err = service.Update(svc)
 			return err
-		}
-		svc.ObjectMeta.ResourceVersion = resource.ResourceVersion
-		svc.Spec.ClusterIP = resource.Spec.ClusterIP
-		_, err = service.Update(svc)
-		if err != nil {
+		}); err != nil {
 			return err
 		}
 	}