Scaling replica count for csi driver to verify volume operations gett…

…ing picked up after FULL SYNC

tests/e2e/e2e_common.go

@@ -26,6 +26,7 @@ import (
 
 const (
 	envSharedDatastoreURL                      = "SHARED_VSPHERE_DATASTORE_URL"
+	envMasterNodeIP                            = "K8S_MASTER_NODE_IP"
 	envNonSharedStorageClassDatastoreURL       = "NONSHARED_VSPHERE_DATASTORE_URL"
 	envInaccessibleZoneDatastoreURL            = "INACCESSIBLE_ZONE_VSPHERE_DATASTORE_URL"
 	scParamDatastoreURL                        = "DatastoreURL"
@@ -57,7 +58,7 @@ const (
 	invalidFSType                              = "ext10"
 	execCommand                                = "/bin/df -T /mnt/volume1 | /bin/awk 'FNR == 2 {print $2}' > /mnt/volume1/fstype && while true ; do sleep 2 ; done"
 	kubeSystemNamespace                        = "kube-system"
-	syncerStatefulsetName                      = "vsphere-csi-metadata-syncer"
+	vSphereCSIControllerPodNamePrefix          = "vsphere-csi-controller"
 )
 
 // GetAndExpectStringEnvVar parses a string from env variable

tests/e2e/fullsynctest.go

@@ -69,7 +69,7 @@ var _ bool = ginkgo.Describe("[csi-block-e2e] full-sync-test", func() {
 		fcdName                  = "BasicStaticFCD"
 		stopVsanHealthOperation  = "stop"
 		startVsanHealthOperation = "start"
-		vcenterPort              = "22"
+		sshdPort                 = "22"
 		numberOfPVC              = 5
 	)
 
@@ -140,7 +140,7 @@ var _ bool = ginkgo.Describe("[csi-block-e2e] full-sync-test", func() {
 		time.Sleep(time.Duration(pandoraSyncWaitTime) * time.Second)
 
 		ginkgo.By(fmt.Sprintln("Stopping vsan-health on the vCenter host"))
-		vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + vcenterPort
+		vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + sshdPort
 		err = invokeVCenterServiceControl(stopVsanHealthOperation, vsanhealthServiceName, vcAddress)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
@@ -187,7 +187,7 @@ var _ bool = ginkgo.Describe("[csi-block-e2e] full-sync-test", func() {
 		pv := pvs[0]
 
 		ginkgo.By(fmt.Sprintln("Stopping vsan-health on the vCenter host"))
-		vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + vcenterPort
+		vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + sshdPort
 		err = invokeVCenterServiceControl(stopVsanHealthOperation, vsanhealthServiceName, vcAddress)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
@@ -267,7 +267,7 @@ var _ bool = ginkgo.Describe("[csi-block-e2e] full-sync-test", func() {
 		}
 		gomega.Expect(datastore).NotTo(gomega.BeNil())
 		ginkgo.By(fmt.Sprintln("Stopping vsan-health on the vCenter host"))
-		vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + vcenterPort
+		vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + sshdPort
 		err = invokeVCenterServiceControl(stopVsanHealthOperation, vsanhealthServiceName, vcAddress)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
@@ -328,7 +328,7 @@ var _ bool = ginkgo.Describe("[csi-block-e2e] full-sync-test", func() {
 		}
 
 		ginkgo.By(fmt.Sprintln("Stopping vsan-health on the vCenter host"))
-		vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + vcenterPort
+		vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + sshdPort
 		err = invokeVCenterServiceControl(stopVsanHealthOperation, vsanhealthServiceName, vcAddress)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
@@ -439,7 +439,7 @@ var _ bool = ginkgo.Describe("[csi-block-e2e] full-sync-test", func() {
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
 		ginkgo.By(fmt.Sprintln("Stopping vsan-health on the vCenter host"))
-		vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + vcenterPort
+		vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + sshdPort
 		err = invokeVCenterServiceControl(stopVsanHealthOperation, vsanhealthServiceName, vcAddress)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
@@ -524,7 +524,7 @@ var _ bool = ginkgo.Describe("[csi-block-e2e] full-sync-test", func() {
 		framework.ExpectNoError(framework.WaitOnPVandPVC(client, namespace, pv, pvc))
 
 		ginkgo.By(fmt.Sprintln("Stopping vsan-health on the vCenter host"))
-		vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + vcenterPort
+		vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + sshdPort
 		err = invokeVCenterServiceControl(stopVsanHealthOperation, vsanhealthServiceName, vcAddress)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
@@ -553,7 +553,7 @@ var _ bool = ginkgo.Describe("[csi-block-e2e] full-sync-test", func() {
 
 	})
 
-	ginkgo.It("Bring down syncer pod and verify PV metadata is created in CNS", func() {
+	ginkgo.It("Bring down syncer container and verify PV metadata is created in CNS", func() {
 		var err error
 
 		ctx, cancel := context.WithCancel(context.Background())
@@ -584,13 +584,9 @@ var _ bool = ginkgo.Describe("[csi-block-e2e] full-sync-test", func() {
 		ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow newly created FCD:%s to sync with pandora", pandoraSyncWaitTime, fcdID))
 		time.Sleep(time.Duration(pandoraSyncWaitTime) * time.Second)
 
-		statefulsetTester := framework.NewStatefulSetTester(client)
-		statefulset := statefulsetTester.GetStatefulSet(kubeSystemNamespace, syncerStatefulsetName)
-		replicas := *(statefulset.Spec.Replicas)
-		ginkgo.By(fmt.Sprintf("Kill syncer statefulset by scaling down statefulsets to number of Replica: %v", replicas-1))
-		_, scaleDownErr := statefulsetTester.Scale(statefulset, replicas-1)
-		gomega.Expect(scaleDownErr).NotTo(gomega.HaveOccurred())
-		statefulsetTester.WaitForStatusReadyReplicas(statefulset, replicas-1)
+		ginkgo.By("Scaling down the csi driver to zero replica")
+		deployment := updateDeploymentReplica(client, 0, vSphereCSIControllerPodNamePrefix, kubeSystemNamespace)
+		ginkgo.By(fmt.Sprintf("Successfully scaled down the csi driver deployment:%s to zero replicas", deployment.Name))
 
 		ginkgo.By(fmt.Sprintf("Creating the PV with the fcdID %s", fcdID))
 		staticPVLabels := make(map[string]string)
@@ -599,11 +595,9 @@ var _ bool = ginkgo.Describe("[csi-block-e2e] full-sync-test", func() {
 		pv, err = client.CoreV1().PersistentVolumes().Create(pv)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
-		ginkgo.By(fmt.Sprintf("Recreate syncer statefulset by scaling up statefulsets to number of Replica: %v", replicas))
-		_, scaleUpErr := statefulsetTester.Scale(statefulset, replicas)
-		gomega.Expect(scaleUpErr).NotTo(gomega.HaveOccurred())
-		statefulsetTester.WaitForStatusReplicas(statefulset, replicas)
-		statefulsetTester.WaitForStatusReadyReplicas(statefulset, replicas)
+		ginkgo.By("Scaling up the csi driver to one replica")
+		deployment = updateDeploymentReplica(client, 1, vSphereCSIControllerPodNamePrefix, kubeSystemNamespace)
+		ginkgo.By(fmt.Sprintf("Successfully scaled up the csi driver deployment:%s to one replica", deployment.Name))
 
 		ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow full sync finish", fullSyncWaitTime))
 		time.Sleep(time.Duration(fullSyncWaitTime) * time.Second)

tests/e2e/util.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/onsi/ginkgo"
 	"github.com/onsi/gomega"
@@ -218,6 +219,18 @@ func createStatefulSetWithOneReplica(client clientset.Interface, manifestPath st
 	return statefulSet
 }
 
+// updateDeploymentReplica helps to update the replica for a deployment
+func updateDeploymentReplica(client clientset.Interface, count int32, name string, namespace string) *appsv1.Deployment {
+	deployment, err := client.AppsV1().Deployments(namespace).Get(name, metav1.GetOptions{})
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
+	*deployment.Spec.Replicas = count
+	_, err = client.AppsV1().Deployments(namespace).Update(deployment)
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
+	ginkgo.By("Waiting for update operation on deployment to take effect")
+	time.Sleep(1 * time.Minute)
+	return deployment
+}
+
 // getLabelsMapFromKeyValue returns map[string]string for given array of vim25types.KeyValue
 func getLabelsMapFromKeyValue(labels []vim25types.KeyValue) map[string]string {
 	labelsMap := make(map[string]string)