Skip to content
Permalink
Browse files

Scaling replica count for csi driver to verify volume operations gett…

…ing picked up after FULL SYNC
  • Loading branch information
chethanv28 committed Sep 24, 2019
1 parent 168eb8a commit 1438def379cea729ab525c38f77f0e67cdb430ee
Showing with 29 additions and 21 deletions.
  1. +2 −1 tests/e2e/e2e_common.go
  2. +14 −20 tests/e2e/fullsynctest.go
  3. +13 −0 tests/e2e/util.go
@@ -26,6 +26,7 @@ import (

const (
envSharedDatastoreURL = "SHARED_VSPHERE_DATASTORE_URL"
envMasterNodeIP = "K8S_MASTER_NODE_IP"
envNonSharedStorageClassDatastoreURL = "NONSHARED_VSPHERE_DATASTORE_URL"
envInaccessibleZoneDatastoreURL = "INACCESSIBLE_ZONE_VSPHERE_DATASTORE_URL"
scParamDatastoreURL = "DatastoreURL"
@@ -57,7 +58,7 @@ const (
invalidFSType = "ext10"
execCommand = "/bin/df -T /mnt/volume1 | /bin/awk 'FNR == 2 {print $2}' > /mnt/volume1/fstype && while true ; do sleep 2 ; done"
kubeSystemNamespace = "kube-system"
syncerStatefulsetName = "vsphere-csi-metadata-syncer"
vSphereCSIControllerPodNamePrefix = "vsphere-csi-controller"
)

// GetAndExpectStringEnvVar parses a string from env variable
@@ -69,7 +69,7 @@ var _ bool = ginkgo.Describe("[csi-block-e2e] full-sync-test", func() {
fcdName = "BasicStaticFCD"
stopVsanHealthOperation = "stop"
startVsanHealthOperation = "start"
vcenterPort = "22"
sshdPort = "22"
numberOfPVC = 5
)

@@ -140,7 +140,7 @@ var _ bool = ginkgo.Describe("[csi-block-e2e] full-sync-test", func() {
time.Sleep(time.Duration(pandoraSyncWaitTime) * time.Second)

ginkgo.By(fmt.Sprintln("Stopping vsan-health on the vCenter host"))
vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + vcenterPort
vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + sshdPort
err = invokeVCenterServiceControl(stopVsanHealthOperation, vsanhealthServiceName, vcAddress)
gomega.Expect(err).NotTo(gomega.HaveOccurred())

@@ -187,7 +187,7 @@ var _ bool = ginkgo.Describe("[csi-block-e2e] full-sync-test", func() {
pv := pvs[0]

ginkgo.By(fmt.Sprintln("Stopping vsan-health on the vCenter host"))
vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + vcenterPort
vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + sshdPort
err = invokeVCenterServiceControl(stopVsanHealthOperation, vsanhealthServiceName, vcAddress)
gomega.Expect(err).NotTo(gomega.HaveOccurred())

@@ -267,7 +267,7 @@ var _ bool = ginkgo.Describe("[csi-block-e2e] full-sync-test", func() {
}
gomega.Expect(datastore).NotTo(gomega.BeNil())
ginkgo.By(fmt.Sprintln("Stopping vsan-health on the vCenter host"))
vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + vcenterPort
vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + sshdPort
err = invokeVCenterServiceControl(stopVsanHealthOperation, vsanhealthServiceName, vcAddress)
gomega.Expect(err).NotTo(gomega.HaveOccurred())

@@ -328,7 +328,7 @@ var _ bool = ginkgo.Describe("[csi-block-e2e] full-sync-test", func() {
}

ginkgo.By(fmt.Sprintln("Stopping vsan-health on the vCenter host"))
vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + vcenterPort
vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + sshdPort
err = invokeVCenterServiceControl(stopVsanHealthOperation, vsanhealthServiceName, vcAddress)
gomega.Expect(err).NotTo(gomega.HaveOccurred())

@@ -439,7 +439,7 @@ var _ bool = ginkgo.Describe("[csi-block-e2e] full-sync-test", func() {
gomega.Expect(err).NotTo(gomega.HaveOccurred())

ginkgo.By(fmt.Sprintln("Stopping vsan-health on the vCenter host"))
vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + vcenterPort
vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + sshdPort
err = invokeVCenterServiceControl(stopVsanHealthOperation, vsanhealthServiceName, vcAddress)
gomega.Expect(err).NotTo(gomega.HaveOccurred())

@@ -524,7 +524,7 @@ var _ bool = ginkgo.Describe("[csi-block-e2e] full-sync-test", func() {
framework.ExpectNoError(framework.WaitOnPVandPVC(client, namespace, pv, pvc))

ginkgo.By(fmt.Sprintln("Stopping vsan-health on the vCenter host"))
vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + vcenterPort
vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + sshdPort
err = invokeVCenterServiceControl(stopVsanHealthOperation, vsanhealthServiceName, vcAddress)
gomega.Expect(err).NotTo(gomega.HaveOccurred())

@@ -553,7 +553,7 @@ var _ bool = ginkgo.Describe("[csi-block-e2e] full-sync-test", func() {

})

ginkgo.It("Bring down syncer pod and verify PV metadata is created in CNS", func() {
ginkgo.It("Scale down csi driver statefulset and verify PV metadata is created in CNS", func() {
var err error

ctx, cancel := context.WithCancel(context.Background())
@@ -584,13 +584,9 @@ var _ bool = ginkgo.Describe("[csi-block-e2e] full-sync-test", func() {
ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow newly created FCD:%s to sync with pandora", pandoraSyncWaitTime, fcdID))
time.Sleep(time.Duration(pandoraSyncWaitTime) * time.Second)

statefulsetTester := framework.NewStatefulSetTester(client)
statefulset := statefulsetTester.GetStatefulSet(kubeSystemNamespace, syncerStatefulsetName)
replicas := *(statefulset.Spec.Replicas)
ginkgo.By(fmt.Sprintf("Kill syncer statefulset by scaling down statefulsets to number of Replica: %v", replicas-1))
_, scaleDownErr := statefulsetTester.Scale(statefulset, replicas-1)
gomega.Expect(scaleDownErr).NotTo(gomega.HaveOccurred())
statefulsetTester.WaitForStatusReadyReplicas(statefulset, replicas-1)
ginkgo.By("Scaling down the csi driver to zero replica")
statefulSet := updateStatefulSetReplica(client, 0, vSphereCSIControllerPodNamePrefix, kubeSystemNamespace)
ginkgo.By(fmt.Sprintf("Successfully scaled down the csi driver statefulset:%s to zero replicas", statefulSet.Name))

ginkgo.By(fmt.Sprintf("Creating the PV with the fcdID %s", fcdID))
staticPVLabels := make(map[string]string)
@@ -599,11 +595,9 @@ var _ bool = ginkgo.Describe("[csi-block-e2e] full-sync-test", func() {
pv, err = client.CoreV1().PersistentVolumes().Create(pv)
gomega.Expect(err).NotTo(gomega.HaveOccurred())

ginkgo.By(fmt.Sprintf("Recreate syncer statefulset by scaling up statefulsets to number of Replica: %v", replicas))
_, scaleUpErr := statefulsetTester.Scale(statefulset, replicas)
gomega.Expect(scaleUpErr).NotTo(gomega.HaveOccurred())
statefulsetTester.WaitForStatusReplicas(statefulset, replicas)
statefulsetTester.WaitForStatusReadyReplicas(statefulset, replicas)
ginkgo.By("Scaling up the csi driver to one replica")
statefulSet = updateStatefulSetReplica(client, 1, vSphereCSIControllerPodNamePrefix, kubeSystemNamespace)
ginkgo.By(fmt.Sprintf("Successfully scaled up the csi driver statefulset:%s to one replica", statefulSet.Name))

ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow full sync finish", fullSyncWaitTime))
time.Sleep(time.Duration(fullSyncWaitTime) * time.Second)
@@ -22,6 +22,7 @@ import (
"fmt"
"path/filepath"
"strings"
"time"

"github.com/onsi/ginkgo"
"github.com/onsi/gomega"
@@ -218,6 +219,18 @@ func createStatefulSetWithOneReplica(client clientset.Interface, manifestPath st
return statefulSet
}

// updateStatefulSetReplica helps to update the replica for a statefulset
func updateStatefulSetReplica(client clientset.Interface, count int32, name string, namespace string) *appsv1.StatefulSet {
statefulSet, err := client.AppsV1().StatefulSets(namespace).Get(name, metav1.GetOptions{})
gomega.Expect(err).NotTo(gomega.HaveOccurred())
*statefulSet.Spec.Replicas = count
_, err = client.AppsV1().StatefulSets(namespace).Update(statefulSet)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
ginkgo.By("Waiting for update operation on statefulset to take effect")
time.Sleep(1 * time.Minute)
return statefulSet
}

// getLabelsMapFromKeyValue returns map[string]string for given array of vim25types.KeyValue
func getLabelsMapFromKeyValue(labels []vim25types.KeyValue) map[string]string {
labelsMap := make(map[string]string)

0 comments on commit 1438def

Please sign in to comment.
You can’t perform that action at this time.