Skip to content

Commit

Permalink
Scaling replica count for csi driver to verify volume operations gett…
Browse files Browse the repository at this point in the history 
…ing picked up after FULL SYNC
  • Loading branch information
@chethanv28
chethanv28 committed Oct 3, 2019
1 parent 168eb8a commit 1438def
Show file tree
Hide file tree
Showing 3 changed files with 29 additions and 21 deletions.
3 changes: 2 additions & 1 deletion tests/e2e/e2e_common.go
View file
Original file line number Original file line Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (


const ( const (
envSharedDatastoreURL = "SHARED_VSPHERE_DATASTORE_URL" envSharedDatastoreURL = "SHARED_VSPHERE_DATASTORE_URL"
envMasterNodeIP = "K8S_MASTER_NODE_IP"
envNonSharedStorageClassDatastoreURL = "NONSHARED_VSPHERE_DATASTORE_URL" envNonSharedStorageClassDatastoreURL = "NONSHARED_VSPHERE_DATASTORE_URL"
envInaccessibleZoneDatastoreURL = "INACCESSIBLE_ZONE_VSPHERE_DATASTORE_URL" envInaccessibleZoneDatastoreURL = "INACCESSIBLE_ZONE_VSPHERE_DATASTORE_URL"
scParamDatastoreURL = "DatastoreURL" scParamDatastoreURL = "DatastoreURL"
Expand Down Expand Up @@ -57,7 +58,7 @@ const (
invalidFSType = "ext10" invalidFSType = "ext10"
execCommand = "/bin/df -T /mnt/volume1 | /bin/awk 'FNR == 2 {print $2}' > /mnt/volume1/fstype && while true ; do sleep 2 ; done" execCommand = "/bin/df -T /mnt/volume1 | /bin/awk 'FNR == 2 {print $2}' > /mnt/volume1/fstype && while true ; do sleep 2 ; done"
kubeSystemNamespace = "kube-system" kubeSystemNamespace = "kube-system"
syncerStatefulsetName = "vsphere-csi-metadata-syncer" vSphereCSIControllerPodNamePrefix = "vsphere-csi-controller"
) )


// GetAndExpectStringEnvVar parses a string from env variable // GetAndExpectStringEnvVar parses a string from env variable
Expand Down
34 changes: 14 additions & 20 deletions tests/e2e/fullsynctest.go
View file
Original file line number Original file line Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ var _ bool = ginkgo.Describe("[csi-block-e2e] full-sync-test", func() {
fcdName = "BasicStaticFCD" fcdName = "BasicStaticFCD"
stopVsanHealthOperation = "stop" stopVsanHealthOperation = "stop"
startVsanHealthOperation = "start" startVsanHealthOperation = "start"
vcenterPort = "22" sshdPort = "22"
numberOfPVC = 5 numberOfPVC = 5
) )


Expand Down Expand Up @@ -140,7 +140,7 @@ var _ bool = ginkgo.Describe("[csi-block-e2e] full-sync-test", func() {
time.Sleep(time.Duration(pandoraSyncWaitTime) * time.Second) time.Sleep(time.Duration(pandoraSyncWaitTime) * time.Second)


ginkgo.By(fmt.Sprintln("Stopping vsan-health on the vCenter host")) ginkgo.By(fmt.Sprintln("Stopping vsan-health on the vCenter host"))
vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + vcenterPort vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + sshdPort
err = invokeVCenterServiceControl(stopVsanHealthOperation, vsanhealthServiceName, vcAddress) err = invokeVCenterServiceControl(stopVsanHealthOperation, vsanhealthServiceName, vcAddress)
gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(err).NotTo(gomega.HaveOccurred())


Expand Down Expand Up @@ -187,7 +187,7 @@ var _ bool = ginkgo.Describe("[csi-block-e2e] full-sync-test", func() {
pv := pvs[0] pv := pvs[0]


ginkgo.By(fmt.Sprintln("Stopping vsan-health on the vCenter host")) ginkgo.By(fmt.Sprintln("Stopping vsan-health on the vCenter host"))
vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + vcenterPort vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + sshdPort
err = invokeVCenterServiceControl(stopVsanHealthOperation, vsanhealthServiceName, vcAddress) err = invokeVCenterServiceControl(stopVsanHealthOperation, vsanhealthServiceName, vcAddress)
gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(err).NotTo(gomega.HaveOccurred())


Expand Down Expand Up @@ -267,7 +267,7 @@ var _ bool = ginkgo.Describe("[csi-block-e2e] full-sync-test", func() {
} }
gomega.Expect(datastore).NotTo(gomega.BeNil()) gomega.Expect(datastore).NotTo(gomega.BeNil())
ginkgo.By(fmt.Sprintln("Stopping vsan-health on the vCenter host")) ginkgo.By(fmt.Sprintln("Stopping vsan-health on the vCenter host"))
vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + vcenterPort vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + sshdPort
err = invokeVCenterServiceControl(stopVsanHealthOperation, vsanhealthServiceName, vcAddress) err = invokeVCenterServiceControl(stopVsanHealthOperation, vsanhealthServiceName, vcAddress)
gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(err).NotTo(gomega.HaveOccurred())


Expand Down Expand Up @@ -328,7 +328,7 @@ var _ bool = ginkgo.Describe("[csi-block-e2e] full-sync-test", func() {
} }


ginkgo.By(fmt.Sprintln("Stopping vsan-health on the vCenter host")) ginkgo.By(fmt.Sprintln("Stopping vsan-health on the vCenter host"))
vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + vcenterPort vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + sshdPort
err = invokeVCenterServiceControl(stopVsanHealthOperation, vsanhealthServiceName, vcAddress) err = invokeVCenterServiceControl(stopVsanHealthOperation, vsanhealthServiceName, vcAddress)
gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(err).NotTo(gomega.HaveOccurred())


Expand Down Expand Up @@ -439,7 +439,7 @@ var _ bool = ginkgo.Describe("[csi-block-e2e] full-sync-test", func() {
gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(err).NotTo(gomega.HaveOccurred())


ginkgo.By(fmt.Sprintln("Stopping vsan-health on the vCenter host")) ginkgo.By(fmt.Sprintln("Stopping vsan-health on the vCenter host"))
vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + vcenterPort vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + sshdPort
err = invokeVCenterServiceControl(stopVsanHealthOperation, vsanhealthServiceName, vcAddress) err = invokeVCenterServiceControl(stopVsanHealthOperation, vsanhealthServiceName, vcAddress)
gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(err).NotTo(gomega.HaveOccurred())


Expand Down Expand Up @@ -524,7 +524,7 @@ var _ bool = ginkgo.Describe("[csi-block-e2e] full-sync-test", func() {
framework.ExpectNoError(framework.WaitOnPVandPVC(client, namespace, pv, pvc)) framework.ExpectNoError(framework.WaitOnPVandPVC(client, namespace, pv, pvc))


ginkgo.By(fmt.Sprintln("Stopping vsan-health on the vCenter host")) ginkgo.By(fmt.Sprintln("Stopping vsan-health on the vCenter host"))
vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + vcenterPort vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + sshdPort
err = invokeVCenterServiceControl(stopVsanHealthOperation, vsanhealthServiceName, vcAddress) err = invokeVCenterServiceControl(stopVsanHealthOperation, vsanhealthServiceName, vcAddress)
gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(err).NotTo(gomega.HaveOccurred())


Expand Down Expand Up @@ -553,7 +553,7 @@ var _ bool = ginkgo.Describe("[csi-block-e2e] full-sync-test", func() {


}) })


ginkgo.It("Bring down syncer pod and verify PV metadata is created in CNS", func() { ginkgo.It("Scale down csi driver statefulset and verify PV metadata is created in CNS", func() {
var err error var err error


ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
Expand Down Expand Up @@ -584,13 +584,9 @@ var _ bool = ginkgo.Describe("[csi-block-e2e] full-sync-test", func() {
ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow newly created FCD:%s to sync with pandora", pandoraSyncWaitTime, fcdID)) ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow newly created FCD:%s to sync with pandora", pandoraSyncWaitTime, fcdID))
time.Sleep(time.Duration(pandoraSyncWaitTime) * time.Second) time.Sleep(time.Duration(pandoraSyncWaitTime) * time.Second)


statefulsetTester := framework.NewStatefulSetTester(client) ginkgo.By("Scaling down the csi driver to zero replica")
statefulset := statefulsetTester.GetStatefulSet(kubeSystemNamespace, syncerStatefulsetName) statefulSet := updateStatefulSetReplica(client, 0, vSphereCSIControllerPodNamePrefix, kubeSystemNamespace)
replicas := *(statefulset.Spec.Replicas) ginkgo.By(fmt.Sprintf("Successfully scaled down the csi driver statefulset:%s to zero replicas", statefulSet.Name))
ginkgo.By(fmt.Sprintf("Kill syncer statefulset by scaling down statefulsets to number of Replica: %v", replicas-1))
_, scaleDownErr := statefulsetTester.Scale(statefulset, replicas-1)
gomega.Expect(scaleDownErr).NotTo(gomega.HaveOccurred())
statefulsetTester.WaitForStatusReadyReplicas(statefulset, replicas-1)


ginkgo.By(fmt.Sprintf("Creating the PV with the fcdID %s", fcdID)) ginkgo.By(fmt.Sprintf("Creating the PV with the fcdID %s", fcdID))
staticPVLabels := make(map[string]string) staticPVLabels := make(map[string]string)
Expand All @@ -599,11 +595,9 @@ var _ bool = ginkgo.Describe("[csi-block-e2e] full-sync-test", func() {
pv, err = client.CoreV1().PersistentVolumes().Create(pv) pv, err = client.CoreV1().PersistentVolumes().Create(pv)
gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(err).NotTo(gomega.HaveOccurred())


ginkgo.By(fmt.Sprintf("Recreate syncer statefulset by scaling up statefulsets to number of Replica: %v", replicas)) ginkgo.By("Scaling up the csi driver to one replica")
_, scaleUpErr := statefulsetTester.Scale(statefulset, replicas) statefulSet = updateStatefulSetReplica(client, 1, vSphereCSIControllerPodNamePrefix, kubeSystemNamespace)
gomega.Expect(scaleUpErr).NotTo(gomega.HaveOccurred()) ginkgo.By(fmt.Sprintf("Successfully scaled up the csi driver statefulset:%s to one replica", statefulSet.Name))
statefulsetTester.WaitForStatusReplicas(statefulset, replicas)
statefulsetTester.WaitForStatusReadyReplicas(statefulset, replicas)


ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow full sync finish", fullSyncWaitTime)) ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow full sync finish", fullSyncWaitTime))
time.Sleep(time.Duration(fullSyncWaitTime) * time.Second) time.Sleep(time.Duration(fullSyncWaitTime) * time.Second)
Expand Down
13 changes: 13 additions & 0 deletions tests/e2e/util.go
View file
Original file line number Original file line Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"fmt" "fmt"
"path/filepath" "path/filepath"
"strings" "strings"
"time"


"github.com/onsi/ginkgo" "github.com/onsi/ginkgo"
"github.com/onsi/gomega" "github.com/onsi/gomega"
Expand Down Expand Up @@ -218,6 +219,18 @@ func createStatefulSetWithOneReplica(client clientset.Interface, manifestPath st
return statefulSet return statefulSet
} }


// updateStatefulSetReplica helps to update the replica for a statefulset
func updateStatefulSetReplica(client clientset.Interface, count int32, name string, namespace string) *appsv1.StatefulSet {
statefulSet, err := client.AppsV1().StatefulSets(namespace).Get(name, metav1.GetOptions{})
gomega.Expect(err).NotTo(gomega.HaveOccurred())
*statefulSet.Spec.Replicas = count
_, err = client.AppsV1().StatefulSets(namespace).Update(statefulSet)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
ginkgo.By("Waiting for update operation on statefulset to take effect")
time.Sleep(1 * time.Minute)
return statefulSet
}

// getLabelsMapFromKeyValue returns map[string]string for given array of vim25types.KeyValue // getLabelsMapFromKeyValue returns map[string]string for given array of vim25types.KeyValue
func getLabelsMapFromKeyValue(labels []vim25types.KeyValue) map[string]string { func getLabelsMapFromKeyValue(labels []vim25types.KeyValue) map[string]string {
labelsMap := make(map[string]string) labelsMap := make(map[string]string)
Expand Down

0 comments on commit 1438def

Please sign in to comment.