replace full sync static sleeps with full sync trigger API calls

kubernetes-sigs · Aug 18, 2023 · e087b0d · e087b0d
1 parent e631444
commit e087b0d
Show file tree

Hide file tree

Showing 18 changed files with 220 additions and 375 deletions.
diff --git a/tests/e2e/csi_cns_telemetry.go b/tests/e2e/csi_cns_telemetry.go
@@ -19,8 +19,6 @@ package e2e
 import (
 	"context"
 	"fmt"
-	"os"
-	"strconv"
 	"time"
 
 	ginkgo "github.com/onsi/ginkgo/v2"
@@ -32,17 +30,21 @@ import (
 	"k8s.io/kubernetes/test/e2e/framework"
 	fnodes "k8s.io/kubernetes/test/e2e/framework/node"
 	fpv "k8s.io/kubernetes/test/e2e/framework/pv"
+	k8sClient "sigs.k8s.io/controller-runtime/pkg/client"
+	cnsoperatorv1alpha1 "sigs.k8s.io/vsphere-csi-driver/v3/pkg/apis/cnsoperator"
+	k8s "sigs.k8s.io/vsphere-csi-driver/v3/pkg/kubernetes"
 )
 
 var _ = ginkgo.Describe("[csi-block-vanilla] [csi-file-vanilla] [csi-block-vanilla-parallelized] "+
 	"CNS-CSI Cluster Distribution Telemetry", func() {
 
 	f := framework.NewDefaultFramework("csi-cns-telemetry")
 	var (
-		client       clientset.Interface
-		namespace    string
-		scParameters map[string]string
-		datastoreURL string
+		client            clientset.Interface
+		namespace         string
+		scParameters      map[string]string
+		datastoreURL      string
+		cnsOperatorClient k8sClient.Client
 	)
 
 	ginkgo.BeforeEach(func() {
@@ -61,6 +63,12 @@ var _ = ginkgo.Describe("[csi-block-vanilla] [csi-file-vanilla] [csi-block-vanil
 
 		// Reset the cluster distribution value to default value "CSI-Vanilla".
 		setClusterDistribution(ctx, client, vanillaClusterDistribution)
+
+		cnsOperatorClient, err = k8s.NewClientForGroup(ctx, f.ClientConfig(), cnsoperatorv1alpha1.GroupName)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		enableFullSyncTriggerFss(ctx, client, csiSystemNamespace, fullSyncFss)
+
 	})
 
 	ginkgo.AfterEach(func() {
@@ -90,16 +98,8 @@ var _ = ginkgo.Describe("[csi-block-vanilla] [csi-file-vanilla] [csi-block-vanil
 		var pvclaim1 *v1.PersistentVolumeClaim
 		var storageclasspvc2 *storagev1.StorageClass
 		var pvclaim2 *v1.PersistentVolumeClaim
-		var fullSyncWaitTime int
 		var err error
 
-		// Read full-sync value.
-		if os.Getenv(envFullSyncWaitTime) != "" {
-			fullSyncWaitTime, err = strconv.Atoi(os.Getenv(envFullSyncWaitTime))
-			framework.Logf("Full-Sync interval time value is = %v", fullSyncWaitTime)
-			gomega.Expect(err).NotTo(gomega.HaveOccurred())
-		}
-
 		ginkgo.By("Creating a PVC")
 		scParameters[scParamDatastoreURL] = datastoreURL
 		storageclasspvc1, pvclaim1, err = createPVCAndStorageClass(client,
@@ -183,9 +183,8 @@ var _ = ginkgo.Describe("[csi-block-vanilla] [csi-file-vanilla] [csi-block-vanil
 
 		// For Old PVCs to reflect the latest Cluster-Distribution Value, wait for
 		// full sync.
-		framework.Logf("Sleeping full-sync interval for all the volumes Metadata " +
-			"to reflect the cluster-distribution value to = Empty")
-		time.Sleep(time.Duration(fullSyncWaitTime) * time.Second)
+		ginkgo.By("Trigger 2 full syncs")
+		triggerFullSync(ctx, client, cnsOperatorClient)
 
 		// Add additional safe wait time for cluster-distribution to reflect on
 		// metadata.
@@ -210,7 +209,9 @@ var _ = ginkgo.Describe("[csi-block-vanilla] [csi-file-vanilla] [csi-block-vanil
 		// full sync.
 		framework.Logf("Sleeping full-sync interval for all the volumes Metadata to reflect "+
 			"the cluster-distribution value to = %s", vanillaClusterDistributionWithSpecialChar)
-		time.Sleep(time.Duration(fullSyncWaitTime) * time.Second)
+
+		ginkgo.By("Trigger 2 full syncs")
+		triggerFullSync(ctx, client, cnsOperatorClient)
 
 		// Add additional safe wait time for cluster-distribution to reflect on
 		// metadata.

diff --git a/tests/e2e/csi_cns_telemetry_vc_reboot.go b/tests/e2e/csi_cns_telemetry_vc_reboot.go
@@ -19,8 +19,6 @@ package e2e
 import (
 	"context"
 	"fmt"
-	"os"
-	"strconv"
 	"time"
 
 	ginkgo "github.com/onsi/ginkgo/v2"
@@ -94,16 +92,8 @@ var _ bool = ginkgo.Describe("[csi-block-vanilla] [csi-file-vanilla] "+
 		var pvclaim *v1.PersistentVolumeClaim
 		var storageclass2 *storagev1.StorageClass
 		var pvclaim2 *v1.PersistentVolumeClaim
-		var fullSyncWaitTime int
 		var err error
 
-		// Read full-sync value.
-		if os.Getenv(envFullSyncWaitTime) != "" {
-			fullSyncWaitTime, err = strconv.Atoi(os.Getenv(envFullSyncWaitTime))
-			framework.Logf("Full-Sync interval time value is = %v", fullSyncWaitTime)
-			gomega.Expect(err).NotTo(gomega.HaveOccurred())
-		}
-
 		ginkgo.By("Creating Storage Class and PVC")
 		// Decide which test setup is available to run.
 		if vanillaCluster {
@@ -279,16 +269,8 @@ var _ bool = ginkgo.Describe("[csi-block-vanilla] [csi-file-vanilla] "+
 		defer cancel()
 		var storageclass *storagev1.StorageClass
 		var pvclaim *v1.PersistentVolumeClaim
-		var fullSyncWaitTime int
 		var err error
 
-		// Read full-sync value.
-		if os.Getenv(envFullSyncWaitTime) != "" {
-			fullSyncWaitTime, err = strconv.Atoi(os.Getenv(envFullSyncWaitTime))
-			framework.Logf("Full-Sync interval time value is = %v", fullSyncWaitTime)
-			gomega.Expect(err).NotTo(gomega.HaveOccurred())
-		}
-
 		ginkgo.By("Rebooting VC")
 		vcAddress := e2eVSphere.Config.Global.VCenterHostname + ":" + sshdPort
 		err = invokeVCenterReboot(vcAddress)

diff --git a/tests/e2e/csi_snapshot_basic.go b/tests/e2e/csi_snapshot_basic.go
@@ -26,6 +26,9 @@ import (
 	"github.com/onsi/gomega"
 	cnstypes "github.com/vmware/govmomi/cns/types"
 	"golang.org/x/crypto/ssh"
+	k8sClient "sigs.k8s.io/controller-runtime/pkg/client"
+	cnsoperatorv1alpha1 "sigs.k8s.io/vsphere-csi-driver/v3/pkg/apis/cnsoperator"
+	k8s "sigs.k8s.io/vsphere-csi-driver/v3/pkg/kubernetes"
 
 	v1 "k8s.io/api/core/v1"
 	storagev1 "k8s.io/api/storage/v1"
@@ -63,6 +66,7 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() {
 		snapc                  *snapclient.Clientset
 		storagePolicyName      string
 		clientIndex            int
+		cnsOperatorClient      k8sClient.Client
 	)
 
 	ginkgo.BeforeEach(func() {
@@ -76,6 +80,8 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() {
 		if !(len(nodeList.Items) > 0) {
 			framework.Failf("Unable to find ready and schedulable Node")
 		}
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
 
 		//Get snapshot client using the rest config
 		if !guestCluster {
@@ -129,6 +135,10 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() {
 			snapc, err = snapclient.NewForConfig(guestClusterRestConfig)
 			gomega.Expect(err).NotTo(gomega.HaveOccurred())
 		}
+		cnsOperatorClient, err = k8s.NewClientForGroup(ctx, f.ClientConfig(), cnsoperatorv1alpha1.GroupName)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		enableFullSyncTriggerFss(ctx, client, csiSystemNamespace, fullSyncFss)
 	})
 
 	ginkgo.AfterEach(func() {
@@ -3775,22 +3785,8 @@ var _ = ginkgo.Describe("Volume Snapshot Basic Test", func() {
 		// After reboot.
 		bootstrap()
 
-		fullSyncWaitTime := 0
-
-		if os.Getenv(envFullSyncWaitTime) != "" {
-			fullSyncWaitTime, err = strconv.Atoi(os.Getenv(envFullSyncWaitTime))
-			gomega.Expect(err).NotTo(gomega.HaveOccurred())
-			// Full sync interval can be 1 min at minimum so full sync wait time
-			// has to be more than 120s.
-			if fullSyncWaitTime < 120 || fullSyncWaitTime > defaultFullSyncWaitTime {
-				framework.Failf("The FullSync Wait time %v is not set correctly", fullSyncWaitTime)
-			}
-		} else {
-			fullSyncWaitTime = defaultFullSyncWaitTime
-		}
-
-		ginkgo.By(fmt.Sprintf("Double Sleeping for %v seconds to allow full sync finish", fullSyncWaitTime))
-		time.Sleep(time.Duration(2*fullSyncWaitTime) * time.Second)
+		ginkgo.By("Trigger 2 full syncs")
+		triggerFullSync(ctx, client, cnsOperatorClient)
 
 		ginkgo.By("Verify volume snapshot is created")
 		for _, snapshot := range volumesnapshots {

diff --git a/tests/e2e/csi_snapshot_negative.go b/tests/e2e/csi_snapshot_negative.go
@@ -23,13 +23,15 @@ import (
 
 	ginkgo "github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
+	k8sClient "sigs.k8s.io/controller-runtime/pkg/client"
+	cnsoperatorv1alpha1 "sigs.k8s.io/vsphere-csi-driver/v3/pkg/apis/cnsoperator"
+	k8s "sigs.k8s.io/vsphere-csi-driver/v3/pkg/kubernetes"
 
 	v1 "k8s.io/api/core/v1"
 	storagev1 "k8s.io/api/storage/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clientset "k8s.io/client-go/kubernetes"
-	restclient "k8s.io/client-go/rest"
 	"k8s.io/kubernetes/test/e2e/framework"
 	fnodes "k8s.io/kubernetes/test/e2e/framework/node"
 	fpod "k8s.io/kubernetes/test/e2e/framework/pod"
@@ -51,9 +53,8 @@ var _ = ginkgo.Describe("[block-snapshot-negative] Volume Snapshot Fault-Injecti
 		namespace           string
 		scParameters        map[string]string
 		datastoreURL        string
-		fullSyncWaitTime    int
+		cnsOperatorClient   k8sClient.Client
 		pvclaims            []*v1.PersistentVolumeClaim
-		restConfig          *restclient.Config
 		snapc               *snapclient.Clientset
 		serviceName         string
 		pandoraSyncWaitTime int
@@ -75,20 +76,13 @@ var _ = ginkgo.Describe("[block-snapshot-negative] Volume Snapshot Fault-Injecti
 		}
 
 		//Get snapshot client using the rest config
-		restConfig = getRestConfigClient()
-		snapc, err = snapclient.NewForConfig(restConfig)
+		snapc, err = snapclient.NewForConfig(f.ClientConfig())
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
-		if os.Getenv(envFullSyncWaitTime) != "" {
-			fullSyncWaitTime, err = strconv.Atoi(os.Getenv(envFullSyncWaitTime))
-			gomega.Expect(err).NotTo(gomega.HaveOccurred())
-			// Full sync interval can be 1 min at minimum so full sync wait time has to be more than 120s
-			if fullSyncWaitTime < 120 || fullSyncWaitTime > defaultFullSyncWaitTime {
-				framework.Failf("The FullSync Wait time %v is not set correctly", fullSyncWaitTime)
-			}
-		} else {
-			fullSyncWaitTime = defaultFullSyncWaitTime
-		}
+		cnsOperatorClient, err = k8s.NewClientForGroup(ctx, f.ClientConfig(), cnsoperatorv1alpha1.GroupName)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		enableFullSyncTriggerFss(ctx, client, csiSystemNamespace, fullSyncFss)
 		csiNamespace = GetAndExpectStringEnvVar(envCSINamespace)
 		csiDeployment, err := client.AppsV1().Deployments(csiNamespace).Get(
 			ctx, vSphereCSIControllerPodNamePrefix, metav1.GetOptions{})
@@ -229,38 +223,38 @@ var _ = ginkgo.Describe("[block-snapshot-negative] Volume Snapshot Fault-Injecti
 	ginkgo.It("create volume snapshot when hostd goes down", func() {
 		serviceName = hostdServiceName
 		snapshotOperationWhileServiceDown(serviceName, namespace, client, snapc, datastoreURL,
-			fullSyncWaitTime, isServiceStopped, true, csiReplicas, pandoraSyncWaitTime)
+			cnsOperatorClient, isServiceStopped, true, csiReplicas, pandoraSyncWaitTime)
 	})
 
 	ginkgo.It("create volume snapshot when CSI restarts", func() {
 		serviceName = "CSI"
 		snapshotOperationWhileServiceDown(serviceName, namespace, client, snapc, datastoreURL,
-			fullSyncWaitTime, isServiceStopped, true, csiReplicas, pandoraSyncWaitTime)
+			cnsOperatorClient, isServiceStopped, true, csiReplicas, pandoraSyncWaitTime)
 	})
 
 	ginkgo.It("create volume snapshot when VPXD goes down", func() {
 		serviceName = vpxdServiceName
 		snapshotOperationWhileServiceDownNegative(serviceName, namespace, client, snapc, datastoreURL,
-			fullSyncWaitTime, isServiceStopped, csiReplicas, pandoraSyncWaitTime)
+			cnsOperatorClient, isServiceStopped, csiReplicas, pandoraSyncWaitTime)
 	})
 
 	ginkgo.It("create volume snapshot when CNS goes down", func() {
 		serviceName = vsanhealthServiceName
 		snapshotOperationWhileServiceDown(serviceName, namespace, client, snapc, datastoreURL,
-			fullSyncWaitTime, isServiceStopped, false, csiReplicas, pandoraSyncWaitTime)
+			cnsOperatorClient, isServiceStopped, false, csiReplicas, pandoraSyncWaitTime)
 	})
 
 	ginkgo.It("create volume snapshot when SPS goes down", func() {
 		serviceName = spsServiceName
 		snapshotOperationWhileServiceDown(serviceName, namespace, client, snapc, datastoreURL,
-			fullSyncWaitTime, isServiceStopped, true, csiReplicas, pandoraSyncWaitTime)
+			cnsOperatorClient, isServiceStopped, true, csiReplicas, pandoraSyncWaitTime)
 	})
 })
 
 // snapshotOperationWhileServiceDown creates the volumesnapshot while the services is down
 func snapshotOperationWhileServiceDown(serviceName string, namespace string,
-	client clientset.Interface, snapc *snapclient.Clientset, datastoreURL string,
-	fullSyncWaitTime int, isServiceStopped bool, isSnapshotCreated bool, csiReplicas int32, pandoraSyncWaitTime int) {
+	client clientset.Interface, snapc *snapclient.Clientset, datastoreURL string, cnsOperatorClient k8sClient.Client,
+	isServiceStopped bool, isSnapshotCreated bool, csiReplicas int32, pandoraSyncWaitTime int) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	var storageclass *storagev1.StorageClass
@@ -360,8 +354,8 @@ func snapshotOperationWhileServiceDown(serviceName string, namespace string,
 		isServiceStopped, err = startCSIPods(ctx, client, csiReplicas, csiSystemNamespace)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
-		ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow full sync finish", fullSyncWaitTime))
-		time.Sleep(time.Duration(fullSyncWaitTime) * time.Second)
+		ginkgo.By("Trigger 2 full syncs")
+		triggerFullSync(ctx, client, cnsOperatorClient)
 	} else if serviceName == hostdServiceName {
 		ginkgo.By("Fetch IPs for the all the hosts in the cluster")
 		hostIPs := getAllHostsIP(ctx)
@@ -422,8 +416,8 @@ func snapshotOperationWhileServiceDown(serviceName string, namespace string,
 		err = waitVCenterServiceToBeInState(serviceName, vcAddress, svcRunningMessage)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
-		ginkgo.By("Sleeping for full sync interval")
-		time.Sleep(time.Duration(fullSyncWaitTime) * time.Second)
+		ginkgo.By("Trigger 2 full syncs")
+		triggerFullSync(ctx, client, cnsOperatorClient)
 	}
 
 	//After service restart
@@ -464,8 +458,8 @@ func snapshotOperationWhileServiceDown(serviceName string, namespace string,
 			isServiceStopped, err = startCSIPods(ctx, client, csiReplicas, csiSystemNamespace)
 			gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
-			ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow full sync finish", fullSyncWaitTime))
-			time.Sleep(time.Duration(fullSyncWaitTime) * time.Second)
+			ginkgo.By("Trigger 2 full syncs")
+			triggerFullSync(ctx, client, cnsOperatorClient)
 		} else if serviceName == hostdServiceName {
 			ginkgo.By("Fetch IPs for the all the hosts in the cluster")
 			hostIPs := getAllHostsIP(ctx)
@@ -526,8 +520,8 @@ func snapshotOperationWhileServiceDown(serviceName string, namespace string,
 			err = waitVCenterServiceToBeInState(serviceName, vcAddress, svcRunningMessage)
 			gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
-			ginkgo.By("Sleeping for full sync interval")
-			time.Sleep(time.Duration(fullSyncWaitTime) * time.Second)
+			ginkgo.By("Trigger 2 full syncs")
+			triggerFullSync(ctx, client, cnsOperatorClient)
 		}
 
 		//After service restart
@@ -567,8 +561,8 @@ func snapshotOperationWhileServiceDown(serviceName string, namespace string,
 			isServiceStopped, err = startCSIPods(ctx, client, csiReplicas, csiSystemNamespace)
 			gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
-			ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow full sync finish", fullSyncWaitTime))
-			time.Sleep(time.Duration(fullSyncWaitTime) * time.Second)
+			ginkgo.By("Trigger 2 full syncs")
+			triggerFullSync(ctx, client, cnsOperatorClient)
 		} else if serviceName == hostdServiceName {
 			ginkgo.By("Fetch IPs for the all the hosts in the cluster")
 			hostIPs := getAllHostsIP(ctx)
@@ -629,8 +623,8 @@ func snapshotOperationWhileServiceDown(serviceName string, namespace string,
 			err = waitVCenterServiceToBeInState(serviceName, vcAddress, svcRunningMessage)
 			gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
-			ginkgo.By("Sleeping for full sync interval")
-			time.Sleep(time.Duration(fullSyncWaitTime) * time.Second)
+			ginkgo.By("Trigger 2 full syncs")
+			triggerFullSync(ctx, client, cnsOperatorClient)
 		}
 	}
 
@@ -646,7 +640,7 @@ func snapshotOperationWhileServiceDown(serviceName string, namespace string,
 // snapshotOperationWhileServiceDownNegative creates the volumesnapshot while the services is down
 func snapshotOperationWhileServiceDownNegative(serviceName string, namespace string,
 	client clientset.Interface, snapc *snapclient.Clientset, datastoreURL string,
-	fullSyncWaitTime int, isServiceStopped bool, csiReplicas int32, pandoraSyncWaitTime int) {
+	cnsOperatorClient k8sClient.Client, isServiceStopped bool, csiReplicas int32, pandoraSyncWaitTime int) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	var storageclass *storagev1.StorageClass
@@ -734,8 +728,8 @@ func snapshotOperationWhileServiceDownNegative(serviceName string, namespace str
 		isServiceStopped, err = startCSIPods(ctx, client, csiReplicas, csiSystemNamespace)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
-		ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow full sync finish", fullSyncWaitTime))
-		time.Sleep(time.Duration(fullSyncWaitTime) * time.Second)
+		ginkgo.By("Trigger 2 full syncs")
+		triggerFullSync(ctx, client, cnsOperatorClient)
 	} else if serviceName == hostdServiceName {
 		ginkgo.By("Fetch IPs for the all the hosts in the cluster")
 		hostIPs := getAllHostsIP(ctx)
@@ -796,8 +790,8 @@ func snapshotOperationWhileServiceDownNegative(serviceName string, namespace str
 		err = waitVCenterServiceToBeInState(serviceName, vcAddress, svcRunningMessage)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
-		ginkgo.By("Sleeping for full sync interval")
-		time.Sleep(time.Duration(fullSyncWaitTime) * time.Second)
+		ginkgo.By("Trigger 2 full syncs")
+		triggerFullSync(ctx, client, cnsOperatorClient)
 	}
 
 	//After service restart