From 23277934c25b3498d71e1dff7d6f542088e2fbcc Mon Sep 17 00:00:00 2001 From: Roman Bednar Date: Wed, 17 Apr 2024 10:15:24 +0200 Subject: [PATCH] add test for vsphere driver snapshot configuration --- test/extended/storage/driver_configuration.go | 326 ++++++++++++++++++ .../generated/zz_generated.annotations.go | 10 + 2 files changed, 336 insertions(+) create mode 100644 test/extended/storage/driver_configuration.go diff --git a/test/extended/storage/driver_configuration.go b/test/extended/storage/driver_configuration.go new file mode 100644 index 000000000000..85ea66480444 --- /dev/null +++ b/test/extended/storage/driver_configuration.go @@ -0,0 +1,326 @@ +package storage + +import ( + "context" + "fmt" + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + operatorv1 "github.com/openshift/api/operator/v1" + exutil "github.com/openshift/origin/test/extended/util" + "gopkg.in/ini.v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/kubernetes/test/e2e/framework" + e2e "k8s.io/kubernetes/test/e2e/framework" + k8simage "k8s.io/kubernetes/test/utils/image" + "strings" + "time" +) + +const ( + projectName = "csi-driver-configuration" + providerName = "csi.vsphere.vmware.com" +) + +// This is [Serial] because it modifies clustercsidriver. +var _ = g.Describe("[sig-storage][Feature:VSphereDriverConfiguration][Serial] vSphere CSI Driver Configuration", func() { + defer g.GinkgoRecover() + var oc = exutil.NewCLI(projectName) + + var ( + originalDriverConfigSpec *operatorv1.CSIDriverConfigSpec + snapshotOptions = map[string]map[string]string{ + "global": { + "clusterCSIDriver": "globalMaxSnapshotsPerBlockVolume", + "cloudConf": "global-max-snapshots-per-block-volume", + }, + "vsan": { + "clusterCSIDriver": "granularMaxSnapshotsPerBlockVolumeInVSAN", + "cloudConf": "granular-max-snapshots-per-block-volume-vsan", + }, + "vvol": { + "clusterCSIDriver": "granularMaxSnapshotsPerBlockVolumeInVVOL", + "cloudConf": "granular-max-snapshots-per-block-volume-vvol", + }, + } + ) + + g.BeforeEach(func() { + //TODO: remove when GA + if !exutil.IsTechPreviewNoUpgrade(oc) { + g.Skip("this test is only expected to work with TechPreviewNoUpgrade clusters") + } + + if !framework.ProviderIs("vsphere") { + g.Skip("this test is only expected to work with vSphere clusters") + } + + originalClusterCSIDriver, err := oc.AdminOperatorClient().OperatorV1().ClusterCSIDrivers().Get(context.Background(), providerName, metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + originalDriverConfigSpec = originalClusterCSIDriver.Spec.DriverConfig.DeepCopy() + e2e.Logf("storing original driver config: %+v", originalDriverConfigSpec) + }) + + g.AfterEach(func() { + if originalDriverConfigSpec == nil { + return + } + + clusterCSIDriver, err := oc.AdminOperatorClient().OperatorV1().ClusterCSIDrivers().Get(context.Background(), providerName, metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + e2e.Logf("restoring original driver config: %+v", originalDriverConfigSpec) + clusterCSIDriver.Spec.DriverConfig = *originalDriverConfigSpec + _, err = oc.AdminOperatorClient().OperatorV1().ClusterCSIDrivers().Update(context.Background(), clusterCSIDriver, metav1.UpdateOptions{}) + if err != nil { + e2e.Failf("failed to update ClusterCSIDriver: %v", err) + } + o.Expect(err).NotTo(o.HaveOccurred()) + }) + + g.Context("snapshot options in clusterCSIDriver", func() { + var tests = []struct { + name string + clusterCSIDriverOptions map[string]int + cloudConfigOptions map[string]int + successfulSnapshotsCreated int // Number of snapshots that should be created successfully, 0 to skip. + }{ + { + name: "global snapshot option", + clusterCSIDriverOptions: map[string]int{ + snapshotOptions["global"]["clusterCSIDriver"]: 4, + }, + cloudConfigOptions: map[string]int{ + snapshotOptions["global"]["cloudConf"]: 4, + }, + successfulSnapshotsCreated: 4, + }, + { + name: "default snapshot limit", + clusterCSIDriverOptions: map[string]int{}, + cloudConfigOptions: map[string]int{}, + successfulSnapshotsCreated: 3, + }, + { + name: "vsan snapshot option", + clusterCSIDriverOptions: map[string]int{ + snapshotOptions["vsan"]["clusterCSIDriver"]: 4, + }, + cloudConfigOptions: map[string]int{ + snapshotOptions["vsan"]["cloudConf"]: 4, + }, + successfulSnapshotsCreated: 0, + }, + { + name: "vvol snapshot option", + clusterCSIDriverOptions: map[string]int{ + snapshotOptions["vvol"]["clusterCSIDriver"]: 4, + }, + cloudConfigOptions: map[string]int{ + snapshotOptions["vvol"]["cloudConf"]: 4, + }, + successfulSnapshotsCreated: 0, + }, + { + name: "all snapshot options", + clusterCSIDriverOptions: map[string]int{ + snapshotOptions["global"]["clusterCSIDriver"]: 5, + snapshotOptions["vsan"]["clusterCSIDriver"]: 10, + snapshotOptions["vvol"]["clusterCSIDriver"]: 15, + }, + cloudConfigOptions: map[string]int{ + snapshotOptions["global"]["cloudConf"]: 5, + snapshotOptions["vsan"]["cloudConf"]: 10, + snapshotOptions["vvol"]["cloudConf"]: 15, + }, + successfulSnapshotsCreated: 0, + }, + } + + for _, t := range tests { + g.It(fmt.Sprintf("%s", t.name), func() { + for option, value := range t.clusterCSIDriverOptions { + e2e.Logf("updating %s to %d in clustercsidriver", option, value) + setClusterCSIDriverSnapshotOptions(oc, option, value) + } + + for option, value := range t.cloudConfigOptions { + o.Eventually(func() error { + return loadAndCheckCloudConf(oc, "Snapshot", option, value) + }, time.Minute, time.Second).Should(o.Succeed()) + } + + if t.successfulSnapshotsCreated > 0 { + pvc, err := createTestPVC(oc, oc.Namespace(), "test-pvc", "1Gi") + o.Expect(err).NotTo(o.HaveOccurred()) + + _, err = createTestPod(oc, pvc.Name, oc.Namespace()) + o.Expect(err).NotTo(o.HaveOccurred()) + + // Wait for pvc to be bound. + o.Eventually(func() error { + pvc, err := oc.AdminKubeClient().CoreV1().PersistentVolumeClaims(oc.Namespace()).Get(context.Background(), "test-pvc", metav1.GetOptions{}) + if err != nil { + return err + } + if pvc.Status.Phase != v1.ClaimBound { + return fmt.Errorf("PVC not bound") + } + return nil + }) + + for i := 0; i < t.successfulSnapshotsCreated; i++ { + err := createSnapshot(oc, oc.Namespace(), fmt.Sprintf("test-snapshot-%d", i), "test-pvc") + o.Expect(err).NotTo(o.HaveOccurred()) + } + + // Next snapshot creation should be over the set limit and fail. + err = createSnapshot(oc, oc.Namespace(), "test-snapshot-failed", "test-pvc") + o.Expect(err).NotTo(o.HaveOccurred()) + + readyToUse, err := oc.Run("get").Args("volumesnapshot/test-snapshot-failed", "-o", "jsonpath={.status.readyToUse}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + errMsg, err := oc.Run("get").Args("volumesnapshot/test-snapshot-failed", "-o", "jsonpath={.status.error.message}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + e2e.Logf("VolumeSnapshot error message: %s readyToUse %s", errMsg, readyToUse) + if !strings.Contains(errMsg, "failed to take snapshot of the volume") && readyToUse != "false" { + e2e.Failf("VolumeSnapshot \"test-snapshot-failed\" should have failed and should not be ready to use") + } + } + }) + + } + }) +}) + +func setClusterCSIDriverSnapshotOptions(oc *exutil.CLI, snapshotOptions string, value int) { + patch := []byte(fmt.Sprintf("{\"spec\":{\"driverConfig\":{\"vSphere\":{\"%s\": %d}}}}", snapshotOptions, value)) + _, err := oc.AdminOperatorClient().OperatorV1().ClusterCSIDrivers().Patch(context.Background(), providerName, types.MergePatchType, patch, metav1.PatchOptions{}) + if err != nil { + e2e.Failf("failed to patch ClusterCSIDriver: %v", err) + } +} + +func loadAndCheckCloudConf(oc *exutil.CLI, sectionName string, keyName string, expectedValue int) error { + cm, err := oc.AdminKubeClient().CoreV1().ConfigMaps("openshift-cluster-csi-drivers").Get(context.Background(), "vsphere-csi-config", metav1.GetOptions{}) + if err != nil { + e2e.Failf("failed to get ConfigMap: %v", err) + return err + } + + cloudConfData, ok := cm.Data["cloud.conf"] + if !ok { + return fmt.Errorf("cloud.conf key not found in ConfigMap") + } + + cfg, err := ini.Load([]byte(cloudConfData)) + if err != nil { + e2e.Failf("failed to load cloud.conf: %v", err) + return err + } + + section, err := cfg.GetSection(sectionName) + if err != nil { + return fmt.Errorf("section %s not found in cloud.conf: %v", sectionName, err) + } + + key, err := section.GetKey(keyName) + if err != nil { + return fmt.Errorf("key %s not found in section %s: %v", keyName, sectionName, err) + } + + o.Expect(key.String()).To(o.Equal(fmt.Sprintf("%d", expectedValue))) + + return nil +} + +func createTestPod(oc *exutil.CLI, pvcName string, namespace string) (*v1.Pod, error) { + allowPrivEsc := false + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod-driver-conf", + Namespace: namespace, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "test", + Image: k8simage.GetE2EImage(k8simage.BusyBox), + VolumeMounts: []v1.VolumeMount{ + { + Name: "pvc-data", + MountPath: "/mnt", + }, + }, + SecurityContext: &v1.SecurityContext{ + AllowPrivilegeEscalation: &allowPrivEsc, + SeccompProfile: &v1.SeccompProfile{ + Type: v1.SeccompProfileTypeRuntimeDefault, + }, + Capabilities: &v1.Capabilities{ + Drop: []v1.Capability{"ALL"}, + }, + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "pvc-data", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvcName, + }, + }, + }, + }, + }, + } + + return oc.AdminKubeClient().CoreV1().Pods(namespace).Create(context.Background(), pod, metav1.CreateOptions{}) +} + +func createTestPVC(oc *exutil.CLI, namespace string, pvcName string, volumeSize string) (*v1.PersistentVolumeClaim, error) { + e2e.Logf("creating PVC %s in namespace %s with size %s", pvcName, namespace, volumeSize) + + pvc := &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvcName, + Namespace: namespace, + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{"ReadWriteOnce"}, + Resources: v1.VolumeResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse(volumeSize), + }, + }, + }, + } + + return oc.AdminKubeClient().CoreV1().PersistentVolumeClaims(namespace).Create(context.Background(), pvc, metav1.CreateOptions{}) +} + +func createSnapshot(oc *exutil.CLI, namespace string, snapshotName string, pvcName string) error { + snapshot := fmt.Sprintf(` +apiVersion: snapshot.storage.k8s.io/v1 +kind: VolumeSnapshot +metadata: + name: %s + namespace: %s +spec: + source: + persistentVolumeClaimName: %s +`, snapshotName, namespace, pvcName) + + err := oc.AsAdmin().Run("apply").Args("-f", "-").InputString(snapshot).Execute() + if err != nil { + return fmt.Errorf("failed to create snapshot: %v", err) + } + + return nil +} diff --git a/test/extended/util/annotate/generated/zz_generated.annotations.go b/test/extended/util/annotate/generated/zz_generated.annotations.go index 289fcecdb8cc..f6d29ed8fb0f 100644 --- a/test/extended/util/annotate/generated/zz_generated.annotations.go +++ b/test/extended/util/annotate/generated/zz_generated.annotations.go @@ -1551,6 +1551,16 @@ var Annotations = map[string]string{ "[sig-storage][Feature:DisableStorageClass][Serial][apigroup:operator.openshift.io] should remove the StorageClass when StorageClassState is Removed": " [Suite:openshift/conformance/serial]", + "[sig-storage][Feature:VSphereDriverConfiguration][Serial] vSphere CSI Driver Configuration snapshot options in clusterCSIDriver all snapshot options": " [Suite:openshift/conformance/serial]", + + "[sig-storage][Feature:VSphereDriverConfiguration][Serial] vSphere CSI Driver Configuration snapshot options in clusterCSIDriver default snapshot limit": " [Suite:openshift/conformance/serial]", + + "[sig-storage][Feature:VSphereDriverConfiguration][Serial] vSphere CSI Driver Configuration snapshot options in clusterCSIDriver global snapshot option": " [Suite:openshift/conformance/serial]", + + "[sig-storage][Feature:VSphereDriverConfiguration][Serial] vSphere CSI Driver Configuration snapshot options in clusterCSIDriver vsan snapshot option": " [Suite:openshift/conformance/serial]", + + "[sig-storage][Feature:VSphereDriverConfiguration][Serial] vSphere CSI Driver Configuration snapshot options in clusterCSIDriver vvol snapshot option": " [Suite:openshift/conformance/serial]", + "[sig-storage][Late] Metrics should report short attach times": " [Skipped:Disconnected] [Suite:openshift/conformance/parallel]", "[sig-storage][Late] Metrics should report short mount times": " [Skipped:Disconnected] [Suite:openshift/conformance/parallel]",