From 6dd94f5663faa99eeb1c5416c0d0f21bbff115b5 Mon Sep 17 00:00:00 2001 From: Hemant Kumar Date: Wed, 9 Nov 2022 15:11:03 -0500 Subject: [PATCH 1/2] Refactor mock tests in different files Change code to use pointer for mock driver setup --- test/e2e/e2e_test.go | 1 + test/e2e/storage/csi_mock/base.go | 932 ++++++ .../e2e/storage/csi_mock/csi_attach_volume.go | 178 + .../e2e/storage/csi_mock/csi_fsgroup_mount.go | 86 + .../storage/csi_mock/csi_fsgroup_policy.go | 124 + .../csi_mock/csi_node_stage_error_cases.go | 330 ++ .../e2e/storage/csi_mock/csi_selinux_mount.go | 122 + .../csi_mock/csi_service_account_token.go | 98 + test/e2e/storage/csi_mock/csi_snapshot.go | 613 ++++ .../storage/csi_mock/csi_storage_capacity.go | 395 +++ .../storage/csi_mock/csi_volume_expansion.go | 214 ++ test/e2e/storage/csi_mock/csi_volume_limit.go | 156 + test/e2e/storage/csi_mock/csi_workload.go | 124 + test/e2e/storage/csi_mock_volume.go | 2858 ----------------- test/e2e/storage/local_volume_resize.go | 4 + 15 files changed, 3377 insertions(+), 2858 deletions(-) create mode 100644 test/e2e/storage/csi_mock/base.go create mode 100644 test/e2e/storage/csi_mock/csi_attach_volume.go create mode 100644 test/e2e/storage/csi_mock/csi_fsgroup_mount.go create mode 100644 test/e2e/storage/csi_mock/csi_fsgroup_policy.go create mode 100644 test/e2e/storage/csi_mock/csi_node_stage_error_cases.go create mode 100644 test/e2e/storage/csi_mock/csi_selinux_mount.go create mode 100644 test/e2e/storage/csi_mock/csi_service_account_token.go create mode 100644 test/e2e/storage/csi_mock/csi_snapshot.go create mode 100644 test/e2e/storage/csi_mock/csi_storage_capacity.go create mode 100644 test/e2e/storage/csi_mock/csi_volume_expansion.go create mode 100644 test/e2e/storage/csi_mock/csi_volume_limit.go create mode 100644 test/e2e/storage/csi_mock/csi_workload.go delete mode 100644 test/e2e/storage/csi_mock_volume.go diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 1b124cc831f15..776e1cd9f9b91 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -61,6 +61,7 @@ import ( _ "k8s.io/kubernetes/test/e2e/node" _ "k8s.io/kubernetes/test/e2e/scheduling" _ "k8s.io/kubernetes/test/e2e/storage" + _ "k8s.io/kubernetes/test/e2e/storage/csi_mock" _ "k8s.io/kubernetes/test/e2e/storage/external" _ "k8s.io/kubernetes/test/e2e/windows" diff --git a/test/e2e/storage/csi_mock/base.go b/test/e2e/storage/csi_mock/base.go new file mode 100644 index 0000000000000..266eee0bbb0e1 --- /dev/null +++ b/test/e2e/storage/csi_mock/base.go @@ -0,0 +1,932 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi_mock + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + "sync/atomic" + "time" + + csipbv1 "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/onsi/ginkgo/v2" + "google.golang.org/grpc/codes" + v1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/fields" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + e2epv "k8s.io/kubernetes/test/e2e/framework/pv" + "k8s.io/kubernetes/test/e2e/storage/drivers" + storageframework "k8s.io/kubernetes/test/e2e/storage/framework" + "k8s.io/kubernetes/test/e2e/storage/testsuites" + "k8s.io/kubernetes/test/e2e/storage/utils" + imageutils "k8s.io/kubernetes/test/utils/image" +) + +const ( + csiNodeLimitUpdateTimeout = 5 * time.Minute + csiPodUnschedulableTimeout = 5 * time.Minute + csiResizeWaitPeriod = 5 * time.Minute + csiVolumeAttachmentTimeout = 7 * time.Minute + // how long to wait for Resizing Condition on PVC to appear + csiResizingConditionWait = 2 * time.Minute + + // Time for starting a pod with a volume. + csiPodRunningTimeout = 5 * time.Minute + + // How log to wait for kubelet to unstage a volume after a pod is deleted + csiUnstageWaitTimeout = 1 * time.Minute +) + +// csiCall represents an expected call from Kubernetes to CSI mock driver and +// expected return value. +// When matching expected csiCall with a real CSI mock driver output, one csiCall +// matches *one or more* calls with the same method and error code. +// This is due to exponential backoff in Kubernetes, where the test cannot expect +// exact number of call repetitions. +type csiCall struct { + expectedMethod string + expectedError codes.Code + // This is a mark for the test itself to delete the tested pod *after* + // this csiCall is received. + deletePod bool +} + +type testParameters struct { + disableAttach bool + attachLimit int + registerDriver bool + lateBinding bool + enableTopology bool + podInfo *bool + storageCapacity *bool + scName string // pre-selected storage class name; must be unique in the cluster + enableResizing bool // enable resizing for both CSI mock driver and storageClass. + enableNodeExpansion bool // enable node expansion for CSI mock driver + // just disable resizing on driver it overrides enableResizing flag for CSI mock driver + disableResizingOnDriver bool + enableSnapshot bool + enableVolumeMountGroup bool // enable the VOLUME_MOUNT_GROUP node capability in the CSI mock driver. + hooks *drivers.Hooks + tokenRequests []storagev1.TokenRequest + requiresRepublish *bool + fsGroupPolicy *storagev1.FSGroupPolicy + enableSELinuxMount *bool +} + +type mockDriverSetup struct { + cs clientset.Interface + config *storageframework.PerTestConfig + testCleanups []func() + pods []*v1.Pod + pvcs []*v1.PersistentVolumeClaim + sc map[string]*storagev1.StorageClass + vsc map[string]*unstructured.Unstructured + driver drivers.MockCSITestDriver + provisioner string + tp testParameters + f *framework.Framework +} + +type volumeType string + +var ( + csiEphemeral = volumeType("CSI") + genericEphemeral = volumeType("Ephemeral") + pvcReference = volumeType("PVC") +) + +const ( + poll = 2 * time.Second + pvcAsSourceProtectionFinalizer = "snapshot.storage.kubernetes.io/pvc-as-source-protection" + volumeSnapshotContentFinalizer = "snapshot.storage.kubernetes.io/volumesnapshotcontent-bound-protection" + volumeSnapshotBoundFinalizer = "snapshot.storage.kubernetes.io/volumesnapshot-bound-protection" + errReasonNotEnoughSpace = "node(s) did not have enough free storage" +) + +var ( + errPodCompleted = fmt.Errorf("pod ran to completion") + errNotEnoughSpace = errors.New(errReasonNotEnoughSpace) +) + +func newMockDriverSetup(f *framework.Framework) *mockDriverSetup { + return &mockDriverSetup{ + cs: f.ClientSet, + sc: make(map[string]*storagev1.StorageClass), + vsc: make(map[string]*unstructured.Unstructured), + f: f, + } +} + +func (m *mockDriverSetup) init(tp testParameters) { + m.cs = m.f.ClientSet + m.tp = tp + + var err error + driverOpts := drivers.CSIMockDriverOpts{ + RegisterDriver: tp.registerDriver, + PodInfo: tp.podInfo, + StorageCapacity: tp.storageCapacity, + EnableTopology: tp.enableTopology, + AttachLimit: tp.attachLimit, + DisableAttach: tp.disableAttach, + EnableResizing: tp.enableResizing, + EnableNodeExpansion: tp.enableNodeExpansion, + EnableSnapshot: tp.enableSnapshot, + EnableVolumeMountGroup: tp.enableVolumeMountGroup, + TokenRequests: tp.tokenRequests, + RequiresRepublish: tp.requiresRepublish, + FSGroupPolicy: tp.fsGroupPolicy, + EnableSELinuxMount: tp.enableSELinuxMount, + } + + // At the moment, only tests which need hooks are + // using the embedded CSI mock driver. The rest run + // the driver inside the cluster although they could + // changed to use embedding merely by setting + // driverOpts.embedded to true. + // + // Not enabling it for all tests minimizes + // the risk that the introduction of embedded breaks + // some existings tests and avoids a dependency + // on port forwarding, which is important if some of + // these tests are supposed to become part of + // conformance testing (port forwarding isn't + // currently required). + if tp.hooks != nil { + driverOpts.Embedded = true + driverOpts.Hooks = *tp.hooks + } + + // this just disable resizing on driver, keeping resizing on SC enabled. + if tp.disableResizingOnDriver { + driverOpts.EnableResizing = false + } + + m.driver = drivers.InitMockCSIDriver(driverOpts) + config := m.driver.PrepareTest(m.f) + m.config = config + m.provisioner = config.GetUniqueDriverName() + + if tp.registerDriver { + err = waitForCSIDriver(m.cs, m.config.GetUniqueDriverName()) + framework.ExpectNoError(err, "Failed to get CSIDriver %v", m.config.GetUniqueDriverName()) + m.testCleanups = append(m.testCleanups, func() { + destroyCSIDriver(m.cs, m.config.GetUniqueDriverName()) + }) + } + + // Wait for the CSIDriver actually get deployed and CSINode object to be generated. + // This indicates the mock CSI driver pod is up and running healthy. + err = drivers.WaitForCSIDriverRegistrationOnNode(m.config.ClientNodeSelection.Name, m.config.GetUniqueDriverName(), m.cs) + framework.ExpectNoError(err, "Failed to register CSIDriver %v", m.config.GetUniqueDriverName()) +} + +func (m *mockDriverSetup) cleanup() { + cs := m.f.ClientSet + var errs []error + + for _, pod := range m.pods { + ginkgo.By(fmt.Sprintf("Deleting pod %s", pod.Name)) + errs = append(errs, e2epod.DeletePodWithWait(cs, pod)) + } + + for _, claim := range m.pvcs { + ginkgo.By(fmt.Sprintf("Deleting claim %s", claim.Name)) + claim, err := cs.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(context.TODO(), claim.Name, metav1.GetOptions{}) + if err == nil { + if err := cs.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(context.TODO(), claim.Name, metav1.DeleteOptions{}); err != nil { + errs = append(errs, err) + } + if claim.Spec.VolumeName != "" { + errs = append(errs, e2epv.WaitForPersistentVolumeDeleted(cs, claim.Spec.VolumeName, framework.Poll, 2*time.Minute)) + } + } + } + + for _, sc := range m.sc { + ginkgo.By(fmt.Sprintf("Deleting storageclass %s", sc.Name)) + cs.StorageV1().StorageClasses().Delete(context.TODO(), sc.Name, metav1.DeleteOptions{}) + } + + for _, vsc := range m.vsc { + ginkgo.By(fmt.Sprintf("Deleting volumesnapshotclass %s", vsc.GetName())) + m.config.Framework.DynamicClient.Resource(utils.SnapshotClassGVR).Delete(context.TODO(), vsc.GetName(), metav1.DeleteOptions{}) + } + ginkgo.By("Cleaning up resources") + for _, cleanupFunc := range m.testCleanups { + cleanupFunc() + } + + err := utilerrors.NewAggregate(errs) + framework.ExpectNoError(err, "while cleaning up after test") +} + +func (m *mockDriverSetup) createPod(withVolume volumeType) (class *storagev1.StorageClass, claim *v1.PersistentVolumeClaim, pod *v1.Pod) { + ginkgo.By("Creating pod") + f := m.f + + sc := m.driver.GetDynamicProvisionStorageClass(m.config, "") + scTest := testsuites.StorageClassTest{ + Name: m.driver.GetDriverInfo().Name, + Timeouts: f.Timeouts, + Provisioner: sc.Provisioner, + Parameters: sc.Parameters, + ClaimSize: "1Gi", + ExpectedSize: "1Gi", + DelayBinding: m.tp.lateBinding, + AllowVolumeExpansion: m.tp.enableResizing, + } + + // The mock driver only works when everything runs on a single node. + nodeSelection := m.config.ClientNodeSelection + switch withVolume { + case csiEphemeral: + pod = startPausePodInline(f.ClientSet, scTest, nodeSelection, f.Namespace.Name) + case genericEphemeral: + class, pod = startPausePodGenericEphemeral(f.ClientSet, scTest, nodeSelection, m.tp.scName, f.Namespace.Name) + if class != nil { + m.sc[class.Name] = class + } + claim = &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: pod.Name + "-" + pod.Spec.Volumes[0].Name, + Namespace: f.Namespace.Name, + }, + } + case pvcReference: + class, claim, pod = startPausePod(f.ClientSet, scTest, nodeSelection, m.tp.scName, f.Namespace.Name) + if class != nil { + m.sc[class.Name] = class + } + if claim != nil { + m.pvcs = append(m.pvcs, claim) + } + } + if pod != nil { + m.pods = append(m.pods, pod) + } + return // result variables set above +} + +func (m *mockDriverSetup) createPodWithPVC(pvc *v1.PersistentVolumeClaim) (*v1.Pod, error) { + f := m.f + + nodeSelection := m.config.ClientNodeSelection + pod, err := startPausePodWithClaim(m.cs, pvc, nodeSelection, f.Namespace.Name) + if pod != nil { + m.pods = append(m.pods, pod) + } + return pod, err +} + +func (m *mockDriverSetup) createPodWithFSGroup(fsGroup *int64) (*storagev1.StorageClass, *v1.PersistentVolumeClaim, *v1.Pod) { + f := m.f + + ginkgo.By("Creating pod with fsGroup") + nodeSelection := m.config.ClientNodeSelection + sc := m.driver.GetDynamicProvisionStorageClass(m.config, "") + scTest := testsuites.StorageClassTest{ + Name: m.driver.GetDriverInfo().Name, + Provisioner: sc.Provisioner, + Parameters: sc.Parameters, + ClaimSize: "1Gi", + ExpectedSize: "1Gi", + DelayBinding: m.tp.lateBinding, + AllowVolumeExpansion: m.tp.enableResizing, + } + class, claim, pod := startBusyBoxPod(f.ClientSet, scTest, nodeSelection, m.tp.scName, f.Namespace.Name, fsGroup) + + if class != nil { + m.sc[class.Name] = class + } + if claim != nil { + m.pvcs = append(m.pvcs, claim) + } + + if pod != nil { + m.pods = append(m.pods, pod) + } + + return class, claim, pod +} + +func (m *mockDriverSetup) createPodWithSELinux(accessModes []v1.PersistentVolumeAccessMode, mountOptions []string, seLinuxOpts *v1.SELinuxOptions) (*storagev1.StorageClass, *v1.PersistentVolumeClaim, *v1.Pod) { + ginkgo.By("Creating pod with SELinux context") + f := m.f + nodeSelection := m.config.ClientNodeSelection + sc := m.driver.GetDynamicProvisionStorageClass(m.config, "") + scTest := testsuites.StorageClassTest{ + Name: m.driver.GetDriverInfo().Name, + Provisioner: sc.Provisioner, + Parameters: sc.Parameters, + ClaimSize: "1Gi", + ExpectedSize: "1Gi", + DelayBinding: m.tp.lateBinding, + AllowVolumeExpansion: m.tp.enableResizing, + MountOptions: mountOptions, + } + class, claim := createClaim(f.ClientSet, scTest, nodeSelection, m.tp.scName, f.Namespace.Name, accessModes) + pod, err := startPausePodWithSELinuxOptions(f.ClientSet, claim, nodeSelection, f.Namespace.Name, seLinuxOpts) + framework.ExpectNoError(err, "Failed to create pause pod with SELinux context %s: %v", seLinuxOpts, err) + + if class != nil { + m.sc[class.Name] = class + } + if claim != nil { + m.pvcs = append(m.pvcs, claim) + } + + if pod != nil { + m.pods = append(m.pods, pod) + } + + return class, claim, pod +} + +func waitForCSIDriver(cs clientset.Interface, driverName string) error { + timeout := 4 * time.Minute + + framework.Logf("waiting up to %v for CSIDriver %q", timeout, driverName) + for start := time.Now(); time.Since(start) < timeout; time.Sleep(framework.Poll) { + _, err := cs.StorageV1().CSIDrivers().Get(context.TODO(), driverName, metav1.GetOptions{}) + if !apierrors.IsNotFound(err) { + return err + } + } + return fmt.Errorf("gave up after waiting %v for CSIDriver %q", timeout, driverName) +} + +func destroyCSIDriver(cs clientset.Interface, driverName string) { + driverGet, err := cs.StorageV1().CSIDrivers().Get(context.TODO(), driverName, metav1.GetOptions{}) + if err == nil { + framework.Logf("deleting %s.%s: %s", driverGet.TypeMeta.APIVersion, driverGet.TypeMeta.Kind, driverGet.ObjectMeta.Name) + // Uncomment the following line to get full dump of CSIDriver object + // framework.Logf("%s", framework.PrettyPrint(driverGet)) + cs.StorageV1().CSIDrivers().Delete(context.TODO(), driverName, metav1.DeleteOptions{}) + } +} + +func newStorageClass(t testsuites.StorageClassTest, ns string, prefix string) *storagev1.StorageClass { + pluginName := t.Provisioner + if pluginName == "" { + pluginName = getDefaultPluginName() + } + if prefix == "" { + prefix = "sc" + } + bindingMode := storagev1.VolumeBindingImmediate + if t.DelayBinding { + bindingMode = storagev1.VolumeBindingWaitForFirstConsumer + } + if t.Parameters == nil { + t.Parameters = make(map[string]string) + } + + if framework.NodeOSDistroIs("windows") { + // fstype might be forced from outside, in that case skip setting a default + if _, exists := t.Parameters["fstype"]; !exists { + t.Parameters["fstype"] = e2epv.GetDefaultFSType() + framework.Logf("settings a default fsType=%s in the storage class", t.Parameters["fstype"]) + } + } + + sc := getStorageClass(pluginName, t.Parameters, &bindingMode, t.MountOptions, ns, prefix) + if t.AllowVolumeExpansion { + sc.AllowVolumeExpansion = &t.AllowVolumeExpansion + } + return sc +} + +func getStorageClass( + provisioner string, + parameters map[string]string, + bindingMode *storagev1.VolumeBindingMode, + mountOptions []string, + ns string, + prefix string, +) *storagev1.StorageClass { + if bindingMode == nil { + defaultBindingMode := storagev1.VolumeBindingImmediate + bindingMode = &defaultBindingMode + } + return &storagev1.StorageClass{ + TypeMeta: metav1.TypeMeta{ + Kind: "StorageClass", + }, + ObjectMeta: metav1.ObjectMeta{ + // Name must be unique, so let's base it on namespace name and the prefix (the prefix is test specific) + GenerateName: ns + "-" + prefix, + }, + Provisioner: provisioner, + Parameters: parameters, + VolumeBindingMode: bindingMode, + MountOptions: mountOptions, + } +} + +func getDefaultPluginName() string { + switch { + case framework.ProviderIs("gke"), framework.ProviderIs("gce"): + return "kubernetes.io/gce-pd" + case framework.ProviderIs("aws"): + return "kubernetes.io/aws-ebs" + case framework.ProviderIs("vsphere"): + return "kubernetes.io/vsphere-volume" + case framework.ProviderIs("azure"): + return "kubernetes.io/azure-disk" + } + return "" +} + +func createSC(cs clientset.Interface, t testsuites.StorageClassTest, scName, ns string) *storagev1.StorageClass { + class := newStorageClass(t, ns, "") + if scName != "" { + class.Name = scName + } + var err error + _, err = cs.StorageV1().StorageClasses().Get(context.TODO(), class.Name, metav1.GetOptions{}) + if err != nil { + class, err = cs.StorageV1().StorageClasses().Create(context.TODO(), class, metav1.CreateOptions{}) + framework.ExpectNoError(err, "Failed to create class: %v", err) + } + + return class +} + +func createClaim(cs clientset.Interface, t testsuites.StorageClassTest, node e2epod.NodeSelection, scName, ns string, accessModes []v1.PersistentVolumeAccessMode) (*storagev1.StorageClass, *v1.PersistentVolumeClaim) { + class := createSC(cs, t, scName, ns) + claim := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{ + ClaimSize: t.ClaimSize, + StorageClassName: &(class.Name), + VolumeMode: &t.VolumeMode, + AccessModes: accessModes, + }, ns) + claim, err := cs.CoreV1().PersistentVolumeClaims(ns).Create(context.TODO(), claim, metav1.CreateOptions{}) + framework.ExpectNoError(err, "Failed to create claim: %v", err) + + if !t.DelayBinding { + pvcClaims := []*v1.PersistentVolumeClaim{claim} + _, err = e2epv.WaitForPVClaimBoundPhase(cs, pvcClaims, framework.ClaimProvisionTimeout) + framework.ExpectNoError(err, "Failed waiting for PVC to be bound: %v", err) + } + return class, claim +} + +func startPausePod(cs clientset.Interface, t testsuites.StorageClassTest, node e2epod.NodeSelection, scName, ns string) (*storagev1.StorageClass, *v1.PersistentVolumeClaim, *v1.Pod) { + class, claim := createClaim(cs, t, node, scName, ns, nil) + + pod, err := startPausePodWithClaim(cs, claim, node, ns) + framework.ExpectNoError(err, "Failed to create pause pod: %v", err) + return class, claim, pod +} + +func startBusyBoxPod(cs clientset.Interface, t testsuites.StorageClassTest, node e2epod.NodeSelection, scName, ns string, fsGroup *int64) (*storagev1.StorageClass, *v1.PersistentVolumeClaim, *v1.Pod) { + class, claim := createClaim(cs, t, node, scName, ns, nil) + pod, err := startBusyBoxPodWithClaim(cs, claim, node, ns, fsGroup) + framework.ExpectNoError(err, "Failed to create busybox pod: %v", err) + return class, claim, pod +} + +func startPausePodInline(cs clientset.Interface, t testsuites.StorageClassTest, node e2epod.NodeSelection, ns string) *v1.Pod { + pod, err := startPausePodWithInlineVolume(cs, + &v1.CSIVolumeSource{ + Driver: t.Provisioner, + }, + node, ns) + framework.ExpectNoError(err, "Failed to create pod: %v", err) + return pod +} + +func startPausePodGenericEphemeral(cs clientset.Interface, t testsuites.StorageClassTest, node e2epod.NodeSelection, scName, ns string) (*storagev1.StorageClass, *v1.Pod) { + class := createSC(cs, t, scName, ns) + claim := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{ + ClaimSize: t.ClaimSize, + StorageClassName: &(class.Name), + VolumeMode: &t.VolumeMode, + }, ns) + pod, err := startPausePodWithVolumeSource(cs, v1.VolumeSource{ + Ephemeral: &v1.EphemeralVolumeSource{ + VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{Spec: claim.Spec}}, + }, node, ns) + framework.ExpectNoError(err, "Failed to create pod: %v", err) + return class, pod +} + +func startPausePodWithClaim(cs clientset.Interface, pvc *v1.PersistentVolumeClaim, node e2epod.NodeSelection, ns string) (*v1.Pod, error) { + return startPausePodWithVolumeSource(cs, + v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc.Name, + ReadOnly: false, + }, + }, + node, ns) +} + +func startBusyBoxPodWithClaim(cs clientset.Interface, pvc *v1.PersistentVolumeClaim, node e2epod.NodeSelection, ns string, fsGroup *int64) (*v1.Pod, error) { + return startBusyBoxPodWithVolumeSource(cs, + v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc.Name, + ReadOnly: false, + }, + }, + node, ns, fsGroup) +} + +func startPausePodWithInlineVolume(cs clientset.Interface, inlineVolume *v1.CSIVolumeSource, node e2epod.NodeSelection, ns string) (*v1.Pod, error) { + return startPausePodWithVolumeSource(cs, + v1.VolumeSource{ + CSI: inlineVolume, + }, + node, ns) +} + +func startPausePodWithVolumeSource(cs clientset.Interface, volumeSource v1.VolumeSource, node e2epod.NodeSelection, ns string) (*v1.Pod, error) { + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "pvc-volume-tester-", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "volume-tester", + Image: imageutils.GetE2EImage(imageutils.Pause), + VolumeMounts: []v1.VolumeMount{ + { + Name: "my-volume", + MountPath: "/mnt/test", + }, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + Volumes: []v1.Volume{ + { + Name: "my-volume", + VolumeSource: volumeSource, + }, + }, + }, + } + e2epod.SetNodeSelection(&pod.Spec, node) + return cs.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{}) +} + +func startBusyBoxPodWithVolumeSource(cs clientset.Interface, volumeSource v1.VolumeSource, node e2epod.NodeSelection, ns string, fsGroup *int64) (*v1.Pod, error) { + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "pvc-volume-tester-", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "volume-tester", + Image: framework.BusyBoxImage, + VolumeMounts: []v1.VolumeMount{ + { + Name: "my-volume", + MountPath: "/mnt/test", + }, + }, + Command: e2epod.GenerateScriptCmd("while true ; do sleep 2; done"), + }, + }, + SecurityContext: &v1.PodSecurityContext{ + FSGroup: fsGroup, + }, + RestartPolicy: v1.RestartPolicyNever, + Volumes: []v1.Volume{ + { + Name: "my-volume", + VolumeSource: volumeSource, + }, + }, + }, + } + e2epod.SetNodeSelection(&pod.Spec, node) + return cs.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{}) +} + +func startPausePodWithSELinuxOptions(cs clientset.Interface, pvc *v1.PersistentVolumeClaim, node e2epod.NodeSelection, ns string, seLinuxOpts *v1.SELinuxOptions) (*v1.Pod, error) { + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "pvc-volume-tester-", + }, + Spec: v1.PodSpec{ + SecurityContext: &v1.PodSecurityContext{ + SELinuxOptions: seLinuxOpts, + }, + Containers: []v1.Container{ + { + Name: "volume-tester", + Image: imageutils.GetE2EImage(imageutils.Pause), + VolumeMounts: []v1.VolumeMount{ + { + Name: "my-volume", + MountPath: "/mnt/test", + }, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + Volumes: []v1.Volume{ + { + Name: "my-volume", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc.Name, + ReadOnly: false, + }, + }, + }, + }, + }, + } + e2epod.SetNodeSelection(&pod.Spec, node) + return cs.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{}) +} + +func checkPodLogs(getCalls func() ([]drivers.MockCSICall, error), pod *v1.Pod, expectPodInfo, ephemeralVolume, csiInlineVolumesEnabled, csiServiceAccountTokenEnabled bool, expectedNumNodePublish int) error { + expectedAttributes := map[string]string{} + if expectPodInfo { + expectedAttributes["csi.storage.k8s.io/pod.name"] = pod.Name + expectedAttributes["csi.storage.k8s.io/pod.namespace"] = pod.Namespace + expectedAttributes["csi.storage.k8s.io/pod.uid"] = string(pod.UID) + expectedAttributes["csi.storage.k8s.io/serviceAccount.name"] = "default" + + } + if csiInlineVolumesEnabled { + // This is only passed in 1.15 when the CSIInlineVolume feature gate is set. + expectedAttributes["csi.storage.k8s.io/ephemeral"] = strconv.FormatBool(ephemeralVolume) + } + + if csiServiceAccountTokenEnabled { + expectedAttributes["csi.storage.k8s.io/serviceAccount.tokens"] = "" + } + + // Find NodePublish in the GRPC calls. + foundAttributes := sets.NewString() + numNodePublishVolume := 0 + numNodeUnpublishVolume := 0 + calls, err := getCalls() + if err != nil { + return err + } + + for _, call := range calls { + switch call.Method { + case "NodePublishVolume": + numNodePublishVolume++ + if numNodePublishVolume == 1 { + // Check that NodePublish had expected attributes for first volume + for k, v := range expectedAttributes { + vv, found := call.Request.VolumeContext[k] + if found && (v == vv || (v == "" && len(vv) != 0)) { + foundAttributes.Insert(k) + framework.Logf("Found volume attribute %s: %s", k, vv) + } + } + } + case "NodeUnpublishVolume": + framework.Logf("Found NodeUnpublishVolume: %+v", call) + numNodeUnpublishVolume++ + } + } + if numNodePublishVolume < expectedNumNodePublish { + return fmt.Errorf("NodePublish should be called at least %d", expectedNumNodePublish) + } + + if numNodeUnpublishVolume == 0 { + return fmt.Errorf("NodeUnpublish was never called") + } + if foundAttributes.Len() != len(expectedAttributes) { + return fmt.Errorf("number of found volume attributes does not match, expected %d, got %d", len(expectedAttributes), foundAttributes.Len()) + } + return nil +} + +// createFSGroupRequestPreHook creates a hook that records the fsGroup passed in +// through NodeStageVolume and NodePublishVolume calls. +func createFSGroupRequestPreHook(nodeStageFsGroup, nodePublishFsGroup *string) *drivers.Hooks { + return &drivers.Hooks{ + Pre: func(ctx context.Context, fullMethod string, request interface{}) (reply interface{}, err error) { + nodeStageRequest, ok := request.(*csipbv1.NodeStageVolumeRequest) + if ok { + mountVolume := nodeStageRequest.GetVolumeCapability().GetMount() + if mountVolume != nil { + *nodeStageFsGroup = mountVolume.VolumeMountGroup + } + } + nodePublishRequest, ok := request.(*csipbv1.NodePublishVolumeRequest) + if ok { + mountVolume := nodePublishRequest.GetVolumeCapability().GetMount() + if mountVolume != nil { + *nodePublishFsGroup = mountVolume.VolumeMountGroup + } + } + return nil, nil + }, + } +} + +// createPreHook counts invocations of a certain method (identified by a substring in the full gRPC method name). +func createPreHook(method string, callback func(counter int64) error) *drivers.Hooks { + var counter int64 + + return &drivers.Hooks{ + Pre: func() func(ctx context.Context, fullMethod string, request interface{}) (reply interface{}, err error) { + return func(ctx context.Context, fullMethod string, request interface{}) (reply interface{}, err error) { + if strings.Contains(fullMethod, method) { + counter := atomic.AddInt64(&counter, 1) + return nil, callback(counter) + } + return nil, nil + } + }(), + } +} + +// compareCSICalls compares expectedCalls with logs of the mock driver. +// It returns index of the first expectedCall that was *not* received +// yet or error when calls do not match. +// All repeated calls to the CSI mock driver (e.g. due to exponential backoff) +// are squashed and checked against single expectedCallSequence item. +// +// Only permanent errors are returned. Other errors are logged and no +// calls are returned. The caller is expected to retry. +func compareCSICalls(trackedCalls []string, expectedCallSequence []csiCall, getCalls func() ([]drivers.MockCSICall, error)) ([]drivers.MockCSICall, int, error) { + allCalls, err := getCalls() + if err != nil { + framework.Logf("intermittent (?) log retrieval error, proceeding without output: %v", err) + return nil, 0, nil + } + + // Remove all repeated and ignored calls + tracked := sets.NewString(trackedCalls...) + var calls []drivers.MockCSICall + var last drivers.MockCSICall + for _, c := range allCalls { + if !tracked.Has(c.Method) { + continue + } + if c.Method != last.Method || c.FullError.Code != last.FullError.Code { + last = c + calls = append(calls, c) + } + // This call is the same as the last one, ignore it. + } + + for i, c := range calls { + if i >= len(expectedCallSequence) { + // Log all unexpected calls first, return error below outside the loop. + framework.Logf("Unexpected CSI driver call: %s (%d)", c.Method, c.FullError) + continue + } + + // Compare current call with expected call + expectedCall := expectedCallSequence[i] + if c.Method != expectedCall.expectedMethod || c.FullError.Code != expectedCall.expectedError { + return allCalls, i, fmt.Errorf("Unexpected CSI call %d: expected %s (%d), got %s (%d)", i, expectedCall.expectedMethod, expectedCall.expectedError, c.Method, c.FullError.Code) + } + } + if len(calls) > len(expectedCallSequence) { + return allCalls, len(expectedCallSequence), fmt.Errorf("Received %d unexpected CSI driver calls", len(calls)-len(expectedCallSequence)) + } + // All calls were correct + return allCalls, len(calls), nil + +} + +// createSELinuxMountPreHook creates a hook that records the mountOptions passed in +// through NodeStageVolume and NodePublishVolume calls. +func createSELinuxMountPreHook(nodeStageMountOpts, nodePublishMountOpts *[]string) *drivers.Hooks { + return &drivers.Hooks{ + Pre: func(ctx context.Context, fullMethod string, request interface{}) (reply interface{}, err error) { + nodeStageRequest, ok := request.(*csipbv1.NodeStageVolumeRequest) + if ok { + mountVolume := nodeStageRequest.GetVolumeCapability().GetMount() + if mountVolume != nil { + *nodeStageMountOpts = mountVolume.MountFlags + } + } + nodePublishRequest, ok := request.(*csipbv1.NodePublishVolumeRequest) + if ok { + mountVolume := nodePublishRequest.GetVolumeCapability().GetMount() + if mountVolume != nil { + *nodePublishMountOpts = mountVolume.MountFlags + } + } + return nil, nil + }, + } +} + +// A lot of this code was copied from e2e/framework. It would be nicer +// if it could be reused - see https://github.com/kubernetes/kubernetes/issues/92754 +func podRunning(ctx context.Context, c clientset.Interface, podName, namespace string) wait.ConditionFunc { + return func() (bool, error) { + pod, err := c.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + return false, err + } + switch pod.Status.Phase { + case v1.PodRunning: + return true, nil + case v1.PodFailed, v1.PodSucceeded: + return false, errPodCompleted + } + return false, nil + } +} + +func podHasStorage(ctx context.Context, c clientset.Interface, podName, namespace string, when time.Time) wait.ConditionFunc { + // Check for events of this pod. Copied from test/e2e/common/container_probe.go. + expectedEvent := fields.Set{ + "involvedObject.kind": "Pod", + "involvedObject.name": podName, + "involvedObject.namespace": namespace, + "reason": "FailedScheduling", + }.AsSelector().String() + options := metav1.ListOptions{ + FieldSelector: expectedEvent, + } + // copied from test/e2e/framework/events/events.go + return func() (bool, error) { + // We cannot be sure here whether it has enough storage, only when + // it hasn't. In that case we abort waiting with a special error. + events, err := c.CoreV1().Events(namespace).List(ctx, options) + if err != nil { + return false, fmt.Errorf("got error while getting events: %w", err) + } + for _, event := range events.Items { + if /* event.CreationTimestamp.After(when) && + */strings.Contains(event.Message, errReasonNotEnoughSpace) { + return false, errNotEnoughSpace + } + } + return false, nil + } +} + +func anyOf(conditions ...wait.ConditionFunc) wait.ConditionFunc { + return func() (bool, error) { + for _, condition := range conditions { + done, err := condition() + if err != nil { + return false, err + } + if done { + return true, nil + } + } + return false, nil + } +} + +func waitForMaxVolumeCondition(pod *v1.Pod, cs clientset.Interface) error { + waitErr := wait.PollImmediate(10*time.Second, csiPodUnschedulableTimeout, func() (bool, error) { + pod, err := cs.CoreV1().Pods(pod.Namespace).Get(context.TODO(), pod.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + for _, c := range pod.Status.Conditions { + // Conformance tests cannot rely on specific output of optional fields (e.g., Reason + // and Message) because these fields are not suject to the deprecation policy. + if c.Type == v1.PodScheduled && c.Status == v1.ConditionFalse && c.Reason != "" && c.Message != "" { + return true, nil + } + } + return false, nil + }) + if waitErr != nil { + return fmt.Errorf("error waiting for pod %s/%s to have max volume condition: %v", pod.Namespace, pod.Name, waitErr) + } + return nil +} diff --git a/test/e2e/storage/csi_mock/csi_attach_volume.go b/test/e2e/storage/csi_mock/csi_attach_volume.go new file mode 100644 index 0000000000000..fb2a9e829ebec --- /dev/null +++ b/test/e2e/storage/csi_mock/csi_attach_volume.go @@ -0,0 +1,178 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi_mock + +import ( + "context" + "fmt" + "time" + + "github.com/onsi/ginkgo/v2" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/kubernetes/pkg/kubelet/events" + "k8s.io/kubernetes/test/e2e/framework" + e2eevents "k8s.io/kubernetes/test/e2e/framework/events" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + e2evolume "k8s.io/kubernetes/test/e2e/framework/volume" + storageframework "k8s.io/kubernetes/test/e2e/storage/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" + admissionapi "k8s.io/pod-security-admission/api" +) + +var _ = utils.SIGDescribe("CSI Mock volume attach", func() { + // The CSIDriverRegistry feature gate is needed for this test in Kubernetes 1.12. + f := framework.NewDefaultFramework("csi-mock-volumes-attach") + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged + m := newMockDriverSetup(f) + + ginkgo.Context("CSI attach test using mock driver", func() { + tests := []struct { + name string + disableAttach bool + deployClusterRegistrar bool + volumeType volumeType + }{ + { + name: "should not require VolumeAttach for drivers without attachment", + disableAttach: true, + deployClusterRegistrar: true, + }, + { + name: "should require VolumeAttach for drivers with attachment", + deployClusterRegistrar: true, + }, + { + name: "should require VolumeAttach for ephemermal volume and drivers with attachment", + deployClusterRegistrar: true, + volumeType: genericEphemeral, + }, + { + name: "should preserve attachment policy when no CSIDriver present", + deployClusterRegistrar: false, + }, + } + for _, t := range tests { + test := t + ginkgo.It(t.name, func(ctx context.Context) { + var err error + m.init(testParameters{registerDriver: test.deployClusterRegistrar, disableAttach: test.disableAttach}) + defer m.cleanup() + + volumeType := test.volumeType + if volumeType == "" { + volumeType = pvcReference + } + _, claim, pod := m.createPod(volumeType) + if pod == nil { + return + } + err = e2epod.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) + framework.ExpectNoError(err, "Failed to start pod: %v", err) + + ginkgo.By("Checking if VolumeAttachment was created for the pod") + testConfig := storageframework.ConvertTestConfig(m.config) + attachmentName := e2evolume.GetVolumeAttachmentName(m.cs, testConfig, m.provisioner, claim.Name, claim.Namespace) + _, err = m.cs.StorageV1().VolumeAttachments().Get(context.TODO(), attachmentName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + if !test.disableAttach { + framework.ExpectNoError(err, "Expected VolumeAttachment but none was found") + } + } else { + framework.ExpectNoError(err, "Failed to find VolumeAttachment") + } + } + if test.disableAttach { + framework.ExpectError(err, "Unexpected VolumeAttachment found") + } + }) + + } + }) + + ginkgo.Context("CSI CSIDriver deployment after pod creation using non-attachable mock driver", func() { + ginkgo.It("should bringup pod after deploying CSIDriver attach=false [Slow]", func(ctx context.Context) { + var err error + m.init(testParameters{registerDriver: false, disableAttach: true}) + defer m.cleanup() + + _, claim, pod := m.createPod(pvcReference) // late binding as specified above + if pod == nil { + return + } + + ginkgo.By("Checking if attaching failed and pod cannot start") + eventSelector := fields.Set{ + "involvedObject.kind": "Pod", + "involvedObject.name": pod.Name, + "involvedObject.namespace": pod.Namespace, + "reason": events.FailedAttachVolume, + }.AsSelector().String() + msg := "AttachVolume.Attach failed for volume" + + err = e2eevents.WaitTimeoutForEvent(m.cs, pod.Namespace, eventSelector, msg, f.Timeouts.PodStart) + if err != nil { + podErr := e2epod.WaitTimeoutForPodRunningInNamespace(m.cs, pod.Name, pod.Namespace, 10*time.Second) + framework.ExpectError(podErr, "Pod should not be in running status because attaching should failed") + // Events are unreliable, don't depend on the event. It's used only to speed up the test. + framework.Logf("Attach should fail and the corresponding event should show up, error: %v", err) + } + + // VolumeAttachment should be created because the default value for CSI attachable is true + ginkgo.By("Checking if VolumeAttachment was created for the pod") + testConfig := storageframework.ConvertTestConfig(m.config) + attachmentName := e2evolume.GetVolumeAttachmentName(m.cs, testConfig, m.provisioner, claim.Name, claim.Namespace) + _, err = m.cs.StorageV1().VolumeAttachments().Get(context.TODO(), attachmentName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + framework.ExpectNoError(err, "Expected VolumeAttachment but none was found") + } else { + framework.ExpectNoError(err, "Failed to find VolumeAttachment") + } + } + + ginkgo.By("Deploy CSIDriver object with attachRequired=false") + driverNamespace := m.config.DriverNamespace + + canAttach := false + o := utils.PatchCSIOptions{ + OldDriverName: "csi-mock", + NewDriverName: "csi-mock-" + f.UniqueName, + CanAttach: &canAttach, + } + cleanupCSIDriver, err := utils.CreateFromManifests(f, driverNamespace, func(item interface{}) error { + return utils.PatchCSIDeployment(f, o, item) + }, "test/e2e/testing-manifests/storage-csi/mock/csi-mock-driverinfo.yaml") + if err != nil { + framework.Failf("fail to deploy CSIDriver object: %v", err) + } + m.testCleanups = append(m.testCleanups, cleanupCSIDriver) + + ginkgo.By("Wait for the pod in running status") + err = e2epod.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) + framework.ExpectNoError(err, "Failed to start pod: %v", err) + + ginkgo.By(fmt.Sprintf("Wait for the volumeattachment to be deleted up to %v", csiVolumeAttachmentTimeout)) + // This step can be slow because we have to wait either a NodeUpdate event happens or + // the detachment for this volume timeout so that we can do a force detach. + err = e2evolume.WaitForVolumeAttachmentTerminated(attachmentName, m.cs, csiVolumeAttachmentTimeout) + framework.ExpectNoError(err, "Failed to delete VolumeAttachment: %v", err) + }) + }) +}) diff --git a/test/e2e/storage/csi_mock/csi_fsgroup_mount.go b/test/e2e/storage/csi_mock/csi_fsgroup_mount.go new file mode 100644 index 0000000000000..3077d578a4afe --- /dev/null +++ b/test/e2e/storage/csi_mock/csi_fsgroup_mount.go @@ -0,0 +1,86 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi_mock + +import ( + "context" + "math/rand" + "strconv" + + "github.com/onsi/ginkgo/v2" + "k8s.io/kubernetes/test/e2e/framework" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" + "k8s.io/kubernetes/test/e2e/storage/utils" + admissionapi "k8s.io/pod-security-admission/api" +) + +var _ = utils.SIGDescribe("CSI Mock fsgroup as mount option", func() { + f := framework.NewDefaultFramework("csi-mock-volumes-fsgroup-mount") + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged + m := newMockDriverSetup(f) + + ginkgo.Context("Delegate FSGroup to CSI driver [LinuxOnly]", func() { + tests := []struct { + name string + enableVolumeMountGroup bool + }{ + { + name: "should pass FSGroup to CSI driver if it is set in pod and driver supports VOLUME_MOUNT_GROUP", + enableVolumeMountGroup: true, + }, + { + name: "should not pass FSGroup to CSI driver if it is set in pod and driver supports VOLUME_MOUNT_GROUP", + enableVolumeMountGroup: false, + }, + } + for _, t := range tests { + t := t + ginkgo.It(t.name, func(ctx context.Context) { + var nodeStageFsGroup, nodePublishFsGroup string + if framework.NodeOSDistroIs("windows") { + e2eskipper.Skipf("FSGroupPolicy is only applied on linux nodes -- skipping") + } + m.init(testParameters{ + disableAttach: true, + registerDriver: true, + enableVolumeMountGroup: t.enableVolumeMountGroup, + hooks: createFSGroupRequestPreHook(&nodeStageFsGroup, &nodePublishFsGroup), + }) + defer m.cleanup() + + fsGroupVal := int64(rand.Int63n(20000) + 1024) + fsGroup := &fsGroupVal + fsGroupStr := strconv.FormatInt(fsGroupVal, 10 /* base */) + + _, _, pod := m.createPodWithFSGroup(fsGroup) /* persistent volume */ + + err := e2epod.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) + framework.ExpectNoError(err, "failed to start pod") + + if t.enableVolumeMountGroup { + framework.ExpectEqual(nodeStageFsGroup, fsGroupStr, "Expect NodeStageVolumeRequest.VolumeCapability.MountVolume.VolumeMountGroup to equal %q; got: %q", fsGroupStr, nodeStageFsGroup) + framework.ExpectEqual(nodePublishFsGroup, fsGroupStr, "Expect NodePublishVolumeRequest.VolumeCapability.MountVolume.VolumeMountGroup to equal %q; got: %q", fsGroupStr, nodePublishFsGroup) + } else { + framework.ExpectEmpty(nodeStageFsGroup, "Expect NodeStageVolumeRequest.VolumeCapability.MountVolume.VolumeMountGroup to be empty; got: %q", nodeStageFsGroup) + framework.ExpectEmpty(nodePublishFsGroup, "Expect NodePublishVolumeRequest.VolumeCapability.MountVolume.VolumeMountGroup to empty; got: %q", nodePublishFsGroup) + } + }) + } + }) + +}) diff --git a/test/e2e/storage/csi_mock/csi_fsgroup_policy.go b/test/e2e/storage/csi_mock/csi_fsgroup_policy.go new file mode 100644 index 0000000000000..5b2b3642184ad --- /dev/null +++ b/test/e2e/storage/csi_mock/csi_fsgroup_policy.go @@ -0,0 +1,124 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi_mock + +import ( + "context" + "fmt" + "math/rand" + "strconv" + "time" + + "github.com/onsi/ginkgo/v2" + storagev1 "k8s.io/api/storage/v1" + "k8s.io/kubernetes/test/e2e/framework" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" + e2evolume "k8s.io/kubernetes/test/e2e/framework/volume" + "k8s.io/kubernetes/test/e2e/storage/utils" + admissionapi "k8s.io/pod-security-admission/api" +) + +var _ = utils.SIGDescribe("CSI Mock volume fsgroup policies", func() { + f := framework.NewDefaultFramework("csi-mock-volumes-fsgroup-policy") + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged + m := newMockDriverSetup(f) + + // These tests *only* work on a cluster which has the CSIVolumeFSGroupPolicy feature enabled. + ginkgo.Context("CSI FSGroupPolicy [LinuxOnly]", func() { + tests := []struct { + name string + fsGroupPolicy storagev1.FSGroupPolicy + modified bool + }{ + { + name: "should modify fsGroup if fsGroupPolicy=default", + fsGroupPolicy: storagev1.ReadWriteOnceWithFSTypeFSGroupPolicy, + modified: true, + }, + { + name: "should modify fsGroup if fsGroupPolicy=File", + fsGroupPolicy: storagev1.FileFSGroupPolicy, + modified: true, + }, + { + name: "should not modify fsGroup if fsGroupPolicy=None", + fsGroupPolicy: storagev1.NoneFSGroupPolicy, + modified: false, + }, + } + for _, t := range tests { + test := t + ginkgo.It(test.name, func(ctx context.Context) { + if framework.NodeOSDistroIs("windows") { + e2eskipper.Skipf("FSGroupPolicy is only applied on linux nodes -- skipping") + } + m.init(testParameters{ + disableAttach: true, + registerDriver: true, + fsGroupPolicy: &test.fsGroupPolicy, + }) + defer m.cleanup() + + // kube-scheduler may need some time before it gets the CSIDriver object. + // Without them, scheduling doesn't run as expected by the test. + syncDelay := 5 * time.Second + time.Sleep(syncDelay) + + fsGroupVal := int64(rand.Int63n(20000) + 1024) + fsGroup := &fsGroupVal + + _, _, pod := m.createPodWithFSGroup(fsGroup) /* persistent volume */ + + mountPath := pod.Spec.Containers[0].VolumeMounts[0].MountPath + dirName := mountPath + "/" + f.UniqueName + fileName := dirName + "/" + f.UniqueName + + err := e2epod.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) + framework.ExpectNoError(err, "failed to start pod") + + // Create the subdirectory to ensure that fsGroup propagates + createDirectory := fmt.Sprintf("mkdir %s", dirName) + _, _, err = e2evolume.PodExec(f, pod, createDirectory) + framework.ExpectNoError(err, "failed: creating the directory: %s", err) + + // Inject the contents onto the mount + createFile := fmt.Sprintf("echo '%s' > '%s'; sync", "filecontents", fileName) + _, _, err = e2evolume.PodExec(f, pod, createFile) + framework.ExpectNoError(err, "failed: writing the contents: %s", err) + + // Delete the created file. This step is mandatory, as the mock driver + // won't clean up the contents automatically. + defer func() { + delete := fmt.Sprintf("rm -fr %s", dirName) + _, _, err = e2evolume.PodExec(f, pod, delete) + framework.ExpectNoError(err, "failed: deleting the directory: %s", err) + }() + + // Ensure that the fsGroup matches what we expect + if test.modified { + utils.VerifyFSGroupInPod(f, fileName, strconv.FormatInt(*fsGroup, 10), pod) + } else { + utils.VerifyFSGroupInPod(f, fileName, "root", pod) + } + + // The created resources will be removed by the cleanup() function, + // so need to delete anything here. + }) + } + }) +}) diff --git a/test/e2e/storage/csi_mock/csi_node_stage_error_cases.go b/test/e2e/storage/csi_mock/csi_node_stage_error_cases.go new file mode 100644 index 0000000000000..800ca0d8e9434 --- /dev/null +++ b/test/e2e/storage/csi_mock/csi_node_stage_error_cases.go @@ -0,0 +1,330 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi_mock + +import ( + "context" + "sync/atomic" + "time" + + "github.com/onsi/ginkgo/v2" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/kubernetes/test/e2e/framework" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + e2epv "k8s.io/kubernetes/test/e2e/framework/pv" + "k8s.io/kubernetes/test/e2e/storage/drivers" + "k8s.io/kubernetes/test/e2e/storage/utils" + admissionapi "k8s.io/pod-security-admission/api" +) + +var _ = utils.SIGDescribe("CSI Mock volume node stage", func() { + f := framework.NewDefaultFramework("csi-mock-volumes-node-stage") + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged + m := newMockDriverSetup(f) + + ginkgo.Context("CSI NodeStage error cases [Slow]", func() { + trackedCalls := []string{ + "NodeStageVolume", + "NodeUnstageVolume", + } + + tests := []struct { + name string + expectPodRunning bool + expectedCalls []csiCall + + // Called for each NodeStateVolume calls, with counter incremented atomically before + // the invocation (i.e. first value will be 1). + nodeStageHook func(counter int64) error + }{ + { + // This is already tested elsewhere, adding simple good case here to test the test framework. + name: "should call NodeUnstage after NodeStage success", + expectPodRunning: true, + expectedCalls: []csiCall{ + {expectedMethod: "NodeStageVolume", expectedError: codes.OK, deletePod: true}, + {expectedMethod: "NodeUnstageVolume", expectedError: codes.OK}, + }, + }, + { + // Kubelet should repeat NodeStage as long as the pod exists + name: "should retry NodeStage after NodeStage final error", + expectPodRunning: true, + expectedCalls: []csiCall{ + // This matches all 3 NodeStage calls with InvalidArgument error + {expectedMethod: "NodeStageVolume", expectedError: codes.InvalidArgument}, + {expectedMethod: "NodeStageVolume", expectedError: codes.OK, deletePod: true}, + {expectedMethod: "NodeUnstageVolume", expectedError: codes.OK}, + }, + // Fail first 3 NodeStage requests, 4th succeeds + nodeStageHook: func(counter int64) error { + if counter < 4 { + return status.Error(codes.InvalidArgument, "fake error") + } + return nil + }, + }, + { + // Kubelet should repeat NodeStage as long as the pod exists + name: "should retry NodeStage after NodeStage ephemeral error", + expectPodRunning: true, + expectedCalls: []csiCall{ + // This matches all 3 NodeStage calls with DeadlineExceeded error + {expectedMethod: "NodeStageVolume", expectedError: codes.DeadlineExceeded}, + {expectedMethod: "NodeStageVolume", expectedError: codes.OK, deletePod: true}, + {expectedMethod: "NodeUnstageVolume", expectedError: codes.OK}, + }, + // Fail first 3 NodeStage requests, 4th succeeds + nodeStageHook: func(counter int64) error { + if counter < 4 { + return status.Error(codes.DeadlineExceeded, "fake error") + } + return nil + }, + }, + { + // After NodeUnstage with ephemeral error, the driver may continue staging the volume. + // Kubelet should call NodeUnstage to make sure the volume is really unstaged after + // the pod is deleted. + name: "should call NodeUnstage after NodeStage ephemeral error", + expectPodRunning: false, + expectedCalls: []csiCall{ + // Delete the pod before NodeStage succeeds - it should get "uncertain" because of ephemeral error + // This matches all repeated NodeStage calls with DeadlineExceeded error (due to exp. backoff). + {expectedMethod: "NodeStageVolume", expectedError: codes.DeadlineExceeded, deletePod: true}, + {expectedMethod: "NodeUnstageVolume", expectedError: codes.OK}, + }, + nodeStageHook: func(counter int64) error { + return status.Error(codes.DeadlineExceeded, "fake error") + }, + }, + { + // After NodeUnstage with final error, kubelet can be sure the volume is not staged. + // The test checks that NodeUnstage is *not* called. + name: "should not call NodeUnstage after NodeStage final error", + expectPodRunning: false, + expectedCalls: []csiCall{ + // Delete the pod before NodeStage succeeds - it should get "globally unmounted" because of final error. + // This matches all repeated NodeStage calls with InvalidArgument error (due to exp. backoff). + {expectedMethod: "NodeStageVolume", expectedError: codes.InvalidArgument, deletePod: true}, + }, + // nodeStageScript: `INVALIDARGUMENT;`, + nodeStageHook: func(counter int64) error { + return status.Error(codes.InvalidArgument, "fake error") + }, + }, + } + for _, t := range tests { + test := t + ginkgo.It(test.name, func(ctx context.Context) { + var hooks *drivers.Hooks + if test.nodeStageHook != nil { + hooks = createPreHook("NodeStageVolume", test.nodeStageHook) + } + m.init(testParameters{ + disableAttach: true, + registerDriver: true, + hooks: hooks, + }) + defer m.cleanup() + + _, claim, pod := m.createPod(pvcReference) + if pod == nil { + return + } + // Wait for PVC to get bound to make sure the CSI driver is fully started. + err := e2epv.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, f.ClientSet, f.Namespace.Name, claim.Name, time.Second, framework.ClaimProvisionTimeout) + framework.ExpectNoError(err, "while waiting for PVC to get provisioned") + + ginkgo.By("Waiting for expected CSI calls") + // Watch for all calls up to deletePod = true + ctx, cancel := context.WithTimeout(ctx, csiPodRunningTimeout) + defer cancel() + for { + if ctx.Err() != nil { + framework.Failf("timed out waiting for the CSI call that indicates that the pod can be deleted: %v", test.expectedCalls) + } + time.Sleep(1 * time.Second) + _, index, err := compareCSICalls(trackedCalls, test.expectedCalls, m.driver.GetCalls) + framework.ExpectNoError(err, "while waiting for initial CSI calls") + if index == 0 { + // No CSI call received yet + continue + } + // Check the last *received* call wanted the pod to be deleted + if test.expectedCalls[index-1].deletePod { + break + } + } + + if test.expectPodRunning { + ginkgo.By("Waiting for pod to be running") + err := e2epod.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) + framework.ExpectNoError(err, "Failed to start pod: %v", err) + } + + ginkgo.By("Deleting the previously created pod") + err = e2epod.DeletePodWithWait(m.cs, pod) + framework.ExpectNoError(err, "while deleting") + + ginkgo.By("Waiting for all remaining expected CSI calls") + err = wait.Poll(time.Second, csiUnstageWaitTimeout, func() (done bool, err error) { + _, index, err := compareCSICalls(trackedCalls, test.expectedCalls, m.driver.GetCalls) + if err != nil { + return true, err + } + if index == 0 { + // No CSI call received yet + return false, nil + } + if len(test.expectedCalls) == index { + // all calls received + return true, nil + } + return false, nil + }) + framework.ExpectNoError(err, "while waiting for all CSI calls") + }) + } + }) + + ginkgo.Context("CSI NodeUnstage error cases [Slow]", func() { + trackedCalls := []string{ + "NodeStageVolume", + "NodeUnstageVolume", + } + + // Each test starts two pods in sequence. + // The first pod always runs successfully, but NodeUnstage hook can set various error conditions. + // The test then checks how NodeStage of the second pod is called. + tests := []struct { + name string + expectedCalls []csiCall + + // Called for each NodeStageVolume calls, with counter incremented atomically before + // the invocation (i.e. first value will be 1) and index of deleted pod (the first pod + // has index 1) + nodeUnstageHook func(counter, pod int64) error + }{ + { + // This is already tested elsewhere, adding simple good case here to test the test framework. + name: "should call NodeStage after NodeUnstage success", + expectedCalls: []csiCall{ + {expectedMethod: "NodeStageVolume", expectedError: codes.OK}, + {expectedMethod: "NodeUnstageVolume", expectedError: codes.OK}, + {expectedMethod: "NodeStageVolume", expectedError: codes.OK}, + {expectedMethod: "NodeUnstageVolume", expectedError: codes.OK}, + }, + }, + { + name: "two pods: should call NodeStage after previous NodeUnstage final error", + expectedCalls: []csiCall{ + {expectedMethod: "NodeStageVolume", expectedError: codes.OK}, + {expectedMethod: "NodeUnstageVolume", expectedError: codes.InvalidArgument}, + {expectedMethod: "NodeStageVolume", expectedError: codes.OK}, + {expectedMethod: "NodeUnstageVolume", expectedError: codes.OK}, + }, + nodeUnstageHook: func(counter, pod int64) error { + if pod == 1 { + return status.Error(codes.InvalidArgument, "fake final error") + } + return nil + }, + }, + { + name: "two pods: should call NodeStage after previous NodeUnstage transient error", + expectedCalls: []csiCall{ + {expectedMethod: "NodeStageVolume", expectedError: codes.OK}, + {expectedMethod: "NodeUnstageVolume", expectedError: codes.DeadlineExceeded}, + {expectedMethod: "NodeStageVolume", expectedError: codes.OK}, + {expectedMethod: "NodeUnstageVolume", expectedError: codes.OK}, + }, + nodeUnstageHook: func(counter, pod int64) error { + if pod == 1 { + return status.Error(codes.DeadlineExceeded, "fake transient error") + } + return nil + }, + }, + } + for _, t := range tests { + test := t + ginkgo.It(test.name, func(ctx context.Context) { + // Index of the last deleted pod. NodeUnstage calls are then related to this pod. + var deletedPodNumber int64 = 1 + var hooks *drivers.Hooks + if test.nodeUnstageHook != nil { + hooks = createPreHook("NodeUnstageVolume", func(counter int64) error { + pod := atomic.LoadInt64(&deletedPodNumber) + return test.nodeUnstageHook(counter, pod) + }) + } + m.init(testParameters{ + disableAttach: true, + registerDriver: true, + hooks: hooks, + }) + defer m.cleanup() + + _, claim, pod := m.createPod(pvcReference) + if pod == nil { + return + } + // Wait for PVC to get bound to make sure the CSI driver is fully started. + err := e2epv.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, f.ClientSet, f.Namespace.Name, claim.Name, time.Second, framework.ClaimProvisionTimeout) + framework.ExpectNoError(err, "while waiting for PVC to get provisioned") + err = e2epod.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) + framework.ExpectNoError(err, "while waiting for the first pod to start") + err = e2epod.DeletePodWithWait(m.cs, pod) + framework.ExpectNoError(err, "while deleting the first pod") + + // Create the second pod + pod, err = m.createPodWithPVC(claim) + framework.ExpectNoError(err, "while creating the second pod") + err = e2epod.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) + framework.ExpectNoError(err, "while waiting for the second pod to start") + // The second pod is running and kubelet can't call NodeUnstage of the first one. + // Therefore incrementing the pod counter is safe here. + atomic.AddInt64(&deletedPodNumber, 1) + err = e2epod.DeletePodWithWait(m.cs, pod) + framework.ExpectNoError(err, "while deleting the second pod") + + ginkgo.By("Waiting for all remaining expected CSI calls") + err = wait.Poll(time.Second, csiUnstageWaitTimeout, func() (done bool, err error) { + _, index, err := compareCSICalls(trackedCalls, test.expectedCalls, m.driver.GetCalls) + if err != nil { + return true, err + } + if index == 0 { + // No CSI call received yet + return false, nil + } + if len(test.expectedCalls) == index { + // all calls received + return true, nil + } + return false, nil + }) + framework.ExpectNoError(err, "while waiting for all CSI calls") + }) + } + }) + +}) diff --git a/test/e2e/storage/csi_mock/csi_selinux_mount.go b/test/e2e/storage/csi_mock/csi_selinux_mount.go new file mode 100644 index 0000000000000..4355d69681c55 --- /dev/null +++ b/test/e2e/storage/csi_mock/csi_selinux_mount.go @@ -0,0 +1,122 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi_mock + +import ( + "context" + + "github.com/onsi/ginkgo/v2" + v1 "k8s.io/api/core/v1" + "k8s.io/kubernetes/test/e2e/framework" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" + "k8s.io/kubernetes/test/e2e/storage/utils" + admissionapi "k8s.io/pod-security-admission/api" +) + +var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { + f := framework.NewDefaultFramework("csi-mock-volumes-selinux") + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged + m := newMockDriverSetup(f) + + ginkgo.Context("SELinuxMount [LinuxOnly][Feature:SELinux][Feature:SELinuxMountReadWriteOncePod]", func() { + // Make sure all options are set so system specific defaults are not used. + seLinuxOpts := v1.SELinuxOptions{ + User: "system_u", + Role: "object_r", + Type: "container_file_t", + Level: "s0:c0,c1", + } + seLinuxMountOption := "context=\"system_u:object_r:container_file_t:s0:c0,c1\"" + + tests := []struct { + name string + seLinuxEnabled bool + seLinuxSetInPod bool + mountOptions []string + volumeMode v1.PersistentVolumeAccessMode + expectedMountOptions []string + }{ + { + name: "should pass SELinux mount option for RWOP volume and Pod with SELinux context set", + seLinuxEnabled: true, + seLinuxSetInPod: true, + volumeMode: v1.ReadWriteOncePod, + expectedMountOptions: []string{seLinuxMountOption}, + }, + { + name: "should add SELinux mount option to existing mount options", + seLinuxEnabled: true, + seLinuxSetInPod: true, + mountOptions: []string{"noexec", "noatime"}, + volumeMode: v1.ReadWriteOncePod, + expectedMountOptions: []string{"noexec", "noatime", seLinuxMountOption}, + }, + { + name: "should not pass SELinux mount option for RWO volume", + seLinuxEnabled: true, + seLinuxSetInPod: true, + volumeMode: v1.ReadWriteOnce, + expectedMountOptions: nil, + }, + { + name: "should not pass SELinux mount option for Pod without SELinux context", + seLinuxEnabled: true, + seLinuxSetInPod: false, + volumeMode: v1.ReadWriteOncePod, + expectedMountOptions: nil, + }, + { + name: "should not pass SELinux mount option for CSI driver that does not support SELinux mount", + seLinuxEnabled: false, + seLinuxSetInPod: true, + volumeMode: v1.ReadWriteOncePod, + expectedMountOptions: nil, + }, + } + for _, t := range tests { + t := t + ginkgo.It(t.name, func(ctx context.Context) { + if framework.NodeOSDistroIs("windows") { + e2eskipper.Skipf("SELinuxMount is only applied on linux nodes -- skipping") + } + var nodeStageMountOpts, nodePublishMountOpts []string + m.init(testParameters{ + disableAttach: true, + registerDriver: true, + enableSELinuxMount: &t.seLinuxEnabled, + hooks: createSELinuxMountPreHook(&nodeStageMountOpts, &nodePublishMountOpts), + }) + defer m.cleanup() + + accessModes := []v1.PersistentVolumeAccessMode{t.volumeMode} + var podSELinuxOpts *v1.SELinuxOptions + if t.seLinuxSetInPod { + // Make sure all options are set so system specific defaults are not used. + podSELinuxOpts = &seLinuxOpts + } + + _, _, pod := m.createPodWithSELinux(accessModes, t.mountOptions, podSELinuxOpts) + err := e2epod.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) + framework.ExpectNoError(err, "failed to start pod") + + framework.ExpectEqual(nodeStageMountOpts, t.expectedMountOptions, "Expect NodeStageVolumeRequest.VolumeCapability.MountVolume. to equal %q; got: %q", t.expectedMountOptions, nodeStageMountOpts) + framework.ExpectEqual(nodePublishMountOpts, t.expectedMountOptions, "Expect NodePublishVolumeRequest.VolumeCapability.MountVolume.VolumeMountGroup to equal %q; got: %q", t.expectedMountOptions, nodeStageMountOpts) + }) + } + }) +}) diff --git a/test/e2e/storage/csi_mock/csi_service_account_token.go b/test/e2e/storage/csi_mock/csi_service_account_token.go new file mode 100644 index 0000000000000..1e079fe447fbf --- /dev/null +++ b/test/e2e/storage/csi_mock/csi_service_account_token.go @@ -0,0 +1,98 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi_mock + +import ( + "context" + "time" + + "github.com/onsi/ginkgo/v2" + storagev1 "k8s.io/api/storage/v1" + "k8s.io/kubernetes/test/e2e/framework" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + "k8s.io/kubernetes/test/e2e/storage/utils" + admissionapi "k8s.io/pod-security-admission/api" + utilptr "k8s.io/utils/pointer" +) + +var _ = utils.SIGDescribe("CSI Mock volume service account token", func() { + f := framework.NewDefaultFramework("csi-mock-volumes-service-token") + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged + m := newMockDriverSetup(f) + + ginkgo.Context("CSIServiceAccountToken", func() { + var ( + err error + ) + tests := []struct { + desc string + deployCSIDriverObject bool + tokenRequests []storagev1.TokenRequest + }{ + { + desc: "token should not be plumbed down when csiServiceAccountTokenEnabled=false", + deployCSIDriverObject: true, + tokenRequests: nil, + }, + { + desc: "token should not be plumbed down when CSIDriver is not deployed", + deployCSIDriverObject: false, + tokenRequests: []storagev1.TokenRequest{{}}, + }, + { + desc: "token should be plumbed down when csiServiceAccountTokenEnabled=true", + deployCSIDriverObject: true, + tokenRequests: []storagev1.TokenRequest{{ExpirationSeconds: utilptr.Int64Ptr(60 * 10)}}, + }, + } + for _, test := range tests { + test := test + csiServiceAccountTokenEnabled := test.tokenRequests != nil + ginkgo.It(test.desc, func(ctx context.Context) { + m.init(testParameters{ + registerDriver: test.deployCSIDriverObject, + tokenRequests: test.tokenRequests, + requiresRepublish: &csiServiceAccountTokenEnabled, + }) + + defer m.cleanup() + + _, _, pod := m.createPod(pvcReference) + if pod == nil { + return + } + err = e2epod.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) + framework.ExpectNoError(err, "Failed to start pod: %v", err) + + // sleep to make sure RequiresRepublish triggers more than 1 NodePublishVolume + numNodePublishVolume := 1 + if test.deployCSIDriverObject && csiServiceAccountTokenEnabled { + time.Sleep(5 * time.Second) + numNodePublishVolume = 2 + } + + ginkgo.By("Deleting the previously created pod") + err = e2epod.DeletePodWithWait(m.cs, pod) + framework.ExpectNoError(err, "while deleting") + + ginkgo.By("Checking CSI driver logs") + err = checkPodLogs(m.driver.GetCalls, pod, false, false, false, test.deployCSIDriverObject && csiServiceAccountTokenEnabled, numNodePublishVolume) + framework.ExpectNoError(err) + }) + } + }) +}) diff --git a/test/e2e/storage/csi_mock/csi_snapshot.go b/test/e2e/storage/csi_mock/csi_snapshot.go new file mode 100644 index 0000000000000..414c248b55fae --- /dev/null +++ b/test/e2e/storage/csi_mock/csi_snapshot.go @@ -0,0 +1,613 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi_mock + +import ( + "context" + "fmt" + "time" + + "github.com/onsi/ginkgo/v2" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + v1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + e2emetrics "k8s.io/kubernetes/test/e2e/framework/metrics" + e2epv "k8s.io/kubernetes/test/e2e/framework/pv" + e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" + "k8s.io/kubernetes/test/e2e/storage/drivers" + storageframework "k8s.io/kubernetes/test/e2e/storage/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" + admissionapi "k8s.io/pod-security-admission/api" +) + +var _ = utils.SIGDescribe("CSI Mock volume snapshot", func() { + f := framework.NewDefaultFramework("csi-mock-volumes-snapshot") + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged + m := newMockDriverSetup(f) + + ginkgo.Context("CSI Volume Snapshots [Feature:VolumeSnapshotDataSource]", func() { + tests := []struct { + name string + createSnapshotHook func(counter int64) error + }{ + { + name: "volumesnapshotcontent and pvc in Bound state with deletion timestamp set should not get deleted while snapshot finalizer exists", + createSnapshotHook: func(counter int64) error { + if counter < 8 { + return status.Error(codes.DeadlineExceeded, "fake error") + } + return nil + }, + }, + } + for _, test := range tests { + test := test + ginkgo.It(test.name, func(ctx context.Context) { + var hooks *drivers.Hooks + if test.createSnapshotHook != nil { + hooks = createPreHook("CreateSnapshot", test.createSnapshotHook) + } + m.init(testParameters{ + disableAttach: true, + registerDriver: true, + enableSnapshot: true, + hooks: hooks, + }) + sDriver, ok := m.driver.(storageframework.SnapshottableTestDriver) + if !ok { + e2eskipper.Skipf("mock driver %s does not support snapshots -- skipping", m.driver.GetDriverInfo().Name) + + } + ctx, cancel := context.WithTimeout(ctx, csiPodRunningTimeout) + defer cancel() + defer m.cleanup() + + sc := m.driver.GetDynamicProvisionStorageClass(m.config, "") + ginkgo.By("Creating storage class") + class, err := m.cs.StorageV1().StorageClasses().Create(context.TODO(), sc, metav1.CreateOptions{}) + framework.ExpectNoError(err, "Failed to create class: %v", err) + m.sc[class.Name] = class + claim := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{ + // Use static name so that the volumesnapshot can be created before the pvc. + Name: "snapshot-test-pvc", + StorageClassName: &(class.Name), + }, f.Namespace.Name) + + ginkgo.By("Creating snapshot") + // TODO: Test VolumeSnapshots with Retain policy + parameters := map[string]string{} + snapshotClass, snapshot := storageframework.CreateSnapshot(sDriver, m.config, storageframework.DynamicSnapshotDelete, claim.Name, claim.Namespace, f.Timeouts, parameters) + framework.ExpectNoError(err, "failed to create snapshot") + m.vsc[snapshotClass.GetName()] = snapshotClass + volumeSnapshotName := snapshot.GetName() + + ginkgo.By(fmt.Sprintf("Creating PVC %s/%s", claim.Namespace, claim.Name)) + claim, err = m.cs.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Create(context.TODO(), claim, metav1.CreateOptions{}) + framework.ExpectNoError(err, "Failed to create claim: %v", err) + + ginkgo.By(fmt.Sprintf("Wait for finalizer to be added to claim %s/%s", claim.Namespace, claim.Name)) + err = e2epv.WaitForPVCFinalizer(ctx, m.cs, claim.Name, claim.Namespace, pvcAsSourceProtectionFinalizer, 1*time.Millisecond, 1*time.Minute) + framework.ExpectNoError(err) + + ginkgo.By("Wait for PVC to be Bound") + _, err = e2epv.WaitForPVClaimBoundPhase(m.cs, []*v1.PersistentVolumeClaim{claim}, 1*time.Minute) + framework.ExpectNoError(err, "Failed to create claim: %v", err) + + ginkgo.By(fmt.Sprintf("Delete PVC %s", claim.Name)) + err = e2epv.DeletePersistentVolumeClaim(m.cs, claim.Name, claim.Namespace) + framework.ExpectNoError(err, "failed to delete pvc") + + ginkgo.By("Get PVC from API server and verify deletion timestamp is set") + claim, err = m.cs.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Get(context.TODO(), claim.Name, metav1.GetOptions{}) + if err != nil { + if !apierrors.IsNotFound(err) { + framework.ExpectNoError(err, "Failed to get claim: %v", err) + } + framework.Logf("PVC not found. Continuing to test VolumeSnapshotContent finalizer") + } else if claim.DeletionTimestamp == nil { + framework.Failf("Expected deletion timestamp to be set on PVC: %v", claim) + } + + ginkgo.By(fmt.Sprintf("Get VolumeSnapshotContent bound to VolumeSnapshot %s", snapshot.GetName())) + snapshotContent := utils.GetSnapshotContentFromSnapshot(m.config.Framework.DynamicClient, snapshot, f.Timeouts.SnapshotCreate) + volumeSnapshotContentName := snapshotContent.GetName() + + ginkgo.By(fmt.Sprintf("Verify VolumeSnapshotContent %s contains finalizer %s", snapshot.GetName(), volumeSnapshotContentFinalizer)) + err = utils.WaitForGVRFinalizer(ctx, m.config.Framework.DynamicClient, utils.SnapshotContentGVR, volumeSnapshotContentName, "", volumeSnapshotContentFinalizer, 1*time.Millisecond, 1*time.Minute) + framework.ExpectNoError(err) + + ginkgo.By(fmt.Sprintf("Delete VolumeSnapshotContent %s", snapshotContent.GetName())) + err = m.config.Framework.DynamicClient.Resource(utils.SnapshotContentGVR).Delete(ctx, snapshotContent.GetName(), metav1.DeleteOptions{}) + framework.ExpectNoError(err, "Failed to delete snapshotcontent: %v", err) + + ginkgo.By("Get VolumeSnapshotContent from API server and verify deletion timestamp is set") + snapshotContent, err = m.config.Framework.DynamicClient.Resource(utils.SnapshotContentGVR).Get(context.TODO(), snapshotContent.GetName(), metav1.GetOptions{}) + framework.ExpectNoError(err) + + if snapshotContent.GetDeletionTimestamp() == nil { + framework.Failf("Expected deletion timestamp to be set on snapshotcontent") + } + + // If the claim is non existent, the Get() call on the API server returns + // an non-nil claim object with all fields unset. + // Refer https://github.com/kubernetes/kubernetes/pull/99167#issuecomment-781670012 + if claim != nil && claim.Spec.VolumeName != "" { + ginkgo.By(fmt.Sprintf("Wait for PV %s to be deleted", claim.Spec.VolumeName)) + err = e2epv.WaitForPersistentVolumeDeleted(m.cs, claim.Spec.VolumeName, framework.Poll, 3*time.Minute) + framework.ExpectNoError(err, fmt.Sprintf("failed to delete PV %s", claim.Spec.VolumeName)) + } + + ginkgo.By(fmt.Sprintf("Verify VolumeSnapshot %s contains finalizer %s", snapshot.GetName(), volumeSnapshotBoundFinalizer)) + err = utils.WaitForGVRFinalizer(ctx, m.config.Framework.DynamicClient, utils.SnapshotGVR, volumeSnapshotName, f.Namespace.Name, volumeSnapshotBoundFinalizer, 1*time.Millisecond, 1*time.Minute) + framework.ExpectNoError(err) + + ginkgo.By("Delete VolumeSnapshot") + err = utils.DeleteAndWaitSnapshot(m.config.Framework.DynamicClient, f.Namespace.Name, volumeSnapshotName, framework.Poll, framework.SnapshotDeleteTimeout) + framework.ExpectNoError(err, fmt.Sprintf("failed to delete VolumeSnapshot %s", volumeSnapshotName)) + + ginkgo.By(fmt.Sprintf("Wait for VolumeSnapshotContent %s to be deleted", volumeSnapshotContentName)) + err = utils.WaitForGVRDeletion(m.config.Framework.DynamicClient, utils.SnapshotContentGVR, volumeSnapshotContentName, framework.Poll, framework.SnapshotDeleteTimeout) + framework.ExpectNoError(err, fmt.Sprintf("failed to delete VolumeSnapshotContent %s", volumeSnapshotContentName)) + }) + } + }) + + ginkgo.Context("CSI Volume Snapshots secrets [Feature:VolumeSnapshotDataSource]", func() { + + var ( + // CSISnapshotterSecretName is the name of the secret to be created + CSISnapshotterSecretName string = "snapshot-secret" + + // CSISnapshotterSecretNameAnnotation is the annotation key for the CSI snapshotter secret name in VolumeSnapshotClass.parameters + CSISnapshotterSecretNameAnnotation string = "csi.storage.k8s.io/snapshotter-secret-name" + + // CSISnapshotterSecretNamespaceAnnotation is the annotation key for the CSI snapshotter secret namespace in VolumeSnapshotClass.parameters + CSISnapshotterSecretNamespaceAnnotation string = "csi.storage.k8s.io/snapshotter-secret-namespace" + + // anotations holds the annotations object + annotations interface{} + ) + + tests := []struct { + name string + createSnapshotHook func(counter int64) error + }{ + { + // volume snapshot should be created using secrets successfully even if there is a failure in the first few attempts, + name: "volume snapshot create/delete with secrets", + // Fail the first 8 calls to create snapshot and succeed the 9th call. + createSnapshotHook: func(counter int64) error { + if counter < 8 { + return status.Error(codes.DeadlineExceeded, "fake error") + } + return nil + }, + }, + } + for _, test := range tests { + test := test + ginkgo.It(test.name, func(ctx context.Context) { + hooks := createPreHook("CreateSnapshot", test.createSnapshotHook) + m.init(testParameters{ + disableAttach: true, + registerDriver: true, + enableSnapshot: true, + hooks: hooks, + }) + + sDriver, ok := m.driver.(storageframework.SnapshottableTestDriver) + if !ok { + e2eskipper.Skipf("mock driver does not support snapshots -- skipping") + } + defer m.cleanup() + + var sc *storagev1.StorageClass + if dDriver, ok := m.driver.(storageframework.DynamicPVTestDriver); ok { + sc = dDriver.GetDynamicProvisionStorageClass(m.config, "") + } + ginkgo.By("Creating storage class") + class, err := m.cs.StorageV1().StorageClasses().Create(context.TODO(), sc, metav1.CreateOptions{}) + framework.ExpectNoError(err, "Failed to create storage class: %v", err) + m.sc[class.Name] = class + pvc := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{ + Name: "snapshot-test-pvc", + StorageClassName: &(class.Name), + }, f.Namespace.Name) + + ginkgo.By(fmt.Sprintf("Creating PVC %s/%s", pvc.Namespace, pvc.Name)) + pvc, err = m.cs.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Create(context.TODO(), pvc, metav1.CreateOptions{}) + framework.ExpectNoError(err, "Failed to create claim: %v", err) + + ginkgo.By("Wait for PVC to be Bound") + _, err = e2epv.WaitForPVClaimBoundPhase(m.cs, []*v1.PersistentVolumeClaim{pvc}, 1*time.Minute) + framework.ExpectNoError(err, "Failed to create claim: %v", err) + + m.pvcs = append(m.pvcs, pvc) + + ginkgo.By("Creating Secret") + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: f.Namespace.Name, + Name: CSISnapshotterSecretName, + }, + Data: map[string][]byte{ + "secret-data": []byte("secret-value-1"), + }, + } + + if secret, err := m.cs.CoreV1().Secrets(f.Namespace.Name).Create(context.TODO(), secret, metav1.CreateOptions{}); err != nil { + framework.Failf("unable to create test secret %s: %v", secret.Name, err) + } + + ginkgo.By("Creating snapshot with secrets") + parameters := map[string]string{ + CSISnapshotterSecretNameAnnotation: CSISnapshotterSecretName, + CSISnapshotterSecretNamespaceAnnotation: f.Namespace.Name, + } + + _, snapshot := storageframework.CreateSnapshot(sDriver, m.config, storageframework.DynamicSnapshotDelete, pvc.Name, pvc.Namespace, f.Timeouts, parameters) + framework.ExpectNoError(err, "failed to create snapshot") + snapshotcontent := utils.GetSnapshotContentFromSnapshot(m.config.Framework.DynamicClient, snapshot, f.Timeouts.SnapshotCreate) + if annotations, ok = snapshotcontent.Object["metadata"].(map[string]interface{})["annotations"]; !ok { + framework.Failf("Unable to get volume snapshot content annotations") + } + + // checks if delete snapshot secrets annotation is applied to the VolumeSnapshotContent. + checkDeleteSnapshotSecrets(m.cs, annotations) + + // delete the snapshot and check if the snapshot is deleted. + deleteSnapshot(m.cs, m.config, snapshot) + }) + } + }) + + ginkgo.Context("CSI Snapshot Controller metrics [Feature:VolumeSnapshotDataSource]", func() { + tests := []struct { + name string + pattern storageframework.TestPattern + }{ + { + name: "snapshot controller should emit dynamic CreateSnapshot, CreateSnapshotAndReady, and DeleteSnapshot metrics", + pattern: storageframework.DynamicSnapshotDelete, + }, + { + name: "snapshot controller should emit pre-provisioned CreateSnapshot, CreateSnapshotAndReady, and DeleteSnapshot metrics", + pattern: storageframework.PreprovisionedSnapshotDelete, + }, + } + for _, test := range tests { + test := test + ginkgo.It(test.name, func(ctx context.Context) { + m.init(testParameters{ + disableAttach: true, + registerDriver: true, + enableSnapshot: true, + }) + + sDriver, ok := m.driver.(storageframework.SnapshottableTestDriver) + if !ok { + e2eskipper.Skipf("mock driver does not support snapshots -- skipping") + } + defer m.cleanup() + + metricsGrabber, err := e2emetrics.NewMetricsGrabber(m.config.Framework.ClientSet, nil, f.ClientConfig(), false, false, false, false, false, true) + if err != nil { + framework.Failf("Error creating metrics grabber : %v", err) + } + + // Grab initial metrics - if this fails, snapshot controller metrics are not setup. Skip in this case. + _, err = metricsGrabber.GrabFromSnapshotController(framework.TestContext.SnapshotControllerPodName, framework.TestContext.SnapshotControllerHTTPPort) + if err != nil { + e2eskipper.Skipf("Snapshot controller metrics not found -- skipping") + } + + ginkgo.By("getting all initial metric values") + metricsTestConfig := newSnapshotMetricsTestConfig("snapshot_controller_operation_total_seconds_count", + "count", + m.config.GetUniqueDriverName(), + "CreateSnapshot", + "success", + "", + test.pattern) + createSnapshotMetrics := newSnapshotControllerMetrics(metricsTestConfig, metricsGrabber) + originalCreateSnapshotCount, _ := createSnapshotMetrics.getSnapshotControllerMetricValue() + metricsTestConfig.operationName = "CreateSnapshotAndReady" + createSnapshotAndReadyMetrics := newSnapshotControllerMetrics(metricsTestConfig, metricsGrabber) + originalCreateSnapshotAndReadyCount, _ := createSnapshotAndReadyMetrics.getSnapshotControllerMetricValue() + + metricsTestConfig.operationName = "DeleteSnapshot" + deleteSnapshotMetrics := newSnapshotControllerMetrics(metricsTestConfig, metricsGrabber) + originalDeleteSnapshotCount, _ := deleteSnapshotMetrics.getSnapshotControllerMetricValue() + + ginkgo.By("Creating storage class") + var sc *storagev1.StorageClass + if dDriver, ok := m.driver.(storageframework.DynamicPVTestDriver); ok { + sc = dDriver.GetDynamicProvisionStorageClass(m.config, "") + } + class, err := m.cs.StorageV1().StorageClasses().Create(context.TODO(), sc, metav1.CreateOptions{}) + framework.ExpectNoError(err, "Failed to create storage class: %v", err) + m.sc[class.Name] = class + pvc := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{ + Name: "snapshot-test-pvc", + StorageClassName: &(class.Name), + }, f.Namespace.Name) + + ginkgo.By(fmt.Sprintf("Creating PVC %s/%s", pvc.Namespace, pvc.Name)) + pvc, err = m.cs.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Create(context.TODO(), pvc, metav1.CreateOptions{}) + framework.ExpectNoError(err, "Failed to create claim: %v", err) + + ginkgo.By("Wait for PVC to be Bound") + _, err = e2epv.WaitForPVClaimBoundPhase(m.cs, []*v1.PersistentVolumeClaim{pvc}, 1*time.Minute) + framework.ExpectNoError(err, "Failed to create claim: %v", err) + + ginkgo.By("Creating snapshot") + parameters := map[string]string{} + sr := storageframework.CreateSnapshotResource(sDriver, m.config, test.pattern, pvc.Name, pvc.Namespace, f.Timeouts, parameters) + framework.ExpectNoError(err, "failed to create snapshot") + + ginkgo.By("Checking for CreateSnapshot metrics") + createSnapshotMetrics.waitForSnapshotControllerMetric(originalCreateSnapshotCount+1.0, f.Timeouts.SnapshotControllerMetrics) + + ginkgo.By("Checking for CreateSnapshotAndReady metrics") + err = utils.WaitForSnapshotReady(m.config.Framework.DynamicClient, pvc.Namespace, sr.Vs.GetName(), framework.Poll, f.Timeouts.SnapshotCreate) + framework.ExpectNoError(err, "failed to wait for snapshot ready") + createSnapshotAndReadyMetrics.waitForSnapshotControllerMetric(originalCreateSnapshotAndReadyCount+1.0, f.Timeouts.SnapshotControllerMetrics) + + // delete the snapshot and check if the snapshot is deleted + deleteSnapshot(m.cs, m.config, sr.Vs) + + ginkgo.By("check for delete metrics") + metricsTestConfig.operationName = "DeleteSnapshot" + deleteSnapshotMetrics.waitForSnapshotControllerMetric(originalDeleteSnapshotCount+1.0, f.Timeouts.SnapshotControllerMetrics) + }) + } + }) +}) + +// checkDeleteSnapshotSecrets checks if delete snapshot secrets annotation is applied to the VolumeSnapshotContent. +func checkDeleteSnapshotSecrets(cs clientset.Interface, annotations interface{}) error { + ginkgo.By("checking if delete snapshot secrets annotation is applied to the VolumeSnapshotContent") + + var ( + annDeletionSecretName string + annDeletionSecretNamespace string + ok bool + err error + + // CSISnapshotterDeleteSecretNameAnnotation is the annotation key for the CSI snapshotter delete secret name in VolumeSnapshotClass.parameters + CSISnapshotterDeleteSecretNameAnnotation string = "snapshot.storage.kubernetes.io/deletion-secret-name" + + // CSISnapshotterDeleteSecretNamespaceAnnotation is the annotation key for the CSI snapshotter delete secret namespace in VolumeSnapshotClass.parameters + CSISnapshotterDeleteSecretNamespaceAnnotation string = "snapshot.storage.kubernetes.io/deletion-secret-namespace" + ) + + annotationsObj, ok := annotations.(map[string]interface{}) + if !ok { + framework.Failf("failed to get annotations from annotations object") + } + + if annDeletionSecretName, ok = annotationsObj[CSISnapshotterDeleteSecretNameAnnotation].(string); !ok { + framework.Failf("unable to get secret annotation name") + } + if annDeletionSecretNamespace, ok = annotationsObj[CSISnapshotterDeleteSecretNamespaceAnnotation].(string); !ok { + framework.Failf("unable to get secret annotation namespace") + } + + // verify if secrets exists + if _, err = cs.CoreV1().Secrets(annDeletionSecretNamespace).Get(context.TODO(), annDeletionSecretName, metav1.GetOptions{}); err != nil { + framework.Failf("unable to get test secret %s: %v", annDeletionSecretName, err) + } + + return err +} + +func deleteSnapshot(cs clientset.Interface, config *storageframework.PerTestConfig, snapshot *unstructured.Unstructured) { + // delete the given snapshot + dc := config.Framework.DynamicClient + err := dc.Resource(utils.SnapshotGVR).Namespace(snapshot.GetNamespace()).Delete(context.TODO(), snapshot.GetName(), metav1.DeleteOptions{}) + framework.ExpectNoError(err) + + // check if the snapshot is deleted + _, err = dc.Resource(utils.SnapshotGVR).Get(context.TODO(), snapshot.GetName(), metav1.GetOptions{}) + framework.ExpectError(err) +} + +type snapshotMetricsTestConfig struct { + // expected values + metricName string + metricType string + driverName string + operationName string + operationStatus string + snapshotType string + le string +} + +type snapshotControllerMetrics struct { + // configuration for metric + cfg snapshotMetricsTestConfig + metricsGrabber *e2emetrics.Grabber + + // results + countMetrics map[string]float64 + sumMetrics map[string]float64 + bucketMetrics map[string]float64 +} + +func newSnapshotMetricsTestConfig(metricName, metricType, driverName, operationName, operationStatus, le string, pattern storageframework.TestPattern) snapshotMetricsTestConfig { + var snapshotType string + switch pattern.SnapshotType { + case storageframework.DynamicCreatedSnapshot: + snapshotType = "dynamic" + + case storageframework.PreprovisionedCreatedSnapshot: + snapshotType = "pre-provisioned" + + default: + framework.Failf("invalid snapshotType: %v", pattern.SnapshotType) + } + + return snapshotMetricsTestConfig{ + metricName: metricName, + metricType: metricType, + driverName: driverName, + operationName: operationName, + operationStatus: operationStatus, + snapshotType: snapshotType, + le: le, + } +} + +func newSnapshotControllerMetrics(cfg snapshotMetricsTestConfig, metricsGrabber *e2emetrics.Grabber) *snapshotControllerMetrics { + return &snapshotControllerMetrics{ + cfg: cfg, + metricsGrabber: metricsGrabber, + + countMetrics: make(map[string]float64), + sumMetrics: make(map[string]float64), + bucketMetrics: make(map[string]float64), + } +} + +func (scm *snapshotControllerMetrics) waitForSnapshotControllerMetric(expectedValue float64, timeout time.Duration) { + metricKey := scm.getMetricKey() + if successful := utils.WaitUntil(10*time.Second, timeout, func() bool { + // get metric value + actualValue, err := scm.getSnapshotControllerMetricValue() + if err != nil { + return false + } + + // Another operation could have finished from a previous test, + // so we check if we have at least the expected value. + if actualValue < expectedValue { + return false + } + + return true + }); successful { + return + } + + scm.showMetricsFailure(metricKey) + framework.Failf("Unable to get valid snapshot controller metrics after %v", timeout) +} + +func (scm *snapshotControllerMetrics) getSnapshotControllerMetricValue() (float64, error) { + metricKey := scm.getMetricKey() + + // grab and parse into readable format + err := scm.grabSnapshotControllerMetrics() + if err != nil { + return 0, err + } + + metrics := scm.getMetricsTable() + actual, ok := metrics[metricKey] + if !ok { + return 0, fmt.Errorf("did not find metric for key %s", metricKey) + } + + return actual, nil +} + +func (scm *snapshotControllerMetrics) getMetricsTable() map[string]float64 { + var metrics map[string]float64 + switch scm.cfg.metricType { + case "count": + metrics = scm.countMetrics + + case "sum": + metrics = scm.sumMetrics + + case "bucket": + metrics = scm.bucketMetrics + } + + return metrics +} + +func (scm *snapshotControllerMetrics) showMetricsFailure(metricKey string) { + framework.Logf("failed to find metric key %s inside of the following metrics:", metricKey) + + metrics := scm.getMetricsTable() + for k, v := range metrics { + framework.Logf("%s: %v", k, v) + } +} + +func (scm *snapshotControllerMetrics) grabSnapshotControllerMetrics() error { + // pull all metrics + metrics, err := scm.metricsGrabber.GrabFromSnapshotController(framework.TestContext.SnapshotControllerPodName, framework.TestContext.SnapshotControllerHTTPPort) + if err != nil { + return err + } + + for method, samples := range metrics { + + for _, sample := range samples { + operationName := string(sample.Metric["operation_name"]) + driverName := string(sample.Metric["driver_name"]) + operationStatus := string(sample.Metric["operation_status"]) + snapshotType := string(sample.Metric["snapshot_type"]) + le := string(sample.Metric["le"]) + key := snapshotMetricKey(scm.cfg.metricName, driverName, operationName, operationStatus, snapshotType, le) + + switch method { + case "snapshot_controller_operation_total_seconds_count": + for _, sample := range samples { + scm.countMetrics[key] = float64(sample.Value) + } + + case "snapshot_controller_operation_total_seconds_sum": + for _, sample := range samples { + scm.sumMetrics[key] = float64(sample.Value) + } + + case "snapshot_controller_operation_total_seconds_bucket": + for _, sample := range samples { + scm.bucketMetrics[key] = float64(sample.Value) + } + } + } + } + + return nil +} + +func (scm *snapshotControllerMetrics) getMetricKey() string { + return snapshotMetricKey(scm.cfg.metricName, scm.cfg.driverName, scm.cfg.operationName, scm.cfg.operationStatus, scm.cfg.snapshotType, scm.cfg.le) +} + +func snapshotMetricKey(metricName, driverName, operationName, operationStatus, snapshotType, le string) string { + key := driverName + + // build key for shorthand metrics storage + for _, s := range []string{metricName, operationName, operationStatus, snapshotType, le} { + if s != "" { + key = fmt.Sprintf("%s_%s", key, s) + } + } + + return key +} diff --git a/test/e2e/storage/csi_mock/csi_storage_capacity.go b/test/e2e/storage/csi_mock/csi_storage_capacity.go new file mode 100644 index 0000000000000..c9265ae9748bb --- /dev/null +++ b/test/e2e/storage/csi_mock/csi_storage_capacity.go @@ -0,0 +1,395 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi_mock + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + v1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apimachinery/pkg/watch" + cachetools "k8s.io/client-go/tools/cache" + watchtools "k8s.io/client-go/tools/watch" + "k8s.io/kubernetes/test/e2e/framework" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + "k8s.io/kubernetes/test/e2e/storage/drivers" + "k8s.io/kubernetes/test/e2e/storage/utils" + admissionapi "k8s.io/pod-security-admission/api" +) + +var _ = utils.SIGDescribe("CSI Mock volume storage capacity", func() { + f := framework.NewDefaultFramework("csi-mock-volumes-capacity") + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged + m := newMockDriverSetup(f) + + ginkgo.Context("storage capacity", func() { + tests := []struct { + name string + resourceExhausted bool + lateBinding bool + topology bool + }{ + { + name: "unlimited", + }, + { + name: "exhausted, immediate binding", + resourceExhausted: true, + }, + { + name: "exhausted, late binding, no topology", + resourceExhausted: true, + lateBinding: true, + }, + { + name: "exhausted, late binding, with topology", + resourceExhausted: true, + lateBinding: true, + topology: true, + }, + } + + createVolume := "CreateVolume" + deleteVolume := "DeleteVolume" + // publishVolume := "NodePublishVolume" + // unpublishVolume := "NodeUnpublishVolume" + // stageVolume := "NodeStageVolume" + // unstageVolume := "NodeUnstageVolume" + + // These calls are assumed to occur in this order for + // each test run. NodeStageVolume and + // NodePublishVolume should also be deterministic and + // only get called once, but sometimes kubelet calls + // both multiple times, which breaks this test + // (https://github.com/kubernetes/kubernetes/issues/90250). + // Therefore they are temporarily commented out until + // that issue is resolved. + // + // NodeUnpublishVolume and NodeUnstageVolume are racing + // with DeleteVolume, so we cannot assume a deterministic + // order and have to ignore them + // (https://github.com/kubernetes/kubernetes/issues/94108). + deterministicCalls := []string{ + createVolume, + // stageVolume, + // publishVolume, + // unpublishVolume, + // unstageVolume, + deleteVolume, + } + + for _, t := range tests { + test := t + ginkgo.It(test.name, func(ctx context.Context) { + var err error + params := testParameters{ + lateBinding: test.lateBinding, + enableTopology: test.topology, + + // Not strictly necessary, but runs a bit faster this way + // and for a while there also was a problem with a two minuted delay + // due to a bug (https://github.com/kubernetes-csi/csi-test/pull/250). + disableAttach: true, + registerDriver: true, + } + + if test.resourceExhausted { + params.hooks = createPreHook("CreateVolume", func(counter int64) error { + if counter%2 != 0 { + return status.Error(codes.ResourceExhausted, "fake error") + } + return nil + }) + } + + m.init(params) + defer m.cleanup() + + ctx, cancel := context.WithTimeout(ctx, csiPodRunningTimeout) + defer cancel() + + // In contrast to the raw watch, RetryWatcher is expected to deliver all events even + // when the underlying raw watch gets closed prematurely + // (https://github.com/kubernetes/kubernetes/pull/93777#discussion_r467932080). + // This is important because below the test is going to make assertions about the + // PVC state changes. + initResource, err := f.ClientSet.CoreV1().PersistentVolumeClaims(f.Namespace.Name).List(ctx, metav1.ListOptions{}) + framework.ExpectNoError(err, "Failed to fetch initial PVC resource") + listWatcher := &cachetools.ListWatch{ + WatchFunc: func(listOptions metav1.ListOptions) (watch.Interface, error) { + return f.ClientSet.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Watch(ctx, listOptions) + }, + } + pvcWatch, err := watchtools.NewRetryWatcher(initResource.GetResourceVersion(), listWatcher) + framework.ExpectNoError(err, "create PVC watch") + defer pvcWatch.Stop() + + sc, claim, pod := m.createPod(pvcReference) + gomega.Expect(pod).NotTo(gomega.BeNil(), "while creating pod") + bindingMode := storagev1.VolumeBindingImmediate + if test.lateBinding { + bindingMode = storagev1.VolumeBindingWaitForFirstConsumer + } + framework.ExpectEqual(*sc.VolumeBindingMode, bindingMode, "volume binding mode") + + err = e2epod.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) + framework.ExpectNoError(err, "failed to start pod") + err = e2epod.DeletePodWithWait(m.cs, pod) + framework.ExpectNoError(err, "failed to delete pod") + err = m.cs.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(ctx, claim.Name, metav1.DeleteOptions{}) + framework.ExpectNoError(err, "failed to delete claim") + + normal := []csiCall{} + for _, method := range deterministicCalls { + normal = append(normal, csiCall{expectedMethod: method}) + } + expected := normal + // When simulating limited capacity, + // we expect exactly two CreateVolume + // calls because the first one should + // have failed. + if test.resourceExhausted { + expected = []csiCall{ + {expectedMethod: createVolume, expectedError: codes.ResourceExhausted}, + } + expected = append(expected, normal...) + } + + var calls []drivers.MockCSICall + err = wait.PollImmediateUntil(time.Second, func() (done bool, err error) { + c, index, err := compareCSICalls(deterministicCalls, expected, m.driver.GetCalls) + if err != nil { + return true, fmt.Errorf("error waiting for expected CSI calls: %s", err) + } + calls = c + if index == 0 { + // No CSI call received yet + return false, nil + } + if len(expected) == index { + // all calls received + return true, nil + } + return false, nil + }, ctx.Done()) + framework.ExpectNoError(err, "while waiting for all CSI calls") + + // The capacity error is dealt with in two different ways. + // + // For delayed binding, the external-provisioner should unset the node annotation + // to give the scheduler the opportunity to reschedule the pod onto a different + // node. + // + // For immediate binding, the external-scheduler must keep retrying. + // + // Unfortunately, the call log is the same in both cases. We have to collect + // additional evidence that rescheduling really happened. What we have observed + // above is how the PVC changed over time. Now we can analyze that. + ginkgo.By("Checking PVC events") + nodeAnnotationSet := false + nodeAnnotationReset := false + watchFailed := false + loop: + for { + select { + case event, ok := <-pvcWatch.ResultChan(): + if !ok { + watchFailed = true + break loop + } + + framework.Logf("PVC event %s: %#v", event.Type, event.Object) + switch event.Type { + case watch.Modified: + pvc, ok := event.Object.(*v1.PersistentVolumeClaim) + if !ok { + framework.Failf("PVC watch sent %#v instead of a PVC", event.Object) + } + _, set := pvc.Annotations["volume.kubernetes.io/selected-node"] + if set { + nodeAnnotationSet = true + } else if nodeAnnotationSet { + nodeAnnotationReset = true + } + case watch.Deleted: + break loop + case watch.Error: + watchFailed = true + break + } + case <-ctx.Done(): + framework.Failf("Timeout while waiting to observe PVC list") + } + } + + // More tests when capacity is limited. + if test.resourceExhausted { + for _, call := range calls { + if call.Method == createVolume { + gomega.Expect(call.Error).To(gomega.ContainSubstring("code = ResourceExhausted"), "first CreateVolume error in\n%s", calls) + break + } + } + + switch { + case watchFailed: + // If the watch failed or stopped prematurely (which can happen at any time), then we cannot + // verify whether the annotation was set as expected. This is still considered a successful + // test. + framework.Logf("PVC watch delivered incomplete data, cannot check annotation") + case test.lateBinding: + gomega.Expect(nodeAnnotationSet).To(gomega.BeTrue(), "selected-node should have been set") + // Whether it gets reset depends on whether we have topology enabled. Without + // it, rescheduling is unnecessary. + if test.topology { + gomega.Expect(nodeAnnotationReset).To(gomega.BeTrue(), "selected-node should have been set") + } else { + gomega.Expect(nodeAnnotationReset).To(gomega.BeFalse(), "selected-node should not have been reset") + } + default: + gomega.Expect(nodeAnnotationSet).To(gomega.BeFalse(), "selected-node should not have been set") + gomega.Expect(nodeAnnotationReset).To(gomega.BeFalse(), "selected-node should not have been reset") + } + } + }) + } + }) + + // These tests *only* work on a cluster which has the CSIStorageCapacity feature enabled. + ginkgo.Context("CSIStorageCapacity", func() { + var ( + err error + yes = true + no = false + ) + // Tests that expect a failure are slow because we have to wait for a while + // to be sure that the volume isn't getting created. + tests := []struct { + name string + storageCapacity *bool + capacities []string + expectFailure bool + }{ + { + name: "CSIStorageCapacity unused", + }, + { + name: "CSIStorageCapacity disabled", + storageCapacity: &no, + }, + { + name: "CSIStorageCapacity used, no capacity", + storageCapacity: &yes, + expectFailure: true, + }, + { + name: "CSIStorageCapacity used, insufficient capacity", + storageCapacity: &yes, + expectFailure: true, + capacities: []string{"1Mi"}, + }, + { + name: "CSIStorageCapacity used, have capacity", + storageCapacity: &yes, + capacities: []string{"100Gi"}, + }, + // We could add more test cases here for + // various situations, but covering those via + // the scheduler binder unit tests is faster. + } + for _, t := range tests { + test := t + ginkgo.It(t.name, func(ctx context.Context) { + scName := "mock-csi-storage-capacity-" + f.UniqueName + m.init(testParameters{ + registerDriver: true, + scName: scName, + storageCapacity: test.storageCapacity, + lateBinding: true, + }) + defer m.cleanup() + + // The storage class uses a random name, therefore we have to create it first + // before adding CSIStorageCapacity objects for it. + for _, capacityStr := range test.capacities { + capacityQuantity := resource.MustParse(capacityStr) + capacity := &storagev1.CSIStorageCapacity{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "fake-capacity-", + }, + // Empty topology, usable by any node. + StorageClassName: scName, + NodeTopology: &metav1.LabelSelector{}, + Capacity: &capacityQuantity, + } + createdCapacity, err := f.ClientSet.StorageV1().CSIStorageCapacities(f.Namespace.Name).Create(context.Background(), capacity, metav1.CreateOptions{}) + framework.ExpectNoError(err, "create CSIStorageCapacity %+v", *capacity) + m.testCleanups = append(m.testCleanups, func() { + f.ClientSet.StorageV1().CSIStorageCapacities(f.Namespace.Name).Delete(context.Background(), createdCapacity.Name, metav1.DeleteOptions{}) + }) + } + + // kube-scheduler may need some time before it gets the CSIDriver and CSIStorageCapacity objects. + // Without them, scheduling doesn't run as expected by the test. + syncDelay := 5 * time.Second + time.Sleep(syncDelay) + + sc, _, pod := m.createPod(pvcReference) // late binding as specified above + framework.ExpectEqual(sc.Name, scName, "pre-selected storage class name not used") + + waitCtx, cancel := context.WithTimeout(ctx, f.Timeouts.PodStart) + defer cancel() + condition := anyOf( + podRunning(waitCtx, f.ClientSet, pod.Name, pod.Namespace), + // We only just created the CSIStorageCapacity objects, therefore + // we have to ignore all older events, plus the syncDelay as our + // safety margin. + podHasStorage(waitCtx, f.ClientSet, pod.Name, pod.Namespace, time.Now().Add(syncDelay)), + ) + err = wait.PollImmediateUntil(poll, condition, waitCtx.Done()) + if test.expectFailure { + switch { + case errors.Is(err, context.DeadlineExceeded), + errors.Is(err, wait.ErrWaitTimeout), + errors.Is(err, errNotEnoughSpace): + // Okay, we expected that. + case err == nil: + framework.Fail("pod unexpectedly started to run") + default: + framework.Failf("unexpected error while waiting for pod: %v", err) + } + } else { + framework.ExpectNoError(err, "failed to start pod") + } + + ginkgo.By("Deleting the previously created pod") + err = e2epod.DeletePodWithWait(m.cs, pod) + framework.ExpectNoError(err, "while deleting") + }) + } + }) +}) diff --git a/test/e2e/storage/csi_mock/csi_volume_expansion.go b/test/e2e/storage/csi_mock/csi_volume_expansion.go new file mode 100644 index 0000000000000..df989c6f7a338 --- /dev/null +++ b/test/e2e/storage/csi_mock/csi_volume_expansion.go @@ -0,0 +1,214 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi_mock + +import ( + "context" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/kubernetes/test/e2e/framework" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + "k8s.io/kubernetes/test/e2e/storage/testsuites" + "k8s.io/kubernetes/test/e2e/storage/utils" + admissionapi "k8s.io/pod-security-admission/api" +) + +var _ = utils.SIGDescribe("CSI Mock volume expansion", func() { + f := framework.NewDefaultFramework("csi-mock-volumes-expansion") + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged + m := newMockDriverSetup(f) + + ginkgo.Context("CSI Volume expansion", func() { + tests := []struct { + name string + nodeExpansionRequired bool + disableAttach bool + disableResizingOnDriver bool + expectFailure bool + }{ + { + name: "should expand volume without restarting pod if nodeExpansion=off", + nodeExpansionRequired: false, + }, + { + name: "should expand volume by restarting pod if attach=on, nodeExpansion=on", + nodeExpansionRequired: true, + }, + { + name: "should expand volume by restarting pod if attach=off, nodeExpansion=on", + disableAttach: true, + nodeExpansionRequired: true, + }, + { + name: "should not expand volume if resizingOnDriver=off, resizingOnSC=on", + disableResizingOnDriver: true, + expectFailure: true, + }, + } + for _, t := range tests { + test := t + ginkgo.It(t.name, func(ctx context.Context) { + var err error + tp := testParameters{ + enableResizing: true, + enableNodeExpansion: test.nodeExpansionRequired, + disableResizingOnDriver: test.disableResizingOnDriver, + } + // disabling attach requires drive registration feature + if test.disableAttach { + tp.disableAttach = true + tp.registerDriver = true + } + + m.init(tp) + defer m.cleanup() + + sc, pvc, pod := m.createPod(pvcReference) + gomega.Expect(pod).NotTo(gomega.BeNil(), "while creating pod for resizing") + + if !*sc.AllowVolumeExpansion { + framework.Fail("failed creating sc with allowed expansion") + } + + err = e2epod.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) + framework.ExpectNoError(err, "Failed to start pod1: %v", err) + + ginkgo.By("Expanding current pvc") + newSize := resource.MustParse("6Gi") + newPVC, err := testsuites.ExpandPVCSize(pvc, newSize, m.cs) + framework.ExpectNoError(err, "While updating pvc for more size") + pvc = newPVC + gomega.Expect(pvc).NotTo(gomega.BeNil()) + + pvcSize := pvc.Spec.Resources.Requests[v1.ResourceStorage] + if pvcSize.Cmp(newSize) != 0 { + framework.Failf("error updating pvc size %q", pvc.Name) + } + if test.expectFailure { + err = testsuites.WaitForResizingCondition(pvc, m.cs, csiResizingConditionWait) + framework.ExpectError(err, "unexpected resizing condition on PVC") + return + } + + ginkgo.By("Waiting for persistent volume resize to finish") + err = testsuites.WaitForControllerVolumeResize(pvc, m.cs, csiResizeWaitPeriod) + framework.ExpectNoError(err, "While waiting for CSI PV resize to finish") + + checkPVCSize := func() { + ginkgo.By("Waiting for PVC resize to finish") + pvc, err = testsuites.WaitForFSResize(pvc, m.cs) + framework.ExpectNoError(err, "while waiting for PVC resize to finish") + + pvcConditions := pvc.Status.Conditions + framework.ExpectEqual(len(pvcConditions), 0, "pvc should not have conditions") + } + + // if node expansion is not required PVC should be resized as well + if !test.nodeExpansionRequired { + checkPVCSize() + } else { + ginkgo.By("Checking for conditions on pvc") + npvc, err := testsuites.WaitForPendingFSResizeCondition(pvc, m.cs) + framework.ExpectNoError(err, "While waiting for pvc to have fs resizing condition") + pvc = npvc + + inProgressConditions := pvc.Status.Conditions + if len(inProgressConditions) > 0 { + framework.ExpectEqual(inProgressConditions[0].Type, v1.PersistentVolumeClaimFileSystemResizePending, "pvc must have fs resizing condition") + } + + ginkgo.By("Deleting the previously created pod") + err = e2epod.DeletePodWithWait(m.cs, pod) + framework.ExpectNoError(err, "while deleting pod for resizing") + + ginkgo.By("Creating a new pod with same volume") + pod2, err := m.createPodWithPVC(pvc) + gomega.Expect(pod2).NotTo(gomega.BeNil(), "while creating pod for csi resizing") + framework.ExpectNoError(err, "while recreating pod for resizing") + + checkPVCSize() + } + }) + } + }) + ginkgo.Context("CSI online volume expansion", func() { + tests := []struct { + name string + disableAttach bool + }{ + { + name: "should expand volume without restarting pod if attach=on, nodeExpansion=on", + }, + { + name: "should expand volume without restarting pod if attach=off, nodeExpansion=on", + disableAttach: true, + }, + } + for _, t := range tests { + test := t + ginkgo.It(test.name, func(ctx context.Context) { + var err error + params := testParameters{enableResizing: true, enableNodeExpansion: true} + if test.disableAttach { + params.disableAttach = true + params.registerDriver = true + } + + m.init(params) + + defer m.cleanup() + + sc, pvc, pod := m.createPod(pvcReference) + gomega.Expect(pod).NotTo(gomega.BeNil(), "while creating pod for resizing") + + if !*sc.AllowVolumeExpansion { + framework.Fail("failed creating sc with allowed expansion") + } + + err = e2epod.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) + framework.ExpectNoError(err, "Failed to start pod1: %v", err) + + ginkgo.By("Expanding current pvc") + newSize := resource.MustParse("6Gi") + newPVC, err := testsuites.ExpandPVCSize(pvc, newSize, m.cs) + framework.ExpectNoError(err, "While updating pvc for more size") + pvc = newPVC + gomega.Expect(pvc).NotTo(gomega.BeNil()) + + pvcSize := pvc.Spec.Resources.Requests[v1.ResourceStorage] + if pvcSize.Cmp(newSize) != 0 { + framework.Failf("error updating pvc size %q", pvc.Name) + } + + ginkgo.By("Waiting for persistent volume resize to finish") + err = testsuites.WaitForControllerVolumeResize(pvc, m.cs, csiResizeWaitPeriod) + framework.ExpectNoError(err, "While waiting for PV resize to finish") + + ginkgo.By("Waiting for PVC resize to finish") + pvc, err = testsuites.WaitForFSResize(pvc, m.cs) + framework.ExpectNoError(err, "while waiting for PVC to finish") + + pvcConditions := pvc.Status.Conditions + framework.ExpectEqual(len(pvcConditions), 0, "pvc should not have conditions") + + }) + } + }) +}) diff --git a/test/e2e/storage/csi_mock/csi_volume_limit.go b/test/e2e/storage/csi_mock/csi_volume_limit.go new file mode 100644 index 0000000000000..3272df669dc9b --- /dev/null +++ b/test/e2e/storage/csi_mock/csi_volume_limit.go @@ -0,0 +1,156 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi_mock + +import ( + "context" + "fmt" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + storagev1 "k8s.io/api/storage/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + "k8s.io/kubernetes/test/e2e/storage/utils" + admissionapi "k8s.io/pod-security-admission/api" +) + +var _ = utils.SIGDescribe("CSI Mock volume limit", func() { + f := framework.NewDefaultFramework("csi-mock-volumes-limit") + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged + m := newMockDriverSetup(f) + + ginkgo.Context("CSI volume limit information using mock driver", func() { + ginkgo.It("should report attach limit when limit is bigger than 0 [Slow]", func(ctx context.Context) { + // define volume limit to be 2 for this test + var err error + m.init(testParameters{attachLimit: 2}) + defer m.cleanup() + nodeName := m.config.ClientNodeSelection.Name + driverName := m.config.GetUniqueDriverName() + + csiNodeAttachLimit, err := checkCSINodeForLimits(nodeName, driverName, m.cs) + framework.ExpectNoError(err, "while checking limits in CSINode: %v", err) + + gomega.Expect(csiNodeAttachLimit).To(gomega.BeNumerically("==", 2)) + + _, _, pod1 := m.createPod(pvcReference) + gomega.Expect(pod1).NotTo(gomega.BeNil(), "while creating first pod") + + err = e2epod.WaitForPodNameRunningInNamespace(m.cs, pod1.Name, pod1.Namespace) + framework.ExpectNoError(err, "Failed to start pod1: %v", err) + + _, _, pod2 := m.createPod(pvcReference) + gomega.Expect(pod2).NotTo(gomega.BeNil(), "while creating second pod") + + err = e2epod.WaitForPodNameRunningInNamespace(m.cs, pod2.Name, pod2.Namespace) + framework.ExpectNoError(err, "Failed to start pod2: %v", err) + + _, _, pod3 := m.createPod(pvcReference) + gomega.Expect(pod3).NotTo(gomega.BeNil(), "while creating third pod") + err = waitForMaxVolumeCondition(pod3, m.cs) + framework.ExpectNoError(err, "while waiting for max volume condition on pod : %+v", pod3) + }) + + ginkgo.It("should report attach limit for generic ephemeral volume when persistent volume is attached [Slow]", func(ctx context.Context) { + // define volume limit to be 2 for this test + var err error + m.init(testParameters{attachLimit: 1}) + defer m.cleanup() + nodeName := m.config.ClientNodeSelection.Name + driverName := m.config.GetUniqueDriverName() + + csiNodeAttachLimit, err := checkCSINodeForLimits(nodeName, driverName, m.cs) + framework.ExpectNoError(err, "while checking limits in CSINode: %v", err) + + gomega.Expect(csiNodeAttachLimit).To(gomega.BeNumerically("==", 1)) + + _, _, pod1 := m.createPod(pvcReference) + gomega.Expect(pod1).NotTo(gomega.BeNil(), "while creating pod with persistent volume") + + err = e2epod.WaitForPodNameRunningInNamespace(m.cs, pod1.Name, pod1.Namespace) + framework.ExpectNoError(err, "Failed to start pod1: %v", err) + + _, _, pod2 := m.createPod(genericEphemeral) + gomega.Expect(pod2).NotTo(gomega.BeNil(), "while creating pod with ephemeral volume") + err = waitForMaxVolumeCondition(pod2, m.cs) + framework.ExpectNoError(err, "while waiting for max volume condition on pod : %+v", pod2) + }) + + ginkgo.It("should report attach limit for persistent volume when generic ephemeral volume is attached [Slow]", func(ctx context.Context) { + // define volume limit to be 2 for this test + var err error + m.init(testParameters{attachLimit: 1}) + defer m.cleanup() + nodeName := m.config.ClientNodeSelection.Name + driverName := m.config.GetUniqueDriverName() + + csiNodeAttachLimit, err := checkCSINodeForLimits(nodeName, driverName, m.cs) + framework.ExpectNoError(err, "while checking limits in CSINode: %v", err) + + gomega.Expect(csiNodeAttachLimit).To(gomega.BeNumerically("==", 1)) + + _, _, pod1 := m.createPod(genericEphemeral) + gomega.Expect(pod1).NotTo(gomega.BeNil(), "while creating pod with persistent volume") + + err = e2epod.WaitForPodNameRunningInNamespace(m.cs, pod1.Name, pod1.Namespace) + framework.ExpectNoError(err, "Failed to start pod1: %v", err) + + _, _, pod2 := m.createPod(pvcReference) + gomega.Expect(pod2).NotTo(gomega.BeNil(), "while creating pod with ephemeral volume") + err = waitForMaxVolumeCondition(pod2, m.cs) + framework.ExpectNoError(err, "while waiting for max volume condition on pod : %+v", pod2) + }) + }) +}) + +func checkCSINodeForLimits(nodeName string, driverName string, cs clientset.Interface) (int32, error) { + var attachLimit int32 + + waitErr := wait.PollImmediate(10*time.Second, csiNodeLimitUpdateTimeout, func() (bool, error) { + csiNode, err := cs.StorageV1().CSINodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return false, err + } + attachLimit = getVolumeLimitFromCSINode(csiNode, driverName) + if attachLimit > 0 { + return true, nil + } + return false, nil + }) + if waitErr != nil { + return 0, fmt.Errorf("error waiting for non-zero volume limit of driver %s on node %s: %v", driverName, nodeName, waitErr) + } + return attachLimit, nil +} + +func getVolumeLimitFromCSINode(csiNode *storagev1.CSINode, driverName string) int32 { + for _, d := range csiNode.Spec.Drivers { + if d.Name != driverName { + continue + } + if d.Allocatable != nil && d.Allocatable.Count != nil { + return *d.Allocatable.Count + } + } + return 0 +} diff --git a/test/e2e/storage/csi_mock/csi_workload.go b/test/e2e/storage/csi_mock/csi_workload.go new file mode 100644 index 0000000000000..0d6d2aa421b0a --- /dev/null +++ b/test/e2e/storage/csi_mock/csi_workload.go @@ -0,0 +1,124 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi_mock + +import ( + "context" + + "github.com/onsi/ginkgo/v2" + "k8s.io/kubernetes/test/e2e/framework" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + "k8s.io/kubernetes/test/e2e/storage/testsuites" + "k8s.io/kubernetes/test/e2e/storage/utils" + admissionapi "k8s.io/pod-security-admission/api" +) + +var _ = utils.SIGDescribe("CSI Mock workload info", func() { + // The CSIDriverRegistry feature gate is needed for this test in Kubernetes 1.12. + f := framework.NewDefaultFramework("csi-mock-volumes-workload") + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged + m := newMockDriverSetup(f) + ginkgo.Context("CSI workload information using mock driver", func() { + var ( + err error + podInfoTrue = true + podInfoFalse = false + ) + tests := []struct { + name string + podInfoOnMount *bool + deployClusterRegistrar bool + expectPodInfo bool + expectEphemeral bool + }{ + { + name: "should not be passed when podInfoOnMount=nil", + podInfoOnMount: nil, + deployClusterRegistrar: true, + expectPodInfo: false, + expectEphemeral: false, + }, + { + name: "should be passed when podInfoOnMount=true", + podInfoOnMount: &podInfoTrue, + deployClusterRegistrar: true, + expectPodInfo: true, + expectEphemeral: false, + }, + { + name: "contain ephemeral=true when using inline volume", + podInfoOnMount: &podInfoTrue, + deployClusterRegistrar: true, + expectPodInfo: true, + expectEphemeral: true, + }, + { + name: "should not be passed when podInfoOnMount=false", + podInfoOnMount: &podInfoFalse, + deployClusterRegistrar: true, + expectPodInfo: false, + expectEphemeral: false, + }, + { + name: "should not be passed when CSIDriver does not exist", + deployClusterRegistrar: false, + expectPodInfo: false, + expectEphemeral: false, + }, + } + for _, t := range tests { + test := t + ginkgo.It(t.name, func(ctx context.Context) { + m.init(testParameters{ + registerDriver: test.deployClusterRegistrar, + podInfo: test.podInfoOnMount}) + + defer m.cleanup() + + withVolume := pvcReference + if test.expectEphemeral { + withVolume = csiEphemeral + } + _, _, pod := m.createPod(withVolume) + if pod == nil { + return + } + err = e2epod.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) + framework.ExpectNoError(err, "Failed to start pod: %v", err) + + // If we expect an ephemeral volume, the feature has to be enabled. + // Otherwise need to check if we expect pod info, because the content + // of that depends on whether the feature is enabled or not. + csiInlineVolumesEnabled := test.expectEphemeral + if test.expectPodInfo { + ginkgo.By("checking for CSIInlineVolumes feature") + csiInlineVolumesEnabled, err = testsuites.CSIInlineVolumesEnabled(ctx, m.cs, f.Timeouts, f.Namespace.Name) + framework.ExpectNoError(err, "failed to test for CSIInlineVolumes") + } + + ginkgo.By("Deleting the previously created pod") + err = e2epod.DeletePodWithWait(m.cs, pod) + framework.ExpectNoError(err, "while deleting") + + ginkgo.By("Checking CSI driver logs") + err = checkPodLogs(m.driver.GetCalls, pod, test.expectPodInfo, test.expectEphemeral, csiInlineVolumesEnabled, false, 1) + framework.ExpectNoError(err) + }) + } + }) + +}) diff --git a/test/e2e/storage/csi_mock_volume.go b/test/e2e/storage/csi_mock_volume.go deleted file mode 100644 index 7e0d84eb27661..0000000000000 --- a/test/e2e/storage/csi_mock_volume.go +++ /dev/null @@ -1,2858 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package storage - -import ( - "context" - "errors" - "fmt" - "math/rand" - "strconv" - "strings" - "sync/atomic" - "time" - - csipbv1 "github.com/container-storage-interface/spec/lib/go/csi" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - v1 "k8s.io/api/core/v1" - storagev1 "k8s.io/api/storage/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/fields" - utilerrors "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/apimachinery/pkg/watch" - clientset "k8s.io/client-go/kubernetes" - cachetools "k8s.io/client-go/tools/cache" - watchtools "k8s.io/client-go/tools/watch" - "k8s.io/kubernetes/pkg/kubelet/events" - "k8s.io/kubernetes/test/e2e/framework" - e2eevents "k8s.io/kubernetes/test/e2e/framework/events" - e2emetrics "k8s.io/kubernetes/test/e2e/framework/metrics" - e2epod "k8s.io/kubernetes/test/e2e/framework/pod" - e2epv "k8s.io/kubernetes/test/e2e/framework/pv" - e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" - e2evolume "k8s.io/kubernetes/test/e2e/framework/volume" - "k8s.io/kubernetes/test/e2e/storage/drivers" - storageframework "k8s.io/kubernetes/test/e2e/storage/framework" - "k8s.io/kubernetes/test/e2e/storage/testsuites" - "k8s.io/kubernetes/test/e2e/storage/utils" - imageutils "k8s.io/kubernetes/test/utils/image" - admissionapi "k8s.io/pod-security-admission/api" - utilptr "k8s.io/utils/pointer" - - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" -) - -const ( - csiNodeLimitUpdateTimeout = 5 * time.Minute - csiPodUnschedulableTimeout = 5 * time.Minute - csiResizeWaitPeriod = 5 * time.Minute - csiVolumeAttachmentTimeout = 7 * time.Minute - // how long to wait for Resizing Condition on PVC to appear - csiResizingConditionWait = 2 * time.Minute - - // Time for starting a pod with a volume. - csiPodRunningTimeout = 5 * time.Minute - - // How log to wait for kubelet to unstage a volume after a pod is deleted - csiUnstageWaitTimeout = 1 * time.Minute -) - -// csiCall represents an expected call from Kubernetes to CSI mock driver and -// expected return value. -// When matching expected csiCall with a real CSI mock driver output, one csiCall -// matches *one or more* calls with the same method and error code. -// This is due to exponential backoff in Kubernetes, where the test cannot expect -// exact number of call repetitions. -type csiCall struct { - expectedMethod string - expectedError codes.Code - // This is a mark for the test itself to delete the tested pod *after* - // this csiCall is received. - deletePod bool -} - -var _ = utils.SIGDescribe("CSI mock volume", func() { - type testParameters struct { - disableAttach bool - attachLimit int - registerDriver bool - lateBinding bool - enableTopology bool - podInfo *bool - storageCapacity *bool - scName string // pre-selected storage class name; must be unique in the cluster - enableResizing bool // enable resizing for both CSI mock driver and storageClass. - enableNodeExpansion bool // enable node expansion for CSI mock driver - // just disable resizing on driver it overrides enableResizing flag for CSI mock driver - disableResizingOnDriver bool - enableSnapshot bool - enableVolumeMountGroup bool // enable the VOLUME_MOUNT_GROUP node capability in the CSI mock driver. - hooks *drivers.Hooks - tokenRequests []storagev1.TokenRequest - requiresRepublish *bool - fsGroupPolicy *storagev1.FSGroupPolicy - enableSELinuxMount *bool - } - - type mockDriverSetup struct { - cs clientset.Interface - config *storageframework.PerTestConfig - testCleanups []func() - pods []*v1.Pod - pvcs []*v1.PersistentVolumeClaim - sc map[string]*storagev1.StorageClass - vsc map[string]*unstructured.Unstructured - driver drivers.MockCSITestDriver - provisioner string - tp testParameters - } - - var m mockDriverSetup - - f := framework.NewDefaultFramework("csi-mock-volumes") - f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged - - init := func(tp testParameters) { - m = mockDriverSetup{ - cs: f.ClientSet, - sc: make(map[string]*storagev1.StorageClass), - vsc: make(map[string]*unstructured.Unstructured), - tp: tp, - } - cs := f.ClientSet - var err error - driverOpts := drivers.CSIMockDriverOpts{ - RegisterDriver: tp.registerDriver, - PodInfo: tp.podInfo, - StorageCapacity: tp.storageCapacity, - EnableTopology: tp.enableTopology, - AttachLimit: tp.attachLimit, - DisableAttach: tp.disableAttach, - EnableResizing: tp.enableResizing, - EnableNodeExpansion: tp.enableNodeExpansion, - EnableSnapshot: tp.enableSnapshot, - EnableVolumeMountGroup: tp.enableVolumeMountGroup, - TokenRequests: tp.tokenRequests, - RequiresRepublish: tp.requiresRepublish, - FSGroupPolicy: tp.fsGroupPolicy, - EnableSELinuxMount: tp.enableSELinuxMount, - } - - // At the moment, only tests which need hooks are - // using the embedded CSI mock driver. The rest run - // the driver inside the cluster although they could - // changed to use embedding merely by setting - // driverOpts.embedded to true. - // - // Not enabling it for all tests minimizes - // the risk that the introduction of embedded breaks - // some existings tests and avoids a dependency - // on port forwarding, which is important if some of - // these tests are supposed to become part of - // conformance testing (port forwarding isn't - // currently required). - if tp.hooks != nil { - driverOpts.Embedded = true - driverOpts.Hooks = *tp.hooks - } - - // this just disable resizing on driver, keeping resizing on SC enabled. - if tp.disableResizingOnDriver { - driverOpts.EnableResizing = false - } - - m.driver = drivers.InitMockCSIDriver(driverOpts) - config := m.driver.PrepareTest(f) - m.config = config - m.provisioner = config.GetUniqueDriverName() - - if tp.registerDriver { - err = waitForCSIDriver(cs, m.config.GetUniqueDriverName()) - framework.ExpectNoError(err, "Failed to get CSIDriver %v", m.config.GetUniqueDriverName()) - ginkgo.DeferCleanup(destroyCSIDriver, cs, m.config.GetUniqueDriverName()) - } - - // Wait for the CSIDriver actually get deployed and CSINode object to be generated. - // This indicates the mock CSI driver pod is up and running healthy. - err = drivers.WaitForCSIDriverRegistrationOnNode(m.config.ClientNodeSelection.Name, m.config.GetUniqueDriverName(), cs) - framework.ExpectNoError(err, "Failed to register CSIDriver %v", m.config.GetUniqueDriverName()) - } - - type volumeType string - csiEphemeral := volumeType("CSI") - genericEphemeral := volumeType("Ephemeral") - pvcReference := volumeType("PVC") - createPod := func(withVolume volumeType) (class *storagev1.StorageClass, claim *v1.PersistentVolumeClaim, pod *v1.Pod) { - ginkgo.By("Creating pod") - sc := m.driver.GetDynamicProvisionStorageClass(m.config, "") - scTest := testsuites.StorageClassTest{ - Name: m.driver.GetDriverInfo().Name, - Timeouts: f.Timeouts, - Provisioner: sc.Provisioner, - Parameters: sc.Parameters, - ClaimSize: "1Gi", - ExpectedSize: "1Gi", - DelayBinding: m.tp.lateBinding, - AllowVolumeExpansion: m.tp.enableResizing, - } - - // The mock driver only works when everything runs on a single node. - nodeSelection := m.config.ClientNodeSelection - switch withVolume { - case csiEphemeral: - pod = startPausePodInline(f.ClientSet, scTest, nodeSelection, f.Namespace.Name) - case genericEphemeral: - class, pod = startPausePodGenericEphemeral(f.ClientSet, scTest, nodeSelection, m.tp.scName, f.Namespace.Name) - if class != nil { - m.sc[class.Name] = class - } - claim = &v1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: pod.Name + "-" + pod.Spec.Volumes[0].Name, - Namespace: f.Namespace.Name, - }, - } - case pvcReference: - class, claim, pod = startPausePod(f.ClientSet, scTest, nodeSelection, m.tp.scName, f.Namespace.Name) - if class != nil { - m.sc[class.Name] = class - } - if claim != nil { - m.pvcs = append(m.pvcs, claim) - } - } - if pod != nil { - m.pods = append(m.pods, pod) - } - return // result variables set above - } - - createPodWithPVC := func(pvc *v1.PersistentVolumeClaim) (*v1.Pod, error) { - nodeSelection := m.config.ClientNodeSelection - pod, err := startPausePodWithClaim(m.cs, pvc, nodeSelection, f.Namespace.Name) - if pod != nil { - m.pods = append(m.pods, pod) - } - return pod, err - } - - createPodWithFSGroup := func(fsGroup *int64) (*storagev1.StorageClass, *v1.PersistentVolumeClaim, *v1.Pod) { - ginkgo.By("Creating pod with fsGroup") - nodeSelection := m.config.ClientNodeSelection - sc := m.driver.GetDynamicProvisionStorageClass(m.config, "") - scTest := testsuites.StorageClassTest{ - Name: m.driver.GetDriverInfo().Name, - Provisioner: sc.Provisioner, - Parameters: sc.Parameters, - ClaimSize: "1Gi", - ExpectedSize: "1Gi", - DelayBinding: m.tp.lateBinding, - AllowVolumeExpansion: m.tp.enableResizing, - } - class, claim, pod := startBusyBoxPod(f.ClientSet, scTest, nodeSelection, m.tp.scName, f.Namespace.Name, fsGroup) - - if class != nil { - m.sc[class.Name] = class - } - if claim != nil { - m.pvcs = append(m.pvcs, claim) - } - - if pod != nil { - m.pods = append(m.pods, pod) - } - - return class, claim, pod - } - - createPodWithSELinux := func(accessModes []v1.PersistentVolumeAccessMode, mountOptions []string, seLinuxOpts *v1.SELinuxOptions) (*storagev1.StorageClass, *v1.PersistentVolumeClaim, *v1.Pod) { - ginkgo.By("Creating pod with SELinux context") - nodeSelection := m.config.ClientNodeSelection - sc := m.driver.GetDynamicProvisionStorageClass(m.config, "") - scTest := testsuites.StorageClassTest{ - Name: m.driver.GetDriverInfo().Name, - Provisioner: sc.Provisioner, - Parameters: sc.Parameters, - ClaimSize: "1Gi", - ExpectedSize: "1Gi", - DelayBinding: m.tp.lateBinding, - AllowVolumeExpansion: m.tp.enableResizing, - MountOptions: mountOptions, - } - class, claim := createClaim(f.ClientSet, scTest, nodeSelection, m.tp.scName, f.Namespace.Name, accessModes) - pod, err := startPausePodWithSELinuxOptions(f.ClientSet, claim, nodeSelection, f.Namespace.Name, seLinuxOpts) - framework.ExpectNoError(err, "Failed to create pause pod with SELinux context %s: %v", seLinuxOpts, err) - - if class != nil { - m.sc[class.Name] = class - } - if claim != nil { - m.pvcs = append(m.pvcs, claim) - } - - if pod != nil { - m.pods = append(m.pods, pod) - } - - return class, claim, pod - } - - cleanup := func() { - cs := f.ClientSet - var errs []error - - for _, pod := range m.pods { - ginkgo.By(fmt.Sprintf("Deleting pod %s", pod.Name)) - errs = append(errs, e2epod.DeletePodWithWait(cs, pod)) - } - - for _, claim := range m.pvcs { - ginkgo.By(fmt.Sprintf("Deleting claim %s", claim.Name)) - claim, err := cs.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(context.TODO(), claim.Name, metav1.GetOptions{}) - if err == nil { - if err := cs.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(context.TODO(), claim.Name, metav1.DeleteOptions{}); err != nil { - errs = append(errs, err) - } - if claim.Spec.VolumeName != "" { - errs = append(errs, e2epv.WaitForPersistentVolumeDeleted(cs, claim.Spec.VolumeName, framework.Poll, 2*time.Minute)) - } - } - } - - for _, sc := range m.sc { - ginkgo.By(fmt.Sprintf("Deleting storageclass %s", sc.Name)) - cs.StorageV1().StorageClasses().Delete(context.TODO(), sc.Name, metav1.DeleteOptions{}) - } - - for _, vsc := range m.vsc { - ginkgo.By(fmt.Sprintf("Deleting volumesnapshotclass %s", vsc.GetName())) - m.config.Framework.DynamicClient.Resource(utils.SnapshotClassGVR).Delete(context.TODO(), vsc.GetName(), metav1.DeleteOptions{}) - } - ginkgo.By("Cleaning up resources") - for _, cleanupFunc := range m.testCleanups { - cleanupFunc() - } - - err := utilerrors.NewAggregate(errs) - framework.ExpectNoError(err, "while cleaning up after test") - } - - // The CSIDriverRegistry feature gate is needed for this test in Kubernetes 1.12. - ginkgo.Context("CSI attach test using mock driver", func() { - tests := []struct { - name string - disableAttach bool - deployClusterRegistrar bool - volumeType volumeType - }{ - { - name: "should not require VolumeAttach for drivers without attachment", - disableAttach: true, - deployClusterRegistrar: true, - }, - { - name: "should require VolumeAttach for drivers with attachment", - deployClusterRegistrar: true, - }, - { - name: "should require VolumeAttach for ephemermal volume and drivers with attachment", - deployClusterRegistrar: true, - volumeType: genericEphemeral, - }, - { - name: "should preserve attachment policy when no CSIDriver present", - deployClusterRegistrar: false, - }, - } - for _, t := range tests { - test := t - ginkgo.It(t.name, func(ctx context.Context) { - var err error - init(testParameters{registerDriver: test.deployClusterRegistrar, disableAttach: test.disableAttach}) - ginkgo.DeferCleanup(cleanup) - - volumeType := test.volumeType - if volumeType == "" { - volumeType = pvcReference - } - _, claim, pod := createPod(volumeType) - if pod == nil { - return - } - err = e2epod.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) - framework.ExpectNoError(err, "Failed to start pod: %v", err) - - ginkgo.By("Checking if VolumeAttachment was created for the pod") - testConfig := storageframework.ConvertTestConfig(m.config) - attachmentName := e2evolume.GetVolumeAttachmentName(m.cs, testConfig, m.provisioner, claim.Name, claim.Namespace) - _, err = m.cs.StorageV1().VolumeAttachments().Get(context.TODO(), attachmentName, metav1.GetOptions{}) - if err != nil { - if apierrors.IsNotFound(err) { - if !test.disableAttach { - framework.ExpectNoError(err, "Expected VolumeAttachment but none was found") - } - } else { - framework.ExpectNoError(err, "Failed to find VolumeAttachment") - } - } - if test.disableAttach { - framework.ExpectError(err, "Unexpected VolumeAttachment found") - } - }) - - } - }) - - ginkgo.Context("CSI CSIDriver deployment after pod creation using non-attachable mock driver", func() { - ginkgo.It("should bringup pod after deploying CSIDriver attach=false [Slow]", func(ctx context.Context) { - var err error - init(testParameters{registerDriver: false, disableAttach: true}) - ginkgo.DeferCleanup(cleanup) - - _, claim, pod := createPod(pvcReference) // late binding as specified above - if pod == nil { - return - } - - ginkgo.By("Checking if attaching failed and pod cannot start") - eventSelector := fields.Set{ - "involvedObject.kind": "Pod", - "involvedObject.name": pod.Name, - "involvedObject.namespace": pod.Namespace, - "reason": events.FailedAttachVolume, - }.AsSelector().String() - msg := "AttachVolume.Attach failed for volume" - - err = e2eevents.WaitTimeoutForEvent(m.cs, pod.Namespace, eventSelector, msg, f.Timeouts.PodStart) - if err != nil { - podErr := e2epod.WaitTimeoutForPodRunningInNamespace(m.cs, pod.Name, pod.Namespace, 10*time.Second) - framework.ExpectError(podErr, "Pod should not be in running status because attaching should failed") - // Events are unreliable, don't depend on the event. It's used only to speed up the test. - framework.Logf("Attach should fail and the corresponding event should show up, error: %v", err) - } - - // VolumeAttachment should be created because the default value for CSI attachable is true - ginkgo.By("Checking if VolumeAttachment was created for the pod") - testConfig := storageframework.ConvertTestConfig(m.config) - attachmentName := e2evolume.GetVolumeAttachmentName(m.cs, testConfig, m.provisioner, claim.Name, claim.Namespace) - _, err = m.cs.StorageV1().VolumeAttachments().Get(context.TODO(), attachmentName, metav1.GetOptions{}) - if err != nil { - if apierrors.IsNotFound(err) { - framework.ExpectNoError(err, "Expected VolumeAttachment but none was found") - } else { - framework.ExpectNoError(err, "Failed to find VolumeAttachment") - } - } - - ginkgo.By("Deploy CSIDriver object with attachRequired=false") - driverNamespace := m.config.DriverNamespace - - canAttach := false - o := utils.PatchCSIOptions{ - OldDriverName: "csi-mock", - NewDriverName: "csi-mock-" + f.UniqueName, - CanAttach: &canAttach, - } - err = utils.CreateFromManifests(f, driverNamespace, func(item interface{}) error { - return utils.PatchCSIDeployment(f, o, item) - }, "test/e2e/testing-manifests/storage-csi/mock/csi-mock-driverinfo.yaml") - if err != nil { - framework.Failf("fail to deploy CSIDriver object: %v", err) - } - - ginkgo.By("Wait for the pod in running status") - err = e2epod.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) - framework.ExpectNoError(err, "Failed to start pod: %v", err) - - ginkgo.By(fmt.Sprintf("Wait for the volumeattachment to be deleted up to %v", csiVolumeAttachmentTimeout)) - // This step can be slow because we have to wait either a NodeUpdate event happens or - // the detachment for this volume timeout so that we can do a force detach. - err = e2evolume.WaitForVolumeAttachmentTerminated(attachmentName, m.cs, csiVolumeAttachmentTimeout) - framework.ExpectNoError(err, "Failed to delete VolumeAttachment: %v", err) - }) - }) - - ginkgo.Context("CSI workload information using mock driver", func() { - var ( - err error - podInfoTrue = true - podInfoFalse = false - ) - tests := []struct { - name string - podInfoOnMount *bool - deployClusterRegistrar bool - expectPodInfo bool - expectEphemeral bool - }{ - { - name: "should not be passed when podInfoOnMount=nil", - podInfoOnMount: nil, - deployClusterRegistrar: true, - expectPodInfo: false, - expectEphemeral: false, - }, - { - name: "should be passed when podInfoOnMount=true", - podInfoOnMount: &podInfoTrue, - deployClusterRegistrar: true, - expectPodInfo: true, - expectEphemeral: false, - }, - { - name: "contain ephemeral=true when using inline volume", - podInfoOnMount: &podInfoTrue, - deployClusterRegistrar: true, - expectPodInfo: true, - expectEphemeral: true, - }, - { - name: "should not be passed when podInfoOnMount=false", - podInfoOnMount: &podInfoFalse, - deployClusterRegistrar: true, - expectPodInfo: false, - expectEphemeral: false, - }, - { - name: "should not be passed when CSIDriver does not exist", - deployClusterRegistrar: false, - expectPodInfo: false, - expectEphemeral: false, - }, - } - for _, t := range tests { - test := t - ginkgo.It(t.name, func(ctx context.Context) { - init(testParameters{ - registerDriver: test.deployClusterRegistrar, - podInfo: test.podInfoOnMount}) - ginkgo.DeferCleanup(cleanup) - - withVolume := pvcReference - if test.expectEphemeral { - withVolume = csiEphemeral - } - _, _, pod := createPod(withVolume) - if pod == nil { - return - } - err = e2epod.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) - framework.ExpectNoError(err, "Failed to start pod: %v", err) - - // If we expect an ephemeral volume, the feature has to be enabled. - // Otherwise need to check if we expect pod info, because the content - // of that depends on whether the feature is enabled or not. - csiInlineVolumesEnabled := test.expectEphemeral - if test.expectPodInfo { - ginkgo.By("checking for CSIInlineVolumes feature") - csiInlineVolumesEnabled, err = testsuites.CSIInlineVolumesEnabled(ctx, m.cs, f.Timeouts, f.Namespace.Name) - framework.ExpectNoError(err, "failed to test for CSIInlineVolumes") - } - - ginkgo.By("Deleting the previously created pod") - err = e2epod.DeletePodWithWait(m.cs, pod) - framework.ExpectNoError(err, "while deleting") - - ginkgo.By("Checking CSI driver logs") - err = checkPodLogs(m.driver.GetCalls, pod, test.expectPodInfo, test.expectEphemeral, csiInlineVolumesEnabled, false, 1) - framework.ExpectNoError(err) - }) - } - }) - - ginkgo.Context("CSI volume limit information using mock driver", func() { - ginkgo.It("should report attach limit when limit is bigger than 0 [Slow]", func(ctx context.Context) { - // define volume limit to be 2 for this test - var err error - init(testParameters{attachLimit: 2}) - ginkgo.DeferCleanup(cleanup) - nodeName := m.config.ClientNodeSelection.Name - driverName := m.config.GetUniqueDriverName() - - csiNodeAttachLimit, err := checkCSINodeForLimits(nodeName, driverName, m.cs) - framework.ExpectNoError(err, "while checking limits in CSINode: %v", err) - - gomega.Expect(csiNodeAttachLimit).To(gomega.BeNumerically("==", 2)) - - _, _, pod1 := createPod(pvcReference) - gomega.Expect(pod1).NotTo(gomega.BeNil(), "while creating first pod") - - err = e2epod.WaitForPodNameRunningInNamespace(m.cs, pod1.Name, pod1.Namespace) - framework.ExpectNoError(err, "Failed to start pod1: %v", err) - - _, _, pod2 := createPod(pvcReference) - gomega.Expect(pod2).NotTo(gomega.BeNil(), "while creating second pod") - - err = e2epod.WaitForPodNameRunningInNamespace(m.cs, pod2.Name, pod2.Namespace) - framework.ExpectNoError(err, "Failed to start pod2: %v", err) - - _, _, pod3 := createPod(pvcReference) - gomega.Expect(pod3).NotTo(gomega.BeNil(), "while creating third pod") - err = waitForMaxVolumeCondition(pod3, m.cs) - framework.ExpectNoError(err, "while waiting for max volume condition on pod : %+v", pod3) - }) - - ginkgo.It("should report attach limit for generic ephemeral volume when persistent volume is attached [Slow]", func(ctx context.Context) { - // define volume limit to be 2 for this test - var err error - init(testParameters{attachLimit: 1}) - ginkgo.DeferCleanup(cleanup) - nodeName := m.config.ClientNodeSelection.Name - driverName := m.config.GetUniqueDriverName() - - csiNodeAttachLimit, err := checkCSINodeForLimits(nodeName, driverName, m.cs) - framework.ExpectNoError(err, "while checking limits in CSINode: %v", err) - - gomega.Expect(csiNodeAttachLimit).To(gomega.BeNumerically("==", 1)) - - _, _, pod1 := createPod(pvcReference) - gomega.Expect(pod1).NotTo(gomega.BeNil(), "while creating pod with persistent volume") - - err = e2epod.WaitForPodNameRunningInNamespace(m.cs, pod1.Name, pod1.Namespace) - framework.ExpectNoError(err, "Failed to start pod1: %v", err) - - _, _, pod2 := createPod(genericEphemeral) - gomega.Expect(pod2).NotTo(gomega.BeNil(), "while creating pod with ephemeral volume") - err = waitForMaxVolumeCondition(pod2, m.cs) - framework.ExpectNoError(err, "while waiting for max volume condition on pod : %+v", pod2) - }) - - ginkgo.It("should report attach limit for persistent volume when generic ephemeral volume is attached [Slow]", func(ctx context.Context) { - // define volume limit to be 2 for this test - var err error - init(testParameters{attachLimit: 1}) - ginkgo.DeferCleanup(cleanup) - nodeName := m.config.ClientNodeSelection.Name - driverName := m.config.GetUniqueDriverName() - - csiNodeAttachLimit, err := checkCSINodeForLimits(nodeName, driverName, m.cs) - framework.ExpectNoError(err, "while checking limits in CSINode: %v", err) - - gomega.Expect(csiNodeAttachLimit).To(gomega.BeNumerically("==", 1)) - - _, _, pod1 := createPod(genericEphemeral) - gomega.Expect(pod1).NotTo(gomega.BeNil(), "while creating pod with persistent volume") - - err = e2epod.WaitForPodNameRunningInNamespace(m.cs, pod1.Name, pod1.Namespace) - framework.ExpectNoError(err, "Failed to start pod1: %v", err) - - _, _, pod2 := createPod(pvcReference) - gomega.Expect(pod2).NotTo(gomega.BeNil(), "while creating pod with ephemeral volume") - err = waitForMaxVolumeCondition(pod2, m.cs) - framework.ExpectNoError(err, "while waiting for max volume condition on pod : %+v", pod2) - }) - }) - - ginkgo.Context("CSI Volume expansion", func() { - tests := []struct { - name string - nodeExpansionRequired bool - disableAttach bool - disableResizingOnDriver bool - expectFailure bool - }{ - { - name: "should expand volume without restarting pod if nodeExpansion=off", - nodeExpansionRequired: false, - }, - { - name: "should expand volume by restarting pod if attach=on, nodeExpansion=on", - nodeExpansionRequired: true, - }, - { - name: "should expand volume by restarting pod if attach=off, nodeExpansion=on", - disableAttach: true, - nodeExpansionRequired: true, - }, - { - name: "should not expand volume if resizingOnDriver=off, resizingOnSC=on", - disableResizingOnDriver: true, - expectFailure: true, - }, - } - for _, t := range tests { - test := t - ginkgo.It(t.name, func(ctx context.Context) { - var err error - tp := testParameters{ - enableResizing: true, - enableNodeExpansion: test.nodeExpansionRequired, - disableResizingOnDriver: test.disableResizingOnDriver, - } - // disabling attach requires drive registration feature - if test.disableAttach { - tp.disableAttach = true - tp.registerDriver = true - } - - init(tp) - ginkgo.DeferCleanup(cleanup) - - sc, pvc, pod := createPod(pvcReference) - gomega.Expect(pod).NotTo(gomega.BeNil(), "while creating pod for resizing") - - if !*sc.AllowVolumeExpansion { - framework.Fail("failed creating sc with allowed expansion") - } - - err = e2epod.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) - framework.ExpectNoError(err, "Failed to start pod1: %v", err) - - ginkgo.By("Expanding current pvc") - newSize := resource.MustParse("6Gi") - newPVC, err := testsuites.ExpandPVCSize(pvc, newSize, m.cs) - framework.ExpectNoError(err, "While updating pvc for more size") - pvc = newPVC - gomega.Expect(pvc).NotTo(gomega.BeNil()) - - pvcSize := pvc.Spec.Resources.Requests[v1.ResourceStorage] - if pvcSize.Cmp(newSize) != 0 { - framework.Failf("error updating pvc size %q", pvc.Name) - } - if test.expectFailure { - err = testsuites.WaitForResizingCondition(pvc, m.cs, csiResizingConditionWait) - framework.ExpectError(err, "unexpected resizing condition on PVC") - return - } - - ginkgo.By("Waiting for persistent volume resize to finish") - err = testsuites.WaitForControllerVolumeResize(pvc, m.cs, csiResizeWaitPeriod) - framework.ExpectNoError(err, "While waiting for CSI PV resize to finish") - - checkPVCSize := func() { - ginkgo.By("Waiting for PVC resize to finish") - pvc, err = testsuites.WaitForFSResize(pvc, m.cs) - framework.ExpectNoError(err, "while waiting for PVC resize to finish") - - pvcConditions := pvc.Status.Conditions - framework.ExpectEqual(len(pvcConditions), 0, "pvc should not have conditions") - } - - // if node expansion is not required PVC should be resized as well - if !test.nodeExpansionRequired { - checkPVCSize() - } else { - ginkgo.By("Checking for conditions on pvc") - npvc, err := testsuites.WaitForPendingFSResizeCondition(pvc, m.cs) - framework.ExpectNoError(err, "While waiting for pvc to have fs resizing condition") - pvc = npvc - - inProgressConditions := pvc.Status.Conditions - if len(inProgressConditions) > 0 { - framework.ExpectEqual(inProgressConditions[0].Type, v1.PersistentVolumeClaimFileSystemResizePending, "pvc must have fs resizing condition") - } - - ginkgo.By("Deleting the previously created pod") - err = e2epod.DeletePodWithWait(m.cs, pod) - framework.ExpectNoError(err, "while deleting pod for resizing") - - ginkgo.By("Creating a new pod with same volume") - pod2, err := createPodWithPVC(pvc) - gomega.Expect(pod2).NotTo(gomega.BeNil(), "while creating pod for csi resizing") - framework.ExpectNoError(err, "while recreating pod for resizing") - - checkPVCSize() - } - }) - } - }) - ginkgo.Context("CSI online volume expansion", func() { - tests := []struct { - name string - disableAttach bool - }{ - { - name: "should expand volume without restarting pod if attach=on, nodeExpansion=on", - }, - { - name: "should expand volume without restarting pod if attach=off, nodeExpansion=on", - disableAttach: true, - }, - } - for _, t := range tests { - test := t - ginkgo.It(test.name, func(ctx context.Context) { - var err error - params := testParameters{enableResizing: true, enableNodeExpansion: true} - if test.disableAttach { - params.disableAttach = true - params.registerDriver = true - } - - init(params) - ginkgo.DeferCleanup(cleanup) - - sc, pvc, pod := createPod(pvcReference) - gomega.Expect(pod).NotTo(gomega.BeNil(), "while creating pod for resizing") - - if !*sc.AllowVolumeExpansion { - framework.Fail("failed creating sc with allowed expansion") - } - - err = e2epod.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) - framework.ExpectNoError(err, "Failed to start pod1: %v", err) - - ginkgo.By("Expanding current pvc") - newSize := resource.MustParse("6Gi") - newPVC, err := testsuites.ExpandPVCSize(pvc, newSize, m.cs) - framework.ExpectNoError(err, "While updating pvc for more size") - pvc = newPVC - gomega.Expect(pvc).NotTo(gomega.BeNil()) - - pvcSize := pvc.Spec.Resources.Requests[v1.ResourceStorage] - if pvcSize.Cmp(newSize) != 0 { - framework.Failf("error updating pvc size %q", pvc.Name) - } - - ginkgo.By("Waiting for persistent volume resize to finish") - err = testsuites.WaitForControllerVolumeResize(pvc, m.cs, csiResizeWaitPeriod) - framework.ExpectNoError(err, "While waiting for PV resize to finish") - - ginkgo.By("Waiting for PVC resize to finish") - pvc, err = testsuites.WaitForFSResize(pvc, m.cs) - framework.ExpectNoError(err, "while waiting for PVC to finish") - - pvcConditions := pvc.Status.Conditions - framework.ExpectEqual(len(pvcConditions), 0, "pvc should not have conditions") - - }) - } - }) - - ginkgo.Context("CSI NodeStage error cases [Slow]", func() { - trackedCalls := []string{ - "NodeStageVolume", - "NodeUnstageVolume", - } - - tests := []struct { - name string - expectPodRunning bool - expectedCalls []csiCall - - // Called for each NodeStateVolume calls, with counter incremented atomically before - // the invocation (i.e. first value will be 1). - nodeStageHook func(counter int64) error - }{ - { - // This is already tested elsewhere, adding simple good case here to test the test framework. - name: "should call NodeUnstage after NodeStage success", - expectPodRunning: true, - expectedCalls: []csiCall{ - {expectedMethod: "NodeStageVolume", expectedError: codes.OK, deletePod: true}, - {expectedMethod: "NodeUnstageVolume", expectedError: codes.OK}, - }, - }, - { - // Kubelet should repeat NodeStage as long as the pod exists - name: "should retry NodeStage after NodeStage final error", - expectPodRunning: true, - expectedCalls: []csiCall{ - // This matches all 3 NodeStage calls with InvalidArgument error - {expectedMethod: "NodeStageVolume", expectedError: codes.InvalidArgument}, - {expectedMethod: "NodeStageVolume", expectedError: codes.OK, deletePod: true}, - {expectedMethod: "NodeUnstageVolume", expectedError: codes.OK}, - }, - // Fail first 3 NodeStage requests, 4th succeeds - nodeStageHook: func(counter int64) error { - if counter < 4 { - return status.Error(codes.InvalidArgument, "fake error") - } - return nil - }, - }, - { - // Kubelet should repeat NodeStage as long as the pod exists - name: "should retry NodeStage after NodeStage ephemeral error", - expectPodRunning: true, - expectedCalls: []csiCall{ - // This matches all 3 NodeStage calls with DeadlineExceeded error - {expectedMethod: "NodeStageVolume", expectedError: codes.DeadlineExceeded}, - {expectedMethod: "NodeStageVolume", expectedError: codes.OK, deletePod: true}, - {expectedMethod: "NodeUnstageVolume", expectedError: codes.OK}, - }, - // Fail first 3 NodeStage requests, 4th succeeds - nodeStageHook: func(counter int64) error { - if counter < 4 { - return status.Error(codes.DeadlineExceeded, "fake error") - } - return nil - }, - }, - { - // After NodeUnstage with ephemeral error, the driver may continue staging the volume. - // Kubelet should call NodeUnstage to make sure the volume is really unstaged after - // the pod is deleted. - name: "should call NodeUnstage after NodeStage ephemeral error", - expectPodRunning: false, - expectedCalls: []csiCall{ - // Delete the pod before NodeStage succeeds - it should get "uncertain" because of ephemeral error - // This matches all repeated NodeStage calls with DeadlineExceeded error (due to exp. backoff). - {expectedMethod: "NodeStageVolume", expectedError: codes.DeadlineExceeded, deletePod: true}, - {expectedMethod: "NodeUnstageVolume", expectedError: codes.OK}, - }, - nodeStageHook: func(counter int64) error { - return status.Error(codes.DeadlineExceeded, "fake error") - }, - }, - { - // After NodeUnstage with final error, kubelet can be sure the volume is not staged. - // The test checks that NodeUnstage is *not* called. - name: "should not call NodeUnstage after NodeStage final error", - expectPodRunning: false, - expectedCalls: []csiCall{ - // Delete the pod before NodeStage succeeds - it should get "globally unmounted" because of final error. - // This matches all repeated NodeStage calls with InvalidArgument error (due to exp. backoff). - {expectedMethod: "NodeStageVolume", expectedError: codes.InvalidArgument, deletePod: true}, - }, - // nodeStageScript: `INVALIDARGUMENT;`, - nodeStageHook: func(counter int64) error { - return status.Error(codes.InvalidArgument, "fake error") - }, - }, - } - for _, t := range tests { - test := t - ginkgo.It(test.name, func(ctx context.Context) { - var hooks *drivers.Hooks - if test.nodeStageHook != nil { - hooks = createPreHook("NodeStageVolume", test.nodeStageHook) - } - init(testParameters{ - disableAttach: true, - registerDriver: true, - hooks: hooks, - }) - ginkgo.DeferCleanup(cleanup) - - _, claim, pod := createPod(pvcReference) - if pod == nil { - return - } - // Wait for PVC to get bound to make sure the CSI driver is fully started. - err := e2epv.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, f.ClientSet, f.Namespace.Name, claim.Name, time.Second, framework.ClaimProvisionTimeout) - framework.ExpectNoError(err, "while waiting for PVC to get provisioned") - - ginkgo.By("Waiting for expected CSI calls") - // Watch for all calls up to deletePod = true - ctx, cancel := context.WithTimeout(ctx, csiPodRunningTimeout) - defer cancel() - for { - if ctx.Err() != nil { - framework.Failf("timed out waiting for the CSI call that indicates that the pod can be deleted: %v", test.expectedCalls) - } - time.Sleep(1 * time.Second) - _, index, err := compareCSICalls(trackedCalls, test.expectedCalls, m.driver.GetCalls) - framework.ExpectNoError(err, "while waiting for initial CSI calls") - if index == 0 { - // No CSI call received yet - continue - } - // Check the last *received* call wanted the pod to be deleted - if test.expectedCalls[index-1].deletePod { - break - } - } - - if test.expectPodRunning { - ginkgo.By("Waiting for pod to be running") - err := e2epod.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) - framework.ExpectNoError(err, "Failed to start pod: %v", err) - } - - ginkgo.By("Deleting the previously created pod") - err = e2epod.DeletePodWithWait(m.cs, pod) - framework.ExpectNoError(err, "while deleting") - - ginkgo.By("Waiting for all remaining expected CSI calls") - err = wait.Poll(time.Second, csiUnstageWaitTimeout, func() (done bool, err error) { - _, index, err := compareCSICalls(trackedCalls, test.expectedCalls, m.driver.GetCalls) - if err != nil { - return true, err - } - if index == 0 { - // No CSI call received yet - return false, nil - } - if len(test.expectedCalls) == index { - // all calls received - return true, nil - } - return false, nil - }) - framework.ExpectNoError(err, "while waiting for all CSI calls") - }) - } - }) - - ginkgo.Context("CSI NodeUnstage error cases [Slow]", func() { - trackedCalls := []string{ - "NodeStageVolume", - "NodeUnstageVolume", - } - - // Each test starts two pods in sequence. - // The first pod always runs successfully, but NodeUnstage hook can set various error conditions. - // The test then checks how NodeStage of the second pod is called. - tests := []struct { - name string - expectedCalls []csiCall - - // Called for each NodeStageVolume calls, with counter incremented atomically before - // the invocation (i.e. first value will be 1) and index of deleted pod (the first pod - // has index 1) - nodeUnstageHook func(counter, pod int64) error - }{ - { - // This is already tested elsewhere, adding simple good case here to test the test framework. - name: "should call NodeStage after NodeUnstage success", - expectedCalls: []csiCall{ - {expectedMethod: "NodeStageVolume", expectedError: codes.OK}, - {expectedMethod: "NodeUnstageVolume", expectedError: codes.OK}, - {expectedMethod: "NodeStageVolume", expectedError: codes.OK}, - {expectedMethod: "NodeUnstageVolume", expectedError: codes.OK}, - }, - }, - { - name: "two pods: should call NodeStage after previous NodeUnstage final error", - expectedCalls: []csiCall{ - {expectedMethod: "NodeStageVolume", expectedError: codes.OK}, - {expectedMethod: "NodeUnstageVolume", expectedError: codes.InvalidArgument}, - {expectedMethod: "NodeStageVolume", expectedError: codes.OK}, - {expectedMethod: "NodeUnstageVolume", expectedError: codes.OK}, - }, - nodeUnstageHook: func(counter, pod int64) error { - if pod == 1 { - return status.Error(codes.InvalidArgument, "fake final error") - } - return nil - }, - }, - { - name: "two pods: should call NodeStage after previous NodeUnstage transient error", - expectedCalls: []csiCall{ - {expectedMethod: "NodeStageVolume", expectedError: codes.OK}, - {expectedMethod: "NodeUnstageVolume", expectedError: codes.DeadlineExceeded}, - {expectedMethod: "NodeStageVolume", expectedError: codes.OK}, - {expectedMethod: "NodeUnstageVolume", expectedError: codes.OK}, - }, - nodeUnstageHook: func(counter, pod int64) error { - if pod == 1 { - return status.Error(codes.DeadlineExceeded, "fake transient error") - } - return nil - }, - }, - } - for _, t := range tests { - test := t - ginkgo.It(test.name, func(ctx context.Context) { - // Index of the last deleted pod. NodeUnstage calls are then related to this pod. - var deletedPodNumber int64 = 1 - var hooks *drivers.Hooks - if test.nodeUnstageHook != nil { - hooks = createPreHook("NodeUnstageVolume", func(counter int64) error { - pod := atomic.LoadInt64(&deletedPodNumber) - return test.nodeUnstageHook(counter, pod) - }) - } - init(testParameters{ - disableAttach: true, - registerDriver: true, - hooks: hooks, - }) - ginkgo.DeferCleanup(cleanup) - - _, claim, pod := createPod(pvcReference) - if pod == nil { - return - } - // Wait for PVC to get bound to make sure the CSI driver is fully started. - err := e2epv.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, f.ClientSet, f.Namespace.Name, claim.Name, time.Second, framework.ClaimProvisionTimeout) - framework.ExpectNoError(err, "while waiting for PVC to get provisioned") - err = e2epod.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) - framework.ExpectNoError(err, "while waiting for the first pod to start") - err = e2epod.DeletePodWithWait(m.cs, pod) - framework.ExpectNoError(err, "while deleting the first pod") - - // Create the second pod - pod, err = createPodWithPVC(claim) - framework.ExpectNoError(err, "while creating the second pod") - err = e2epod.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) - framework.ExpectNoError(err, "while waiting for the second pod to start") - // The second pod is running and kubelet can't call NodeUnstage of the first one. - // Therefore incrementing the pod counter is safe here. - atomic.AddInt64(&deletedPodNumber, 1) - err = e2epod.DeletePodWithWait(m.cs, pod) - framework.ExpectNoError(err, "while deleting the second pod") - - ginkgo.By("Waiting for all remaining expected CSI calls") - err = wait.Poll(time.Second, csiUnstageWaitTimeout, func() (done bool, err error) { - _, index, err := compareCSICalls(trackedCalls, test.expectedCalls, m.driver.GetCalls) - if err != nil { - return true, err - } - if index == 0 { - // No CSI call received yet - return false, nil - } - if len(test.expectedCalls) == index { - // all calls received - return true, nil - } - return false, nil - }) - framework.ExpectNoError(err, "while waiting for all CSI calls") - }) - } - }) - - ginkgo.Context("storage capacity", func() { - tests := []struct { - name string - resourceExhausted bool - lateBinding bool - topology bool - }{ - { - name: "unlimited", - }, - { - name: "exhausted, immediate binding", - resourceExhausted: true, - }, - { - name: "exhausted, late binding, no topology", - resourceExhausted: true, - lateBinding: true, - }, - { - name: "exhausted, late binding, with topology", - resourceExhausted: true, - lateBinding: true, - topology: true, - }, - } - - createVolume := "CreateVolume" - deleteVolume := "DeleteVolume" - // publishVolume := "NodePublishVolume" - // unpublishVolume := "NodeUnpublishVolume" - // stageVolume := "NodeStageVolume" - // unstageVolume := "NodeUnstageVolume" - - // These calls are assumed to occur in this order for - // each test run. NodeStageVolume and - // NodePublishVolume should also be deterministic and - // only get called once, but sometimes kubelet calls - // both multiple times, which breaks this test - // (https://github.com/kubernetes/kubernetes/issues/90250). - // Therefore they are temporarily commented out until - // that issue is resolved. - // - // NodeUnpublishVolume and NodeUnstageVolume are racing - // with DeleteVolume, so we cannot assume a deterministic - // order and have to ignore them - // (https://github.com/kubernetes/kubernetes/issues/94108). - deterministicCalls := []string{ - createVolume, - // stageVolume, - // publishVolume, - // unpublishVolume, - // unstageVolume, - deleteVolume, - } - - for _, t := range tests { - test := t - ginkgo.It(test.name, func(ctx context.Context) { - var err error - params := testParameters{ - lateBinding: test.lateBinding, - enableTopology: test.topology, - - // Not strictly necessary, but runs a bit faster this way - // and for a while there also was a problem with a two minuted delay - // due to a bug (https://github.com/kubernetes-csi/csi-test/pull/250). - disableAttach: true, - registerDriver: true, - } - - if test.resourceExhausted { - params.hooks = createPreHook("CreateVolume", func(counter int64) error { - if counter%2 != 0 { - return status.Error(codes.ResourceExhausted, "fake error") - } - return nil - }) - } - - ctx, cancel := context.WithTimeout(context.Background(), csiPodRunningTimeout) - defer cancel() - init(params) - ginkgo.DeferCleanup(cleanup) - - // In contrast to the raw watch, RetryWatcher is expected to deliver all events even - // when the underlying raw watch gets closed prematurely - // (https://github.com/kubernetes/kubernetes/pull/93777#discussion_r467932080). - // This is important because below the test is going to make assertions about the - // PVC state changes. - initResource, err := f.ClientSet.CoreV1().PersistentVolumeClaims(f.Namespace.Name).List(ctx, metav1.ListOptions{}) - framework.ExpectNoError(err, "Failed to fetch initial PVC resource") - listWatcher := &cachetools.ListWatch{ - WatchFunc: func(listOptions metav1.ListOptions) (watch.Interface, error) { - return f.ClientSet.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Watch(ctx, listOptions) - }, - } - pvcWatch, err := watchtools.NewRetryWatcher(initResource.GetResourceVersion(), listWatcher) - framework.ExpectNoError(err, "create PVC watch") - defer pvcWatch.Stop() - - sc, claim, pod := createPod(pvcReference) - gomega.Expect(pod).NotTo(gomega.BeNil(), "while creating pod") - bindingMode := storagev1.VolumeBindingImmediate - if test.lateBinding { - bindingMode = storagev1.VolumeBindingWaitForFirstConsumer - } - framework.ExpectEqual(*sc.VolumeBindingMode, bindingMode, "volume binding mode") - - err = e2epod.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) - framework.ExpectNoError(err, "failed to start pod") - err = e2epod.DeletePodWithWait(m.cs, pod) - framework.ExpectNoError(err, "failed to delete pod") - err = m.cs.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(ctx, claim.Name, metav1.DeleteOptions{}) - framework.ExpectNoError(err, "failed to delete claim") - - normal := []csiCall{} - for _, method := range deterministicCalls { - normal = append(normal, csiCall{expectedMethod: method}) - } - expected := normal - // When simulating limited capacity, - // we expect exactly two CreateVolume - // calls because the first one should - // have failed. - if test.resourceExhausted { - expected = []csiCall{ - {expectedMethod: createVolume, expectedError: codes.ResourceExhausted}, - } - expected = append(expected, normal...) - } - - var calls []drivers.MockCSICall - err = wait.PollImmediateUntil(time.Second, func() (done bool, err error) { - c, index, err := compareCSICalls(deterministicCalls, expected, m.driver.GetCalls) - if err != nil { - return true, fmt.Errorf("error waiting for expected CSI calls: %s", err) - } - calls = c - if index == 0 { - // No CSI call received yet - return false, nil - } - if len(expected) == index { - // all calls received - return true, nil - } - return false, nil - }, ctx.Done()) - framework.ExpectNoError(err, "while waiting for all CSI calls") - - // The capacity error is dealt with in two different ways. - // - // For delayed binding, the external-provisioner should unset the node annotation - // to give the scheduler the opportunity to reschedule the pod onto a different - // node. - // - // For immediate binding, the external-scheduler must keep retrying. - // - // Unfortunately, the call log is the same in both cases. We have to collect - // additional evidence that rescheduling really happened. What we have observed - // above is how the PVC changed over time. Now we can analyze that. - ginkgo.By("Checking PVC events") - nodeAnnotationSet := false - nodeAnnotationReset := false - watchFailed := false - loop: - for { - select { - case event, ok := <-pvcWatch.ResultChan(): - if !ok { - watchFailed = true - break loop - } - - framework.Logf("PVC event %s: %#v", event.Type, event.Object) - switch event.Type { - case watch.Modified: - pvc, ok := event.Object.(*v1.PersistentVolumeClaim) - if !ok { - framework.Failf("PVC watch sent %#v instead of a PVC", event.Object) - } - _, set := pvc.Annotations["volume.kubernetes.io/selected-node"] - if set { - nodeAnnotationSet = true - } else if nodeAnnotationSet { - nodeAnnotationReset = true - } - case watch.Deleted: - break loop - case watch.Error: - watchFailed = true - break - } - case <-ctx.Done(): - framework.Failf("Timeout while waiting to observe PVC list") - } - } - - // More tests when capacity is limited. - if test.resourceExhausted { - for _, call := range calls { - if call.Method == createVolume { - gomega.Expect(call.Error).To(gomega.ContainSubstring("code = ResourceExhausted"), "first CreateVolume error in\n%s", calls) - break - } - } - - switch { - case watchFailed: - // If the watch failed or stopped prematurely (which can happen at any time), then we cannot - // verify whether the annotation was set as expected. This is still considered a successful - // test. - framework.Logf("PVC watch delivered incomplete data, cannot check annotation") - case test.lateBinding: - gomega.Expect(nodeAnnotationSet).To(gomega.BeTrue(), "selected-node should have been set") - // Whether it gets reset depends on whether we have topology enabled. Without - // it, rescheduling is unnecessary. - if test.topology { - gomega.Expect(nodeAnnotationReset).To(gomega.BeTrue(), "selected-node should have been set") - } else { - gomega.Expect(nodeAnnotationReset).To(gomega.BeFalse(), "selected-node should not have been reset") - } - default: - gomega.Expect(nodeAnnotationSet).To(gomega.BeFalse(), "selected-node should not have been set") - gomega.Expect(nodeAnnotationReset).To(gomega.BeFalse(), "selected-node should not have been reset") - } - } - }) - } - }) - - // These tests *only* work on a cluster which has the CSIStorageCapacity feature enabled. - ginkgo.Context("CSIStorageCapacity", func() { - var ( - err error - yes = true - no = false - ) - // Tests that expect a failure are slow because we have to wait for a while - // to be sure that the volume isn't getting created. - tests := []struct { - name string - storageCapacity *bool - capacities []string - expectFailure bool - }{ - { - name: "CSIStorageCapacity unused", - }, - { - name: "CSIStorageCapacity disabled", - storageCapacity: &no, - }, - { - name: "CSIStorageCapacity used, no capacity", - storageCapacity: &yes, - expectFailure: true, - }, - { - name: "CSIStorageCapacity used, insufficient capacity", - storageCapacity: &yes, - expectFailure: true, - capacities: []string{"1Mi"}, - }, - { - name: "CSIStorageCapacity used, have capacity", - storageCapacity: &yes, - capacities: []string{"100Gi"}, - }, - // We could add more test cases here for - // various situations, but covering those via - // the scheduler binder unit tests is faster. - } - for _, t := range tests { - test := t - ginkgo.It(t.name, func(ctx context.Context) { - scName := "mock-csi-storage-capacity-" + f.UniqueName - init(testParameters{ - registerDriver: true, - scName: scName, - storageCapacity: test.storageCapacity, - lateBinding: true, - }) - ginkgo.DeferCleanup(cleanup) - - // The storage class uses a random name, therefore we have to create it first - // before adding CSIStorageCapacity objects for it. - for _, capacityStr := range test.capacities { - capacityQuantity := resource.MustParse(capacityStr) - capacity := &storagev1.CSIStorageCapacity{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "fake-capacity-", - }, - // Empty topology, usable by any node. - StorageClassName: scName, - NodeTopology: &metav1.LabelSelector{}, - Capacity: &capacityQuantity, - } - createdCapacity, err := f.ClientSet.StorageV1().CSIStorageCapacities(f.Namespace.Name).Create(context.Background(), capacity, metav1.CreateOptions{}) - framework.ExpectNoError(err, "create CSIStorageCapacity %+v", *capacity) - ginkgo.DeferCleanup(framework.IgnoreNotFound(f.ClientSet.StorageV1().CSIStorageCapacities(f.Namespace.Name).Delete), createdCapacity.Name, metav1.DeleteOptions{}) - } - - // kube-scheduler may need some time before it gets the CSIDriver and CSIStorageCapacity objects. - // Without them, scheduling doesn't run as expected by the test. - syncDelay := 5 * time.Second - time.Sleep(syncDelay) - - sc, _, pod := createPod(pvcReference) // late binding as specified above - framework.ExpectEqual(sc.Name, scName, "pre-selected storage class name not used") - - waitCtx, cancel := context.WithTimeout(context.Background(), f.Timeouts.PodStart) - defer cancel() - condition := anyOf( - podRunning(waitCtx, f.ClientSet, pod.Name, pod.Namespace), - // We only just created the CSIStorageCapacity objects, therefore - // we have to ignore all older events, plus the syncDelay as our - // safety margin. - podHasStorage(waitCtx, f.ClientSet, pod.Name, pod.Namespace, time.Now().Add(syncDelay)), - ) - err = wait.PollImmediateUntil(poll, condition, waitCtx.Done()) - if test.expectFailure { - switch { - case errors.Is(err, context.DeadlineExceeded), - errors.Is(err, wait.ErrWaitTimeout), - errors.Is(err, errNotEnoughSpace): - // Okay, we expected that. - case err == nil: - framework.Fail("pod unexpectedly started to run") - default: - framework.Failf("unexpected error while waiting for pod: %v", err) - } - } else { - framework.ExpectNoError(err, "failed to start pod") - } - - ginkgo.By("Deleting the previously created pod") - err = e2epod.DeletePodWithWait(m.cs, pod) - framework.ExpectNoError(err, "while deleting") - }) - } - }) - - ginkgo.Context("CSI Volume Snapshots [Feature:VolumeSnapshotDataSource]", func() { - tests := []struct { - name string - createSnapshotHook func(counter int64) error - }{ - { - name: "volumesnapshotcontent and pvc in Bound state with deletion timestamp set should not get deleted while snapshot finalizer exists", - createSnapshotHook: func(counter int64) error { - if counter < 8 { - return status.Error(codes.DeadlineExceeded, "fake error") - } - return nil - }, - }, - } - for _, test := range tests { - test := test - ginkgo.It(test.name, func(ctx context.Context) { - var hooks *drivers.Hooks - if test.createSnapshotHook != nil { - hooks = createPreHook("CreateSnapshot", test.createSnapshotHook) - } - init(testParameters{ - disableAttach: true, - registerDriver: true, - enableSnapshot: true, - hooks: hooks, - }) - sDriver, ok := m.driver.(storageframework.SnapshottableTestDriver) - if !ok { - e2eskipper.Skipf("mock driver %s does not support snapshots -- skipping", m.driver.GetDriverInfo().Name) - - } - ctx, cancel := context.WithTimeout(ctx, csiPodRunningTimeout) - defer cancel() - ginkgo.DeferCleanup(cleanup) - - sc := m.driver.GetDynamicProvisionStorageClass(m.config, "") - ginkgo.By("Creating storage class") - class, err := m.cs.StorageV1().StorageClasses().Create(context.TODO(), sc, metav1.CreateOptions{}) - framework.ExpectNoError(err, "Failed to create class: %v", err) - m.sc[class.Name] = class - claim := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{ - // Use static name so that the volumesnapshot can be created before the pvc. - Name: "snapshot-test-pvc", - StorageClassName: &(class.Name), - }, f.Namespace.Name) - - ginkgo.By("Creating snapshot") - // TODO: Test VolumeSnapshots with Retain policy - parameters := map[string]string{} - snapshotClass, snapshot := storageframework.CreateSnapshot(sDriver, m.config, storageframework.DynamicSnapshotDelete, claim.Name, claim.Namespace, f.Timeouts, parameters) - framework.ExpectNoError(err, "failed to create snapshot") - m.vsc[snapshotClass.GetName()] = snapshotClass - volumeSnapshotName := snapshot.GetName() - - ginkgo.By(fmt.Sprintf("Creating PVC %s/%s", claim.Namespace, claim.Name)) - claim, err = m.cs.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Create(context.TODO(), claim, metav1.CreateOptions{}) - framework.ExpectNoError(err, "Failed to create claim: %v", err) - - ginkgo.By(fmt.Sprintf("Wait for finalizer to be added to claim %s/%s", claim.Namespace, claim.Name)) - err = e2epv.WaitForPVCFinalizer(ctx, m.cs, claim.Name, claim.Namespace, pvcAsSourceProtectionFinalizer, 1*time.Millisecond, 1*time.Minute) - framework.ExpectNoError(err) - - ginkgo.By("Wait for PVC to be Bound") - _, err = e2epv.WaitForPVClaimBoundPhase(m.cs, []*v1.PersistentVolumeClaim{claim}, 1*time.Minute) - framework.ExpectNoError(err, "Failed to create claim: %v", err) - - ginkgo.By(fmt.Sprintf("Delete PVC %s", claim.Name)) - err = e2epv.DeletePersistentVolumeClaim(m.cs, claim.Name, claim.Namespace) - framework.ExpectNoError(err, "failed to delete pvc") - - ginkgo.By("Get PVC from API server and verify deletion timestamp is set") - claim, err = m.cs.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Get(context.TODO(), claim.Name, metav1.GetOptions{}) - if err != nil { - if !apierrors.IsNotFound(err) { - framework.ExpectNoError(err, "Failed to get claim: %v", err) - } - framework.Logf("PVC not found. Continuing to test VolumeSnapshotContent finalizer") - } else if claim.DeletionTimestamp == nil { - framework.Failf("Expected deletion timestamp to be set on PVC: %v", claim) - } - - ginkgo.By(fmt.Sprintf("Get VolumeSnapshotContent bound to VolumeSnapshot %s", snapshot.GetName())) - snapshotContent := utils.GetSnapshotContentFromSnapshot(m.config.Framework.DynamicClient, snapshot, f.Timeouts.SnapshotCreate) - volumeSnapshotContentName := snapshotContent.GetName() - - ginkgo.By(fmt.Sprintf("Verify VolumeSnapshotContent %s contains finalizer %s", snapshot.GetName(), volumeSnapshotContentFinalizer)) - err = utils.WaitForGVRFinalizer(ctx, m.config.Framework.DynamicClient, utils.SnapshotContentGVR, volumeSnapshotContentName, "", volumeSnapshotContentFinalizer, 1*time.Millisecond, 1*time.Minute) - framework.ExpectNoError(err) - - ginkgo.By(fmt.Sprintf("Delete VolumeSnapshotContent %s", snapshotContent.GetName())) - err = m.config.Framework.DynamicClient.Resource(utils.SnapshotContentGVR).Delete(ctx, snapshotContent.GetName(), metav1.DeleteOptions{}) - framework.ExpectNoError(err, "Failed to delete snapshotcontent: %v", err) - - ginkgo.By("Get VolumeSnapshotContent from API server and verify deletion timestamp is set") - snapshotContent, err = m.config.Framework.DynamicClient.Resource(utils.SnapshotContentGVR).Get(context.TODO(), snapshotContent.GetName(), metav1.GetOptions{}) - framework.ExpectNoError(err) - - if snapshotContent.GetDeletionTimestamp() == nil { - framework.Failf("Expected deletion timestamp to be set on snapshotcontent") - } - - // If the claim is non existent, the Get() call on the API server returns - // an non-nil claim object with all fields unset. - // Refer https://github.com/kubernetes/kubernetes/pull/99167#issuecomment-781670012 - if claim != nil && claim.Spec.VolumeName != "" { - ginkgo.By(fmt.Sprintf("Wait for PV %s to be deleted", claim.Spec.VolumeName)) - err = e2epv.WaitForPersistentVolumeDeleted(m.cs, claim.Spec.VolumeName, framework.Poll, 3*time.Minute) - framework.ExpectNoError(err, fmt.Sprintf("failed to delete PV %s", claim.Spec.VolumeName)) - } - - ginkgo.By(fmt.Sprintf("Verify VolumeSnapshot %s contains finalizer %s", snapshot.GetName(), volumeSnapshotBoundFinalizer)) - err = utils.WaitForGVRFinalizer(ctx, m.config.Framework.DynamicClient, utils.SnapshotGVR, volumeSnapshotName, f.Namespace.Name, volumeSnapshotBoundFinalizer, 1*time.Millisecond, 1*time.Minute) - framework.ExpectNoError(err) - - ginkgo.By("Delete VolumeSnapshot") - err = utils.DeleteAndWaitSnapshot(m.config.Framework.DynamicClient, f.Namespace.Name, volumeSnapshotName, framework.Poll, framework.SnapshotDeleteTimeout) - framework.ExpectNoError(err, fmt.Sprintf("failed to delete VolumeSnapshot %s", volumeSnapshotName)) - - ginkgo.By(fmt.Sprintf("Wait for VolumeSnapshotContent %s to be deleted", volumeSnapshotContentName)) - err = utils.WaitForGVRDeletion(m.config.Framework.DynamicClient, utils.SnapshotContentGVR, volumeSnapshotContentName, framework.Poll, framework.SnapshotDeleteTimeout) - framework.ExpectNoError(err, fmt.Sprintf("failed to delete VolumeSnapshotContent %s", volumeSnapshotContentName)) - }) - } - }) - - ginkgo.Context("CSIServiceAccountToken", func() { - var ( - err error - ) - tests := []struct { - desc string - deployCSIDriverObject bool - tokenRequests []storagev1.TokenRequest - }{ - { - desc: "token should not be plumbed down when csiServiceAccountTokenEnabled=false", - deployCSIDriverObject: true, - tokenRequests: nil, - }, - { - desc: "token should not be plumbed down when CSIDriver is not deployed", - deployCSIDriverObject: false, - tokenRequests: []storagev1.TokenRequest{{}}, - }, - { - desc: "token should be plumbed down when csiServiceAccountTokenEnabled=true", - deployCSIDriverObject: true, - tokenRequests: []storagev1.TokenRequest{{ExpirationSeconds: utilptr.Int64Ptr(60 * 10)}}, - }, - } - for _, test := range tests { - test := test - csiServiceAccountTokenEnabled := test.tokenRequests != nil - ginkgo.It(test.desc, func(ctx context.Context) { - init(testParameters{ - registerDriver: test.deployCSIDriverObject, - tokenRequests: test.tokenRequests, - requiresRepublish: &csiServiceAccountTokenEnabled, - }) - ginkgo.DeferCleanup(cleanup) - - _, _, pod := createPod(pvcReference) - if pod == nil { - return - } - err = e2epod.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) - framework.ExpectNoError(err, "Failed to start pod: %v", err) - - // sleep to make sure RequiresRepublish triggers more than 1 NodePublishVolume - numNodePublishVolume := 1 - if test.deployCSIDriverObject && csiServiceAccountTokenEnabled { - time.Sleep(5 * time.Second) - numNodePublishVolume = 2 - } - - ginkgo.By("Deleting the previously created pod") - err = e2epod.DeletePodWithWait(m.cs, pod) - framework.ExpectNoError(err, "while deleting") - - ginkgo.By("Checking CSI driver logs") - err = checkPodLogs(m.driver.GetCalls, pod, false, false, false, test.deployCSIDriverObject && csiServiceAccountTokenEnabled, numNodePublishVolume) - framework.ExpectNoError(err) - }) - } - }) - // These tests *only* work on a cluster which has the CSIVolumeFSGroupPolicy feature enabled. - ginkgo.Context("CSI FSGroupPolicy [LinuxOnly]", func() { - tests := []struct { - name string - fsGroupPolicy storagev1.FSGroupPolicy - modified bool - }{ - { - name: "should modify fsGroup if fsGroupPolicy=default", - fsGroupPolicy: storagev1.ReadWriteOnceWithFSTypeFSGroupPolicy, - modified: true, - }, - { - name: "should modify fsGroup if fsGroupPolicy=File", - fsGroupPolicy: storagev1.FileFSGroupPolicy, - modified: true, - }, - { - name: "should not modify fsGroup if fsGroupPolicy=None", - fsGroupPolicy: storagev1.NoneFSGroupPolicy, - modified: false, - }, - } - for _, t := range tests { - test := t - ginkgo.It(test.name, func(ctx context.Context) { - if framework.NodeOSDistroIs("windows") { - e2eskipper.Skipf("FSGroupPolicy is only applied on linux nodes -- skipping") - } - init(testParameters{ - disableAttach: true, - registerDriver: true, - fsGroupPolicy: &test.fsGroupPolicy, - }) - ginkgo.DeferCleanup(cleanup) - - // kube-scheduler may need some time before it gets the CSIDriver object. - // Without them, scheduling doesn't run as expected by the test. - syncDelay := 5 * time.Second - time.Sleep(syncDelay) - - fsGroupVal := int64(rand.Int63n(20000) + 1024) - fsGroup := &fsGroupVal - - _, _, pod := createPodWithFSGroup(fsGroup) /* persistent volume */ - - mountPath := pod.Spec.Containers[0].VolumeMounts[0].MountPath - dirName := mountPath + "/" + f.UniqueName - fileName := dirName + "/" + f.UniqueName - - err := e2epod.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) - framework.ExpectNoError(err, "failed to start pod") - - // Create the subdirectory to ensure that fsGroup propagates - createDirectory := fmt.Sprintf("mkdir %s", dirName) - _, _, err = e2evolume.PodExec(f, pod, createDirectory) - framework.ExpectNoError(err, "failed: creating the directory: %s", err) - - // Inject the contents onto the mount - createFile := fmt.Sprintf("echo '%s' > '%s'; sync", "filecontents", fileName) - _, _, err = e2evolume.PodExec(f, pod, createFile) - framework.ExpectNoError(err, "failed: writing the contents: %s", err) - - // Delete the created file. This step is mandatory, as the mock driver - // won't clean up the contents automatically. - defer func() { - delete := fmt.Sprintf("rm -fr %s", dirName) - _, _, err = e2evolume.PodExec(f, pod, delete) - framework.ExpectNoError(err, "failed: deleting the directory: %s", err) - }() - - // Ensure that the fsGroup matches what we expect - if test.modified { - utils.VerifyFSGroupInPod(f, fileName, strconv.FormatInt(*fsGroup, 10), pod) - } else { - utils.VerifyFSGroupInPod(f, fileName, "root", pod) - } - - // The created resources will be removed by the cleanup() function, - // so need to delete anything here. - }) - } - }) - - ginkgo.Context("Delegate FSGroup to CSI driver [LinuxOnly]", func() { - tests := []struct { - name string - enableVolumeMountGroup bool - }{ - { - name: "should pass FSGroup to CSI driver if it is set in pod and driver supports VOLUME_MOUNT_GROUP", - enableVolumeMountGroup: true, - }, - { - name: "should not pass FSGroup to CSI driver if it is set in pod and driver supports VOLUME_MOUNT_GROUP", - enableVolumeMountGroup: false, - }, - } - for _, t := range tests { - t := t - ginkgo.It(t.name, func(ctx context.Context) { - var nodeStageFsGroup, nodePublishFsGroup string - if framework.NodeOSDistroIs("windows") { - e2eskipper.Skipf("FSGroupPolicy is only applied on linux nodes -- skipping") - } - init(testParameters{ - disableAttach: true, - registerDriver: true, - enableVolumeMountGroup: t.enableVolumeMountGroup, - hooks: createFSGroupRequestPreHook(&nodeStageFsGroup, &nodePublishFsGroup), - }) - ginkgo.DeferCleanup(cleanup) - - fsGroupVal := int64(rand.Int63n(20000) + 1024) - fsGroup := &fsGroupVal - fsGroupStr := strconv.FormatInt(fsGroupVal, 10 /* base */) - - _, _, pod := createPodWithFSGroup(fsGroup) /* persistent volume */ - - err := e2epod.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) - framework.ExpectNoError(err, "failed to start pod") - - if t.enableVolumeMountGroup { - framework.ExpectEqual(nodeStageFsGroup, fsGroupStr, "Expect NodeStageVolumeRequest.VolumeCapability.MountVolume.VolumeMountGroup to equal %q; got: %q", fsGroupStr, nodeStageFsGroup) - framework.ExpectEqual(nodePublishFsGroup, fsGroupStr, "Expect NodePublishVolumeRequest.VolumeCapability.MountVolume.VolumeMountGroup to equal %q; got: %q", fsGroupStr, nodePublishFsGroup) - } else { - framework.ExpectEmpty(nodeStageFsGroup, "Expect NodeStageVolumeRequest.VolumeCapability.MountVolume.VolumeMountGroup to be empty; got: %q", nodeStageFsGroup) - framework.ExpectEmpty(nodePublishFsGroup, "Expect NodePublishVolumeRequest.VolumeCapability.MountVolume.VolumeMountGroup to empty; got: %q", nodePublishFsGroup) - } - }) - } - }) - - ginkgo.Context("CSI Volume Snapshots secrets [Feature:VolumeSnapshotDataSource]", func() { - - var ( - // CSISnapshotterSecretName is the name of the secret to be created - CSISnapshotterSecretName string = "snapshot-secret" - - // CSISnapshotterSecretNameAnnotation is the annotation key for the CSI snapshotter secret name in VolumeSnapshotClass.parameters - CSISnapshotterSecretNameAnnotation string = "csi.storage.k8s.io/snapshotter-secret-name" - - // CSISnapshotterSecretNamespaceAnnotation is the annotation key for the CSI snapshotter secret namespace in VolumeSnapshotClass.parameters - CSISnapshotterSecretNamespaceAnnotation string = "csi.storage.k8s.io/snapshotter-secret-namespace" - - // anotations holds the annotations object - annotations interface{} - ) - - tests := []struct { - name string - createSnapshotHook func(counter int64) error - }{ - { - // volume snapshot should be created using secrets successfully even if there is a failure in the first few attempts, - name: "volume snapshot create/delete with secrets", - // Fail the first 8 calls to create snapshot and succeed the 9th call. - createSnapshotHook: func(counter int64) error { - if counter < 8 { - return status.Error(codes.DeadlineExceeded, "fake error") - } - return nil - }, - }, - } - for _, test := range tests { - test := test - ginkgo.It(test.name, func(ctx context.Context) { - hooks := createPreHook("CreateSnapshot", test.createSnapshotHook) - init(testParameters{ - disableAttach: true, - registerDriver: true, - enableSnapshot: true, - hooks: hooks, - }) - - sDriver, ok := m.driver.(storageframework.SnapshottableTestDriver) - if !ok { - e2eskipper.Skipf("mock driver does not support snapshots -- skipping") - } - ginkgo.DeferCleanup(cleanup) - - var sc *storagev1.StorageClass - if dDriver, ok := m.driver.(storageframework.DynamicPVTestDriver); ok { - sc = dDriver.GetDynamicProvisionStorageClass(m.config, "") - } - ginkgo.By("Creating storage class") - class, err := m.cs.StorageV1().StorageClasses().Create(context.TODO(), sc, metav1.CreateOptions{}) - framework.ExpectNoError(err, "Failed to create storage class: %v", err) - m.sc[class.Name] = class - pvc := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{ - Name: "snapshot-test-pvc", - StorageClassName: &(class.Name), - }, f.Namespace.Name) - - ginkgo.By(fmt.Sprintf("Creating PVC %s/%s", pvc.Namespace, pvc.Name)) - pvc, err = m.cs.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Create(context.TODO(), pvc, metav1.CreateOptions{}) - framework.ExpectNoError(err, "Failed to create claim: %v", err) - - ginkgo.By("Wait for PVC to be Bound") - _, err = e2epv.WaitForPVClaimBoundPhase(m.cs, []*v1.PersistentVolumeClaim{pvc}, 1*time.Minute) - framework.ExpectNoError(err, "Failed to create claim: %v", err) - - m.pvcs = append(m.pvcs, pvc) - - ginkgo.By("Creating Secret") - secret := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: f.Namespace.Name, - Name: CSISnapshotterSecretName, - }, - Data: map[string][]byte{ - "secret-data": []byte("secret-value-1"), - }, - } - - if secret, err := m.cs.CoreV1().Secrets(f.Namespace.Name).Create(context.TODO(), secret, metav1.CreateOptions{}); err != nil { - framework.Failf("unable to create test secret %s: %v", secret.Name, err) - } - - ginkgo.By("Creating snapshot with secrets") - parameters := map[string]string{ - CSISnapshotterSecretNameAnnotation: CSISnapshotterSecretName, - CSISnapshotterSecretNamespaceAnnotation: f.Namespace.Name, - } - - _, snapshot := storageframework.CreateSnapshot(sDriver, m.config, storageframework.DynamicSnapshotDelete, pvc.Name, pvc.Namespace, f.Timeouts, parameters) - framework.ExpectNoError(err, "failed to create snapshot") - snapshotcontent := utils.GetSnapshotContentFromSnapshot(m.config.Framework.DynamicClient, snapshot, f.Timeouts.SnapshotCreate) - if annotations, ok = snapshotcontent.Object["metadata"].(map[string]interface{})["annotations"]; !ok { - framework.Failf("Unable to get volume snapshot content annotations") - } - - // checks if delete snapshot secrets annotation is applied to the VolumeSnapshotContent. - checkDeleteSnapshotSecrets(m.cs, annotations) - - // delete the snapshot and check if the snapshot is deleted. - deleteSnapshot(m.cs, m.config, snapshot) - }) - } - }) - - ginkgo.Context("CSI Snapshot Controller metrics [Feature:VolumeSnapshotDataSource]", func() { - tests := []struct { - name string - pattern storageframework.TestPattern - }{ - { - name: "snapshot controller should emit dynamic CreateSnapshot, CreateSnapshotAndReady, and DeleteSnapshot metrics", - pattern: storageframework.DynamicSnapshotDelete, - }, - { - name: "snapshot controller should emit pre-provisioned CreateSnapshot, CreateSnapshotAndReady, and DeleteSnapshot metrics", - pattern: storageframework.PreprovisionedSnapshotDelete, - }, - } - for _, test := range tests { - test := test - ginkgo.It(test.name, func(ctx context.Context) { - init(testParameters{ - disableAttach: true, - registerDriver: true, - enableSnapshot: true, - }) - - sDriver, ok := m.driver.(storageframework.SnapshottableTestDriver) - if !ok { - e2eskipper.Skipf("mock driver does not support snapshots -- skipping") - } - ginkgo.DeferCleanup(cleanup) - - metricsGrabber, err := e2emetrics.NewMetricsGrabber(m.config.Framework.ClientSet, nil, f.ClientConfig(), false, false, false, false, false, true) - if err != nil { - framework.Failf("Error creating metrics grabber : %v", err) - } - - // Grab initial metrics - if this fails, snapshot controller metrics are not setup. Skip in this case. - _, err = metricsGrabber.GrabFromSnapshotController(framework.TestContext.SnapshotControllerPodName, framework.TestContext.SnapshotControllerHTTPPort) - if err != nil { - e2eskipper.Skipf("Snapshot controller metrics not found -- skipping") - } - - ginkgo.By("getting all initial metric values") - metricsTestConfig := newSnapshotMetricsTestConfig("snapshot_controller_operation_total_seconds_count", - "count", - m.config.GetUniqueDriverName(), - "CreateSnapshot", - "success", - "", - test.pattern) - createSnapshotMetrics := newSnapshotControllerMetrics(metricsTestConfig, metricsGrabber) - originalCreateSnapshotCount, _ := createSnapshotMetrics.getSnapshotControllerMetricValue() - metricsTestConfig.operationName = "CreateSnapshotAndReady" - createSnapshotAndReadyMetrics := newSnapshotControllerMetrics(metricsTestConfig, metricsGrabber) - originalCreateSnapshotAndReadyCount, _ := createSnapshotAndReadyMetrics.getSnapshotControllerMetricValue() - - metricsTestConfig.operationName = "DeleteSnapshot" - deleteSnapshotMetrics := newSnapshotControllerMetrics(metricsTestConfig, metricsGrabber) - originalDeleteSnapshotCount, _ := deleteSnapshotMetrics.getSnapshotControllerMetricValue() - - ginkgo.By("Creating storage class") - var sc *storagev1.StorageClass - if dDriver, ok := m.driver.(storageframework.DynamicPVTestDriver); ok { - sc = dDriver.GetDynamicProvisionStorageClass(m.config, "") - } - class, err := m.cs.StorageV1().StorageClasses().Create(context.TODO(), sc, metav1.CreateOptions{}) - framework.ExpectNoError(err, "Failed to create storage class: %v", err) - m.sc[class.Name] = class - pvc := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{ - Name: "snapshot-test-pvc", - StorageClassName: &(class.Name), - }, f.Namespace.Name) - - ginkgo.By(fmt.Sprintf("Creating PVC %s/%s", pvc.Namespace, pvc.Name)) - pvc, err = m.cs.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Create(context.TODO(), pvc, metav1.CreateOptions{}) - framework.ExpectNoError(err, "Failed to create claim: %v", err) - - ginkgo.By("Wait for PVC to be Bound") - _, err = e2epv.WaitForPVClaimBoundPhase(m.cs, []*v1.PersistentVolumeClaim{pvc}, 1*time.Minute) - framework.ExpectNoError(err, "Failed to create claim: %v", err) - - ginkgo.By("Creating snapshot") - parameters := map[string]string{} - sr := storageframework.CreateSnapshotResource(sDriver, m.config, test.pattern, pvc.Name, pvc.Namespace, f.Timeouts, parameters) - framework.ExpectNoError(err, "failed to create snapshot") - - ginkgo.By("Checking for CreateSnapshot metrics") - createSnapshotMetrics.waitForSnapshotControllerMetric(originalCreateSnapshotCount+1.0, f.Timeouts.SnapshotControllerMetrics) - - ginkgo.By("Checking for CreateSnapshotAndReady metrics") - err = utils.WaitForSnapshotReady(m.config.Framework.DynamicClient, pvc.Namespace, sr.Vs.GetName(), framework.Poll, f.Timeouts.SnapshotCreate) - framework.ExpectNoError(err, "failed to wait for snapshot ready") - createSnapshotAndReadyMetrics.waitForSnapshotControllerMetric(originalCreateSnapshotAndReadyCount+1.0, f.Timeouts.SnapshotControllerMetrics) - - // delete the snapshot and check if the snapshot is deleted - deleteSnapshot(m.cs, m.config, sr.Vs) - - ginkgo.By("check for delete metrics") - metricsTestConfig.operationName = "DeleteSnapshot" - deleteSnapshotMetrics.waitForSnapshotControllerMetric(originalDeleteSnapshotCount+1.0, f.Timeouts.SnapshotControllerMetrics) - }) - } - }) - - ginkgo.Context("SELinuxMount [LinuxOnly][Feature:SELinux][Feature:SELinuxMountReadWriteOncePod]", func() { - // Make sure all options are set so system specific defaults are not used. - seLinuxOpts := v1.SELinuxOptions{ - User: "system_u", - Role: "object_r", - Type: "container_file_t", - Level: "s0:c0,c1", - } - seLinuxMountOption := "context=\"system_u:object_r:container_file_t:s0:c0,c1\"" - - tests := []struct { - name string - seLinuxEnabled bool - seLinuxSetInPod bool - mountOptions []string - volumeMode v1.PersistentVolumeAccessMode - expectedMountOptions []string - }{ - { - name: "should pass SELinux mount option for RWOP volume and Pod with SELinux context set", - seLinuxEnabled: true, - seLinuxSetInPod: true, - volumeMode: v1.ReadWriteOncePod, - expectedMountOptions: []string{seLinuxMountOption}, - }, - { - name: "should add SELinux mount option to existing mount options", - seLinuxEnabled: true, - seLinuxSetInPod: true, - mountOptions: []string{"noexec", "noatime"}, - volumeMode: v1.ReadWriteOncePod, - expectedMountOptions: []string{"noexec", "noatime", seLinuxMountOption}, - }, - { - name: "should not pass SELinux mount option for RWO volume", - seLinuxEnabled: true, - seLinuxSetInPod: true, - volumeMode: v1.ReadWriteOnce, - expectedMountOptions: nil, - }, - { - name: "should not pass SELinux mount option for Pod without SELinux context", - seLinuxEnabled: true, - seLinuxSetInPod: false, - volumeMode: v1.ReadWriteOncePod, - expectedMountOptions: nil, - }, - { - name: "should not pass SELinux mount option for CSI driver that does not support SELinux mount", - seLinuxEnabled: false, - seLinuxSetInPod: true, - volumeMode: v1.ReadWriteOncePod, - expectedMountOptions: nil, - }, - } - for _, t := range tests { - t := t - ginkgo.It(t.name, func(ctx context.Context) { - if framework.NodeOSDistroIs("windows") { - e2eskipper.Skipf("SELinuxMount is only applied on linux nodes -- skipping") - } - var nodeStageMountOpts, nodePublishMountOpts []string - init(testParameters{ - disableAttach: true, - registerDriver: true, - enableSELinuxMount: &t.seLinuxEnabled, - hooks: createSELinuxMountPreHook(&nodeStageMountOpts, &nodePublishMountOpts), - }) - ginkgo.DeferCleanup(cleanup) - - accessModes := []v1.PersistentVolumeAccessMode{t.volumeMode} - var podSELinuxOpts *v1.SELinuxOptions - if t.seLinuxSetInPod { - // Make sure all options are set so system specific defaults are not used. - podSELinuxOpts = &seLinuxOpts - } - - _, _, pod := createPodWithSELinux(accessModes, t.mountOptions, podSELinuxOpts) - err := e2epod.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) - framework.ExpectNoError(err, "failed to start pod") - - framework.ExpectEqual(nodeStageMountOpts, t.expectedMountOptions, "Expect NodeStageVolumeRequest.VolumeCapability.MountVolume. to equal %q; got: %q", t.expectedMountOptions, nodeStageMountOpts) - framework.ExpectEqual(nodePublishMountOpts, t.expectedMountOptions, "Expect NodePublishVolumeRequest.VolumeCapability.MountVolume.VolumeMountGroup to equal %q; got: %q", t.expectedMountOptions, nodeStageMountOpts) - }) - } - }) - -}) - -func deleteSnapshot(cs clientset.Interface, config *storageframework.PerTestConfig, snapshot *unstructured.Unstructured) { - // delete the given snapshot - dc := config.Framework.DynamicClient - err := dc.Resource(utils.SnapshotGVR).Namespace(snapshot.GetNamespace()).Delete(context.TODO(), snapshot.GetName(), metav1.DeleteOptions{}) - framework.ExpectNoError(err) - - // check if the snapshot is deleted - _, err = dc.Resource(utils.SnapshotGVR).Get(context.TODO(), snapshot.GetName(), metav1.GetOptions{}) - framework.ExpectError(err) -} - -// A lot of this code was copied from e2e/framework. It would be nicer -// if it could be reused - see https://github.com/kubernetes/kubernetes/issues/92754 -func podRunning(ctx context.Context, c clientset.Interface, podName, namespace string) wait.ConditionFunc { - return func() (bool, error) { - pod, err := c.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) - if err != nil { - return false, err - } - switch pod.Status.Phase { - case v1.PodRunning: - return true, nil - case v1.PodFailed, v1.PodSucceeded: - return false, errPodCompleted - } - return false, nil - } -} - -const ( - poll = 2 * time.Second - pvcAsSourceProtectionFinalizer = "snapshot.storage.kubernetes.io/pvc-as-source-protection" - volumeSnapshotContentFinalizer = "snapshot.storage.kubernetes.io/volumesnapshotcontent-bound-protection" - volumeSnapshotBoundFinalizer = "snapshot.storage.kubernetes.io/volumesnapshot-bound-protection" - errReasonNotEnoughSpace = "node(s) did not have enough free storage" -) - -var ( - errPodCompleted = fmt.Errorf("pod ran to completion") - errNotEnoughSpace = errors.New(errReasonNotEnoughSpace) -) - -func podHasStorage(ctx context.Context, c clientset.Interface, podName, namespace string, when time.Time) wait.ConditionFunc { - // Check for events of this pod. Copied from test/e2e/common/container_probe.go. - expectedEvent := fields.Set{ - "involvedObject.kind": "Pod", - "involvedObject.name": podName, - "involvedObject.namespace": namespace, - "reason": "FailedScheduling", - }.AsSelector().String() - options := metav1.ListOptions{ - FieldSelector: expectedEvent, - } - // copied from test/e2e/framework/events/events.go - return func() (bool, error) { - // We cannot be sure here whether it has enough storage, only when - // it hasn't. In that case we abort waiting with a special error. - events, err := c.CoreV1().Events(namespace).List(ctx, options) - if err != nil { - return false, fmt.Errorf("got error while getting events: %w", err) - } - for _, event := range events.Items { - if /* event.CreationTimestamp.After(when) && - */strings.Contains(event.Message, errReasonNotEnoughSpace) { - return false, errNotEnoughSpace - } - } - return false, nil - } -} - -func anyOf(conditions ...wait.ConditionFunc) wait.ConditionFunc { - return func() (bool, error) { - for _, condition := range conditions { - done, err := condition() - if err != nil { - return false, err - } - if done { - return true, nil - } - } - return false, nil - } -} - -func waitForMaxVolumeCondition(pod *v1.Pod, cs clientset.Interface) error { - waitErr := wait.PollImmediate(10*time.Second, csiPodUnschedulableTimeout, func() (bool, error) { - pod, err := cs.CoreV1().Pods(pod.Namespace).Get(context.TODO(), pod.Name, metav1.GetOptions{}) - if err != nil { - return false, err - } - for _, c := range pod.Status.Conditions { - // Conformance tests cannot rely on specific output of optional fields (e.g., Reason - // and Message) because these fields are not suject to the deprecation policy. - if c.Type == v1.PodScheduled && c.Status == v1.ConditionFalse && c.Reason != "" && c.Message != "" { - return true, nil - } - } - return false, nil - }) - if waitErr != nil { - return fmt.Errorf("error waiting for pod %s/%s to have max volume condition: %v", pod.Namespace, pod.Name, waitErr) - } - return nil -} - -func checkCSINodeForLimits(nodeName string, driverName string, cs clientset.Interface) (int32, error) { - var attachLimit int32 - - waitErr := wait.PollImmediate(10*time.Second, csiNodeLimitUpdateTimeout, func() (bool, error) { - csiNode, err := cs.StorageV1().CSINodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - return false, err - } - attachLimit = getVolumeLimitFromCSINode(csiNode, driverName) - if attachLimit > 0 { - return true, nil - } - return false, nil - }) - if waitErr != nil { - return 0, fmt.Errorf("error waiting for non-zero volume limit of driver %s on node %s: %v", driverName, nodeName, waitErr) - } - return attachLimit, nil -} - -func createSC(cs clientset.Interface, t testsuites.StorageClassTest, scName, ns string) *storagev1.StorageClass { - class := newStorageClass(t, ns, "") - if scName != "" { - class.Name = scName - } - var err error - _, err = cs.StorageV1().StorageClasses().Get(context.TODO(), class.Name, metav1.GetOptions{}) - if err != nil { - class, err = cs.StorageV1().StorageClasses().Create(context.TODO(), class, metav1.CreateOptions{}) - framework.ExpectNoError(err, "Failed to create class: %v", err) - } - - return class -} - -func createClaim(cs clientset.Interface, t testsuites.StorageClassTest, node e2epod.NodeSelection, scName, ns string, accessModes []v1.PersistentVolumeAccessMode) (*storagev1.StorageClass, *v1.PersistentVolumeClaim) { - class := createSC(cs, t, scName, ns) - claim := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{ - ClaimSize: t.ClaimSize, - StorageClassName: &(class.Name), - VolumeMode: &t.VolumeMode, - AccessModes: accessModes, - }, ns) - claim, err := cs.CoreV1().PersistentVolumeClaims(ns).Create(context.TODO(), claim, metav1.CreateOptions{}) - framework.ExpectNoError(err, "Failed to create claim: %v", err) - - if !t.DelayBinding { - pvcClaims := []*v1.PersistentVolumeClaim{claim} - _, err = e2epv.WaitForPVClaimBoundPhase(cs, pvcClaims, framework.ClaimProvisionTimeout) - framework.ExpectNoError(err, "Failed waiting for PVC to be bound: %v", err) - } - return class, claim -} - -func startPausePod(cs clientset.Interface, t testsuites.StorageClassTest, node e2epod.NodeSelection, scName, ns string) (*storagev1.StorageClass, *v1.PersistentVolumeClaim, *v1.Pod) { - class, claim := createClaim(cs, t, node, scName, ns, nil) - - pod, err := startPausePodWithClaim(cs, claim, node, ns) - framework.ExpectNoError(err, "Failed to create pause pod: %v", err) - return class, claim, pod -} - -func startBusyBoxPod(cs clientset.Interface, t testsuites.StorageClassTest, node e2epod.NodeSelection, scName, ns string, fsGroup *int64) (*storagev1.StorageClass, *v1.PersistentVolumeClaim, *v1.Pod) { - class, claim := createClaim(cs, t, node, scName, ns, nil) - pod, err := startBusyBoxPodWithClaim(cs, claim, node, ns, fsGroup) - framework.ExpectNoError(err, "Failed to create busybox pod: %v", err) - return class, claim, pod -} - -func startPausePodInline(cs clientset.Interface, t testsuites.StorageClassTest, node e2epod.NodeSelection, ns string) *v1.Pod { - pod, err := startPausePodWithInlineVolume(cs, - &v1.CSIVolumeSource{ - Driver: t.Provisioner, - }, - node, ns) - framework.ExpectNoError(err, "Failed to create pod: %v", err) - return pod -} - -func startPausePodGenericEphemeral(cs clientset.Interface, t testsuites.StorageClassTest, node e2epod.NodeSelection, scName, ns string) (*storagev1.StorageClass, *v1.Pod) { - class := createSC(cs, t, scName, ns) - claim := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{ - ClaimSize: t.ClaimSize, - StorageClassName: &(class.Name), - VolumeMode: &t.VolumeMode, - }, ns) - pod, err := startPausePodWithVolumeSource(cs, v1.VolumeSource{ - Ephemeral: &v1.EphemeralVolumeSource{ - VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{Spec: claim.Spec}}, - }, node, ns) - framework.ExpectNoError(err, "Failed to create pod: %v", err) - return class, pod -} - -func startPausePodWithClaim(cs clientset.Interface, pvc *v1.PersistentVolumeClaim, node e2epod.NodeSelection, ns string) (*v1.Pod, error) { - return startPausePodWithVolumeSource(cs, - v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: pvc.Name, - ReadOnly: false, - }, - }, - node, ns) -} - -func startBusyBoxPodWithClaim(cs clientset.Interface, pvc *v1.PersistentVolumeClaim, node e2epod.NodeSelection, ns string, fsGroup *int64) (*v1.Pod, error) { - return startBusyBoxPodWithVolumeSource(cs, - v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: pvc.Name, - ReadOnly: false, - }, - }, - node, ns, fsGroup) -} - -func startPausePodWithInlineVolume(cs clientset.Interface, inlineVolume *v1.CSIVolumeSource, node e2epod.NodeSelection, ns string) (*v1.Pod, error) { - return startPausePodWithVolumeSource(cs, - v1.VolumeSource{ - CSI: inlineVolume, - }, - node, ns) -} - -func startPausePodWithVolumeSource(cs clientset.Interface, volumeSource v1.VolumeSource, node e2epod.NodeSelection, ns string) (*v1.Pod, error) { - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "pvc-volume-tester-", - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "volume-tester", - Image: imageutils.GetE2EImage(imageutils.Pause), - VolumeMounts: []v1.VolumeMount{ - { - Name: "my-volume", - MountPath: "/mnt/test", - }, - }, - }, - }, - RestartPolicy: v1.RestartPolicyNever, - Volumes: []v1.Volume{ - { - Name: "my-volume", - VolumeSource: volumeSource, - }, - }, - }, - } - e2epod.SetNodeSelection(&pod.Spec, node) - return cs.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{}) -} - -func startBusyBoxPodWithVolumeSource(cs clientset.Interface, volumeSource v1.VolumeSource, node e2epod.NodeSelection, ns string, fsGroup *int64) (*v1.Pod, error) { - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "pvc-volume-tester-", - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "volume-tester", - Image: framework.BusyBoxImage, - VolumeMounts: []v1.VolumeMount{ - { - Name: "my-volume", - MountPath: "/mnt/test", - }, - }, - Command: e2epod.GenerateScriptCmd("while true ; do sleep 2; done"), - }, - }, - SecurityContext: &v1.PodSecurityContext{ - FSGroup: fsGroup, - }, - RestartPolicy: v1.RestartPolicyNever, - Volumes: []v1.Volume{ - { - Name: "my-volume", - VolumeSource: volumeSource, - }, - }, - }, - } - e2epod.SetNodeSelection(&pod.Spec, node) - return cs.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{}) -} - -func startPausePodWithSELinuxOptions(cs clientset.Interface, pvc *v1.PersistentVolumeClaim, node e2epod.NodeSelection, ns string, seLinuxOpts *v1.SELinuxOptions) (*v1.Pod, error) { - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "pvc-volume-tester-", - }, - Spec: v1.PodSpec{ - SecurityContext: &v1.PodSecurityContext{ - SELinuxOptions: seLinuxOpts, - }, - Containers: []v1.Container{ - { - Name: "volume-tester", - Image: imageutils.GetE2EImage(imageutils.Pause), - VolumeMounts: []v1.VolumeMount{ - { - Name: "my-volume", - MountPath: "/mnt/test", - }, - }, - }, - }, - RestartPolicy: v1.RestartPolicyNever, - Volumes: []v1.Volume{ - { - Name: "my-volume", - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: pvc.Name, - ReadOnly: false, - }, - }, - }, - }, - }, - } - e2epod.SetNodeSelection(&pod.Spec, node) - return cs.CoreV1().Pods(ns).Create(context.TODO(), pod, metav1.CreateOptions{}) -} - -// checkPodLogs tests that NodePublish was called with expected volume_context and (for ephemeral inline volumes) -// has the matching NodeUnpublish -func checkPodLogs(getCalls func() ([]drivers.MockCSICall, error), pod *v1.Pod, expectPodInfo, ephemeralVolume, csiInlineVolumesEnabled, csiServiceAccountTokenEnabled bool, expectedNumNodePublish int) error { - expectedAttributes := map[string]string{} - if expectPodInfo { - expectedAttributes["csi.storage.k8s.io/pod.name"] = pod.Name - expectedAttributes["csi.storage.k8s.io/pod.namespace"] = pod.Namespace - expectedAttributes["csi.storage.k8s.io/pod.uid"] = string(pod.UID) - expectedAttributes["csi.storage.k8s.io/serviceAccount.name"] = "default" - - } - if csiInlineVolumesEnabled { - // This is only passed in 1.15 when the CSIInlineVolume feature gate is set. - expectedAttributes["csi.storage.k8s.io/ephemeral"] = strconv.FormatBool(ephemeralVolume) - } - - if csiServiceAccountTokenEnabled { - expectedAttributes["csi.storage.k8s.io/serviceAccount.tokens"] = "" - } - - // Find NodePublish in the GRPC calls. - foundAttributes := sets.NewString() - numNodePublishVolume := 0 - numNodeUnpublishVolume := 0 - calls, err := getCalls() - if err != nil { - return err - } - - for _, call := range calls { - switch call.Method { - case "NodePublishVolume": - numNodePublishVolume++ - if numNodePublishVolume == 1 { - // Check that NodePublish had expected attributes for first volume - for k, v := range expectedAttributes { - vv, found := call.Request.VolumeContext[k] - if found && (v == vv || (v == "" && len(vv) != 0)) { - foundAttributes.Insert(k) - framework.Logf("Found volume attribute %s: %s", k, vv) - } - } - } - case "NodeUnpublishVolume": - framework.Logf("Found NodeUnpublishVolume: %+v", call) - numNodeUnpublishVolume++ - } - } - if numNodePublishVolume < expectedNumNodePublish { - return fmt.Errorf("NodePublish should be called at least %d", expectedNumNodePublish) - } - - if numNodeUnpublishVolume == 0 { - return fmt.Errorf("NodeUnpublish was never called") - } - if foundAttributes.Len() != len(expectedAttributes) { - return fmt.Errorf("number of found volume attributes does not match, expected %d, got %d", len(expectedAttributes), foundAttributes.Len()) - } - return nil -} - -// compareCSICalls compares expectedCalls with logs of the mock driver. -// It returns index of the first expectedCall that was *not* received -// yet or error when calls do not match. -// All repeated calls to the CSI mock driver (e.g. due to exponential backoff) -// are squashed and checked against single expectedCallSequence item. -// -// Only permanent errors are returned. Other errors are logged and no -// calls are returned. The caller is expected to retry. -func compareCSICalls(trackedCalls []string, expectedCallSequence []csiCall, getCalls func() ([]drivers.MockCSICall, error)) ([]drivers.MockCSICall, int, error) { - allCalls, err := getCalls() - if err != nil { - framework.Logf("intermittent (?) log retrieval error, proceeding without output: %v", err) - return nil, 0, nil - } - - // Remove all repeated and ignored calls - tracked := sets.NewString(trackedCalls...) - var calls []drivers.MockCSICall - var last drivers.MockCSICall - for _, c := range allCalls { - if !tracked.Has(c.Method) { - continue - } - if c.Method != last.Method || c.FullError.Code != last.FullError.Code { - last = c - calls = append(calls, c) - } - // This call is the same as the last one, ignore it. - } - - for i, c := range calls { - if i >= len(expectedCallSequence) { - // Log all unexpected calls first, return error below outside the loop. - framework.Logf("Unexpected CSI driver call: %s (%d)", c.Method, c.FullError) - continue - } - - // Compare current call with expected call - expectedCall := expectedCallSequence[i] - if c.Method != expectedCall.expectedMethod || c.FullError.Code != expectedCall.expectedError { - return allCalls, i, fmt.Errorf("Unexpected CSI call %d: expected %s (%d), got %s (%d)", i, expectedCall.expectedMethod, expectedCall.expectedError, c.Method, c.FullError.Code) - } - } - if len(calls) > len(expectedCallSequence) { - return allCalls, len(expectedCallSequence), fmt.Errorf("Received %d unexpected CSI driver calls", len(calls)-len(expectedCallSequence)) - } - // All calls were correct - return allCalls, len(calls), nil - -} - -func waitForCSIDriver(cs clientset.Interface, driverName string) error { - timeout := 4 * time.Minute - - framework.Logf("waiting up to %v for CSIDriver %q", timeout, driverName) - for start := time.Now(); time.Since(start) < timeout; time.Sleep(framework.Poll) { - _, err := cs.StorageV1().CSIDrivers().Get(context.TODO(), driverName, metav1.GetOptions{}) - if !apierrors.IsNotFound(err) { - return err - } - } - return fmt.Errorf("gave up after waiting %v for CSIDriver %q", timeout, driverName) -} - -func destroyCSIDriver(cs clientset.Interface, driverName string) { - driverGet, err := cs.StorageV1().CSIDrivers().Get(context.TODO(), driverName, metav1.GetOptions{}) - if err == nil { - framework.Logf("deleting %s.%s: %s", driverGet.TypeMeta.APIVersion, driverGet.TypeMeta.Kind, driverGet.ObjectMeta.Name) - // Uncomment the following line to get full dump of CSIDriver object - // framework.Logf("%s", framework.PrettyPrint(driverGet)) - cs.StorageV1().CSIDrivers().Delete(context.TODO(), driverName, metav1.DeleteOptions{}) - } -} - -func getVolumeLimitFromCSINode(csiNode *storagev1.CSINode, driverName string) int32 { - for _, d := range csiNode.Spec.Drivers { - if d.Name != driverName { - continue - } - if d.Allocatable != nil && d.Allocatable.Count != nil { - return *d.Allocatable.Count - } - } - return 0 -} - -// checkDeleteSnapshotSecrets checks if delete snapshot secrets annotation is applied to the VolumeSnapshotContent. -func checkDeleteSnapshotSecrets(cs clientset.Interface, annotations interface{}) error { - ginkgo.By("checking if delete snapshot secrets annotation is applied to the VolumeSnapshotContent") - - var ( - annDeletionSecretName string - annDeletionSecretNamespace string - ok bool - err error - - // CSISnapshotterDeleteSecretNameAnnotation is the annotation key for the CSI snapshotter delete secret name in VolumeSnapshotClass.parameters - CSISnapshotterDeleteSecretNameAnnotation string = "snapshot.storage.kubernetes.io/deletion-secret-name" - - // CSISnapshotterDeleteSecretNamespaceAnnotation is the annotation key for the CSI snapshotter delete secret namespace in VolumeSnapshotClass.parameters - CSISnapshotterDeleteSecretNamespaceAnnotation string = "snapshot.storage.kubernetes.io/deletion-secret-namespace" - ) - - annotationsObj, ok := annotations.(map[string]interface{}) - if !ok { - framework.Failf("failed to get annotations from annotations object") - } - - if annDeletionSecretName, ok = annotationsObj[CSISnapshotterDeleteSecretNameAnnotation].(string); !ok { - framework.Failf("unable to get secret annotation name") - } - if annDeletionSecretNamespace, ok = annotationsObj[CSISnapshotterDeleteSecretNamespaceAnnotation].(string); !ok { - framework.Failf("unable to get secret annotation namespace") - } - - // verify if secrets exists - if _, err = cs.CoreV1().Secrets(annDeletionSecretNamespace).Get(context.TODO(), annDeletionSecretName, metav1.GetOptions{}); err != nil { - framework.Failf("unable to get test secret %s: %v", annDeletionSecretName, err) - } - - return err -} - -// createPreHook counts invocations of a certain method (identified by a substring in the full gRPC method name). -func createPreHook(method string, callback func(counter int64) error) *drivers.Hooks { - var counter int64 - - return &drivers.Hooks{ - Pre: func() func(ctx context.Context, fullMethod string, request interface{}) (reply interface{}, err error) { - return func(ctx context.Context, fullMethod string, request interface{}) (reply interface{}, err error) { - if strings.Contains(fullMethod, method) { - counter := atomic.AddInt64(&counter, 1) - return nil, callback(counter) - } - return nil, nil - } - }(), - } -} - -// createFSGroupRequestPreHook creates a hook that records the fsGroup passed in -// through NodeStageVolume and NodePublishVolume calls. -func createFSGroupRequestPreHook(nodeStageFsGroup, nodePublishFsGroup *string) *drivers.Hooks { - return &drivers.Hooks{ - Pre: func(ctx context.Context, fullMethod string, request interface{}) (reply interface{}, err error) { - nodeStageRequest, ok := request.(*csipbv1.NodeStageVolumeRequest) - if ok { - mountVolume := nodeStageRequest.GetVolumeCapability().GetMount() - if mountVolume != nil { - *nodeStageFsGroup = mountVolume.VolumeMountGroup - } - } - nodePublishRequest, ok := request.(*csipbv1.NodePublishVolumeRequest) - if ok { - mountVolume := nodePublishRequest.GetVolumeCapability().GetMount() - if mountVolume != nil { - *nodePublishFsGroup = mountVolume.VolumeMountGroup - } - } - return nil, nil - }, - } -} - -// createSELinuxMountPreHook creates a hook that records the mountOptions passed in -// through NodeStageVolume and NodePublishVolume calls. -func createSELinuxMountPreHook(nodeStageMountOpts, nodePublishMountOpts *[]string) *drivers.Hooks { - return &drivers.Hooks{ - Pre: func(ctx context.Context, fullMethod string, request interface{}) (reply interface{}, err error) { - nodeStageRequest, ok := request.(*csipbv1.NodeStageVolumeRequest) - if ok { - mountVolume := nodeStageRequest.GetVolumeCapability().GetMount() - if mountVolume != nil { - *nodeStageMountOpts = mountVolume.MountFlags - } - } - nodePublishRequest, ok := request.(*csipbv1.NodePublishVolumeRequest) - if ok { - mountVolume := nodePublishRequest.GetVolumeCapability().GetMount() - if mountVolume != nil { - *nodePublishMountOpts = mountVolume.MountFlags - } - } - return nil, nil - }, - } -} - -type snapshotMetricsTestConfig struct { - // expected values - metricName string - metricType string - driverName string - operationName string - operationStatus string - snapshotType string - le string -} - -type snapshotControllerMetrics struct { - // configuration for metric - cfg snapshotMetricsTestConfig - metricsGrabber *e2emetrics.Grabber - - // results - countMetrics map[string]float64 - sumMetrics map[string]float64 - bucketMetrics map[string]float64 -} - -func newSnapshotMetricsTestConfig(metricName, metricType, driverName, operationName, operationStatus, le string, pattern storageframework.TestPattern) snapshotMetricsTestConfig { - var snapshotType string - switch pattern.SnapshotType { - case storageframework.DynamicCreatedSnapshot: - snapshotType = "dynamic" - - case storageframework.PreprovisionedCreatedSnapshot: - snapshotType = "pre-provisioned" - - default: - framework.Failf("invalid snapshotType: %v", pattern.SnapshotType) - } - - return snapshotMetricsTestConfig{ - metricName: metricName, - metricType: metricType, - driverName: driverName, - operationName: operationName, - operationStatus: operationStatus, - snapshotType: snapshotType, - le: le, - } -} - -func newSnapshotControllerMetrics(cfg snapshotMetricsTestConfig, metricsGrabber *e2emetrics.Grabber) *snapshotControllerMetrics { - return &snapshotControllerMetrics{ - cfg: cfg, - metricsGrabber: metricsGrabber, - - countMetrics: make(map[string]float64), - sumMetrics: make(map[string]float64), - bucketMetrics: make(map[string]float64), - } -} - -func (scm *snapshotControllerMetrics) waitForSnapshotControllerMetric(expectedValue float64, timeout time.Duration) { - metricKey := scm.getMetricKey() - if successful := utils.WaitUntil(10*time.Second, timeout, func() bool { - // get metric value - actualValue, err := scm.getSnapshotControllerMetricValue() - if err != nil { - return false - } - - // Another operation could have finished from a previous test, - // so we check if we have at least the expected value. - if actualValue < expectedValue { - return false - } - - return true - }); successful { - return - } - - scm.showMetricsFailure(metricKey) - framework.Failf("Unable to get valid snapshot controller metrics after %v", timeout) -} - -func (scm *snapshotControllerMetrics) getSnapshotControllerMetricValue() (float64, error) { - metricKey := scm.getMetricKey() - - // grab and parse into readable format - err := scm.grabSnapshotControllerMetrics() - if err != nil { - return 0, err - } - - metrics := scm.getMetricsTable() - actual, ok := metrics[metricKey] - if !ok { - return 0, fmt.Errorf("did not find metric for key %s", metricKey) - } - - return actual, nil -} - -func (scm *snapshotControllerMetrics) getMetricsTable() map[string]float64 { - var metrics map[string]float64 - switch scm.cfg.metricType { - case "count": - metrics = scm.countMetrics - - case "sum": - metrics = scm.sumMetrics - - case "bucket": - metrics = scm.bucketMetrics - } - - return metrics -} - -func (scm *snapshotControllerMetrics) showMetricsFailure(metricKey string) { - framework.Logf("failed to find metric key %s inside of the following metrics:", metricKey) - - metrics := scm.getMetricsTable() - for k, v := range metrics { - framework.Logf("%s: %v", k, v) - } -} - -func (scm *snapshotControllerMetrics) grabSnapshotControllerMetrics() error { - // pull all metrics - metrics, err := scm.metricsGrabber.GrabFromSnapshotController(framework.TestContext.SnapshotControllerPodName, framework.TestContext.SnapshotControllerHTTPPort) - if err != nil { - return err - } - - for method, samples := range metrics { - - for _, sample := range samples { - operationName := string(sample.Metric["operation_name"]) - driverName := string(sample.Metric["driver_name"]) - operationStatus := string(sample.Metric["operation_status"]) - snapshotType := string(sample.Metric["snapshot_type"]) - le := string(sample.Metric["le"]) - key := snapshotMetricKey(scm.cfg.metricName, driverName, operationName, operationStatus, snapshotType, le) - - switch method { - case "snapshot_controller_operation_total_seconds_count": - for _, sample := range samples { - scm.countMetrics[key] = float64(sample.Value) - } - - case "snapshot_controller_operation_total_seconds_sum": - for _, sample := range samples { - scm.sumMetrics[key] = float64(sample.Value) - } - - case "snapshot_controller_operation_total_seconds_bucket": - for _, sample := range samples { - scm.bucketMetrics[key] = float64(sample.Value) - } - } - } - } - - return nil -} - -func (scm *snapshotControllerMetrics) getMetricKey() string { - return snapshotMetricKey(scm.cfg.metricName, scm.cfg.driverName, scm.cfg.operationName, scm.cfg.operationStatus, scm.cfg.snapshotType, scm.cfg.le) -} - -func snapshotMetricKey(metricName, driverName, operationName, operationStatus, snapshotType, le string) string { - key := driverName - - // build key for shorthand metrics storage - for _, s := range []string{metricName, operationName, operationStatus, snapshotType, le} { - if s != "" { - key = fmt.Sprintf("%s_%s", key, s) - } - } - - return key -} diff --git a/test/e2e/storage/local_volume_resize.go b/test/e2e/storage/local_volume_resize.go index b66e61588c110..8bb428eaafd62 100644 --- a/test/e2e/storage/local_volume_resize.go +++ b/test/e2e/storage/local_volume_resize.go @@ -39,6 +39,10 @@ import ( admissionapi "k8s.io/pod-security-admission/api" ) +const ( + csiResizeWaitPeriod = 5 * time.Minute +) + var _ = utils.SIGDescribe("PersistentVolumes-expansion ", func() { f := framework.NewDefaultFramework("persistent-local-volumes-expansion") f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged From 9e5d0828e08effbadbf6bb097ae5d70254fe2b5e Mon Sep 17 00:00:00 2001 From: Hemant Kumar Date: Tue, 13 Dec 2022 15:54:55 -0500 Subject: [PATCH 2/2] use defercleanup in all the places --- test/e2e/storage/csi_mock/base.go | 4 +--- test/e2e/storage/csi_mock/csi_attach_volume.go | 7 +++---- test/e2e/storage/csi_mock/csi_fsgroup_mount.go | 2 +- test/e2e/storage/csi_mock/csi_fsgroup_policy.go | 2 +- test/e2e/storage/csi_mock/csi_node_stage_error_cases.go | 4 ++-- test/e2e/storage/csi_mock/csi_selinux_mount.go | 2 +- test/e2e/storage/csi_mock/csi_service_account_token.go | 2 +- test/e2e/storage/csi_mock/csi_snapshot.go | 6 +++--- test/e2e/storage/csi_mock/csi_storage_capacity.go | 8 +++----- test/e2e/storage/csi_mock/csi_volume_expansion.go | 5 ++--- test/e2e/storage/csi_mock/csi_volume_limit.go | 9 ++++++--- test/e2e/storage/csi_mock/csi_workload.go | 2 +- 12 files changed, 25 insertions(+), 28 deletions(-) diff --git a/test/e2e/storage/csi_mock/base.go b/test/e2e/storage/csi_mock/base.go index 266eee0bbb0e1..4b21fdebc942f 100644 --- a/test/e2e/storage/csi_mock/base.go +++ b/test/e2e/storage/csi_mock/base.go @@ -196,9 +196,7 @@ func (m *mockDriverSetup) init(tp testParameters) { if tp.registerDriver { err = waitForCSIDriver(m.cs, m.config.GetUniqueDriverName()) framework.ExpectNoError(err, "Failed to get CSIDriver %v", m.config.GetUniqueDriverName()) - m.testCleanups = append(m.testCleanups, func() { - destroyCSIDriver(m.cs, m.config.GetUniqueDriverName()) - }) + ginkgo.DeferCleanup(destroyCSIDriver, m.cs, m.config.GetUniqueDriverName()) } // Wait for the CSIDriver actually get deployed and CSINode object to be generated. diff --git a/test/e2e/storage/csi_mock/csi_attach_volume.go b/test/e2e/storage/csi_mock/csi_attach_volume.go index fb2a9e829ebec..5fe1d69fa303a 100644 --- a/test/e2e/storage/csi_mock/csi_attach_volume.go +++ b/test/e2e/storage/csi_mock/csi_attach_volume.go @@ -72,7 +72,7 @@ var _ = utils.SIGDescribe("CSI Mock volume attach", func() { ginkgo.It(t.name, func(ctx context.Context) { var err error m.init(testParameters{registerDriver: test.deployClusterRegistrar, disableAttach: test.disableAttach}) - defer m.cleanup() + ginkgo.DeferCleanup(m.cleanup) volumeType := test.volumeType if volumeType == "" { @@ -110,7 +110,7 @@ var _ = utils.SIGDescribe("CSI Mock volume attach", func() { ginkgo.It("should bringup pod after deploying CSIDriver attach=false [Slow]", func(ctx context.Context) { var err error m.init(testParameters{registerDriver: false, disableAttach: true}) - defer m.cleanup() + ginkgo.DeferCleanup(m.cleanup) _, claim, pod := m.createPod(pvcReference) // late binding as specified above if pod == nil { @@ -156,13 +156,12 @@ var _ = utils.SIGDescribe("CSI Mock volume attach", func() { NewDriverName: "csi-mock-" + f.UniqueName, CanAttach: &canAttach, } - cleanupCSIDriver, err := utils.CreateFromManifests(f, driverNamespace, func(item interface{}) error { + err = utils.CreateFromManifests(f, driverNamespace, func(item interface{}) error { return utils.PatchCSIDeployment(f, o, item) }, "test/e2e/testing-manifests/storage-csi/mock/csi-mock-driverinfo.yaml") if err != nil { framework.Failf("fail to deploy CSIDriver object: %v", err) } - m.testCleanups = append(m.testCleanups, cleanupCSIDriver) ginkgo.By("Wait for the pod in running status") err = e2epod.WaitForPodNameRunningInNamespace(m.cs, pod.Name, pod.Namespace) diff --git a/test/e2e/storage/csi_mock/csi_fsgroup_mount.go b/test/e2e/storage/csi_mock/csi_fsgroup_mount.go index 3077d578a4afe..bade243bc9e26 100644 --- a/test/e2e/storage/csi_mock/csi_fsgroup_mount.go +++ b/test/e2e/storage/csi_mock/csi_fsgroup_mount.go @@ -61,7 +61,7 @@ var _ = utils.SIGDescribe("CSI Mock fsgroup as mount option", func() { enableVolumeMountGroup: t.enableVolumeMountGroup, hooks: createFSGroupRequestPreHook(&nodeStageFsGroup, &nodePublishFsGroup), }) - defer m.cleanup() + ginkgo.DeferCleanup(m.cleanup) fsGroupVal := int64(rand.Int63n(20000) + 1024) fsGroup := &fsGroupVal diff --git a/test/e2e/storage/csi_mock/csi_fsgroup_policy.go b/test/e2e/storage/csi_mock/csi_fsgroup_policy.go index 5b2b3642184ad..65d157ace4ff4 100644 --- a/test/e2e/storage/csi_mock/csi_fsgroup_policy.go +++ b/test/e2e/storage/csi_mock/csi_fsgroup_policy.go @@ -72,7 +72,7 @@ var _ = utils.SIGDescribe("CSI Mock volume fsgroup policies", func() { registerDriver: true, fsGroupPolicy: &test.fsGroupPolicy, }) - defer m.cleanup() + ginkgo.DeferCleanup(m.cleanup) // kube-scheduler may need some time before it gets the CSIDriver object. // Without them, scheduling doesn't run as expected by the test. diff --git a/test/e2e/storage/csi_mock/csi_node_stage_error_cases.go b/test/e2e/storage/csi_mock/csi_node_stage_error_cases.go index 800ca0d8e9434..09b71dd973f5f 100644 --- a/test/e2e/storage/csi_mock/csi_node_stage_error_cases.go +++ b/test/e2e/storage/csi_mock/csi_node_stage_error_cases.go @@ -143,7 +143,7 @@ var _ = utils.SIGDescribe("CSI Mock volume node stage", func() { registerDriver: true, hooks: hooks, }) - defer m.cleanup() + ginkgo.DeferCleanup(m.cleanup) _, claim, pod := m.createPod(pvcReference) if pod == nil { @@ -281,7 +281,7 @@ var _ = utils.SIGDescribe("CSI Mock volume node stage", func() { registerDriver: true, hooks: hooks, }) - defer m.cleanup() + ginkgo.DeferCleanup(m.cleanup) _, claim, pod := m.createPod(pvcReference) if pod == nil { diff --git a/test/e2e/storage/csi_mock/csi_selinux_mount.go b/test/e2e/storage/csi_mock/csi_selinux_mount.go index 4355d69681c55..743c9bfc8962a 100644 --- a/test/e2e/storage/csi_mock/csi_selinux_mount.go +++ b/test/e2e/storage/csi_mock/csi_selinux_mount.go @@ -101,7 +101,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount", func() { enableSELinuxMount: &t.seLinuxEnabled, hooks: createSELinuxMountPreHook(&nodeStageMountOpts, &nodePublishMountOpts), }) - defer m.cleanup() + ginkgo.DeferCleanup(m.cleanup) accessModes := []v1.PersistentVolumeAccessMode{t.volumeMode} var podSELinuxOpts *v1.SELinuxOptions diff --git a/test/e2e/storage/csi_mock/csi_service_account_token.go b/test/e2e/storage/csi_mock/csi_service_account_token.go index 1e079fe447fbf..53811ab9881ad 100644 --- a/test/e2e/storage/csi_mock/csi_service_account_token.go +++ b/test/e2e/storage/csi_mock/csi_service_account_token.go @@ -69,7 +69,7 @@ var _ = utils.SIGDescribe("CSI Mock volume service account token", func() { requiresRepublish: &csiServiceAccountTokenEnabled, }) - defer m.cleanup() + ginkgo.DeferCleanup(m.cleanup) _, _, pod := m.createPod(pvcReference) if pod == nil { diff --git a/test/e2e/storage/csi_mock/csi_snapshot.go b/test/e2e/storage/csi_mock/csi_snapshot.go index 414c248b55fae..f4b97e02c80f4 100644 --- a/test/e2e/storage/csi_mock/csi_snapshot.go +++ b/test/e2e/storage/csi_mock/csi_snapshot.go @@ -80,7 +80,7 @@ var _ = utils.SIGDescribe("CSI Mock volume snapshot", func() { } ctx, cancel := context.WithTimeout(ctx, csiPodRunningTimeout) defer cancel() - defer m.cleanup() + ginkgo.DeferCleanup(m.cleanup) sc := m.driver.GetDynamicProvisionStorageClass(m.config, "") ginkgo.By("Creating storage class") @@ -219,7 +219,7 @@ var _ = utils.SIGDescribe("CSI Mock volume snapshot", func() { if !ok { e2eskipper.Skipf("mock driver does not support snapshots -- skipping") } - defer m.cleanup() + ginkgo.DeferCleanup(m.cleanup) var sc *storagev1.StorageClass if dDriver, ok := m.driver.(storageframework.DynamicPVTestDriver); ok { @@ -308,7 +308,7 @@ var _ = utils.SIGDescribe("CSI Mock volume snapshot", func() { if !ok { e2eskipper.Skipf("mock driver does not support snapshots -- skipping") } - defer m.cleanup() + ginkgo.DeferCleanup(m.cleanup) metricsGrabber, err := e2emetrics.NewMetricsGrabber(m.config.Framework.ClientSet, nil, f.ClientConfig(), false, false, false, false, false, true) if err != nil { diff --git a/test/e2e/storage/csi_mock/csi_storage_capacity.go b/test/e2e/storage/csi_mock/csi_storage_capacity.go index c9265ae9748bb..fa09628198f08 100644 --- a/test/e2e/storage/csi_mock/csi_storage_capacity.go +++ b/test/e2e/storage/csi_mock/csi_storage_capacity.go @@ -127,7 +127,7 @@ var _ = utils.SIGDescribe("CSI Mock volume storage capacity", func() { } m.init(params) - defer m.cleanup() + ginkgo.DeferCleanup(m.cleanup) ctx, cancel := context.WithTimeout(ctx, csiPodRunningTimeout) defer cancel() @@ -331,7 +331,7 @@ var _ = utils.SIGDescribe("CSI Mock volume storage capacity", func() { storageCapacity: test.storageCapacity, lateBinding: true, }) - defer m.cleanup() + ginkgo.DeferCleanup(m.cleanup) // The storage class uses a random name, therefore we have to create it first // before adding CSIStorageCapacity objects for it. @@ -348,9 +348,7 @@ var _ = utils.SIGDescribe("CSI Mock volume storage capacity", func() { } createdCapacity, err := f.ClientSet.StorageV1().CSIStorageCapacities(f.Namespace.Name).Create(context.Background(), capacity, metav1.CreateOptions{}) framework.ExpectNoError(err, "create CSIStorageCapacity %+v", *capacity) - m.testCleanups = append(m.testCleanups, func() { - f.ClientSet.StorageV1().CSIStorageCapacities(f.Namespace.Name).Delete(context.Background(), createdCapacity.Name, metav1.DeleteOptions{}) - }) + ginkgo.DeferCleanup(framework.IgnoreNotFound(f.ClientSet.StorageV1().CSIStorageCapacities(f.Namespace.Name).Delete), createdCapacity.Name, metav1.DeleteOptions{}) } // kube-scheduler may need some time before it gets the CSIDriver and CSIStorageCapacity objects. diff --git a/test/e2e/storage/csi_mock/csi_volume_expansion.go b/test/e2e/storage/csi_mock/csi_volume_expansion.go index df989c6f7a338..a36bf78f4b45b 100644 --- a/test/e2e/storage/csi_mock/csi_volume_expansion.go +++ b/test/e2e/storage/csi_mock/csi_volume_expansion.go @@ -78,7 +78,7 @@ var _ = utils.SIGDescribe("CSI Mock volume expansion", func() { } m.init(tp) - defer m.cleanup() + ginkgo.DeferCleanup(m.cleanup) sc, pvc, pod := m.createPod(pvcReference) gomega.Expect(pod).NotTo(gomega.BeNil(), "while creating pod for resizing") @@ -172,8 +172,7 @@ var _ = utils.SIGDescribe("CSI Mock volume expansion", func() { } m.init(params) - - defer m.cleanup() + ginkgo.DeferCleanup(m.cleanup) sc, pvc, pod := m.createPod(pvcReference) gomega.Expect(pod).NotTo(gomega.BeNil(), "while creating pod for resizing") diff --git a/test/e2e/storage/csi_mock/csi_volume_limit.go b/test/e2e/storage/csi_mock/csi_volume_limit.go index 3272df669dc9b..bc0714c04d57c 100644 --- a/test/e2e/storage/csi_mock/csi_volume_limit.go +++ b/test/e2e/storage/csi_mock/csi_volume_limit.go @@ -44,7 +44,8 @@ var _ = utils.SIGDescribe("CSI Mock volume limit", func() { // define volume limit to be 2 for this test var err error m.init(testParameters{attachLimit: 2}) - defer m.cleanup() + ginkgo.DeferCleanup(m.cleanup) + nodeName := m.config.ClientNodeSelection.Name driverName := m.config.GetUniqueDriverName() @@ -75,7 +76,8 @@ var _ = utils.SIGDescribe("CSI Mock volume limit", func() { // define volume limit to be 2 for this test var err error m.init(testParameters{attachLimit: 1}) - defer m.cleanup() + ginkgo.DeferCleanup(m.cleanup) + nodeName := m.config.ClientNodeSelection.Name driverName := m.config.GetUniqueDriverName() @@ -100,7 +102,8 @@ var _ = utils.SIGDescribe("CSI Mock volume limit", func() { // define volume limit to be 2 for this test var err error m.init(testParameters{attachLimit: 1}) - defer m.cleanup() + ginkgo.DeferCleanup(m.cleanup) + nodeName := m.config.ClientNodeSelection.Name driverName := m.config.GetUniqueDriverName() diff --git a/test/e2e/storage/csi_mock/csi_workload.go b/test/e2e/storage/csi_mock/csi_workload.go index 0d6d2aa421b0a..6d34229ccac12 100644 --- a/test/e2e/storage/csi_mock/csi_workload.go +++ b/test/e2e/storage/csi_mock/csi_workload.go @@ -87,7 +87,7 @@ var _ = utils.SIGDescribe("CSI Mock workload info", func() { registerDriver: test.deployClusterRegistrar, podInfo: test.podInfoOnMount}) - defer m.cleanup() + ginkgo.DeferCleanup(m.cleanup) withVolume := pvcReference if test.expectEphemeral {