Skip to content

Commit

Permalink
Merge pull request kubernetes#111746 from RomanBednar/retro-sc-assign…
Browse files Browse the repository at this point in the history 
…ment-int

Add integration test for Retroactive default StorageClass assignement
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot committed Aug 10, 2022
2 parents 518e0ac + 77d904f commit 3b945fd
Showing 1 changed file with 249 additions and 0 deletions.
249 changes: 249 additions & 0 deletions test/integration/volume/persistent_volumes_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@ package volume
import (
"context"
"fmt"
utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/kubernetes/pkg/features"
"math/rand"
"os"
"strconv"
Expand All @@ -40,6 +43,7 @@ import (
persistentvolumecontroller "k8s.io/kubernetes/pkg/controller/volume/persistentvolume"
"k8s.io/kubernetes/pkg/volume"
volumetest "k8s.io/kubernetes/pkg/volume/testing"
"k8s.io/kubernetes/pkg/volume/util"
"k8s.io/kubernetes/test/integration/framework"

"k8s.io/klog/v2"
Expand Down Expand Up @@ -1036,6 +1040,196 @@ func TestPersistentVolumeMultiPVsDiffAccessModes(t *testing.T) {
t.Log("volume released")
}

// TestRetroactiveStorageClassAssignment tests PVC retroactive storage class
// assignment and binding of PVCs with storage class name set to nil or "" with
// and without presence of a default SC.
func TestRetroactiveStorageClassAssignment(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RetroactiveDefaultStorageClass, true)()
s := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=DefaultStorageClass"}, framework.SharedEtcd())
defer s.TearDownFn()
namespaceName := "retro-pvc-sc"
defaultStorageClassName := "gold"
storageClassName := "silver"

testClient, binder, informers, watchPV, watchPVC := createClients(namespaceName, t, s, defaultSyncPeriod)
defer watchPV.Stop()
defer watchPVC.Stop()

ns := framework.CreateNamespaceOrDie(testClient, namespaceName, t)
defer framework.DeleteNamespaceOrDie(testClient, ns, t)

// NOTE: This test cannot run in parallel, because it is creating and deleting
// non-namespaced objects (PersistenceVolumes and StorageClasses).
defer testClient.CoreV1().PersistentVolumes().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{})
defer testClient.CoreV1().PersistentVolumeClaims(namespaceName).DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{})
defer testClient.StorageV1().StorageClasses().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{})

// Create non default SC (extra SC - should not be used by any PVC in this test).
nonDefaultSC := storage.StorageClass{
TypeMeta: metav1.TypeMeta{
Kind: "StorageClass",
},
ObjectMeta: metav1.ObjectMeta{
Name: storageClassName,
Annotations: map[string]string{
util.IsDefaultStorageClassAnnotation: "false",
},
},
Provisioner: provisionerPluginName,
}
if _, err := testClient.StorageV1().StorageClasses().Create(context.TODO(), &nonDefaultSC, metav1.CreateOptions{}); err != nil {
t.Errorf("Failed to create a storage class: %v", err)
}

ctx, cancel := context.WithCancel(context.TODO())
informers.Start(ctx.Done())
go binder.Run(ctx)
defer cancel()

klog.V(2).Infof("TestRetroactiveStorageClassAssignment: start")

// 1. Test that PV with SC set to "" binds to PVC with SC set to nil while default SC does not exist (verifies that feature enablement does not break old behavior).
pv1 := createPVWithStorageClass("pv-1", "/tmp/foo", "5G", "",
[]v1.PersistentVolumeAccessMode{v1.ReadWriteMany}, v1.PersistentVolumeReclaimRetain)
_, err := testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pv1, metav1.CreateOptions{})
if err != nil {
t.Errorf("Failed to create PersistentVolume: %v", err)
}

pvc1 := createPVCWithNilStorageClass("pvc-1", ns.Name, "5G", []v1.PersistentVolumeAccessMode{v1.ReadWriteMany})
_, err = testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), pvc1, metav1.CreateOptions{})
if err != nil {
t.Errorf("Failed to create PersistentVolumeClaim: %v", err)
}

// Wait until the controller pairs the volume.
waitForPersistentVolumePhase(testClient, pv1.Name, watchPV, v1.VolumeBound)
t.Log("volume bound")
waitForPersistentVolumeClaimPhase(testClient, pvc1.Name, ns.Name, watchPVC, v1.ClaimBound)
t.Log("claim bound")

pv, err := testClient.CoreV1().PersistentVolumes().Get(context.TODO(), "pv-1", metav1.GetOptions{})
if err != nil {
t.Fatalf("Unexpected error getting pv: %v", err)
}
if pv.Spec.ClaimRef == nil {
t.Fatalf("PV %s with \"\" storage class should have been bound to PVC %s that has nil storage class", pv1.Name, pvc1.Name)
}
if pv.Spec.ClaimRef.Name != pvc1.Name {
t.Fatalf("Bind mismatch! Expected %s but got %s", pvc1.Name, pv.Spec.ClaimRef.Name)
}

// 2. Test that retroactive SC assignment works - default SC is created after creation of PVC with nil SC.
pvcRetro := createPVCWithNilStorageClass("pvc-provision-noclass", ns.Name, "5G", []v1.PersistentVolumeAccessMode{v1.ReadWriteMany})
if _, err := testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), pvcRetro, metav1.CreateOptions{}); err != nil {
t.Errorf("Failed to create PVC: %v", err)
}
t.Log("claim created")

// Create default SC.
defaultSC := storage.StorageClass{
TypeMeta: metav1.TypeMeta{
Kind: "StorageClass",
},
ObjectMeta: metav1.ObjectMeta{
Name: defaultStorageClassName,
Annotations: map[string]string{
util.IsDefaultStorageClassAnnotation: "true",
},
},
Provisioner: provisionerPluginName,
}
if _, err := testClient.StorageV1().StorageClasses().Create(context.TODO(), &defaultSC, metav1.CreateOptions{}); err != nil {
t.Errorf("Failed to create a storage class: %v", err)
}

// Verify SC was assigned retroactively to PVC.
if _, ok := waitForPersistentVolumeClaimStorageClass(t, pvcRetro.Name, defaultStorageClassName, watchPVC, 20*time.Second); !ok {
t.Errorf("Expected claim %s to get a storage class %s assigned retroactively", pvcRetro.Name, defaultStorageClassName)
}

waitForPersistentVolumeClaimPhase(testClient, pvcRetro.Name, ns.Name, watchPVC, v1.ClaimBound)

// 3. Test that a new claim with nil class will still bind to PVs with SC set to "" (if available) and SC will not be assigned retroactively.
pv3 := createPVWithStorageClass("pv-3", "/tmp/bar", "5G", "",
[]v1.PersistentVolumeAccessMode{v1.ReadWriteMany}, v1.PersistentVolumeReclaimRetain)
_, err = testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pv3, metav1.CreateOptions{})
if err != nil {
t.Errorf("Failed to create PersistentVolume: %v", err)
}
waitForPersistentVolumePhase(testClient, pv3.Name, watchPV, v1.VolumeAvailable)

pvc3 := createPVCWithNilStorageClass("pvc-3", ns.Name, "5G", []v1.PersistentVolumeAccessMode{v1.ReadWriteMany})
if _, err := testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), pvc3, metav1.CreateOptions{}); err != nil {
t.Errorf("Failed to create PVC: %v", err)
}
t.Log("claim created")

waitForPersistentVolumeClaimPhase(testClient, pvc3.Name, ns.Name, watchPVC, v1.ClaimBound)

pvc, err := testClient.CoreV1().PersistentVolumeClaims(ns.Name).Get(context.TODO(), "pvc-3", metav1.GetOptions{})
if err != nil {
t.Fatalf("Unexpected error getting pv: %v", err)
}
if pvc.Spec.StorageClassName != nil {
t.Errorf("claim %s should still have nil storage class because it bound to existing PV", pvc.Name)
}

// Create another PV which should remain unbound.
pvUnbound := createPVWithStorageClass("pv-unbound", "/tmp/bar", "5G", "",
[]v1.PersistentVolumeAccessMode{v1.ReadWriteMany}, v1.PersistentVolumeReclaimRetain)
_, err = testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pvUnbound, metav1.CreateOptions{})
if err != nil {
t.Errorf("Failed to create PersistentVolume: %v", err)
}

waitForPersistentVolumePhase(testClient, pvUnbound.Name, watchPV, v1.VolumeAvailable)

pv, err = testClient.CoreV1().PersistentVolumes().Get(context.TODO(), "pv-unbound", metav1.GetOptions{})
if err != nil {
t.Fatalf("Unexpected error getting pv: %v", err)
}
if pv.Spec.ClaimRef != nil {
t.Fatalf("PV %s shouldn't be bound", pvUnbound.Name)
}

// Remove the PV to not interfere with next test.
testClient.CoreV1().PersistentVolumes().Delete(context.TODO(), pvUnbound.Name, metav1.DeleteOptions{})

// 4. Test that PV with SC set to "" binds to PVC with SC set to "" while default SC exists.
// This tests that the feature enablement and default SC presence does not break this binding.
// If this breaks there would be no way to ever bind PVs with SC set to "".
pvc4 := createPVC("pvc-4", ns.Name, "5G", []v1.PersistentVolumeAccessMode{v1.ReadWriteMany}, "")
_, err = testClient.CoreV1().PersistentVolumeClaims(ns.Name).Create(context.TODO(), pvc4, metav1.CreateOptions{})
if err != nil {
t.Errorf("Failed to create PersistentVolumeClaim: %v", err)
}

pv4 := createPVWithStorageClass("pv-4", "/tmp/bar", "5G", "",
[]v1.PersistentVolumeAccessMode{v1.ReadWriteMany}, v1.PersistentVolumeReclaimRetain)
_, err = testClient.CoreV1().PersistentVolumes().Create(context.TODO(), pv4, metav1.CreateOptions{})
if err != nil {
t.Errorf("Failed to create PersistentVolume: %v", err)
}

// Wait until the controller pairs the volume.
waitForPersistentVolumePhase(testClient, pv4.Name, watchPV, v1.VolumeBound)
t.Log("volume bound")
waitForPersistentVolumeClaimPhase(testClient, pvc4.Name, ns.Name, watchPVC, v1.ClaimBound)
t.Log("claim bound")

pv, err = testClient.CoreV1().PersistentVolumes().Get(context.TODO(), "pv-4", metav1.GetOptions{})
if err != nil {
t.Fatalf("Unexpected error getting pv: %v", err)
}
if pv.Spec.ClaimRef == nil {
t.Fatalf("PV %s with \"\" storage class should have been bound to PVC %s that also has \"\" storage class", pv4.Name, pvc4.Name)
}
if pv.Spec.ClaimRef.Name != pvc4.Name {
t.Fatalf("Bind mismatch! Expected PV %s to bind to PVC %s but instead it bound to PVC %s", pv.Name, pvc4.Name, pv.Spec.ClaimRef.Name)
}
}

func waitForPersistentVolumePhase(client *clientset.Clientset, pvName string, w watch.Interface, phase v1.PersistentVolumePhase) {
// Check if the volume is already in requested phase
volume, err := client.CoreV1().PersistentVolumes().Get(context.TODO(), pvName, metav1.GetOptions{})
Expand Down Expand Up @@ -1106,6 +1300,35 @@ func waitForAnyPersistentVolumeClaimPhase(w watch.Interface, phase v1.Persistent
}
}

func waitForPersistentVolumeClaimStorageClass(t *testing.T, claimName, scName string, w watch.Interface, duration time.Duration) (*v1.PersistentVolumeClaim, bool) {
stopTimer := time.NewTimer(duration)
defer stopTimer.Stop()

// Wait for the storage class
for {
select {
case event := <-w.ResultChan():
claim, ok := event.Object.(*v1.PersistentVolumeClaim)
if ok {
t.Logf("Watching claim %s", claim.Name)
} else {
t.Errorf("Watch closed unexpectedly")
}
if claim.Spec.StorageClassName == nil {
t.Logf("Claim %v does not yet have expected storage class %v", claim.Name, scName)
continue
}
if *claim.Spec.StorageClassName == scName && claim.Name == claimName {
t.Logf("Claim %s now has expected storage class %s", claim.Name, *claim.Spec.StorageClassName)
return claim, true
}
case <-stopTimer.C:
return nil, false
}

}
}

func createClients(namespaceName string, t *testing.T, s *kubeapiservertesting.TestServer, syncPeriod time.Duration) (*clientset.Clientset, *persistentvolumecontroller.PersistentVolumeController, informers.SharedInformerFactory, watch.Interface, watch.Interface) {
// Use higher QPS and Burst, there is a test for race conditions which
// creates many objects and default values were too low.
Expand Down Expand Up @@ -1175,6 +1398,19 @@ func createPV(name, path, cap string, mode []v1.PersistentVolumeAccessMode, recl
}
}

func createPVWithStorageClass(name, path, cap, scName string, mode []v1.PersistentVolumeAccessMode, reclaim v1.PersistentVolumeReclaimPolicy) *v1.PersistentVolume {
return &v1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{Name: name},
Spec: v1.PersistentVolumeSpec{
PersistentVolumeSource: v1.PersistentVolumeSource{HostPath: &v1.HostPathVolumeSource{Path: path}},
Capacity: v1.ResourceList{v1.ResourceName(v1.ResourceStorage): resource.MustParse(cap)},
AccessModes: mode,
PersistentVolumeReclaimPolicy: reclaim,
StorageClassName: scName,
},
}
}

func createPVC(name, namespace, cap string, mode []v1.PersistentVolumeAccessMode, class string) *v1.PersistentVolumeClaim {
return &v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Expand All @@ -1188,3 +1424,16 @@ func createPVC(name, namespace, cap string, mode []v1.PersistentVolumeAccessMode
},
}
}

func createPVCWithNilStorageClass(name, namespace, cap string, mode []v1.PersistentVolumeAccessMode) *v1.PersistentVolumeClaim {
return &v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
},
Spec: v1.PersistentVolumeClaimSpec{
Resources: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceName(v1.ResourceStorage): resource.MustParse(cap)}},
AccessModes: mode,
},
}
}

0 comments on commit 3b945fd

Please sign in to comment.