From 53b3a6c1d9b6ae42106b263889b641b529c18655 Mon Sep 17 00:00:00 2001 From: Matthew Cary Date: Wed, 30 Jun 2021 18:41:55 -0700 Subject: [PATCH] controller change for statefulset auto-delete (tests) Change-Id: I16b50e6853bba65fc89c793d2b9b335581c02407 --- .../statefulset/stateful_pod_control_test.go | 424 ++++++- .../statefulset/stateful_set_control_test.go | 1024 ++++++++++------- .../statefulset/stateful_set_test.go | 457 ++++++-- .../statefulset/stateful_set_utils_test.go | 403 ++++++- test/e2e/apps/statefulset.go | 249 ++++ 5 files changed, 2040 insertions(+), 517 deletions(-) diff --git a/pkg/controller/statefulset/stateful_pod_control_test.go b/pkg/controller/statefulset/stateful_pod_control_test.go index 82e535707ed1..4dcdfe90e3d1 100644 --- a/pkg/controller/statefulset/stateful_pod_control_test.go +++ b/pkg/controller/statefulset/stateful_pod_control_test.go @@ -19,23 +19,27 @@ package statefulset import ( "context" "errors" + "fmt" "strings" "testing" "time" + apps "k8s.io/api/apps/v1" + "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - + "k8s.io/apimachinery/pkg/types" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/client-go/kubernetes/fake" + corelisters "k8s.io/client-go/listers/core/v1" core "k8s.io/client-go/testing" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" - - "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes/fake" - corelisters "k8s.io/client-go/listers/core/v1" + featuregatetesting "k8s.io/component-base/featuregate/testing" _ "k8s.io/kubernetes/pkg/apis/apps/install" _ "k8s.io/kubernetes/pkg/apis/core/install" + "k8s.io/kubernetes/pkg/features" ) func TestStatefulPodControlCreatesPods(t *testing.T) { @@ -43,14 +47,15 @@ func TestStatefulPodControlCreatesPods(t *testing.T) { set := newStatefulSet(3) pod := newStatefulSetPod(set, 0) fakeClient := &fake.Clientset{} - pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) - pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) - control := NewRealStatefulPodControl(fakeClient, nil, nil, pvcLister, recorder) + claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + claimLister := corelisters.NewPersistentVolumeClaimLister(claimIndexer) + control := NewStatefulPodControl(fakeClient, nil, claimLister, recorder) fakeClient.AddReactor("get", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { return true, nil, apierrors.NewNotFound(action.GetResource().GroupResource(), action.GetResource().Resource) }) fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { create := action.(core.CreateAction) + claimIndexer.Add(create.GetObject()) return true, create.GetObject(), nil }) fakeClient.AddReactor("create", "pods", func(action core.Action) (bool, runtime.Object, error) { @@ -83,7 +88,7 @@ func TestStatefulPodControlCreatePodExists(t *testing.T) { pvcIndexer.Add(&pvc) } pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) - control := NewRealStatefulPodControl(fakeClient, nil, nil, pvcLister, recorder) + control := NewStatefulPodControl(fakeClient, nil, pvcLister, recorder) fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { create := action.(core.CreateAction) return true, create.GetObject(), nil @@ -110,7 +115,7 @@ func TestStatefulPodControlCreatePodPvcCreateFailure(t *testing.T) { fakeClient := &fake.Clientset{} pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) - control := NewRealStatefulPodControl(fakeClient, nil, nil, pvcLister, recorder) + control := NewStatefulPodControl(fakeClient, nil, pvcLister, recorder) fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { return true, nil, apierrors.NewInternalError(errors.New("API server down")) }) @@ -131,7 +136,7 @@ func TestStatefulPodControlCreatePodPvcCreateFailure(t *testing.T) { } } } -func TestStatefulPodControlCreatePodPvcDeleting(t *testing.T) { +func TestStatefulPodControlCreatePodPVCDeleting(t *testing.T) { recorder := record.NewFakeRecorder(10) set := newStatefulSet(3) pod := newStatefulSetPod(set, 0) @@ -145,7 +150,7 @@ func TestStatefulPodControlCreatePodPvcDeleting(t *testing.T) { pvcIndexer.Add(&pvc) } pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) - control := NewRealStatefulPodControl(fakeClient, nil, nil, pvcLister, recorder) + control := NewStatefulPodControl(fakeClient, nil, pvcLister, recorder) fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { create := action.(core.CreateAction) return true, create.GetObject(), nil @@ -184,7 +189,7 @@ func TestStatefulPodControlCreatePodPvcGetFailure(t *testing.T) { fakeClient := &fake.Clientset{} pvcIndexer := &fakeIndexer{getError: errors.New("API server down")} pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) - control := NewRealStatefulPodControl(fakeClient, nil, nil, pvcLister, recorder) + control := NewStatefulPodControl(fakeClient, nil, pvcLister, recorder) fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { return true, nil, apierrors.NewInternalError(errors.New("API server down")) }) @@ -213,7 +218,7 @@ func TestStatefulPodControlCreatePodFailed(t *testing.T) { fakeClient := &fake.Clientset{} pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) - control := NewRealStatefulPodControl(fakeClient, nil, nil, pvcLister, recorder) + control := NewStatefulPodControl(fakeClient, nil, pvcLister, recorder) fakeClient.AddReactor("create", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { create := action.(core.CreateAction) return true, create.GetObject(), nil @@ -232,7 +237,6 @@ func TestStatefulPodControlCreatePodFailed(t *testing.T) { } else if !strings.Contains(events[1], v1.EventTypeWarning) { t.Errorf("Found unexpected non-warning event %s", events[1]) - } } @@ -241,7 +245,14 @@ func TestStatefulPodControlNoOpUpdate(t *testing.T) { set := newStatefulSet(3) pod := newStatefulSetPod(set, 0) fakeClient := &fake.Clientset{} - control := NewRealStatefulPodControl(fakeClient, nil, nil, nil, recorder) + claims := getPersistentVolumeClaims(set, pod) + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + for k := range claims { + claim := claims[k] + indexer.Add(&claim) + } + claimLister := corelisters.NewPersistentVolumeClaimLister(indexer) + control := NewStatefulPodControl(fakeClient, nil, claimLister, recorder) fakeClient.AddReactor("*", "*", func(action core.Action) (bool, runtime.Object, error) { t.Error("no-op update should not make any client invocation") return true, nil, apierrors.NewInternalError(errors.New("if we are here we have a problem")) @@ -260,7 +271,9 @@ func TestStatefulPodControlUpdatesIdentity(t *testing.T) { set := newStatefulSet(3) pod := newStatefulSetPod(set, 0) fakeClient := fake.NewSimpleClientset(set, pod) - control := NewRealStatefulPodControl(fakeClient, nil, nil, nil, recorder) + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + claimLister := corelisters.NewPersistentVolumeClaimLister(indexer) + control := NewStatefulPodControl(fakeClient, nil, claimLister, recorder) var updated *v1.Pod fakeClient.PrependReactor("update", "pods", func(action core.Action) (bool, runtime.Object, error) { update := action.(core.UpdateAction) @@ -287,12 +300,14 @@ func TestStatefulPodControlUpdateIdentityFailure(t *testing.T) { set := newStatefulSet(3) pod := newStatefulSetPod(set, 0) fakeClient := &fake.Clientset{} - indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + podIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) gooPod := newStatefulSetPod(set, 0) gooPod.Name = "goo-0" - indexer.Add(gooPod) - podLister := corelisters.NewPodLister(indexer) - control := NewRealStatefulPodControl(fakeClient, nil, podLister, nil, recorder) + podIndexer.Add(gooPod) + podLister := corelisters.NewPodLister(podIndexer) + claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + claimLister := corelisters.NewPersistentVolumeClaimLister(claimIndexer) + control := NewStatefulPodControl(fakeClient, podLister, claimLister, recorder) fakeClient.AddReactor("update", "pods", func(action core.Action) (bool, runtime.Object, error) { pod.Name = "goo-0" return true, nil, apierrors.NewInternalError(errors.New("API server down")) @@ -319,7 +334,7 @@ func TestStatefulPodControlUpdatesPodStorage(t *testing.T) { fakeClient := &fake.Clientset{} pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) - control := NewRealStatefulPodControl(fakeClient, nil, nil, pvcLister, recorder) + control := NewStatefulPodControl(fakeClient, nil, pvcLister, recorder) pvcs := getPersistentVolumeClaims(set, pod) volumes := make([]v1.Volume, 0, len(pod.Spec.Volumes)) for i := range pod.Spec.Volumes { @@ -366,7 +381,7 @@ func TestStatefulPodControlUpdatePodStorageFailure(t *testing.T) { fakeClient := &fake.Clientset{} pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) pvcLister := corelisters.NewPersistentVolumeClaimLister(pvcIndexer) - control := NewRealStatefulPodControl(fakeClient, nil, nil, pvcLister, recorder) + control := NewStatefulPodControl(fakeClient, nil, pvcLister, recorder) pvcs := getPersistentVolumeClaims(set, pod) volumes := make([]v1.Volume, 0, len(pod.Spec.Volumes)) for i := range pod.Spec.Volumes { @@ -401,12 +416,19 @@ func TestStatefulPodControlUpdatePodConflictSuccess(t *testing.T) { set := newStatefulSet(3) pod := newStatefulSetPod(set, 0) fakeClient := &fake.Clientset{} - indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + podIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + podLister := corelisters.NewPodLister(podIndexer) + claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + claimLister := corelisters.NewPersistentVolumeClaimLister(podIndexer) gooPod := newStatefulSetPod(set, 0) - gooPod.Name = "goo-0" - indexer.Add(gooPod) - podLister := corelisters.NewPodLister(indexer) - control := NewRealStatefulPodControl(fakeClient, nil, podLister, nil, recorder) + gooPod.Labels[apps.StatefulSetPodNameLabel] = "goo-starts" + podIndexer.Add(gooPod) + claims := getPersistentVolumeClaims(set, gooPod) + for k := range claims { + claim := claims[k] + claimIndexer.Add(&claim) + } + control := NewStatefulPodControl(fakeClient, podLister, claimLister, recorder) conflict := false fakeClient.AddReactor("update", "pods", func(action core.Action) (bool, runtime.Object, error) { update := action.(core.UpdateAction) @@ -417,7 +439,7 @@ func TestStatefulPodControlUpdatePodConflictSuccess(t *testing.T) { return true, update.GetObject(), nil }) - pod.Name = "goo-0" + pod.Labels[apps.StatefulSetPodNameLabel] = "goo-0" if err := control.UpdateStatefulPod(set, pod); err != nil { t.Errorf("Successful update returned an error: %s", err) } @@ -437,7 +459,7 @@ func TestStatefulPodControlDeletesStatefulPod(t *testing.T) { set := newStatefulSet(3) pod := newStatefulSetPod(set, 0) fakeClient := &fake.Clientset{} - control := NewRealStatefulPodControl(fakeClient, nil, nil, nil, recorder) + control := NewStatefulPodControl(fakeClient, nil, nil, recorder) fakeClient.AddReactor("delete", "pods", func(action core.Action) (bool, runtime.Object, error) { return true, nil, nil }) @@ -457,7 +479,7 @@ func TestStatefulPodControlDeleteFailure(t *testing.T) { set := newStatefulSet(3) pod := newStatefulSetPod(set, 0) fakeClient := &fake.Clientset{} - control := NewRealStatefulPodControl(fakeClient, nil, nil, nil, recorder) + control := NewStatefulPodControl(fakeClient, nil, nil, recorder) fakeClient.AddReactor("delete", "pods", func(action core.Action) (bool, runtime.Object, error) { return true, nil, apierrors.NewInternalError(errors.New("API server down")) }) @@ -472,6 +494,344 @@ func TestStatefulPodControlDeleteFailure(t *testing.T) { } } +func TestStatefulPodControlClaimsMatchDeletionPolcy(t *testing.T) { + // The claimOwnerMatchesSetAndPod is tested exhaustively in stateful_set_utils_test; this + // test is for the wiring to the method tested there. + fakeClient := &fake.Clientset{} + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + claimLister := corelisters.NewPersistentVolumeClaimLister(indexer) + set := newStatefulSet(3) + pod := newStatefulSetPod(set, 0) + claims := getPersistentVolumeClaims(set, pod) + for k := range claims { + claim := claims[k] + indexer.Add(&claim) + } + control := NewStatefulPodControl(fakeClient, nil, claimLister, &noopRecorder{}) + set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + } + if matches, err := control.ClaimsMatchRetentionPolicy(set, pod); err != nil { + t.Errorf("Unexpected error for ClaimsMatchRetentionPolicy (retain): %v", err) + } else if !matches { + t.Error("Unexpected non-match for ClaimsMatchRetentionPolicy (retain)") + } + set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType, + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + } + if matches, err := control.ClaimsMatchRetentionPolicy(set, pod); err != nil { + t.Errorf("Unexpected error for ClaimsMatchRetentionPolicy (set deletion): %v", err) + } else if matches { + t.Error("Unexpected match for ClaimsMatchRetentionPolicy (set deletion)") + } +} + +func TestStatefulPodControlUpdatePodClaimForRetentionPolicy(t *testing.T) { + // All the update conditions are tested exhaustively in stateful_set_utils_test. This + // tests the wiring from the pod control to that method. + testFn := func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, true)() + fakeClient := &fake.Clientset{} + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + claimLister := corelisters.NewPersistentVolumeClaimLister(indexer) + set := newStatefulSet(3) + set.GetObjectMeta().SetUID("set-123") + pod := newStatefulSetPod(set, 0) + claims := getPersistentVolumeClaims(set, pod) + for k := range claims { + claim := claims[k] + indexer.Add(&claim) + } + control := NewStatefulPodControl(fakeClient, nil, claimLister, &noopRecorder{}) + set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType, + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + } + if err := control.UpdatePodClaimForRetentionPolicy(set, pod); err != nil { + t.Errorf("Unexpected error for UpdatePodClaimForRetentionPolicy (retain): %v", err) + } + expectRef := utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoDeletePVC) + for k := range claims { + claim, err := claimLister.PersistentVolumeClaims(claims[k].Namespace).Get(claims[k].Name) + if err != nil { + t.Errorf("Unexpected error getting Claim %s/%s: %v", claim.Namespace, claim.Name, err) + } + if hasOwnerRef(claim, set) != expectRef { + t.Errorf("Claim %s/%s bad set owner ref", claim.Namespace, claim.Name) + } + } + } + t.Run("StatefulSetAutoDeletePVCEnabled", func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, true)() + testFn(t) + }) + t.Run("StatefulSetAutoDeletePVCDisabled", func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, false)() + testFn(t) + }) +} + +func TestPodClaimIsStale(t *testing.T) { + const missing = "missing" + const exists = "exists" + const stale = "stale" + const withRef = "with-ref" + testCases := []struct { + name string + claimStates []string + expected bool + skipPodUID bool + }{ + { + name: "all missing", + claimStates: []string{missing, missing}, + expected: false, + }, + { + name: "no claims", + claimStates: []string{}, + expected: false, + }, + { + name: "exists", + claimStates: []string{missing, exists}, + expected: false, + }, + { + name: "all refs", + claimStates: []string{withRef, withRef}, + expected: false, + }, + { + name: "stale & exists", + claimStates: []string{stale, exists}, + expected: true, + }, + { + name: "stale & missing", + claimStates: []string{stale, missing}, + expected: true, + }, + { + name: "withRef & stale", + claimStates: []string{withRef, stale}, + expected: true, + }, + { + name: "withRef, no UID", + claimStates: []string{withRef}, + skipPodUID: true, + expected: true, + }, + } + for _, tc := range testCases { + set := apps.StatefulSet{} + set.Name = "set" + set.Namespace = "default" + set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType, + } + set.Spec.Selector = &metav1.LabelSelector{MatchLabels: map[string]string{"key": "value"}} + claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + for i, claimState := range tc.claimStates { + claim := v1.PersistentVolumeClaim{} + claim.Name = fmt.Sprintf("claim-%d", i) + set.Spec.VolumeClaimTemplates = append(set.Spec.VolumeClaimTemplates, claim) + claim.Name = fmt.Sprintf("%s-set-3", claim.Name) + claim.Namespace = set.Namespace + switch claimState { + case missing: + // Do nothing, the claim shouldn't exist. + case exists: + claimIndexer.Add(&claim) + case stale: + claim.SetOwnerReferences([]metav1.OwnerReference{ + {Name: "set-3", UID: types.UID("stale")}, + }) + claimIndexer.Add(&claim) + case withRef: + claim.SetOwnerReferences([]metav1.OwnerReference{ + {Name: "set-3", UID: types.UID("123")}, + }) + claimIndexer.Add(&claim) + } + } + pod := v1.Pod{} + pod.Name = "set-3" + if !tc.skipPodUID { + pod.SetUID("123") + } + claimLister := corelisters.NewPersistentVolumeClaimLister(claimIndexer) + control := NewStatefulPodControl(&fake.Clientset{}, nil, claimLister, &noopRecorder{}) + expected := tc.expected + // Note that the error isn't / can't be tested. + if stale, _ := control.PodClaimIsStale(&set, &pod); stale != expected { + t.Errorf("unexpected stale for %s", tc.name) + } + } +} + +func TestStatefulPodControlRetainDeletionPolicyUpdate(t *testing.T) { + testFn := func(t *testing.T) { + recorder := record.NewFakeRecorder(10) + set := newStatefulSet(1) + set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + } + pod := newStatefulSetPod(set, 0) + fakeClient := &fake.Clientset{} + podIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + podLister := corelisters.NewPodLister(podIndexer) + claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + claimLister := corelisters.NewPersistentVolumeClaimLister(claimIndexer) + podIndexer.Add(pod) + claims := getPersistentVolumeClaims(set, pod) + if len(claims) < 1 { + t.Errorf("Unexpected missing PVCs") + } + for k := range claims { + claim := claims[k] + setOwnerRef(&claim, set, &set.TypeMeta) // This ownerRef should be removed in the update. + claimIndexer.Add(&claim) + } + control := NewStatefulPodControl(fakeClient, podLister, claimLister, recorder) + if err := control.UpdateStatefulPod(set, pod); err != nil { + t.Errorf("Successful update returned an error: %s", err) + } + for k := range claims { + claim := claims[k] + if hasOwnerRef(&claim, set) { + t.Errorf("ownerRef not removed: %s/%s", claim.Namespace, claim.Name) + } + } + events := collectEvents(recorder.Events) + if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoDeletePVC) { + if eventCount := len(events); eventCount != 1 { + t.Errorf("delete failed: got %d events, but want 1", eventCount) + } + } else { + if len(events) != 0 { + t.Errorf("delete failed: expected no events, but got %v", events) + } + } + } + t.Run("StatefulSetAutoDeletePVCEnabled", func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, true)() + testFn(t) + }) + t.Run("StatefulSetAutoDeletePVCDisabled", func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, false)() + testFn(t) + }) +} + +func TestStatefulPodControlRetentionPolicyUpdate(t *testing.T) { + // Only applicable when the feature gate is on; the off case is tested in TestStatefulPodControlRetainRetentionPolicyUpdate. + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, true)() + + recorder := record.NewFakeRecorder(10) + set := newStatefulSet(1) + set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType, + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + } + pod := newStatefulSetPod(set, 0) + fakeClient := &fake.Clientset{} + podIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + podIndexer.Add(pod) + claims := getPersistentVolumeClaims(set, pod) + if len(claims) != 1 { + t.Errorf("Unexpected or missing PVCs") + } + var claim v1.PersistentVolumeClaim + for k := range claims { + claim = claims[k] + claimIndexer.Add(&claim) + } + fakeClient.AddReactor("update", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { + update := action.(core.UpdateAction) + claimIndexer.Update(update.GetObject()) + return true, update.GetObject(), nil + }) + podLister := corelisters.NewPodLister(podIndexer) + claimLister := corelisters.NewPersistentVolumeClaimLister(claimIndexer) + control := NewStatefulPodControl(fakeClient, podLister, claimLister, recorder) + if err := control.UpdateStatefulPod(set, pod); err != nil { + t.Errorf("Successful update returned an error: %s", err) + } + updatedClaim, err := claimLister.PersistentVolumeClaims(claim.Namespace).Get(claim.Name) + if err != nil { + t.Errorf("Error retrieving claim %s/%s: %v", claim.Namespace, claim.Name, err) + } + if !hasOwnerRef(updatedClaim, set) { + t.Errorf("ownerRef not added: %s/%s", claim.Namespace, claim.Name) + } + events := collectEvents(recorder.Events) + if eventCount := len(events); eventCount != 1 { + t.Errorf("update failed: got %d events, but want 1", eventCount) + } +} + +func TestStatefulPodControlRetentionPolicyUpdateMissingClaims(t *testing.T) { + // Only applicable when the feature gate is on. + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, true)() + + recorder := record.NewFakeRecorder(10) + set := newStatefulSet(1) + set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType, + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + } + pod := newStatefulSetPod(set, 0) + fakeClient := &fake.Clientset{} + podIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + podLister := corelisters.NewPodLister(podIndexer) + claimIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + claimLister := corelisters.NewPersistentVolumeClaimLister(claimIndexer) + podIndexer.Add(pod) + fakeClient.AddReactor("update", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) { + update := action.(core.UpdateAction) + claimIndexer.Update(update.GetObject()) + return true, update.GetObject(), nil + }) + control := NewStatefulPodControl(fakeClient, podLister, claimLister, recorder) + if err := control.UpdateStatefulPod(set, pod); err != nil { + t.Error("Unexpected error on pod update when PVCs are missing") + } + claims := getPersistentVolumeClaims(set, pod) + if len(claims) != 1 { + t.Errorf("Unexpected or missing PVCs") + } + var claim v1.PersistentVolumeClaim + for k := range claims { + claim = claims[k] + claimIndexer.Add(&claim) + } + + if err := control.UpdateStatefulPod(set, pod); err != nil { + t.Errorf("Expected update to succeed, saw error %v", err) + } + updatedClaim, err := claimLister.PersistentVolumeClaims(claim.Namespace).Get(claim.Name) + if err != nil { + t.Errorf("Error retrieving claim %s/%s: %v", claim.Namespace, claim.Name, err) + } + if !hasOwnerRef(updatedClaim, set) { + t.Errorf("ownerRef not added: %s/%s", claim.Namespace, claim.Name) + } + events := collectEvents(recorder.Events) + if eventCount := len(events); eventCount != 1 { + t.Errorf("update failed: got %d events, but want 2", eventCount) + } + if !strings.Contains(events[0], "SuccessfulUpdate") { + t.Errorf("expected first event to be a successful update: %s", events[1]) + } +} + func collectEvents(source <-chan string) []string { done := false events := make([]string, 0) diff --git a/pkg/controller/statefulset/stateful_set_control_test.go b/pkg/controller/statefulset/stateful_set_control_test.go index 098cf321f795..f9d65e6da8ad 100644 --- a/pkg/controller/statefulset/stateful_set_control_test.go +++ b/pkg/controller/statefulset/stateful_set_control_test.go @@ -34,17 +34,16 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" utilerrors "k8s.io/apimachinery/pkg/util/errors" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/informers" appsinformers "k8s.io/client-go/informers/apps/v1" - coreinformers "k8s.io/client-go/informers/core/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" appslisters "k8s.io/client-go/listers/apps/v1" corelisters "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" - "k8s.io/client-go/tools/record" featuregatetesting "k8s.io/component-base/featuregate/testing" podutil "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/kubernetes/pkg/controller" @@ -52,13 +51,14 @@ import ( "k8s.io/kubernetes/pkg/features" ) -type invariantFunc func(set *apps.StatefulSet, spc *fakeStatefulPodControl) error +type invariantFunc func(set *apps.StatefulSet, om *fakeObjectManager) error -func setupController(client clientset.Interface) (*fakeStatefulPodControl, *fakeStatefulSetStatusUpdater, StatefulSetControlInterface, chan struct{}) { +func setupController(client clientset.Interface) (*fakeObjectManager, *fakeStatefulSetStatusUpdater, StatefulSetControlInterface, chan struct{}) { informerFactory := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc()) - spc := newFakeStatefulPodControl(informerFactory.Core().V1().Pods(), informerFactory.Apps().V1().StatefulSets(), informerFactory.Apps().V1().ControllerRevisions()) + om := newFakeObjectManager(informerFactory) + spc := NewStatefulPodControlFromManager(om, &noopRecorder{}) ssu := newFakeStatefulSetStatusUpdater(informerFactory.Apps().V1().StatefulSets()) - recorder := record.NewFakeRecorder(10) + recorder := &noopRecorder{} ssc := NewDefaultStatefulSetControl(spc, ssu, history.NewFakeHistory(informerFactory.Apps().V1().ControllerRevisions()), recorder) stop := make(chan struct{}) @@ -69,7 +69,7 @@ func setupController(client clientset.Interface) (*fakeStatefulPodControl, *fake informerFactory.Core().V1().Pods().Informer().HasSynced, informerFactory.Apps().V1().ControllerRevisions().Informer().HasSynced, ) - return spc, ssu, ssc, stop + return om, ssu, ssc, stop } func burst(set *apps.StatefulSet) *apps.StatefulSet { @@ -82,6 +82,64 @@ func setMinReadySeconds(set *apps.StatefulSet, minReadySeconds int32) *apps.Stat return set } +func runTestOverPVCRetentionPolicies(t *testing.T, testName string, testFn func(*testing.T, *apps.StatefulSetPersistentVolumeClaimRetentionPolicy)) { + subtestName := "StatefulSetAutoDeletePVCDisabled" + if testName != "" { + subtestName = fmt.Sprintf("%s/%s", testName, subtestName) + } + t.Run(subtestName, func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, false)() + testFn(t, &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + }) + }) + + for _, policy := range []*apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + { + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + }, + { + WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + }, + { + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType, + }, + { + WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType, + }, + } { + subtestName := pvcDeletePolicyString(policy) + "/StatefulSetAutoDeletePVCEnabled" + if testName != "" { + subtestName = fmt.Sprintf("%s/%s", testName, subtestName) + } + t.Run(subtestName, func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, true)() + testFn(t, policy) + }) + } +} + +func pvcDeletePolicyString(policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) string { + const retain = apps.RetainPersistentVolumeClaimRetentionPolicyType + const delete = apps.DeletePersistentVolumeClaimRetentionPolicyType + switch { + case policy.WhenScaled == retain && policy.WhenDeleted == retain: + return "Retain" + case policy.WhenScaled == retain && policy.WhenDeleted == delete: + return "SetDeleteOnly" + case policy.WhenScaled == delete && policy.WhenDeleted == retain: + return "ScaleDownOnly" + case policy.WhenScaled == delete && policy.WhenDeleted == delete: + return "Delete" + } + return "invalid" +} + func TestStatefulSetControl(t *testing.T) { simpleSetFn := func() *apps.StatefulSet { return newStatefulSet(3) } largeSetFn := func() *apps.StatefulSet { return newStatefulSet(5) } @@ -106,17 +164,24 @@ func TestStatefulSetControl(t *testing.T) { if i := strings.LastIndex(fnName, "."); i != -1 { fnName = fnName[i+1:] } - t.Run( + testObj := testCase.obj + testFn := testCase.fn + runTestOverPVCRetentionPolicies( + t, fmt.Sprintf("%s/Monotonic", fnName), - func(t *testing.T) { - testCase.fn(t, testCase.obj(), assertMonotonicInvariants) + func(t *testing.T, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { + set := testObj() + set.Spec.PersistentVolumeClaimRetentionPolicy = policy + testFn(t, set, assertMonotonicInvariants) }, ) - t.Run( + runTestOverPVCRetentionPolicies( + t, fmt.Sprintf("%s/Burst", fnName), - func(t *testing.T) { - set := burst(testCase.obj()) - testCase.fn(t, set, assertBurstInvariants) + func(t *testing.T, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { + set := burst(testObj()) + set.Spec.PersistentVolumeClaimRetentionPolicy = policy + testFn(t, set, assertBurstInvariants) }, ) } @@ -124,14 +189,14 @@ func TestStatefulSetControl(t *testing.T) { func CreatesPods(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) { client := fake.NewSimpleClientset(set) - spc, _, ssc, stop := setupController(client) + om, _, ssc, stop := setupController(client) defer close(stop) - if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil { + if err := scaleUpStatefulSetControl(set, ssc, om, invariants); err != nil { t.Errorf("Failed to turn up StatefulSet : %s", err) } var err error - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("Error getting updated StatefulSet: %v", err) } @@ -148,18 +213,18 @@ func CreatesPods(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) func ScalesUp(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) { client := fake.NewSimpleClientset(set) - spc, _, ssc, stop := setupController(client) + om, _, ssc, stop := setupController(client) defer close(stop) - if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil { + if err := scaleUpStatefulSetControl(set, ssc, om, invariants); err != nil { t.Errorf("Failed to turn up StatefulSet : %s", err) } *set.Spec.Replicas = 4 - if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil { + if err := scaleUpStatefulSetControl(set, ssc, om, invariants); err != nil { t.Errorf("Failed to scale StatefulSet : %s", err) } var err error - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("Error getting updated StatefulSet: %v", err) } @@ -176,16 +241,27 @@ func ScalesUp(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) { func ScalesDown(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) { client := fake.NewSimpleClientset(set) - spc, _, ssc, stop := setupController(client) + om, _, ssc, stop := setupController(client) defer close(stop) - if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil { + if err := scaleUpStatefulSetControl(set, ssc, om, invariants); err != nil { t.Errorf("Failed to turn up StatefulSet : %s", err) } + var err error + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated StatefulSet: %v", err) + } *set.Spec.Replicas = 0 - if err := scaleDownStatefulSetControl(set, ssc, spc, invariants); err != nil { + if err := scaleDownStatefulSetControl(set, ssc, om, invariants); err != nil { t.Errorf("Failed to scale StatefulSet : %s", err) } + + // Check updated set. + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated StatefulSet: %v", err) + } if set.Status.Replicas != 0 { t.Error("Failed to scale statefulset to 0 replicas") } @@ -199,14 +275,14 @@ func ScalesDown(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) { func ReplacesPods(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) { client := fake.NewSimpleClientset(set) - spc, _, ssc, stop := setupController(client) + om, _, ssc, stop := setupController(client) defer close(stop) - if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil { + if err := scaleUpStatefulSetControl(set, ssc, om, invariants); err != nil { t.Errorf("Failed to turn up StatefulSet : %s", err) } var err error - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("Error getting updated StatefulSet: %v", err) } @@ -217,48 +293,62 @@ func ReplacesPods(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) if err != nil { t.Error(err) } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + claims, err := om.claimsLister.PersistentVolumeClaims(set.Namespace).List(selector) if err != nil { t.Error(err) } + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Error(err) + } + for _, pod := range pods { + podClaims := getPersistentVolumeClaims(set, pod) + for _, claim := range claims { + if _, found := podClaims[claim.Name]; found { + if hasOwnerRef(claim, pod) { + t.Errorf("Unexpected ownerRef on %s", claim.Name) + } + } + } + } sort.Sort(ascendingOrdinal(pods)) - spc.podsIndexer.Delete(pods[0]) - spc.podsIndexer.Delete(pods[2]) - spc.podsIndexer.Delete(pods[4]) + om.podsIndexer.Delete(pods[0]) + om.podsIndexer.Delete(pods[2]) + om.podsIndexer.Delete(pods[4]) for i := 0; i < 5; i += 2 { - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Error(err) } if _, err = ssc.UpdateStatefulSet(context.TODO(), set, pods); err != nil { t.Errorf("Failed to update StatefulSet : %s", err) } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("Error getting updated StatefulSet: %v", err) } - if pods, err = spc.setPodRunning(set, i); err != nil { + if pods, err = om.setPodRunning(set, i); err != nil { t.Error(err) } if _, err = ssc.UpdateStatefulSet(context.TODO(), set, pods); err != nil { t.Errorf("Failed to update StatefulSet : %s", err) } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("Error getting updated StatefulSet: %v", err) } - if _, err = spc.setPodReady(set, i); err != nil { + if _, err = om.setPodReady(set, i); err != nil { t.Error(err) } } - pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + pods, err = om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Error(err) } if _, err := ssc.UpdateStatefulSet(context.TODO(), set, pods); err != nil { t.Errorf("Failed to update StatefulSet : %s", err) } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("Error getting updated StatefulSet: %v", err) } @@ -269,35 +359,35 @@ func ReplacesPods(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) func RecreatesFailedPod(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) { client := fake.NewSimpleClientset() - spc, _, ssc, stop := setupController(client) + om, _, ssc, stop := setupController(client) defer close(stop) selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { t.Error(err) } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Error(err) } if _, err := ssc.UpdateStatefulSet(context.TODO(), set, pods); err != nil { t.Errorf("Error updating StatefulSet %s", err) } - if err := invariants(set, spc); err != nil { + if err := invariants(set, om); err != nil { t.Error(err) } - pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + pods, err = om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Error(err) } pods[0].Status.Phase = v1.PodFailed - spc.podsIndexer.Update(pods[0]) + om.podsIndexer.Update(pods[0]) if _, err := ssc.UpdateStatefulSet(context.TODO(), set, pods); err != nil { t.Errorf("Error updating StatefulSet %s", err) } - if err := invariants(set, spc); err != nil { + if err := invariants(set, om); err != nil { t.Error(err) } - pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + pods, err = om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Error(err) } @@ -308,18 +398,23 @@ func RecreatesFailedPod(t *testing.T, set *apps.StatefulSet, invariants invarian func CreatePodFailure(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) { client := fake.NewSimpleClientset(set) - spc, _, ssc, stop := setupController(client) + om, _, ssc, stop := setupController(client) defer close(stop) - spc.SetCreateStatefulPodError(apierrors.NewInternalError(errors.New("API server failed")), 2) + om.SetCreateStatefulPodError(apierrors.NewInternalError(errors.New("API server failed")), 2) - if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil && isOrHasInternalError(err) { + if err := scaleUpStatefulSetControl(set, ssc, om, invariants); err != nil && isOrHasInternalError(err) { t.Errorf("StatefulSetControl did not return InternalError found %s", err) } - if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil { + // Update so set.Status is set for the next scaleUpStatefulSetControl call. + var err error + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated StatefulSet: %v", err) + } + if err := scaleUpStatefulSetControl(set, ssc, om, invariants); err != nil { t.Errorf("Failed to turn up StatefulSet : %s", err) } - var err error - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("Error getting updated StatefulSet: %v", err) } @@ -336,16 +431,16 @@ func CreatePodFailure(t *testing.T, set *apps.StatefulSet, invariants invariantF func UpdatePodFailure(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) { client := fake.NewSimpleClientset(set) - spc, _, ssc, stop := setupController(client) + om, _, ssc, stop := setupController(client) defer close(stop) - spc.SetUpdateStatefulPodError(apierrors.NewInternalError(errors.New("API server failed")), 0) + om.SetUpdateStatefulPodError(apierrors.NewInternalError(errors.New("API server failed")), 0) // have to have 1 successful loop first - if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil { + if err := scaleUpStatefulSetControl(set, ssc, om, invariants); err != nil { t.Fatalf("Unexpected error: %v", err) } var err error - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("Error getting updated StatefulSet: %v", err) } @@ -360,7 +455,7 @@ func UpdatePodFailure(t *testing.T, set *apps.StatefulSet, invariants invariantF } // now mutate a pod's identity - pods, err := spc.podsLister.List(labels.Everything()) + pods, err := om.podsLister.List(labels.Everything()) if err != nil { t.Fatalf("Error listing pods: %v", err) } @@ -369,7 +464,7 @@ func UpdatePodFailure(t *testing.T, set *apps.StatefulSet, invariants invariantF } sort.Sort(ascendingOrdinal(pods)) pods[0].Name = "goo-0" - spc.podsIndexer.Update(pods[0]) + om.podsIndexer.Update(pods[0]) // now it should fail if _, err := ssc.UpdateStatefulSet(context.TODO(), set, pods); err != nil && isOrHasInternalError(err) { @@ -379,18 +474,23 @@ func UpdatePodFailure(t *testing.T, set *apps.StatefulSet, invariants invariantF func UpdateSetStatusFailure(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) { client := fake.NewSimpleClientset(set) - spc, ssu, ssc, stop := setupController(client) + om, ssu, ssc, stop := setupController(client) defer close(stop) ssu.SetUpdateStatefulSetStatusError(apierrors.NewInternalError(errors.New("API server failed")), 2) - if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil && isOrHasInternalError(err) { + if err := scaleUpStatefulSetControl(set, ssc, om, invariants); err != nil && isOrHasInternalError(err) { t.Errorf("StatefulSetControl did not return InternalError found %s", err) } - if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil { + // Update so set.Status is set for the next scaleUpStatefulSetControl call. + var err error + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated StatefulSet: %v", err) + } + if err := scaleUpStatefulSetControl(set, ssc, om, invariants); err != nil { t.Errorf("Failed to turn up StatefulSet : %s", err) } - var err error - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("Error getting updated StatefulSet: %v", err) } @@ -407,43 +507,43 @@ func UpdateSetStatusFailure(t *testing.T, set *apps.StatefulSet, invariants inva func PodRecreateDeleteFailure(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) { client := fake.NewSimpleClientset(set) - spc, _, ssc, stop := setupController(client) + om, _, ssc, stop := setupController(client) defer close(stop) selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { t.Error(err) } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Error(err) } if _, err := ssc.UpdateStatefulSet(context.TODO(), set, pods); err != nil { t.Errorf("Error updating StatefulSet %s", err) } - if err := invariants(set, spc); err != nil { + if err := invariants(set, om); err != nil { t.Error(err) } - pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + pods, err = om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Error(err) } pods[0].Status.Phase = v1.PodFailed - spc.podsIndexer.Update(pods[0]) - spc.SetDeleteStatefulPodError(apierrors.NewInternalError(errors.New("API server failed")), 0) + om.podsIndexer.Update(pods[0]) + om.SetDeleteStatefulPodError(apierrors.NewInternalError(errors.New("API server failed")), 0) if _, err := ssc.UpdateStatefulSet(context.TODO(), set, pods); err != nil && isOrHasInternalError(err) { t.Errorf("StatefulSet failed to %s", err) } - if err := invariants(set, spc); err != nil { + if err := invariants(set, om); err != nil { t.Error(err) } if _, err := ssc.UpdateStatefulSet(context.TODO(), set, pods); err != nil { t.Errorf("Error updating StatefulSet %s", err) } - if err := invariants(set, spc); err != nil { + if err := invariants(set, om); err != nil { t.Error(err) } - pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + pods, err = om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Error(err) } @@ -453,41 +553,49 @@ func PodRecreateDeleteFailure(t *testing.T, set *apps.StatefulSet, invariants in } func TestStatefulSetControlScaleDownDeleteError(t *testing.T) { - invariants := assertMonotonicInvariants - set := newStatefulSet(3) - client := fake.NewSimpleClientset(set) - spc, _, ssc, stop := setupController(client) - defer close(stop) - - if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil { - t.Errorf("Failed to turn up StatefulSet : %s", err) - } - var err error - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) - if err != nil { - t.Fatalf("Error getting updated StatefulSet: %v", err) - } - *set.Spec.Replicas = 0 - spc.SetDeleteStatefulPodError(apierrors.NewInternalError(errors.New("API server failed")), 2) - if err := scaleDownStatefulSetControl(set, ssc, spc, invariants); err != nil && isOrHasInternalError(err) { - t.Errorf("StatefulSetControl failed to throw error on delete %s", err) - } - if err := scaleDownStatefulSetControl(set, ssc, spc, invariants); err != nil { - t.Errorf("Failed to turn down StatefulSet %s", err) - } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) - if err != nil { - t.Fatalf("Error getting updated StatefulSet: %v", err) - } - if set.Status.Replicas != 0 { - t.Error("Failed to scale statefulset to 0 replicas") - } - if set.Status.ReadyReplicas != 0 { - t.Error("Failed to set readyReplicas to 0") - } - if set.Status.UpdatedReplicas != 0 { - t.Error("Failed to set updatedReplicas to 0") - } + runTestOverPVCRetentionPolicies( + t, "", func(t *testing.T, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { + set := newStatefulSet(3) + set.Spec.PersistentVolumeClaimRetentionPolicy = policy + invariants := assertMonotonicInvariants + client := fake.NewSimpleClientset(set) + om, _, ssc, stop := setupController(client) + defer close(stop) + + if err := scaleUpStatefulSetControl(set, ssc, om, invariants); err != nil { + t.Errorf("Failed to turn up StatefulSet : %s", err) + } + var err error + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated StatefulSet: %v", err) + } + *set.Spec.Replicas = 0 + om.SetDeleteStatefulPodError(apierrors.NewInternalError(errors.New("API server failed")), 2) + if err := scaleDownStatefulSetControl(set, ssc, om, invariants); err != nil && isOrHasInternalError(err) { + t.Errorf("StatefulSetControl failed to throw error on delete %s", err) + } + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated StatefulSet: %v", err) + } + if err := scaleDownStatefulSetControl(set, ssc, om, invariants); err != nil { + t.Errorf("Failed to turn down StatefulSet %s", err) + } + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated StatefulSet: %v", err) + } + if set.Status.Replicas != 0 { + t.Error("Failed to scale statefulset to 0 replicas") + } + if set.Status.ReadyReplicas != 0 { + t.Error("Failed to set readyReplicas to 0") + } + if set.Status.UpdatedReplicas != 0 { + t.Error("Failed to set updatedReplicas to 0") + } + }) } func TestStatefulSetControl_getSetRevisions(t *testing.T) { @@ -504,9 +612,9 @@ func TestStatefulSetControl_getSetRevisions(t *testing.T) { testFn := func(test *testcase, t *testing.T) { client := fake.NewSimpleClientset() informerFactory := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc()) - spc := newFakeStatefulPodControl(informerFactory.Core().V1().Pods(), informerFactory.Apps().V1().StatefulSets(), informerFactory.Apps().V1().ControllerRevisions()) + spc := NewStatefulPodControlFromManager(newFakeObjectManager(informerFactory), &noopRecorder{}) ssu := newFakeStatefulSetStatusUpdater(informerFactory.Apps().V1().StatefulSets()) - recorder := record.NewFakeRecorder(10) + recorder := &noopRecorder{} ssc := defaultStatefulSetControl{spc, ssu, history.NewFakeHistory(informerFactory.Apps().V1().ControllerRevisions()), recorder} stop := make(chan struct{}) @@ -560,66 +668,70 @@ func TestStatefulSetControl_getSetRevisions(t *testing.T) { return clone } - set := newStatefulSet(3) - set.Status.CollisionCount = new(int32) - rev0 := newRevisionOrDie(set, 1) - set1 := set.DeepCopy() - set1.Spec.Template.Spec.Containers[0].Image = "foo" - set1.Status.CurrentRevision = rev0.Name - set1.Status.CollisionCount = new(int32) - rev1 := newRevisionOrDie(set1, 2) - set2 := set1.DeepCopy() - set2.Spec.Template.Labels["new"] = "label" - set2.Status.CurrentRevision = rev0.Name - set2.Status.CollisionCount = new(int32) - rev2 := newRevisionOrDie(set2, 3) - tests := []testcase{ - { - name: "creates initial revision", - existing: nil, - set: set, - expectedCount: 1, - expectedCurrent: rev0, - expectedUpdate: rev0, - err: false, - }, - { - name: "creates revision on update", - existing: []*apps.ControllerRevision{rev0}, - set: set1, - expectedCount: 2, - expectedCurrent: rev0, - expectedUpdate: rev1, - err: false, - }, - { - name: "must not recreate a new revision of same set", - existing: []*apps.ControllerRevision{rev0, rev1}, - set: set1, - expectedCount: 2, - expectedCurrent: rev0, - expectedUpdate: rev1, - err: false, - }, - { - name: "must rollback to a previous revision", - existing: []*apps.ControllerRevision{rev0, rev1, rev2}, - set: set1, - expectedCount: 3, - expectedCurrent: rev0, - expectedUpdate: updateRevision(rev1, 4), - err: false, - }, - } - for i := range tests { - testFn(&tests[i], t) - } + runTestOverPVCRetentionPolicies( + t, "", func(t *testing.T, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { + set := newStatefulSet(3) + set.Spec.PersistentVolumeClaimRetentionPolicy = policy + set.Status.CollisionCount = new(int32) + rev0 := newRevisionOrDie(set, 1) + set1 := set.DeepCopy() + set1.Spec.Template.Spec.Containers[0].Image = "foo" + set1.Status.CurrentRevision = rev0.Name + set1.Status.CollisionCount = new(int32) + rev1 := newRevisionOrDie(set1, 2) + set2 := set1.DeepCopy() + set2.Spec.Template.Labels["new"] = "label" + set2.Status.CurrentRevision = rev0.Name + set2.Status.CollisionCount = new(int32) + rev2 := newRevisionOrDie(set2, 3) + tests := []testcase{ + { + name: "creates initial revision", + existing: nil, + set: set, + expectedCount: 1, + expectedCurrent: rev0, + expectedUpdate: rev0, + err: false, + }, + { + name: "creates revision on update", + existing: []*apps.ControllerRevision{rev0}, + set: set1, + expectedCount: 2, + expectedCurrent: rev0, + expectedUpdate: rev1, + err: false, + }, + { + name: "must not recreate a new revision of same set", + existing: []*apps.ControllerRevision{rev0, rev1}, + set: set1, + expectedCount: 2, + expectedCurrent: rev0, + expectedUpdate: rev1, + err: false, + }, + { + name: "must rollback to a previous revision", + existing: []*apps.ControllerRevision{rev0, rev1, rev2}, + set: set1, + expectedCount: 3, + expectedCurrent: rev0, + expectedUpdate: updateRevision(rev1, 4), + err: false, + }, + } + for i := range tests { + testFn(&tests[i], t) + } + }) } func TestStatefulSetControlRollingUpdate(t *testing.T) { type testcase struct { name string - invariants func(set *apps.StatefulSet, spc *fakeStatefulPodControl) error + invariants func(set *apps.StatefulSet, om *fakeObjectManager) error initial func() *apps.StatefulSet update func(set *apps.StatefulSet) *apps.StatefulSet validate func(set *apps.StatefulSet, pods []*v1.Pod) error @@ -628,28 +740,28 @@ func TestStatefulSetControlRollingUpdate(t *testing.T) { testFn := func(test *testcase, t *testing.T) { set := test.initial() client := fake.NewSimpleClientset(set) - spc, _, ssc, stop := setupController(client) + om, _, ssc, stop := setupController(client) defer close(stop) - if err := scaleUpStatefulSetControl(set, ssc, spc, test.invariants); err != nil { + if err := scaleUpStatefulSetControl(set, ssc, om, test.invariants); err != nil { t.Fatalf("%s: %s", test.name, err) } - set, err := spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err := om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("%s: %s", test.name, err) } set = test.update(set) - if err := updateStatefulSetControl(set, ssc, spc, assertUpdateInvariants); err != nil { + if err := updateStatefulSetControl(set, ssc, om, assertUpdateInvariants); err != nil { t.Fatalf("%s: %s", test.name, err) } selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { t.Fatalf("%s: %s", test.name, err) } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Fatalf("%s: %s", test.name, err) } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("%s: %s", test.name, err) } @@ -792,7 +904,7 @@ func TestStatefulSetControlRollingUpdate(t *testing.T) { func TestStatefulSetControlOnDeleteUpdate(t *testing.T) { type testcase struct { name string - invariants func(set *apps.StatefulSet, spc *fakeStatefulPodControl) error + invariants func(set *apps.StatefulSet, om *fakeObjectManager) error initial func() *apps.StatefulSet update func(set *apps.StatefulSet) *apps.StatefulSet validateUpdate func(set *apps.StatefulSet, pods []*v1.Pod) error @@ -801,33 +913,48 @@ func TestStatefulSetControlOnDeleteUpdate(t *testing.T) { originalImage := newStatefulSet(3).Spec.Template.Spec.Containers[0].Image - testFn := func(test *testcase, t *testing.T) { + testFn := func(t *testing.T, test *testcase, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { set := test.initial() + set.Spec.PersistentVolumeClaimRetentionPolicy = policy set.Spec.UpdateStrategy = apps.StatefulSetUpdateStrategy{Type: apps.OnDeleteStatefulSetStrategyType} client := fake.NewSimpleClientset(set) - spc, _, ssc, stop := setupController(client) + om, _, ssc, stop := setupController(client) defer close(stop) - if err := scaleUpStatefulSetControl(set, ssc, spc, test.invariants); err != nil { + if err := scaleUpStatefulSetControl(set, ssc, om, test.invariants); err != nil { t.Fatalf("%s: %s", test.name, err) } - set, err := spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err := om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("%s: %s", test.name, err) } set = test.update(set) - if err := updateStatefulSetControl(set, ssc, spc, assertUpdateInvariants); err != nil { + if err := updateStatefulSetControl(set, ssc, om, assertUpdateInvariants); err != nil { t.Fatalf("%s: %s", test.name, err) } + // Pods may have been deleted in the update. Delete any claims with a pod ownerRef. selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { t.Fatalf("%s: %s", test.name, err) } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + claims, err := om.claimsLister.PersistentVolumeClaims(set.Namespace).List(selector) if err != nil { t.Fatalf("%s: %s", test.name, err) } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + for _, claim := range claims { + for _, ref := range claim.GetOwnerReferences() { + if strings.HasPrefix(ref.Name, "foo-") { + om.claimsIndexer.Delete(claim) + break + } + } + } + + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Fatalf("%s: %s", test.name, err) } @@ -838,24 +965,49 @@ func TestStatefulSetControlOnDeleteUpdate(t *testing.T) { t.Fatalf("%s: %s", test.name, err) } + claims, err = om.claimsLister.PersistentVolumeClaims(set.Namespace).List(selector) + if err != nil { + t.Fatalf("%s: %s", test.name, err) + } + for _, claim := range claims { + for _, ref := range claim.GetOwnerReferences() { + if strings.HasPrefix(ref.Name, "foo-") { + t.Fatalf("Unexpected pod reference on %s: %v", claim.Name, claim.GetOwnerReferences()) + } + } + } + replicas := *set.Spec.Replicas *set.Spec.Replicas = 0 - if err := scaleDownStatefulSetControl(set, ssc, spc, test.invariants); err != nil { + if err := scaleDownStatefulSetControl(set, ssc, om, test.invariants); err != nil { t.Fatalf("%s: %s", test.name, err) } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("%s: %s", test.name, err) } *set.Spec.Replicas = replicas - if err := scaleUpStatefulSetControl(set, ssc, spc, test.invariants); err != nil { + + claims, err = om.claimsLister.PersistentVolumeClaims(set.Namespace).List(selector) + if err != nil { t.Fatalf("%s: %s", test.name, err) } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + for _, claim := range claims { + for _, ref := range claim.GetOwnerReferences() { + if strings.HasPrefix(ref.Name, "foo-") { + t.Fatalf("Unexpected pod reference on %s: %v", claim.Name, claim.GetOwnerReferences()) + } + } + } + + if err := scaleUpStatefulSetControl(set, ssc, om, test.invariants); err != nil { + t.Fatalf("%s: %s", test.name, err) + } + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("%s: %s", test.name, err) } - pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + pods, err = om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Fatalf("%s: %s", test.name, err) } @@ -1050,23 +1202,26 @@ func TestStatefulSetControlOnDeleteUpdate(t *testing.T) { }, }, } - for i := range tests { - testFn(&tests[i], t) - } + runTestOverPVCRetentionPolicies(t, "", func(t *testing.T, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { + for i := range tests { + testFn(t, &tests[i], policy) + } + }) } func TestStatefulSetControlRollingUpdateWithPartition(t *testing.T) { type testcase struct { name string partition int32 - invariants func(set *apps.StatefulSet, spc *fakeStatefulPodControl) error + invariants func(set *apps.StatefulSet, om *fakeObjectManager) error initial func() *apps.StatefulSet update func(set *apps.StatefulSet) *apps.StatefulSet validate func(set *apps.StatefulSet, pods []*v1.Pod) error } - testFn := func(test *testcase, t *testing.T) { + testFn := func(t *testing.T, test *testcase, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { set := test.initial() + set.Spec.PersistentVolumeClaimRetentionPolicy = policy set.Spec.UpdateStrategy = apps.StatefulSetUpdateStrategy{ Type: apps.RollingUpdateStatefulSetStrategyType, RollingUpdate: func() *apps.RollingUpdateStatefulSetStrategy { @@ -1074,28 +1229,28 @@ func TestStatefulSetControlRollingUpdateWithPartition(t *testing.T) { }(), } client := fake.NewSimpleClientset(set) - spc, _, ssc, stop := setupController(client) + om, _, ssc, stop := setupController(client) defer close(stop) - if err := scaleUpStatefulSetControl(set, ssc, spc, test.invariants); err != nil { + if err := scaleUpStatefulSetControl(set, ssc, om, test.invariants); err != nil { t.Fatalf("%s: %s", test.name, err) } - set, err := spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err := om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("%s: %s", test.name, err) } set = test.update(set) - if err := updateStatefulSetControl(set, ssc, spc, assertUpdateInvariants); err != nil { + if err := updateStatefulSetControl(set, ssc, om, assertUpdateInvariants); err != nil { t.Fatalf("%s: %s", test.name, err) } selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { t.Fatalf("%s: %s", test.name, err) } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Fatalf("%s: %s", test.name, err) } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("%s: %s", test.name, err) } @@ -1206,80 +1361,85 @@ func TestStatefulSetControlRollingUpdateWithPartition(t *testing.T) { }, }, } - for i := range tests { - testFn(&tests[i], t) - } + runTestOverPVCRetentionPolicies(t, "", func(t *testing.T, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { + for i := range tests { + testFn(t, &tests[i], policy) + } + }) } func TestStatefulSetHonorRevisionHistoryLimit(t *testing.T) { - invariants := assertMonotonicInvariants - set := newStatefulSet(3) - client := fake.NewSimpleClientset(set) - spc, ssu, ssc, stop := setupController(client) - defer close(stop) - - if err := scaleUpStatefulSetControl(set, ssc, spc, invariants); err != nil { - t.Errorf("Failed to turn up StatefulSet : %s", err) - } - var err error - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) - if err != nil { - t.Fatalf("Error getting updated StatefulSet: %v", err) - } + runTestOverPVCRetentionPolicies(t, "", func(t *testing.T, policy *apps.StatefulSetPersistentVolumeClaimRetentionPolicy) { + invariants := assertMonotonicInvariants + set := newStatefulSet(3) + set.Spec.PersistentVolumeClaimRetentionPolicy = policy + client := fake.NewSimpleClientset(set) + om, ssu, ssc, stop := setupController(client) + defer close(stop) - for i := 0; i < int(*set.Spec.RevisionHistoryLimit)+5; i++ { - set.Spec.Template.Spec.Containers[0].Image = fmt.Sprintf("foo-%d", i) - ssu.SetUpdateStatefulSetStatusError(apierrors.NewInternalError(errors.New("API server failed")), 2) - updateStatefulSetControl(set, ssc, spc, assertUpdateInvariants) - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) - if err != nil { - t.Fatalf("Error getting updated StatefulSet: %v", err) + if err := scaleUpStatefulSetControl(set, ssc, om, invariants); err != nil { + t.Errorf("Failed to turn up StatefulSet : %s", err) } - revisions, err := ssc.ListRevisions(set) + var err error + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { - t.Fatalf("Error listing revisions: %v", err) + t.Fatalf("Error getting updated StatefulSet: %v", err) } - // the extra 2 revisions are `currentRevision` and `updateRevision` - // They're considered as `live`, and truncateHistory only cleans up non-live revisions - if len(revisions) > int(*set.Spec.RevisionHistoryLimit)+2 { - t.Fatalf("%s: %d greater than limit %d", "", len(revisions), *set.Spec.RevisionHistoryLimit) + + for i := 0; i < int(*set.Spec.RevisionHistoryLimit)+5; i++ { + set.Spec.Template.Spec.Containers[0].Image = fmt.Sprintf("foo-%d", i) + ssu.SetUpdateStatefulSetStatusError(apierrors.NewInternalError(errors.New("API server failed")), 2) + updateStatefulSetControl(set, ssc, om, assertUpdateInvariants) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) + if err != nil { + t.Fatalf("Error getting updated StatefulSet: %v", err) + } + revisions, err := ssc.ListRevisions(set) + if err != nil { + t.Fatalf("Error listing revisions: %v", err) + } + // the extra 2 revisions are `currentRevision` and `updateRevision` + // They're considered as `live`, and truncateHistory only cleans up non-live revisions + if len(revisions) > int(*set.Spec.RevisionHistoryLimit)+2 { + t.Fatalf("%s: %d greater than limit %d", "", len(revisions), *set.Spec.RevisionHistoryLimit) + } } - } + }) } func TestStatefulSetControlLimitsHistory(t *testing.T) { type testcase struct { name string - invariants func(set *apps.StatefulSet, spc *fakeStatefulPodControl) error + invariants func(set *apps.StatefulSet, om *fakeObjectManager) error initial func() *apps.StatefulSet } - testFn := func(test *testcase, t *testing.T) { + testFn := func(t *testing.T, test *testcase) { set := test.initial() client := fake.NewSimpleClientset(set) - spc, _, ssc, stop := setupController(client) + om, _, ssc, stop := setupController(client) defer close(stop) - if err := scaleUpStatefulSetControl(set, ssc, spc, test.invariants); err != nil { + if err := scaleUpStatefulSetControl(set, ssc, om, test.invariants); err != nil { t.Fatalf("%s: %s", test.name, err) } - set, err := spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err := om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("%s: %s", test.name, err) } for i := 0; i < 10; i++ { set.Spec.Template.Spec.Containers[0].Image = fmt.Sprintf("foo-%d", i) - if err := updateStatefulSetControl(set, ssc, spc, assertUpdateInvariants); err != nil { + if err := updateStatefulSetControl(set, ssc, om, assertUpdateInvariants); err != nil { t.Fatalf("%s: %s", test.name, err) } selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { t.Fatalf("%s: %s", test.name, err) } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Fatalf("%s: %s", test.name, err) } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("%s: %s", test.name, err) } @@ -1314,14 +1474,14 @@ func TestStatefulSetControlLimitsHistory(t *testing.T) { }, } for i := range tests { - testFn(&tests[i], t) + testFn(t, &tests[i]) } } func TestStatefulSetControlRollback(t *testing.T) { type testcase struct { name string - invariants func(set *apps.StatefulSet, spc *fakeStatefulPodControl) error + invariants func(set *apps.StatefulSet, om *fakeObjectManager) error initial func() *apps.StatefulSet update func(set *apps.StatefulSet) *apps.StatefulSet validateUpdate func(set *apps.StatefulSet, pods []*v1.Pod) error @@ -1330,31 +1490,31 @@ func TestStatefulSetControlRollback(t *testing.T) { originalImage := newStatefulSet(3).Spec.Template.Spec.Containers[0].Image - testFn := func(test *testcase, t *testing.T) { + testFn := func(t *testing.T, test *testcase) { set := test.initial() client := fake.NewSimpleClientset(set) - spc, _, ssc, stop := setupController(client) + om, _, ssc, stop := setupController(client) defer close(stop) - if err := scaleUpStatefulSetControl(set, ssc, spc, test.invariants); err != nil { + if err := scaleUpStatefulSetControl(set, ssc, om, test.invariants); err != nil { t.Fatalf("%s: %s", test.name, err) } - set, err := spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err := om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("%s: %s", test.name, err) } set = test.update(set) - if err := updateStatefulSetControl(set, ssc, spc, assertUpdateInvariants); err != nil { + if err := updateStatefulSetControl(set, ssc, om, assertUpdateInvariants); err != nil { t.Fatalf("%s: %s", test.name, err) } selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { t.Fatalf("%s: %s", test.name, err) } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Fatalf("%s: %s", test.name, err) } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("%s: %s", test.name, err) } @@ -1370,17 +1530,17 @@ func TestStatefulSetControlRollback(t *testing.T) { if err != nil { t.Fatalf("%s: %s", test.name, err) } - if err := updateStatefulSetControl(set, ssc, spc, assertUpdateInvariants); err != nil { + if err := updateStatefulSetControl(set, ssc, om, assertUpdateInvariants); err != nil { t.Fatalf("%s: %s", test.name, err) } if err != nil { t.Fatalf("%s: %s", test.name, err) } - pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + pods, err = om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Fatalf("%s: %s", test.name, err) } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { t.Fatalf("%s: %s", test.name, err) } @@ -1570,7 +1730,7 @@ func TestStatefulSetControlRollback(t *testing.T) { }, } for i := range tests { - testFn(&tests[i], t) + testFn(t, &tests[i]) } } @@ -1659,7 +1819,7 @@ func (rt *requestTracker) reset() { rt.after = 0 } -type fakeStatefulPodControl struct { +type fakeObjectManager struct { podsLister corelisters.PodLister claimsLister corelisters.PersistentVolumeClaimLister setsLister appslisters.StatefulSetLister @@ -1672,14 +1832,18 @@ type fakeStatefulPodControl struct { deletePodTracker requestTracker } -func newFakeStatefulPodControl(podInformer coreinformers.PodInformer, setInformer appsinformers.StatefulSetInformer, revisionInformer appsinformers.ControllerRevisionInformer) *fakeStatefulPodControl { - claimsIndexer := cache.NewIndexer(controller.KeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) - return &fakeStatefulPodControl{ +func newFakeObjectManager(informerFactory informers.SharedInformerFactory) *fakeObjectManager { + podInformer := informerFactory.Core().V1().Pods() + claimInformer := informerFactory.Core().V1().PersistentVolumeClaims() + setInformer := informerFactory.Apps().V1().StatefulSets() + revisionInformer := informerFactory.Apps().V1().ControllerRevisions() + + return &fakeObjectManager{ podInformer.Lister(), - corelisters.NewPersistentVolumeClaimLister(claimsIndexer), + claimInformer.Lister(), setInformer.Lister(), podInformer.Informer().GetIndexer(), - claimsIndexer, + claimInformer.Informer().GetIndexer(), setInformer.Informer().GetIndexer(), revisionInformer.Informer().GetIndexer(), requestTracker{0, nil, 0}, @@ -1687,27 +1851,82 @@ func newFakeStatefulPodControl(podInformer coreinformers.PodInformer, setInforme requestTracker{0, nil, 0}} } -func (spc *fakeStatefulPodControl) SetCreateStatefulPodError(err error, after int) { - spc.createPodTracker.err = err - spc.createPodTracker.after = after +func (om *fakeObjectManager) CreatePod(ctx context.Context, pod *v1.Pod) error { + defer om.createPodTracker.inc() + if om.createPodTracker.errorReady() { + defer om.createPodTracker.reset() + return om.createPodTracker.err + } + pod.SetUID(types.UID(pod.Name + "-uid")) + return om.podsIndexer.Update(pod) } -func (spc *fakeStatefulPodControl) SetUpdateStatefulPodError(err error, after int) { - spc.updatePodTracker.err = err - spc.updatePodTracker.after = after +func (om *fakeObjectManager) GetPod(namespace, podName string) (*v1.Pod, error) { + return om.podsLister.Pods(namespace).Get(podName) } -func (spc *fakeStatefulPodControl) SetDeleteStatefulPodError(err error, after int) { - spc.deletePodTracker.err = err - spc.deletePodTracker.after = after +func (om *fakeObjectManager) UpdatePod(pod *v1.Pod) error { + return om.podsIndexer.Update(pod) } -func (spc *fakeStatefulPodControl) setPodPending(set *apps.StatefulSet, ordinal int) ([]*v1.Pod, error) { +func (om *fakeObjectManager) DeletePod(pod *v1.Pod) error { + defer om.deletePodTracker.inc() + if om.deletePodTracker.errorReady() { + defer om.deletePodTracker.reset() + return om.deletePodTracker.err + } + if key, err := controller.KeyFunc(pod); err != nil { + return err + } else if obj, found, err := om.podsIndexer.GetByKey(key); err != nil { + return err + } else if found { + return om.podsIndexer.Delete(obj) + } + return nil // Not found, no error in deleting. +} + +func (om *fakeObjectManager) CreateClaim(claim *v1.PersistentVolumeClaim) error { + om.claimsIndexer.Update(claim) + return nil +} + +func (om *fakeObjectManager) GetClaim(namespace, claimName string) (*v1.PersistentVolumeClaim, error) { + return om.claimsLister.PersistentVolumeClaims(namespace).Get(claimName) +} + +func (om *fakeObjectManager) UpdateClaim(claim *v1.PersistentVolumeClaim) error { + // Validate ownerRefs. + refs := claim.GetOwnerReferences() + for _, ref := range refs { + if ref.APIVersion == "" || ref.Kind == "" || ref.Name == "" { + return fmt.Errorf("invalid ownerRefs: %s %v", claim.Name, refs) + } + } + om.claimsIndexer.Update(claim) + return nil +} + +func (om *fakeObjectManager) SetCreateStatefulPodError(err error, after int) { + om.createPodTracker.err = err + om.createPodTracker.after = after +} + +func (om *fakeObjectManager) SetUpdateStatefulPodError(err error, after int) { + om.updatePodTracker.err = err + om.updatePodTracker.after = after +} + +func (om *fakeObjectManager) SetDeleteStatefulPodError(err error, after int) { + om.deletePodTracker.err = err + om.deletePodTracker.after = after +} + +func (om *fakeObjectManager) setPodPending(set *apps.StatefulSet, ordinal int) ([]*v1.Pod, error) { selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { return nil, err } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return nil, err } @@ -1718,16 +1937,16 @@ func (spc *fakeStatefulPodControl) setPodPending(set *apps.StatefulSet, ordinal pod := pods[ordinal].DeepCopy() pod.Status.Phase = v1.PodPending fakeResourceVersion(pod) - spc.podsIndexer.Update(pod) - return spc.podsLister.Pods(set.Namespace).List(selector) + om.podsIndexer.Update(pod) + return om.podsLister.Pods(set.Namespace).List(selector) } -func (spc *fakeStatefulPodControl) setPodRunning(set *apps.StatefulSet, ordinal int) ([]*v1.Pod, error) { +func (om *fakeObjectManager) setPodRunning(set *apps.StatefulSet, ordinal int) ([]*v1.Pod, error) { selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { return nil, err } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return nil, err } @@ -1738,16 +1957,16 @@ func (spc *fakeStatefulPodControl) setPodRunning(set *apps.StatefulSet, ordinal pod := pods[ordinal].DeepCopy() pod.Status.Phase = v1.PodRunning fakeResourceVersion(pod) - spc.podsIndexer.Update(pod) - return spc.podsLister.Pods(set.Namespace).List(selector) + om.podsIndexer.Update(pod) + return om.podsLister.Pods(set.Namespace).List(selector) } -func (spc *fakeStatefulPodControl) setPodReady(set *apps.StatefulSet, ordinal int) ([]*v1.Pod, error) { +func (om *fakeObjectManager) setPodReady(set *apps.StatefulSet, ordinal int) ([]*v1.Pod, error) { selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { return nil, err } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return nil, err } @@ -1759,16 +1978,16 @@ func (spc *fakeStatefulPodControl) setPodReady(set *apps.StatefulSet, ordinal in condition := v1.PodCondition{Type: v1.PodReady, Status: v1.ConditionTrue} podutil.UpdatePodCondition(&pod.Status, &condition) fakeResourceVersion(pod) - spc.podsIndexer.Update(pod) - return spc.podsLister.Pods(set.Namespace).List(selector) + om.podsIndexer.Update(pod) + return om.podsLister.Pods(set.Namespace).List(selector) } -func (spc *fakeStatefulPodControl) setPodAvailable(set *apps.StatefulSet, ordinal int, lastTransitionTime time.Time) ([]*v1.Pod, error) { +func (om *fakeObjectManager) setPodAvailable(set *apps.StatefulSet, ordinal int, lastTransitionTime time.Time) ([]*v1.Pod, error) { selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { return nil, err } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return nil, err } @@ -1792,90 +2011,41 @@ func (spc *fakeStatefulPodControl) setPodAvailable(set *apps.StatefulSet, ordina } podutil.UpdatePodCondition(&pod.Status, &condition) fakeResourceVersion(pod) - spc.podsIndexer.Update(pod) - return spc.podsLister.Pods(set.Namespace).List(selector) + om.podsIndexer.Update(pod) + return om.podsLister.Pods(set.Namespace).List(selector) } -func (spc *fakeStatefulPodControl) addTerminatingPod(set *apps.StatefulSet, ordinal int) ([]*v1.Pod, error) { +func (om *fakeObjectManager) addTerminatingPod(set *apps.StatefulSet, ordinal int) ([]*v1.Pod, error) { pod := newStatefulSetPod(set, ordinal) + pod.SetUID(types.UID(pod.Name + "-uid")) // To match fakeObjectManager.CreatePod pod.Status.Phase = v1.PodRunning deleted := metav1.NewTime(time.Now()) pod.DeletionTimestamp = &deleted condition := v1.PodCondition{Type: v1.PodReady, Status: v1.ConditionTrue} fakeResourceVersion(pod) podutil.UpdatePodCondition(&pod.Status, &condition) - spc.podsIndexer.Update(pod) + om.podsIndexer.Update(pod) selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { return nil, err } - return spc.podsLister.Pods(set.Namespace).List(selector) + return om.podsLister.Pods(set.Namespace).List(selector) } -func (spc *fakeStatefulPodControl) setPodTerminated(set *apps.StatefulSet, ordinal int) ([]*v1.Pod, error) { +func (om *fakeObjectManager) setPodTerminated(set *apps.StatefulSet, ordinal int) ([]*v1.Pod, error) { pod := newStatefulSetPod(set, ordinal) deleted := metav1.NewTime(time.Now()) pod.DeletionTimestamp = &deleted fakeResourceVersion(pod) - spc.podsIndexer.Update(pod) + om.podsIndexer.Update(pod) selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { return nil, err } - return spc.podsLister.Pods(set.Namespace).List(selector) -} - -func (spc *fakeStatefulPodControl) CreateStatefulPod(ctx context.Context, set *apps.StatefulSet, pod *v1.Pod) error { - defer spc.createPodTracker.inc() - if spc.createPodTracker.errorReady() { - defer spc.createPodTracker.reset() - return spc.createPodTracker.err - } - - for _, claim := range getPersistentVolumeClaims(set, pod) { - spc.claimsIndexer.Update(&claim) - } - spc.podsIndexer.Update(pod) - return nil -} - -func (spc *fakeStatefulPodControl) UpdateStatefulPod(set *apps.StatefulSet, pod *v1.Pod) error { - defer spc.updatePodTracker.inc() - if spc.updatePodTracker.errorReady() { - defer spc.updatePodTracker.reset() - return spc.updatePodTracker.err - } - if !identityMatches(set, pod) { - updateIdentity(set, pod) - } - if !storageMatches(set, pod) { - updateStorage(set, pod) - for _, claim := range getPersistentVolumeClaims(set, pod) { - spc.claimsIndexer.Update(&claim) - } - } - spc.podsIndexer.Update(pod) - return nil + return om.podsLister.Pods(set.Namespace).List(selector) } -func (spc *fakeStatefulPodControl) DeleteStatefulPod(set *apps.StatefulSet, pod *v1.Pod) error { - defer spc.deletePodTracker.inc() - if spc.deletePodTracker.errorReady() { - defer spc.deletePodTracker.reset() - return spc.deletePodTracker.err - } - if key, err := controller.KeyFunc(pod); err != nil { - return err - } else if obj, found, err := spc.podsIndexer.GetByKey(key); err != nil { - return err - } else if found { - spc.podsIndexer.Delete(obj) - } - - return nil -} - -var _ StatefulPodControlInterface = &fakeStatefulPodControl{} +var _ StatefulPodControlObjectManager = &fakeObjectManager{} type fakeStatefulSetStatusUpdater struct { setsLister appslisters.StatefulSetLister @@ -1909,12 +2079,12 @@ func (ssu *fakeStatefulSetStatusUpdater) SetUpdateStatefulSetStatusError(err err var _ StatefulSetStatusUpdaterInterface = &fakeStatefulSetStatusUpdater{} -func assertMonotonicInvariants(set *apps.StatefulSet, spc *fakeStatefulPodControl) error { +func assertMonotonicInvariants(set *apps.StatefulSet, om *fakeObjectManager) error { selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { return err } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return err } @@ -1933,13 +2103,10 @@ func assertMonotonicInvariants(set *apps.StatefulSet, spc *fakeStatefulPodContro } for _, claim := range getPersistentVolumeClaims(set, pods[ord]) { - claim, err := spc.claimsLister.PersistentVolumeClaims(set.Namespace).Get(claim.Name) - if err != nil { + claim, _ := om.claimsLister.PersistentVolumeClaims(set.Namespace).Get(claim.Name) + if err := checkClaimInvarients(set, pods[ord], claim, ord); err != nil { return err } - if claim == nil { - return fmt.Errorf("claim %s for Pod %s was not created", claim.Name, pods[ord].Name) - } } if !identityMatches(set, pods[ord]) { @@ -1949,12 +2116,12 @@ func assertMonotonicInvariants(set *apps.StatefulSet, spc *fakeStatefulPodContro return nil } -func assertBurstInvariants(set *apps.StatefulSet, spc *fakeStatefulPodControl) error { +func assertBurstInvariants(set *apps.StatefulSet, om *fakeObjectManager) error { selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { return err } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return err } @@ -1965,12 +2132,12 @@ func assertBurstInvariants(set *apps.StatefulSet, spc *fakeStatefulPodControl) e } for _, claim := range getPersistentVolumeClaims(set, pods[ord]) { - claim, err := spc.claimsLister.PersistentVolumeClaims(set.Namespace).Get(claim.Name) + claim, err := om.claimsLister.PersistentVolumeClaims(set.Namespace).Get(claim.Name) if err != nil { return err } - if claim == nil { - return fmt.Errorf("claim %s for Pod %s was not created", claim.Name, pods[ord].Name) + if err := checkClaimInvarients(set, pods[ord], claim, ord); err != nil { + return err } } @@ -1983,12 +2150,12 @@ func assertBurstInvariants(set *apps.StatefulSet, spc *fakeStatefulPodControl) e return nil } -func assertUpdateInvariants(set *apps.StatefulSet, spc *fakeStatefulPodControl) error { +func assertUpdateInvariants(set *apps.StatefulSet, om *fakeObjectManager) error { selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { return err } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return err } @@ -2000,12 +2167,12 @@ func assertUpdateInvariants(set *apps.StatefulSet, spc *fakeStatefulPodControl) } for _, claim := range getPersistentVolumeClaims(set, pods[ord]) { - claim, err := spc.claimsLister.PersistentVolumeClaims(set.Namespace).Get(claim.Name) + claim, err := om.claimsLister.PersistentVolumeClaims(set.Namespace).Get(claim.Name) if err != nil { return err } - if claim == nil { - return fmt.Errorf("claim %s for Pod %s was not created", claim.Name, pods[ord].Name) + if err := checkClaimInvarients(set, pods[ord], claim, ord); err != nil { + return err } } @@ -2031,6 +2198,68 @@ func assertUpdateInvariants(set *apps.StatefulSet, spc *fakeStatefulPodControl) return nil } +func checkClaimInvarients(set *apps.StatefulSet, pod *v1.Pod, claim *v1.PersistentVolumeClaim, ordinal int) error { + policy := apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + } + if set.Spec.PersistentVolumeClaimRetentionPolicy != nil && utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoDeletePVC) { + policy = *set.Spec.PersistentVolumeClaimRetentionPolicy + } + claimShouldBeRetained := policy.WhenScaled == apps.RetainPersistentVolumeClaimRetentionPolicyType + if claim == nil { + if claimShouldBeRetained { + return fmt.Errorf("claim for Pod %s was not created", pod.Name) + } + return nil // A non-retained claim has no invariants to satisfy. + } + + if pod.Status.Phase != v1.PodRunning || !podutil.IsPodReady(pod) { + // The pod has spun up yet, we do not expect the owner refs on the claim to have been set. + return nil + } + + const retain = apps.RetainPersistentVolumeClaimRetentionPolicyType + const delete = apps.DeletePersistentVolumeClaimRetentionPolicyType + switch { + case policy.WhenScaled == retain && policy.WhenDeleted == retain: + if hasOwnerRef(claim, set) { + return fmt.Errorf("claim %s has unexpected owner ref on %s for StatefulSet retain", claim.Name, set.Name) + } + if hasOwnerRef(claim, pod) { + return fmt.Errorf("claim %s has unexpected owner ref on pod %s for StatefulSet retain", claim.Name, pod.Name) + } + case policy.WhenScaled == retain && policy.WhenDeleted == delete: + if !hasOwnerRef(claim, set) { + return fmt.Errorf("claim %s does not have owner ref on %s for StatefulSet deletion", claim.Name, set.Name) + } + if hasOwnerRef(claim, pod) { + return fmt.Errorf("claim %s has unexpected owner ref on pod %s for StatefulSet deletion", claim.Name, pod.Name) + } + case policy.WhenScaled == delete && policy.WhenDeleted == retain: + if hasOwnerRef(claim, set) { + return fmt.Errorf("claim %s has unexpected owner ref on %s for scaledown only", claim.Name, set.Name) + } + if ordinal >= int(*set.Spec.Replicas) && !hasOwnerRef(claim, pod) { + return fmt.Errorf("claim %s does not have owner ref on condemned pod %s for scaledown delete", claim.Name, pod.Name) + } + if ordinal < int(*set.Spec.Replicas) && hasOwnerRef(claim, pod) { + return fmt.Errorf("claim %s has unexpected owner ref on condemned pod %s for scaledown delete", claim.Name, pod.Name) + } + case policy.WhenScaled == delete && policy.WhenDeleted == delete: + if ordinal >= int(*set.Spec.Replicas) { + if !hasOwnerRef(claim, pod) || hasOwnerRef(claim, set) { + return fmt.Errorf("condemned claim %s has bad owner refs: %v", claim.Name, claim.GetOwnerReferences()) + } + } else { + if hasOwnerRef(claim, pod) || !hasOwnerRef(claim, set) { + return fmt.Errorf("live claim %s has bad owner refs: %v", claim.Name, claim.GetOwnerReferences()) + } + } + } + return nil +} + func fakeResourceVersion(object interface{}) { obj, isObj := object.(metav1.Object) if !isObj { @@ -2045,32 +2274,28 @@ func fakeResourceVersion(object interface{}) { func scaleUpStatefulSetControl(set *apps.StatefulSet, ssc StatefulSetControlInterface, - spc *fakeStatefulPodControl, + om *fakeObjectManager, invariants invariantFunc) error { selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { return err } for set.Status.ReadyReplicas < *set.Spec.Replicas { - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return err } sort.Sort(ascendingOrdinal(pods)) // ensure all pods are valid (have a phase) - initialized := false for ord, pod := range pods { if pod.Status.Phase == "" { - if pods, err = spc.setPodPending(set, ord); err != nil { + if pods, err = om.setPodPending(set, ord); err != nil { return err } break } } - if initialized { - continue - } // select one of the pods and move it forward in status if len(pods) > 0 { @@ -2078,11 +2303,11 @@ func scaleUpStatefulSetControl(set *apps.StatefulSet, pod := pods[ord] switch pod.Status.Phase { case v1.PodPending: - if pods, err = spc.setPodRunning(set, ord); err != nil { + if pods, err = om.setPodRunning(set, ord); err != nil { return err } case v1.PodRunning: - if pods, err = spc.setPodReady(set, ord); err != nil { + if pods, err = om.setPodReady(set, ord); err != nil { return err } default: @@ -2094,25 +2319,26 @@ func scaleUpStatefulSetControl(set *apps.StatefulSet, if err != nil { return err } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { return err } - if err := invariants(set, spc); err != nil { + if err := invariants(set, om); err != nil { return err } //fmt.Printf("Ravig pod conditions %v %v", set.Status.ReadyReplicas, *set.Spec.Replicas) } - return invariants(set, spc) + return invariants(set, om) } -func scaleDownStatefulSetControl(set *apps.StatefulSet, ssc StatefulSetControlInterface, spc *fakeStatefulPodControl, invariants invariantFunc) error { +func scaleDownStatefulSetControl(set *apps.StatefulSet, ssc StatefulSetControlInterface, om *fakeObjectManager, invariants invariantFunc) error { selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { return err } + for set.Status.Replicas > *set.Spec.Replicas { - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return err } @@ -2121,42 +2347,72 @@ func scaleDownStatefulSetControl(set *apps.StatefulSet, ssc StatefulSetControlIn if _, err := ssc.UpdateStatefulSet(context.TODO(), set, pods); err != nil { return err } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { return err } - if pods, err = spc.addTerminatingPod(set, ordinal); err != nil { + if pods, err = om.addTerminatingPod(set, ordinal); err != nil { return err } if _, err = ssc.UpdateStatefulSet(context.TODO(), set, pods); err != nil { return err } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { return err } - pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + pods, err = om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return err } sort.Sort(ascendingOrdinal(pods)) if len(pods) > 0 { - spc.podsIndexer.Delete(pods[len(pods)-1]) + om.podsIndexer.Delete(pods[len(pods)-1]) } } if _, err := ssc.UpdateStatefulSet(context.TODO(), set, pods); err != nil { return err } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { return err } - if err := invariants(set, spc); err != nil { + + if err := invariants(set, om); err != nil { return err } } - return invariants(set, spc) + // If there are claims with ownerRefs on pods that have been deleted, delete them. + pods, err := om.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + return err + } + currentPods := map[string]bool{} + for _, pod := range pods { + currentPods[pod.Name] = true + } + claims, err := om.claimsLister.PersistentVolumeClaims(set.Namespace).List(selector) + if err != nil { + return err + } + for _, claim := range claims { + claimPodName := getClaimPodName(set, claim) + if claimPodName == "" { + continue // Skip claims not related to a stateful set pod. + } + if _, found := currentPods[claimPodName]; found { + continue // Skip claims which still have a current pod. + } + for _, refs := range claim.GetOwnerReferences() { + if refs.Name == claimPodName { + om.claimsIndexer.Delete(claim) + break + } + } + } + + return invariants(set, om) } func updateComplete(set *apps.StatefulSet, pods []*v1.Pod) bool { @@ -2198,13 +2454,13 @@ func updateComplete(set *apps.StatefulSet, pods []*v1.Pod) bool { func updateStatefulSetControl(set *apps.StatefulSet, ssc StatefulSetControlInterface, - spc *fakeStatefulPodControl, + om *fakeObjectManager, invariants invariantFunc) error { selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { return err } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return err } @@ -2212,16 +2468,16 @@ func updateStatefulSetControl(set *apps.StatefulSet, return err } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { return err } - pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + pods, err = om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return err } for !updateComplete(set, pods) { - pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + pods, err = om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return err } @@ -2229,7 +2485,7 @@ func updateStatefulSetControl(set *apps.StatefulSet, initialized := false for ord, pod := range pods { if pod.Status.Phase == "" { - if pods, err = spc.setPodPending(set, ord); err != nil { + if pods, err = om.setPodPending(set, ord); err != nil { return err } break @@ -2244,11 +2500,11 @@ func updateStatefulSetControl(set *apps.StatefulSet, pod := pods[ord] switch pod.Status.Phase { case v1.PodPending: - if pods, err = spc.setPodRunning(set, ord); err != nil { + if pods, err = om.setPodRunning(set, ord); err != nil { return err } case v1.PodRunning: - if pods, err = spc.setPodReady(set, ord); err != nil { + if pods, err = om.setPodReady(set, ord); err != nil { return err } default: @@ -2259,19 +2515,19 @@ func updateStatefulSetControl(set *apps.StatefulSet, if _, err = ssc.UpdateStatefulSet(context.TODO(), set, pods); err != nil { return err } - set, err = spc.setsLister.StatefulSets(set.Namespace).Get(set.Name) + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) if err != nil { return err } - if err := invariants(set, spc); err != nil { + if err := invariants(set, om); err != nil { return err } - pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + pods, err = om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return err } } - return invariants(set, spc) + return invariants(set, om) } func newRevisionOrDie(set *apps.StatefulSet, revision int64) *apps.ControllerRevision { diff --git a/pkg/controller/statefulset/stateful_set_test.go b/pkg/controller/statefulset/stateful_set_test.go index 31bd9c332058..1f2e98614a0b 100644 --- a/pkg/controller/statefulset/stateful_set_test.go +++ b/pkg/controller/statefulset/stateful_set_test.go @@ -20,21 +20,28 @@ import ( "bytes" "context" "encoding/json" + "fmt" "sort" "testing" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/strategicpatch" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" + core "k8s.io/client-go/testing" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" + featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/controller/history" + "k8s.io/kubernetes/pkg/features" ) var parentKind = apps.SchemeGroupVersion.WithKind("StatefulSet") @@ -43,11 +50,11 @@ func alwaysReady() bool { return true } func TestStatefulSetControllerCreates(t *testing.T) { set := newStatefulSet(3) - ssc, spc, _ := newFakeStatefulSetController(set) - if err := scaleUpStatefulSetController(set, ssc, spc); err != nil { + ssc, spc, om, _ := newFakeStatefulSetController(set) + if err := scaleUpStatefulSetController(set, ssc, spc, om); err != nil { t.Errorf("Failed to turn up StatefulSet : %s", err) } - if obj, _, err := spc.setsIndexer.Get(set); err != nil { + if obj, _, err := om.setsIndexer.Get(set); err != nil { t.Error(err) } else { set = obj.(*apps.StatefulSet) @@ -59,11 +66,11 @@ func TestStatefulSetControllerCreates(t *testing.T) { func TestStatefulSetControllerDeletes(t *testing.T) { set := newStatefulSet(3) - ssc, spc, _ := newFakeStatefulSetController(set) - if err := scaleUpStatefulSetController(set, ssc, spc); err != nil { + ssc, spc, om, _ := newFakeStatefulSetController(set) + if err := scaleUpStatefulSetController(set, ssc, spc, om); err != nil { t.Errorf("Failed to turn up StatefulSet : %s", err) } - if obj, _, err := spc.setsIndexer.Get(set); err != nil { + if obj, _, err := om.setsIndexer.Get(set); err != nil { t.Error(err) } else { set = obj.(*apps.StatefulSet) @@ -72,10 +79,10 @@ func TestStatefulSetControllerDeletes(t *testing.T) { t.Errorf("set.Status.Replicas = %v; want 3", set.Status.Replicas) } *set.Spec.Replicas = 0 - if err := scaleDownStatefulSetController(set, ssc, spc); err != nil { + if err := scaleDownStatefulSetController(set, ssc, spc, om); err != nil { t.Errorf("Failed to turn down StatefulSet : %s", err) } - if obj, _, err := spc.setsIndexer.Get(set); err != nil { + if obj, _, err := om.setsIndexer.Get(set); err != nil { t.Error(err) } else { set = obj.(*apps.StatefulSet) @@ -87,11 +94,11 @@ func TestStatefulSetControllerDeletes(t *testing.T) { func TestStatefulSetControllerRespectsTermination(t *testing.T) { set := newStatefulSet(3) - ssc, spc, _ := newFakeStatefulSetController(set) - if err := scaleUpStatefulSetController(set, ssc, spc); err != nil { + ssc, spc, om, _ := newFakeStatefulSetController(set) + if err := scaleUpStatefulSetController(set, ssc, spc, om); err != nil { t.Errorf("Failed to turn up StatefulSet : %s", err) } - if obj, _, err := spc.setsIndexer.Get(set); err != nil { + if obj, _, err := om.setsIndexer.Get(set); err != nil { t.Error(err) } else { set = obj.(*apps.StatefulSet) @@ -99,11 +106,11 @@ func TestStatefulSetControllerRespectsTermination(t *testing.T) { if set.Status.Replicas != 3 { t.Errorf("set.Status.Replicas = %v; want 3", set.Status.Replicas) } - _, err := spc.addTerminatingPod(set, 3) + _, err := om.addTerminatingPod(set, 3) if err != nil { t.Error(err) } - pods, err := spc.addTerminatingPod(set, 4) + pods, err := om.addTerminatingPod(set, 4) if err != nil { t.Error(err) } @@ -112,7 +119,7 @@ func TestStatefulSetControllerRespectsTermination(t *testing.T) { if err != nil { t.Error(err) } - pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + pods, err = om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Error(err) } @@ -123,10 +130,10 @@ func TestStatefulSetControllerRespectsTermination(t *testing.T) { spc.DeleteStatefulPod(set, pods[3]) spc.DeleteStatefulPod(set, pods[4]) *set.Spec.Replicas = 0 - if err := scaleDownStatefulSetController(set, ssc, spc); err != nil { + if err := scaleDownStatefulSetController(set, ssc, spc, om); err != nil { t.Errorf("Failed to turn down StatefulSet : %s", err) } - if obj, _, err := spc.setsIndexer.Get(set); err != nil { + if obj, _, err := om.setsIndexer.Get(set); err != nil { t.Error(err) } else { set = obj.(*apps.StatefulSet) @@ -138,11 +145,11 @@ func TestStatefulSetControllerRespectsTermination(t *testing.T) { func TestStatefulSetControllerBlocksScaling(t *testing.T) { set := newStatefulSet(3) - ssc, spc, _ := newFakeStatefulSetController(set) - if err := scaleUpStatefulSetController(set, ssc, spc); err != nil { + ssc, spc, om, _ := newFakeStatefulSetController(set) + if err := scaleUpStatefulSetController(set, ssc, spc, om); err != nil { t.Errorf("Failed to turn up StatefulSet : %s", err) } - if obj, _, err := spc.setsIndexer.Get(set); err != nil { + if obj, _, err := om.setsIndexer.Get(set); err != nil { t.Error(err) } else { set = obj.(*apps.StatefulSet) @@ -152,8 +159,8 @@ func TestStatefulSetControllerBlocksScaling(t *testing.T) { } *set.Spec.Replicas = 5 fakeResourceVersion(set) - spc.setsIndexer.Update(set) - _, err := spc.setPodTerminated(set, 0) + om.setsIndexer.Update(set) + _, err := om.setPodTerminated(set, 0) if err != nil { t.Error("Failed to set pod terminated at ordinal 0") } @@ -163,7 +170,7 @@ func TestStatefulSetControllerBlocksScaling(t *testing.T) { if err != nil { t.Error(err) } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Error(err) } @@ -174,7 +181,7 @@ func TestStatefulSetControllerBlocksScaling(t *testing.T) { spc.DeleteStatefulPod(set, pods[0]) ssc.enqueueStatefulSet(set) fakeWorker(ssc) - pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + pods, err = om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Error(err) } @@ -186,9 +193,9 @@ func TestStatefulSetControllerBlocksScaling(t *testing.T) { func TestStatefulSetControllerDeletionTimestamp(t *testing.T) { set := newStatefulSet(3) set.DeletionTimestamp = new(metav1.Time) - ssc, spc, _ := newFakeStatefulSetController(set) + ssc, _, om, _ := newFakeStatefulSetController(set) - spc.setsIndexer.Add(set) + om.setsIndexer.Add(set) // Force a sync. It should not try to create any Pods. ssc.enqueueStatefulSet(set) @@ -198,7 +205,7 @@ func TestStatefulSetControllerDeletionTimestamp(t *testing.T) { if err != nil { t.Fatal(err) } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Fatal(err) } @@ -211,17 +218,17 @@ func TestStatefulSetControllerDeletionTimestampRace(t *testing.T) { set := newStatefulSet(3) // The bare client says it IS deleted. set.DeletionTimestamp = new(metav1.Time) - ssc, spc, ssh := newFakeStatefulSetController(set) + ssc, _, om, ssh := newFakeStatefulSetController(set) // The lister (cache) says it's NOT deleted. set2 := *set set2.DeletionTimestamp = nil - spc.setsIndexer.Add(&set2) + om.setsIndexer.Add(&set2) // The recheck occurs in the presence of a matching orphan. pod := newStatefulSetPod(set, 1) pod.OwnerReferences = nil - spc.podsIndexer.Add(pod) + om.podsIndexer.Add(pod) set.Status.CollisionCount = new(int32) revision, err := newRevision(set, 1, set.Status.CollisionCount) if err != nil { @@ -241,7 +248,7 @@ func TestStatefulSetControllerDeletionTimestampRace(t *testing.T) { if err != nil { t.Fatal(err) } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { t.Fatal(err) } @@ -272,13 +279,13 @@ func TestStatefulSetControllerDeletionTimestampRace(t *testing.T) { } func TestStatefulSetControllerAddPod(t *testing.T) { - ssc, spc, _ := newFakeStatefulSetController() + ssc, _, om, _ := newFakeStatefulSetController() set1 := newStatefulSet(3) set2 := newStatefulSet(3) pod1 := newStatefulSetPod(set1, 0) pod2 := newStatefulSetPod(set2, 0) - spc.setsIndexer.Add(set1) - spc.setsIndexer.Add(set2) + om.setsIndexer.Add(set1) + om.setsIndexer.Add(set2) ssc.addPod(pod1) key, done := ssc.queue.Get() @@ -304,7 +311,7 @@ func TestStatefulSetControllerAddPod(t *testing.T) { } func TestStatefulSetControllerAddPodOrphan(t *testing.T) { - ssc, spc, _ := newFakeStatefulSetController() + ssc, _, om, _ := newFakeStatefulSetController() set1 := newStatefulSet(3) set2 := newStatefulSet(3) set2.Name = "foo2" @@ -312,9 +319,9 @@ func TestStatefulSetControllerAddPodOrphan(t *testing.T) { set3.Name = "foo3" set3.Spec.Selector.MatchLabels = map[string]string{"foo3": "bar"} pod := newStatefulSetPod(set1, 0) - spc.setsIndexer.Add(set1) - spc.setsIndexer.Add(set2) - spc.setsIndexer.Add(set3) + om.setsIndexer.Add(set1) + om.setsIndexer.Add(set2) + om.setsIndexer.Add(set3) // Make pod an orphan. Expect matching sets to be queued. pod.OwnerReferences = nil @@ -325,7 +332,7 @@ func TestStatefulSetControllerAddPodOrphan(t *testing.T) { } func TestStatefulSetControllerAddPodNoSet(t *testing.T) { - ssc, _, _ := newFakeStatefulSetController() + ssc, _, _, _ := newFakeStatefulSetController() set := newStatefulSet(3) pod := newStatefulSetPod(set, 0) ssc.addPod(pod) @@ -337,14 +344,14 @@ func TestStatefulSetControllerAddPodNoSet(t *testing.T) { } func TestStatefulSetControllerUpdatePod(t *testing.T) { - ssc, spc, _ := newFakeStatefulSetController() + ssc, _, om, _ := newFakeStatefulSetController() set1 := newStatefulSet(3) set2 := newStatefulSet(3) set2.Name = "foo2" pod1 := newStatefulSetPod(set1, 0) pod2 := newStatefulSetPod(set2, 0) - spc.setsIndexer.Add(set1) - spc.setsIndexer.Add(set2) + om.setsIndexer.Add(set1) + om.setsIndexer.Add(set2) prev := *pod1 fakeResourceVersion(pod1) @@ -372,7 +379,7 @@ func TestStatefulSetControllerUpdatePod(t *testing.T) { } func TestStatefulSetControllerUpdatePodWithNoSet(t *testing.T) { - ssc, _, _ := newFakeStatefulSetController() + ssc, _, _, _ := newFakeStatefulSetController() set := newStatefulSet(3) pod := newStatefulSetPod(set, 0) prev := *pod @@ -386,10 +393,10 @@ func TestStatefulSetControllerUpdatePodWithNoSet(t *testing.T) { } func TestStatefulSetControllerUpdatePodWithSameVersion(t *testing.T) { - ssc, spc, _ := newFakeStatefulSetController() + ssc, _, om, _ := newFakeStatefulSetController() set := newStatefulSet(3) pod := newStatefulSetPod(set, 0) - spc.setsIndexer.Add(set) + om.setsIndexer.Add(set) ssc.updatePod(pod, pod) ssc.queue.ShutDown() key, _ := ssc.queue.Get() @@ -399,14 +406,14 @@ func TestStatefulSetControllerUpdatePodWithSameVersion(t *testing.T) { } func TestStatefulSetControllerUpdatePodOrphanWithNewLabels(t *testing.T) { - ssc, spc, _ := newFakeStatefulSetController() + ssc, _, om, _ := newFakeStatefulSetController() set := newStatefulSet(3) pod := newStatefulSetPod(set, 0) pod.OwnerReferences = nil set2 := newStatefulSet(3) set2.Name = "foo2" - spc.setsIndexer.Add(set) - spc.setsIndexer.Add(set2) + om.setsIndexer.Add(set) + om.setsIndexer.Add(set2) clone := *pod clone.Labels = map[string]string{"foo2": "bar2"} fakeResourceVersion(&clone) @@ -417,14 +424,14 @@ func TestStatefulSetControllerUpdatePodOrphanWithNewLabels(t *testing.T) { } func TestStatefulSetControllerUpdatePodChangeControllerRef(t *testing.T) { - ssc, spc, _ := newFakeStatefulSetController() + ssc, _, om, _ := newFakeStatefulSetController() set := newStatefulSet(3) set2 := newStatefulSet(3) set2.Name = "foo2" pod := newStatefulSetPod(set, 0) pod2 := newStatefulSetPod(set2, 0) - spc.setsIndexer.Add(set) - spc.setsIndexer.Add(set2) + om.setsIndexer.Add(set) + om.setsIndexer.Add(set2) clone := *pod clone.OwnerReferences = pod2.OwnerReferences fakeResourceVersion(&clone) @@ -435,13 +442,13 @@ func TestStatefulSetControllerUpdatePodChangeControllerRef(t *testing.T) { } func TestStatefulSetControllerUpdatePodRelease(t *testing.T) { - ssc, spc, _ := newFakeStatefulSetController() + ssc, _, om, _ := newFakeStatefulSetController() set := newStatefulSet(3) set2 := newStatefulSet(3) set2.Name = "foo2" pod := newStatefulSetPod(set, 0) - spc.setsIndexer.Add(set) - spc.setsIndexer.Add(set2) + om.setsIndexer.Add(set) + om.setsIndexer.Add(set2) clone := *pod clone.OwnerReferences = nil fakeResourceVersion(&clone) @@ -452,14 +459,14 @@ func TestStatefulSetControllerUpdatePodRelease(t *testing.T) { } func TestStatefulSetControllerDeletePod(t *testing.T) { - ssc, spc, _ := newFakeStatefulSetController() + ssc, _, om, _ := newFakeStatefulSetController() set1 := newStatefulSet(3) set2 := newStatefulSet(3) set2.Name = "foo2" pod1 := newStatefulSetPod(set1, 0) pod2 := newStatefulSetPod(set2, 0) - spc.setsIndexer.Add(set1) - spc.setsIndexer.Add(set2) + om.setsIndexer.Add(set1) + om.setsIndexer.Add(set2) ssc.deletePod(pod1) key, done := ssc.queue.Get() @@ -483,13 +490,13 @@ func TestStatefulSetControllerDeletePod(t *testing.T) { } func TestStatefulSetControllerDeletePodOrphan(t *testing.T) { - ssc, spc, _ := newFakeStatefulSetController() + ssc, _, om, _ := newFakeStatefulSetController() set1 := newStatefulSet(3) set2 := newStatefulSet(3) set2.Name = "foo2" pod1 := newStatefulSetPod(set1, 0) - spc.setsIndexer.Add(set1) - spc.setsIndexer.Add(set2) + om.setsIndexer.Add(set1) + om.setsIndexer.Add(set2) pod1.OwnerReferences = nil ssc.deletePod(pod1) @@ -499,10 +506,10 @@ func TestStatefulSetControllerDeletePodOrphan(t *testing.T) { } func TestStatefulSetControllerDeletePodTombstone(t *testing.T) { - ssc, spc, _ := newFakeStatefulSetController() + ssc, _, om, _ := newFakeStatefulSetController() set := newStatefulSet(3) pod := newStatefulSetPod(set, 0) - spc.setsIndexer.Add(set) + om.setsIndexer.Add(set) tombstoneKey, _ := controller.KeyFunc(pod) tombstone := cache.DeletedFinalStateUnknown{Key: tombstoneKey, Obj: pod} ssc.deletePod(tombstone) @@ -517,14 +524,14 @@ func TestStatefulSetControllerDeletePodTombstone(t *testing.T) { } func TestStatefulSetControllerGetStatefulSetsForPod(t *testing.T) { - ssc, spc, _ := newFakeStatefulSetController() + ssc, _, om, _ := newFakeStatefulSetController() set1 := newStatefulSet(3) set2 := newStatefulSet(3) set2.Name = "foo2" pod := newStatefulSetPod(set1, 0) - spc.setsIndexer.Add(set1) - spc.setsIndexer.Add(set2) - spc.podsIndexer.Add(pod) + om.setsIndexer.Add(set1) + om.setsIndexer.Add(set2) + om.podsIndexer.Add(pod) sets := ssc.getStatefulSetsForPod(pod) if got, want := len(sets), 2; got != want { t.Errorf("len(sets) = %v, want %v", got, want) @@ -546,12 +553,12 @@ func TestGetPodsForStatefulSetAdopt(t *testing.T) { pod4.OwnerReferences = nil pod4.Name = "x" + pod4.Name - ssc, spc, _ := newFakeStatefulSetController(set, pod1, pod2, pod3, pod4) + ssc, _, om, _ := newFakeStatefulSetController(set, pod1, pod2, pod3, pod4) - spc.podsIndexer.Add(pod1) - spc.podsIndexer.Add(pod2) - spc.podsIndexer.Add(pod3) - spc.podsIndexer.Add(pod4) + om.podsIndexer.Add(pod1) + om.podsIndexer.Add(pod2) + om.podsIndexer.Add(pod3) + om.podsIndexer.Add(pod4) selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { t.Fatal(err) @@ -588,10 +595,10 @@ func TestAdoptOrphanRevisions(t *testing.T) { ss1Rev2.Namespace = ss1.Namespace ss1Rev2.OwnerReferences = []metav1.OwnerReference{} - ssc, spc, _ := newFakeStatefulSetController(ss1, ss1Rev1, ss1Rev2) + ssc, _, om, _ := newFakeStatefulSetController(ss1, ss1Rev1, ss1Rev2) - spc.revisionsIndexer.Add(ss1Rev1) - spc.revisionsIndexer.Add(ss1Rev2) + om.revisionsIndexer.Add(ss1Rev1) + om.revisionsIndexer.Add(ss1Rev2) err = ssc.adoptOrphanRevisions(context.TODO(), ss1) if err != nil { @@ -615,7 +622,7 @@ func TestAdoptOrphanRevisions(t *testing.T) { func TestGetPodsForStatefulSetRelease(t *testing.T) { set := newStatefulSet(3) - ssc, spc, _ := newFakeStatefulSetController(set) + ssc, _, om, _ := newFakeStatefulSetController(set) pod1 := newStatefulSetPod(set, 1) // pod2 is owned but has wrong name. pod2 := newStatefulSetPod(set, 2) @@ -628,9 +635,10 @@ func TestGetPodsForStatefulSetRelease(t *testing.T) { pod4.OwnerReferences = nil pod4.Labels = nil - spc.podsIndexer.Add(pod1) - spc.podsIndexer.Add(pod2) - spc.podsIndexer.Add(pod3) + om.podsIndexer.Add(pod1) + om.podsIndexer.Add(pod2) + om.podsIndexer.Add(pod3) + om.podsIndexer.Add(pod4) selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { t.Fatal(err) @@ -651,10 +659,255 @@ func TestGetPodsForStatefulSetRelease(t *testing.T) { } } -func newFakeStatefulSetController(initialObjects ...runtime.Object) (*StatefulSetController, *fakeStatefulPodControl, history.Interface) { +func TestOrphanedPodsWithPVCDeletePolicy(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, true)() + + testFn := func(t *testing.T, scaledownPolicy, deletionPolicy apps.PersistentVolumeClaimRetentionPolicyType) { + set := newStatefulSet(4) + *set.Spec.Replicas = 2 + set.Spec.PersistentVolumeClaimRetentionPolicy.WhenScaled = scaledownPolicy + set.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted = deletionPolicy + ssc, _, om, _ := newFakeStatefulSetController(set) + om.setsIndexer.Add(set) + + pods := []*v1.Pod{} + pods = append(pods, newStatefulSetPod(set, 0)) + // pod1 is orphaned + pods = append(pods, newStatefulSetPod(set, 1)) + pods[1].OwnerReferences = nil + // pod2 is owned but has wrong name. + pods = append(pods, newStatefulSetPod(set, 2)) + pods[2].Name = "x" + pods[2].Name + + ssc.kubeClient.(*fake.Clientset).PrependReactor("patch", "pods", func(action core.Action) (bool, runtime.Object, error) { + patch := action.(core.PatchAction).GetPatch() + target := action.(core.PatchAction).GetName() + var pod *v1.Pod + for _, p := range pods { + if p.Name == target { + pod = p + break + } + } + if pod == nil { + t.Fatalf("Can't find patch target %s", target) + } + original, err := json.Marshal(pod) + if err != nil { + t.Fatalf("failed to marshal original pod %s: %v", pod.Name, err) + } + updated, err := strategicpatch.StrategicMergePatch(original, patch, v1.Pod{}) + if err != nil { + t.Fatalf("failed to apply strategic merge patch %q on node %s: %v", patch, pod.Name, err) + } + if err := json.Unmarshal(updated, pod); err != nil { + t.Fatalf("failed to unmarshal updated pod %s: %v", pod.Name, err) + } + + return true, pod, nil + }) + + for _, pod := range pods { + om.podsIndexer.Add(pod) + claims := getPersistentVolumeClaims(set, pod) + for _, claim := range claims { + om.CreateClaim(&claim) + } + } + + for i := range pods { + if _, err := om.setPodReady(set, i); err != nil { + t.Errorf("%d: %v", i, err) + } + if _, err := om.setPodRunning(set, i); err != nil { + t.Errorf("%d: %v", i, err) + } + } + + // First sync to manage orphaned pod, then set replicas. + ssc.enqueueStatefulSet(set) + fakeWorker(ssc) + *set.Spec.Replicas = 0 // Put an ownerRef for all scale-down deleted PVCs. + ssc.enqueueStatefulSet(set) + fakeWorker(ssc) + + hasNamedOwnerRef := func(claim *v1.PersistentVolumeClaim, name string) bool { + for _, ownerRef := range claim.GetOwnerReferences() { + if ownerRef.Name == name { + return true + } + } + return false + } + verifyOwnerRefs := func(claim *v1.PersistentVolumeClaim, condemned bool) { + podName := getClaimPodName(set, claim) + const retain = apps.RetainPersistentVolumeClaimRetentionPolicyType + const delete = apps.DeletePersistentVolumeClaimRetentionPolicyType + switch { + case scaledownPolicy == retain && deletionPolicy == retain: + if hasNamedOwnerRef(claim, podName) || hasNamedOwnerRef(claim, set.Name) { + t.Errorf("bad claim ownerRefs: %s: %v", claim.Name, claim.GetOwnerReferences()) + } + case scaledownPolicy == retain && deletionPolicy == delete: + if hasNamedOwnerRef(claim, podName) || !hasNamedOwnerRef(claim, set.Name) { + t.Errorf("bad claim ownerRefs: %s: %v", claim.Name, claim.GetOwnerReferences()) + } + case scaledownPolicy == delete && deletionPolicy == retain: + if hasNamedOwnerRef(claim, podName) != condemned || hasNamedOwnerRef(claim, set.Name) { + t.Errorf("bad claim ownerRefs: %s: %v", claim.Name, claim.GetOwnerReferences()) + } + case scaledownPolicy == delete && deletionPolicy == delete: + if hasNamedOwnerRef(claim, podName) != condemned || !hasNamedOwnerRef(claim, set.Name) { + t.Errorf("bad claim ownerRefs: %s: %v", claim.Name, claim.GetOwnerReferences()) + } + } + } + + claims, _ := om.claimsLister.PersistentVolumeClaims(set.Namespace).List(labels.Everything()) + if len(claims) != len(pods) { + t.Errorf("Unexpected number of claims: %d", len(claims)) + } + for _, claim := range claims { + // Only the first pod and the reclaimed orphan pod should have owner refs. + switch claim.Name { + case "datadir-foo-0", "datadir-foo-1": + verifyOwnerRefs(claim, false) + case "datadir-foo-2": + if hasNamedOwnerRef(claim, getClaimPodName(set, claim)) || hasNamedOwnerRef(claim, set.Name) { + t.Errorf("unexpected ownerRefs for %s: %v", claim.Name, claim.GetOwnerReferences()) + } + default: + t.Errorf("Unexpected claim %s", claim.Name) + } + } + } + policies := []apps.PersistentVolumeClaimRetentionPolicyType{ + apps.RetainPersistentVolumeClaimRetentionPolicyType, + apps.DeletePersistentVolumeClaimRetentionPolicyType, + } + for _, scaledownPolicy := range policies { + for _, deletionPolicy := range policies { + testName := fmt.Sprintf("ScaleDown:%s/SetDeletion:%s", scaledownPolicy, deletionPolicy) + t.Run(testName, func(t *testing.T) { testFn(t, scaledownPolicy, deletionPolicy) }) + } + } +} + +func TestStaleOwnerRefOnScaleup(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetAutoDeletePVC, true)() + + for _, policy := range []*apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + { + WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + }, + { + WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.DeletePersistentVolumeClaimRetentionPolicyType, + }, + } { + onPolicy := func(msg string, args ...interface{}) string { + return fmt.Sprintf(fmt.Sprintf("(%s) %s", policy, msg), args...) + } + set := newStatefulSet(3) + set.Spec.PersistentVolumeClaimRetentionPolicy = policy + ssc, spc, om, _ := newFakeStatefulSetController(set) + if err := scaleUpStatefulSetController(set, ssc, spc, om); err != nil { + t.Errorf(onPolicy("Failed to turn up StatefulSet : %s", err)) + } + var err error + if set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name); err != nil { + t.Errorf(onPolicy("Could not get scaled up set: %v", err)) + } + if set.Status.Replicas != 3 { + t.Errorf(onPolicy("set.Status.Replicas = %v; want 3", set.Status.Replicas)) + } + *set.Spec.Replicas = 2 + if err := scaleDownStatefulSetController(set, ssc, spc, om); err != nil { + t.Errorf(onPolicy("Failed to scale down StatefulSet : msg, %s", err)) + } + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) + if err != nil { + t.Errorf(onPolicy("Could not get scaled down StatefulSet: %v", err)) + } + if set.Status.Replicas != 2 { + t.Errorf(onPolicy("Failed to scale statefulset to 2 replicas")) + } + + var claim *v1.PersistentVolumeClaim + claim, err = om.claimsLister.PersistentVolumeClaims(set.Namespace).Get("datadir-foo-2") + if err != nil { + t.Errorf(onPolicy("Could not find expected pvc datadir-foo-2")) + } + refs := claim.GetOwnerReferences() + if len(refs) != 1 { + t.Errorf(onPolicy("Expected only one refs: %v", refs)) + } + // Make the pod ref stale. + for i := range refs { + if refs[i].Name == "foo-2" { + refs[i].UID = "stale" + break + } + } + claim.SetOwnerReferences(refs) + if err = om.claimsIndexer.Update(claim); err != nil { + t.Errorf(onPolicy("Could not update claim with new owner ref: %v", err)) + } + + *set.Spec.Replicas = 3 + // Until the stale PVC goes away, the scale up should never finish. Run 10 iterations, then delete the PVC. + if err := scaleUpStatefulSetControllerBounded(set, ssc, spc, om, 10); err != nil { + t.Errorf(onPolicy("Failed attempt to scale StatefulSet back up: %v", err)) + } + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) + if err != nil { + t.Errorf(onPolicy("Could not get scaled down StatefulSet: %v", err)) + } + if set.Status.Replicas != 2 { + t.Errorf(onPolicy("Expected set to stay at two replicas")) + } + + claim, err = om.claimsLister.PersistentVolumeClaims(set.Namespace).Get("datadir-foo-2") + if err != nil { + t.Errorf(onPolicy("Could not find expected pvc datadir-foo-2")) + } + refs = claim.GetOwnerReferences() + if len(refs) != 1 { + t.Errorf(onPolicy("Unexpected change to condemned pvc ownerRefs: %v", refs)) + } + foundPodRef := false + for i := range refs { + if refs[i].UID == "stale" { + foundPodRef = true + break + } + } + if !foundPodRef { + t.Errorf(onPolicy("Claim ref unexpectedly changed: %v", refs)) + } + if err = om.claimsIndexer.Delete(claim); err != nil { + t.Errorf(onPolicy("Could not delete stale pvc: %v", err)) + } + + if err := scaleUpStatefulSetController(set, ssc, spc, om); err != nil { + t.Errorf(onPolicy("Failed to scale StatefulSet back up: %v", err)) + } + set, err = om.setsLister.StatefulSets(set.Namespace).Get(set.Name) + if err != nil { + t.Errorf(onPolicy("Could not get scaled down StatefulSet: %v", err)) + } + if set.Status.Replicas != 3 { + t.Errorf(onPolicy("Failed to scale set back up once PVC was deleted")) + } + } +} + +func newFakeStatefulSetController(initialObjects ...runtime.Object) (*StatefulSetController, *StatefulPodControl, *fakeObjectManager, history.Interface) { client := fake.NewSimpleClientset(initialObjects...) informerFactory := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc()) - fpc := newFakeStatefulPodControl(informerFactory.Core().V1().Pods(), informerFactory.Apps().V1().StatefulSets(), informerFactory.Apps().V1().ControllerRevisions()) + om := newFakeObjectManager(informerFactory) + spc := NewStatefulPodControlFromManager(om, &noopRecorder{}) ssu := newFakeStatefulSetStatusUpdater(informerFactory.Apps().V1().StatefulSets()) ssc := NewStatefulSetController( informerFactory.Core().V1().Pods(), @@ -667,9 +920,9 @@ func newFakeStatefulSetController(initialObjects ...runtime.Object) (*StatefulSe ssc.podListerSynced = alwaysReady ssc.setListerSynced = alwaysReady recorder := record.NewFakeRecorder(10) - ssc.control = NewDefaultStatefulSetControl(fpc, ssu, ssh, recorder) + ssc.control = NewDefaultStatefulSetControl(spc, ssu, ssh, recorder) - return ssc, fpc, ssh + return ssc, spc, om, ssh } func fakeWorker(ssc *StatefulSetController) { @@ -687,21 +940,27 @@ func getPodAtOrdinal(pods []*v1.Pod, ordinal int) *v1.Pod { return pods[ordinal] } -func scaleUpStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetController, spc *fakeStatefulPodControl) error { - spc.setsIndexer.Add(set) +func scaleUpStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetController, spc *StatefulPodControl, om *fakeObjectManager) error { + return scaleUpStatefulSetControllerBounded(set, ssc, spc, om, -1) +} + +func scaleUpStatefulSetControllerBounded(set *apps.StatefulSet, ssc *StatefulSetController, spc *StatefulPodControl, om *fakeObjectManager, maxIterations int) error { + om.setsIndexer.Add(set) ssc.enqueueStatefulSet(set) fakeWorker(ssc) selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { return err } - for set.Status.ReadyReplicas < *set.Spec.Replicas { - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + iterations := 0 + for (maxIterations < 0 || iterations < maxIterations) && set.Status.ReadyReplicas < *set.Spec.Replicas { + iterations++ + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return err } ord := len(pods) - 1 - if pods, err = spc.setPodPending(set, ord); err != nil { + if pods, err = om.setPodPending(set, ord); err != nil { return err } pod := getPodAtOrdinal(pods, ord) @@ -709,7 +968,7 @@ func scaleUpStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetControl fakeWorker(ssc) pod = getPodAtOrdinal(pods, ord) prev := *pod - if pods, err = spc.setPodRunning(set, ord); err != nil { + if pods, err = om.setPodRunning(set, ord); err != nil { return err } pod = getPodAtOrdinal(pods, ord) @@ -717,31 +976,31 @@ func scaleUpStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetControl fakeWorker(ssc) pod = getPodAtOrdinal(pods, ord) prev = *pod - if pods, err = spc.setPodReady(set, ord); err != nil { + if pods, err = om.setPodReady(set, ord); err != nil { return err } pod = getPodAtOrdinal(pods, ord) ssc.updatePod(&prev, pod) fakeWorker(ssc) - if err := assertMonotonicInvariants(set, spc); err != nil { + if err := assertMonotonicInvariants(set, om); err != nil { return err } - obj, _, err := spc.setsIndexer.Get(set) + obj, _, err := om.setsIndexer.Get(set) if err != nil { return err } set = obj.(*apps.StatefulSet) } - return assertMonotonicInvariants(set, spc) + return assertMonotonicInvariants(set, om) } -func scaleDownStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetController, spc *fakeStatefulPodControl) error { +func scaleDownStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetController, spc *StatefulPodControl, om *fakeObjectManager) error { selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) if err != nil { return err } - pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + pods, err := om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return err } @@ -749,10 +1008,10 @@ func scaleDownStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetContr pod := getPodAtOrdinal(pods, ord) prev := *pod fakeResourceVersion(set) - spc.setsIndexer.Add(set) + om.setsIndexer.Add(set) ssc.enqueueStatefulSet(set) fakeWorker(ssc) - pods, err = spc.addTerminatingPod(set, ord) + pods, err = om.addTerminatingPod(set, ord) if err != nil { return err } @@ -763,13 +1022,13 @@ func scaleDownStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetContr ssc.deletePod(pod) fakeWorker(ssc) for set.Status.Replicas > *set.Spec.Replicas { - pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + pods, err = om.podsLister.Pods(set.Namespace).List(selector) if err != nil { return err } ord := len(pods) - pods, err = spc.addTerminatingPod(set, ord) + pods, err = om.addTerminatingPod(set, ord) if err != nil { return err } @@ -779,14 +1038,14 @@ func scaleDownStatefulSetController(set *apps.StatefulSet, ssc *StatefulSetContr spc.DeleteStatefulPod(set, pod) ssc.deletePod(pod) fakeWorker(ssc) - obj, _, err := spc.setsIndexer.Get(set) + obj, _, err := om.setsIndexer.Get(set) if err != nil { return err } set = obj.(*apps.StatefulSet) } - return assertMonotonicInvariants(set, spc) + return assertMonotonicInvariants(set, om) } func rawTemplate(template *v1.PodTemplateSpec) runtime.RawExtension { diff --git a/pkg/controller/statefulset/stateful_set_utils_test.go b/pkg/controller/statefulset/stateful_set_utils_test.go index 1b9f4693e6da..f86dbd3eca2b 100644 --- a/pkg/controller/statefulset/stateful_set_utils_test.go +++ b/pkg/controller/statefulset/stateful_set_utils_test.go @@ -20,6 +20,7 @@ import ( "fmt" "math/rand" "reflect" + "regexp" "sort" "strconv" "testing" @@ -27,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" apps "k8s.io/api/apps/v1" @@ -35,6 +37,28 @@ import ( "k8s.io/kubernetes/pkg/controller/history" ) +// noopRecorder is an EventRecorder that does nothing. record.FakeRecorder has a fixed +// buffer size, which causes tests to hang if that buffer's exceeded. +type noopRecorder struct{} + +func (r *noopRecorder) Event(object runtime.Object, eventtype, reason, message string) {} +func (r *noopRecorder) Eventf(object runtime.Object, eventtype, reason, messageFmt string, args ...interface{}) { +} +func (r *noopRecorder) AnnotatedEventf(object runtime.Object, annotations map[string]string, eventtype, reason, messageFmt string, args ...interface{}) { +} + +// getClaimPodName gets the name of the Pod associated with the Claim, or an empty string if this doesn't look matching. +func getClaimPodName(set *apps.StatefulSet, claim *v1.PersistentVolumeClaim) string { + podName := "" + + statefulClaimRegex := regexp.MustCompile(fmt.Sprintf(".*-(%s-[0-9]+)$", set.Name)) + matches := statefulClaimRegex.FindStringSubmatch(claim.Name) + if len(matches) != 2 { + return podName + } + return matches[1] +} + func TestGetParentNameAndOrdinal(t *testing.T) { set := newStatefulSet(3) pod := newStatefulSetPod(set, 1) @@ -51,6 +75,28 @@ func TestGetParentNameAndOrdinal(t *testing.T) { } } +func TestGetClaimPodName(t *testing.T) { + set := apps.StatefulSet{} + set.Name = "my-set" + claim := v1.PersistentVolumeClaim{} + claim.Name = "volume-my-set-2" + if pod := getClaimPodName(&set, &claim); pod != "my-set-2" { + t.Errorf("Expected my-set-2 found %s", pod) + } + claim.Name = "long-volume-my-set-20" + if pod := getClaimPodName(&set, &claim); pod != "my-set-20" { + t.Errorf("Expected my-set-20 found %s", pod) + } + claim.Name = "volume-2-my-set" + if pod := getClaimPodName(&set, &claim); pod != "" { + t.Errorf("Expected empty string found %s", pod) + } + claim.Name = "volume-pod-2" + if pod := getClaimPodName(&set, &claim); pod != "" { + t.Errorf("Expected empty string found %s", pod) + } +} + func TestIsMemberOf(t *testing.T) { set := newStatefulSet(3) set2 := newStatefulSet(3) @@ -180,6 +226,350 @@ func TestUpdateStorage(t *testing.T) { } } +func TestGetPersistentVolumeClaimRetentionPolicy(t *testing.T) { + retainPolicy := apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + } + scaledownPolicy := apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: apps.DeletePersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + } + + set := apps.StatefulSet{} + set.Spec.PersistentVolumeClaimRetentionPolicy = &retainPolicy + got := getPersistentVolumeClaimRetentionPolicy(&set) + if got.WhenScaled != apps.RetainPersistentVolumeClaimRetentionPolicyType || got.WhenDeleted != apps.RetainPersistentVolumeClaimRetentionPolicyType { + t.Errorf("Expected retain policy") + } + set.Spec.PersistentVolumeClaimRetentionPolicy = &scaledownPolicy + got = getPersistentVolumeClaimRetentionPolicy(&set) + if got.WhenScaled != apps.DeletePersistentVolumeClaimRetentionPolicyType || got.WhenDeleted != apps.RetainPersistentVolumeClaimRetentionPolicyType { + t.Errorf("Expected scaledown policy") + } +} + +func TestClaimOwnerMatchesSetAndPod(t *testing.T) { + testCases := []struct { + name string + scaleDownPolicy apps.PersistentVolumeClaimRetentionPolicyType + setDeletePolicy apps.PersistentVolumeClaimRetentionPolicyType + needsPodRef bool + needsSetRef bool + replicas int32 + ordinal int + }{ + { + name: "retain", + scaleDownPolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, + needsPodRef: false, + needsSetRef: false, + }, + { + name: "on SS delete", + scaleDownPolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + needsPodRef: false, + needsSetRef: true, + }, + { + name: "on scaledown only, condemned", + scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, + needsPodRef: true, + needsSetRef: false, + replicas: 2, + ordinal: 2, + }, + { + name: "on scaledown only, remains", + scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, + needsPodRef: false, + needsSetRef: false, + replicas: 2, + ordinal: 1, + }, + { + name: "on both, condemned", + scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + needsPodRef: true, + needsSetRef: false, + replicas: 2, + ordinal: 2, + }, + { + name: "on both, remains", + scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + needsPodRef: false, + needsSetRef: true, + replicas: 2, + ordinal: 1, + }, + } + + for _, tc := range testCases { + for _, useOtherRefs := range []bool{false, true} { + for _, setPodRef := range []bool{false, true} { + for _, setSetRef := range []bool{false, true} { + claim := v1.PersistentVolumeClaim{} + claim.Name = "target-claim" + pod := v1.Pod{} + pod.Name = fmt.Sprintf("pod-%d", tc.ordinal) + pod.GetObjectMeta().SetUID("pod-123") + set := apps.StatefulSet{} + set.Name = "stateful-set" + set.GetObjectMeta().SetUID("ss-456") + set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: tc.scaleDownPolicy, + WhenDeleted: tc.setDeletePolicy, + } + set.Spec.Replicas = &tc.replicas + if setPodRef { + setOwnerRef(&claim, &pod, &pod.TypeMeta) + } + if setSetRef { + setOwnerRef(&claim, &set, &set.TypeMeta) + } + if useOtherRefs { + randomObject1 := v1.Pod{} + randomObject1.Name = "rand1" + randomObject1.GetObjectMeta().SetUID("rand1-abc") + randomObject2 := v1.Pod{} + randomObject2.Name = "rand2" + randomObject2.GetObjectMeta().SetUID("rand2-def") + setOwnerRef(&claim, &randomObject1, &randomObject1.TypeMeta) + setOwnerRef(&claim, &randomObject2, &randomObject2.TypeMeta) + } + shouldMatch := setPodRef == tc.needsPodRef && setSetRef == tc.needsSetRef + if claimOwnerMatchesSetAndPod(&claim, &set, &pod) != shouldMatch { + t.Errorf("Bad match for %s with pod=%v,set=%v,others=%v", tc.name, setPodRef, setSetRef, useOtherRefs) + } + } + } + } + } +} + +func TestUpdateClaimOwnerRefForSetAndPod(t *testing.T) { + testCases := []struct { + name string + scaleDownPolicy apps.PersistentVolumeClaimRetentionPolicyType + setDeletePolicy apps.PersistentVolumeClaimRetentionPolicyType + condemned bool + needsPodRef bool + needsSetRef bool + }{ + { + name: "retain", + scaleDownPolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, + condemned: false, + needsPodRef: false, + needsSetRef: false, + }, + { + name: "delete with set", + scaleDownPolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + condemned: false, + needsPodRef: false, + needsSetRef: true, + }, + { + name: "delete with scaledown, not condemned", + scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, + condemned: false, + needsPodRef: false, + needsSetRef: false, + }, + { + name: "delete on scaledown, condemned", + scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.RetainPersistentVolumeClaimRetentionPolicyType, + condemned: true, + needsPodRef: true, + needsSetRef: false, + }, + { + name: "delete on both, not condemned", + scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + condemned: false, + needsPodRef: false, + needsSetRef: true, + }, + { + name: "delete on both, condemned", + scaleDownPolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + setDeletePolicy: apps.DeletePersistentVolumeClaimRetentionPolicyType, + condemned: true, + needsPodRef: true, + needsSetRef: false, + }, + } + for _, tc := range testCases { + for _, hasPodRef := range []bool{true, false} { + for _, hasSetRef := range []bool{true, false} { + set := apps.StatefulSet{} + set.Name = "ss" + numReplicas := int32(5) + set.Spec.Replicas = &numReplicas + set.SetUID("ss-123") + set.Spec.PersistentVolumeClaimRetentionPolicy = &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: tc.scaleDownPolicy, + WhenDeleted: tc.setDeletePolicy, + } + pod := v1.Pod{} + if tc.condemned { + pod.Name = "pod-8" + } else { + pod.Name = "pod-1" + } + pod.SetUID("pod-456") + claim := v1.PersistentVolumeClaim{} + if hasPodRef { + setOwnerRef(&claim, &pod, &pod.TypeMeta) + } + if hasSetRef { + setOwnerRef(&claim, &set, &set.TypeMeta) + } + needsUpdate := hasPodRef != tc.needsPodRef || hasSetRef != tc.needsSetRef + shouldUpdate := updateClaimOwnerRefForSetAndPod(&claim, &set, &pod) + if shouldUpdate != needsUpdate { + t.Errorf("Bad update for %s hasPodRef=%v hasSetRef=%v", tc.name, hasPodRef, hasSetRef) + } + if hasOwnerRef(&claim, &pod) != tc.needsPodRef { + t.Errorf("Bad pod ref for %s hasPodRef=%v hasSetRef=%v", tc.name, hasPodRef, hasSetRef) + } + if hasOwnerRef(&claim, &set) != tc.needsSetRef { + t.Errorf("Bad set ref for %s hasPodRef=%v hasSetRef=%v", tc.name, hasPodRef, hasSetRef) + } + } + } + } +} + +func TestHasOwnerRef(t *testing.T) { + target := v1.Pod{} + target.SetOwnerReferences([]metav1.OwnerReference{ + {UID: "123"}, {UID: "456"}}) + ownerA := v1.Pod{} + ownerA.GetObjectMeta().SetUID("123") + ownerB := v1.Pod{} + ownerB.GetObjectMeta().SetUID("789") + if !hasOwnerRef(&target, &ownerA) { + t.Error("Missing owner") + } + if hasOwnerRef(&target, &ownerB) { + t.Error("Unexpected owner") + } +} + +func TestHasStaleOwnerRef(t *testing.T) { + target := v1.Pod{} + target.SetOwnerReferences([]metav1.OwnerReference{ + {Name: "bob", UID: "123"}, {Name: "shirley", UID: "456"}}) + ownerA := v1.Pod{} + ownerA.SetUID("123") + ownerA.Name = "bob" + ownerB := v1.Pod{} + ownerB.Name = "shirley" + ownerB.SetUID("789") + ownerC := v1.Pod{} + ownerC.Name = "yvonne" + ownerC.SetUID("345") + if hasStaleOwnerRef(&target, &ownerA) { + t.Error("ownerA should not be stale") + } + if !hasStaleOwnerRef(&target, &ownerB) { + t.Error("ownerB should be stale") + } + if hasStaleOwnerRef(&target, &ownerC) { + t.Error("ownerC should not be stale") + } +} + +func TestSetOwnerRef(t *testing.T) { + target := v1.Pod{} + ownerA := v1.Pod{} + ownerA.Name = "A" + ownerA.GetObjectMeta().SetUID("ABC") + if setOwnerRef(&target, &ownerA, &ownerA.TypeMeta) != true { + t.Errorf("Unexpected lack of update") + } + ownerRefs := target.GetObjectMeta().GetOwnerReferences() + if len(ownerRefs) != 1 { + t.Errorf("Unexpected owner ref count: %d", len(ownerRefs)) + } + if ownerRefs[0].UID != "ABC" { + t.Errorf("Unexpected owner UID %v", ownerRefs[0].UID) + } + if setOwnerRef(&target, &ownerA, &ownerA.TypeMeta) != false { + t.Errorf("Unexpected update") + } + if len(target.GetObjectMeta().GetOwnerReferences()) != 1 { + t.Error("Unexpected duplicate reference") + } + ownerB := v1.Pod{} + ownerB.Name = "B" + ownerB.GetObjectMeta().SetUID("BCD") + if setOwnerRef(&target, &ownerB, &ownerB.TypeMeta) != true { + t.Error("Unexpected lack of second update") + } + ownerRefs = target.GetObjectMeta().GetOwnerReferences() + if len(ownerRefs) != 2 { + t.Errorf("Unexpected owner ref count: %d", len(ownerRefs)) + } + if ownerRefs[0].UID != "ABC" || ownerRefs[1].UID != "BCD" { + t.Errorf("Bad second ownerRefs: %v", ownerRefs) + } +} + +func TestRemoveOwnerRef(t *testing.T) { + target := v1.Pod{} + ownerA := v1.Pod{} + ownerA.Name = "A" + ownerA.GetObjectMeta().SetUID("ABC") + if removeOwnerRef(&target, &ownerA) != false { + t.Error("Unexpected update on empty remove") + } + setOwnerRef(&target, &ownerA, &ownerA.TypeMeta) + if removeOwnerRef(&target, &ownerA) != true { + t.Error("Unexpected lack of update") + } + if len(target.GetObjectMeta().GetOwnerReferences()) != 0 { + t.Error("Unexpected owner reference remains") + } + + ownerB := v1.Pod{} + ownerB.Name = "B" + ownerB.GetObjectMeta().SetUID("BCD") + + setOwnerRef(&target, &ownerA, &ownerA.TypeMeta) + if removeOwnerRef(&target, &ownerB) != false { + t.Error("Unexpected update for mismatched owner") + } + if len(target.GetObjectMeta().GetOwnerReferences()) != 1 { + t.Error("Missing ref after no-op remove") + } + setOwnerRef(&target, &ownerB, &ownerB.TypeMeta) + if removeOwnerRef(&target, &ownerA) != true { + t.Error("Missing update for second remove") + } + ownerRefs := target.GetObjectMeta().GetOwnerReferences() + if len(ownerRefs) != 1 { + t.Error("Extra ref after second remove") + } + if ownerRefs[0].UID != "BCD" { + t.Error("Bad UID after second remove") + } +} + func TestIsRunningAndReady(t *testing.T) { set := newStatefulSet(3) pod := newStatefulSetPod(set, 1) @@ -387,7 +777,8 @@ func newPod() *v1.Pod { func newPVC(name string) v1.PersistentVolumeClaim { return v1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ - Name: name, + Namespace: "default", + Name: name, }, Spec: v1.PersistentVolumeClaimSpec{ Resources: v1.ResourceRequirements{ @@ -452,6 +843,10 @@ func newStatefulSetWithVolumes(replicas int, name string, petMounts []v1.VolumeM VolumeClaimTemplates: claims, ServiceName: "governingsvc", UpdateStrategy: apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType}, + PersistentVolumeClaimRetentionPolicy: &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + }, RevisionHistoryLimit: func() *int32 { limit := int32(2) return &limit @@ -499,6 +894,10 @@ func newStatefulSetWithLabels(replicas int, name string, uid types.UID, labels m MatchExpressions: testMatchExpressions, }, Replicas: func() *int32 { i := int32(replicas); return &i }(), + PersistentVolumeClaimRetentionPolicy: &apps.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: apps.RetainPersistentVolumeClaimRetentionPolicyType, + WhenDeleted: apps.RetainPersistentVolumeClaimRetentionPolicyType, + }, Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: labels, @@ -525,7 +924,7 @@ func newStatefulSetWithLabels(replicas int, name string, uid types.UID, labels m }, VolumeClaimTemplates: []v1.PersistentVolumeClaim{ { - ObjectMeta: metav1.ObjectMeta{Name: "datadir"}, + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "datadir"}, Spec: v1.PersistentVolumeClaimSpec{ Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{ diff --git a/test/e2e/apps/statefulset.go b/test/e2e/apps/statefulset.go index 1f0bd7627429..6a347e171b17 100644 --- a/test/e2e/apps/statefulset.go +++ b/test/e2e/apps/statefulset.go @@ -20,6 +20,8 @@ import ( "context" "encoding/json" "fmt" + "regexp" + "strconv" "strings" "sync" "time" @@ -1142,6 +1144,7 @@ var _ = SIGDescribe("StatefulSet", func() { appTester.run() }) }) + // Make sure minReadySeconds is honored // Don't mark it as conformance yet ginkgo.It("MinReadySeconds should be honored when enabled", func() { @@ -1202,6 +1205,144 @@ var _ = SIGDescribe("StatefulSet", func() { framework.Failf("invalid number of availableReplicas: expected=%v received=%v", 2, out) } }) + + ginkgo.Describe("Non-retain StatefulSetPersistentVolumeClaimPolicy [Feature:StatefulSetAutoDeletePVC]", func() { + ssName := "ss" + labels := map[string]string{ + "foo": "bar", + "baz": "blah", + } + headlessSvcName := "test" + var statefulPodMounts, podMounts []v1.VolumeMount + var ss *appsv1.StatefulSet + + ginkgo.BeforeEach(func() { + statefulPodMounts = []v1.VolumeMount{{Name: "datadir", MountPath: "/data/"}} + podMounts = []v1.VolumeMount{{Name: "home", MountPath: "/home"}} + ss = e2estatefulset.NewStatefulSet(ssName, ns, headlessSvcName, 2, statefulPodMounts, podMounts, labels) + + ginkgo.By("Creating service " + headlessSvcName + " in namespace " + ns) + headlessService := e2eservice.CreateServiceSpec(headlessSvcName, "", true, labels) + _, err := c.CoreV1().Services(ns).Create(context.TODO(), headlessService, metav1.CreateOptions{}) + framework.ExpectNoError(err) + }) + + ginkgo.AfterEach(func() { + if ginkgo.CurrentGinkgoTestDescription().Failed { + framework.DumpDebugInfo(c, ns) + } + framework.Logf("Deleting all statefulset in ns %v", ns) + e2estatefulset.DeleteAllStatefulSets(c, ns) + }) + + ginkgo.It("should delete PVCs with a WhenDeleted policy", func() { + e2epv.SkipIfNoDefaultStorageClass(c) + ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns) + *(ss.Spec.Replicas) = 3 + ss.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: appsv1.DeletePersistentVolumeClaimRetentionPolicyType, + } + _, err := c.AppsV1().StatefulSets(ns).Create(context.TODO(), ss, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + ginkgo.By("Confirming all 3 PVCs exist with their owner refs") + err = verifyStatefulSetPVCsExistWithOwnerRefs(c, ss, []int{0, 1, 2}, true, false) + framework.ExpectNoError(err) + + ginkgo.By("Deleting stateful set " + ss.Name) + err = c.AppsV1().StatefulSets(ns).Delete(context.TODO(), ss.Name, metav1.DeleteOptions{}) + framework.ExpectNoError(err) + + ginkgo.By("Verifying PVCs deleted") + err = verifyStatefulSetPVCsExist(c, ss, []int{}) + framework.ExpectNoError(err) + }) + + ginkgo.It("should delete PVCs with a OnScaledown policy", func() { + e2epv.SkipIfNoDefaultStorageClass(c) + ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns) + *(ss.Spec.Replicas) = 3 + ss.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: appsv1.DeletePersistentVolumeClaimRetentionPolicyType, + } + _, err := c.AppsV1().StatefulSets(ns).Create(context.TODO(), ss, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + ginkgo.By("Confirming all 3 PVCs exist") + err = verifyStatefulSetPVCsExist(c, ss, []int{0, 1, 2}) + framework.ExpectNoError(err) + + ginkgo.By("Scaling stateful set " + ss.Name + " to one replica") + ss, err = e2estatefulset.Scale(c, ss, 1) + framework.ExpectNoError(err) + + ginkgo.By("Verifying all but one PVC deleted") + err = verifyStatefulSetPVCsExist(c, ss, []int{0}) + framework.ExpectNoError(err) + }) + + ginkgo.It("should delete PVCs after adopting pod (WhenDeleted)", func() { + e2epv.SkipIfNoDefaultStorageClass(c) + ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns) + *(ss.Spec.Replicas) = 3 + ss.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: appsv1.DeletePersistentVolumeClaimRetentionPolicyType, + } + _, err := c.AppsV1().StatefulSets(ns).Create(context.TODO(), ss, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + ginkgo.By("Confirming all 3 PVCs exist with their owner refs") + err = verifyStatefulSetPVCsExistWithOwnerRefs(c, ss, []int{0, 1, 2}, true, false) + framework.ExpectNoError(err) + + ginkgo.By("Orphaning the 3rd pod") + patch, err := json.Marshal(metav1.ObjectMeta{ + OwnerReferences: []metav1.OwnerReference{}, + }) + framework.ExpectNoError(err, "Could not Marshal JSON for patch payload") + _, err = c.CoreV1().Pods(ns).Patch(context.TODO(), fmt.Sprintf("%s-2", ss.Name), types.StrategicMergePatchType, []byte(patch), metav1.PatchOptions{}, "") + framework.ExpectNoError(err, "Could not patch payload") + + ginkgo.By("Deleting stateful set " + ss.Name) + err = c.AppsV1().StatefulSets(ns).Delete(context.TODO(), ss.Name, metav1.DeleteOptions{}) + framework.ExpectNoError(err) + + ginkgo.By("Verifying PVCs deleted") + err = verifyStatefulSetPVCsExist(c, ss, []int{}) + framework.ExpectNoError(err) + }) + + ginkgo.It("should delete PVCs after adopting pod (WhenScaled) [Feature:StatefulSetAutoDeletePVC]", func() { + e2epv.SkipIfNoDefaultStorageClass(c) + ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns) + *(ss.Spec.Replicas) = 3 + ss.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenScaled: appsv1.DeletePersistentVolumeClaimRetentionPolicyType, + } + _, err := c.AppsV1().StatefulSets(ns).Create(context.TODO(), ss, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + ginkgo.By("Confirming all 3 PVCs exist") + err = verifyStatefulSetPVCsExist(c, ss, []int{0, 1, 2}) + framework.ExpectNoError(err) + + ginkgo.By("Orphaning the 3rd pod") + patch, err := json.Marshal(metav1.ObjectMeta{ + OwnerReferences: []metav1.OwnerReference{}, + }) + framework.ExpectNoError(err, "Could not Marshal JSON for patch payload") + _, err = c.CoreV1().Pods(ns).Patch(context.TODO(), fmt.Sprintf("%s-2", ss.Name), types.StrategicMergePatchType, []byte(patch), metav1.PatchOptions{}, "") + framework.ExpectNoError(err, "Could not patch payload") + + ginkgo.By("Scaling stateful set " + ss.Name + " to one replica") + ss, err = e2estatefulset.Scale(c, ss, 1) + framework.ExpectNoError(err) + + ginkgo.By("Verifying all but one PVC deleted") + err = verifyStatefulSetPVCsExist(c, ss, []int{0}) + framework.ExpectNoError(err) + }) + }) }) func kubectlExecWithRetries(ns string, args ...string) (out string) { @@ -1645,3 +1786,111 @@ func getStatefulSet(c clientset.Interface, namespace, name string) *appsv1.State } return ss } + +// verifyStatefulSetPVCsExist confirms that exactly the PVCs for ss with the specified ids exist. This polls until the situation occurs, an error happens, or until timeout (in the latter case an error is also returned). Beware that this cannot tell if a PVC will be deleted at some point in the future, so if used to confirm that no PVCs are deleted, the caller should wait for some event giving the PVCs a reasonable chance to be deleted, before calling this function. +func verifyStatefulSetPVCsExist(c clientset.Interface, ss *appsv1.StatefulSet, claimIds []int) error { + idSet := map[int]struct{}{} + for _, id := range claimIds { + idSet[id] = struct{}{} + } + return wait.PollImmediate(e2estatefulset.StatefulSetPoll, e2estatefulset.StatefulSetTimeout, func() (bool, error) { + pvcList, err := c.CoreV1().PersistentVolumeClaims(ss.Namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: klabels.Everything().String()}) + if err != nil { + framework.Logf("WARNING: Failed to list pvcs for verification, retrying: %v", err) + return false, nil + } + for _, claim := range ss.Spec.VolumeClaimTemplates { + pvcNameRE := regexp.MustCompile(fmt.Sprintf("^%s-%s-([0-9]+)$", claim.Name, ss.Name)) + seenPVCs := map[int]struct{}{} + for _, pvc := range pvcList.Items { + matches := pvcNameRE.FindStringSubmatch(pvc.Name) + if len(matches) != 2 { + continue + } + ordinal, err := strconv.ParseInt(matches[1], 10, 32) + if err != nil { + framework.Logf("ERROR: bad pvc name %s (%v)", pvc.Name, err) + return false, err + } + if _, found := idSet[int(ordinal)]; !found { + return false, nil // Retry until the PVCs are consistent. + } else { + seenPVCs[int(ordinal)] = struct{}{} + } + } + if len(seenPVCs) != len(idSet) { + framework.Logf("Found %d of %d PVCs", len(seenPVCs), len(idSet)) + return false, nil // Retry until the PVCs are consistent. + } + } + return true, nil + }) +} + +// verifyStatefulSetPVCsExistWithOwnerRefs works as verifyStatefulSetPVCsExist, but also waits for the ownerRefs to match. +func verifyStatefulSetPVCsExistWithOwnerRefs(c clientset.Interface, ss *appsv1.StatefulSet, claimIndicies []int, wantSetRef, wantPodRef bool) error { + indexSet := map[int]struct{}{} + for _, id := range claimIndicies { + indexSet[id] = struct{}{} + } + set := getStatefulSet(c, ss.Namespace, ss.Name) + setUID := set.GetUID() + if setUID == "" { + framework.Failf("Statefulset %s mising UID", ss.Name) + } + return wait.PollImmediate(e2estatefulset.StatefulSetPoll, e2estatefulset.StatefulSetTimeout, func() (bool, error) { + pvcList, err := c.CoreV1().PersistentVolumeClaims(ss.Namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: klabels.Everything().String()}) + if err != nil { + framework.Logf("WARNING: Failed to list pvcs for verification, retrying: %v", err) + return false, nil + } + for _, claim := range ss.Spec.VolumeClaimTemplates { + pvcNameRE := regexp.MustCompile(fmt.Sprintf("^%s-%s-([0-9]+)$", claim.Name, ss.Name)) + seenPVCs := map[int]struct{}{} + for _, pvc := range pvcList.Items { + matches := pvcNameRE.FindStringSubmatch(pvc.Name) + if len(matches) != 2 { + continue + } + ordinal, err := strconv.ParseInt(matches[1], 10, 32) + if err != nil { + framework.Logf("ERROR: bad pvc name %s (%v)", pvc.Name, err) + return false, err + } + if _, found := indexSet[int(ordinal)]; !found { + framework.Logf("Unexpected, retrying") + return false, nil // Retry until the PVCs are consistent. + } + var foundSetRef, foundPodRef bool + for _, ref := range pvc.GetOwnerReferences() { + if ref.Kind == "StatefulSet" && ref.UID == setUID { + foundSetRef = true + } + if ref.Kind == "Pod" { + podName := fmt.Sprintf("%s-%d", ss.Name, ordinal) + pod, err := c.CoreV1().Pods(ss.Namespace).Get(context.TODO(), podName, metav1.GetOptions{}) + if err != nil { + framework.Logf("Pod %s not found, retrying (%v)", podName, err) + return false, nil + } + podUID := pod.GetUID() + if podUID == "" { + framework.Failf("Pod %s is missing UID", pod.Name) + } + if ref.UID == podUID { + foundPodRef = true + } + } + } + if foundSetRef == wantSetRef && foundPodRef == wantPodRef { + seenPVCs[int(ordinal)] = struct{}{} + } + } + if len(seenPVCs) != len(indexSet) { + framework.Logf("Only %d PVCs, retrying", len(seenPVCs)) + return false, nil // Retry until the PVCs are consistent. + } + } + return true, nil + }) +}