Implement RetentionPolicy for VolumeSnapshot (#926)

Author: suaas21

pkg/cmds/create_volumesnapshot.go

@@ -22,6 +22,7 @@ import (
 	"stash.appscode.dev/stash/pkg/restic"
 	"stash.appscode.dev/stash/pkg/status"
 	"stash.appscode.dev/stash/pkg/util"
+	"stash.appscode.dev/stash/pkg/volumesnapshot"
 )
 
 type VSoption struct {
@@ -139,6 +140,12 @@ func (opt *VSoption) createVolumeSnapshot() (*restic.BackupOutput, error) {
 		}
 
 	}
+
+	err = volumesnapshot.CleanupSnapshots(backupConfig.Spec.RetentionPolicy, backupOutput.HostBackupStats, backupSession.Namespace, opt.snapshotClient)
+	if err != nil {
+		return nil, err
+	}
+
 	return backupOutput, nil
 }
 

pkg/rbac/volume_snapshot.go

@@ -80,7 +80,7 @@ func ensureVolumeSnapshotterJobClusterRole(kubeClient kubernetes.Interface, labe
 			{
 				APIGroups: []string{crdv1.GroupName},
 				Resources: []string{"volumesnapshots", "volumesnapshotcontents", "volumesnapshotclasses"},
-				Verbs:     []string{"create", "get", "list", "watch", "patch"},
+				Verbs:     []string{"create", "get", "list", "watch", "patch", "delete"},
 			},
 		}
 		return in

pkg/volumesnapshot/volumesnapshot_cleanup_policy.go

@@ -0,0 +1,163 @@
+package volumesnapshot
+
+import (
+	"sort"
+	"time"
+
+	vs_api "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1alpha1"
+	vs_cs "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned"
+	kerr "k8s.io/apimachinery/pkg/api/errors"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"stash.appscode.dev/stash/apis/stash/v1alpha1"
+	"stash.appscode.dev/stash/apis/stash/v1beta1"
+)
+
+// Some of the code of this file has been copied from restic/restic repository.
+// ref: https://github.com/restic/restic/blob/604b18aa7426148a55f76307ca729e829ff6b61d/internal/restic/snapshot_policy.go#L152:6
+
+// isPolicyEmpty returns true if no policy has been configured (all values zero).
+func isPolicyEmpty(policy v1alpha1.RetentionPolicy) bool {
+	if policy.KeepLast > 0 ||
+		policy.KeepHourly > 0 ||
+		policy.KeepDaily > 0 ||
+		policy.KeepMonthly > 0 ||
+		policy.KeepWeekly > 0 ||
+		policy.KeepYearly > 0 {
+		return true
+	}
+	return false
+}
+
+// ymdh returns an integer in the form YYYYMMDDHH.
+func ymdh(d time.Time, _ int) int {
+	return d.Year()*1000000 + int(d.Month())*10000 + d.Day()*100 + d.Hour()
+}
+
+// ymd returns an integer in the form YYYYMMDD.
+func ymd(d time.Time, _ int) int {
+	return d.Year()*10000 + int(d.Month())*100 + d.Day()
+}
+
+// yw returns an integer in the form YYYYWW, where WW is the week number.
+func yw(d time.Time, _ int) int {
+	year, week := d.ISOWeek()
+	return year*100 + week
+}
+
+// ym returns an integer in the form YYYYMM.
+func ym(d time.Time, _ int) int {
+	return d.Year()*100 + int(d.Month())
+}
+
+// y returns the year of d.
+func y(d time.Time, _ int) int {
+	return d.Year()
+}
+
+// always returns a unique number for d.
+func always(d time.Time, nr int) int {
+	return nr
+}
+
+type VolumeSnapshot struct {
+	VolumeSnap vs_api.VolumeSnapshot
+}
+
+type VolumeSnapshots []VolumeSnapshot
+
+func (vs VolumeSnapshots) Len() int {
+	return len(vs)
+}
+func (vs VolumeSnapshots) Less(i, j int) bool {
+	return vs[i].VolumeSnap.CreationTimestamp.Time.After(vs[j].VolumeSnap.CreationTimestamp.Time)
+}
+func (vs VolumeSnapshots) Swap(i, j int) {
+	vs[i], vs[j] = vs[j], vs[i]
+}
+
+// ApplyRetentionPolicy do the following steps:
+// 1. sorts all the VolumeSnapshot according to CreationTimeStamp.
+// 2. then list that are to be kept and removed according to the policy.
+// 3. remove VolumeSnapshot that are not necessary according to RetentionPolicy
+func applyRetentionPolicy(policy v1alpha1.RetentionPolicy, volumeSnapshots VolumeSnapshots, namespace string, vsClient vs_cs.Interface) error {
+
+	// sorts the VolumeSnapshots according to CreationTimeStamp
+	sort.Sort(VolumeSnapshots(volumeSnapshots))
+
+	if !isPolicyEmpty(policy) {
+		return nil
+	}
+
+	var buckets = [6]struct {
+		Count     int
+		LastAdded func(d time.Time, nr int) int
+		Last      int
+	}{
+		{policy.KeepLast, always, -1},
+		{policy.KeepHourly, ymdh, -1},
+		{policy.KeepDaily, ymd, -1},
+		{policy.KeepWeekly, yw, -1},
+		{policy.KeepMonthly, ym, -1},
+		{policy.KeepYearly, y, -1},
+	}
+
+	var kept, removed VolumeSnapshots
+	for nr, vs := range volumeSnapshots {
+		var keepSnap bool
+		// keep VolumeSnapshot that are matched with the policy
+		for i, b := range buckets {
+			if b.Count > 0 {
+				val := b.LastAdded(vs.VolumeSnap.CreationTimestamp.Time, nr)
+				if val != b.Last {
+					keepSnap = true
+					buckets[i].Last = val
+					buckets[i].Count--
+				}
+			}
+		}
+
+		if keepSnap {
+			kept = append(kept, vs)
+		} else {
+			removed = append(removed, vs)
+		}
+	}
+
+	for _, vs := range removed {
+		err := vsClient.SnapshotV1alpha1().VolumeSnapshots(namespace).Delete(vs.VolumeSnap.Name, &v1.DeleteOptions{})
+		if err != nil {
+			if kerr.IsNotFound(err) {
+				return nil
+			}
+			return err
+		}
+	}
+
+	return nil
+}
+
+func CleanupSnapshots(policy v1alpha1.RetentionPolicy, hostBackupStats []v1beta1.HostBackupStats, namespace string, vsClient vs_cs.Interface) error {
+	vsList, err := vsClient.SnapshotV1alpha1().VolumeSnapshots(namespace).List(v1.ListOptions{})
+	if err != nil {
+		if kerr.IsNotFound(err) || len(vsList.Items) == 0 {
+			return nil
+		}
+		return err
+	}
+	// filter VolumeSnapshots according to PVC source
+	// then apply RetentionPolicy rule
+	for _, host := range hostBackupStats {
+		var volumeSnapshots VolumeSnapshots
+		for _, vs := range vsList.Items {
+			if host.Hostname == vs.Spec.Source.Name {
+				volumeSnapshots = append(volumeSnapshots, VolumeSnapshot{VolumeSnap: vs})
+			}
+		}
+		err := applyRetentionPolicy(policy, volumeSnapshots, namespace, vsClient)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

pkg/volumesnapshot/volumesnapshot_cleanup_policy_test.go

@@ -0,0 +1,194 @@
+package volumesnapshot
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/appscode/go/strings"
+	type_util "github.com/appscode/go/types"
+	crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1alpha1"
+	vsfake "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned/fake"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"stash.appscode.dev/stash/apis"
+	"stash.appscode.dev/stash/apis/stash/v1alpha1"
+	"stash.appscode.dev/stash/apis/stash/v1beta1"
+)
+
+type snapInfo struct {
+	name         string
+	creationTime string
+	pvcName      string
+}
+
+type testInfo struct {
+	description       string
+	policy            v1alpha1.RetentionPolicy
+	hostBackupStats   []v1beta1.HostBackupStats
+	expectedSnapshots []string
+}
+
+const testNamespace = "vs-retention-policy-test"
+
+func TestCleanupSnapshots(t *testing.T) {
+
+	snapMeta := []snapInfo{
+		{name: "snap-1", creationTime: "2019-12-10T05:36:07Z", pvcName: "pvc-1"},
+		{name: "snap-2", creationTime: "2019-11-10T05:36:07Z", pvcName: "pvc-1"},
+		{name: "snap-3", creationTime: "2019-10-10T05:36:07Z", pvcName: "pvc-1"},
+		{name: "snap-4", creationTime: "2019-10-10T05:30:07Z", pvcName: "pvc-1"},
+		{name: "snap-5", creationTime: "2019-10-10T05:15:07Z", pvcName: "pvc-1"},
+		{name: "snap-6", creationTime: "2018-10-11T05:36:07Z", pvcName: "pvc-1"},
+		{name: "snap-7", creationTime: "2018-10-10T05:36:07Z", pvcName: "pvc-1"},
+		{name: "snap-8", creationTime: "2017-12-10T05:36:07Z", pvcName: "pvc-1"},
+		{name: "snap-9", creationTime: "2017-10-10T05:36:07Z", pvcName: "pvc-1"},
+		{name: "snap-10", creationTime: "2019-12-10T05:36:07Z", pvcName: "pvc-2"},
+		{name: "snap-11", creationTime: "2019-10-10T05:36:07Z", pvcName: "pvc-2"},
+		{name: "snap-12", creationTime: "2019-10-10T05:36:07Z", pvcName: "pvc-2"},
+		{name: "snap-13", creationTime: "2019-10-09T05:30:07Z", pvcName: "pvc-2"},
+	}
+
+	testCases := []testInfo{
+		{
+			description:       "No Policy",
+			policy:            v1alpha1.RetentionPolicy{},
+			hostBackupStats:   []v1beta1.HostBackupStats{{Hostname: "pvc-1"}, {Hostname: "pvc-2"}},
+			expectedSnapshots: []string{"snap-1", "snap-2", "snap-3", "snap-4", "snap-5", "snap-6", "snap-7", "snap-8", "snap-9", "snap-10", "snap-11", "snap-12", "snap-13"}, // should keep all snapshots
+		},
+		{
+			description:       "KeepLast",
+			policy:            v1alpha1.RetentionPolicy{KeepLast: 3},
+			hostBackupStats:   []v1beta1.HostBackupStats{{Hostname: "pvc-1"}, {Hostname: "pvc-2"}},
+			expectedSnapshots: []string{"snap-1", "snap-2", "snap-3", "snap-10", "snap-11", "snap-12"}, // should keep last 3 snapshots of claim 1 and last 3 snapshots of claim 2
+		},
+		{
+			description:       "KeepHourly",
+			policy:            v1alpha1.RetentionPolicy{KeepHourly: 3},
+			hostBackupStats:   []v1beta1.HostBackupStats{{Hostname: "pvc-1"}, {Hostname: "pvc-2"}},
+			expectedSnapshots: []string{"snap-1", "snap-2", "snap-3", "snap-10", "snap-11", "snap-13"},
+		},
+		{
+			description:       "KeepDaily",
+			policy:            v1alpha1.RetentionPolicy{KeepDaily: 3},
+			hostBackupStats:   []v1beta1.HostBackupStats{{Hostname: "pvc-1"}, {Hostname: "pvc-2"}},
+			expectedSnapshots: []string{"snap-1", "snap-2", "snap-3", "snap-10", "snap-11", "snap-13"},
+		},
+		{
+			description:       "KeepMonthly",
+			policy:            v1alpha1.RetentionPolicy{KeepMonthly: 3},
+			hostBackupStats:   []v1beta1.HostBackupStats{{Hostname: "pvc-1"}, {Hostname: "pvc-2"}},
+			expectedSnapshots: []string{"snap-1", "snap-2", "snap-3", "snap-10", "snap-11"},
+		},
+		{
+			description:       "KeepYearly",
+			policy:            v1alpha1.RetentionPolicy{KeepYearly: 3},
+			hostBackupStats:   []v1beta1.HostBackupStats{{Hostname: "pvc-1"}, {Hostname: "pvc-2"}},
+			expectedSnapshots: []string{"snap-1", "snap-6", "snap-8", "snap-10"},
+		},
+		{
+			description:       "KeepLast & KeepDaily",
+			policy:            v1alpha1.RetentionPolicy{KeepLast: 3, KeepDaily: 3},
+			hostBackupStats:   []v1beta1.HostBackupStats{{Hostname: "pvc-1"}, {Hostname: "pvc-2"}},
+			expectedSnapshots: []string{"snap-1", "snap-2", "snap-3", "snap-10", "snap-11", "snap-12", "snap-13"},
+		},
+		{
+			description:       "KeepWeekly & KeepMonthly",
+			policy:            v1alpha1.RetentionPolicy{KeepWeekly: 2, KeepMonthly: 2},
+			hostBackupStats:   []v1beta1.HostBackupStats{{Hostname: "pvc-1"}, {Hostname: "pvc-2"}},
+			expectedSnapshots: []string{"snap-1", "snap-2", "snap-10", "snap-11"},
+		},
+		{
+			description:       "KeepWeekly & KeepMonthly & KeepYearly",
+			policy:            v1alpha1.RetentionPolicy{KeepWeekly: 2, KeepMonthly: 3, KeepYearly: 4},
+			hostBackupStats:   []v1beta1.HostBackupStats{{Hostname: "pvc-1"}, {Hostname: "pvc-2"}},
+			expectedSnapshots: []string{"snap-1", "snap-2", "snap-3", "snap-6", "snap-8", "snap-10", "snap-11"},
+		},
+	}
+
+	for _, test := range testCases {
+		t.Run(test.description, func(t *testing.T) {
+			volumeSnasphots, err := getVolumeSnapshots(snapMeta)
+			if err != nil {
+				t.Errorf("Failed to generate VolumeSnasphots. Reason: %v", err)
+				return
+			}
+			vsClient := vsfake.NewSimpleClientset(volumeSnasphots...)
+			err = CleanupSnapshots(test.policy, test.hostBackupStats, testNamespace, vsClient)
+			if err != nil {
+				t.Errorf("Failed to cleanup VolumeSnapshots. Reason: %v", err)
+				return
+			}
+			vsList, err := vsClient.SnapshotV1alpha1().VolumeSnapshots(testNamespace).List(metav1.ListOptions{})
+			if err != nil {
+				t.Errorf("Failed to list remaining VolumeSnapshots. Reason: %v", err)
+				return
+			}
+			if len(test.expectedSnapshots) != len(vsList.Items) {
+				var remainingSnapshots []string
+				for i := range vsList.Items {
+					remainingSnapshots = append(remainingSnapshots, vsList.Items[i].Name)
+				}
+
+				t.Errorf("Remaining VolumeSnapshot number did not match with expected number."+
+					"\nExpected: %d"+
+					"\nFound: %d"+
+					"\nExpected Snapshots: %q"+
+					"\nRemaining Snapshots: %q", len(test.expectedSnapshots), len(vsList.Items), test.expectedSnapshots, remainingSnapshots)
+				return
+			}
+
+			for _, vs := range vsList.Items {
+				if !strings.Contains(test.expectedSnapshots, vs.Name) {
+					t.Errorf("VolumeSnapshot %s should be deleted according to retention-policy: %s.", vs.Name, test.description)
+				}
+			}
+		})
+	}
+
+}
+
+func getVolumeSnapshots(snapMetas []snapInfo) ([]runtime.Object, error) {
+	snapshots := make([]runtime.Object, 0)
+	for i := range snapMetas {
+		snapshot, err := newSnapshot(snapMetas[i])
+		if err != nil {
+			return nil, err
+		}
+
+		snapshots = append(snapshots, snapshot)
+	}
+	return snapshots, nil
+}
+
+func newSnapshot(snapMeta snapInfo) (*crdv1.VolumeSnapshot, error) {
+	creationTimestamp, err := time.Parse(time.RFC3339, snapMeta.creationTime)
+	if err != nil {
+		return nil, err
+	}
+
+	return &crdv1.VolumeSnapshot{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:              snapMeta.name,
+			Namespace:         testNamespace,
+			UID:               types.UID(snapMeta.name),
+			ResourceVersion:   "1",
+			SelfLink:          "/apis/snapshot.storage.k8s.io/v1alpha1/namespaces/" + testNamespace + "/volumesnapshots/" + snapMeta.name,
+			CreationTimestamp: metav1.Time{Time: creationTimestamp},
+		},
+		Spec: crdv1.VolumeSnapshotSpec{
+			VolumeSnapshotClassName: type_util.StringP("standard"),
+			SnapshotContentName:     fmt.Sprintf("snapshot-content-%s", snapMeta.name),
+			Source: &v1.TypedLocalObjectReference{
+				Name: snapMeta.pvcName,
+				Kind: apis.KindPersistentVolumeClaim,
+			},
+		},
+		Status: crdv1.VolumeSnapshotStatus{
+			ReadyToUse: true,
+			Error:      nil,
+		},
+	}, nil
+}