Cleanup stale volume operation requests from snapshot operations

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/grpc-ecosystem/go-grpc-middleware v1.2.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.14.5 // indirect
 	github.com/kubernetes-csi/csi-lib-utils v0.7.0
+	github.com/kubernetes-csi/external-snapshotter/client/v4 v4.1.0
 	github.com/onsi/ginkgo v1.16.4
 	github.com/onsi/gomega v1.13.0
 	github.com/prometheus/client_golang v1.11.0

go.sum

@@ -440,6 +440,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kubernetes-csi/csi-lib-utils v0.7.0 h1:t1cS7HTD7z5D7h9iAdjWuHtMxJPb9s1fIv34rxytzqs=
 github.com/kubernetes-csi/csi-lib-utils v0.7.0/go.mod h1:bze+2G9+cmoHxN6+WyG1qT4MDxgZJMLGwc7V4acPNm0=
+github.com/kubernetes-csi/external-snapshotter/client/v4 v4.1.0 h1:DecASDOSUnp0ftwi4aU87joEpZfLv9iMPwNYzrGb9Lc=
+github.com/kubernetes-csi/external-snapshotter/client/v4 v4.1.0/go.mod h1:YBCo4DoEeDndqvAn6eeu0vWM7QdXmHEeI9cFWplmBys=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
 github.com/libopenstorage/openstorage v1.0.0/go.mod h1:Sp1sIObHjat1BeXhfMqLZ14wnOzEhNx2YQedreMcUyc=
 github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0=
@@ -684,8 +686,6 @@ github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1
 github.com/vmware-tanzu/vm-operator-api v0.1.3 h1:4vxewu0jAN3fSoCBI6FhjmRGJ7ci0R2WNu/I6hacTYs=
 github.com/vmware-tanzu/vm-operator-api v0.1.3/go.mod h1:mubK0QMyaA2TbeAmGsu2GVfiqDFppNUAUqoMPoKFgzM=
 github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU=
-github.com/vmware/govmomi v0.26.1-0.20210616182200-66b9538af589 h1:3zexZbYzLhWvG92k4xUWu2pdb05LOFrpsGA3/2Oc4T4=
-github.com/vmware/govmomi v0.26.1-0.20210616182200-66b9538af589/go.mod h1:daTuJEcQosNMXYJOeku0qdBJP9SOLLWB3Mqz8THtv6o=
 github.com/vmware/govmomi v0.26.2-0.20210907233321-5196d831d88e h1:irZ+ONb0L2RGUxFkwt/5BAKxb2nvnP+w+U+6/asl4Qg=
 github.com/vmware/govmomi v0.26.2-0.20210907233321-5196d831d88e/go.mod h1:daTuJEcQosNMXYJOeku0qdBJP9SOLLWB3Mqz8THtv6o=
 github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk=
@@ -919,6 +919,7 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE=
 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

pkg/common/cns-lib/volume/manager.go

@@ -2018,7 +2018,7 @@ func (m *defaultManager) deleteSnapshotWithImprovedIdempotencyCheck(
 		// Reference to the DeleteVolume task on CNS.
 		deleteSnapshotTask *object.Task
 		// Name of the CnsVolumeOperationRequest instance.
-		instanceName = "deletesnapshot-" + snapshotID + "-" + volumeID
+		instanceName = "deletesnapshot-" + volumeID + "+" + snapshotID
 		// Local instance of DeleteSnapshot details that needs to be persisted.
 		volumeOperationDetails *cnsvolumeoperationrequest.VolumeOperationRequestDetails
 		// error

pkg/internalapis/cnsvolumeoperationrequest/cnsvolumeoperationrequest.go

@@ -346,6 +346,35 @@ func (or *operationRequestStore) cleanupStaleInstances(cleanupInterval int) {
 			}
 		}
 
+		snapshotterClient, err := k8s.NewSnapshotterClient(ctx)
+		if err != nil {
+			log.Errorf("failed to get snapshotterClient with error: %v. Abandoning CnsVolumeOperationRequests "+
+				"clean up ...", err)
+			continue
+		}
+
+		vscList, err := snapshotterClient.SnapshotV1().VolumeSnapshotContents().List(ctx, metav1.ListOptions{})
+		if err != nil {
+			log.Errorf("failed to list VolumeSnapshotContents with error %v. Abandoning "+
+				"CnsVolumeOperationRequests clean up ...", err)
+			continue
+		}
+
+		for _, vsc := range vscList.Items {
+			if vsc.Spec.Driver != csitypes.Name {
+				continue
+			}
+			volumeHandle := vsc.Spec.Source.VolumeHandle
+			if volumeHandle != nil {
+				// CnsVolumeOperation instance for CreateSnapshot
+				instanceMap[strings.TrimPrefix(vsc.Name, "snapcontent-")+"-"+*volumeHandle] = true
+			}
+			if vsc.Status != nil && vsc.Status.SnapshotHandle != nil {
+				// CnsVolumeOperation instance for DeleteSnapshot
+				instanceMap[*vsc.Status.SnapshotHandle] = true
+			}
+		}
+
 		for _, instance := range cnsVolumeOperationRequestList.Items {
 			latestOperationDetailsLength := len(instance.Status.LatestOperationDetails)
 			if latestOperationDetailsLength != 0 &&
@@ -361,6 +390,10 @@ func (or *operationRequestStore) cleanupStaleInstances(cleanupInterval int) {
 				trimmedName = strings.TrimPrefix(instance.Name, "delete-")
 			case strings.HasPrefix(instance.Name, "expand"):
 				trimmedName = strings.TrimPrefix(instance.Name, "expand-")
+			case strings.HasPrefix(instance.Name, "snapshot"):
+				trimmedName = strings.TrimPrefix(instance.Name, "snapshot-")
+			case strings.HasPrefix(instance.Name, "deletesnapshot"):
+				trimmedName = strings.TrimPrefix(instance.Name, "deletesnapshot-")
 			}
 			if _, ok := instanceMap[trimmedName]; !ok {
 				err = or.deleteRequestDetails(ctx, instance.Name)

pkg/kubernetes/kubernetes.go

@@ -47,6 +47,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	apiutils "sigs.k8s.io/controller-runtime/pkg/client/apiutil"
 
+	snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned"
 	cnsoperatorv1alpha1 "sigs.k8s.io/vsphere-csi-driver/v2/pkg/apis/cnsoperator"
 	migrationv1alpha1 "sigs.k8s.io/vsphere-csi-driver/v2/pkg/apis/migration/v1alpha1"
 	cnsvsphere "sigs.k8s.io/vsphere-csi-driver/v2/pkg/common/cns-lib/vsphere"
@@ -104,6 +105,17 @@ func NewClient(ctx context.Context) (clientset.Interface, error) {
 	return clientset.NewForConfig(config)
 }
 
+// NewSnapshotterClient creates a new external-snapshotter client based on a service account.
+func NewSnapshotterClient(ctx context.Context) (snapshotterClientSet.Interface, error) {
+	log := logger.GetLogger(ctx)
+	config, err := GetKubeConfig(ctx)
+	if err != nil {
+		log.Errorf("Failed to get KubeConfig. err: %v", err)
+		return nil, err
+	}
+	return snapshotterClientSet.NewForConfig(config)
+}
+
 // GetRestClientConfigForSupervisor returns restclient config for given
 // endpoint, port, certificate and token.
 func GetRestClientConfigForSupervisor(ctx context.Context, endpoint string, port string) *restclient.Config {