Allow deletion pvc when tenant is deleted (#268)

Add finalizer for the tenant.

operator-kustomize/cluster-role.yaml

@@ -3,6 +3,14 @@ kind: ClusterRole
 metadata:
   name: minio-operator-role
 rules:
+  - apiGroups:
+      - ""
+    resources:
+      - persistentvolumeclaims
+    verbs:
+      - get
+      - update
+      - list
   - apiGroups:
       - ""
     resources:

operator-kustomize/crds/minio.min.io_tenants.yaml

@@ -479,6 +479,12 @@ spec:
                 importance of a Pod relative to other Pods. This is applied to MinIO
                 pods only. Refer Kubernetes documentation for details https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass
               type: string
+            purgePVCOnTenantDelete:
+              description: Delete PVCs on tenant deletion. Defaults to (false) preserving
+                PVCs if tenant is deleted. PVCs will need cleanup post tenant deletion
+                if not set to true. If set to true ALL PVs for the tenant will be
+                deleted.
+              type: boolean
             requestAutoCert:
               description: 'RequestAutoCert allows user to enable Kubernetes based
                 TLS cert generation and signing as explained here: https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/'

pkg/apis/minio.min.io/v1/names.go

@@ -39,6 +39,9 @@ const ConsoleContainerName = "console"
 // InitContainerImage name for init container.
 const InitContainerImage = "busybox:1.32"
 
+// Finalizer name used for resources operator wants to track for deletion
+const Finalizer = "finalizer.minio.min.io"
+
 // MinIO Related Names
 
 // MinIOStatefulSetNameForZone returns the name for MinIO StatefulSet

pkg/apis/minio.min.io/v1/types.go

@@ -51,6 +51,10 @@ type TenantScheduler struct {
 type TenantSpec struct {
 	// Definition for Cluster in given MinIO cluster
 	Zones []Zone `json:"zones"`
+	// Delete PVCs on tenant deletion. Defaults to (false) preserving PVCs if tenant is deleted. PVCs will
+	// need cleanup post tenant deletion if not set to true. If set to true ALL PVs for the tenant will be deleted.
+	// +optional
+	PurgePVCOnTenantDelete bool `json:"purgePVCOnTenantDelete,omitempty"`
 	// Image defines the Tenant Docker image.
 	// +optional
 	Image string `json:"image,omitempty"`

pkg/controller/cluster/main-controller.go

@@ -705,6 +705,13 @@ func (c *Controller) syncHandler(key string) error {
 	}
 	// Set any required default values and init Global variables
 	nsName := types.NamespacedName{Namespace: namespace, Name: name}
+
+	// Update tenant before setting defaults
+	mi, err = c.applyFinalizer(ctx, mi)
+	if err != nil {
+		return err
+	}
+
 	mi.EnsureDefaults()
 	miniov1.InitGlobals(mi)
 
@@ -723,6 +730,16 @@ func (c *Controller) syncHandler(key string) error {
 		return err
 	}
 
+	// Process deletion
+	if !mi.ObjectMeta.DeletionTimestamp.IsZero() {
+		klog.Infof("deleting tenant:%s/%s", mi.Namespace, mi.Name)
+		err = c.processDelete(ctx, mi)
+		if err != nil {
+			return err
+		}
+		return err
+	}
+
 	// check if both auto certificate creation and external secret with certificate is passed,
 	// this is an error as only one of this is allowed in one Tenant
 	if mi.AutoCert() && (mi.ExternalCert() || mi.ExternalClientCert() || mi.KESExternalCert() || mi.ConsoleExternalCert()) {
@@ -887,6 +904,13 @@ func (c *Controller) syncHandler(key string) error {
 					return err
 				}
 			}
+
+			// Set the PVC purge  behavior needed.
+			err = c.processPurgePVCsOnDeleteFlag(ctx, mi)
+			if err != nil {
+				return err
+			}
+
 		}
 
 		// If the StatefulSet is not controlled by this Tenant resource, we should log
@@ -1296,3 +1320,104 @@ func (c *Controller) checkAndCreateConsoleCSR(ctx context.Context, nsName types.
 	}
 	return nil
 }
+
+// getPodsForTenant returns a list of pods running on a given tenant
+func (c *Controller) getPodsForTenant(tenant *miniov1.Tenant) ([]corev1.Pod, error) {
+	pods, err := c.kubeClientSet.CoreV1().Pods(tenant.Namespace).List(context.Background(), metav1.ListOptions{
+		LabelSelector: fmt.Sprintf("%s=%s", miniov1.TenantLabel, tenant.Name),
+	})
+	if err != nil {
+		return nil, err
+	}
+	return pods.Items, nil
+}
+
+// processPurgePVCsOnDelete sets the owner reference for PVC to the tenant. This allows automatic deletion
+// of the PVCs when the tenant is deleted.
+func (c *Controller) processPurgePVCsOnDeleteFlag(ctx context.Context, mi *miniov1.Tenant) error {
+
+	pods, err := c.getPodsForTenant(mi)
+	if err != nil {
+		// No pods created for tenant
+		if k8serrors.IsNotFound(err) {
+			return nil
+		}
+		return err
+	}
+	for _, pod := range pods {
+		for _, pvc := range pod.Spec.Volumes {
+			if pvc.PersistentVolumeClaim != nil {
+				p, err := c.kubeClientSet.CoreV1().PersistentVolumeClaims(mi.Namespace).Get(ctx, pvc.PersistentVolumeClaim.ClaimName, metav1.GetOptions{})
+				if err != nil {
+					// No claims found
+					if k8serrors.IsNotFound(err) {
+						return nil
+					}
+					return err
+				}
+				update := true
+				if mi.Spec.PurgePVCOnTenantDelete {
+					// No need to add if already present
+					for _, o := range p.OwnerReferences {
+						if o.UID == mi.OwnerRef()[0].UID {
+							update = false
+							break
+						}
+					}
+					if update {
+						p.OwnerReferences = append(p.OwnerReferences, mi.OwnerRef()...)
+					}
+				} else {
+					update = false
+					// If UID is set remove and update
+					for i, o := range mi.OwnerReferences {
+						// Loop through ALL to remove the UID reference to tenant.
+						if o.UID == mi.OwnerRef()[0].UID {
+							p.OwnerReferences = append(p.OwnerReferences[:i], p.OwnerReferences[i+1:]...)
+							update = true
+						}
+					}
+				}
+				if update {
+					_, err = c.kubeClientSet.CoreV1().PersistentVolumeClaims(mi.Namespace).Update(ctx, p, metav1.UpdateOptions{})
+					if err != nil {
+						return err
+					}
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func (c *Controller) applyFinalizer(ctx context.Context, mi *miniov1.Tenant) (*miniov1.Tenant, error) {
+	found := false
+	for _, f := range mi.Finalizers {
+		if f == miniov1.Finalizer {
+			found = true
+			break
+		}
+	}
+	var err error
+	if !found {
+		mi.ObjectMeta.Finalizers = append(mi.ObjectMeta.Finalizers, miniov1.Finalizer)
+		mi, err = c.minioClientSet.MinioV1().Tenants(mi.Namespace).Update(ctx, mi, metav1.UpdateOptions{})
+	}
+	return mi, err
+}
+
+func (c *Controller) processDelete(ctx context.Context, mi *miniov1.Tenant) error {
+	// Set the PVC purge behavior needed. This will ensure that k8s correctly deletes all PVCs.
+	err := c.processPurgePVCsOnDeleteFlag(ctx, mi)
+	if err != nil {
+		return err
+	}
+	// Remove finalizer
+	for i, f := range mi.ObjectMeta.Finalizers {
+		if f == miniov1.Finalizer {
+			mi.ObjectMeta.Finalizers = append(mi.Finalizers[:i], mi.ObjectMeta.Finalizers[i+1:]...)
+		}
+	}
+	_, err = c.minioClientSet.MinioV1().Tenants(mi.Namespace).Update(ctx, mi, metav1.UpdateOptions{})
+	return err
+}

pkg/resources/statefulsets/minio-statefulset.go

@@ -447,6 +447,15 @@ func NewForMinIOZone(t *miniov1.Tenant, wsSecret *v1.Secret, zone *miniov1.Zone,
 
 	if zone.VolumeClaimTemplate != nil {
 		pvClaim := *zone.VolumeClaimTemplate
+		if t.Spec.PurgePVCOnTenantDelete {
+			pvClaim.OwnerReferences = []metav1.OwnerReference{
+				*metav1.NewControllerRef(t, schema.GroupVersionKind{
+					Group:   miniov1.SchemeGroupVersion.Group,
+					Version: miniov1.SchemeGroupVersion.Version,
+					Kind:    miniov1.MinIOCRDResourceKind,
+				}),
+			}
+		}
 		name := pvClaim.Name
 		for i := 0; i < int(zone.VolumesPerServer); i++ {
 			pvClaim.Name = name + strconv.Itoa(i)