[v0.9] Keep CRDs when deleting a Bundle (#2024)

Co-authored-by: Corentin Néau <tan.neau@suse.com>

charts/fleet-crd/templates/crds.yaml

@@ -159,6 +159,10 @@ spec:
                         enforce or lock down the deployment to a specific namespace.
                       nullable: true
                       type: string
+                    deleteCRDResources:
+                      description: DeleteCRDResources deletes CRDs. Warning! this
+                        will also delete all your Custom Resources.
+                      type: boolean
                     diff:
                       description: Diff can be used to ignore the modified state of
                         objects which are amended at runtime.
@@ -464,6 +468,10 @@ spec:
                         enforce or lock down the deployment to a specific namespace.
                       nullable: true
                       type: string
+                    deleteCRDResources:
+                      description: DeleteCRDResources deletes CRDs. Warning! this
+                        will also delete all your Custom Resources.
+                      type: boolean
                     diff:
                       description: Diff can be used to ignore the modified state of
                         objects which are amended at runtime.
@@ -1105,6 +1113,10 @@ spec:
                     or lock down the deployment to a specific namespace.
                   nullable: true
                   type: string
+                deleteCRDResources:
+                  description: DeleteCRDResources deletes CRDs. Warning! this will
+                    also delete all your Custom Resources.
+                  type: boolean
                 dependsOn:
                   description: DependsOn refers to the bundles which must be ready
                     before this bundle can be deployed.
@@ -1873,6 +1885,10 @@ spec:
                           namespace.
                         nullable: true
                         type: string
+                      deleteCRDResources:
+                        description: DeleteCRDResources deletes CRDs. Warning! this
+                          will also delete all your Custom Resources.
+                        type: boolean
                       diff:
                         description: Diff can be used to ignore the modified state
                           of objects which are amended at runtime.

internal/helmdeployer/deployer.go

@@ -46,6 +46,7 @@ const (
 	KeepResourcesAnnotation      = "fleet.cattle.io/keep-resources"
 	HelmUpgradeInterruptedError  = "another operation (install/upgrade/rollback) is in progress"
 	MaxHelmHistory               = 2
+	CRDKind                      = "CustomResourceDefinition"
 )
 
 var (
@@ -166,6 +167,10 @@ func (p *postRender) Run(renderedManifests *bytes.Buffer) (modifiedManifests *by
 		if err != nil {
 			return nil, err
 		}
+		if !p.opts.DeleteCRDResources &&
+			obj.GetObjectKind().GroupVersionKind().Kind == CRDKind {
+			annotations[kube.ResourcePolicyAnno] = kube.KeepPolicy
+		}
 		m.SetLabels(mergeMaps(m.GetLabels(), labels))
 		m.SetAnnotations(mergeMaps(m.GetAnnotations(), annotations))
 
@@ -185,7 +190,6 @@ func (p *postRender) Run(renderedManifests *bytes.Buffer) (modifiedManifests *by
 			m.SetNamespace(p.opts.TargetNamespace)
 		}
 	}
-
 	data, err = yaml.ToBytes(objs)
 	return bytes.NewBuffer(data), err
 }

internal/helmdeployer/deployer_test.go

@@ -1,14 +1,24 @@
 package helmdeployer
 
 import (
+	"bytes"
 	"fmt"
 	"runtime"
 	"testing"
 
-	"github.com/stretchr/testify/assert"
+	"github.com/rancher/fleet/internal/manifest"
+	"github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
+	"github.com/rancher/wrangler/pkg/yaml"
 
+	"github.com/google/go-cmp/cmp"
+	"github.com/stretchr/testify/assert"
+	"helm.sh/helm/v3/pkg/chart"
+	"helm.sh/helm/v3/pkg/kube"
 	corev1 "k8s.io/api/core/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	kruntime "k8s.io/apimachinery/pkg/runtime"
 )
 
 func TestValuesFrom(t *testing.T) {
@@ -60,3 +70,103 @@ func TestValuesFrom(t *testing.T) {
 	totalValues = mergeValues(totalValues, configMapValues)
 	a.Equal(expected, totalValues)
 }
+
+func TestPostRenderer_Run_DeleteCRDs(t *testing.T) {
+	tests := map[string]struct {
+		obj                 kruntime.Object
+		opts                v1alpha1.BundleDeploymentOptions
+		expectedAnnotations map[string]string
+	}{
+		"default (no DeleteCRDResources specified)": {
+			obj: &apiextensionsv1.CustomResourceDefinition{
+				TypeMeta: metav1.TypeMeta{
+					Kind:       CRDKind,
+					APIVersion: "apiextensions.k8s.io/v1",
+				},
+			},
+			opts: v1alpha1.BundleDeploymentOptions{},
+			expectedAnnotations: map[string]string{
+				kube.ResourcePolicyAnno:      kube.KeepPolicy,
+				"objectset.rio.cattle.io/id": "-",
+			},
+		},
+		"DeleteCRDResources set to true": {
+			obj: &apiextensionsv1.CustomResourceDefinition{
+				TypeMeta: metav1.TypeMeta{
+					Kind:       CRDKind,
+					APIVersion: "apiextensions.k8s.io/v1",
+				},
+			},
+			opts: v1alpha1.BundleDeploymentOptions{
+				DeleteCRDResources: true,
+			},
+			expectedAnnotations: map[string]string{
+				"objectset.rio.cattle.io/id": "-",
+			},
+		},
+		"DeleteCRDResources set to false": {
+			obj: &apiextensionsv1.CustomResourceDefinition{
+				TypeMeta: metav1.TypeMeta{
+					Kind:       CRDKind,
+					APIVersion: "apiextensions.k8s.io/v1",
+				},
+			},
+			opts: v1alpha1.BundleDeploymentOptions{
+				DeleteCRDResources: false,
+			},
+			expectedAnnotations: map[string]string{
+				kube.ResourcePolicyAnno:      kube.KeepPolicy,
+				"objectset.rio.cattle.io/id": "-",
+			},
+		},
+		"Annotation not added for non CRDs resources": {
+			obj: &corev1.Pod{
+				TypeMeta: metav1.TypeMeta{
+					Kind: "Pod",
+				},
+			},
+			opts: v1alpha1.BundleDeploymentOptions{
+				DeleteCRDResources: false,
+			},
+			expectedAnnotations: map[string]string{
+				"objectset.rio.cattle.io/id": "-",
+			},
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			data, err := yaml.ToBytes([]kruntime.Object{test.obj})
+			if err != nil {
+				t.Errorf("unexpected error %v", err)
+			}
+			renderedManifests := bytes.NewBuffer(data)
+
+			pr := postRender{
+				manifest: &manifest.Manifest{
+					Resources: []v1alpha1.BundleResource{},
+				},
+				chart: &chart.Chart{},
+				opts:  test.opts,
+			}
+			postRenderedManifests, err := pr.Run(renderedManifests)
+			if err != nil {
+				t.Errorf("unexpected error %v", err)
+			}
+
+			data = postRenderedManifests.Bytes()
+			objs, err := yaml.ToObjects(bytes.NewBuffer(data))
+			if err != nil {
+				t.Errorf("unexpected error %v", err)
+			}
+
+			m, err := meta.Accessor(objs[0])
+			if err != nil {
+				t.Errorf("unexpected error %v", err)
+			}
+			if !cmp.Equal(m.GetAnnotations(), test.expectedAnnotations) {
+				t.Errorf("expected %s, got %s", test.expectedAnnotations, m.GetAnnotations())
+			}
+		})
+	}
+}

pkg/apis/fleet.cattle.io/v1alpha1/bundledeployment_types.go

@@ -78,6 +78,9 @@ type BundleDeploymentOptions struct {
 
 	// NamespaceAnnotations are annotations that will be appended to the namespace created by Fleet.
 	NamespaceAnnotations *map[string]string `json:"namespaceAnnotations,omitempty"`
+
+	// DeleteCRDResources deletes CRDs. Warning! this will also delete all your Custom Resources.
+	DeleteCRDResources bool `json:"deleteCRDResources,omitempty"`
 }
 
 type DiffOptions struct {