feat: store only secrets that are possible to be used in object cache of dataplane

CHANGELOG.md

@@ -79,6 +79,11 @@ Adding a new version? You'll need three changes:
 - Secrets validation introduced: CA certificates won't be synchronized
   to Kong if the certificate is expired.
   [#3063](https://github.com/Kong/kubernetes-ingress-controller/pull/3063)
+- Changed the logic of storing secrets into object cache. Now only the secrets
+  that are possibly used in Kong configuration are stored into cache, and the 
+  irrelevant secrets (e.g: service account tokens) are not stored. This change
+  is made to reduce memory usage of the cache.
+  [#3047](https://github.com/Kong/kubernetes-ingress-controller/pull/3047)
 
 ### Fixed
 

hack/generators/controllers/networking/main.go

@@ -43,6 +43,7 @@ var inputControllersNeeded = &typesNeeded{
 		NeedsStatusPermissions:            true,
 		AcceptsIngressClassNameAnnotation: false,
 		AcceptsIngressClassNameSpec:       false,
+		NeedsUpdateReferences:             true,
 		RBACVerbs:                         []string{"get", "list", "watch"},
 	},
 	typeNeeded{
@@ -59,20 +60,6 @@ var inputControllersNeeded = &typesNeeded{
 		AcceptsIngressClassNameSpec:       false,
 		RBACVerbs:                         []string{"list", "watch"},
 	},
-	typeNeeded{
-		Group:                             "\"\"",
-		Version:                           "v1",
-		Kind:                              "Secret",
-		PackageImportAlias:                "corev1",
-		PackageAlias:                      "CoreV1",
-		Package:                           corev1,
-		Plural:                            "secrets",
-		CacheType:                         "Secret",
-		NeedsStatusPermissions:            true,
-		AcceptsIngressClassNameAnnotation: false,
-		AcceptsIngressClassNameSpec:       false,
-		RBACVerbs:                         []string{"list", "watch"},
-	},
 	typeNeeded{
 		Group:                             "networking.k8s.io",
 		Version:                           "v1",
@@ -86,6 +73,7 @@ var inputControllersNeeded = &typesNeeded{
 		CapableOfStatusUpdates:            true,
 		AcceptsIngressClassNameAnnotation: true,
 		AcceptsIngressClassNameSpec:       true,
+		NeedsUpdateReferences:             true,
 		RBACVerbs:                         []string{"get", "list", "watch"},
 	},
 	typeNeeded{
@@ -115,6 +103,7 @@ var inputControllersNeeded = &typesNeeded{
 		CapableOfStatusUpdates:            true,
 		AcceptsIngressClassNameAnnotation: true,
 		AcceptsIngressClassNameSpec:       true,
+		NeedsUpdateReferences:             true,
 		RBACVerbs:                         []string{"get", "list", "watch"},
 	},
 	typeNeeded{
@@ -130,6 +119,7 @@ var inputControllersNeeded = &typesNeeded{
 		CapableOfStatusUpdates:            true,
 		AcceptsIngressClassNameAnnotation: true,
 		AcceptsIngressClassNameSpec:       true,
+		NeedsUpdateReferences:             true,
 		RBACVerbs:                         []string{"get", "list", "watch"},
 	},
 	typeNeeded{
@@ -158,6 +148,7 @@ var inputControllersNeeded = &typesNeeded{
 		NeedsStatusPermissions:            true,
 		AcceptsIngressClassNameAnnotation: false,
 		AcceptsIngressClassNameSpec:       false,
+		NeedsUpdateReferences:             true,
 		RBACVerbs:                         []string{"get", "list", "watch"},
 	},
 	typeNeeded{
@@ -172,6 +163,7 @@ var inputControllersNeeded = &typesNeeded{
 		NeedsStatusPermissions:            true,
 		AcceptsIngressClassNameAnnotation: true,
 		AcceptsIngressClassNameSpec:       false,
+		NeedsUpdateReferences:             true,
 		RBACVerbs:                         []string{"get", "list", "watch"},
 	},
 	typeNeeded{
@@ -186,6 +178,7 @@ var inputControllersNeeded = &typesNeeded{
 		NeedsStatusPermissions:            true,
 		AcceptsIngressClassNameAnnotation: true,
 		AcceptsIngressClassNameSpec:       false,
+		NeedsUpdateReferences:             true,
 		RBACVerbs:                         []string{"get", "list", "watch"},
 	},
 	typeNeeded{
@@ -201,6 +194,7 @@ var inputControllersNeeded = &typesNeeded{
 		CapableOfStatusUpdates:            true,
 		AcceptsIngressClassNameAnnotation: true,
 		AcceptsIngressClassNameSpec:       false,
+		NeedsUpdateReferences:             true,
 		RBACVerbs:                         []string{"get", "list", "watch"},
 	},
 	typeNeeded{
@@ -355,6 +349,10 @@ type typeNeeded struct {
 	// CapableOfStatusUpdates indicates that the controllers should manage status
 	// updates for the resource.
 	CapableOfStatusUpdates bool
+
+	// NeedUpdateReferences is true if we need to update the reference relationships
+	// between reconciled object and other objects.
+	NeedsUpdateReferences bool
 }
 
 func (t *typeNeeded) generate(contents *bytes.Buffer) error {
@@ -411,6 +409,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"sigs.k8s.io/controller-runtime/pkg/source"
 
+	ctrlref "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/reference"
 	ctrlutils "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/utils"
 	"github.com/kong/kubernetes-ingress-controller/v2/internal/dataplane"
 	"github.com/kong/kubernetes-ingress-controller/v2/internal/util"
@@ -452,6 +451,9 @@ type {{.PackageAlias}}{{.Kind}}Reconciler struct {
 	IngressClassName string
 	DisableIngressClassLookups bool
 {{- end}}
+{{- if .NeedsUpdateReferences}}
+	ReferenceIndexers ctrlref.CacheIndexers
+{{- end}}
 }
 
 // SetupWithManager sets up the controller with the Manager.
@@ -542,6 +544,12 @@ func (r *{{.PackageAlias}}{{.Kind}}Reconciler) Reconcile(ctx context.Context, re
 		if errors.IsNotFound(err) {
 			obj.Namespace = req.Namespace
 			obj.Name = req.Name
+			{{if .NeedsUpdateReferences}}
+			// remove reference record where the {{.Kind}} is the referrer
+			if err := ctrlref.DeleteReferencesByReferrer(r.ReferenceIndexers, r.DataplaneClient, obj); err != nil {
+				return ctrl.Result{}, err
+			}
+			{{end}}
 			return ctrl.Result{}, r.DataplaneClient.DeleteObject(obj)
 		}
 		return ctrl.Result{}, err
@@ -551,6 +559,12 @@ func (r *{{.PackageAlias}}{{.Kind}}Reconciler) Reconcile(ctx context.Context, re
 	// clean the object up if it's being deleted
 	if !obj.DeletionTimestamp.IsZero() && time.Now().After(obj.DeletionTimestamp.Time) {
 		log.V(util.DebugLevel).Info("resource is being deleted, its configuration will be removed", "type", "{{.Kind}}", "namespace", req.Namespace, "name", req.Name)
+		{{if .NeedsUpdateReferences}}
+		// remove reference record where the {{.Kind}} is the referrer
+		if err := ctrlref.DeleteReferencesByReferrer(r.ReferenceIndexers, r.DataplaneClient, obj); err != nil {
+			return ctrl.Result{}, err
+		}
+		{{end}}
 		objectExistsInCache, err := r.DataplaneClient.ObjectExists(obj)
 		if err != nil {
 			return ctrl.Result{}, err
@@ -589,6 +603,19 @@ func (r *{{.PackageAlias}}{{.Kind}}Reconciler) Reconcile(ctx context.Context, re
 		return ctrl.Result{}, err
 	}
 
+{{- if .NeedsUpdateReferences }}
+	// update reference relationship from the {{.Kind}} to other objects.
+	if err := updateReferredObjects(ctx, r.Client, r.ReferenceIndexers, r.DataplaneClient, obj); err != nil {
+		if errors.IsNotFound(err) {
+			// reconcile again if the secret does not exist yet
+			return ctrl.Result{
+				Requeue: true,
+			}, nil
+		}
+		return ctrl.Result{}, err
+	}
+{{- end }}
+
 {{- if .CapableOfStatusUpdates}}
 	// if status updates are enabled report the status for the object
 	if r.DataplaneClient.AreKubernetesObjectReportsEnabled() {

internal/controllers/configuration/object_references.go

@@ -0,0 +1,157 @@
+package configuration
+
+import (
+	"context"
+
+	corev1 "k8s.io/api/core/v1"
+	extv1beta1 "k8s.io/api/extensions/v1beta1"
+	netv1 "k8s.io/api/networking/v1"
+	netv1beta1 "k8s.io/api/networking/v1beta1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/kong/kubernetes-ingress-controller/v2/internal/annotations"
+	ctrlref "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/reference"
+	"github.com/kong/kubernetes-ingress-controller/v2/internal/dataplane"
+	kongv1 "github.com/kong/kubernetes-ingress-controller/v2/pkg/apis/configuration/v1"
+	kongv1beta1 "github.com/kong/kubernetes-ingress-controller/v2/pkg/apis/configuration/v1beta1"
+)
+
+// updateReferredObjects updates reference records where the referrer is the object in parameter obj.
+// currently it only updates reference records to secrets, since we wanted to limit cache size of secrets:
+// https://github.com/Kong/kubernetes-ingress-controller/issues/2868
+func updateReferredObjects(
+	ctx context.Context, client client.Client, refIndexers ctrlref.CacheIndexers, dataplaneClient *dataplane.KongClient, obj client.Object) error {
+
+	var referredSecretNameMap = make(map[types.NamespacedName]struct{})
+	var referredSecretList []types.NamespacedName
+	switch obj := obj.(type) {
+	// functions update***ReferredSecrets first list the secrets referred by object,
+	// then call UpdateReferencesToSecret to store referrence records between the object and referred secrets,
+	// and also to remove the outdated reference records in cache where the secret is not referred by the obj specification anymore.
+	case *corev1.Service:
+		referredSecretList = listCoreV1ServiceReferredSecrets(obj)
+	case *netv1.Ingress:
+		referredSecretList = listNetV1IngressReferredSecrets(obj)
+	case *netv1beta1.Ingress:
+		referredSecretList = listNetV1beta1IngressReferredSecrets(obj)
+	case *extv1beta1.Ingress:
+		referredSecretList = listExtensionV1beta1IngressReferredSecrets(obj)
+	case *kongv1.KongPlugin:
+		referredSecretList = listKongPluginReferredSecrets(obj)
+	case *kongv1.KongClusterPlugin:
+		referredSecretList = listKongClusterPluginReferredSecrets(obj)
+	case *kongv1.KongConsumer:
+		referredSecretList = listKongConsumerReferredSecrets(obj)
+	case *kongv1beta1.TCPIngress:
+		referredSecretList = listTCPIngressReferredSecrets(obj)
+	}
+
+	for _, nsName := range referredSecretList {
+		referredSecretNameMap[nsName] = struct{}{}
+	}
+	return ctrlref.UpdateReferencesToSecret(ctx, client, refIndexers, dataplaneClient, obj, referredSecretNameMap)
+}
+
+func listCoreV1ServiceReferredSecrets(service *corev1.Service) []types.NamespacedName {
+
+	if service.Annotations == nil {
+		return nil
+	}
+
+	referredSecretNames := make([]types.NamespacedName, 0, 1)
+	secretName := annotations.ExtractClientCertificate(service.Annotations)
+	if secretName != "" {
+		nsName := types.NamespacedName{
+			Namespace: service.Namespace,
+			Name:      secretName,
+		}
+
+		referredSecretNames = append(referredSecretNames, nsName)
+	}
+	return referredSecretNames
+}
+
+func listNetV1IngressReferredSecrets(ingress *netv1.Ingress) []types.NamespacedName {
+	referredSecretNames := make([]types.NamespacedName, 0, len(ingress.Spec.TLS))
+	for _, tls := range ingress.Spec.TLS {
+		nsName := types.NamespacedName{
+			Namespace: ingress.Namespace,
+			Name:      tls.SecretName,
+		}
+		referredSecretNames = append(referredSecretNames, nsName)
+	}
+	return referredSecretNames
+}
+
+func listNetV1beta1IngressReferredSecrets(ingress *netv1beta1.Ingress) []types.NamespacedName {
+	referredSecretNames := make([]types.NamespacedName, 0, len(ingress.Spec.TLS))
+	for _, tls := range ingress.Spec.TLS {
+		nsName := types.NamespacedName{
+			Namespace: ingress.Namespace,
+			Name:      tls.SecretName,
+		}
+		referredSecretNames = append(referredSecretNames, nsName)
+	}
+	return referredSecretNames
+}
+
+func listExtensionV1beta1IngressReferredSecrets(ingress *extv1beta1.Ingress) []types.NamespacedName {
+	referredSecretNames := make([]types.NamespacedName, 0, len(ingress.Spec.TLS))
+	for _, tls := range ingress.Spec.TLS {
+		nsName := types.NamespacedName{
+			Namespace: ingress.Namespace,
+			Name:      tls.SecretName,
+		}
+		referredSecretNames = append(referredSecretNames, nsName)
+	}
+	return referredSecretNames
+}
+
+func listKongPluginReferredSecrets(plugin *kongv1.KongPlugin) []types.NamespacedName {
+	referredSecretNames := make([]types.NamespacedName, 0, 1)
+	if plugin.ConfigFrom != nil {
+		nsName := types.NamespacedName{
+			Namespace: plugin.Namespace,
+			Name:      plugin.ConfigFrom.SecretValue.Secret,
+		}
+		referredSecretNames = append(referredSecretNames, nsName)
+	}
+	return referredSecretNames
+}
+
+func listKongClusterPluginReferredSecrets(plugin *kongv1.KongClusterPlugin) []types.NamespacedName {
+	referredSecretNames := make([]types.NamespacedName, 0, 1)
+	if plugin.ConfigFrom != nil {
+		nsName := types.NamespacedName{
+			Namespace: plugin.ConfigFrom.SecretValue.Namespace,
+			Name:      plugin.ConfigFrom.SecretValue.Secret,
+		}
+		referredSecretNames = append(referredSecretNames, nsName)
+	}
+	return referredSecretNames
+}
+
+func listKongConsumerReferredSecrets(consumer *kongv1.KongConsumer) []types.NamespacedName {
+	referredSecretNames := make([]types.NamespacedName, 0, len(consumer.Credentials))
+	for _, secretName := range consumer.Credentials {
+		nsName := types.NamespacedName{
+			Namespace: consumer.Namespace,
+			Name:      secretName,
+		}
+		referredSecretNames = append(referredSecretNames, nsName)
+	}
+	return referredSecretNames
+}
+
+func listTCPIngressReferredSecrets(tcpIngress *kongv1beta1.TCPIngress) []types.NamespacedName {
+	referredSecretNames := make([]types.NamespacedName, 0, len(tcpIngress.Spec.TLS))
+	for _, tls := range tcpIngress.Spec.TLS {
+		nsName := types.NamespacedName{
+			Namespace: tcpIngress.Namespace,
+			Name:      tls.SecretName,
+		}
+		referredSecretNames = append(referredSecretNames, nsName)
+	}
+	return referredSecretNames
+}