feat(httproute): add ReferenceGrant watch to controller (#3759)

Co-authored-by: Grzegorz Burzyński <czeslavo@gmail.com>
Co-authored-by: Travis Raines <571832+rainest@users.noreply.github.com>

CHANGELOG.md

@@ -86,6 +86,8 @@ Adding a new version? You'll need three changes:
 - Adding the `konghq.com/tags: csv,of,tags` annotation will add tags to
   generated resources.
   [#3778](https://github.com/Kong/kubernetes-ingress-controller/pull/3778)
+- `HTTPRoute` reconciler now watches relevant `ReferenceGrant`s for changes.
+  [#3759](https://github.com/Kong/kubernetes-ingress-controller/pull/3759)
 - Bumped Kong version in manifests to 3.2.
   [#3804](https://github.com/Kong/kubernetes-ingress-controller/pull/3804)
 

internal/controllers/configuration/kongadminapi_controller_envtest_test.go

@@ -60,7 +60,7 @@ func startKongAdminAPIServiceReconciler(ctx context.Context, t *testing.T, clien
 
 	mgr, err := ctrl.NewManager(cfg, ctrl.Options{
 		Logger:             logrusr.New(logrus.New()),
-		Scheme:             scheme.Scheme,
+		Scheme:             client.Scheme(),
 		SyncPeriod:         lo.ToPtr(2 * time.Second),
 		MetricsBindAddress: "0",
 	})
@@ -106,7 +106,7 @@ func TestKongAdminAPIController(t *testing.T) {
 	// In tests below we use a deferred cancel to stop the manager and not wait
 	// for its timeout.
 
-	cfg := envtest.Setup(t)
+	cfg := envtest.Setup(t, scheme.Scheme)
 	client, err := ctrlclient.New(cfg, ctrlclient.Options{})
 	require.NoError(t, err)
 

internal/controllers/controller.go

@@ -0,0 +1,11 @@
+package controllers
+
+import (
+	"github.com/go-logr/logr"
+	ctrl "sigs.k8s.io/controller-runtime"
+)
+
+type Reconciler interface {
+	SetupWithManager(ctrl.Manager) error
+	SetLogger(logr.Logger)
+}

internal/controllers/gateway/dataplane_client.go

@@ -0,0 +1,26 @@
+package gateway
+
+import (
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	k8sobj "github.com/kong/kubernetes-ingress-controller/v2/internal/util/kubernetes/object"
+)
+
+// DataPlane is a common interface that is used by reconcilers to interact
+// with the dataplane.
+//
+// TODO: This can probably be used in other reconcilers as well.
+// Related issue: https://github.com/Kong/kubernetes-ingress-controller/issues/3794
+type DataPlane interface {
+	DataPlaneClient
+
+	AreKubernetesObjectReportsEnabled() bool
+	KubernetesObjectConfigurationStatus(obj client.Object) k8sobj.ConfigurationStatus
+}
+
+// DataPlaneClient is a common client interface that is used by reconcilers to interact
+// with the dataplane to perform CRUD operations on provided objects.
+type DataPlaneClient interface {
+	UpdateObject(obj client.Object) error
+	DeleteObject(obj client.Object) error
+}

internal/controllers/gateway/dataplane_mock_test.go

@@ -0,0 +1,32 @@
+package gateway
+
+import (
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	k8sobj "github.com/kong/kubernetes-ingress-controller/v2/internal/util/kubernetes/object"
+)
+
+type DataplaneMock struct {
+	KubernetesObjectReportsEnabled bool
+	// Mapping namespace to name to status
+	// Note: this will come in useful when implementing
+	// https://github.com/Kong/kubernetes-ingress-controller/issues/3793
+	// which requires the status to be reported for route objects.
+	ObjectsStatuses map[string]map[string]k8sobj.ConfigurationStatus
+}
+
+func (d DataplaneMock) UpdateObject(_ client.Object) error {
+	return nil
+}
+
+func (d DataplaneMock) DeleteObject(_ client.Object) error {
+	return nil
+}
+
+func (d DataplaneMock) AreKubernetesObjectReportsEnabled() bool {
+	return d.KubernetesObjectReportsEnabled
+}
+
+func (d DataplaneMock) KubernetesObjectConfigurationStatus(obj client.Object) k8sobj.ConfigurationStatus {
+	return d.ObjectsStatuses[obj.GetNamespace()][obj.GetName()]
+}

internal/controllers/gateway/gateway_utils.go

@@ -83,7 +83,8 @@ func isObjectUnmanaged(anns map[string]string) bool {
 // isGatewayClassControlledAndUnmanaged returns boolean if the GatewayClass
 // is controlled by this controller and is configured for unmanaged mode.
 func isGatewayClassControlledAndUnmanaged(gatewayClass *GatewayClass) bool {
-	return gatewayClass.Spec.ControllerName == GetControllerName() && isObjectUnmanaged(gatewayClass.Annotations)
+	isUnamanaged := isObjectUnmanaged(gatewayClass.Annotations)
+	return gatewayClass.Spec.ControllerName == GetControllerName() && isUnamanaged
 }
 
 // getRefFromPublishService splits a publish service string in the format namespace/name into a types.NamespacedName

internal/controllers/gateway/httproute_controller.go

@@ -23,7 +23,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/source"
 	gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
 
-	"github.com/kong/kubernetes-ingress-controller/v2/internal/dataplane"
 	"github.com/kong/kubernetes-ingress-controller/v2/internal/util"
 	k8sobj "github.com/kong/kubernetes-ingress-controller/v2/internal/util/kubernetes/object"
 )
@@ -38,7 +37,7 @@ type HTTPRouteReconciler struct {
 
 	Log             logr.Logger
 	Scheme          *runtime.Scheme
-	DataplaneClient *dataplane.KongClient
+	DataplaneClient DataPlane
 	// If EnableReferenceGrant is true, we will check for ReferenceGrant if backend in another
 	// namespace is in backendRefs.
 	// If it is false, referencing backend in different namespace will be rejected.
@@ -87,6 +86,16 @@ func (r *HTTPRouteReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		return err
 	}
 
+	if r.EnableReferenceGrant {
+		if err := c.Watch(
+			&source.Kind{Type: &gatewayv1beta1.ReferenceGrant{}},
+			handler.EnqueueRequestsFromMapFunc(r.listReferenceGrantsForHTTPRoute),
+			predicate.NewPredicateFuncs(referenceGrantHasHTTPRouteFrom),
+		); err != nil {
+			return err
+		}
+	}
+
 	// because of the additional burden of having to manage reference data-plane
 	// configurations for HTTPRoute objects in the underlying Kong Gateway, we
 	// simply reconcile ALL HTTPRoute objects. This allows us to drop the backend
@@ -102,6 +111,54 @@ func (r *HTTPRouteReconciler) SetupWithManager(mgr ctrl.Manager) error {
 // HTTPRoute Controller - Event Handlers
 // -----------------------------------------------------------------------------
 
+// listReferenceGrantsForHTTPRoute is a watch predicate which finds all HTTPRoutes
+// mentioned in a From clause for a ReferenceGrant.
+func (r *HTTPRouteReconciler) listReferenceGrantsForHTTPRoute(obj client.Object) []reconcile.Request {
+	grant, ok := obj.(*gatewayv1beta1.ReferenceGrant)
+	if !ok {
+		r.Log.Error(
+			fmt.Errorf("unexpected object type"),
+			"referencegrant watch predicate received unexpected object type",
+			"expected", "*gatewayv1beta1.ReferenceGrant", "found", reflect.TypeOf(obj),
+		)
+		return nil
+	}
+	httproutes := &gatewayv1beta1.HTTPRouteList{}
+	if err := r.Client.List(context.Background(), httproutes); err != nil {
+		r.Log.Error(err, "failed to list httproutes in watch", "referencegrant", grant.Name)
+		return nil
+	}
+	recs := []reconcile.Request{}
+	for _, gateway := range httproutes.Items {
+		for _, from := range grant.Spec.From {
+			if string(from.Namespace) == gateway.Namespace &&
+				from.Kind == gatewayv1beta1.Kind("HTTPRoute") &&
+				from.Group == gatewayv1beta1.Group("gateway.networking.k8s.io") {
+				recs = append(recs, reconcile.Request{
+					NamespacedName: types.NamespacedName{
+						Namespace: gateway.Namespace,
+						Name:      gateway.Name,
+					},
+				})
+			}
+		}
+	}
+	return recs
+}
+
+func referenceGrantHasHTTPRouteFrom(obj client.Object) bool {
+	grant, ok := obj.(*gatewayv1beta1.ReferenceGrant)
+	if !ok {
+		return false
+	}
+	for _, from := range grant.Spec.From {
+		if from.Kind == "HTTPRoute" && from.Group == "gateway.networking.k8s.io" {
+			return true
+		}
+	}
+	return false
+}
+
 // listHTTPRoutesForGatewayClass is a controller-runtime event.Handler which
 // produces a list of HTTPRoutes which were bound to a Gateway which is or was
 // bound to this GatewayClass. This implementation effectively does a map-reduce
@@ -634,3 +691,8 @@ func (r *HTTPRouteReconciler) getHTTPRouteRuleReason(ctx context.Context, httpRo
 	}
 	return gatewayv1beta1.RouteReasonResolvedRefs, nil
 }
+
+// SetLogger sets the logger.
+func (r *HTTPRouteReconciler) SetLogger(l logr.Logger) {
+	r.Log = l
+}