diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f196cdd47..85643ca2aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -110,6 +110,10 @@ Adding a new version? You'll need three changes: strategy that prevents KIC from exceeding API calls limits. [#3989](https://github.com/Kong/kubernetes-ingress-controller/pull/3989) [#4015](https://github.com/Kong/kubernetes-ingress-controller/pull/4015) +- When Gateway API CRDs are not installed, the controllers of those are not started + during the start-up phase. From now on, they will be dynamically started in runtime + once their installation is detected, making restarting the process unnecessary. + [#3996](https://github.com/Kong/kubernetes-ingress-controller/pull/3996) ### Fixed diff --git a/Makefile b/Makefile index 581afceb9b..f94647a410 100644 --- a/Makefile +++ b/Makefile @@ -204,6 +204,7 @@ manifests.rbac: controller-gen $(CONTROLLER_GEN) rbac:roleName=kong-ingress paths="./internal/controllers/configuration/" $(CONTROLLER_GEN) rbac:roleName=kong-ingress-knative paths="./internal/controllers/knative/" output:rbac:artifacts:config=config/rbac/knative $(CONTROLLER_GEN) rbac:roleName=kong-ingress-gateway paths="./internal/controllers/gateway/" output:rbac:artifacts:config=config/rbac/gateway + $(CONTROLLER_GEN) rbac:roleName=kong-ingress-crds paths="./internal/controllers/crds/" output:rbac:artifacts:config=config/rbac/crds .PHONY: manifests.single manifests.single: kustomize ## Compose single-file deployment manifests from building blocks diff --git a/config/base/kustomization.yaml b/config/base/kustomization.yaml index 3c8df10981..594a45ae19 100644 --- a/config/base/kustomization.yaml +++ b/config/base/kustomization.yaml @@ -5,6 +5,7 @@ resources: - ../rbac - ../rbac/gateway - ../rbac/knative +- ../rbac/crds - ingressclass.yaml - service.yaml - serviceaccount.yaml diff --git a/config/rbac/crds/kustomization.yaml b/config/rbac/crds/kustomization.yaml new file mode 100644 index 0000000000..b228b96dae --- /dev/null +++ b/config/rbac/crds/kustomization.yaml @@ -0,0 +1,3 @@ +resources: +- role.yaml +- role_binding.yaml diff --git a/config/rbac/crds/role.yaml b/config/rbac/crds/role.yaml new file mode 100644 index 0000000000..0ef77edd2c --- /dev/null +++ b/config/rbac/crds/role.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kong-ingress-crds +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch diff --git a/config/rbac/crds/role_binding.yaml b/config/rbac/crds/role_binding.yaml new file mode 100644 index 0000000000..d8c850b49f --- /dev/null +++ b/config/rbac/crds/role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kong-ingress-crds +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-crds +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong diff --git a/deploy/single/all-in-one-dbless-enterprise.yaml b/deploy/single/all-in-one-dbless-enterprise.yaml index 362ce4fe4b..aa33f83ede 100644 --- a/deploy/single/all-in-one-dbless-enterprise.yaml +++ b/deploy/single/all-in-one-dbless-enterprise.yaml @@ -1366,6 +1366,19 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: kong-ingress-crds +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: kong-ingress-gateway rules: @@ -1542,6 +1555,19 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + name: kong-ingress-crds +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-crds +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: name: kong-ingress-gateway roleRef: diff --git a/deploy/single/all-in-one-dbless-k4k8s-enterprise.yaml b/deploy/single/all-in-one-dbless-k4k8s-enterprise.yaml index cd2d4c5af3..73124d5c19 100644 --- a/deploy/single/all-in-one-dbless-k4k8s-enterprise.yaml +++ b/deploy/single/all-in-one-dbless-k4k8s-enterprise.yaml @@ -1366,6 +1366,19 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: kong-ingress-crds +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: kong-ingress-gateway rules: @@ -1542,6 +1555,19 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + name: kong-ingress-crds +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-crds +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: name: kong-ingress-gateway roleRef: diff --git a/deploy/single/all-in-one-dbless-konnect-enterprise.yaml b/deploy/single/all-in-one-dbless-konnect-enterprise.yaml index 1690d3441e..b47199c48b 100644 --- a/deploy/single/all-in-one-dbless-konnect-enterprise.yaml +++ b/deploy/single/all-in-one-dbless-konnect-enterprise.yaml @@ -1366,6 +1366,19 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: kong-ingress-crds +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: kong-ingress-gateway rules: @@ -1542,6 +1555,19 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + name: kong-ingress-crds +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-crds +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: name: kong-ingress-gateway roleRef: diff --git a/deploy/single/all-in-one-dbless-konnect.yaml b/deploy/single/all-in-one-dbless-konnect.yaml index 2b3f344e11..a36cee64f5 100644 --- a/deploy/single/all-in-one-dbless-konnect.yaml +++ b/deploy/single/all-in-one-dbless-konnect.yaml @@ -1366,6 +1366,19 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: kong-ingress-crds +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: kong-ingress-gateway rules: @@ -1542,6 +1555,19 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + name: kong-ingress-crds +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-crds +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: name: kong-ingress-gateway roleRef: diff --git a/deploy/single/all-in-one-dbless-legacy.yaml b/deploy/single/all-in-one-dbless-legacy.yaml index 7694b12c5a..de68879516 100644 --- a/deploy/single/all-in-one-dbless-legacy.yaml +++ b/deploy/single/all-in-one-dbless-legacy.yaml @@ -1366,6 +1366,19 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: kong-ingress-crds +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: kong-ingress-gateway rules: @@ -1542,6 +1555,19 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + name: kong-ingress-crds +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-crds +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: name: kong-ingress-gateway roleRef: diff --git a/deploy/single/all-in-one-dbless.yaml b/deploy/single/all-in-one-dbless.yaml index 2697ca82a9..5705bc0171 100644 --- a/deploy/single/all-in-one-dbless.yaml +++ b/deploy/single/all-in-one-dbless.yaml @@ -1366,6 +1366,19 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: kong-ingress-crds +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: kong-ingress-gateway rules: @@ -1542,6 +1555,19 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + name: kong-ingress-crds +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-crds +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: name: kong-ingress-gateway roleRef: diff --git a/deploy/single/all-in-one-postgres-enterprise.yaml b/deploy/single/all-in-one-postgres-enterprise.yaml index dd1b18a31e..a963f02339 100644 --- a/deploy/single/all-in-one-postgres-enterprise.yaml +++ b/deploy/single/all-in-one-postgres-enterprise.yaml @@ -1366,6 +1366,19 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: kong-ingress-crds +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: kong-ingress-gateway rules: @@ -1542,6 +1555,19 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + name: kong-ingress-crds +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-crds +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: name: kong-ingress-gateway roleRef: diff --git a/deploy/single/all-in-one-postgres.yaml b/deploy/single/all-in-one-postgres.yaml index 530d12e24d..af257ba4ca 100644 --- a/deploy/single/all-in-one-postgres.yaml +++ b/deploy/single/all-in-one-postgres.yaml @@ -1366,6 +1366,19 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: kong-ingress-crds +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: kong-ingress-gateway rules: @@ -1542,6 +1555,19 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + name: kong-ingress-crds +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-crds +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: name: kong-ingress-gateway roleRef: diff --git a/internal/controllers/crds/dynamic_controller.go b/internal/controllers/crds/dynamic_controller.go new file mode 100644 index 0000000000..8cae2a0003 --- /dev/null +++ b/internal/controllers/crds/dynamic_controller.go @@ -0,0 +1,126 @@ +package crds + +import ( + "context" + "sync" + "time" + + "github.com/go-logr/logr" + "github.com/samber/lo" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/utils" + "github.com/kong/kubernetes-ingress-controller/v2/internal/util" +) + +// +kubebuilder:rbac:groups="apiextensions.k8s.io",resources=customresourcedefinitions,verbs=list;watch + +type Controller interface { + SetupWithManager(mgr ctrl.Manager) error +} + +// DynamicCRDController ensures that RequiredCRDs are installed in the cluster and only then sets up its Controller +// that depends on them. +// In case the CRDs are not installed at start-up time, DynamicCRDController will set up a watch for CustomResourceDefinition +// and will dynamically set up its Controller once it detects that all RequiredCRDs are already in place. +type DynamicCRDController struct { + Log logr.Logger + Manager ctrl.Manager + CacheSyncTimeout time.Duration + Controller Controller + RequiredCRDs []schema.GroupVersionResource + + startControllersOnce sync.Once +} + +func (r *DynamicCRDController) SetupWithManager(mgr ctrl.Manager) error { + if r.allRequiredCRDsInstalled() { + r.Log.V(util.DebugLevel).Info("All required CustomResourceDefinitions are installed, skipping DynamicCRDController set up") + return r.setupController(mgr) + } + + r.Log.Info("Required CustomResourceDefinitions are not installed, setting up a watch for them in case they are installed afterward") + + c, err := controller.New("DynamicCRDController", mgr, controller.Options{ + Reconciler: r, + LogConstructor: func(_ *reconcile.Request) logr.Logger { + return r.Log + }, + CacheSyncTimeout: r.CacheSyncTimeout, + }) + if err != nil { + return err + } + + return c.Watch( + &source.Kind{Type: &apiextensionsv1.CustomResourceDefinition{}}, + &handler.EnqueueRequestForObject{}, + predicate.NewPredicateFuncs(r.isOneOfRequiredCRDs), + ) +} + +func (r *DynamicCRDController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := r.Log.WithValues("CustomResourceDefinition", req.NamespacedName) + + crd := new(apiextensionsv1.CustomResourceDefinition) + if err := r.Manager.GetClient().Get(ctx, req.NamespacedName, crd); err != nil { + if apierrors.IsNotFound(err) { + log.V(util.DebugLevel).Info("Object enqueued no longer exists, skipping", "name", req.Name) + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + log.V(util.DebugLevel).Info("Processing CustomResourceDefinition", "name", req.Name) + + if !r.allRequiredCRDsInstalled() { + log.V(util.DebugLevel).Info("Still not all required CustomResourceDefinitions are installed, waiting") + return ctrl.Result{}, nil + } + + var startControllerErr error + r.startControllersOnce.Do(func() { + log.V(util.InfoLevel).Info("All required CustomResourceDefinitions are installed, setting up the controller") + startControllerErr = r.setupController(r.Manager) + }) + if startControllerErr != nil { + return ctrl.Result{}, startControllerErr + } + + return ctrl.Result{}, nil +} + +func (r *DynamicCRDController) allRequiredCRDsInstalled() bool { + return lo.EveryBy(r.RequiredCRDs, func(gvr schema.GroupVersionResource) bool { + return utils.CRDExists(r.Manager.GetClient().RESTMapper(), gvr) + }) +} + +func (r *DynamicCRDController) isOneOfRequiredCRDs(obj client.Object) bool { + crd, ok := obj.(*apiextensionsv1.CustomResourceDefinition) + if !ok { + return false + } + + return lo.ContainsBy(r.RequiredCRDs, func(gvr schema.GroupVersionResource) bool { + versionMatches := lo.ContainsBy(crd.Spec.Versions, func(crdv apiextensionsv1.CustomResourceDefinitionVersion) bool { + return crdv.Name == gvr.Version + }) + + return crd.Spec.Group == gvr.Group && + crd.Status.AcceptedNames.Plural == gvr.Resource && + versionMatches + }) +} + +func (r *DynamicCRDController) setupController(mgr ctrl.Manager) error { + return r.Controller.SetupWithManager(mgr) +} diff --git a/internal/controllers/gateway/gateway_controller.go b/internal/controllers/gateway/gateway_controller.go index 1fda6e94ef..fef8942fda 100644 --- a/internal/controllers/gateway/gateway_controller.go +++ b/internal/controllers/gateway/gateway_controller.go @@ -15,6 +15,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -28,6 +29,7 @@ import ( "github.com/kong/kubernetes-ingress-controller/v2/internal/annotations" ctrlref "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/reference" + ctrlutils "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/utils" "github.com/kong/kubernetes-ingress-controller/v2/internal/dataplane" "github.com/kong/kubernetes-ingress-controller/v2/internal/util" ) @@ -53,16 +55,18 @@ type GatewayReconciler struct { //nolint:revive Scheme *runtime.Scheme DataplaneClient *dataplane.KongClient - WatchNamespaces []string - // If EnableReferenceGrant is true, controller will watch ReferenceGrants - // to invalidate or allow cross-namespace TLSConfigs in gateways. - EnableReferenceGrant bool - CacheSyncTimeout time.Duration + WatchNamespaces []string + CacheSyncTimeout time.Duration ReferenceIndexers ctrlref.CacheIndexers PublishServiceRef types.NamespacedName PublishServiceUDPRef mo.Option[types.NamespacedName] + + // If enableReferenceGrant is true, controller will watch ReferenceGrants + // to invalidate or allow cross-namespace TLSConfigs in gateways. + // It's resolved on SetupWithManager call. + enableReferenceGrant bool } // SetupWithManager sets up the controller with the Manager. @@ -72,6 +76,17 @@ func (r *GatewayReconciler) SetupWithManager(mgr ctrl.Manager) error { return fmt.Errorf("publish service must be configured") } + // We're verifying whether ReferenceGrant CRD is installed at setup of the GatewayReconciler + // to decide whether we should run additional ReferenceGrant watch and handle ReferenceGrants + // when reconciling Gateways. + // Once the GatewayReconciler is set up without ReferenceGrant, there's no possibility to enable + // ReferenceGrant handling again in this reconciler at runtime. + r.enableReferenceGrant = ctrlutils.CRDExists(mgr.GetRESTMapper(), schema.GroupVersionResource{ + Group: gatewayv1beta1.GroupVersion.Group, + Version: gatewayv1beta1.GroupVersion.Version, + Resource: "referencegrants", + }) + // generate the controller object and attach it to the manager and link the reconciler object c, err := controller.New("gateway-controller", mgr, controller.Options{ Reconciler: r, @@ -116,7 +131,7 @@ func (r *GatewayReconciler) SetupWithManager(mgr ctrl.Manager) error { } // watch ReferenceGrants, which may invalidate or allow cross-namespace TLSConfigs - if r.EnableReferenceGrant { + if r.enableReferenceGrant { if err := c.Watch( &source.Kind{Type: &gatewayv1beta1.ReferenceGrant{}}, handler.EnqueueRequestsFromMapFunc(r.listReferenceGrantsForGateway), @@ -490,7 +505,7 @@ func (r *GatewayReconciler) reconcileUnmanagedGateway(ctx context.Context, log l // the ReferenceGrants need to be retrieved to ensure that all gateway listeners reference // TLS secrets they are granted for referenceGrantList := &gatewayv1beta1.ReferenceGrantList{} - if r.EnableReferenceGrant { + if r.enableReferenceGrant { if err := r.Client.List(ctx, referenceGrantList); err != nil { return ctrl.Result{}, err } diff --git a/internal/controllers/gateway/httproute_controller.go b/internal/controllers/gateway/httproute_controller.go index f594878be0..b8b2315c3b 100644 --- a/internal/controllers/gateway/httproute_controller.go +++ b/internal/controllers/gateway/httproute_controller.go @@ -12,6 +12,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -23,6 +24,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + ctrlutils "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/utils" "github.com/kong/kubernetes-ingress-controller/v2/internal/util" k8sobj "github.com/kong/kubernetes-ingress-controller/v2/internal/util/kubernetes/object" ) @@ -35,14 +37,16 @@ import ( type HTTPRouteReconciler struct { client.Client - Log logr.Logger - Scheme *runtime.Scheme - DataplaneClient DataPlane - // If EnableReferenceGrant is true, we will check for ReferenceGrant if backend in another + Log logr.Logger + Scheme *runtime.Scheme + DataplaneClient DataPlane + CacheSyncTimeout time.Duration + + // If enableReferenceGrant is true, we will check for ReferenceGrant if backend in another // namespace is in backendRefs. // If it is false, referencing backend in different namespace will be rejected. - EnableReferenceGrant bool - CacheSyncTimeout time.Duration + // It's resolved on SetupWithManager call. + enableReferenceGrant bool } // SetupWithManager sets up the controller with the Manager. @@ -58,6 +62,17 @@ func (r *HTTPRouteReconciler) SetupWithManager(mgr ctrl.Manager) error { return err } + // We're verifying whether ReferenceGrant CRD is installed at setup of the HTTPRouteReconciler + // to decide whether we should run additional ReferenceGrant watch and handle ReferenceGrants + // when reconciling HTTPRoutes. + // Once the HTTPRouteReconciler is set up without ReferenceGrant, there's no possibility to enable + // ReferenceGrant handling again in this reconciler at runtime. + r.enableReferenceGrant = ctrlutils.CRDExists(mgr.GetRESTMapper(), schema.GroupVersionResource{ + Group: gatewayv1beta1.GroupVersion.Group, + Version: gatewayv1beta1.GroupVersion.Version, + Resource: "referencegrants", + }) + // if a GatewayClass updates then we need to enqueue the linked HTTPRoutes to // ensure that any route objects that may have been orphaned by that change get // removed from data-plane configurations, and any routes that are now supported @@ -86,7 +101,7 @@ func (r *HTTPRouteReconciler) SetupWithManager(mgr ctrl.Manager) error { return err } - if r.EnableReferenceGrant { + if r.enableReferenceGrant { if err := c.Watch( &source.Kind{Type: &gatewayv1beta1.ReferenceGrant{}}, handler.EnqueueRequestsFromMapFunc(r.listReferenceGrantsForHTTPRoute), @@ -132,8 +147,8 @@ func (r *HTTPRouteReconciler) listReferenceGrantsForHTTPRoute(obj client.Object) for _, gateway := range httproutes.Items { for _, from := range grant.Spec.From { if string(from.Namespace) == gateway.Namespace && - from.Kind == gatewayv1beta1.Kind("HTTPRoute") && - from.Group == gatewayv1beta1.Group("gateway.networking.k8s.io") { + from.Kind == ("HTTPRoute") && + from.Group == ("gateway.networking.k8s.io") { recs = append(recs, reconcile.Request{ NamespacedName: types.NamespacedName{ Namespace: gateway.Namespace, @@ -665,7 +680,7 @@ func (r *HTTPRouteReconciler) getHTTPRouteRuleReason(ctx context.Context, httpRo // Check if the object referenced is in another namespace, // and if there is grant for that reference if httpRoute.Namespace != backendNamespace { - if !r.EnableReferenceGrant { + if !r.enableReferenceGrant { return gatewayv1beta1.RouteReasonRefNotPermitted, nil } diff --git a/internal/controllers/gateway/httproute_controller_envtest_test.go b/internal/controllers/gateway/httproute_controller_envtest_test.go index fe9d6150b3..c085dff079 100644 --- a/internal/controllers/gateway/httproute_controller_envtest_test.go +++ b/internal/controllers/gateway/httproute_controller_envtest_test.go @@ -52,282 +52,236 @@ func TestHTTPRouteReconcilerProperlyReactsToReferenceGrant(t *testing.T) { require.NoError(t, err) } - // In tests below we use a deferred cancel to stop the manager and not wait - // for its timeout. - - testcases := []struct { - name string - reconciler *gateway.HTTPRouteReconciler - }{ - { - name: "with ReferenceGrant enabled", - reconciler: &gateway.HTTPRouteReconciler{ - Client: client, - EnableReferenceGrant: true, - }, - }, - { - name: "with ReferenceGrant disabled", - reconciler: &gateway.HTTPRouteReconciler{ - Client: client, - EnableReferenceGrant: false, - }, - }, + reconciler := &gateway.HTTPRouteReconciler{ + Client: client, + DataplaneClient: gateway.DataplaneMock{}, } - for _, tc := range testcases { - tc := tc - t.Run(tc.name, func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + // We use a deferred cancel to stop the manager and not wait for its timeout. + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() - ns := envtest.CreateNamespace(ctx, t, client) - nsRoute := envtest.CreateNamespace(ctx, t, client) + ns := envtest.CreateNamespace(ctx, t, client) + nsRoute := envtest.CreateNamespace(ctx, t, client) - svc := corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: ns.Name, - Name: "backend-1", - }, - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - { - Name: "http", - Protocol: corev1.ProtocolTCP, - Port: 80, - TargetPort: intstr.FromInt(80), - }, - }, + svc := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns.Name, + Name: "backend-1", + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "http", + Protocol: corev1.ProtocolTCP, + Port: 80, + TargetPort: intstr.FromInt(80), }, - } - require.NoError(t, client.Create(ctx, &svc)) - - tc.reconciler.DataplaneClient = gateway.DataplaneMock{} - envtest.StartReconciler(ctx, t, client.Scheme(), cfg, tc.reconciler, nil) + }, + }, + } + require.NoError(t, client.Create(ctx, &svc)) + envtest.StartReconciler(ctx, t, client.Scheme(), cfg, reconciler, nil) - gwc := gatewayv1beta1.GatewayClass{ - Spec: gatewayv1beta1.GatewayClassSpec{ - ControllerName: gateway.GetControllerName(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: uuid.NewString(), - Annotations: map[string]string{ - "konghq.com/gatewayclass-unmanaged": "placeholder", - }, - }, - } - require.NoError(t, client.Create(ctx, &gwc)) - t.Cleanup(func() { _ = client.Delete(ctx, &gwc) }) + gwc := gatewayv1beta1.GatewayClass{ + Spec: gatewayv1beta1.GatewayClassSpec{ + ControllerName: gateway.GetControllerName(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: uuid.NewString(), + Annotations: map[string]string{ + "konghq.com/gatewayclass-unmanaged": "placeholder", + }, + }, + } + require.NoError(t, client.Create(ctx, &gwc)) + t.Cleanup(func() { _ = client.Delete(ctx, &gwc) }) - gw := gatewayv1beta1.Gateway{ - Spec: gatewayv1beta1.GatewaySpec{ - GatewayClassName: gatewayv1beta1.ObjectName(gwc.Name), - Listeners: []gatewayv1beta1.Listener{ - { - Name: gatewayv1beta1.SectionName("http"), - Port: gatewayv1beta1.PortNumber(80), - Protocol: gatewayv1beta1.HTTPProtocolType, - AllowedRoutes: &gatewayv1beta1.AllowedRoutes{ - Namespaces: &gatewayv1beta1.RouteNamespaces{ - From: lo.ToPtr(gatewayv1beta1.NamespacesFromAll), - }, - }, + gw := gatewayv1beta1.Gateway{ + Spec: gatewayv1beta1.GatewaySpec{ + GatewayClassName: gatewayv1beta1.ObjectName(gwc.Name), + Listeners: []gatewayv1beta1.Listener{ + { + Name: gatewayv1beta1.SectionName("http"), + Port: gatewayv1beta1.PortNumber(80), + Protocol: gatewayv1beta1.HTTPProtocolType, + AllowedRoutes: &gatewayv1beta1.AllowedRoutes{ + Namespaces: &gatewayv1beta1.RouteNamespaces{ + From: lo.ToPtr(gatewayv1beta1.NamespacesFromAll), }, }, }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: ns.Name, - Name: uuid.NewString(), - }, - } - require.NoError(t, client.Create(ctx, &gw)) + }, + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns.Name, + Name: uuid.NewString(), + }, + } + require.NoError(t, client.Create(ctx, &gw)) - gwOld := gw.DeepCopy() - gw.Status = gatewayv1beta1.GatewayStatus{ - Addresses: []gatewayv1beta1.GatewayAddress{ - { - Type: lo.ToPtr(gatewayv1beta1.IPAddressType), - Value: "10.0.0.1", - }, - }, + gwOld := gw.DeepCopy() + gw.Status = gatewayv1beta1.GatewayStatus{ + Addresses: []gatewayv1beta1.GatewayAddress{ + { + Type: lo.ToPtr(gatewayv1beta1.IPAddressType), + Value: "10.0.0.1", + }, + }, + Conditions: []metav1.Condition{ + { + Type: "Ready", + Status: metav1.ConditionTrue, + Reason: "Ready", + LastTransitionTime: metav1.Now(), + ObservedGeneration: gw.Generation, + }, + { + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + LastTransitionTime: metav1.Now(), + ObservedGeneration: gw.Generation, + }, + { + Type: "Programmed", + Status: metav1.ConditionTrue, + Reason: "Programmed", + LastTransitionTime: metav1.Now(), + ObservedGeneration: gw.Generation, + }, + }, + Listeners: []gatewayv1beta1.ListenerStatus{ + { + Name: "http", Conditions: []metav1.Condition{ - { - Type: "Ready", - Status: metav1.ConditionTrue, - Reason: "Ready", - LastTransitionTime: metav1.Now(), - ObservedGeneration: gw.Generation, - }, { Type: "Accepted", Status: metav1.ConditionTrue, Reason: "Accepted", LastTransitionTime: metav1.Now(), - ObservedGeneration: gw.Generation, }, { - Type: "Programmed", + Type: "Ready", Status: metav1.ConditionTrue, - Reason: "Programmed", + Reason: "Ready", LastTransitionTime: metav1.Now(), - ObservedGeneration: gw.Generation, }, }, - Listeners: []gatewayv1beta1.ListenerStatus{ + SupportedKinds: []gatewayv1beta1.RouteGroupKind{ { - Name: gatewayv1beta1.SectionName("http"), - Conditions: []metav1.Condition{ - { - Type: "Accepted", - Status: metav1.ConditionTrue, - Reason: "Accepted", - LastTransitionTime: metav1.Now(), - }, - { - Type: "Ready", - Status: metav1.ConditionTrue, - Reason: "Ready", - LastTransitionTime: metav1.Now(), - }, - }, - SupportedKinds: []gatewayv1beta1.RouteGroupKind{ - { - Group: lo.ToPtr(gatewayv1beta1.Group(gatewayv1beta1.GroupVersion.Group)), - Kind: "HTTPRoute", - }, - }, + Group: lo.ToPtr(gatewayv1beta1.Group(gatewayv1beta1.GroupVersion.Group)), + Kind: "HTTPRoute", }, }, - } - require.NoError(t, client.Status().Patch(ctx, &gw, ctrlclient.MergeFrom(gwOld))) + }, + }, + } + require.NoError(t, client.Status().Patch(ctx, &gw, ctrlclient.MergeFrom(gwOld))) - route := gatewayv1beta1.HTTPRoute{ - TypeMeta: metav1.TypeMeta{ - Kind: "HTTPRoute", - APIVersion: "v1beta1", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: nsRoute.Name, - Name: uuid.NewString(), - }, - Spec: gatewayv1beta1.HTTPRouteSpec{ - CommonRouteSpec: gatewayv1beta1.CommonRouteSpec{ - ParentRefs: []gatewayv1beta1.ParentReference{{ - Name: gatewayv1beta1.ObjectName(gw.Name), - Namespace: lo.ToPtr(gatewayv1beta1.Namespace(ns.Name)), - }}, - }, - Rules: []gatewayv1beta1.HTTPRouteRule{{ - BackendRefs: builder.NewHTTPBackendRef("backend-1").WithNamespace(ns.Name).ToSlice(), - }}, - }, - } - require.NoError(t, client.Create(ctx, &route)) + route := gatewayv1beta1.HTTPRoute{ + TypeMeta: metav1.TypeMeta{ + Kind: "HTTPRoute", + APIVersion: "v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: nsRoute.Name, + Name: uuid.NewString(), + }, + Spec: gatewayv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gatewayv1beta1.CommonRouteSpec{ + ParentRefs: []gatewayv1beta1.ParentReference{{ + Name: gatewayv1beta1.ObjectName(gw.Name), + Namespace: lo.ToPtr(gatewayv1beta1.Namespace(ns.Name)), + }}, + }, + Rules: []gatewayv1beta1.HTTPRouteRule{{ + BackendRefs: builder.NewHTTPBackendRef("backend-1").WithNamespace(ns.Name).ToSlice(), + }}, + }, + } + require.NoError(t, client.Create(ctx, &route)) - nn := types.NamespacedName{ - Namespace: route.GetNamespace(), - Name: route.GetName(), - } + nn := types.NamespacedName{ + Namespace: route.GetNamespace(), + Name: route.GetName(), + } - t.Logf("verifying that HTTPRoute has ResolvedRefs set to Status False and Reason RefNotPermitted") - if !assert.Eventually(t, - helpers.HTTPRouteEventuallyContainsConditions(ctx, t, client, nn, - metav1.Condition{ - Type: "ResolvedRefs", - Status: "False", - Reason: "RefNotPermitted", - }, - ), - waitDuration, tickDuration, - ) { - t.Fatal(printHTTPRoutesConditions(ctx, client, nn)) - } + t.Logf("verifying that HTTPRoute has ResolvedRefs set to Status False and Reason RefNotPermitted") + if !assert.Eventually(t, + helpers.HTTPRouteEventuallyContainsConditions(ctx, t, client, nn, + metav1.Condition{ + Type: "ResolvedRefs", + Status: "False", + Reason: "RefNotPermitted", + }, + ), + waitDuration, tickDuration, + ) { + t.Fatal(printHTTPRoutesConditions(ctx, client, nn)) + } - rg := gatewayv1beta1.ReferenceGrant{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: ns.Name, - Name: uuid.NewString(), + rg := gatewayv1beta1.ReferenceGrant{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns.Name, + Name: uuid.NewString(), + }, + Spec: gatewayv1beta1.ReferenceGrantSpec{ + From: []gatewayv1beta1.ReferenceGrantFrom{ + { + Group: gatewayv1beta1.Group(gatewayv1beta1.GroupVersion.Group), + Kind: "HTTPRoute", + Namespace: gatewayv1beta1.Namespace(nsRoute.Name), }, - Spec: gatewayv1beta1.ReferenceGrantSpec{ - From: []gatewayv1beta1.ReferenceGrantFrom{ - { - Group: gatewayv1beta1.Group(gatewayv1beta1.GroupVersion.Group), - Kind: "HTTPRoute", - Namespace: gatewayv1beta1.Namespace(nsRoute.Name), - }, - }, - To: []gatewayv1beta1.ReferenceGrantTo{ - { - Group: "", - Kind: "Service", - }, - }, + }, + To: []gatewayv1beta1.ReferenceGrantTo{ + { + Group: "", + Kind: "Service", }, - } - require.NoError(t, client.Create(ctx, &rg)) - if tc.reconciler.EnableReferenceGrant { - t.Logf("verifying that HTTPRoute gets accepted by HTTPRouteReconciler after relevant ReferenceGrant gets created") - if !assert.Eventually(t, - helpers.HTTPRouteEventuallyContainsConditions(ctx, t, client, nn, - metav1.Condition{ - Type: "ResolvedRefs", - Status: "True", - Reason: "ResolvedRefs", - }, - metav1.Condition{ - Type: "Accepted", - Status: "True", - Reason: "Accepted", - }, - // Programmed condition requires a bit more work with mocks. - // It's set only when KubernetesObjectReports are enabled in the underlying - // dataplane client and then it relies on what's returned by - // dataplane client in KubernetesObjectConfigurationStatus(). - // This can be done but it's not the main focus of this test. - // Related: https://github.com/Kong/kubernetes-ingress-controller/issues/3793 - ), - waitDuration, tickDuration, - ) { - t.Fatal(printHTTPRoutesConditions(ctx, client, nn)) - } - } else { - t.Logf("verifying that HTTPRoute's status doesn't change after relevant ReferenceGrant gets created") - - if !assert.Eventually(t, - helpers.HTTPRouteEventuallyNotContainsConditions(ctx, t, client, nn, - metav1.Condition{ - Type: "ResolvedRefs", - Status: "True", - Reason: "ResolvedRefs", - }, - metav1.Condition{ - Type: "Accepted", - Status: "True", - Reason: "Accepted", - }, - ), - waitDuration, tickDuration, - ) { - t.Fatal(printHTTPRoutesConditions(ctx, client, nn)) - } - } + }, + }, + } + require.NoError(t, client.Create(ctx, &rg)) + t.Logf("verifying that HTTPRoute gets accepted by HTTPRouteReconciler after relevant ReferenceGrant gets created") + if !assert.Eventually(t, + helpers.HTTPRouteEventuallyContainsConditions(ctx, t, client, nn, + metav1.Condition{ + Type: "ResolvedRefs", + Status: "True", + Reason: "ResolvedRefs", + }, + metav1.Condition{ + Type: "Accepted", + Status: "True", + Reason: "Accepted", + }, + // Programmed condition requires a bit more work with mocks. + // It's set only when KubernetesObjectReports are enabled in the underlying + // dataplane client and then it relies on what's returned by + // dataplane client in KubernetesObjectConfigurationStatus(). + // This can be done but it's not the main focus of this test. + // Related: https://github.com/Kong/kubernetes-ingress-controller/issues/3793 + ), + waitDuration, tickDuration, + ) { + t.Fatal(printHTTPRoutesConditions(ctx, client, nn)) + } - require.NoError(t, client.Delete(ctx, &rg)) - t.Logf("verifying that HTTPRoute gets its ResolvedRefs condition to Status False and Reason RefNotPermitted when relevant ReferenceGrant gets deleted") + require.NoError(t, client.Delete(ctx, &rg)) + t.Logf("verifying that HTTPRoute gets its ResolvedRefs condition to Status False and Reason RefNotPermitted when relevant ReferenceGrant gets deleted") - if !assert.Eventually(t, - helpers.HTTPRouteEventuallyContainsConditions(ctx, t, client, nn, - metav1.Condition{ - Type: "ResolvedRefs", - Status: "False", - Reason: "RefNotPermitted", - }, - ), - waitDuration, tickDuration, - ) { - t.Fatal(printHTTPRoutesConditions(ctx, client, nn)) - } - }) + if !assert.Eventually(t, + helpers.HTTPRouteEventuallyContainsConditions(ctx, t, client, nn, + metav1.Condition{ + Type: "ResolvedRefs", + Status: "False", + Reason: "RefNotPermitted", + }, + ), + waitDuration, tickDuration, + ) { + t.Fatal(printHTTPRoutesConditions(ctx, client, nn)) } } diff --git a/internal/manager/controllerdef.go b/internal/manager/controllerdef.go index 4549348f0e..12c7362bae 100644 --- a/internal/manager/controllerdef.go +++ b/internal/manager/controllerdef.go @@ -13,6 +13,7 @@ import ( gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/configuration" + "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/crds" "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/gateway" "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/knative" ctrlref "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/reference" @@ -75,15 +76,6 @@ func setupControllers( return nil, fmt.Errorf("ingress version picker failed: %w", err) } - referenceGrantsEnabled := featureGates[featuregates.GatewayFeature] && ShouldEnableCRDController( - schema.GroupVersionResource{ - Group: gatewayv1beta1.GroupVersion.Group, - Version: gatewayv1beta1.GroupVersion.Version, - Resource: "referencegrants", - }, - restMapper, - ) - referenceIndexers := ctrlref.NewCacheIndexers() controllers := []ControllerDef{ @@ -331,127 +323,165 @@ func setupControllers( // Gateway API Controllers - Beta APIs // --------------------------------------------------------------------------- { - Enabled: featureGates[featuregates.GatewayFeature] && ShouldEnableCRDController( - schema.GroupVersionResource{ - Group: gatewayv1beta1.GroupVersion.Group, - Version: gatewayv1beta1.GroupVersion.Version, - Resource: "gateways", + Enabled: featureGates[featuregates.GatewayFeature], + Controller: &crds.DynamicCRDController{ + Manager: mgr, + Log: ctrl.Log.WithName("controllers").WithName("Dynamic/Gateway"), + CacheSyncTimeout: c.CacheSyncTimeout, + RequiredCRDs: baseGatewayCRDs(), + Controller: &gateway.GatewayReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("Gateway"), + Scheme: mgr.GetScheme(), + DataplaneClient: dataplaneClient, + PublishServiceRef: c.PublishService.OrEmpty(), + PublishServiceUDPRef: c.PublishServiceUDP, + WatchNamespaces: c.WatchNamespaces, + CacheSyncTimeout: c.CacheSyncTimeout, + ReferenceIndexers: referenceIndexers, }, - restMapper, - ), - Controller: &gateway.GatewayReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName(featuregates.GatewayFeature), - Scheme: mgr.GetScheme(), - DataplaneClient: dataplaneClient, - PublishServiceRef: c.PublishService.OrEmpty(), - PublishServiceUDPRef: c.PublishServiceUDP, - WatchNamespaces: c.WatchNamespaces, - EnableReferenceGrant: referenceGrantsEnabled, - CacheSyncTimeout: c.CacheSyncTimeout, - ReferenceIndexers: referenceIndexers, }, }, { - Enabled: featureGates[featuregates.GatewayFeature] && ShouldEnableCRDController( - schema.GroupVersionResource{ + Enabled: featureGates[featuregates.GatewayFeature], + Controller: &crds.DynamicCRDController{ + Manager: mgr, + Log: ctrl.Log.WithName("controllers").WithName("Dynamic/HTTPRoute"), + CacheSyncTimeout: c.CacheSyncTimeout, + RequiredCRDs: append(baseGatewayCRDs(), schema.GroupVersionResource{ Group: gatewayv1beta1.GroupVersion.Group, Version: gatewayv1beta1.GroupVersion.Version, Resource: "httproutes", + }), + Controller: &gateway.HTTPRouteReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("HTTPRoute"), + Scheme: mgr.GetScheme(), + DataplaneClient: dataplaneClient, + CacheSyncTimeout: c.CacheSyncTimeout, }, - restMapper, - ), - Controller: &gateway.HTTPRouteReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("HTTPRoute"), - Scheme: mgr.GetScheme(), - DataplaneClient: dataplaneClient, - EnableReferenceGrant: referenceGrantsEnabled, - CacheSyncTimeout: c.CacheSyncTimeout, }, }, // --------------------------------------------------------------------------- // Gateway API Controllers - Alpha APIs // --------------------------------------------------------------------------- { - Enabled: referenceGrantsEnabled, - Controller: &gateway.ReferenceGrantReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("ReferenceGrant"), - Scheme: mgr.GetScheme(), - DataplaneClient: dataplaneClient, + Enabled: featureGates[featuregates.GatewayAlphaFeature], + Controller: &crds.DynamicCRDController{ + Manager: mgr, + Log: ctrl.Log.WithName("controllers").WithName("Dynamic/ReferenceGrant"), CacheSyncTimeout: c.CacheSyncTimeout, + RequiredCRDs: append(baseGatewayCRDs(), schema.GroupVersionResource{ + Group: gatewayv1beta1.GroupVersion.Group, + Version: gatewayv1beta1.GroupVersion.Version, + Resource: "referencegrants", + }), + Controller: &gateway.ReferenceGrantReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("ReferenceGrant"), + Scheme: mgr.GetScheme(), + DataplaneClient: dataplaneClient, + CacheSyncTimeout: c.CacheSyncTimeout, + }, }, }, { - Enabled: featureGates[featuregates.GatewayAlphaFeature] && ShouldEnableCRDController( - schema.GroupVersionResource{ + Enabled: featureGates[featuregates.GatewayAlphaFeature], + Controller: &crds.DynamicCRDController{ + Manager: mgr, + Log: ctrl.Log.WithName("controllers").WithName("Dynamic/UDPRoute"), + CacheSyncTimeout: c.CacheSyncTimeout, + RequiredCRDs: append(baseGatewayCRDs(), schema.GroupVersionResource{ Group: gatewayv1alpha2.GroupVersion.Group, Version: gatewayv1alpha2.GroupVersion.Version, Resource: "udproutes", + }), + Controller: &gateway.UDPRouteReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("UDPRoute"), + Scheme: mgr.GetScheme(), + DataplaneClient: dataplaneClient, + CacheSyncTimeout: c.CacheSyncTimeout, }, - restMapper, - ), - Controller: &gateway.UDPRouteReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("UDPRoute"), - Scheme: mgr.GetScheme(), - DataplaneClient: dataplaneClient, - CacheSyncTimeout: c.CacheSyncTimeout, }, }, { - Enabled: featureGates[featuregates.GatewayAlphaFeature] && ShouldEnableCRDController( - schema.GroupVersionResource{ + Enabled: featureGates[featuregates.GatewayAlphaFeature], + Controller: &crds.DynamicCRDController{ + Manager: mgr, + Log: ctrl.Log.WithName("controllers").WithName("Dynamic/TCPRoute"), + CacheSyncTimeout: c.CacheSyncTimeout, + RequiredCRDs: append(baseGatewayCRDs(), schema.GroupVersionResource{ Group: gatewayv1alpha2.GroupVersion.Group, Version: gatewayv1alpha2.GroupVersion.Version, Resource: "tcproutes", + }), + Controller: &gateway.TCPRouteReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("TCPRoute"), + Scheme: mgr.GetScheme(), + DataplaneClient: dataplaneClient, + CacheSyncTimeout: c.CacheSyncTimeout, }, - restMapper, - ), - Controller: &gateway.TCPRouteReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("TCPRoute"), - Scheme: mgr.GetScheme(), - DataplaneClient: dataplaneClient, - CacheSyncTimeout: c.CacheSyncTimeout, }, }, { - Enabled: featureGates[featuregates.GatewayAlphaFeature] && ShouldEnableCRDController( - schema.GroupVersionResource{ + Enabled: featureGates[featuregates.GatewayAlphaFeature], + Controller: &crds.DynamicCRDController{ + Manager: mgr, + Log: ctrl.Log.WithName("controllers").WithName("Dynamic/TLSRoute"), + CacheSyncTimeout: c.CacheSyncTimeout, + RequiredCRDs: append(baseGatewayCRDs(), schema.GroupVersionResource{ Group: gatewayv1alpha2.GroupVersion.Group, Version: gatewayv1alpha2.GroupVersion.Version, Resource: "tlsroutes", + }), + Controller: &gateway.TLSRouteReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("TLSRoute"), + Scheme: mgr.GetScheme(), + DataplaneClient: dataplaneClient, + CacheSyncTimeout: c.CacheSyncTimeout, }, - restMapper, - ), - Controller: &gateway.TLSRouteReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("TLSRoute"), - Scheme: mgr.GetScheme(), - DataplaneClient: dataplaneClient, - CacheSyncTimeout: c.CacheSyncTimeout, }, }, { - Enabled: featureGates[featuregates.GatewayAlphaFeature] && ShouldEnableCRDController( - schema.GroupVersionResource{ + Enabled: featureGates[featuregates.GatewayAlphaFeature], + Controller: &crds.DynamicCRDController{ + Manager: mgr, + Log: ctrl.Log.WithName("controllers").WithName("Dynamic/GRPCRoute"), + CacheSyncTimeout: c.CacheSyncTimeout, + RequiredCRDs: append(baseGatewayCRDs(), schema.GroupVersionResource{ Group: gatewayv1alpha2.GroupVersion.Group, Version: gatewayv1alpha2.GroupVersion.Version, Resource: "grpcroutes", + }), + Controller: &gateway.GRPCRouteReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("GRPCRoute"), + Scheme: mgr.GetScheme(), + DataplaneClient: dataplaneClient, + CacheSyncTimeout: c.CacheSyncTimeout, }, - restMapper, - ), - Controller: &gateway.GRPCRouteReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("GRPCRoute"), - Scheme: mgr.GetScheme(), - DataplaneClient: dataplaneClient, - CacheSyncTimeout: c.CacheSyncTimeout, }, }, } return controllers, nil } + +// baseGatewayCRDs returns a slice of base CRDs required for running all the Gateway API controllers. +func baseGatewayCRDs() []schema.GroupVersionResource { + return []schema.GroupVersionResource{ + { + Group: gatewayv1beta1.GroupVersion.Group, + Version: gatewayv1beta1.GroupVersion.Version, + Resource: "gateways", + }, + { + Group: gatewayv1beta1.GroupVersion.Group, + Version: gatewayv1beta1.GroupVersion.Version, + Resource: "gatewayclasses", + }, + } +} diff --git a/internal/manager/scheme/scheme.go b/internal/manager/scheme/scheme.go index 8e3ba4901e..e1318326ed 100644 --- a/internal/manager/scheme/scheme.go +++ b/internal/manager/scheme/scheme.go @@ -1,6 +1,7 @@ package scheme import ( + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" knativev1alpha1 "knative.dev/networking/pkg/apis/networking/v1alpha1" @@ -18,6 +19,10 @@ import ( func Get(fg map[string]bool) (*runtime.Scheme, error) { scheme := runtime.NewScheme() + if err := apiextensionsv1.AddToScheme(scheme); err != nil { + return nil, err + } + if err := clientgoscheme.AddToScheme(scheme); err != nil { return nil, err } diff --git a/test/e2e/features_test.go b/test/e2e/features_test.go index fa0c89fdf0..d010d53a9f 100644 --- a/test/e2e/features_test.go +++ b/test/e2e/features_test.go @@ -305,23 +305,31 @@ func TestDeployAllInOneDBLESSGateway(t *testing.T) { LabelSelector: "app=" + controllerDeploymentNN.Name, } - t.Log("verifying that KIC disabled controllers for Gateway API and printed proper log") + t.Log("updating controller deployment to enable alpha Gateway feature gate") + controllerDeployment := deployments.GetController(ctx, t, env) + for i, container := range controllerDeployment.Spec.Template.Spec.Containers { + if container.Name == controllerContainerName { + controllerDeployment.Spec.Template.Spec.Containers[i].Env = append(controllerDeployment.Spec.Template.Spec.Containers[i].Env, + corev1.EnvVar{Name: "CONTROLLER_FEATURE_GATES", Value: consts.DefaultFeatureGates}) + } + } + + _, err := env.Cluster().Client().AppsV1().Deployments(namespace).Update(ctx, controllerDeployment, metav1.UpdateOptions{}) + require.NoError(t, err) + + t.Log("verifying that KIC waits for Gateway API CRDs and prints proper log") require.Eventually(t, func() bool { pods, err := env.Cluster().Client().CoreV1().Pods(controllerDeploymentNN.Namespace).List(ctx, controllerDeploymentListOptions) - gatewayGVR := schema.GroupVersionResource{ - Group: gatewayv1beta1.GroupVersion.Group, - Version: gatewayv1beta1.GroupVersion.Version, - Resource: "gateways", - } - msg := fmt.Sprintf("Disabling controller for Group=%s, Resource=%s due to missing CRD", gatewayGVR.GroupVersion(), gatewayGVR.Resource) require.NoError(t, err) + + expectedMsg := "Required CustomResourceDefinitions are not installed, setting up a watch for them in case they are installed afterward" for _, pod := range pods.Items { logs, err := getPodLogs(ctx, t, env, pod.Namespace, pod.Name) if err != nil { t.Logf("failed to get logs of pods %s/%s, error %v", pod.Namespace, pod.Name, err) return false } - if !strings.Contains(logs, msg) { + if !strings.Contains(logs, expectedMsg) { return false } } @@ -331,14 +339,6 @@ func TestDeployAllInOneDBLESSGateway(t *testing.T) { t.Logf("deploying Gateway APIs CRDs in standard channel from %s", consts.GatewayStandardCRDsKustomizeURL) require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), consts.GatewayStandardCRDsKustomizeURL)) - t.Logf("deleting KIC pods to restart them after Gateway APIs CRDs installed") - pods, err := env.Cluster().Client().CoreV1().Pods(controllerDeploymentNN.Namespace).List(ctx, controllerDeploymentListOptions) - require.NoError(t, err) - for _, pod := range pods.Items { - err = env.Cluster().Client().CoreV1().Pods(controllerDeploymentNN.Namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{}) - require.NoError(t, err) - } - t.Log("verifying controller updates associated Gateway resoures") gw := deployGateway(ctx, t, env) verifyGateway(ctx, t, env, gw) @@ -411,17 +411,6 @@ func TestDeployAllInOneDBLESSGateway(t *testing.T) { t.Logf("deploying Gateway APIs CRDs in experimental channel from %s", consts.GatewayExperimentalCRDsKustomizeURL) require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), consts.GatewayExperimentalCRDsKustomizeURL)) - t.Log("updating controller deployment to enable Gateway feature gate") - controllerDeployment := deployments.GetController(ctx, t, env) - for i, container := range controllerDeployment.Spec.Template.Spec.Containers { - if container.Name == controllerContainerName { - controllerDeployment.Spec.Template.Spec.Containers[i].Env = append(controllerDeployment.Spec.Template.Spec.Containers[i].Env, - corev1.EnvVar{Name: "CONTROLLER_FEATURE_GATES", Value: consts.DefaultFeatureGates}) - } - } - _, err = env.Cluster().Client().AppsV1().Deployments(namespace).Update(ctx, controllerDeployment, metav1.UpdateOptions{}) - require.NoError(t, err) - t.Log("updating proxy deployment to enable TCP listener") proxyDeployment := deployments.GetProxy(ctx, t, env) for i, container := range proxyDeployment.Spec.Template.Spec.Containers {