diff --git a/Makefile b/Makefile index 581afceb9b..f94647a410 100644 --- a/Makefile +++ b/Makefile @@ -204,6 +204,7 @@ manifests.rbac: controller-gen $(CONTROLLER_GEN) rbac:roleName=kong-ingress paths="./internal/controllers/configuration/" $(CONTROLLER_GEN) rbac:roleName=kong-ingress-knative paths="./internal/controllers/knative/" output:rbac:artifacts:config=config/rbac/knative $(CONTROLLER_GEN) rbac:roleName=kong-ingress-gateway paths="./internal/controllers/gateway/" output:rbac:artifacts:config=config/rbac/gateway + $(CONTROLLER_GEN) rbac:roleName=kong-ingress-crds paths="./internal/controllers/crds/" output:rbac:artifacts:config=config/rbac/crds .PHONY: manifests.single manifests.single: kustomize ## Compose single-file deployment manifests from building blocks diff --git a/config/base/kustomization.yaml b/config/base/kustomization.yaml index 3c8df10981..594a45ae19 100644 --- a/config/base/kustomization.yaml +++ b/config/base/kustomization.yaml @@ -5,6 +5,7 @@ resources: - ../rbac - ../rbac/gateway - ../rbac/knative +- ../rbac/crds - ingressclass.yaml - service.yaml - serviceaccount.yaml diff --git a/config/rbac/crds/kustomization.yaml b/config/rbac/crds/kustomization.yaml new file mode 100644 index 0000000000..b228b96dae --- /dev/null +++ b/config/rbac/crds/kustomization.yaml @@ -0,0 +1,3 @@ +resources: +- role.yaml +- role_binding.yaml diff --git a/config/rbac/crds/role.yaml b/config/rbac/crds/role.yaml new file mode 100644 index 0000000000..0ef77edd2c --- /dev/null +++ b/config/rbac/crds/role.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kong-ingress-crds +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch diff --git a/config/rbac/crds/role_binding.yaml b/config/rbac/crds/role_binding.yaml new file mode 100644 index 0000000000..d8c850b49f --- /dev/null +++ b/config/rbac/crds/role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kong-ingress-crds +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-crds +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong diff --git a/deploy/single/all-in-one-dbless-enterprise.yaml b/deploy/single/all-in-one-dbless-enterprise.yaml index 4b19168672..036d2865f3 100644 --- a/deploy/single/all-in-one-dbless-enterprise.yaml +++ b/deploy/single/all-in-one-dbless-enterprise.yaml @@ -1374,6 +1374,19 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: kong-ingress-crds +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: kong-ingress-gateway rules: @@ -1550,6 +1563,19 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + name: kong-ingress-crds +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-crds +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: name: kong-ingress-gateway roleRef: diff --git a/deploy/single/all-in-one-dbless-k4k8s-enterprise.yaml b/deploy/single/all-in-one-dbless-k4k8s-enterprise.yaml index 26ff98f993..f41f8f7e54 100644 --- a/deploy/single/all-in-one-dbless-k4k8s-enterprise.yaml +++ b/deploy/single/all-in-one-dbless-k4k8s-enterprise.yaml @@ -1374,6 +1374,19 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: kong-ingress-crds +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: kong-ingress-gateway rules: @@ -1550,6 +1563,19 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + name: kong-ingress-crds +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-crds +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: name: kong-ingress-gateway roleRef: diff --git a/deploy/single/all-in-one-dbless-konnect-enterprise.yaml b/deploy/single/all-in-one-dbless-konnect-enterprise.yaml index 3263bfe42e..4ef5697275 100644 --- a/deploy/single/all-in-one-dbless-konnect-enterprise.yaml +++ b/deploy/single/all-in-one-dbless-konnect-enterprise.yaml @@ -1374,6 +1374,19 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: kong-ingress-crds +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: kong-ingress-gateway rules: @@ -1550,6 +1563,19 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + name: kong-ingress-crds +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-crds +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: name: kong-ingress-gateway roleRef: diff --git a/deploy/single/all-in-one-dbless-konnect.yaml b/deploy/single/all-in-one-dbless-konnect.yaml index 5ec982c9dd..61a7adb9aa 100644 --- a/deploy/single/all-in-one-dbless-konnect.yaml +++ b/deploy/single/all-in-one-dbless-konnect.yaml @@ -1374,6 +1374,19 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: kong-ingress-crds +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: kong-ingress-gateway rules: @@ -1550,6 +1563,19 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + name: kong-ingress-crds +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-crds +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: name: kong-ingress-gateway roleRef: diff --git a/deploy/single/all-in-one-dbless-legacy.yaml b/deploy/single/all-in-one-dbless-legacy.yaml index 95c1993c7d..3e33417d99 100644 --- a/deploy/single/all-in-one-dbless-legacy.yaml +++ b/deploy/single/all-in-one-dbless-legacy.yaml @@ -1374,6 +1374,19 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: kong-ingress-crds +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: kong-ingress-gateway rules: @@ -1550,6 +1563,19 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + name: kong-ingress-crds +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-crds +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: name: kong-ingress-gateway roleRef: diff --git a/deploy/single/all-in-one-dbless.yaml b/deploy/single/all-in-one-dbless.yaml index 02ffb9b505..2ed3166055 100644 --- a/deploy/single/all-in-one-dbless.yaml +++ b/deploy/single/all-in-one-dbless.yaml @@ -1374,6 +1374,19 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: kong-ingress-crds +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: kong-ingress-gateway rules: @@ -1550,6 +1563,19 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + name: kong-ingress-crds +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-crds +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: name: kong-ingress-gateway roleRef: diff --git a/deploy/single/all-in-one-postgres-enterprise.yaml b/deploy/single/all-in-one-postgres-enterprise.yaml index bad34705bd..9ac0355bc7 100644 --- a/deploy/single/all-in-one-postgres-enterprise.yaml +++ b/deploy/single/all-in-one-postgres-enterprise.yaml @@ -1374,6 +1374,19 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: kong-ingress-crds +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: kong-ingress-gateway rules: @@ -1550,6 +1563,19 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + name: kong-ingress-crds +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-crds +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: name: kong-ingress-gateway roleRef: diff --git a/deploy/single/all-in-one-postgres.yaml b/deploy/single/all-in-one-postgres.yaml index af5cc18ca5..72f140458c 100644 --- a/deploy/single/all-in-one-postgres.yaml +++ b/deploy/single/all-in-one-postgres.yaml @@ -1374,6 +1374,19 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: kong-ingress-crds +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: kong-ingress-gateway rules: @@ -1550,6 +1563,19 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + name: kong-ingress-crds +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-crds +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: name: kong-ingress-gateway roleRef: diff --git a/internal/controllers/crds/dynamic_controller.go b/internal/controllers/crds/dynamic_controller.go new file mode 100644 index 0000000000..368cf4bf91 --- /dev/null +++ b/internal/controllers/crds/dynamic_controller.go @@ -0,0 +1,134 @@ +package crds + +import ( + "context" + "errors" + "sync" + "time" + + "github.com/go-logr/logr" + "github.com/samber/lo" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/utils" + "github.com/kong/kubernetes-ingress-controller/v2/internal/util" +) + +// +kubebuilder:rbac:groups="apiextensions.k8s.io",resources=customresourcedefinitions,verbs=list;watch + +type Controller interface { + SetupWithManager(mgr ctrl.Manager) error +} + +// DynamicController ensures that RequiredCRDs are installed in the cluster and only then sets up all of its Controllers +// that depends on them. +// In case the CRDs are not installed at start-up time, DynamicController will set up a watch for CustomResourceDefinition +// and will dynamically set up its Controllers once it detects that all RequiredCRDs are already in place. +type DynamicController struct { + Log logr.Logger + Manager ctrl.Manager + CacheSyncTimeout time.Duration + Controllers []Controller + RequiredCRDs []schema.GroupVersionResource + + startControllersOnce sync.Once +} + +func (r *DynamicController) SetupWithManager(mgr ctrl.Manager) error { + if r.allRequiredCRDsInstalled() { + r.Log.V(util.DebugLevel).Info("All required CustomResourceDefinitions are installed, skipping DynamicController set up") + return r.setupControllers(mgr) + } + + r.Log.Info("Required CustomResourceDefinitions are not installed, setting up a watch for them in case they are installed afterward") + + c, err := controller.New("DynamicController", mgr, controller.Options{ + Reconciler: r, + LogConstructor: func(_ *reconcile.Request) logr.Logger { + return r.Log + }, + CacheSyncTimeout: r.CacheSyncTimeout, + }) + if err != nil { + return err + } + + return c.Watch( + &source.Kind{Type: &apiextensionsv1.CustomResourceDefinition{}}, + &handler.EnqueueRequestForObject{}, + predicate.NewPredicateFuncs(r.isOneOfRequiredCRDs), + ) +} + +func (r *DynamicController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := r.Log.WithValues("CustomResourceDefinition", req.NamespacedName) + + crd := new(apiextensionsv1.CustomResourceDefinition) + if err := r.Manager.GetClient().Get(ctx, req.NamespacedName, crd); err != nil { + if apierrors.IsNotFound(err) { + log.V(util.DebugLevel).Info("Object enqueued no longer exists, skipping", "name", req.Name) + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + log.V(util.DebugLevel).Info("Processing CustomResourceDefinition", "name", req.Name) + + if !r.allRequiredCRDsInstalled() { + log.V(util.DebugLevel).Info("Still not all required CustomResourceDefinitions are installed, waiting") + return ctrl.Result{}, nil + } + + var startControllersErr error + r.startControllersOnce.Do(func() { + log.V(util.InfoLevel).Info("All required CustomResourceDefinitions are installed, setting up the controllers") + startControllersErr = r.setupControllers(r.Manager) + }) + if startControllersErr != nil { + return ctrl.Result{}, startControllersErr + } + + return ctrl.Result{}, nil +} + +func (r *DynamicController) allRequiredCRDsInstalled() bool { + return lo.EveryBy(r.RequiredCRDs, func(gvr schema.GroupVersionResource) bool { + return utils.CRDExists(r.Manager.GetClient().RESTMapper(), gvr) + }) +} + +func (r *DynamicController) isOneOfRequiredCRDs(obj client.Object) bool { + crd, ok := obj.(*apiextensionsv1.CustomResourceDefinition) + if !ok { + return false + } + + return lo.ContainsBy(r.RequiredCRDs, func(gvr schema.GroupVersionResource) bool { + versionMatches := lo.ContainsBy(crd.Spec.Versions, func(crdv apiextensionsv1.CustomResourceDefinitionVersion) bool { + return crdv.Name == gvr.Version + }) + + return crd.Spec.Group == gvr.Group && + crd.Status.AcceptedNames.Plural == gvr.Resource && + versionMatches + }) +} + +func (r *DynamicController) setupControllers(mgr ctrl.Manager) error { + errs := lo.FilterMap(r.Controllers, func(c Controller, _ int) (error, bool) { + if err := c.SetupWithManager(mgr); err != nil { + return err, true + } + return nil, false + }) + + return errors.Join(errs...) +} diff --git a/internal/controllers/gateway/gateway_controller.go b/internal/controllers/gateway/gateway_controller.go index 1fda6e94ef..dda3b078d0 100644 --- a/internal/controllers/gateway/gateway_controller.go +++ b/internal/controllers/gateway/gateway_controller.go @@ -15,6 +15,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -28,6 +29,7 @@ import ( "github.com/kong/kubernetes-ingress-controller/v2/internal/annotations" ctrlref "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/reference" + ctrlutils "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/utils" "github.com/kong/kubernetes-ingress-controller/v2/internal/dataplane" "github.com/kong/kubernetes-ingress-controller/v2/internal/util" ) @@ -54,9 +56,9 @@ type GatewayReconciler struct { //nolint:revive DataplaneClient *dataplane.KongClient WatchNamespaces []string - // If EnableReferenceGrant is true, controller will watch ReferenceGrants + // If enableReferenceGrant is true, controller will watch ReferenceGrants // to invalidate or allow cross-namespace TLSConfigs in gateways. - EnableReferenceGrant bool + enableReferenceGrant bool CacheSyncTimeout time.Duration ReferenceIndexers ctrlref.CacheIndexers @@ -72,6 +74,12 @@ func (r *GatewayReconciler) SetupWithManager(mgr ctrl.Manager) error { return fmt.Errorf("publish service must be configured") } + r.enableReferenceGrant = ctrlutils.CRDExists(mgr.GetRESTMapper(), schema.GroupVersionResource{ + Group: gatewayv1beta1.GroupVersion.Group, + Version: gatewayv1beta1.GroupVersion.Version, + Resource: "referencegrants", + }) + // generate the controller object and attach it to the manager and link the reconciler object c, err := controller.New("gateway-controller", mgr, controller.Options{ Reconciler: r, @@ -116,7 +124,7 @@ func (r *GatewayReconciler) SetupWithManager(mgr ctrl.Manager) error { } // watch ReferenceGrants, which may invalidate or allow cross-namespace TLSConfigs - if r.EnableReferenceGrant { + if r.enableReferenceGrant { if err := c.Watch( &source.Kind{Type: &gatewayv1beta1.ReferenceGrant{}}, handler.EnqueueRequestsFromMapFunc(r.listReferenceGrantsForGateway), @@ -490,7 +498,7 @@ func (r *GatewayReconciler) reconcileUnmanagedGateway(ctx context.Context, log l // the ReferenceGrants need to be retrieved to ensure that all gateway listeners reference // TLS secrets they are granted for referenceGrantList := &gatewayv1beta1.ReferenceGrantList{} - if r.EnableReferenceGrant { + if r.enableReferenceGrant { if err := r.Client.List(ctx, referenceGrantList); err != nil { return ctrl.Result{}, err } diff --git a/internal/controllers/gateway/httproute_controller.go b/internal/controllers/gateway/httproute_controller.go index f594878be0..a1c9367f35 100644 --- a/internal/controllers/gateway/httproute_controller.go +++ b/internal/controllers/gateway/httproute_controller.go @@ -12,6 +12,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -23,6 +24,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + ctrlutils "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/utils" "github.com/kong/kubernetes-ingress-controller/v2/internal/util" k8sobj "github.com/kong/kubernetes-ingress-controller/v2/internal/util/kubernetes/object" ) @@ -58,6 +60,12 @@ func (r *HTTPRouteReconciler) SetupWithManager(mgr ctrl.Manager) error { return err } + r.EnableReferenceGrant = ctrlutils.CRDExists(mgr.GetRESTMapper(), schema.GroupVersionResource{ + Group: gatewayv1beta1.GroupVersion.Group, + Version: gatewayv1beta1.GroupVersion.Version, + Resource: "referencegrants", + }) + // if a GatewayClass updates then we need to enqueue the linked HTTPRoutes to // ensure that any route objects that may have been orphaned by that change get // removed from data-plane configurations, and any routes that are now supported diff --git a/internal/manager/controllerdef.go b/internal/manager/controllerdef.go index 4549348f0e..f7c0bfcc11 100644 --- a/internal/manager/controllerdef.go +++ b/internal/manager/controllerdef.go @@ -13,6 +13,7 @@ import ( gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/configuration" + "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/crds" "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/gateway" "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/knative" ctrlref "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/reference" @@ -75,15 +76,6 @@ func setupControllers( return nil, fmt.Errorf("ingress version picker failed: %w", err) } - referenceGrantsEnabled := featureGates[featuregates.GatewayFeature] && ShouldEnableCRDController( - schema.GroupVersionResource{ - Group: gatewayv1beta1.GroupVersion.Group, - Version: gatewayv1beta1.GroupVersion.Version, - Resource: "referencegrants", - }, - restMapper, - ) - referenceIndexers := ctrlref.NewCacheIndexers() controllers := []ControllerDef{ @@ -331,124 +323,123 @@ func setupControllers( // Gateway API Controllers - Beta APIs // --------------------------------------------------------------------------- { - Enabled: featureGates[featuregates.GatewayFeature] && ShouldEnableCRDController( - schema.GroupVersionResource{ - Group: gatewayv1beta1.GroupVersion.Group, - Version: gatewayv1beta1.GroupVersion.Version, - Resource: "gateways", + Enabled: featureGates[featuregates.GatewayFeature], + Controller: &crds.DynamicController{ + Manager: mgr, + Log: ctrl.Log.WithName("controllers").WithName("Dynamic/gatewayv1beta1"), + CacheSyncTimeout: c.CacheSyncTimeout, + RequiredCRDs: []schema.GroupVersionResource{ + { + Group: gatewayv1beta1.GroupVersion.Group, + Version: gatewayv1beta1.GroupVersion.Version, + Resource: "gateways", + }, + { + Group: gatewayv1beta1.GroupVersion.Group, + Version: gatewayv1beta1.GroupVersion.Version, + Resource: "gatewayclasses", + }, + { + Group: gatewayv1beta1.GroupVersion.Group, + Version: gatewayv1beta1.GroupVersion.Version, + Resource: "httproutes", + }, }, - restMapper, - ), - Controller: &gateway.GatewayReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName(featuregates.GatewayFeature), - Scheme: mgr.GetScheme(), - DataplaneClient: dataplaneClient, - PublishServiceRef: c.PublishService.OrEmpty(), - PublishServiceUDPRef: c.PublishServiceUDP, - WatchNamespaces: c.WatchNamespaces, - EnableReferenceGrant: referenceGrantsEnabled, - CacheSyncTimeout: c.CacheSyncTimeout, - ReferenceIndexers: referenceIndexers, - }, - }, - { - Enabled: featureGates[featuregates.GatewayFeature] && ShouldEnableCRDController( - schema.GroupVersionResource{ - Group: gatewayv1beta1.GroupVersion.Group, - Version: gatewayv1beta1.GroupVersion.Version, - Resource: "httproutes", + Controllers: []crds.Controller{ + &gateway.GatewayReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("Gateway"), + Scheme: mgr.GetScheme(), + DataplaneClient: dataplaneClient, + PublishServiceRef: c.PublishService.OrEmpty(), + PublishServiceUDPRef: c.PublishServiceUDP, + WatchNamespaces: c.WatchNamespaces, + CacheSyncTimeout: c.CacheSyncTimeout, + ReferenceIndexers: referenceIndexers, + }, + &gateway.HTTPRouteReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("HTTPRoute"), + Scheme: mgr.GetScheme(), + DataplaneClient: dataplaneClient, + CacheSyncTimeout: c.CacheSyncTimeout, + }, }, - restMapper, - ), - Controller: &gateway.HTTPRouteReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("HTTPRoute"), - Scheme: mgr.GetScheme(), - DataplaneClient: dataplaneClient, - EnableReferenceGrant: referenceGrantsEnabled, - CacheSyncTimeout: c.CacheSyncTimeout, }, }, // --------------------------------------------------------------------------- // Gateway API Controllers - Alpha APIs // --------------------------------------------------------------------------- { - Enabled: referenceGrantsEnabled, - Controller: &gateway.ReferenceGrantReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("ReferenceGrant"), - Scheme: mgr.GetScheme(), - DataplaneClient: dataplaneClient, + Enabled: featureGates[featuregates.GatewayAlphaFeature], + Controller: &crds.DynamicController{ + Manager: mgr, + Log: ctrl.Log.WithName("controllers").WithName("Dynamic/gatewayv1alpha2"), CacheSyncTimeout: c.CacheSyncTimeout, - }, - }, - { - Enabled: featureGates[featuregates.GatewayAlphaFeature] && ShouldEnableCRDController( - schema.GroupVersionResource{ - Group: gatewayv1alpha2.GroupVersion.Group, - Version: gatewayv1alpha2.GroupVersion.Version, - Resource: "udproutes", + RequiredCRDs: []schema.GroupVersionResource{ + { + Group: gatewayv1beta1.GroupVersion.Group, + Version: gatewayv1beta1.GroupVersion.Version, + Resource: "referencegrants", + }, + { + Group: gatewayv1alpha2.GroupVersion.Group, + Version: gatewayv1alpha2.GroupVersion.Version, + Resource: "udproutes", + }, + { + Group: gatewayv1alpha2.GroupVersion.Group, + Version: gatewayv1alpha2.GroupVersion.Version, + Resource: "tcproutes", + }, + { + Group: gatewayv1alpha2.GroupVersion.Group, + Version: gatewayv1alpha2.GroupVersion.Version, + Resource: "tlsroutes", + }, + { + Group: gatewayv1alpha2.GroupVersion.Group, + Version: gatewayv1alpha2.GroupVersion.Version, + Resource: "grpcroutes", + }, }, - restMapper, - ), - Controller: &gateway.UDPRouteReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("UDPRoute"), - Scheme: mgr.GetScheme(), - DataplaneClient: dataplaneClient, - CacheSyncTimeout: c.CacheSyncTimeout, - }, - }, - { - Enabled: featureGates[featuregates.GatewayAlphaFeature] && ShouldEnableCRDController( - schema.GroupVersionResource{ - Group: gatewayv1alpha2.GroupVersion.Group, - Version: gatewayv1alpha2.GroupVersion.Version, - Resource: "tcproutes", + Controllers: []crds.Controller{ + &gateway.ReferenceGrantReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("ReferenceGrant"), + Scheme: mgr.GetScheme(), + DataplaneClient: dataplaneClient, + CacheSyncTimeout: c.CacheSyncTimeout, + }, + &gateway.UDPRouteReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("UDPRoute"), + Scheme: mgr.GetScheme(), + DataplaneClient: dataplaneClient, + CacheSyncTimeout: c.CacheSyncTimeout, + }, + &gateway.TCPRouteReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("TCPRoute"), + Scheme: mgr.GetScheme(), + DataplaneClient: dataplaneClient, + CacheSyncTimeout: c.CacheSyncTimeout, + }, + &gateway.TLSRouteReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("TLSRoute"), + Scheme: mgr.GetScheme(), + DataplaneClient: dataplaneClient, + CacheSyncTimeout: c.CacheSyncTimeout, + }, + &gateway.GRPCRouteReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("GRPCRoute"), + Scheme: mgr.GetScheme(), + DataplaneClient: dataplaneClient, + CacheSyncTimeout: c.CacheSyncTimeout, + }, }, - restMapper, - ), - Controller: &gateway.TCPRouteReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("TCPRoute"), - Scheme: mgr.GetScheme(), - DataplaneClient: dataplaneClient, - CacheSyncTimeout: c.CacheSyncTimeout, - }, - }, - { - Enabled: featureGates[featuregates.GatewayAlphaFeature] && ShouldEnableCRDController( - schema.GroupVersionResource{ - Group: gatewayv1alpha2.GroupVersion.Group, - Version: gatewayv1alpha2.GroupVersion.Version, - Resource: "tlsroutes", - }, - restMapper, - ), - Controller: &gateway.TLSRouteReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("TLSRoute"), - Scheme: mgr.GetScheme(), - DataplaneClient: dataplaneClient, - CacheSyncTimeout: c.CacheSyncTimeout, - }, - }, - { - Enabled: featureGates[featuregates.GatewayAlphaFeature] && ShouldEnableCRDController( - schema.GroupVersionResource{ - Group: gatewayv1alpha2.GroupVersion.Group, - Version: gatewayv1alpha2.GroupVersion.Version, - Resource: "grpcroutes", - }, - restMapper, - ), - Controller: &gateway.GRPCRouteReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("GRPCRoute"), - Scheme: mgr.GetScheme(), - DataplaneClient: dataplaneClient, - CacheSyncTimeout: c.CacheSyncTimeout, }, }, } diff --git a/internal/manager/scheme/scheme.go b/internal/manager/scheme/scheme.go index 8e3ba4901e..e1318326ed 100644 --- a/internal/manager/scheme/scheme.go +++ b/internal/manager/scheme/scheme.go @@ -1,6 +1,7 @@ package scheme import ( + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" knativev1alpha1 "knative.dev/networking/pkg/apis/networking/v1alpha1" @@ -18,6 +19,10 @@ import ( func Get(fg map[string]bool) (*runtime.Scheme, error) { scheme := runtime.NewScheme() + if err := apiextensionsv1.AddToScheme(scheme); err != nil { + return nil, err + } + if err := clientgoscheme.AddToScheme(scheme); err != nil { return nil, err } diff --git a/test/e2e/features_test.go b/test/e2e/features_test.go index fa0c89fdf0..d010d53a9f 100644 --- a/test/e2e/features_test.go +++ b/test/e2e/features_test.go @@ -305,23 +305,31 @@ func TestDeployAllInOneDBLESSGateway(t *testing.T) { LabelSelector: "app=" + controllerDeploymentNN.Name, } - t.Log("verifying that KIC disabled controllers for Gateway API and printed proper log") + t.Log("updating controller deployment to enable alpha Gateway feature gate") + controllerDeployment := deployments.GetController(ctx, t, env) + for i, container := range controllerDeployment.Spec.Template.Spec.Containers { + if container.Name == controllerContainerName { + controllerDeployment.Spec.Template.Spec.Containers[i].Env = append(controllerDeployment.Spec.Template.Spec.Containers[i].Env, + corev1.EnvVar{Name: "CONTROLLER_FEATURE_GATES", Value: consts.DefaultFeatureGates}) + } + } + + _, err := env.Cluster().Client().AppsV1().Deployments(namespace).Update(ctx, controllerDeployment, metav1.UpdateOptions{}) + require.NoError(t, err) + + t.Log("verifying that KIC waits for Gateway API CRDs and prints proper log") require.Eventually(t, func() bool { pods, err := env.Cluster().Client().CoreV1().Pods(controllerDeploymentNN.Namespace).List(ctx, controllerDeploymentListOptions) - gatewayGVR := schema.GroupVersionResource{ - Group: gatewayv1beta1.GroupVersion.Group, - Version: gatewayv1beta1.GroupVersion.Version, - Resource: "gateways", - } - msg := fmt.Sprintf("Disabling controller for Group=%s, Resource=%s due to missing CRD", gatewayGVR.GroupVersion(), gatewayGVR.Resource) require.NoError(t, err) + + expectedMsg := "Required CustomResourceDefinitions are not installed, setting up a watch for them in case they are installed afterward" for _, pod := range pods.Items { logs, err := getPodLogs(ctx, t, env, pod.Namespace, pod.Name) if err != nil { t.Logf("failed to get logs of pods %s/%s, error %v", pod.Namespace, pod.Name, err) return false } - if !strings.Contains(logs, msg) { + if !strings.Contains(logs, expectedMsg) { return false } } @@ -331,14 +339,6 @@ func TestDeployAllInOneDBLESSGateway(t *testing.T) { t.Logf("deploying Gateway APIs CRDs in standard channel from %s", consts.GatewayStandardCRDsKustomizeURL) require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), consts.GatewayStandardCRDsKustomizeURL)) - t.Logf("deleting KIC pods to restart them after Gateway APIs CRDs installed") - pods, err := env.Cluster().Client().CoreV1().Pods(controllerDeploymentNN.Namespace).List(ctx, controllerDeploymentListOptions) - require.NoError(t, err) - for _, pod := range pods.Items { - err = env.Cluster().Client().CoreV1().Pods(controllerDeploymentNN.Namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{}) - require.NoError(t, err) - } - t.Log("verifying controller updates associated Gateway resoures") gw := deployGateway(ctx, t, env) verifyGateway(ctx, t, env, gw) @@ -411,17 +411,6 @@ func TestDeployAllInOneDBLESSGateway(t *testing.T) { t.Logf("deploying Gateway APIs CRDs in experimental channel from %s", consts.GatewayExperimentalCRDsKustomizeURL) require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), consts.GatewayExperimentalCRDsKustomizeURL)) - t.Log("updating controller deployment to enable Gateway feature gate") - controllerDeployment := deployments.GetController(ctx, t, env) - for i, container := range controllerDeployment.Spec.Template.Spec.Containers { - if container.Name == controllerContainerName { - controllerDeployment.Spec.Template.Spec.Containers[i].Env = append(controllerDeployment.Spec.Template.Spec.Containers[i].Env, - corev1.EnvVar{Name: "CONTROLLER_FEATURE_GATES", Value: consts.DefaultFeatureGates}) - } - } - _, err = env.Cluster().Client().AppsV1().Deployments(namespace).Update(ctx, controllerDeployment, metav1.UpdateOptions{}) - require.NoError(t, err) - t.Log("updating proxy deployment to enable TCP listener") proxyDeployment := deployments.GetProxy(ctx, t, env) for i, container := range proxyDeployment.Spec.Template.Spec.Containers {