diff --git a/Makefile b/Makefile index 9504782e..e91b521c 100644 --- a/Makefile +++ b/Makefile @@ -62,7 +62,7 @@ GATEAY_API_VERSION ?= v1.3.0 SUPPORTED_EXTENDED_FEATURES = "HTTPRouteDestinationPortMatching,HTTPRouteMethodMatching,HTTPRoutePortRedirect,HTTPRouteRequestMirror,HTTPRouteSchemeRedirect,GatewayAddressEmpty,HTTPRouteResponseHeaderModification,GatewayPort8080,HTTPRouteHostRewrite,HTTPRouteQueryParamMatching,HTTPRoutePathRewrite" CONFORMANCE_TEST_REPORT_OUTPUT ?= $(DIR)/apisix-ingress-controller-conformance-report.yaml ## https://github.com/kubernetes-sigs/gateway-api/blob/v1.3.0/conformance/utils/suite/profiles.go -CONFORMANCE_PROFILES ?= GATEWAY-HTTP,GATEWAY-GRPC +CONFORMANCE_PROFILES ?= GATEWAY-HTTP,GATEWAY-GRPC,GATEWAY-TLS # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) diff --git a/api/v2/shared_types.go b/api/v2/shared_types.go index 06dae1d6..6c2c2934 100644 --- a/api/v2/shared_types.go +++ b/api/v2/shared_types.go @@ -101,6 +101,8 @@ const ( SchemeTCP = "tcp" // SchemeUDP represents the UDP protocol. SchemeUDP = "udp" + // SchemeTLS represents the TLS protocol. + SchemeTLS = "tls" ) const ( diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index d2bd08a0..07d8175b 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -94,6 +94,7 @@ rules: - httproutes/status - referencegrants/status - tcproutes/status + - tlsroutes/status - udproutes/status verbs: - get @@ -106,6 +107,7 @@ rules: - httproutes - referencegrants - tcproutes + - tlsroutes - udproutes verbs: - get diff --git a/docs/en/latest/concepts/gateway-api.md b/docs/en/latest/concepts/gateway-api.md index 4efb8a76..8a5e5864 100644 --- a/docs/en/latest/concepts/gateway-api.md +++ b/docs/en/latest/concepts/gateway-api.md @@ -50,7 +50,7 @@ By supporting Gateway API, the APISIX Ingress controller can realize richer func | HTTPRoute | Supported | Partially supported | Not supported | v1 | | GRPCRoute | Supported | Supported | Not supported | v1 | | ReferenceGrant | Supported | Not supported | Not supported | v1beta1 | -| TLSRoute | Not supported | Not supported | Not supported | v1alpha2 | +| TLSRoute | Supported | Supported | Not supported | v1alpha2 | | TCPRoute | Supported | Supported | Not supported | v1alpha2 | | UDPRoute | Supported | Supported | Not supported | v1alpha2 | | BackendTLSPolicy | Not supported | Not supported | Not supported | v1alpha3 | @@ -59,7 +59,7 @@ By supporting Gateway API, the APISIX Ingress controller can realize richer func For configuration examples, see the Gateway API tabs in [Configuration Examples](../reference/example.md). -For a complete list of configuration options, refer to the [Gateway API Reference](https://gateway-api.sigs.k8s.io/reference/main/spec/). Be aware that some fields are not supported, or partially supported. +For a complete list of configuration options, refer to the [Gateway API Reference](https://gateway-api.sigs.k8s.io/reference/spec/). Be aware that some fields are not supported, or partially supported. ## Unsupported / Partially Supported Fields diff --git a/internal/adc/translator/tlsroute.go b/internal/adc/translator/tlsroute.go new file mode 100644 index 00000000..85bfba0c --- /dev/null +++ b/internal/adc/translator/tlsroute.go @@ -0,0 +1,159 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package translator + +import ( + "fmt" + + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + + adctypes "github.com/apache/apisix-ingress-controller/api/adc" + apiv2 "github.com/apache/apisix-ingress-controller/api/v2" + "github.com/apache/apisix-ingress-controller/internal/controller/label" + "github.com/apache/apisix-ingress-controller/internal/id" + "github.com/apache/apisix-ingress-controller/internal/provider" + "github.com/apache/apisix-ingress-controller/internal/types" +) + +func (t *Translator) TranslateTLSRoute(tctx *provider.TranslateContext, tlsRoute *gatewayv1alpha2.TLSRoute) (*TranslateResult, error) { + result := &TranslateResult{} + rules := tlsRoute.Spec.Rules + labels := label.GenLabel(tlsRoute) + hosts := make([]string, 0, len(tlsRoute.Spec.Hostnames)) + for _, hostname := range tlsRoute.Spec.Hostnames { + hosts = append(hosts, string(hostname)) + } + for ruleIndex, rule := range rules { + service := adctypes.NewDefaultService() + service.Labels = labels + service.Name = adctypes.ComposeServiceNameWithStream(tlsRoute.Namespace, tlsRoute.Name, fmt.Sprintf("%d", ruleIndex), "TLS") + service.ID = id.GenID(service.Name) + var ( + upstreams = make([]*adctypes.Upstream, 0) + weightedUpstreams = make([]adctypes.TrafficSplitConfigRuleWeightedUpstream, 0) + ) + for _, backend := range rule.BackendRefs { + if backend.Namespace == nil { + namespace := gatewayv1.Namespace(tlsRoute.Namespace) + backend.Namespace = &namespace + } + upstream := newDefaultUpstreamWithoutScheme() + upNodes, err := t.translateBackendRef(tctx, backend, DefaultEndpointFilter) + if err != nil { + continue + } + if len(upNodes) == 0 { + continue + } + // TODO: Confirm BackendTrafficPolicy attachment with e2e test case. + t.AttachBackendTrafficPolicyToUpstream(backend, tctx.BackendTrafficPolicies, upstream) + upstream.Nodes = upNodes + var ( + kind string + port int32 + ) + if backend.Kind == nil { + kind = types.KindService + } else { + kind = string(*backend.Kind) + } + if backend.Port != nil { + port = int32(*backend.Port) + } + namespace := string(*backend.Namespace) + name := string(backend.Name) + upstreamName := adctypes.ComposeUpstreamNameForBackendRef(kind, namespace, name, port) + upstream.Name = upstreamName + upstream.ID = id.GenID(upstreamName) + upstreams = append(upstreams, upstream) + } + + // Handle multiple backends with traffic-split plugin + if len(upstreams) == 0 { + // Create a default upstream if no valid backends + upstream := adctypes.NewDefaultUpstream() + service.Upstream = upstream + } else if len(upstreams) == 1 { + // Single backend - use directly as service upstream + service.Upstream = upstreams[0] + // remove the id and name of the service.upstream, adc schema does not need id and name for it + service.Upstream.ID = "" + service.Upstream.Name = "" + } else { + // Multiple backends - use traffic-split plugin + service.Upstream = upstreams[0] + // remove the id and name of the service.upstream, adc schema does not need id and name for it + service.Upstream.ID = "" + service.Upstream.Name = "" + + upstreams = upstreams[1:] + + if len(upstreams) > 0 { + service.Upstreams = upstreams + } + + // Set weight in traffic-split for the default upstream + weight := apiv2.DefaultWeight + if rule.BackendRefs[0].Weight != nil { + weight = int(*rule.BackendRefs[0].Weight) + } + weightedUpstreams = append(weightedUpstreams, adctypes.TrafficSplitConfigRuleWeightedUpstream{ + Weight: weight, + }) + + // Set other upstreams in traffic-split using upstream_id + for i, upstream := range upstreams { + weight := apiv2.DefaultWeight + // get weight from the backend refs starting from the second backend + if i+1 < len(rule.BackendRefs) && rule.BackendRefs[i+1].Weight != nil { + weight = int(*rule.BackendRefs[i+1].Weight) + } + weightedUpstreams = append(weightedUpstreams, adctypes.TrafficSplitConfigRuleWeightedUpstream{ + UpstreamID: upstream.ID, + Weight: weight, + }) + } + + if len(weightedUpstreams) > 0 { + if service.Plugins == nil { + service.Plugins = make(map[string]any) + } + service.Plugins["traffic-split"] = &adctypes.TrafficSplitConfig{ + Rules: []adctypes.TrafficSplitConfigRule{ + { + WeightedUpstreams: weightedUpstreams, + }, + }, + } + } + } + + for _, host := range hosts { + streamRoute := adctypes.NewDefaultStreamRoute() + streamRouteName := adctypes.ComposeStreamRouteName(tlsRoute.Namespace, tlsRoute.Name, fmt.Sprintf("%d", ruleIndex), "TLS") + streamRoute.Name = streamRouteName + streamRoute.ID = id.GenID(streamRouteName) + streamRoute.SNI = host + streamRoute.Labels = labels + service.StreamRoutes = append(service.StreamRoutes, streamRoute) + } + result.Services = append(result.Services, service) + } + return result, nil +} diff --git a/internal/controller/indexer/indexer.go b/internal/controller/indexer/indexer.go index dc41b426..629f7c24 100644 --- a/internal/controller/indexer/indexer.go +++ b/internal/controller/indexer/indexer.go @@ -67,6 +67,7 @@ func SetupIndexer(mgr ctrl.Manager) error { &gatewayv1.GRPCRoute{}: setupGRPCRouteIndexer, &gatewayv1alpha2.TCPRoute{}: setupTCPRouteIndexer, &gatewayv1alpha2.UDPRoute{}: setupUDPRouteIndexer, + &gatewayv1alpha2.TLSRoute{}: setupTLSRouteIndexer, &gatewayv1.GatewayClass{}: setupGatewayClassIndexer, &v1alpha1.Consumer{}: setupConsumerIndexer, } { diff --git a/internal/controller/indexer/tlsroute.go b/internal/controller/indexer/tlsroute.go new file mode 100644 index 00000000..acef5317 --- /dev/null +++ b/internal/controller/indexer/tlsroute.go @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package indexer + +import ( + "context" + + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + + internaltypes "github.com/apache/apisix-ingress-controller/internal/types" +) + +func setupTLSRouteIndexer(mgr ctrl.Manager) error { + if err := mgr.GetFieldIndexer().IndexField( + context.Background(), + &gatewayv1alpha2.TLSRoute{}, + ParentRefs, + TLSRouteParentRefsIndexFunc, + ); err != nil { + return err + } + + if err := mgr.GetFieldIndexer().IndexField( + context.Background(), + &gatewayv1alpha2.TLSRoute{}, + ServiceIndexRef, + TLSPRouteServiceIndexFunc, + ); err != nil { + return err + } + return nil +} + +func TLSRouteParentRefsIndexFunc(rawObj client.Object) []string { + tr := rawObj.(*gatewayv1alpha2.TLSRoute) + keys := make([]string, 0, len(tr.Spec.ParentRefs)) + for _, ref := range tr.Spec.ParentRefs { + ns := tr.GetNamespace() + if ref.Namespace != nil { + ns = string(*ref.Namespace) + } + keys = append(keys, GenIndexKey(ns, string(ref.Name))) + } + return keys +} + +func TLSPRouteServiceIndexFunc(rawObj client.Object) []string { + tr := rawObj.(*gatewayv1alpha2.TLSRoute) + keys := make([]string, 0, len(tr.Spec.Rules)) + for _, rule := range tr.Spec.Rules { + for _, backend := range rule.BackendRefs { + namespace := tr.GetNamespace() + if backend.Kind != nil && *backend.Kind != internaltypes.KindService { + continue + } + if backend.Namespace != nil { + namespace = string(*backend.Namespace) + } + keys = append(keys, GenIndexKey(namespace, string(backend.Name))) + } + } + return keys +} diff --git a/internal/controller/tlsroute_controller.go b/internal/controller/tlsroute_controller.go new file mode 100644 index 00000000..f5f97721 --- /dev/null +++ b/internal/controller/tlsroute_controller.go @@ -0,0 +1,505 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package controller + +import ( + "cmp" + "context" + "fmt" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + k8stypes "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + "sigs.k8s.io/gateway-api/apis/v1beta1" + + "github.com/apache/apisix-ingress-controller/api/v1alpha1" + "github.com/apache/apisix-ingress-controller/internal/controller/indexer" + "github.com/apache/apisix-ingress-controller/internal/controller/status" + "github.com/apache/apisix-ingress-controller/internal/manager/readiness" + "github.com/apache/apisix-ingress-controller/internal/provider" + "github.com/apache/apisix-ingress-controller/internal/types" + "github.com/apache/apisix-ingress-controller/internal/utils" +) + +// TLSRouteReconciler reconciles a TLSRoute object. +type TLSRouteReconciler struct { //nolint:revive + client.Client + Scheme *runtime.Scheme + + Log logr.Logger + + Provider provider.Provider + + Updater status.Updater + Readier readiness.ReadinessManager +} + +// SetupWithManager sets up the controller with the Manager. +func (r *TLSRouteReconciler) SetupWithManager(mgr ctrl.Manager) error { + + bdr := ctrl.NewControllerManagedBy(mgr). + For(&gatewayv1alpha2.TLSRoute{}). + WithEventFilter(predicate.GenerationChangedPredicate{}). + Watches(&discoveryv1.EndpointSlice{}, + handler.EnqueueRequestsFromMapFunc(r.listTLSRoutesByServiceRef), + ). + Watches(&gatewayv1.Gateway{}, + handler.EnqueueRequestsFromMapFunc(r.listTLSRoutesForGateway), + builder.WithPredicates( + predicate.Funcs{ + GenericFunc: func(e event.GenericEvent) bool { + return false + }, + DeleteFunc: func(e event.DeleteEvent) bool { + return false + }, + CreateFunc: func(e event.CreateEvent) bool { + return true + }, + UpdateFunc: func(e event.UpdateEvent) bool { + return true + }, + }, + ), + ). + Watches(&v1alpha1.BackendTrafficPolicy{}, + handler.EnqueueRequestsFromMapFunc(r.listTLSRoutesForBackendTrafficPolicy), + ). + Watches(&v1alpha1.GatewayProxy{}, + handler.EnqueueRequestsFromMapFunc(r.listTLSRoutesForGatewayProxy), + ) + + if GetEnableReferenceGrant() { + bdr.Watches(&v1beta1.ReferenceGrant{}, + handler.EnqueueRequestsFromMapFunc(r.listTLSRoutesForReferenceGrant), + builder.WithPredicates(referenceGrantPredicates(types.KindTLSRoute)), + ) + } + + return bdr.Complete(r) +} + +func (r *TLSRouteReconciler) listTLSRoutesForBackendTrafficPolicy(ctx context.Context, obj client.Object) []reconcile.Request { + policy, ok := obj.(*v1alpha1.BackendTrafficPolicy) + if !ok { + r.Log.Error(fmt.Errorf("unexpected object type"), "failed to convert object to BackendTrafficPolicy") + return nil + } + + tlsrouteList := []gatewayv1alpha2.TLSRoute{} + for _, targetRef := range policy.Spec.TargetRefs { + service := &corev1.Service{} + if err := r.Get(ctx, client.ObjectKey{ + Namespace: policy.Namespace, + Name: string(targetRef.Name), + }, service); err != nil { + if client.IgnoreNotFound(err) != nil { + r.Log.Error(err, "failed to get service", "namespace", policy.Namespace, "name", targetRef.Name) + } + continue + } + trList := &gatewayv1alpha2.TLSRouteList{} + if err := r.List(ctx, trList, client.MatchingFields{ + indexer.ServiceIndexRef: indexer.GenIndexKey(policy.Namespace, string(targetRef.Name)), + }); err != nil { + r.Log.Error(err, "failed to list tlsroutes by service reference", "service", targetRef.Name) + return nil + } + tlsrouteList = append(tlsrouteList, trList.Items...) + } + var namespacedNameMap = make(map[k8stypes.NamespacedName]struct{}) + requests := make([]reconcile.Request, 0, len(tlsrouteList)) + for _, tr := range tlsrouteList { + key := k8stypes.NamespacedName{ + Namespace: tr.Namespace, + Name: tr.Name, + } + if _, ok := namespacedNameMap[key]; !ok { + namespacedNameMap[key] = struct{}{} + requests = append(requests, reconcile.Request{ + NamespacedName: client.ObjectKey{ + Namespace: tr.Namespace, + Name: tr.Name, + }, + }) + } + } + return requests +} + +func (r *TLSRouteReconciler) listTLSRoutesForGateway(ctx context.Context, obj client.Object) []reconcile.Request { + gateway, ok := obj.(*gatewayv1.Gateway) + if !ok { + r.Log.Error(fmt.Errorf("unexpected object type"), "failed to convert object to Gateway") + } + trList := &gatewayv1alpha2.TLSRouteList{} + if err := r.List(ctx, trList, client.MatchingFields{ + indexer.ParentRefs: indexer.GenIndexKey(gateway.Namespace, gateway.Name), + }); err != nil { + r.Log.Error(err, "failed to list tlsroutes by gateway", "gateway", gateway.Name) + return nil + } + + requests := make([]reconcile.Request, 0, len(trList.Items)) + for _, tcr := range trList.Items { + requests = append(requests, reconcile.Request{ + NamespacedName: client.ObjectKey{ + Namespace: tcr.Namespace, + Name: tcr.Name, + }, + }) + } + return requests +} + +// listTLSRoutesForGatewayProxy list all TLSRoute resources that are affected by a given GatewayProxy +func (r *TLSRouteReconciler) listTLSRoutesForGatewayProxy(ctx context.Context, obj client.Object) []reconcile.Request { + gatewayProxy, ok := obj.(*v1alpha1.GatewayProxy) + if !ok { + r.Log.Error(fmt.Errorf("unexpected object type"), "failed to convert object to GatewayProxy") + return nil + } + + namespace := gatewayProxy.GetNamespace() + name := gatewayProxy.GetName() + + // find all gateways that reference this gateway proxy + gatewayList := &gatewayv1.GatewayList{} + if err := r.List(ctx, gatewayList, client.MatchingFields{ + indexer.ParametersRef: indexer.GenIndexKey(namespace, name), + }); err != nil { + r.Log.Error(err, "failed to list gateways for gateway proxy", "gatewayproxy", gatewayProxy.GetName()) + return nil + } + + var requests []reconcile.Request + + // for each gateway, find all TLSRoute resources that reference it + for _, gateway := range gatewayList.Items { + tlsRouteList := &gatewayv1alpha2.TLSRouteList{} + if err := r.List(ctx, tlsRouteList, client.MatchingFields{ + indexer.ParentRefs: indexer.GenIndexKey(gateway.Namespace, gateway.Name), + }); err != nil { + r.Log.Error(err, "failed to list tlsroutes for gateway", "gateway", gateway.Name) + continue + } + + for _, tlsRoute := range tlsRouteList.Items { + requests = append(requests, reconcile.Request{ + NamespacedName: client.ObjectKey{ + Namespace: tlsRoute.Namespace, + Name: tlsRoute.Name, + }, + }) + } + } + + return requests +} + +func (r *TLSRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + defer r.Readier.Done(&gatewayv1alpha2.TLSRoute{}, req.NamespacedName) + tr := new(gatewayv1alpha2.TLSRoute) + if err := r.Get(ctx, req.NamespacedName, tr); err != nil { + if client.IgnoreNotFound(err) == nil { + tr.Namespace = req.Namespace + tr.Name = req.Name + + tr.TypeMeta = metav1.TypeMeta{ + Kind: types.KindTLSRoute, + APIVersion: gatewayv1alpha2.GroupVersion.String(), + } + + if err := r.Provider.Delete(ctx, tr); err != nil { + r.Log.Error(err, "failed to delete tlsroute", "tlsroute", tr) + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + type ResourceStatus struct { + status bool + msg string + } + + acceptStatus := ResourceStatus{ + status: true, + msg: "Route is accepted", + } + + gateways, err := ParseRouteParentRefs(ctx, r.Client, r.Log, tr, tr.Spec.ParentRefs) + if err != nil { + return ctrl.Result{}, err + } + + if len(gateways) == 0 { + return ctrl.Result{}, nil + } + + tctx := provider.NewDefaultTranslateContext(ctx) + + tctx.RouteParentRefs = tr.Spec.ParentRefs + rk := utils.NamespacedNameKind(tr) + for _, gateway := range gateways { + if err := ProcessGatewayProxy(r.Client, r.Log, tctx, gateway.Gateway, rk); err != nil { + acceptStatus.status = false + acceptStatus.msg = err.Error() + } + } + + var backendRefErr error + if err := r.processTLSRoute(tctx, tr); err != nil { + // When encountering a backend reference error, it should not affect the acceptance status + if types.IsSomeReasonError(err, gatewayv1.RouteReasonInvalidKind) { + backendRefErr = err + } else { + acceptStatus.status = false + acceptStatus.msg = err.Error() + } + } + + // Store the backend reference error for later use. + // If the backend reference error is because of an invalid kind, use this error first + if err := r.processTLSRouteBackendRefs(tctx, req.NamespacedName); err != nil && backendRefErr == nil { + backendRefErr = err + } + + ProcessBackendTrafficPolicy(r.Client, r.Log, tctx) + tr.Status.Parents = make([]gatewayv1.RouteParentStatus, 0, len(gateways)) + for _, gateway := range gateways { + parentStatus := gatewayv1.RouteParentStatus{} + SetRouteParentRef(&parentStatus, gateway.Gateway.Name, gateway.Gateway.Namespace) + for _, condition := range gateway.Conditions { + parentStatus.Conditions = MergeCondition(parentStatus.Conditions, condition) + } + SetRouteConditionAccepted(&parentStatus, tr.GetGeneration(), acceptStatus.status, acceptStatus.msg) + SetRouteConditionResolvedRefs(&parentStatus, tr.GetGeneration(), backendRefErr) + + tr.Status.Parents = append(tr.Status.Parents, parentStatus) + } + + r.Updater.Update(status.Update{ + NamespacedName: utils.NamespacedName(tr), + Resource: &gatewayv1alpha2.TLSRoute{}, + Mutator: status.MutatorFunc(func(obj client.Object) client.Object { + t, ok := obj.(*gatewayv1alpha2.TLSRoute) + if !ok { + err := fmt.Errorf("unsupported object type %T", obj) + panic(err) + } + tCopy := t.DeepCopy() + tCopy.Status = tr.Status + return tCopy + }), + }) + UpdateStatus(r.Updater, r.Log, tctx) + if isRouteAccepted(gateways) { + routeToUpdate := tr + if err := r.Provider.Update(ctx, tctx, routeToUpdate); err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{}, nil +} + +func (r *TLSRouteReconciler) processTLSRoute(tctx *provider.TranslateContext, tlsRoute *gatewayv1alpha2.TLSRoute) error { + var terror error + for _, rule := range tlsRoute.Spec.Rules { + for _, backend := range rule.BackendRefs { + if backend.Kind != nil && *backend.Kind != KindService { + terror = types.NewInvalidKindError(*backend.Kind) + continue + } + tctx.BackendRefs = append(tctx.BackendRefs, gatewayv1.BackendRef{ + BackendObjectReference: gatewayv1.BackendObjectReference{ + Name: backend.Name, + Namespace: cmp.Or(backend.Namespace, (*gatewayv1.Namespace)(&tlsRoute.Namespace)), + Port: backend.Port, + }, + }) + } + } + + return terror +} + +func (r *TLSRouteReconciler) processTLSRouteBackendRefs(tctx *provider.TranslateContext, trNN k8stypes.NamespacedName) error { + var terr error + for _, backend := range tctx.BackendRefs { + targetNN := k8stypes.NamespacedName{ + Namespace: trNN.Namespace, + Name: string(backend.Name), + } + if backend.Namespace != nil { + targetNN.Namespace = string(*backend.Namespace) + } + + if backend.Kind != nil && *backend.Kind != KindService { + terr = types.NewInvalidKindError(*backend.Kind) + continue + } + + if backend.Port == nil { + terr = fmt.Errorf("port is required") + continue + } + + var service corev1.Service + if err := r.Get(tctx, targetNN, &service); err != nil { + terr = err + if client.IgnoreNotFound(err) == nil { + terr = types.ReasonError{ + Reason: string(gatewayv1.RouteReasonBackendNotFound), + Message: fmt.Sprintf("Service %s not found", targetNN), + } + } + continue + } + + // if cross namespaces between TLSRoute and referenced Service, check ReferenceGrant + if trNN.Namespace != targetNN.Namespace { + if permitted := checkReferenceGrant(tctx, + r.Client, + v1beta1.ReferenceGrantFrom{ + Group: gatewayv1.GroupName, + Kind: types.KindTLSRoute, + Namespace: v1beta1.Namespace(trNN.Namespace), + }, + gatewayv1.ObjectReference{ + Group: corev1.GroupName, + Kind: types.KindService, + Name: gatewayv1.ObjectName(targetNN.Name), + Namespace: (*gatewayv1.Namespace)(&targetNN.Namespace), + }, + ); !permitted { + terr = types.ReasonError{ + Reason: string(v1beta1.RouteReasonRefNotPermitted), + Message: fmt.Sprintf("%s is in a different namespace than the TLSRoute %s and no ReferenceGrant allowing reference is configured", targetNN, trNN), + } + continue + } + } + + if service.Spec.Type == corev1.ServiceTypeExternalName { + tctx.Services[targetNN] = &service + continue + } + + portExists := false + for _, port := range service.Spec.Ports { + if port.Port == int32(*backend.Port) { + portExists = true + break + } + } + if !portExists { + terr = fmt.Errorf("port %d not found in service %s", *backend.Port, targetNN.Name) + continue + } + tctx.Services[targetNN] = &service + + endpointSliceList := new(discoveryv1.EndpointSliceList) + if err := r.List(tctx, endpointSliceList, + client.InNamespace(targetNN.Namespace), + client.MatchingLabels{ + discoveryv1.LabelServiceName: targetNN.Name, + }, + ); err != nil { + r.Log.Error(err, "failed to list endpoint slices", "Service", targetNN) + terr = err + continue + } + + tctx.EndpointSlices[targetNN] = endpointSliceList.Items + } + return terr +} + +func (r *TLSRouteReconciler) listTLSRoutesForReferenceGrant(ctx context.Context, obj client.Object) (requests []reconcile.Request) { + grant, ok := obj.(*v1beta1.ReferenceGrant) + if !ok { + r.Log.Error(fmt.Errorf("unexpected object type"), "failed to convert object to ReferenceGrant") + return nil + } + + var tlsRouteList gatewayv1alpha2.TLSRouteList + if err := r.List(ctx, &tlsRouteList); err != nil { + r.Log.Error(err, "failed to list tlsroutes for reference ReferenceGrant", "ReferenceGrant", k8stypes.NamespacedName{Namespace: obj.GetNamespace(), Name: obj.GetName()}) + return nil + } + + for _, tlsRoute := range tlsRouteList.Items { + tr := v1beta1.ReferenceGrantFrom{ + Group: gatewayv1.GroupName, + Kind: types.KindTLSRoute, + Namespace: v1beta1.Namespace(tlsRoute.GetNamespace()), + } + for _, from := range grant.Spec.From { + if from == tr { + requests = append(requests, reconcile.Request{ + NamespacedName: client.ObjectKey{ + Namespace: tlsRoute.GetNamespace(), + Name: tlsRoute.GetName(), + }, + }) + } + } + } + return requests +} + +func (r *TLSRouteReconciler) listTLSRoutesByServiceRef(ctx context.Context, obj client.Object) []reconcile.Request { + endpointSlice, ok := obj.(*discoveryv1.EndpointSlice) + if !ok { + r.Log.Error(fmt.Errorf("unexpected object type"), "failed to convert object to EndpointSlice") + return nil + } + namespace := endpointSlice.GetNamespace() + serviceName := endpointSlice.Labels[discoveryv1.LabelServiceName] + + trList := &gatewayv1alpha2.TLSRouteList{} + if err := r.List(ctx, trList, client.MatchingFields{ + indexer.ServiceIndexRef: indexer.GenIndexKey(namespace, serviceName), + }); err != nil { + r.Log.Error(err, "failed to list tlsroutes by service", "service", serviceName) + return nil + } + requests := make([]reconcile.Request, 0, len(trList.Items)) + for _, tr := range trList.Items { + requests = append(requests, reconcile.Request{ + NamespacedName: client.ObjectKey{ + Namespace: tr.Namespace, + Name: tr.Name, + }, + }) + } + return requests +} diff --git a/internal/controller/utils.go b/internal/controller/utils.go index 98bed4fd..35fdcd96 100644 --- a/internal/controller/utils.go +++ b/internal/controller/utils.go @@ -509,6 +509,8 @@ func routeHostnamesIntersectsWithListenerHostname(route client.Object, listener return true // TCPRoute and UDPRoute don't have Hostnames to match case *gatewayv1.GRPCRoute: return listenerHostnameIntersectWithRouteHostnames(listener, r.Spec.Hostnames) + case *gatewayv1alpha2.TLSRoute: + return listenerHostnameIntersectWithRouteHostnames(listener, r.Spec.Hostnames) default: return false } @@ -681,6 +683,10 @@ func routeMatchesListenerType(route client.Object, listener gatewayv1.Listener) if listener.Protocol != gatewayv1.UDPProtocolType { return false } + case *gatewayv1alpha2.TLSRoute: + if listener.Protocol != gatewayv1.TLSProtocolType { + return false + } default: return false } diff --git a/internal/manager/controllers.go b/internal/manager/controllers.go index 9ceb79a9..f8251e1d 100644 --- a/internal/manager/controllers.go +++ b/internal/manager/controllers.go @@ -95,6 +95,8 @@ import ( // +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=referencegrants/status,verbs=get;update // +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=grpcroutes,verbs=get;list;watch // +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=grpcroutes/status,verbs=get;update +// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=tlsroutes,verbs=get;list;watch +// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=tlsroutes/status,verbs=get;update // Networking // +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch @@ -173,6 +175,14 @@ func setupControllers(ctx context.Context, mgr manager.Manager, pro provider.Pro Updater: updater, Readier: readier, }, + &gatewayv1alpha2.TLSRoute{}: &controller.TLSRouteReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Log: ctrl.LoggerFrom(ctx).WithName("controllers").WithName(types.KindTLSRoute), + Provider: pro, + Updater: updater, + Readier: readier, + }, &v1alpha1.Consumer{}: &controller.ConsumerReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), @@ -336,6 +346,9 @@ func registerGatewayAPIForReadinessGVK(mgr manager.Manager, readier readiness.Re if utils.HasAPIResource(mgr, &gatewayv1alpha2.UDPRoute{}) { gvks = append(gvks, types.GvkOf(&gatewayv1alpha2.UDPRoute{})) } + if utils.HasAPIResource(mgr, &gatewayv1alpha2.TLSRoute{}) { + gvks = append(gvks, types.GvkOf(&gatewayv1alpha2.TLSRoute{})) + } if len(gvks) == 0 { return } diff --git a/internal/provider/api7ee/provider.go b/internal/provider/api7ee/provider.go index c3597adb..aab86a0a 100644 --- a/internal/provider/api7ee/provider.go +++ b/internal/provider/api7ee/provider.go @@ -111,6 +111,9 @@ func (d *api7eeProvider) Update(ctx context.Context, tctx *provider.TranslateCon case *gatewayv1alpha2.UDPRoute: result, err = d.translator.TranslateUDPRoute(tctx, t.DeepCopy()) resourceTypes = append(resourceTypes, adctypes.TypeService) + case *gatewayv1alpha2.TLSRoute: + result, err = d.translator.TranslateTLSRoute(tctx, t.DeepCopy()) + resourceTypes = append(resourceTypes, adctypes.TypeService) case *gatewayv1.GRPCRoute: result, err = d.translator.TranslateGRPCRoute(tctx, t.DeepCopy()) resourceTypes = append(resourceTypes, "service") @@ -192,7 +195,7 @@ func (d *api7eeProvider) Delete(ctx context.Context, obj client.Object) error { var resourceTypes []string var labels map[string]string switch obj.(type) { - case *gatewayv1.HTTPRoute, *apiv2.ApisixRoute, *gatewayv1.GRPCRoute, *gatewayv1alpha2.TCPRoute, *gatewayv1alpha2.UDPRoute: + case *gatewayv1.HTTPRoute, *apiv2.ApisixRoute, *gatewayv1.GRPCRoute, *gatewayv1alpha2.TCPRoute, *gatewayv1alpha2.UDPRoute, *gatewayv1alpha2.TLSRoute: resourceTypes = append(resourceTypes, "service") labels = label.GenLabel(obj) case *gatewayv1.Gateway: diff --git a/internal/provider/api7ee/status.go b/internal/provider/api7ee/status.go index bda06bd7..0fc5fb79 100644 --- a/internal/provider/api7ee/status.go +++ b/internal/provider/api7ee/status.go @@ -248,6 +248,41 @@ func (d *api7eeProvider) updateStatus(nnk types.NamespacedNameKind, condition me return cp }), }) + case types.KindTLSRoute: + parentRefs := d.client.ConfigManager.GetConfigRefsByResourceKey(nnk) + d.log.V(1).Info("updating TLSRoute status", "parentRefs", parentRefs) + gatewayRefs := map[types.NamespacedNameKind]struct{}{} + for _, parentRef := range parentRefs { + if parentRef.Kind == types.KindGateway { + gatewayRefs[parentRef] = struct{}{} + } + } + d.updater.Update(status.Update{ + NamespacedName: nnk.NamespacedName(), + Resource: &gatewayv1alpha2.TLSRoute{}, + Mutator: status.MutatorFunc(func(obj client.Object) client.Object { + cp := obj.(*gatewayv1alpha2.TLSRoute).DeepCopy() + gatewayNs := cp.GetNamespace() + for i, ref := range cp.Status.Parents { + ns := gatewayNs + if ref.ParentRef.Namespace != nil { + ns = string(*ref.ParentRef.Namespace) + } + if ref.ParentRef.Kind == nil || *ref.ParentRef.Kind == types.KindGateway { + nnk := types.NamespacedNameKind{ + Name: string(ref.ParentRef.Name), + Namespace: ns, + Kind: types.KindGateway, + } + if _, ok := gatewayRefs[nnk]; ok { + ref.Conditions = cutils.MergeCondition(ref.Conditions, condition) + cp.Status.Parents[i] = ref + } + } + } + return cp + }), + }) } } diff --git a/internal/provider/apisix/provider.go b/internal/provider/apisix/provider.go index bec6eaa6..d7974e9a 100644 --- a/internal/provider/apisix/provider.go +++ b/internal/provider/apisix/provider.go @@ -118,6 +118,9 @@ func (d *apisixProvider) Update(ctx context.Context, tctx *provider.TranslateCon case *gatewayv1alpha2.UDPRoute: result, err = d.translator.TranslateUDPRoute(tctx, t.DeepCopy()) resourceTypes = append(resourceTypes, adctypes.TypeService) + case *gatewayv1alpha2.TLSRoute: + result, err = d.translator.TranslateTLSRoute(tctx, t.DeepCopy()) + resourceTypes = append(resourceTypes, adctypes.TypeService) case *gatewayv1.GRPCRoute: result, err = d.translator.TranslateGRPCRoute(tctx, t.DeepCopy()) resourceTypes = append(resourceTypes, adctypes.TypeService) @@ -195,7 +198,7 @@ func (d *apisixProvider) Delete(ctx context.Context, obj client.Object) error { var resourceTypes []string var labels map[string]string switch obj.(type) { - case *gatewayv1.HTTPRoute, *apiv2.ApisixRoute, *gatewayv1.GRPCRoute, *gatewayv1alpha2.TCPRoute, *gatewayv1alpha2.UDPRoute: + case *gatewayv1.HTTPRoute, *apiv2.ApisixRoute, *gatewayv1.GRPCRoute, *gatewayv1alpha2.TCPRoute, *gatewayv1alpha2.UDPRoute, *gatewayv1alpha2.TLSRoute: resourceTypes = append(resourceTypes, adctypes.TypeService) labels = label.GenLabel(obj) case *gatewayv1.Gateway: diff --git a/internal/provider/apisix/status.go b/internal/provider/apisix/status.go index 4e60a624..327856ec 100644 --- a/internal/provider/apisix/status.go +++ b/internal/provider/apisix/status.go @@ -249,6 +249,41 @@ func (d *apisixProvider) updateStatus(nnk types.NamespacedNameKind, condition me return cp }), }) + case types.KindTLSRoute: + parentRefs := d.client.ConfigManager.GetConfigRefsByResourceKey(nnk) + d.log.V(1).Info("updating TLSRoute status", "parentRefs", parentRefs) + gatewayRefs := map[types.NamespacedNameKind]struct{}{} + for _, parentRef := range parentRefs { + if parentRef.Kind == types.KindGateway { + gatewayRefs[parentRef] = struct{}{} + } + } + d.updater.Update(status.Update{ + NamespacedName: nnk.NamespacedName(), + Resource: &gatewayv1alpha2.TLSRoute{}, + Mutator: status.MutatorFunc(func(obj client.Object) client.Object { + cp := obj.(*gatewayv1alpha2.TLSRoute).DeepCopy() + gatewayNs := cp.GetNamespace() + for i, ref := range cp.Status.Parents { + ns := gatewayNs + if ref.ParentRef.Namespace != nil { + ns = string(*ref.ParentRef.Namespace) + } + if ref.ParentRef.Kind == nil || *ref.ParentRef.Kind == types.KindGateway { + nnk := types.NamespacedNameKind{ + Name: string(ref.ParentRef.Name), + Namespace: ns, + Kind: types.KindGateway, + } + if _, ok := gatewayRefs[nnk]; ok { + ref.Conditions = cutils.MergeCondition(ref.Conditions, condition) + cp.Status.Parents[i] = ref + } + } + } + return cp + }), + }) } } diff --git a/internal/types/k8s.go b/internal/types/k8s.go index 321582c8..7b160d5d 100644 --- a/internal/types/k8s.go +++ b/internal/types/k8s.go @@ -44,6 +44,7 @@ const ( KindTCPRoute = "TCPRoute" KindUDPRoute = "UDPRoute" KindGRPCRoute = "GRPCRoute" + KindTLSRoute = "TLSRoute" KindGatewayClass = "GatewayClass" KindIngress = "Ingress" KindIngressClass = "IngressClass" @@ -75,6 +76,8 @@ func KindOf(obj any) string { return KindHTTPRoute case *gatewayv1.GRPCRoute: return KindGRPCRoute + case *gatewayv1alpha2.TLSRoute: + return KindTLSRoute case *gatewayv1.GatewayClass: return KindGatewayClass case *netv1.Ingress: @@ -137,9 +140,7 @@ func GvkOf(obj any) schema.GroupVersionKind { switch obj.(type) { case *gatewayv1.Gateway, *gatewayv1.HTTPRoute, *gatewayv1.GatewayClass, *gatewayv1.GRPCRoute: return gatewayv1.SchemeGroupVersion.WithKind(kind) - case *gatewayv1alpha2.TCPRoute: - return gatewayv1alpha2.SchemeGroupVersion.WithKind(kind) - case *gatewayv1alpha2.UDPRoute: + case *gatewayv1alpha2.TCPRoute, *gatewayv1alpha2.UDPRoute, *gatewayv1alpha2.TLSRoute: return gatewayv1alpha2.SchemeGroupVersion.WithKind(kind) case *gatewayv1beta1.ReferenceGrant: return gatewayv1beta1.SchemeGroupVersion.WithKind(kind) diff --git a/test/conformance/conformance_test.go b/test/conformance/conformance_test.go index 42161ad5..8574f7f6 100644 --- a/test/conformance/conformance_test.go +++ b/test/conformance/conformance_test.go @@ -34,6 +34,9 @@ import ( var skippedTestsForSSL = []string{ tests.HTTPRouteHTTPSListener.ShortName, tests.HTTPRouteRedirectPortAndScheme.ShortName, + + // TODO: APISIX does not support TLSRoute passthrough. + tests.TLSRouteSimpleSameNamespace.ShortName, } // TODO: HTTPRoute hostname intersection and listener hostname matching diff --git a/test/e2e/framework/manifests/apisix-standalone.yaml b/test/e2e/framework/manifests/apisix-standalone.yaml index 4b7adfe9..0eda2bc8 100644 --- a/test/e2e/framework/manifests/apisix-standalone.yaml +++ b/test/e2e/framework/manifests/apisix-standalone.yaml @@ -40,6 +40,8 @@ data: stream_proxy: # TCP/UDP proxy tcp: # TCP proxy port list - 9100 + - addr: 9110 + tls: true udp: # UDP proxy port list - 9200 discovery: @@ -101,6 +103,9 @@ spec: - name: udp containerPort: 9200 protocol: UDP + - name: tls + containerPort: 9110 + protocol: TCP volumeMounts: - name: config-writable mountPath: /usr/local/apisix/conf @@ -139,6 +144,10 @@ spec: port: 9200 protocol: UDP targetPort: 9200 + - name: tls + port: 9110 + protocol: TCP + targetPort: 9110 selector: app.kubernetes.io/name: apisix type: {{ .ServiceType | default "NodePort" }} diff --git a/test/e2e/framework/manifests/apisix.yaml b/test/e2e/framework/manifests/apisix.yaml index ae8a1396..31581bcc 100644 --- a/test/e2e/framework/manifests/apisix.yaml +++ b/test/e2e/framework/manifests/apisix.yaml @@ -47,6 +47,8 @@ data: stream_proxy: # TCP/UDP proxy tcp: # TCP proxy port list - 9100 + - addr: 9110 + tls: true udp: # UDP proxy port list - 9200 discovery: @@ -111,6 +113,9 @@ spec: - name: udp containerPort: 9200 protocol: UDP + - name: tls + containerPort: 9110 + protocol: TCP volumeMounts: - name: config-writable mountPath: /usr/local/apisix/conf @@ -156,6 +161,10 @@ spec: port: 9200 protocol: UDP targetPort: 9200 + - name: tls + port: 9110 + protocol: TCP + targetPort: 9110 selector: app.kubernetes.io/name: apisix type: {{ .ServiceType | default "NodePort" }} diff --git a/test/e2e/framework/manifests/dp.yaml b/test/e2e/framework/manifests/dp.yaml index b34d4363..d4692e9b 100644 --- a/test/e2e/framework/manifests/dp.yaml +++ b/test/e2e/framework/manifests/dp.yaml @@ -34,6 +34,8 @@ data: stream_proxy: tcp: - 9100 + - addr: 9110 + tls: true udp: - 9200 nginx_config: @@ -211,7 +213,7 @@ spec: name: admin protocol: TCP - containerPort: 9443 - name: tls + name: https protocol: TCP - containerPort: 9090 name: control-api @@ -222,6 +224,9 @@ spec: - containerPort: 9200 name: udp protocol: UDP + - containerPort: 9110 + name: tls + protocol: TCP readinessProbe: failureThreshold: 10 initialDelaySeconds: 3 @@ -286,6 +291,10 @@ spec: port: 9200 protocol: UDP targetPort: 9200 + - name: tls + port: 9110 + protocol: TCP + targetPort: 9110 selector: app.kubernetes.io/instance: api7ee3 app.kubernetes.io/name: apisix diff --git a/test/e2e/framework/manifests/ingress.yaml b/test/e2e/framework/manifests/ingress.yaml index f854b04d..b7838494 100644 --- a/test/e2e/framework/manifests/ingress.yaml +++ b/test/e2e/framework/manifests/ingress.yaml @@ -159,6 +159,7 @@ rules: - httproutes/status - referencegrants/status - tcproutes/status + - tlsroutes/status - udproutes/status verbs: - get @@ -171,6 +172,7 @@ rules: - httproutes - referencegrants - tcproutes + - tlsroutes - udproutes verbs: - get diff --git a/test/e2e/gatewayapi/tlsroute.go b/test/e2e/gatewayapi/tlsroute.go new file mode 100644 index 00000000..74fc1b93 --- /dev/null +++ b/test/e2e/gatewayapi/tlsroute.go @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package gatewayapi + +import ( + "fmt" + "net/http" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/apache/apisix-ingress-controller/test/e2e/scaffold" +) + +var _ = Describe("Test TLSRoute", Label("networking.k8s.io", "tlsroute"), func() { + s := scaffold.NewDefaultScaffold() + + Context("TLSRoute Base", func() { + var ( + host = "api6.com" + secretName = _secretName + tlsGateway = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: tls-gateway +spec: + gatewayClassName: %s + listeners: + - name: https + protocol: TLS + port: 443 + hostname: api6.com + tls: + certificateRefs: + - kind: Secret + group: "" + name: %s + infrastructure: + parametersRef: + group: apisix.apache.org + kind: GatewayProxy + name: apisix-proxy-config +` + tlsRoute = ` +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: TLSRoute +metadata: + name: tls-route +spec: + parentRefs: + - name: tls-gateway + hostnames: ["api6.com"] + rules: + - backendRefs: + - name: httpbin-service-e2e-test + port: 80 +` + ) + BeforeEach(func() { + createSecret(s, secretName) + By("create GatewayProxy") + Expect(s.CreateResourceFromString(s.GetGatewayProxySpec())).NotTo(HaveOccurred(), "creating GatewayProxy") + + By("create GatewayClass") + Expect(s.CreateResourceFromString(s.GetGatewayClassYaml())).NotTo(HaveOccurred(), "creating GatewayClass") + + // Create Gateway with TCP listener + By("create Gateway") + Expect(s.CreateResourceFromString(fmt.Sprintf(tlsGateway, s.Namespace(), secretName))).NotTo(HaveOccurred(), "creating Gateway") + }) + It("Basic", func() { + s.ResourceApplied("TLSRoute", "tls-route", tlsRoute, 1) + + client := s.NewAPISIXClientWithTLSProxy(host) + s.RequestAssert(&scaffold.RequestAssert{ + Client: client, + Method: http.MethodGet, + Path: "/ip", + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) + s.RequestAssert(&scaffold.RequestAssert{ + Client: client, + Method: http.MethodGet, + Path: "/notfound", + Check: scaffold.WithExpectedStatus(http.StatusNotFound), + }) + + Expect(s.DeleteResourceFromString(tlsRoute)).NotTo(HaveOccurred(), "deleting TLSRoute") + + s.RetryAssertion(func() string { + var errMsg string + reporter := &scaffold.ErrorReporter{} + _ = client.GET("/ip").WithReporter(reporter).Expect() + if reporter.Err() != nil { + errMsg = reporter.Err().Error() + } + return errMsg + }).Should(ContainSubstring("EOF"), "should get EOF after deleting TLSRoute") + }) + }) +}) diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go index 1468b11f..9256a97d 100644 --- a/test/e2e/scaffold/scaffold.go +++ b/test/e2e/scaffold/scaffold.go @@ -86,6 +86,7 @@ type Tunnels struct { HTTPS *k8s.Tunnel TCP *k8s.Tunnel HTTP2 *k8s.Tunnel + TLS *k8s.Tunnel } func (t *Tunnels) Close() { @@ -105,6 +106,10 @@ func (t *Tunnels) Close() { t.safeClose(t.HTTP2.Close) t.HTTP2 = nil } + if t.TLS != nil { + t.safeClose(t.TLS.Close) + t.TLS = nil + } } func (t *Tunnels) safeClose(close func()) { @@ -284,6 +289,31 @@ func (s *Scaffold) NewAPISIXClientWithTCPProxy() *httpexpect.Expect { }) } +func (s *Scaffold) NewAPISIXClientWithTLSProxy(host string) *httpexpect.Expect { + u := url.URL{ + Scheme: apiv2.SchemeHTTPS, + Host: s.apisixTunnels.TLS.Endpoint(), + } + return httpexpect.WithConfig(httpexpect.Config{ + BaseURL: u.String(), + Client: &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + // accept any certificate; for testing only! + InsecureSkipVerify: true, + ServerName: host, + }, + }, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + Reporter: httpexpect.NewAssertReporter( + httpexpect.NewAssertReporter(s.GinkgoT), + ), + }) +} + func (s *Scaffold) DefaultDataplaneResource() DataplaneResource { return s.Deployer.DefaultDataplaneResource() } @@ -374,6 +404,7 @@ func (s *Scaffold) createDataplaneTunnels( httpsPort int tcpPort int http2Port int + tlsPort int ) for _, port := range svc.Spec.Ports { @@ -386,6 +417,8 @@ func (s *Scaffold) createDataplaneTunnels( tcpPort = int(port.Port) case "http2": http2Port = int(port.Port) + case apiv2.SchemeTLS: + tlsPort = int(port.Port) } } @@ -400,6 +433,8 @@ func (s *Scaffold) createDataplaneTunnels( 0, tcpPort) http2Tunnel := k8s.NewTunnel(kubectlOpts, k8s.ResourceTypeService, serviceName, 0, http2Port) + tlsTunnel := k8s.NewTunnel(kubectlOpts, k8s.ResourceTypeService, serviceName, + 0, tlsPort) if err := httpTunnel.ForwardPortE(s.t); err != nil { return nil, err @@ -415,6 +450,10 @@ func (s *Scaffold) createDataplaneTunnels( return nil, err } tunnels.TCP = tcpTunnel + if err := tlsTunnel.ForwardPortE(s.t); err != nil { + return nil, err + } + tunnels.TLS = tlsTunnel if http2Port != 0 { if err := http2Tunnel.ForwardPortE(s.t); err != nil {