From 8ec38d8819aaa88be3860d2c64b8671b7ef15082 Mon Sep 17 00:00:00 2001 From: ShyunnY <1147212064@qq.com> Date: Thu, 23 May 2024 20:51:34 +0800 Subject: [PATCH 1/2] feat: support caCerts in secrets (#2777) Signed-off-by: ShyunnY <1147212064@qq.com> --- internal/gatewayapi/backendtlspolicy.go | 48 +++-- .../backendtlspolicy-ca-only-secret.in.yaml | 117 +++++++++++++ .../backendtlspolicy-ca-only-secret.out.yaml | 165 ++++++++++++++++++ internal/provider/kubernetes/controller.go | 44 +++-- 4 files changed, 345 insertions(+), 29 deletions(-) create mode 100644 internal/gatewayapi/testdata/backendtlspolicy-ca-only-secret.in.yaml create mode 100644 internal/gatewayapi/testdata/backendtlspolicy-ca-only-secret.out.yaml diff --git a/internal/gatewayapi/backendtlspolicy.go b/internal/gatewayapi/backendtlspolicy.go index cde6d26a151..f51b011229f 100644 --- a/internal/gatewayapi/backendtlspolicy.go +++ b/internal/gatewayapi/backendtlspolicy.go @@ -8,7 +8,6 @@ package gatewayapi import ( "fmt" - corev1 "k8s.io/api/core/v1" "k8s.io/utils/ptr" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" @@ -29,7 +28,7 @@ func (t *Translator) processBackendTLSPolicy( return nil } - tlsBundle, err := getBackendTLSBundle(policy, resources.ConfigMaps) + tlsBundle, err := getBackendTLSBundle(policy, resources) if err == nil && tlsBundle == nil { return nil } @@ -104,7 +103,7 @@ func getBackendTLSPolicy(policies []*gwapiv1a3.BackendTLSPolicy, backendRef gwap return nil } -func getBackendTLSBundle(backendTLSPolicy *gwapiv1a3.BackendTLSPolicy, configmaps []*corev1.ConfigMap) (*ir.TLSUpstreamConfig, error) { +func getBackendTLSBundle(backendTLSPolicy *gwapiv1a3.BackendTLSPolicy, resources *Resources) (*ir.TLSUpstreamConfig, error) { tlsBundle := &ir.TLSUpstreamConfig{ SNI: string(backendTLSPolicy.Spec.Validation.Hostname), UseSystemTrustStore: ptr.Deref(backendTLSPolicy.Spec.Validation.WellKnownCACertificates, "") == gwapiv1a3.WellKnownCACertificatesSystem, @@ -113,23 +112,36 @@ func getBackendTLSBundle(backendTLSPolicy *gwapiv1a3.BackendTLSPolicy, configmap return tlsBundle, nil } - caRefMap := make(map[string]string) - - for _, caRef := range backendTLSPolicy.Spec.Validation.CACertificateRefs { - caRefMap[string(caRef.Name)] = string(caRef.Kind) - } - ca := "" - - for _, cmap := range configmaps { - if kind, ok := caRefMap[cmap.Name]; ok && kind == cmap.Kind { - if crt, dataOk := cmap.Data["ca.crt"]; dataOk { - if ca != "" { - ca += "\n" + for _, caRef := range backendTLSPolicy.Spec.Validation.CACertificateRefs { + kind := string(caRef.Kind) + + switch kind { + case KindConfigMap: + for _, cmap := range resources.ConfigMaps { + if cmap.Name == string(caRef.Name) { + if crt, dataOk := cmap.Data["ca.crt"]; dataOk { + if ca != "" { + ca += "\n" + } + ca += crt + } else { + return nil, fmt.Errorf("no ca found in configmap %s", cmap.Name) + } + } + } + case KindSecret: + for _, secret := range resources.Secrets { + if secret.Name == string(caRef.Name) { + if crt, dataOk := secret.Data["ca.crt"]; dataOk { + if ca != "" { + ca += "\n" + } + ca += string(crt) + } else { + return nil, fmt.Errorf("no ca found in secret %s", secret.Name) + } } - ca += crt - } else { - return nil, fmt.Errorf("no ca found in configmap %s", cmap.Name) } } } diff --git a/internal/gatewayapi/testdata/backendtlspolicy-ca-only-secret.in.yaml b/internal/gatewayapi/testdata/backendtlspolicy-ca-only-secret.in.yaml new file mode 100644 index 00000000000..53d85667b30 --- /dev/null +++ b/internal/gatewayapi/testdata/backendtlspolicy-ca-only-secret.in.yaml @@ -0,0 +1,117 @@ +gateways: + - apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + name: gateway-btls + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All +httpRoutes: + - apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + name: httproute-btls + namespace: envoy-gateway + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-btls + sectionName: http + rules: + - matches: + - path: + type: Exact + value: "/exact" + backendRefs: + - name: http-backend + namespace: backends + port: 8080 + +referenceGrants: + - apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: ReferenceGrant + metadata: + name: refg-route-svc + namespace: backends + spec: + from: + - group: gateway.networking.k8s.io + kind: HTTPRoute + namespace: envoy-gateway + - group: gateway.networking.k8s.io + kind: Gateway + namespace: envoy-gateway + - group: gateway.networking.k8s.io + kind: BackendTLSPolicy + namespace: policies + to: + - group: "" + kind: Service + +services: + - apiVersion: v1 + kind: Service + metadata: + name: http-backend + namespace: backends + spec: + clusterIP: 10.11.12.13 + ports: + - port: 8080 + name: http + protocol: TCP + targetPort: 8080 + + +endpointSlices: + - apiVersion: discovery.k8s.io/v1 + kind: EndpointSlice + metadata: + name: endpointslice-http-backend + namespace: backends + labels: + kubernetes.io/service-name: http-backend + addressType: IPv4 + ports: + - name: http + protocol: TCP + port: 8080 + endpoints: + - addresses: + - "10.244.0.11" + conditions: + ready: true + +secrets: + - apiVersion: v1 + kind: Secret + metadata: + name: ca-secret + namespace: policies + data: + ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURKekNDQWcrZ0F3SUJBZ0lVQWw2VUtJdUttenRlODFjbGx6NVBmZE4ySWxJd0RRWUpLb1pJaHZjTkFRRUwKQlFBd0l6RVFNQTRHQTFVRUF3d0hiWGxqYVdWdWRERVBNQTBHQTFVRUNnd0dhM1ZpWldSaU1CNFhEVEl6TVRBdwpNakExTkRFMU4xb1hEVEkwTVRBd01UQTFOREUxTjFvd0l6RVFNQTRHQTFVRUF3d0hiWGxqYVdWdWRERVBNQTBHCkExVUVDZ3dHYTNWaVpXUmlNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQXdTVGMKMXlqOEhXNjJueW5rRmJYbzRWWEt2MmpDMFBNN2RQVmt5ODdGd2VaY1RLTG9XUVZQUUUycDJrTERLNk9Fc3ptTQp5eXIreHhXdHlpdmVyZW1yV3FuS2tOVFloTGZZUGhnUWtjemliN2VVYWxtRmpVYmhXZEx2SGFrYkVnQ29kbjNiCmt6NTdtSW5YMlZwaURPS2c0a3lIZml1WFdwaUJxckN4MEtOTHB4bzNERVFjRmNzUVRlVEh6aDQ3NTJHVjA0UlUKVGkvR0VXeXpJc2w0Umc3dEd0QXdtY0lQZ1VOVWZZMlEzOTBGR3FkSDRhaG4rbXcvNmFGYlczMVc2M2Q5WUpWcQppb3lPVmNhTUlwTTVCL2M3UWM4U3VoQ0kxWUdoVXlnNGNSSExFdzVWdGlraW95RTNYMDRrbmEzalFBajU0WWJSCmJwRWhjMzVhcEtMQjIxSE9VUUlEQVFBQm8xTXdVVEFkQmdOVkhRNEVGZ1FVeXZsMFZJNXZKVlN1WUZYdTdCNDgKNlBiTUVBb3dId1lEVlIwakJCZ3dGb0FVeXZsMFZJNXZKVlN1WUZYdTdCNDg2UGJNRUFvd0R3WURWUjBUQVFILwpCQVV3QXdFQi96QU5CZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFNTHhyZ0ZWTXVOUnEyd0F3Y0J0N1NuTlI1Q2Z6CjJNdlhxNUVVbXVhd0lVaTlrYVlqd2RWaURSRUdTams3SlcxN3ZsNTc2SGpEa2RmUndpNEUyOFN5ZFJJblpmNkoKaThIWmNaN2NhSDZEeFIzMzVmZ0hWekxpNU5pVGNlL09qTkJRelEyTUpYVkRkOERCbUc1ZnlhdEppT0pRNGJXRQpBN0ZsUDBSZFAzQ08zR1dFME01aVhPQjJtMXFXa0UyZXlPNFVIdndUcU5RTGRyZEFYZ0RRbGJhbTllNEJHM0dnCmQvNnRoQWtXRGJ0L1FOVCtFSkhEQ3ZoRFJLaDFSdUdIeWcrWSsvbmViVFdXckZXc2t0UnJiT29IQ1ppQ3BYSTEKM2VYRTZudDBZa2d0RHhHMjJLcW5ocEFnOWdVU3MyaGxob3h5dmt6eUYwbXU2TmhQbHdBZ25xNysvUT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K +backendTLSPolicies: + - apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: BackendTLSPolicy + metadata: + name: policy-btls + namespace: backends + spec: + targetRefs: + - group: "" + kind: Service + name: http-backend + sectionName: "8080" + validation: + caCertificateRefs: + - name: ca-secret + group: "" + kind: Secret + hostname: example.com \ No newline at end of file diff --git a/internal/gatewayapi/testdata/backendtlspolicy-ca-only-secret.out.yaml b/internal/gatewayapi/testdata/backendtlspolicy-ca-only-secret.out.yaml new file mode 100644 index 00000000000..b817977e727 --- /dev/null +++ b/internal/gatewayapi/testdata/backendtlspolicy-ca-only-secret.out.yaml @@ -0,0 +1,165 @@ +backendTLSPolicies: +- apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: BackendTLSPolicy + metadata: + creationTimestamp: null + name: policy-btls + namespace: backends + spec: + targetRefs: + - group: "" + kind: Service + name: http-backend + sectionName: "8080" + validation: + caCertificateRefs: + - group: "" + kind: Secret + name: ca-secret + hostname: example.com + status: + ancestors: + - ancestorRef: + name: gateway-btls + namespace: envoy-gateway + sectionName: http + conditions: + - lastTransitionTime: null + message: Policy has been accepted. + reason: Accepted + status: "True" + type: Accepted + controllerName: gateway.envoyproxy.io/gatewayclass-controller +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + creationTimestamp: null + name: gateway-btls + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: All + name: http + port: 80 + protocol: HTTP + status: + listeners: + - attachedRoutes: 1 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + creationTimestamp: null + name: httproute-btls + namespace: envoy-gateway + spec: + parentRefs: + - name: gateway-btls + namespace: envoy-gateway + sectionName: http + rules: + - backendRefs: + - name: http-backend + namespace: backends + port: 8080 + matches: + - path: + type: Exact + value: /exact + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Resolved all the Object references for the Route + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-btls + namespace: envoy-gateway + sectionName: http +infraIR: + envoy-gateway/gateway-btls: + proxy: + listeners: + - address: null + name: envoy-gateway/gateway-btls/http + ports: + - containerPort: 10080 + name: http-80 + protocol: HTTP + servicePort: 80 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-btls + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway/gateway-btls +xdsIR: + envoy-gateway/gateway-btls: + accessLog: + text: + - path: /dev/stdout + http: + - address: 0.0.0.0 + hostnames: + - '*' + isHTTP2: false + name: envoy-gateway/gateway-btls/http + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10080 + routes: + - destination: + name: httproute/envoy-gateway/httproute-btls/rule/0 + settings: + - addressType: IP + endpoints: + - host: 10.244.0.11 + port: 8080 + protocol: HTTP + tls: + caCertificate: + certificate: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURKekNDQWcrZ0F3SUJBZ0lVQWw2VUtJdUttenRlODFjbGx6NVBmZE4ySWxJd0RRWUpLb1pJaHZjTkFRRUwKQlFBd0l6RVFNQTRHQTFVRUF3d0hiWGxqYVdWdWRERVBNQTBHQTFVRUNnd0dhM1ZpWldSaU1CNFhEVEl6TVRBdwpNakExTkRFMU4xb1hEVEkwTVRBd01UQTFOREUxTjFvd0l6RVFNQTRHQTFVRUF3d0hiWGxqYVdWdWRERVBNQTBHCkExVUVDZ3dHYTNWaVpXUmlNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQXdTVGMKMXlqOEhXNjJueW5rRmJYbzRWWEt2MmpDMFBNN2RQVmt5ODdGd2VaY1RLTG9XUVZQUUUycDJrTERLNk9Fc3ptTQp5eXIreHhXdHlpdmVyZW1yV3FuS2tOVFloTGZZUGhnUWtjemliN2VVYWxtRmpVYmhXZEx2SGFrYkVnQ29kbjNiCmt6NTdtSW5YMlZwaURPS2c0a3lIZml1WFdwaUJxckN4MEtOTHB4bzNERVFjRmNzUVRlVEh6aDQ3NTJHVjA0UlUKVGkvR0VXeXpJc2w0Umc3dEd0QXdtY0lQZ1VOVWZZMlEzOTBGR3FkSDRhaG4rbXcvNmFGYlczMVc2M2Q5WUpWcQppb3lPVmNhTUlwTTVCL2M3UWM4U3VoQ0kxWUdoVXlnNGNSSExFdzVWdGlraW95RTNYMDRrbmEzalFBajU0WWJSCmJwRWhjMzVhcEtMQjIxSE9VUUlEQVFBQm8xTXdVVEFkQmdOVkhRNEVGZ1FVeXZsMFZJNXZKVlN1WUZYdTdCNDgKNlBiTUVBb3dId1lEVlIwakJCZ3dGb0FVeXZsMFZJNXZKVlN1WUZYdTdCNDg2UGJNRUFvd0R3WURWUjBUQVFILwpCQVV3QXdFQi96QU5CZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFNTHhyZ0ZWTXVOUnEyd0F3Y0J0N1NuTlI1Q2Z6CjJNdlhxNUVVbXVhd0lVaTlrYVlqd2RWaURSRUdTams3SlcxN3ZsNTc2SGpEa2RmUndpNEUyOFN5ZFJJblpmNkoKaThIWmNaN2NhSDZEeFIzMzVmZ0hWekxpNU5pVGNlL09qTkJRelEyTUpYVkRkOERCbUc1ZnlhdEppT0pRNGJXRQpBN0ZsUDBSZFAzQ08zR1dFME01aVhPQjJtMXFXa0UyZXlPNFVIdndUcU5RTGRyZEFYZ0RRbGJhbTllNEJHM0dnCmQvNnRoQWtXRGJ0L1FOVCtFSkhEQ3ZoRFJLaDFSdUdIeWcrWSsvbmViVFdXckZXc2t0UnJiT29IQ1ppQ3BYSTEKM2VYRTZudDBZa2d0RHhHMjJLcW5ocEFnOWdVU3MyaGxob3h5dmt6eUYwbXU2TmhQbHdBZ25xNysvUT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + name: policy-btls/backends-ca + sni: example.com + weight: 1 + hostname: '*' + isHTTP2: false + name: httproute/envoy-gateway/httproute-btls/rule/0/match/0/* + pathMatch: + distinct: false + exact: /exact + name: "" diff --git a/internal/provider/kubernetes/controller.go b/internal/provider/kubernetes/controller.go index a11a54391ae..829010b2603 100644 --- a/internal/provider/kubernetes/controller.go +++ b/internal/provider/kubernetes/controller.go @@ -897,7 +897,7 @@ func (r *gatewayAPIReconciler) processBackendTLSPolicies( } // Add the referenced Secrets and ConfigMaps in BackendTLSPolicies to the resourceTree. - r.processBackendTLSPolicyConfigMapRefs(ctx, resourceTree, resourceMap) + r.processBackendTLSPolicyRefs(ctx, resourceTree, resourceMap) return nil } @@ -1575,27 +1575,49 @@ func (r *gatewayAPIReconciler) serviceImportCRDExists(mgr manager.Manager) bool return serviceImportFound } -func (r *gatewayAPIReconciler) processBackendTLSPolicyConfigMapRefs(ctx context.Context, resourceTree *gatewayapi.Resources, resourceMap *resourceMappings) { +func (r *gatewayAPIReconciler) processBackendTLSPolicyRefs( + ctx context.Context, + resourceTree *gatewayapi.Resources, + resourceMap *resourceMappings, +) { for _, policy := range resourceTree.BackendTLSPolicies { tls := policy.Spec.Validation if tls.CACertificateRefs != nil { for _, caCertRef := range tls.CACertificateRefs { - if string(caCertRef.Kind) == gatewayapi.KindConfigMap { + // if kind is not Secret or ConfigMap, we skip early to avoid further calculation overhead + if string(caCertRef.Kind) == gatewayapi.KindConfigMap || + string(caCertRef.Kind) == gatewayapi.KindSecret { + + var err error caRefNew := gwapiv1b1.SecretObjectReference{ Group: gatewayapi.GroupPtr(string(caCertRef.Group)), Kind: gatewayapi.KindPtr(string(caCertRef.Kind)), Name: caCertRef.Name, Namespace: gatewayapi.NamespacePtr(policy.Namespace), } - if err := r.processConfigMapRef( - ctx, - resourceMap, - resourceTree, - gatewayapi.KindBackendTLSPolicy, - policy.Namespace, - policy.Name, - caRefNew); err != nil { + switch string(caCertRef.Kind) { + case gatewayapi.KindConfigMap: + err = r.processConfigMapRef( + ctx, + resourceMap, + resourceTree, + gatewayapi.KindBackendTLSPolicy, + policy.Namespace, + policy.Name, + caRefNew) + + case gatewayapi.KindSecret: + err = r.processSecretRef( + ctx, + resourceMap, + resourceTree, + gatewayapi.KindBackendTLSPolicy, + policy.Namespace, + policy.Name, + caRefNew) + } + if err != nil { // we don't return an error here, because we want to continue // reconciling the rest of the ClientTrafficPolicies despite that this // reference is invalid. From 9fd9a084938b077015e4219f88cc60991811ea9a Mon Sep 17 00:00:00 2001 From: ShyunnY <1147212064@qq.com> Date: Thu, 23 May 2024 21:09:39 +0800 Subject: [PATCH 2/2] fix Signed-off-by: ShyunnY <1147212064@qq.com> --- .../gatewayapi/testdata/backendtlspolicy-ca-only-secret.in.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/gatewayapi/testdata/backendtlspolicy-ca-only-secret.in.yaml b/internal/gatewayapi/testdata/backendtlspolicy-ca-only-secret.in.yaml index 53d85667b30..b701ad9800f 100644 --- a/internal/gatewayapi/testdata/backendtlspolicy-ca-only-secret.in.yaml +++ b/internal/gatewayapi/testdata/backendtlspolicy-ca-only-secret.in.yaml @@ -114,4 +114,4 @@ backendTLSPolicies: - name: ca-secret group: "" kind: Secret - hostname: example.com \ No newline at end of file + hostname: example.com