diff --git a/config/samples/gateway-with-custom-listeners.yaml b/config/samples/gateway-with-multiple-listeners.yaml similarity index 75% rename from config/samples/gateway-with-custom-listeners.yaml rename to config/samples/gateway-with-multiple-listeners.yaml index fdf607a7..7646c8ad 100644 --- a/config/samples/gateway-with-custom-listeners.yaml +++ b/config/samples/gateway-with-multiple-listeners.yaml @@ -4,11 +4,6 @@ metadata: name: kong spec: controllerName: konghq.com/gateway-operator - parametersRef: - group: gateway-operator.konghq.com - kind: GatewayConfiguration - name: kong - namespace: default --- kind: Gateway apiVersion: gateway.networking.k8s.io/v1 @@ -24,3 +19,8 @@ spec: - name: http2 protocol: HTTP port: 8089 + - name: tls + protocol: TLS + port: 9443 + tls: + mode: Passthrough diff --git a/controller/controlplane/controller_utils_test.go b/controller/controlplane/controller_utils_test.go index 04e4015b..9bd03040 100644 --- a/controller/controlplane/controller_utils_test.go +++ b/controller/controlplane/controller_utils_test.go @@ -434,6 +434,14 @@ func TestSetControlPlaneDefaults(t *testing.T) { Name: "CONTROLLER_ADMISSION_WEBHOOK_LISTEN", Value: consts.ControlPlaneAdmissionWebhookEnvVarValue, }, + { + Name: "", + Value: consts.ControlPlaneAdmissionWebhookEnvVarValue, + }, + { + Name: "CONTROLLER_FEATURE_GATES", + Value: "GatewayAlpha=true", + }, }, }, }, @@ -511,6 +519,10 @@ func TestSetControlPlaneDefaults(t *testing.T) { Name: "CONTROLLER_ADMISSION_WEBHOOK_LISTEN", Value: consts.ControlPlaneAdmissionWebhookEnvVarValue, }, + { + Name: "CONTROLLER_FEATURE_GATES", + Value: "GatewayAlpha=true", + }, }, }, }, diff --git a/controller/gateway/controller_conditions.go b/controller/gateway/controller_conditions.go index 6fa8a351..7206b05d 100644 --- a/controller/gateway/controller_conditions.go +++ b/controller/gateway/controller_conditions.go @@ -32,4 +32,10 @@ const ( // to express that more than one TLS secret has been set in the listener // TLS configuration. ListenerReasonTooManyTLSSecrets k8sutils.ConditionReason = "TooManyTLSSecrets" + + // ListenereReasonInvalidTLSMode must be used with the Accepted condition + // to express that the listener has an invalid TLS mode. + // HTTPS can only be configured with mode Terminate, while TLS can only be + // be configured with mode Passthrough. + ListenereReasonInvalidTLSMode k8sutils.ConditionReason = "InvalidTLSMode" ) diff --git a/controller/gateway/controller_reconciler_utils.go b/controller/gateway/controller_reconciler_utils.go index 6995691f..f098c8a3 100644 --- a/controller/gateway/controller_reconciler_utils.go +++ b/controller/gateway/controller_reconciler_utils.go @@ -299,8 +299,8 @@ func generateDataPlaneNetworkPolicy( var ( protocolTCP = corev1.ProtocolTCP adminAPISSLPort = intstr.FromInt(consts.DataPlaneAdminAPIPort) - proxyPort = intstr.FromInt(consts.DataPlaneProxyPort) - proxySSLPort = intstr.FromInt(consts.DataPlaneProxySSLPort) + proxyPort = intstr.FromInt(consts.DataPlaneProxyHTTPPort) + proxySSLPort = intstr.FromInt(consts.DataPlaneProxyHTTPSPort) metricsPort = intstr.FromInt(consts.DataPlaneMetricsPort) ) @@ -527,9 +527,9 @@ func supportedRoutesByProtocol() map[gatewayv1.ProtocolType]map[gatewayv1.Kind]s return map[gatewayv1.ProtocolType]map[gatewayv1.Kind]struct{}{ gatewayv1.HTTPProtocolType: {"HTTPRoute": {}}, gatewayv1.HTTPSProtocolType: {"HTTPRoute": {}}, + gatewayv1.TLSProtocolType: {"TLSRoute": {}}, - // L4 routes not supported yet - // gatewayv1.TLSProtocolType: {"TLSRoute": {}}, + // TCP and UDP routes not supported yet // gatewayv1.TCPProtocolType: {"TCPRoute": {}}, // gatewayv1.UDPProtocolType: {"UDPRoute": {}}, } @@ -590,10 +590,25 @@ func (g *gatewayConditionsAndListenersAwareT) setAccepted() { LastTransitionTime: metav1.Now(), ObservedGeneration: g.Generation, } - if listener.Protocol != gatewayv1.HTTPProtocolType && listener.Protocol != gatewayv1.HTTPSProtocolType { + if listener.Protocol != gatewayv1.HTTPProtocolType && + listener.Protocol != gatewayv1.HTTPSProtocolType && + listener.Protocol != gatewayv1.TLSProtocolType { acceptedCondition.Status = metav1.ConditionFalse acceptedCondition.Reason = string(gatewayv1.ListenerReasonUnsupportedProtocol) } + // Only TLS terminate mode supported with HTTPS Listeners. + if listener.Protocol == gatewayv1.HTTPSProtocolType && *listener.TLS.Mode != gatewayv1.TLSModeTerminate { + acceptedCondition.Status = metav1.ConditionFalse + acceptedCondition.Reason = string(ListenereReasonInvalidTLSMode) + acceptedCondition.Message = "Only Terminate mode is supported with HTTPS listeners" + } + + // Only TLS passthrough mode supported with TLS Listeners. + if listener.Protocol == gatewayv1.TLSProtocolType && *listener.TLS.Mode != gatewayv1.TLSModePassthrough { + acceptedCondition.Status = metav1.ConditionFalse + acceptedCondition.Reason = string(ListenereReasonInvalidTLSMode) + acceptedCondition.Message = "Only Passthrough mode is supported with TLS listeners" + } listenerConditionsAware := listenerConditionsAware(&g.Status.Listeners[i]) listenerConditionsAware.SetConditions(append(listenerConditionsAware.Conditions, acceptedCondition)) } @@ -685,9 +700,11 @@ func setDataPlaneIngressServicePorts(opts *operatorv1beta1.DataPlaneOptions, lis } switch l.Protocol { case gatewayv1.HTTPSProtocolType: - port.TargetPort = intstr.FromInt(consts.DataPlaneProxySSLPort) + port.TargetPort = intstr.FromInt(consts.DataPlaneProxyHTTPSPort) case gatewayv1.HTTPProtocolType: - port.TargetPort = intstr.FromInt(consts.DataPlaneProxyPort) + port.TargetPort = intstr.FromInt(consts.DataPlaneProxyHTTPPort) + case gatewayv1.TLSProtocolType: + port.TargetPort = intstr.FromInt(consts.DataPlaneProxyTLSPort) default: errs = errors.Join(errs, fmt.Errorf("listener %d uses unsupported protocol %s", i, l.Protocol)) continue @@ -712,67 +729,66 @@ func getSupportedKindsWithResolvedRefsCondition(ctx context.Context, c client.Cl message := "" if listener.TLS != nil { - // We currently do not support TLSRoutes, hence only TLS termination supported. - if *listener.TLS.Mode != gatewayv1.TLSModeTerminate { - resolvedRefsCondition.Status = metav1.ConditionFalse - resolvedRefsCondition.Reason = string(gatewayv1.ListenerReasonInvalidCertificateRef) - message = conditionMessage(message, "Only Terminate mode is supported") - } // We currently do not support more that one listener certificate. - if len(listener.TLS.CertificateRefs) != 1 { + if len(listener.TLS.CertificateRefs) > 1 { resolvedRefsCondition.Reason = string(ListenerReasonTooManyTLSSecrets) message = conditionMessage(message, "Only one certificate per listener is supported") } else { - isValidGroupKind := true - certificateRef := listener.TLS.CertificateRefs[0] - if certificateRef.Group != nil && *certificateRef.Group != "" && *certificateRef.Group != gatewayv1.Group(corev1.SchemeGroupVersion.Group) { - resolvedRefsCondition.Reason = string(gatewayv1.ListenerReasonInvalidCertificateRef) - message = conditionMessage(message, fmt.Sprintf("Group %s not supported in CertificateRef", *certificateRef.Group)) - isValidGroupKind = false - } - if certificateRef.Kind != nil && *certificateRef.Kind != "" && *certificateRef.Kind != gatewayv1.Kind("Secret") { - resolvedRefsCondition.Reason = string(gatewayv1.ListenerReasonInvalidCertificateRef) - message = conditionMessage(message, fmt.Sprintf("Kind %s not supported in CertificateRef", *certificateRef.Kind)) - isValidGroupKind = false - } - secretNamespace := gatewayNamespace - if certificateRef.Namespace != nil && *certificateRef.Namespace != "" { - secretNamespace = string(*certificateRef.Namespace) - } - - var secretExists bool - if isValidGroupKind { - // Get the secret and check it exists. - certificateSecret := &corev1.Secret{} - err = c.Get(ctx, types.NamespacedName{ - Namespace: secretNamespace, - Name: string(certificateRef.Name), - }, certificateSecret) - if err != nil { - if !k8serrors.IsNotFound(err) { - return - } + // check certificate references only when Terminate mode is used. + // Passthrough mode does not need a certificate. + if len(listener.TLS.CertificateRefs) != 0 { + isValidGroupKind := true + certificateRef := listener.TLS.CertificateRefs[0] + if certificateRef.Group != nil && *certificateRef.Group != "" && *certificateRef.Group != gatewayv1.Group(corev1.SchemeGroupVersion.Group) { resolvedRefsCondition.Reason = string(gatewayv1.ListenerReasonInvalidCertificateRef) - message = conditionMessage(message, fmt.Sprintf("Referenced secret %s/%s does not exist", secretNamespace, certificateRef.Name)) - } else { - secretExists = true + message = conditionMessage(message, fmt.Sprintf("Group %s not supported in CertificateRef", *certificateRef.Group)) + isValidGroupKind = false + } + if certificateRef.Kind != nil && *certificateRef.Kind != "" && *certificateRef.Kind != gatewayv1.Kind("Secret") { + resolvedRefsCondition.Reason = string(gatewayv1.ListenerReasonInvalidCertificateRef) + message = conditionMessage(message, fmt.Sprintf("Kind %s not supported in CertificateRef", *certificateRef.Kind)) + isValidGroupKind = false + } + secretNamespace := gatewayNamespace + if certificateRef.Namespace != nil && *certificateRef.Namespace != "" { + secretNamespace = string(*certificateRef.Namespace) } - } - if secretExists { - // In case there is a cross-namespace reference, check if there is any referenceGrant allowing it. - if secretNamespace != gatewayNamespace { - referenceGrantList := &gatewayv1beta1.ReferenceGrantList{} - err = c.List(ctx, referenceGrantList, client.InNamespace(secretNamespace)) + var secretExists bool + if isValidGroupKind { + // Get the secret and check it exists. + certificateSecret := &corev1.Secret{} + err = c.Get(ctx, types.NamespacedName{ + Namespace: secretNamespace, + Name: string(certificateRef.Name), + }, certificateSecret) if err != nil { - return + if !k8serrors.IsNotFound(err) { + return + } + resolvedRefsCondition.Reason = string(gatewayv1.ListenerReasonInvalidCertificateRef) + message = conditionMessage(message, fmt.Sprintf("Referenced secret %s/%s does not exist", secretNamespace, certificateRef.Name)) + } else { + secretExists = true } - if !isSecretCrossReferenceGranted(gatewayv1.Namespace(gatewayNamespace), certificateRef.Name, referenceGrantList.Items) { - resolvedRefsCondition.Reason = string(gatewayv1.ListenerReasonRefNotPermitted) - message = conditionMessage(message, fmt.Sprintf("Secret %s/%s reference not allowed by any referenceGrant", secretNamespace, certificateRef.Name)) + } + + if secretExists { + // In case there is a cross-namespace reference, check if there is any referenceGrant allowing it. + if secretNamespace != gatewayNamespace { + referenceGrantList := &gatewayv1beta1.ReferenceGrantList{} + err = c.List(ctx, referenceGrantList, client.InNamespace(secretNamespace)) + if err != nil { + return + } + if !isSecretCrossReferenceGranted(gatewayv1.Namespace(gatewayNamespace), certificateRef.Name, referenceGrantList.Items) { + resolvedRefsCondition.Reason = string(gatewayv1.ListenerReasonRefNotPermitted) + message = conditionMessage(message, fmt.Sprintf("Secret %s/%s reference not allowed by any referenceGrant", secretNamespace, certificateRef.Name)) + } } } } + } } diff --git a/controller/gateway/controller_reconciler_utils_test.go b/controller/gateway/controller_reconciler_utils_test.go index b3a4825a..f243b592 100644 --- a/controller/gateway/controller_reconciler_utils_test.go +++ b/controller/gateway/controller_reconciler_utils_test.go @@ -553,12 +553,12 @@ func TestSetDataPlaneIngressServicePorts(t *testing.T) { { Name: "http", Port: 80, - TargetPort: intstr.FromInt(consts.DataPlaneProxyPort), + TargetPort: intstr.FromInt(consts.DataPlaneProxyHTTPPort), }, { Name: "https", Port: 443, - TargetPort: intstr.FromInt(consts.DataPlaneProxySSLPort), + TargetPort: intstr.FromInt(consts.DataPlaneProxyHTTPSPort), }, }, }, @@ -580,7 +580,7 @@ func TestSetDataPlaneIngressServicePorts(t *testing.T) { { Name: "http", Port: 80, - TargetPort: intstr.FromInt(consts.DataPlaneProxyPort), + TargetPort: intstr.FromInt(consts.DataPlaneProxyHTTPPort), }, }, expectedError: errors.New("listener 1 uses unsupported protocol UDP"), @@ -979,42 +979,6 @@ func TestGetSupportedKindsWithResolvedRefsCondition(t *testing.T) { ObservedGeneration: generation, }, }, - { - name: "tls with passthrough, HTTPS protocol, no allowed routes", - gatewayNamespace: "default", - listener: gatewayv1.Listener{ - Protocol: gatewayv1.HTTPSProtocolType, - TLS: &gatewayv1.GatewayTLSConfig{ - Mode: lo.ToPtr(gatewayv1.TLSModePassthrough), - CertificateRefs: []gatewayv1.SecretObjectReference{ - { - Name: "test-secret", - }, - }, - }, - }, - secrets: []client.Object{ - &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-secret", - Namespace: "default", - }, - }, - }, - expectedSupportedKinds: []gatewayv1.RouteGroupKind{ - { - Group: (*gatewayv1.Group)(&gatewayv1.GroupVersion.Group), - Kind: "HTTPRoute", - }, - }, - expectedResolvedRefsCondition: metav1.Condition{ - Type: string(gatewayv1.ListenerConditionResolvedRefs), - Status: metav1.ConditionFalse, - Reason: string(gatewayv1.ListenerReasonInvalidCertificateRef), - Message: "Only Terminate mode is supported.", - ObservedGeneration: generation, - }, - }, { name: "tls bad-formed, multiple TLS secrets no cross-namespace reference", gatewayNamespace: "default", diff --git a/controller/pkg/controlplane/controlplane.go b/controller/pkg/controlplane/controlplane.go index a9db4feb..1ab326fa 100644 --- a/controller/pkg/controlplane/controlplane.go +++ b/controller/pkg/controlplane/controlplane.go @@ -92,6 +92,16 @@ func SetDefaults( changed = true } + // If the feature gates are not set, set them to the default value. + const ( + featureGates = "CONTROLLER_FEATURE_GATES" + defaultFeatureGates = "GatewayAlpha=true" + ) + if k8sutils.EnvValueByName(container.Env, featureGates) == "" { + container.Env = k8sutils.UpdateEnv(container.Env, featureGates, defaultFeatureGates) + changed = true + } + if args.Namespace != "" && args.DataPlaneIngressServiceName != "" { if _, isOverrideDisabled := dontOverride["CONTROLLER_PUBLISH_SERVICE"]; !isOverrideDisabled { publishServiceNN := k8stypes.NamespacedName{Namespace: args.Namespace, Name: args.DataPlaneIngressServiceName}.String() diff --git a/internal/utils/dataplane/config.go b/internal/utils/dataplane/config.go index 90392d49..d785af23 100644 --- a/internal/utils/dataplane/config.go +++ b/internal/utils/dataplane/config.go @@ -26,8 +26,9 @@ var KongDefaults = map[string]string{ "KONG_PORT_MAPS": "80:8000, 443:8443", "KONG_PROXY_ACCESS_LOG": "/dev/stdout", "KONG_PROXY_ERROR_LOG": "/dev/stderr", - "KONG_PROXY_LISTEN": fmt.Sprintf("0.0.0.0:%d reuseport backlog=16384, 0.0.0.0:%d http2 ssl reuseport backlog=16384", consts.DataPlaneProxyPort, consts.DataPlaneProxySSLPort), + "KONG_PROXY_LISTEN": fmt.Sprintf("0.0.0.0:%d reuseport backlog=16384, 0.0.0.0:%d http2 ssl reuseport backlog=16384", consts.DataPlaneProxyHTTPPort, consts.DataPlaneProxyHTTPSPort), "KONG_STATUS_LISTEN": fmt.Sprintf("0.0.0.0:%d", consts.DataPlaneStatusPort), + "KONG_STREAM_LISTEN": fmt.Sprintf("0.0.0.0:%d ssl", consts.DataPlaneProxyTLSPort), // TODO: reconfigure following https://github.com/Kong/gateway-operator/issues/7 "KONG_ADMIN_LISTEN": fmt.Sprintf("0.0.0.0:%d ssl reuseport backlog=16384", consts.DataPlaneAdminAPIPort), diff --git a/pkg/consts/dataplane.go b/pkg/consts/dataplane.go index a420608b..af882412 100644 --- a/pkg/consts/dataplane.go +++ b/pkg/consts/dataplane.go @@ -135,10 +135,13 @@ const ( DataPlaneAdminAPIPort = 8444 // DataPlaneHTTPSPort is the port that the dataplane uses for HTTP. - DataPlaneProxyPort = 8000 + DataPlaneProxyHTTPPort = 8000 // DataPlaneHTTPSPort is the port that the dataplane uses for HTTPS. - DataPlaneProxySSLPort = 8443 + DataPlaneProxyHTTPSPort = 8443 + + // DataPlaneProxyTLSPort is the port that the dataplane uses for TLS. + DataPlaneProxyTLSPort = 9443 // DataPlaneMetricsPort is the port that the dataplane uses for metrics. DataPlaneMetricsPort = 8100 diff --git a/pkg/utils/kubernetes/resources/deployments.go b/pkg/utils/kubernetes/resources/deployments.go index 79d14647..9aa6acf9 100644 --- a/pkg/utils/kubernetes/resources/deployments.go +++ b/pkg/utils/kubernetes/resources/deployments.go @@ -295,12 +295,17 @@ func GenerateDataPlaneContainer(image string) corev1.Container { Ports: []corev1.ContainerPort{ { Name: "proxy", - ContainerPort: consts.DataPlaneProxyPort, + ContainerPort: consts.DataPlaneProxyHTTPPort, Protocol: corev1.ProtocolTCP, }, { Name: "proxy-ssl", - ContainerPort: consts.DataPlaneProxySSLPort, + ContainerPort: consts.DataPlaneProxyHTTPSPort, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "proxy-tls", + ContainerPort: consts.DataPlaneProxyTLSPort, Protocol: corev1.ProtocolTCP, }, { diff --git a/pkg/utils/kubernetes/resources/services.go b/pkg/utils/kubernetes/resources/services.go index b8ffecc3..232876e9 100644 --- a/pkg/utils/kubernetes/resources/services.go +++ b/pkg/utils/kubernetes/resources/services.go @@ -91,13 +91,13 @@ var DefaultDataPlaneIngressServicePorts = []corev1.ServicePort{ Name: "http", Protocol: corev1.ProtocolTCP, Port: consts.DefaultHTTPPort, - TargetPort: intstr.FromInt(consts.DataPlaneProxyPort), + TargetPort: intstr.FromInt(consts.DataPlaneProxyHTTPPort), }, { Name: "https", Protocol: corev1.ProtocolTCP, Port: consts.DefaultHTTPSPort, - TargetPort: intstr.FromInt(consts.DataPlaneProxySSLPort), + TargetPort: intstr.FromInt(consts.DataPlaneProxyHTTPSPort), }, } @@ -145,7 +145,7 @@ func ServicePortsFromDataPlaneIngressOpt(dataplane *operatorv1beta1.DataPlane) S newPorts := make([]corev1.ServicePort, 0) alreadyUsedPorts := make(map[int32]struct{}) for _, p := range dataplane.Spec.Network.Services.Ingress.Ports { - targetPort := intstr.FromInt(consts.DataPlaneProxyPort) + targetPort := intstr.FromInt(consts.DataPlaneProxyHTTPPort) if !cmp.Equal(p.TargetPort, intstr.IntOrString{}) { targetPort = p.TargetPort } diff --git a/pkg/utils/test/consts.go b/pkg/utils/test/consts.go index adc67db3..6c362631 100644 --- a/pkg/utils/test/consts.go +++ b/pkg/utils/test/consts.go @@ -69,7 +69,28 @@ const ( // Running on Pod tcp-echo-58ccd6b78d-hn9t8. // In namespace foo. // With IP address 10.244.0.13. - TCPEchoImage = "kong/go-echo:0.1.0" + TCPEchoImage = "kong/go-echo:0.4.0" + + // HTTPBinPort is the port that the HTTPBin container listens on. + HTTPBinPort int32 = 80 + + // TCPEchoTLSPort is the port that the TCPEcho container listens on for TLS traffic. + TCPEchoTLSPort int32 = 1030 +) + +// ----------------------------------------------------------------------------- +// Global Gateway API Testing Vars & Consts +// ----------------------------------------------------------------------------- + +const ( + // DefaultHTTPListenerPort is the default port for HTTP listeners. + DefaultHTTPListenerPort int32 = 80 + + // DefaultHTTPSListenerPort is the default port for HTTPS listeners. + DefaultHTTPSListenerPort int32 = 443 + + // DefaultTLSListenerPort is the default port for TLS listeners. + DefaultTLSListenerPort int32 = 9443 ) // ----------------------------------------------------------------------------- diff --git a/test/helpers/generators.go b/test/helpers/generators.go index 0827cb81..109e8b98 100644 --- a/test/helpers/generators.go +++ b/test/helpers/generators.go @@ -1,6 +1,7 @@ package helpers import ( + "fmt" "testing" "github.com/google/uuid" @@ -11,60 +12,54 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" operatorv1beta1 "github.com/kong/gateway-operator/api/v1beta1" gwtypes "github.com/kong/gateway-operator/internal/types" "github.com/kong/gateway-operator/pkg/consts" "github.com/kong/gateway-operator/pkg/vars" + "github.com/kong/kubernetes-testing-framework/pkg/utils/kubernetes/generators" ) -// MustGenerateGatewayClass generates the default GatewayClass to be used in tests -func MustGenerateGatewayClass(t *testing.T, parametersRefs ...gatewayv1.ParametersReference) *gatewayv1.GatewayClass { - t.Helper() +// ----------------------------------------------------------------------------- +// Kubernetes resources generators +// ----------------------------------------------------------------------------- - if len(parametersRefs) > 1 { - require.Fail(t, "only one ParametersReference is allowed") - } - var parametersRef *gatewayv1.ParametersReference - if len(parametersRefs) == 1 { - parametersRef = ¶metersRefs[0] - } - gatewayClass := &gatewayv1.GatewayClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: uuid.NewString(), +// This is copy-pasted from the test package in the kong/kubernetes-ingress-controller repo +// https://github.com/Kong/kubernetes-ingress-controller/blob/main/test/integration/tlsroute_test.go#L782 +func GenerateTLSEchoContainer(tpcEchoImage string, tlsEchoPort int32, sendMsg string, tlsSecretName string) corev1.Container { //nolint:unparam + container := generators.NewContainer("tcpecho-"+sendMsg, tpcEchoImage, tlsEchoPort) + const tlsCertDir = "/var/run/certs" + container.Env = append(container.Env, + corev1.EnvVar{ + Name: "POD_NAME", + Value: sendMsg, }, - Spec: gatewayv1.GatewayClassSpec{ - ControllerName: gatewayv1.GatewayController(vars.ControllerName()), - ParametersRef: parametersRef, + corev1.EnvVar{ + Name: "TLS_PORT", + Value: fmt.Sprint(tlsEchoPort), }, - } - return gatewayClass -} - -// GenerateGateway generates a Gateway to be used in tests -func GenerateGateway(gatewayNSN types.NamespacedName, gatewayClass *gatewayv1.GatewayClass, opts ...func(gateway *gatewayv1.Gateway)) *gwtypes.Gateway { - gateway := &gwtypes.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: gatewayNSN.Namespace, - Name: gatewayNSN.Name, + corev1.EnvVar{ + Name: "TLS_CERT_FILE", + Value: tlsCertDir + "/tls.crt", }, - Spec: gatewayv1.GatewaySpec{ - GatewayClassName: gatewayv1.ObjectName(gatewayClass.Name), - Listeners: []gatewayv1.Listener{{ - Name: "http", - Protocol: gatewayv1.HTTPProtocolType, - Port: gatewayv1.PortNumber(80), - }}, + corev1.EnvVar{ + Name: "TLS_KEY_FILE", + Value: tlsCertDir + "/tls.key", }, - } - - for _, opt := range opts { - opt(gateway) - } - - return gateway + ) + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ + Name: tlsSecretName, + ReadOnly: true, + MountPath: tlsCertDir, + }) + return container } +// ----------------------------------------------------------------------------- +// Gateway Operator generators +// ----------------------------------------------------------------------------- + // GenerateGatewayConfiguration generates a GatewayConfiguration to be used in tests func GenerateGatewayConfiguration(namespace string) *operatorv1beta1.GatewayConfiguration { return &operatorv1beta1.GatewayConfiguration{ @@ -130,6 +125,12 @@ func GenerateGatewayConfiguration(namespace string) *operatorv1beta1.GatewayConf }, }, }, + Env: []corev1.EnvVar{ + { + Name: "KONG_NGINX_ADMIN_SSL_VERIFY_CLIENT", + Value: "off", + }, + }, }, }, }, @@ -141,8 +142,59 @@ func GenerateGatewayConfiguration(namespace string) *operatorv1beta1.GatewayConf } } +// ----------------------------------------------------------------------------- +// Gateway API generators +// ----------------------------------------------------------------------------- + +// MustGenerateGatewayClass generates the default GatewayClass to be used in tests +func MustGenerateGatewayClass(t *testing.T, parametersRefs ...gatewayv1.ParametersReference) *gatewayv1.GatewayClass { + t.Helper() + + if len(parametersRefs) > 1 { + require.Fail(t, "only one ParametersReference is allowed") + } + var parametersRef *gatewayv1.ParametersReference + if len(parametersRefs) == 1 { + parametersRef = ¶metersRefs[0] + } + gatewayClass := &gatewayv1.GatewayClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: uuid.NewString(), + }, + Spec: gatewayv1.GatewayClassSpec{ + ControllerName: gatewayv1.GatewayController(vars.ControllerName()), + ParametersRef: parametersRef, + }, + } + return gatewayClass +} + +// GenerateGateway generates a Gateway to be used in tests +func GenerateGateway(gatewayNSN types.NamespacedName, gatewayClass *gatewayv1.GatewayClass, opts ...func(gateway *gatewayv1.Gateway)) *gwtypes.Gateway { + gateway := &gwtypes.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: gatewayNSN.Namespace, + Name: gatewayNSN.Name, + }, + Spec: gatewayv1.GatewaySpec{ + GatewayClassName: gatewayv1.ObjectName(gatewayClass.Name), + Listeners: []gatewayv1.Listener{{ + Name: "http", + Protocol: gatewayv1.HTTPProtocolType, + Port: gatewayv1.PortNumber(80), + }}, + }, + } + + for _, opt := range opts { + opt(gateway) + } + + return gateway +} + // GenerateHTTPRoute generates an HTTPRoute to be used in tests -func GenerateHTTPRoute(namespace string, gatewayName, serviceName string, opts ...func(*gatewayv1.HTTPRoute)) *gatewayv1.HTTPRoute { +func GenerateHTTPRoute(namespace string, gatewayName, serviceName string, servicePort int32, opts ...func(*gatewayv1.HTTPRoute)) *gatewayv1.HTTPRoute { httpRoute := &gatewayv1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, @@ -172,7 +224,7 @@ func GenerateHTTPRoute(namespace string, gatewayName, serviceName string, opts . BackendRef: gatewayv1.BackendRef{ BackendObjectReference: gatewayv1.BackendObjectReference{ Name: gatewayv1.ObjectName(serviceName), - Port: lo.ToPtr(gatewayv1.PortNumber(80)), + Port: lo.ToPtr(gatewayv1.PortNumber(servicePort)), Kind: lo.ToPtr(gatewayv1.Kind("Service")), }, }, @@ -189,3 +241,42 @@ func GenerateHTTPRoute(namespace string, gatewayName, serviceName string, opts . return httpRoute } + +// GenerateTLSRoute generates a TLSRoute to be used in tests +func GenerateTLSRoute(namespace string, gatewayName, serviceName string, servicePort int32, opts ...func(*gatewayv1alpha2.TLSRoute)) *gatewayv1alpha2.TLSRoute { + tlsRoute := &gatewayv1alpha2.TLSRoute{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: uuid.NewString(), + Annotations: map[string]string{ + "konghq.com/strip-path": "true", + }, + }, + Spec: gatewayv1alpha2.TLSRouteSpec{ + CommonRouteSpec: gatewayv1.CommonRouteSpec{ + ParentRefs: []gatewayv1.ParentReference{{ + Name: gatewayv1.ObjectName(gatewayName), + }}, + }, + Rules: []gatewayv1alpha2.TLSRouteRule{ + { + BackendRefs: []gatewayv1alpha2.BackendRef{ + { + BackendObjectReference: gatewayv1alpha2.BackendObjectReference{ + Name: gatewayv1alpha2.ObjectName(serviceName), + Port: lo.ToPtr(gatewayv1alpha2.PortNumber(servicePort)), + Kind: lo.ToPtr(gatewayv1alpha2.Kind("Service")), + }, + }, + }, + }, + }, + }, + } + + for _, opt := range opts { + opt(tlsRoute) + } + + return tlsRoute +} diff --git a/test/helpers/tcp.go b/test/helpers/tcp.go new file mode 100644 index 00000000..57d77fb8 --- /dev/null +++ b/test/helpers/tcp.go @@ -0,0 +1,95 @@ +package helpers + +import ( + "bytes" + "crypto/tls" + "fmt" + "net" + "time" + + corev1 "k8s.io/api/core/v1" +) + +// tlsEchoResponds takes a TLS address URL and a Pod name and checks if a go-echo +// instance is running on that Pod at that address using hostname for SNI. It sends +// a message and checks if returned one matches. It returns an error with +// an explanation if it is not (typical network related errors like io.EOF or +// syscall.ECONNRESET are returned directly). +// +// This is copy-paste from the test package in the kong/kubernetes-ingress-controller repo +// https://github.com/Kong/kubernetes-ingress-controller/tree/main/test/integration/tlsroute_test.go#L709 +func TLSEchoResponds( + url string, podName string, certHostname string, tlsSecret *corev1.Secret, passthrough bool, +) error { + tlsConfig, err := createTLSClientConfig(tlsSecret, certHostname) + if err != nil { + return err + } + tlsConfig.InsecureSkipVerify = true + dialer := net.Dialer{Timeout: time.Second * 10} + conn, err := tls.DialWithDialer(&dialer, + "tcp", + url, + tlsConfig, + ) + if err != nil { + return err + } + defer conn.Close() + + cert := conn.ConnectionState().PeerCertificates[0] + if cert.Subject.CommonName != certHostname { + return fmt.Errorf("expected certificate with cn=%s, got cn=%s", certHostname, cert.Subject.CommonName) + } + + header := []byte(fmt.Sprintf("Running on Pod %s.", podName)) + // if we are testing with passthrough, the go-echo service should return a message + // noting that it is listening in TLS mode. + if passthrough { + header = append(header, []byte("\nThrough TLS connection.")...) + } + message := []byte("testing tlsroute") + + wrote, err := conn.Write(message) + if err != nil { + return err + } + + if wrote != len(message) { + return fmt.Errorf("wrote message of size %d, expected %d", wrote, len(message)) + } + + if err := conn.SetDeadline(time.Now().Add(time.Second * 5)); err != nil { + return err + } + + headerResponse := make([]byte, len(header)+1) + read, err := conn.Read(headerResponse) + if err != nil { + return err + } + + if read != len(header)+1 { // add 1 for newline + return fmt.Errorf("read %d bytes but expected %d", read, len(header)+1) + } + + if !bytes.Contains(headerResponse, header) { + return fmt.Errorf(`expected header response "%s", received: "%s"`, string(header), string(headerResponse)) + } + + messageResponse := make([]byte, wrote+1) + read, err = conn.Read(messageResponse) + if err != nil { + return err + } + + if read != len(message) { + return fmt.Errorf("read %d bytes but expected %d", read, len(message)) + } + + if !bytes.Contains(messageResponse, message) { + return fmt.Errorf(`expected message response "%s", received: "%s"`, string(message), string(messageResponse)) + } + + return nil +} diff --git a/test/integration/run_integration_test.go b/test/integration/run_integration_test.go index 9813a027..52dc23dd 100644 --- a/test/integration/run_integration_test.go +++ b/test/integration/run_integration_test.go @@ -20,7 +20,8 @@ func TestMain(m *testing.M) { helpers.SetDefaultDataPlaneImage(consts.DefaultDataPlaneImage) helpers.SetDefaultDataPlaneBaseImage(consts.DefaultDataPlaneBaseImage) - testSuiteToRun = helpers.ParseGoTestFlags(TestIntegration, testSuiteToRun) + //testSuiteToRun = helpers.ParseGoTestFlags(TestIntegration, testSuiteToRun) + testSuiteToRun = []func(t *testing.T){integration.TestTLSRoute} cfg := integration.DefaultControllerConfigForTests() managerToTest := func(startedChan chan struct{}) error { return manager.Run(cfg, scheme.Get(), manager.SetupControllersShim, admission.NewRequestHandler, startedChan) diff --git a/test/integration/test_gateway.go b/test/integration/test_gateway.go index a971dde5..cd0ec195 100644 --- a/test/integration/test_gateway.go +++ b/test/integration/test_gateway.go @@ -729,8 +729,8 @@ func TestGatewayDataPlaneNetworkPolicy(t *testing.T) { t.Log("verifying that the DataPlane's proxy ingress traffic is allowed") var expectAllowProxyIngress networkPolicyIngressRuleDecorator - expectAllowProxyIngress.withProtocolPort(corev1.ProtocolTCP, consts.DataPlaneProxyPort) - expectAllowProxyIngress.withProtocolPort(corev1.ProtocolTCP, consts.DataPlaneProxySSLPort) + expectAllowProxyIngress.withProtocolPort(corev1.ProtocolTCP, consts.DataPlaneProxyHTTPPort) + expectAllowProxyIngress.withProtocolPort(corev1.ProtocolTCP, consts.DataPlaneProxyHTTPSPort) t.Log("verifying that the DataPlane's metrics ingress traffic is allowed") var expectAllowMetricsIngress networkPolicyIngressRuleDecorator @@ -778,7 +778,7 @@ func TestGatewayDataPlaneNetworkPolicy(t *testing.T) { testutils.GatewayNetworkPolicyForGatewayContainsRules(t, GetCtx(), gateway, clients, expectedUpdatedProxyListenPort.Rule), testutils.SubresourceReadinessWait, time.Second) var notExpectedUpdatedProxyListenPort networkPolicyIngressRuleDecorator - notExpectedUpdatedProxyListenPort.withProtocolPort(corev1.ProtocolTCP, consts.DataPlaneProxyPort) + notExpectedUpdatedProxyListenPort.withProtocolPort(corev1.ProtocolTCP, consts.DataPlaneProxyHTTPPort) require.Eventually(t, testutils.Not( testutils.GatewayNetworkPolicyForGatewayContainsRules(t, GetCtx(), gateway, clients, notExpectedUpdatedProxyListenPort.Rule), diff --git a/test/integration/test_gatewayclass.go b/test/integration/test_gatewayclass.go index b7f62d10..13314abc 100644 --- a/test/integration/test_gatewayclass.go +++ b/test/integration/test_gatewayclass.go @@ -61,7 +61,7 @@ func TestGatewayClassUpdates(t *testing.T) { Listeners: []gatewayv1.Listener{{ Name: "http", Protocol: gatewayv1.HTTPProtocolType, - Port: gatewayv1.PortNumber(80), + Port: gatewayv1.PortNumber(testutils.DefaultHTTPListenerPort), }}, }, } @@ -110,7 +110,7 @@ func TestGatewayClassCreation(t *testing.T) { Listeners: []gatewayv1.Listener{{ Name: "http", Protocol: gatewayv1.HTTPProtocolType, - Port: gatewayv1.PortNumber(80), + Port: gatewayv1.PortNumber(testutils.DefaultHTTPListenerPort), }}, }, } diff --git a/test/integration/test_gatewayconfiguration.go b/test/integration/test_gatewayconfiguration.go index c355e140..8bc32fdf 100644 --- a/test/integration/test_gatewayconfiguration.go +++ b/test/integration/test_gatewayconfiguration.go @@ -151,7 +151,7 @@ func TestGatewayConfigurationEssentials(t *testing.T) { Listeners: []gatewayv1.Listener{{ Name: "http", Protocol: gatewayv1.HTTPProtocolType, - Port: gatewayv1.PortNumber(80), + Port: gatewayv1.PortNumber(testutils.DefaultHTTPListenerPort), }}, }, } diff --git a/test/integration/test_httproute.go b/test/integration/test_httproute.go index 95100738..a7fbcaf4 100644 --- a/test/integration/test_httproute.go +++ b/test/integration/test_httproute.go @@ -67,7 +67,7 @@ func TestHTTPRoute(t *testing.T) { gatewayIPAddress := gateway.Status.Addresses[0].Value t.Log("deploying backend deployment (httpbin) of HTTPRoute") - container := generators.NewContainer("httpbin", testutils.HTTPBinImage, 80) + container := generators.NewContainer("httpbin", testutils.HTTPBinImage, testutils.HTTPBinPort) deployment := generators.NewDeploymentForContainer(container) deployment, err = GetEnv().Cluster().Client().AppsV1().Deployments(namespace.Name).Create(GetCtx(), deployment, metav1.CreateOptions{}) require.NoError(t, err) @@ -77,7 +77,7 @@ func TestHTTPRoute(t *testing.T) { _, err = GetEnv().Cluster().Client().CoreV1().Services(namespace.Name).Create(GetCtx(), service, metav1.CreateOptions{}) require.NoError(t, err) - httpRoute := helpers.GenerateHTTPRoute(namespace.Name, gateway.Name, service.Name) + httpRoute := helpers.GenerateHTTPRoute(namespace.Name, gateway.Name, service.Name, testutils.HTTPBinPort) t.Logf("creating httproute %s/%s to access deployment %s via kong", httpRoute.Namespace, httpRoute.Name, deployment.Name) require.EventuallyWithT(t, func(c *assert.CollectT) { @@ -146,7 +146,7 @@ func TestHTTPRouteWithTLS(t *testing.T) { Namespace: namespace.Name, } - const host = "integration.tests.org" + const host = "httproute.integration.tests.org" cert, key := certificate.MustGenerateSelfSignedCertPEMFormat(certificate.WithDNSNames(host)) secret := &corev1.Secret{ @@ -166,7 +166,7 @@ func TestHTTPRouteWithTLS(t *testing.T) { gateway := helpers.GenerateGateway(gatewayNSN, gatewayClass, func(gateway *gatewayv1.Gateway) { gateway.Spec.Listeners[0].Protocol = gatewayv1.HTTPSProtocolType - gateway.Spec.Listeners[0].Port = gatewayv1.PortNumber(443) + gateway.Spec.Listeners[0].Port = gatewayv1.PortNumber(testutils.DefaultHTTPSListenerPort) gateway.Spec.Listeners[0].TLS = &gatewayv1.GatewayTLSConfig{ CertificateRefs: []gatewayv1.SecretObjectReference{ { @@ -196,7 +196,7 @@ func TestHTTPRouteWithTLS(t *testing.T) { gatewayIPAddress := gateway.Status.Addresses[0].Value t.Log("deploying httpbin backend deployment") - container := generators.NewContainer("httpbin", testutils.HTTPBinImage, 80) + container := generators.NewContainer("httpbin", testutils.HTTPBinImage, testutils.DefaultHTTPListenerPort) deployment := generators.NewDeploymentForContainer(container) deployment, err = GetEnv().Cluster().Client().AppsV1().Deployments(namespace.Name).Create(GetCtx(), deployment, metav1.CreateOptions{}) require.NoError(t, err) @@ -206,7 +206,7 @@ func TestHTTPRouteWithTLS(t *testing.T) { _, err = GetEnv().Cluster().Client().CoreV1().Services(namespace.Name).Create(GetCtx(), service, metav1.CreateOptions{}) require.NoError(t, err) - httpRoute := helpers.GenerateHTTPRoute(namespace.Name, gateway.Name, service.Name, func(h *gatewayv1.HTTPRoute) { + httpRoute := helpers.GenerateHTTPRoute(namespace.Name, gateway.Name, service.Name, testutils.HTTPBinPort, func(h *gatewayv1.HTTPRoute) { h.Spec.Hostnames = []gatewayv1.Hostname{gatewayv1.Hostname(host)} }) diff --git a/test/integration/test_ingress.go b/test/integration/test_ingress.go index 09f70269..56c5656f 100644 --- a/test/integration/test_ingress.go +++ b/test/integration/test_ingress.go @@ -66,7 +66,7 @@ func TestIngressEssentials(t *testing.T) { Listeners: []gatewayv1.Listener{{ Name: "http", Protocol: gatewayv1.HTTPProtocolType, - Port: gatewayv1.PortNumber(80), + Port: gatewayv1.PortNumber(testutils.DefaultHTTPListenerPort), }}, }, } @@ -137,7 +137,7 @@ func TestIngressEssentials(t *testing.T) { require.NoError(t, err) t.Log("deploying a minimal HTTP container deployment to test Ingress routes") - container := generators.NewContainer("httpbin", testutils.HTTPBinImage, 80) + container := generators.NewContainer("httpbin", testutils.HTTPBinImage, testutils.DefaultHTTPListenerPort) deployment := generators.NewDeploymentForContainer(container) deployment, err = GetEnv().Cluster().Client().AppsV1().Deployments(namespace.Name).Create(GetCtx(), deployment, metav1.CreateOptions{}) require.NoError(t, err) diff --git a/test/integration/test_manual_upgrades_and_downgrades.go b/test/integration/test_manual_upgrades_and_downgrades.go index ccf822c9..5a3ce26d 100644 --- a/test/integration/test_manual_upgrades_and_downgrades.go +++ b/test/integration/test_manual_upgrades_and_downgrades.go @@ -125,7 +125,7 @@ func TestManualGatewayUpgradesAndDowngrades(t *testing.T) { Listeners: []gatewayv1.Listener{{ Name: "http", Protocol: gatewayv1.HTTPProtocolType, - Port: gatewayv1.PortNumber(80), + Port: gatewayv1.PortNumber(testutils.DefaultHTTPListenerPort), }}, }, } diff --git a/test/integration/test_tlsroute.go b/test/integration/test_tlsroute.go new file mode 100644 index 00000000..8250ec8a --- /dev/null +++ b/test/integration/test_tlsroute.go @@ -0,0 +1,142 @@ +package integration + +import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/google/uuid" + operatorv1beta1 "github.com/kong/gateway-operator/api/v1beta1" + testutils "github.com/kong/gateway-operator/pkg/utils/test" + "github.com/kong/gateway-operator/test/helpers" + "github.com/kong/gateway-operator/test/helpers/certificate" + "github.com/kong/kubernetes-testing-framework/pkg/utils/kubernetes/generators" + "github.com/samber/lo" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" +) + +func TestTLSRoute(t *testing.T) { + t.Parallel() + + namespace, cleaner := helpers.SetupTestEnv(t, GetCtx(), GetEnv()) + + gatewayConfig := helpers.GenerateGatewayConfiguration(namespace.Name) + t.Logf("deploying GatewayConfiguration %s/%s", gatewayConfig.Namespace, gatewayConfig.Name) + gatewayConfig, err := GetClients().OperatorClient.ApisV1beta1().GatewayConfigurations(namespace.Name).Create(GetCtx(), gatewayConfig, metav1.CreateOptions{}) + require.NoError(t, err) + cleaner.Add(gatewayConfig) + + gatewayClass := helpers.MustGenerateGatewayClass(t, gatewayv1.ParametersReference{ + Group: gatewayv1.Group(operatorv1beta1.SchemeGroupVersion.Group), + Kind: gatewayv1.Kind("GatewayConfiguration"), + Namespace: (*gatewayv1.Namespace)(&gatewayConfig.Namespace), + Name: gatewayConfig.Name, + }) + t.Logf("deploying GatewayClass %s", gatewayClass.Name) + gatewayClass, err = GetClients().GatewayClient.GatewayV1().GatewayClasses().Create(GetCtx(), gatewayClass, metav1.CreateOptions{}) + require.NoError(t, err) + cleaner.Add(gatewayClass) + + gatewayNSN := types.NamespacedName{ + Name: uuid.NewString(), + Namespace: namespace.Name, + } + + gateway := helpers.GenerateGateway(gatewayNSN, gatewayClass, func(gateway *gatewayv1.Gateway) { + gateway.Spec.Listeners[0].Protocol = gatewayv1.TLSProtocolType + gateway.Spec.Listeners[0].Port = gatewayv1.PortNumber(testutils.DefaultTLSListenerPort) + gateway.Spec.Listeners[0].TLS = &gatewayv1.GatewayTLSConfig{ + Mode: lo.ToPtr(gatewayv1.TLSModePassthrough), + } + }) + t.Logf("deploying Gateway %s/%s", gateway.Namespace, gateway.Name) + gateway, err = GetClients().GatewayClient.GatewayV1().Gateways(namespace.Name).Create(GetCtx(), gateway, metav1.CreateOptions{}) + require.NoError(t, err) + cleaner.Add(gateway) + + t.Logf("verifying Gateway %s/%s gets marked as Accepted", gateway.Namespace, gateway.Name) + require.Eventually(t, testutils.GatewayIsAccepted(t, GetCtx(), gatewayNSN, clients), testutils.GatewaySchedulingTimeLimit, time.Second) + + t.Logf("verifying Gateway %s/%s gets marked as Programmed", gateway.Namespace, gateway.Name) + require.Eventually(t, testutils.GatewayIsProgrammed(t, GetCtx(), gatewayNSN, clients), testutils.GatewayReadyTimeLimit, time.Second) + t.Logf("verifying Gateway %s/%s Listeners get marked as Programmed", gateway.Namespace, gateway.Name) + require.Eventually(t, testutils.GatewayListenersAreProgrammed(t, GetCtx(), gatewayNSN, clients), testutils.GatewayReadyTimeLimit, time.Second) + + t.Logf("verifying Gateway %s/%s gets an IP address", gateway.Namespace, gateway.Name) + require.Eventually(t, testutils.GatewayIPAddressExist(t, GetCtx(), gatewayNSN, clients), testutils.SubresourceReadinessWait, time.Second) + gateway = testutils.MustGetGateway(t, GetCtx(), gatewayNSN, clients) + gatewayIPAddress := gateway.Status.Addresses[0].Value + + const host = "tlsroute.integration.tests.org" + cert, key := certificate.MustGenerateSelfSignedCertPEMFormat(certificate.WithDNSNames(host)) + + tlsSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace.Name, + Name: strings.ReplaceAll(host, ".", "-"), + }, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{ + corev1.TLSCertKey: cert, + corev1.TLSPrivateKeyKey: key, + }, + } + t.Logf("deploying Secret %s/%s", tlsSecret.Namespace, tlsSecret.Name) + tlsSecret, err = GetClients().K8sClient.CoreV1().Secrets(namespace.Name).Create(GetCtx(), tlsSecret, metav1.CreateOptions{}) + require.NoError(t, err) + + t.Log("creating a tcpecho pod to test TLSRoute traffic routing") + testUUID := uuid.NewString() // go-echo sends a "Running on Pod ." immediately on connecting + deployment := generators.NewDeploymentForContainer(helpers.GenerateTLSEchoContainer(testutils.TCPEchoImage, testutils.TCPEchoTLSPort, testUUID, tlsSecret.Name)) + deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: tlsSecret.Name, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: tlsSecret.Name, + }, + }, + }) + deployment, err = GetEnv().Cluster().Client().AppsV1().Deployments(namespace.Name).Create(GetCtx(), deployment, metav1.CreateOptions{}) + require.NoError(t, err) + + t.Logf("exposing deployment %s via service", deployment.Name) + service := generators.NewServiceForDeployment(deployment, corev1.ServiceTypeClusterIP) + _, err = GetEnv().Cluster().Client().CoreV1().Services(namespace.Name).Create(GetCtx(), service, metav1.CreateOptions{}) + require.NoError(t, err) + + tlsRoute := helpers.GenerateTLSRoute(namespace.Name, gateway.Name, service.Name, testutils.TCPEchoTLSPort, func(h *gatewayv1alpha2.TLSRoute) { + h.Spec.Hostnames = []gatewayv1alpha2.Hostname{gatewayv1alpha2.Hostname(host)} + }) + + t.Logf("creating tlsroute %s/%s to access deployment %s via kong", tlsRoute.Namespace, tlsRoute.Name, deployment.Name) + require.EventuallyWithT(t, + func(c *assert.CollectT) { + result, err := GetClients().GatewayClient.GatewayV1alpha2().TLSRoutes(namespace.Name).Create(GetCtx(), tlsRoute, metav1.CreateOptions{}) + if err != nil { + t.Logf("failed to deploy tlsroute: %v", err) + c.Errorf("failed to deploy tlsroute: %v", err) + return + } + cleaner.Add(result) + }, + testutils.DefaultIngressWait, testutils.WaitIngressTick, + ) + + t.Log("verifying that the tcpecho is responding properly over TLS") + proxyTLSURL := fmt.Sprintf("%s:%d", gatewayIPAddress, testutils.DefaultTLSListenerPort) + require.Eventually(t, func() bool { + if err := helpers.TLSEchoResponds(proxyTLSURL, testUUID, host, tlsSecret, true); err != nil { + t.Logf("failed accessing tcpecho at %s, err: %v", proxyTLSURL, err) + return false + } + return true + }, testutils.DefaultIngressWait, testutils.WaitIngressTick) + +}