From c1ef7429771104e79f2e87b236b21495cb5765f0 Mon Sep 17 00:00:00 2001 From: Joris Vergeer Date: Tue, 27 Feb 2024 10:54:04 +0100 Subject: [PATCH] Allow to use internal node IPs for NodePort services --- .../kubernetes-crd-definition-v1.yml | 49 +++++ .../kubernetes-crd-rbac.yml | 1 + .../traefik.io_ingressroutes.yaml | 7 + .../traefik.io_ingressroutetcps.yaml | 7 + .../traefik.io_ingressrouteudps.yaml | 7 + .../traefik.io_middlewares.yaml | 7 + .../traefik.io_traefikservices.yaml | 21 ++ .../routing/providers/kubernetes-crd.md | 101 +++++----- .../routing/providers/kubernetes-ingress.md | 10 + integration/fixtures/k8s/01-traefik-crd.yml | 49 +++++ pkg/provider/kubernetes/crd/client.go | 27 ++- .../kubernetes/crd/fixtures/services.yml | 15 ++ .../kubernetes/crd/fixtures/tcp/services.yml | 15 ++ .../tcp/with_node_port_service_lb.yml | 26 +++ .../kubernetes/crd/fixtures/udp/services.yml | 15 ++ .../udp/with_node_port_service_lb.yml | 25 +++ .../crd/fixtures/with_node_port_lb.yml | 27 +++ .../kubernetes/crd/kubernetes_http.go | 33 ++++ pkg/provider/kubernetes/crd/kubernetes_tcp.go | 28 +++ .../kubernetes/crd/kubernetes_test.go | 183 ++++++++++++++++++ pkg/provider/kubernetes/crd/kubernetes_udp.go | 28 +++ .../crd/traefikio/v1alpha1/ingressroute.go | 5 + .../crd/traefikio/v1alpha1/ingressroutetcp.go | 5 + .../crd/traefikio/v1alpha1/ingressrouteudp.go | 5 + .../kubernetes/ingress/annotations.go | 1 + pkg/provider/kubernetes/ingress/client.go | 21 ++ .../kubernetes/ingress/client_mock_test.go | 12 ++ .../fixtures/Ingress-with-node-port-lb.yml | 45 +++++ pkg/provider/kubernetes/ingress/kubernetes.go | 35 ++++ .../kubernetes/ingress/kubernetes_test.go | 52 +++++ pkg/provider/kubernetes/k8s/parser.go | 2 +- 31 files changed, 813 insertions(+), 51 deletions(-) create mode 100644 pkg/provider/kubernetes/crd/fixtures/tcp/with_node_port_service_lb.yml create mode 100644 pkg/provider/kubernetes/crd/fixtures/udp/with_node_port_service_lb.yml create mode 100644 pkg/provider/kubernetes/crd/fixtures/with_node_port_lb.yml create mode 100644 pkg/provider/kubernetes/ingress/fixtures/Ingress-with-node-port-lb.yml diff --git a/docs/content/reference/dynamic-configuration/kubernetes-crd-definition-v1.yml b/docs/content/reference/dynamic-configuration/kubernetes-crd-definition-v1.yml index a95cf1153c..89ed129c66 100644 --- a/docs/content/reference/dynamic-configuration/kubernetes-crd-definition-v1.yml +++ b/docs/content/reference/dynamic-configuration/kubernetes-crd-definition-v1.yml @@ -120,6 +120,13 @@ spec: The Kubernetes Service itself does load-balance to the pods. By default, NativeLB is false. type: boolean + nodePortLB: + description: |- + NodePortLB controls, when creating the load-balancer, + whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort. + It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes. + By default, NodePortLB is false. + type: boolean passHostHeader: description: |- PassHostHeader defines whether the client Host header is forwarded to the upstream Kubernetes Service. @@ -401,6 +408,13 @@ spec: The Kubernetes Service itself does load-balance to the pods. By default, NativeLB is false. type: boolean + nodePortLB: + description: |- + NodePortLB controls, when creating the load-balancer, + whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort. + It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes. + By default, NodePortLB is false. + type: boolean port: anyOf: - type: integer @@ -612,6 +626,13 @@ spec: The Kubernetes Service itself does load-balance to the pods. By default, NativeLB is false. type: boolean + nodePortLB: + description: |- + NodePortLB controls, when creating the load-balancer, + whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort. + It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes. + By default, NodePortLB is false. + type: boolean port: anyOf: - type: integer @@ -916,6 +937,13 @@ spec: The Kubernetes Service itself does load-balance to the pods. By default, NativeLB is false. type: boolean + nodePortLB: + description: |- + NodePortLB controls, when creating the load-balancer, + whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort. + It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes. + By default, NodePortLB is false. + type: boolean passHostHeader: description: |- PassHostHeader defines whether the client Host header is forwarded to the upstream Kubernetes Service. @@ -2304,6 +2332,13 @@ spec: The Kubernetes Service itself does load-balance to the pods. By default, NativeLB is false. type: boolean + nodePortLB: + description: |- + NodePortLB controls, when creating the load-balancer, + whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort. + It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes. + By default, NodePortLB is false. + type: boolean passHostHeader: description: |- PassHostHeader defines whether the client Host header is forwarded to the upstream Kubernetes Service. @@ -2410,6 +2445,13 @@ spec: The Kubernetes Service itself does load-balance to the pods. By default, NativeLB is false. type: boolean + nodePortLB: + description: |- + NodePortLB controls, when creating the load-balancer, + whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort. + It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes. + By default, NodePortLB is false. + type: boolean passHostHeader: description: |- PassHostHeader defines whether the client Host header is forwarded to the upstream Kubernetes Service. @@ -2524,6 +2566,13 @@ spec: The Kubernetes Service itself does load-balance to the pods. By default, NativeLB is false. type: boolean + nodePortLB: + description: |- + NodePortLB controls, when creating the load-balancer, + whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort. + It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes. + By default, NodePortLB is false. + type: boolean passHostHeader: description: |- PassHostHeader defines whether the client Host header is forwarded to the upstream Kubernetes Service. diff --git a/docs/content/reference/dynamic-configuration/kubernetes-crd-rbac.yml b/docs/content/reference/dynamic-configuration/kubernetes-crd-rbac.yml index 301ec51662..6d0afb7a3c 100644 --- a/docs/content/reference/dynamic-configuration/kubernetes-crd-rbac.yml +++ b/docs/content/reference/dynamic-configuration/kubernetes-crd-rbac.yml @@ -10,6 +10,7 @@ rules: - services - endpoints - secrets + - nodes verbs: - get - list diff --git a/docs/content/reference/dynamic-configuration/traefik.io_ingressroutes.yaml b/docs/content/reference/dynamic-configuration/traefik.io_ingressroutes.yaml index 9031689c05..71d4374919 100644 --- a/docs/content/reference/dynamic-configuration/traefik.io_ingressroutes.yaml +++ b/docs/content/reference/dynamic-configuration/traefik.io_ingressroutes.yaml @@ -120,6 +120,13 @@ spec: The Kubernetes Service itself does load-balance to the pods. By default, NativeLB is false. type: boolean + nodePortLB: + description: |- + NodePortLB controls, when creating the load-balancer, + whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort. + It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes. + By default, NodePortLB is false. + type: boolean passHostHeader: description: |- PassHostHeader defines whether the client Host header is forwarded to the upstream Kubernetes Service. diff --git a/docs/content/reference/dynamic-configuration/traefik.io_ingressroutetcps.yaml b/docs/content/reference/dynamic-configuration/traefik.io_ingressroutetcps.yaml index 930b06c041..f3eea5e744 100644 --- a/docs/content/reference/dynamic-configuration/traefik.io_ingressroutetcps.yaml +++ b/docs/content/reference/dynamic-configuration/traefik.io_ingressroutetcps.yaml @@ -103,6 +103,13 @@ spec: The Kubernetes Service itself does load-balance to the pods. By default, NativeLB is false. type: boolean + nodePortLB: + description: |- + NodePortLB controls, when creating the load-balancer, + whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort. + It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes. + By default, NodePortLB is false. + type: boolean port: anyOf: - type: integer diff --git a/docs/content/reference/dynamic-configuration/traefik.io_ingressrouteudps.yaml b/docs/content/reference/dynamic-configuration/traefik.io_ingressrouteudps.yaml index 245194c62c..19bbfe62eb 100644 --- a/docs/content/reference/dynamic-configuration/traefik.io_ingressrouteudps.yaml +++ b/docs/content/reference/dynamic-configuration/traefik.io_ingressrouteudps.yaml @@ -74,6 +74,13 @@ spec: The Kubernetes Service itself does load-balance to the pods. By default, NativeLB is false. type: boolean + nodePortLB: + description: |- + NodePortLB controls, when creating the load-balancer, + whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort. + It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes. + By default, NodePortLB is false. + type: boolean port: anyOf: - type: integer diff --git a/docs/content/reference/dynamic-configuration/traefik.io_middlewares.yaml b/docs/content/reference/dynamic-configuration/traefik.io_middlewares.yaml index 4ef178a57f..ef78733f8f 100644 --- a/docs/content/reference/dynamic-configuration/traefik.io_middlewares.yaml +++ b/docs/content/reference/dynamic-configuration/traefik.io_middlewares.yaml @@ -274,6 +274,13 @@ spec: The Kubernetes Service itself does load-balance to the pods. By default, NativeLB is false. type: boolean + nodePortLB: + description: |- + NodePortLB controls, when creating the load-balancer, + whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort. + It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes. + By default, NodePortLB is false. + type: boolean passHostHeader: description: |- PassHostHeader defines whether the client Host header is forwarded to the upstream Kubernetes Service. diff --git a/docs/content/reference/dynamic-configuration/traefik.io_traefikservices.yaml b/docs/content/reference/dynamic-configuration/traefik.io_traefikservices.yaml index 7c8f58a3e7..ee99b7b19e 100644 --- a/docs/content/reference/dynamic-configuration/traefik.io_traefikservices.yaml +++ b/docs/content/reference/dynamic-configuration/traefik.io_traefikservices.yaml @@ -88,6 +88,13 @@ spec: The Kubernetes Service itself does load-balance to the pods. By default, NativeLB is false. type: boolean + nodePortLB: + description: |- + NodePortLB controls, when creating the load-balancer, + whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort. + It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes. + By default, NodePortLB is false. + type: boolean passHostHeader: description: |- PassHostHeader defines whether the client Host header is forwarded to the upstream Kubernetes Service. @@ -194,6 +201,13 @@ spec: The Kubernetes Service itself does load-balance to the pods. By default, NativeLB is false. type: boolean + nodePortLB: + description: |- + NodePortLB controls, when creating the load-balancer, + whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort. + It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes. + By default, NodePortLB is false. + type: boolean passHostHeader: description: |- PassHostHeader defines whether the client Host header is forwarded to the upstream Kubernetes Service. @@ -308,6 +322,13 @@ spec: The Kubernetes Service itself does load-balance to the pods. By default, NativeLB is false. type: boolean + nodePortLB: + description: |- + NodePortLB controls, when creating the load-balancer, + whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort. + It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes. + By default, NodePortLB is false. + type: boolean passHostHeader: description: |- PassHostHeader defines whether the client Host header is forwarded to the upstream Kubernetes Service. diff --git a/docs/content/routing/providers/kubernetes-crd.md b/docs/content/routing/providers/kubernetes-crd.md index 55f25494a4..73929cfe81 100644 --- a/docs/content/routing/providers/kubernetes-crd.md +++ b/docs/content/routing/providers/kubernetes-crd.md @@ -352,15 +352,16 @@ Register the `IngressRoute` [kind](../../reference/dynamic-configuration/kuberne strategy: RoundRobin weight: 10 nativeLB: true # [11] - tls: # [12] - secretName: supersecret # [13] - options: # [14] - name: opt # [15] - namespace: default # [16] - certResolver: foo # [17] - domains: # [18] - - main: example.net # [19] - sans: # [20] + nodePortLB: true # [12] + tls: # [13] + secretName: supersecret # [14] + options: # [15] + name: opt # [16] + namespace: default # [17] + certResolver: foo # [18] + domains: # [19] + - main: example.net # [20] + sans: # [21] - a.example.net - b.example.net ``` @@ -378,15 +379,16 @@ Register the `IngressRoute` [kind](../../reference/dynamic-configuration/kuberne | [9] | `services[n].port` | Defines the port of a [Kubernetes service](https://kubernetes.io/docs/concepts/services-networking/service/). This can be a reference to a named port. | | [10] | `services[n].serversTransport` | Defines the reference to a [ServersTransport](#kind-serverstransport). The ServersTransport namespace is assumed to be the [Kubernetes service](https://kubernetes.io/docs/concepts/services-networking/service/) namespace (see [ServersTransport reference](#serverstransport-reference)). | | [11] | `services[n].nativeLB` | Controls, when creating the load-balancer, whether the LB's children are directly the pods IPs or if the only child is the Kubernetes Service clusterIP. | -| [12] | `tls` | Defines [TLS](../routers/index.md#tls) certificate configuration | -| [13] | `tls.secretName` | Defines the [secret](https://kubernetes.io/docs/concepts/configuration/secret/) name used to store the certificate (in the `IngressRoute` namespace) | -| [14] | `tls.options` | Defines the reference to a [TLSOption](#kind-tlsoption) | -| [15] | `options.name` | Defines the [TLSOption](#kind-tlsoption) name | -| [16] | `options.namespace` | Defines the [TLSOption](#kind-tlsoption) namespace | -| [17] | `tls.certResolver` | Defines the reference to a [CertResolver](../routers/index.md#certresolver) | -| [18] | `tls.domains` | List of [domains](../routers/index.md#domains) | -| [19] | `domains[n].main` | Defines the main domain name | -| [20] | `domains[n].sans` | List of SANs (alternative domains) | +| [12] | `services[n].nodePortLB` | Controls, when creating the load-balancer, whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort. | +| [13] | `tls` | Defines [TLS](../routers/index.md#tls) certificate configuration | +| [14] | `tls.secretName` | Defines the [secret](https://kubernetes.io/docs/concepts/configuration/secret/) name used to store the certificate (in the `IngressRoute` namespace) | +| [15] | `tls.options` | Defines the reference to a [TLSOption](#kind-tlsoption) | +| [16] | `options.name` | Defines the [TLSOption](#kind-tlsoption) name | +| [17] | `options.namespace` | Defines the [TLSOption](#kind-tlsoption) namespace | +| [18] | `tls.certResolver` | Defines the reference to a [CertResolver](../routers/index.md#certresolver) | +| [19] | `tls.domains` | List of [domains](../routers/index.md#domains) | +| [20] | `domains[n].main` | Defines the main domain name | +| [21] | `domains[n].sans` | List of SANs (alternative domains) | ??? example "Declaring an IngressRoute" @@ -1149,18 +1151,20 @@ Register the `IngressRouteTCP` [kind](../../reference/dynamic-configuration/kube version: 1 # [12] serversTransport: transport # [13] nativeLB: true # [14] - tls: # [15] - secretName: supersecret # [16] - options: # [17] - name: opt # [18] - namespace: default # [19] - certResolver: foo # [20] - domains: # [21] - - main: example.net # [22] - sans: # [23] + nodePortLB: true # [15] + + tls: # [16] + secretName: supersecret # [17] + options: # [18] + name: opt # [19] + namespace: default # [20] + certResolver: foo # [21] + domains: # [22] + - main: example.net # [23] + sans: # [24] - a.example.net - b.example.net - passthrough: false # [24] + passthrough: false # [25] ``` | Ref | Attribute | Purpose | @@ -1179,16 +1183,17 @@ Register the `IngressRouteTCP` [kind](../../reference/dynamic-configuration/kube | [12] | `services[n].proxyProtocol.version` | Defines the [PROXY protocol](../services/index.md#proxy-protocol) version | | [13] | `services[n].serversTransport` | Defines the reference to a [ServersTransportTCP](#kind-serverstransporttcp). The ServersTransport namespace is assumed to be the [Kubernetes service](https://kubernetes.io/docs/concepts/services-networking/service/) namespace (see [ServersTransport reference](#serverstransport-reference)). | | [14] | `services[n].nativeLB` | Controls, when creating the load-balancer, whether the LB's children are directly the pods IPs or if the only child is the Kubernetes Service clusterIP. | -| [15] | `tls` | Defines [TLS](../routers/index.md#tls_1) certificate configuration | -| [16] | `tls.secretName` | Defines the [secret](https://kubernetes.io/docs/concepts/configuration/secret/) name used to store the certificate (in the `IngressRoute` namespace) | -| [17] | `tls.options` | Defines the reference to a [TLSOption](#kind-tlsoption) | -| [18] | `tls.options.name` | Defines the [TLSOption](#kind-tlsoption) name | -| [19] | `tls.options.namespace` | Defines the [TLSOption](#kind-tlsoption) namespace | -| [20] | `tls.certResolver` | Defines the reference to a [CertResolver](../routers/index.md#certresolver_1) | -| [21] | `tls.domains` | List of [domains](../routers/index.md#domains_1) | -| [22] | `tls.domains[n].main` | Defines the main domain name | -| [23] | `tls.domains[n].sans` | List of SANs (alternative domains) | -| [24] | `tls.passthrough` | If `true`, delegates the TLS termination to the backend | +| [15] | `services[n].nodePortLB` | Controls, when creating the load-balancer, whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is | +| [16] | `tls` | Defines [TLS](../routers/index.md#tls_1) certificate configuration | +| [17] | `tls.secretName` | Defines the [secret](https://kubernetes.io/docs/concepts/configuration/secret/) name used to store the certificate (in the `IngressRoute` namespace) | +| [18] | `tls.options` | Defines the reference to a [TLSOption](#kind-tlsoption) | +| [19] | `tls.options.name` | Defines the [TLSOption](#kind-tlsoption) name | +| [20] | `tls.options.namespace` | Defines the [TLSOption](#kind-tlsoption) namespace | +| [21] | `tls.certResolver` | Defines the reference to a [CertResolver](../routers/index.md#certresolver_1) | +| [22] | `tls.domains` | List of [domains](../routers/index.md#domains_1) | +| [23] | `tls.domains[n].main` | Defines the main domain name | +| [24] | `tls.domains[n].sans` | List of SANs (alternative domains) | +| [25] | `tls.passthrough` | If `true`, delegates the TLS termination to the backend | ??? example "Declaring an IngressRouteTCP" @@ -1433,17 +1438,19 @@ Register the `IngressRouteUDP` [kind](../../reference/dynamic-configuration/kube port: 8080 # [5] weight: 10 # [6] nativeLB: true # [7] + nodePortLB: true # [8] ``` -| Ref | Attribute | Purpose | -|-----|-------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------| -| [1] | `entryPoints` | List of [entrypoints](../routers/index.md#entrypoints_1) names | -| [2] | `routes` | List of routes | -| [3] | `routes[n].services` | List of [Kubernetes service](https://kubernetes.io/docs/concepts/services-networking/service/) definitions (See below for `ExternalName Service` setup) | -| [4] | `services[n].name` | Defines the name of a [Kubernetes service](https://kubernetes.io/docs/concepts/services-networking/service/) | -| [5] | `services[n].port` | Defines the port of a [Kubernetes service](https://kubernetes.io/docs/concepts/services-networking/service/). This can be a reference to a named port. | -| [6] | `services[n].weight` | Defines the weight to apply to the server load balancing | -| [7] | `services[n].nativeLB` | Controls, when creating the load-balancer, whether the LB's children are directly the pods IPs or if the only child is the Kubernetes Service clusterIP. | +| Ref | Attribute | Purpose | +|-----|-------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [1] | `entryPoints` | List of [entrypoints](../routers/index.md#entrypoints_1) names | +| [2] | `routes` | List of routes | +| [3] | `routes[n].services` | List of [Kubernetes service](https://kubernetes.io/docs/concepts/services-networking/service/) definitions (See below for `ExternalName Service` setup) | +| [4] | `services[n].name` | Defines the name of a [Kubernetes service](https://kubernetes.io/docs/concepts/services-networking/service/) | +| [5] | `services[n].port` | Defines the port of a [Kubernetes service](https://kubernetes.io/docs/concepts/services-networking/service/). This can be a reference to a named port. | +| [6] | `services[n].weight` | Defines the weight to apply to the server load balancing | +| [7] | `services[n].nativeLB` | Controls, when creating the load-balancer, whether the LB's children are directly the pods IPs or if the only child is the Kubernetes Service clusterIP. | +| [8] | `services[n].nodePortLB` | Controls, when creating the load-balancer, whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort. | ??? example "Declaring an IngressRouteUDP" diff --git a/docs/content/routing/providers/kubernetes-ingress.md b/docs/content/routing/providers/kubernetes-ingress.md index 01f1d51661..eabf2f2004 100644 --- a/docs/content/routing/providers/kubernetes-ingress.md +++ b/docs/content/routing/providers/kubernetes-ingress.md @@ -287,6 +287,16 @@ which in turn will create the resulting routers, services, handlers, etc. traefik.ingress.kubernetes.io/service.nativelb: "true" ``` +??? info "`traefik.ingress.kubernetes.io/service.nodeportlb`" + + Controls, when creating the load-balancer, whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort. + It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes. + By default, NodePortLB is false. + + ```yaml + traefik.ingress.kubernetes.io/service.nodeportlb: "true" + ``` + ??? info "`traefik.ingress.kubernetes.io/service.serversscheme`" Overrides the default scheme. diff --git a/integration/fixtures/k8s/01-traefik-crd.yml b/integration/fixtures/k8s/01-traefik-crd.yml index a95cf1153c..89ed129c66 100644 --- a/integration/fixtures/k8s/01-traefik-crd.yml +++ b/integration/fixtures/k8s/01-traefik-crd.yml @@ -120,6 +120,13 @@ spec: The Kubernetes Service itself does load-balance to the pods. By default, NativeLB is false. type: boolean + nodePortLB: + description: |- + NodePortLB controls, when creating the load-balancer, + whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort. + It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes. + By default, NodePortLB is false. + type: boolean passHostHeader: description: |- PassHostHeader defines whether the client Host header is forwarded to the upstream Kubernetes Service. @@ -401,6 +408,13 @@ spec: The Kubernetes Service itself does load-balance to the pods. By default, NativeLB is false. type: boolean + nodePortLB: + description: |- + NodePortLB controls, when creating the load-balancer, + whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort. + It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes. + By default, NodePortLB is false. + type: boolean port: anyOf: - type: integer @@ -612,6 +626,13 @@ spec: The Kubernetes Service itself does load-balance to the pods. By default, NativeLB is false. type: boolean + nodePortLB: + description: |- + NodePortLB controls, when creating the load-balancer, + whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort. + It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes. + By default, NodePortLB is false. + type: boolean port: anyOf: - type: integer @@ -916,6 +937,13 @@ spec: The Kubernetes Service itself does load-balance to the pods. By default, NativeLB is false. type: boolean + nodePortLB: + description: |- + NodePortLB controls, when creating the load-balancer, + whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort. + It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes. + By default, NodePortLB is false. + type: boolean passHostHeader: description: |- PassHostHeader defines whether the client Host header is forwarded to the upstream Kubernetes Service. @@ -2304,6 +2332,13 @@ spec: The Kubernetes Service itself does load-balance to the pods. By default, NativeLB is false. type: boolean + nodePortLB: + description: |- + NodePortLB controls, when creating the load-balancer, + whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort. + It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes. + By default, NodePortLB is false. + type: boolean passHostHeader: description: |- PassHostHeader defines whether the client Host header is forwarded to the upstream Kubernetes Service. @@ -2410,6 +2445,13 @@ spec: The Kubernetes Service itself does load-balance to the pods. By default, NativeLB is false. type: boolean + nodePortLB: + description: |- + NodePortLB controls, when creating the load-balancer, + whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort. + It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes. + By default, NodePortLB is false. + type: boolean passHostHeader: description: |- PassHostHeader defines whether the client Host header is forwarded to the upstream Kubernetes Service. @@ -2524,6 +2566,13 @@ spec: The Kubernetes Service itself does load-balance to the pods. By default, NativeLB is false. type: boolean + nodePortLB: + description: |- + NodePortLB controls, when creating the load-balancer, + whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort. + It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes. + By default, NodePortLB is false. + type: boolean passHostHeader: description: |- PassHostHeader defines whether the client Host header is forwarded to the upstream Kubernetes Service. diff --git a/pkg/provider/kubernetes/crd/client.go b/pkg/provider/kubernetes/crd/client.go index 83c4834ba3..66c325ebe1 100644 --- a/pkg/provider/kubernetes/crd/client.go +++ b/pkg/provider/kubernetes/crd/client.go @@ -46,6 +46,7 @@ type Client interface { GetService(namespace, name string) (*corev1.Service, bool, error) GetSecret(namespace, name string) (*corev1.Secret, bool, error) GetEndpoints(namespace, name string) (*corev1.Endpoints, bool, error) + GetNodes() ([]*corev1.Node, bool, error) } // TODO: add tests for the clientWrapper (and its methods) itself. @@ -53,9 +54,10 @@ type clientWrapper struct { csCrd traefikclientset.Interface csKube kclientset.Interface - factoriesCrd map[string]traefikinformers.SharedInformerFactory - factoriesKube map[string]kinformers.SharedInformerFactory - factoriesSecret map[string]kinformers.SharedInformerFactory + factoryClusterScope kinformers.SharedInformerFactory + factoriesCrd map[string]traefikinformers.SharedInformerFactory + factoriesKube map[string]kinformers.SharedInformerFactory + factoriesSecret map[string]kinformers.SharedInformerFactory labelSelector string @@ -232,11 +234,18 @@ func (c *clientWrapper) WatchAll(namespaces []string, stopCh <-chan struct{}) (< c.factoriesSecret[ns] = factorySecret } + c.factoryClusterScope = kinformers.NewSharedInformerFactory(c.csKube, resyncPeriod) + _, err := c.factoryClusterScope.Core().V1().Nodes().Informer().AddEventHandler(eventHandler) + if err != nil { + return nil, err + } + for _, ns := range namespaces { c.factoriesCrd[ns].Start(stopCh) c.factoriesKube[ns].Start(stopCh) c.factoriesSecret[ns].Start(stopCh) } + c.factoryClusterScope.Start(stopCh) for _, ns := range namespaces { for t, ok := range c.factoriesCrd[ns].WaitForCacheSync(stopCh) { @@ -258,6 +267,12 @@ func (c *clientWrapper) WatchAll(namespaces []string, stopCh <-chan struct{}) (< } } + for t, ok := range c.factoryClusterScope.WaitForCacheSync(stopCh) { + if !ok { + return nil, fmt.Errorf("timed out waiting for controller caches to sync %s", t.String()) + } + } + return eventCh, nil } @@ -450,6 +465,12 @@ func (c *clientWrapper) GetSecret(namespace, name string) (*corev1.Secret, bool, return secret, exist, err } +func (c *clientWrapper) GetNodes() ([]*corev1.Node, bool, error) { + nodes, err := c.factoryClusterScope.Core().V1().Nodes().Lister().List(labels.Everything()) + exist, err := translateNotFoundError(err) + return nodes, exist, err +} + // lookupNamespace returns the lookup namespace key for the given namespace. // When listening on all namespaces, it returns the client-go identifier ("") // for all-namespaces. Otherwise, it returns the given namespace. diff --git a/pkg/provider/kubernetes/crd/fixtures/services.yml b/pkg/provider/kubernetes/crd/fixtures/services.yml index 818a5bd3d3..825bb9106c 100644 --- a/pkg/provider/kubernetes/crd/fixtures/services.yml +++ b/pkg/provider/kubernetes/crd/fixtures/services.yml @@ -266,3 +266,18 @@ spec: port: 80 type: ClusterIP clusterIP: 10.10.0.1 + +--- +apiVersion: v1 +kind: Service +metadata: + name: nodeport-svc + namespace: default + +spec: + ports: + - name: web + port: 80 + nodePort: 32456 + type: NodePort + clusterIP: 10.10.0.1 diff --git a/pkg/provider/kubernetes/crd/fixtures/tcp/services.yml b/pkg/provider/kubernetes/crd/fixtures/tcp/services.yml index 1c49644f22..40a2e90e56 100644 --- a/pkg/provider/kubernetes/crd/fixtures/tcp/services.yml +++ b/pkg/provider/kubernetes/crd/fixtures/tcp/services.yml @@ -262,3 +262,18 @@ spec: port: 8000 type: ClusterIP clusterIP: 10.10.0.1 + +--- +apiVersion: v1 +kind: Service +metadata: + name: nodeport-svc-tcp + namespace: default + +spec: + ports: + - name: myapp + port: 8000 + nodePort: 32456 + type: NodePort + clusterIP: 10.10.0.1 diff --git a/pkg/provider/kubernetes/crd/fixtures/tcp/with_node_port_service_lb.yml b/pkg/provider/kubernetes/crd/fixtures/tcp/with_node_port_service_lb.yml new file mode 100644 index 0000000000..2e9519956c --- /dev/null +++ b/pkg/provider/kubernetes/crd/fixtures/tcp/with_node_port_service_lb.yml @@ -0,0 +1,26 @@ +apiVersion: traefik.io/v1alpha1 +kind: IngressRouteTCP +metadata: + name: test.route + namespace: default + +spec: + entryPoints: + - foo + + routes: + - match: HostSNI(`foo.com`) + services: + - name: nodeport-svc-tcp + port: 8000 + nodePortLB: true + +--- +kind: Node +apiVersion: v1 +metadata: + name: traefik-node +status: + addresses: + - type: InternalIP + address: 172.16.4.4 diff --git a/pkg/provider/kubernetes/crd/fixtures/udp/services.yml b/pkg/provider/kubernetes/crd/fixtures/udp/services.yml index 90d75d0590..9692818862 100644 --- a/pkg/provider/kubernetes/crd/fixtures/udp/services.yml +++ b/pkg/provider/kubernetes/crd/fixtures/udp/services.yml @@ -221,3 +221,18 @@ spec: port: 8000 type: ClusterIP clusterIP: 10.10.0.1 + +--- +apiVersion: v1 +kind: Service +metadata: + name: nodeport-svc-udp + namespace: default + +spec: + ports: + - name: myapp + port: 8000 + nodePort: 32456 + type: NodePort + clusterIP: 10.10.0.1 diff --git a/pkg/provider/kubernetes/crd/fixtures/udp/with_node_port_service_lb.yml b/pkg/provider/kubernetes/crd/fixtures/udp/with_node_port_service_lb.yml new file mode 100644 index 0000000000..b32f86e111 --- /dev/null +++ b/pkg/provider/kubernetes/crd/fixtures/udp/with_node_port_service_lb.yml @@ -0,0 +1,25 @@ +apiVersion: traefik.io/v1alpha1 +kind: IngressRouteUDP +metadata: + name: test.route + namespace: default + +spec: + entryPoints: + - foo + + routes: + - services: + - name: nodeport-svc-udp + port: 8000 + nodePortLB: true + +--- +kind: Node +apiVersion: v1 +metadata: + name: traefik-node +status: + addresses: + - type: InternalIP + address: 172.16.4.4 diff --git a/pkg/provider/kubernetes/crd/fixtures/with_node_port_lb.yml b/pkg/provider/kubernetes/crd/fixtures/with_node_port_lb.yml new file mode 100644 index 0000000000..7d73c19799 --- /dev/null +++ b/pkg/provider/kubernetes/crd/fixtures/with_node_port_lb.yml @@ -0,0 +1,27 @@ +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: test.route + namespace: default + +spec: + entryPoints: + - foo + + routes: + - match: Host(`foo.com`) + kind: Rule + services: + - name: nodeport-svc + port: 80 + nodePortLB: true + +--- +kind: Node +apiVersion: v1 +metadata: + name: traefik-node +status: + addresses: + - type: InternalIP + address: 172.16.4.4 diff --git a/pkg/provider/kubernetes/crd/kubernetes_http.go b/pkg/provider/kubernetes/crd/kubernetes_http.go index 6e3b1f2a53..03a054fc55 100644 --- a/pkg/provider/kubernetes/crd/kubernetes_http.go +++ b/pkg/provider/kubernetes/crd/kubernetes_http.go @@ -409,6 +409,39 @@ func (c configBuilder) loadServers(parentNamespace string, svc traefikv1alpha1.L }), nil } + if service.Spec.Type == corev1.ServiceTypeNodePort && svc.NodePortLB { + nodes, nodesExists, nodesErr := c.client.GetNodes() + if nodesErr != nil { + return nil, nodesErr + } + if !nodesExists || len(nodes) == 0 { + return nil, fmt.Errorf("nodes not found for NodePort service %s/%s", namespace, sanitizedName) + } + + protocol, err := parseServiceProtocol(svc.Scheme, svcPort.Name, svcPort.Port) + if err != nil { + return nil, err + } + + for _, node := range nodes { + for _, addr := range node.Status.Addresses { + if addr.Type == corev1.NodeInternalIP { + hostPort := net.JoinHostPort(addr.Address, strconv.Itoa(int(svcPort.NodePort))) + + servers = append(servers, dynamic.Server{ + URL: fmt.Sprintf("%s://%s", protocol, hostPort), + }) + } + } + } + + if len(servers) == 0 { + return nil, fmt.Errorf("no servers were generated for service %s in namespace", sanitizedName) + } + + return servers, nil + } + endpoints, endpointsExists, endpointsErr := c.client.GetEndpoints(namespace, sanitizedName) if endpointsErr != nil { return nil, endpointsErr diff --git a/pkg/provider/kubernetes/crd/kubernetes_tcp.go b/pkg/provider/kubernetes/crd/kubernetes_tcp.go index f41fe125e7..657dcf7739 100644 --- a/pkg/provider/kubernetes/crd/kubernetes_tcp.go +++ b/pkg/provider/kubernetes/crd/kubernetes_tcp.go @@ -247,6 +247,34 @@ func (p *Provider) loadTCPServers(client Client, namespace string, svc traefikv1 } var servers []dynamic.TCPServer + + if service.Spec.Type == corev1.ServiceTypeNodePort && svc.NodePortLB { + nodes, nodesExists, nodesErr := client.GetNodes() + if nodesErr != nil { + return nil, nodesErr + } + + if !nodesExists || len(nodes) == 0 { + return nil, fmt.Errorf("nodes not found for NodePort service %s/%s", svc.Namespace, svc.Name) + } + + for _, node := range nodes { + for _, addr := range node.Status.Addresses { + if addr.Type == corev1.NodeInternalIP { + servers = append(servers, dynamic.TCPServer{ + Address: net.JoinHostPort(addr.Address, strconv.Itoa(int(svcPort.NodePort))), + }) + } + } + } + + if len(servers) == 0 { + return nil, fmt.Errorf("no servers were generated for service %s/%s", svc.Namespace, svc.Name) + } + + return servers, nil + } + if service.Spec.Type == corev1.ServiceTypeExternalName { servers = append(servers, dynamic.TCPServer{ Address: net.JoinHostPort(service.Spec.ExternalName, strconv.Itoa(int(svcPort.Port))), diff --git a/pkg/provider/kubernetes/crd/kubernetes_test.go b/pkg/provider/kubernetes/crd/kubernetes_test.go index c851140c9d..a525dce1fe 100644 --- a/pkg/provider/kubernetes/crd/kubernetes_test.go +++ b/pkg/provider/kubernetes/crd/kubernetes_test.go @@ -7020,6 +7020,189 @@ func TestNativeLB(t *testing.T) { } } +func TestNodePortLB(t *testing.T) { + testCases := []struct { + desc string + ingressClass string + paths []string + expected *dynamic.Configuration + }{ + { + desc: "Empty", + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + Routers: map[string]*dynamic.TCPRouter{}, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + ServersTransports: map[string]*dynamic.ServersTransport{}, + Routers: map[string]*dynamic.Router{}, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "HTTP with node port LB", + paths: []string{"services.yml", "with_node_port_lb.yml"}, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + Routers: map[string]*dynamic.TCPRouter{}, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + ServersTransports: map[string]*dynamic.ServersTransport{}, + Routers: map[string]*dynamic.Router{ + "default-test-route-6f97418635c7e18853da": { + EntryPoints: []string{"foo"}, + Service: "default-test-route-6f97418635c7e18853da", + Rule: "Host(`foo.com`)", + Priority: 0, + }, + }, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{ + "default-test-route-6f97418635c7e18853da": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + ResponseForwarding: &dynamic.ResponseForwarding{FlushInterval: dynamic.DefaultFlushInterval}, + Servers: []dynamic.Server{ + { + URL: "http://172.16.4.4:32456", + }, + }, + PassHostHeader: Bool(true), + }, + }, + }, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "TCP with native Service LB", + paths: []string{"tcp/services.yml", "tcp/with_node_port_service_lb.yml"}, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + ServersTransports: map[string]*dynamic.ServersTransport{}, + Routers: map[string]*dynamic.Router{}, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{}, + }, + TCP: &dynamic.TCPConfiguration{ + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + Routers: map[string]*dynamic.TCPRouter{ + "default-test.route-fdd3e9338e47a45efefc": { + EntryPoints: []string{"foo"}, + Service: "default-test.route-fdd3e9338e47a45efefc", + Rule: "HostSNI(`foo.com`)", + }, + }, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{ + "default-test.route-fdd3e9338e47a45efefc": { + LoadBalancer: &dynamic.TCPServersLoadBalancer{ + Servers: []dynamic.TCPServer{ + { + Address: "172.16.4.4:32456", + Port: "", + }, + }, + }, + }, + }, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "UDP with native Service LB", + paths: []string{"udp/services.yml", "udp/with_node_port_service_lb.yml"}, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{ + "default-test.route-0": { + EntryPoints: []string{"foo"}, + Service: "default-test.route-0", + }, + }, + Services: map[string]*dynamic.UDPService{ + "default-test.route-0": { + LoadBalancer: &dynamic.UDPServersLoadBalancer{ + Servers: []dynamic.UDPServer{ + { + Address: "172.16.4.4:32456", + Port: "", + }, + }, + }, + }, + }, + }, + HTTP: &dynamic.HTTPConfiguration{ + ServersTransports: map[string]*dynamic.ServersTransport{}, + Routers: map[string]*dynamic.Router{}, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{}, + }, + TCP: &dynamic.TCPConfiguration{ + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + Routers: map[string]*dynamic.TCPRouter{}, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + } + + for _, test := range testCases { + test := test + + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + k8sObjects, crdObjects := readResources(t, test.paths) + + kubeClient := kubefake.NewSimpleClientset(k8sObjects...) + crdClient := traefikcrdfake.NewSimpleClientset(crdObjects...) + + client := newClientImpl(kubeClient, crdClient) + + stopCh := make(chan struct{}) + + eventCh, err := client.WatchAll([]string{"default", "cross-ns"}, stopCh) + require.NoError(t, err) + + if k8sObjects != nil || crdObjects != nil { + // just wait for the first event + <-eventCh + } + + p := Provider{} + + conf := p.loadConfigurationFromCRD(context.Background(), client) + assert.Equal(t, test.expected, conf) + }) + } +} + func TestCreateBasicAuthCredentials(t *testing.T) { var k8sObjects []runtime.Object yamlContent, err := os.ReadFile(filepath.FromSlash("./fixtures/basic_auth_secrets.yml")) diff --git a/pkg/provider/kubernetes/crd/kubernetes_udp.go b/pkg/provider/kubernetes/crd/kubernetes_udp.go index 417e90d996..6afb7990e8 100644 --- a/pkg/provider/kubernetes/crd/kubernetes_udp.go +++ b/pkg/provider/kubernetes/crd/kubernetes_udp.go @@ -131,6 +131,34 @@ func (p *Provider) loadUDPServers(client Client, namespace string, svc traefikv1 } var servers []dynamic.UDPServer + + if service.Spec.Type == corev1.ServiceTypeNodePort && svc.NodePortLB { + nodes, nodesExists, nodesErr := client.GetNodes() + if nodesErr != nil { + return nil, nodesErr + } + + if !nodesExists || len(nodes) == 0 { + return nil, fmt.Errorf("nodes not found for NodePort service %s/%s", svc.Namespace, svc.Name) + } + + for _, node := range nodes { + for _, addr := range node.Status.Addresses { + if addr.Type == corev1.NodeInternalIP { + servers = append(servers, dynamic.UDPServer{ + Address: net.JoinHostPort(addr.Address, strconv.Itoa(int(svcPort.NodePort))), + }) + } + } + } + + if len(servers) == 0 { + return nil, fmt.Errorf("no servers were generated for service %s/%s", svc.Namespace, svc.Name) + } + + return servers, nil + } + if service.Spec.Type == corev1.ServiceTypeExternalName { servers = append(servers, dynamic.UDPServer{ Address: net.JoinHostPort(service.Spec.ExternalName, strconv.Itoa(int(svcPort.Port))), diff --git a/pkg/provider/kubernetes/crd/traefikio/v1alpha1/ingressroute.go b/pkg/provider/kubernetes/crd/traefikio/v1alpha1/ingressroute.go index 475d160d44..03ec08dfb3 100644 --- a/pkg/provider/kubernetes/crd/traefikio/v1alpha1/ingressroute.go +++ b/pkg/provider/kubernetes/crd/traefikio/v1alpha1/ingressroute.go @@ -126,6 +126,11 @@ type LoadBalancerSpec struct { // The Kubernetes Service itself does load-balance to the pods. // By default, NativeLB is false. NativeLB bool `json:"nativeLB,omitempty"` + // NodePortLB controls, when creating the load-balancer, + // whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort. + // It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes. + // By default, NodePortLB is false. + NodePortLB bool `json:"nodePortLB,omitempty"` } type ResponseForwarding struct { diff --git a/pkg/provider/kubernetes/crd/traefikio/v1alpha1/ingressroutetcp.go b/pkg/provider/kubernetes/crd/traefikio/v1alpha1/ingressroutetcp.go index 9ff6897db4..73f37a3f6a 100644 --- a/pkg/provider/kubernetes/crd/traefikio/v1alpha1/ingressroutetcp.go +++ b/pkg/provider/kubernetes/crd/traefikio/v1alpha1/ingressroutetcp.go @@ -93,6 +93,11 @@ type ServiceTCP struct { // The Kubernetes Service itself does load-balance to the pods. // By default, NativeLB is false. NativeLB bool `json:"nativeLB,omitempty"` + // NodePortLB controls, when creating the load-balancer, + // whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort. + // It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes. + // By default, NodePortLB is false. + NodePortLB bool `json:"nodePortLB,omitempty"` } // +genclient diff --git a/pkg/provider/kubernetes/crd/traefikio/v1alpha1/ingressrouteudp.go b/pkg/provider/kubernetes/crd/traefikio/v1alpha1/ingressrouteudp.go index 18773f4372..b2c185f0eb 100644 --- a/pkg/provider/kubernetes/crd/traefikio/v1alpha1/ingressrouteudp.go +++ b/pkg/provider/kubernetes/crd/traefikio/v1alpha1/ingressrouteudp.go @@ -38,6 +38,11 @@ type ServiceUDP struct { // The Kubernetes Service itself does load-balance to the pods. // By default, NativeLB is false. NativeLB bool `json:"nativeLB,omitempty"` + // NodePortLB controls, when creating the load-balancer, + // whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort. + // It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes. + // By default, NodePortLB is false. + NodePortLB bool `json:"nodePortLB,omitempty"` } // +genclient diff --git a/pkg/provider/kubernetes/ingress/annotations.go b/pkg/provider/kubernetes/ingress/annotations.go index 134b2eda2d..fcc021672a 100644 --- a/pkg/provider/kubernetes/ingress/annotations.go +++ b/pkg/provider/kubernetes/ingress/annotations.go @@ -46,6 +46,7 @@ type ServiceIng struct { PassHostHeader *bool `json:"passHostHeader"` Sticky *dynamic.Sticky `json:"sticky,omitempty" label:"allowEmpty"` NativeLB bool `json:"nativeLB,omitempty"` + NodePortLB bool `json:"nodePortLB,omitempty"` } // SetDefaults sets the default values. diff --git a/pkg/provider/kubernetes/ingress/client.go b/pkg/provider/kubernetes/ingress/client.go index c29f3c6b30..cccc3ce95b 100644 --- a/pkg/provider/kubernetes/ingress/client.go +++ b/pkg/provider/kubernetes/ingress/client.go @@ -39,12 +39,14 @@ type Client interface { GetIngressClasses() ([]*netv1.IngressClass, error) GetService(namespace, name string) (*corev1.Service, bool, error) GetSecret(namespace, name string) (*corev1.Secret, bool, error) + GetNodes() ([]*corev1.Node, bool, error) GetEndpoints(namespace, name string) (*corev1.Endpoints, bool, error) UpdateIngressStatus(ing *netv1.Ingress, ingStatus []netv1.IngressLoadBalancerIngress) error } type clientWrapper struct { clientset kclientset.Interface + factoryClusterScope kinformers.SharedInformerFactory factoriesKube map[string]kinformers.SharedInformerFactory factoriesSecret map[string]kinformers.SharedInformerFactory factoriesIngress map[string]kinformers.SharedInformerFactory @@ -196,11 +198,18 @@ func (c *clientWrapper) WatchAll(namespaces []string, stopCh <-chan struct{}) (< c.factoriesSecret[ns] = factorySecret } + c.factoryClusterScope = kinformers.NewSharedInformerFactory(c.clientset, resyncPeriod) + _, err = c.factoryClusterScope.Core().V1().Nodes().Informer().AddEventHandler(eventHandler) + if err != nil { + return nil, err + } + for _, ns := range namespaces { c.factoriesIngress[ns].Start(stopCh) c.factoriesKube[ns].Start(stopCh) c.factoriesSecret[ns].Start(stopCh) } + c.factoryClusterScope.Start(stopCh) for _, ns := range namespaces { for typ, ok := range c.factoriesIngress[ns].WaitForCacheSync(stopCh) { @@ -222,6 +231,12 @@ func (c *clientWrapper) WatchAll(namespaces []string, stopCh <-chan struct{}) (< } } + for t, ok := range c.factoryClusterScope.WaitForCacheSync(stopCh) { + if !ok { + return nil, fmt.Errorf("timed out waiting for controller caches to sync %s", t.String()) + } + } + if !c.disableIngressClassInformer { c.clusterFactory = kinformers.NewSharedInformerFactoryWithOptions(c.clientset, resyncPeriod) @@ -346,6 +361,12 @@ func (c *clientWrapper) GetSecret(namespace, name string) (*corev1.Secret, bool, return secret, exist, err } +func (c *clientWrapper) GetNodes() ([]*corev1.Node, bool, error) { + nodes, err := c.factoryClusterScope.Core().V1().Nodes().Lister().List(labels.Everything()) + exist, err := translateNotFoundError(err) + return nodes, exist, err +} + func (c *clientWrapper) GetIngressClasses() ([]*netv1.IngressClass, error) { if c.clusterFactory == nil { return nil, errors.New("cluster factory not loaded") diff --git a/pkg/provider/kubernetes/ingress/client_mock_test.go b/pkg/provider/kubernetes/ingress/client_mock_test.go index 6f565b25cd..02b8ec4c0c 100644 --- a/pkg/provider/kubernetes/ingress/client_mock_test.go +++ b/pkg/provider/kubernetes/ingress/client_mock_test.go @@ -16,11 +16,13 @@ type clientMock struct { services []*corev1.Service secrets []*corev1.Secret endpoints []*corev1.Endpoints + nodes []*corev1.Node ingressClasses []*netv1.IngressClass apiServiceError error apiSecretError error apiEndpointsError error + apiNodesError error apiIngressStatusError error watchChan chan interface{} @@ -43,6 +45,8 @@ func newClientMock(path string) clientMock { c.secrets = append(c.secrets, o) case *corev1.Endpoints: c.endpoints = append(c.endpoints, o) + case *corev1.Node: + c.nodes = append(c.nodes, o) case *netv1.Ingress: c.ingresses = append(c.ingresses, o) case *netv1.IngressClass: @@ -86,6 +90,14 @@ func (c clientMock) GetEndpoints(namespace, name string) (*corev1.Endpoints, boo return &corev1.Endpoints{}, false, nil } +func (c clientMock) GetNodes() ([]*corev1.Node, bool, error) { + if c.apiNodesError != nil { + return nil, false, c.apiNodesError + } + + return c.nodes, true, nil +} + func (c clientMock) GetSecret(namespace, name string) (*corev1.Secret, bool, error) { if c.apiSecretError != nil { return nil, false, c.apiSecretError diff --git a/pkg/provider/kubernetes/ingress/fixtures/Ingress-with-node-port-lb.yml b/pkg/provider/kubernetes/ingress/fixtures/Ingress-with-node-port-lb.yml new file mode 100644 index 0000000000..46e360bf6a --- /dev/null +++ b/pkg/provider/kubernetes/ingress/fixtures/Ingress-with-node-port-lb.yml @@ -0,0 +1,45 @@ +kind: Ingress +apiVersion: networking.k8s.io/v1 +metadata: + name: "" + namespace: testing + +spec: + rules: + - host: traefik.tchouk + http: + paths: + - path: /bar + backend: + service: + name: service1 + port: + number: 8080 + pathType: Prefix + +--- +kind: Service +apiVersion: v1 +metadata: + name: service1 + namespace: testing + annotations: + traefik.ingress.kubernetes.io/service.nodeportlb: "true" + +spec: + ports: + - port: 8080 + nodePort: 32456 + clusterIP: 10.0.0.1 + type: NodePort + externalName: traefik.wtf + +--- +kind: Node +apiVersion: v1 +metadata: + name: traefik-node +status: + addresses: + - type: InternalIP + address: 172.16.4.4 diff --git a/pkg/provider/kubernetes/ingress/kubernetes.go b/pkg/provider/kubernetes/ingress/kubernetes.go index 8ccbf4f58e..4d50749f46 100644 --- a/pkg/provider/kubernetes/ingress/kubernetes.go +++ b/pkg/provider/kubernetes/ingress/kubernetes.go @@ -599,6 +599,41 @@ func (p *Provider) loadService(client Client, namespace string, backend netv1.In return svc, nil } + + if svcConfig.Service.NodePortLB && service.Spec.Type == corev1.ServiceTypeNodePort { + nodes, nodesExists, nodesErr := client.GetNodes() + if nodesErr != nil { + return nil, nodesErr + } + + if !nodesExists || len(nodes) == 0 { + return nil, fmt.Errorf("nodes not found in namespace %s", namespace) + } + + protocol := getProtocol(portSpec, portSpec.Name, svcConfig) + + var servers []dynamic.Server + + for _, node := range nodes { + for _, addr := range node.Status.Addresses { + if addr.Type == corev1.NodeInternalIP { + hostPort := net.JoinHostPort(addr.Address, strconv.Itoa(int(portSpec.NodePort))) + + servers = append(servers, dynamic.Server{ + URL: fmt.Sprintf("%s://%s", protocol, hostPort), + }) + } + } + } + + if len(servers) == 0 { + return nil, fmt.Errorf("no servers were generated for service %s in namespace", backend.Service.Name) + } + + svc.LoadBalancer.Servers = servers + + return svc, nil + } } if service.Spec.Type == corev1.ServiceTypeExternalName { diff --git a/pkg/provider/kubernetes/ingress/kubernetes_test.go b/pkg/provider/kubernetes/ingress/kubernetes_test.go index cffccbbcbf..7394177383 100644 --- a/pkg/provider/kubernetes/ingress/kubernetes_test.go +++ b/pkg/provider/kubernetes/ingress/kubernetes_test.go @@ -1694,6 +1694,58 @@ func TestLoadConfigurationFromIngressesWithNativeLB(t *testing.T) { } } +func TestLoadConfigurationFromIngressesWithNodePortLB(t *testing.T) { + testCases := []struct { + desc string + ingressClass string + expected *dynamic.Configuration + }{ + { + desc: "Ingress with node port lb", + expected: &dynamic.Configuration{ + TCP: &dynamic.TCPConfiguration{}, + HTTP: &dynamic.HTTPConfiguration{ + Middlewares: map[string]*dynamic.Middleware{}, + Routers: map[string]*dynamic.Router{ + "testing-traefik-tchouk-bar": { + Rule: "Host(`traefik.tchouk`) && PathPrefix(`/bar`)", + Service: "testing-service1-8080", + }, + }, + Services: map[string]*dynamic.Service{ + "testing-service1-8080": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + ResponseForwarding: &dynamic.ResponseForwarding{FlushInterval: dynamic.DefaultFlushInterval}, + PassHostHeader: Bool(true), + Servers: []dynamic.Server{ + { + URL: "http://172.16.4.4:32456", + }, + }, + }, + }, + }, + }, + }, + }, + } + + for _, test := range testCases { + test := test + + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + clientMock := newClientMock(generateTestFilename(test.desc)) + + p := Provider{IngressClass: test.ingressClass} + conf := p.loadConfigurationFromIngresses(context.Background(), clientMock) + + assert.Equal(t, test.expected, conf) + }) + } +} + func generateTestFilename(desc string) string { return filepath.Join("fixtures", strings.ReplaceAll(desc, " ", "-")+".yml") } diff --git a/pkg/provider/kubernetes/k8s/parser.go b/pkg/provider/kubernetes/k8s/parser.go index ddcd69b06c..1f1a0d0e91 100644 --- a/pkg/provider/kubernetes/k8s/parser.go +++ b/pkg/provider/kubernetes/k8s/parser.go @@ -12,7 +12,7 @@ import ( // MustParseYaml parses a YAML to objects. func MustParseYaml(content []byte) []runtime.Object { - acceptedK8sTypes := regexp.MustCompile(`^(Namespace|Deployment|Endpoints|Service|Ingress|IngressRoute|IngressRouteTCP|IngressRouteUDP|Middleware|MiddlewareTCP|Secret|TLSOption|TLSStore|TraefikService|IngressClass|ServersTransport|ServersTransportTCP|GatewayClass|Gateway|HTTPRoute|TCPRoute|TLSRoute|ReferenceGrant)$`) + acceptedK8sTypes := regexp.MustCompile(`^(Namespace|Deployment|Endpoints|Node|Service|Ingress|IngressRoute|IngressRouteTCP|IngressRouteUDP|Middleware|MiddlewareTCP|Secret|TLSOption|TLSStore|TraefikService|IngressClass|ServersTransport|ServersTransportTCP|GatewayClass|Gateway|HTTPRoute|TCPRoute|TLSRoute|ReferenceGrant)$`) files := strings.Split(string(content), "---\n") retVal := make([]runtime.Object, 0, len(files))