From 68a606d48a38b55966ab0f0c989b78351c8fb42a Mon Sep 17 00:00:00 2001 From: Arthur Outhenin-Chalandre Date: Tue, 7 Mar 2023 13:28:36 +0100 Subject: [PATCH] calico: cilium: use localhost lb by default on kube-proxy replacement (#9718) This commit removes the variable `use_localhost_as_kubeapi_loadbalancer` and rather detects that we are in a situation where we can use the localhost apiserver loadbalancer (meaning that we use the localhost load balancer and that the same ports are used for both the load balancer and the kube-apiserver). This also cleanups the calico code to use `kube_apiserver_global_endpoint` rather than implementing the same logic all over again. Signed-off-by: Arthur Outhenin-Chalandre --- docs/calico.md | 7 +--- docs/cilium.md | 8 ++--- inventory/sample/group_vars/all/all.yml | 5 --- roles/kubespray-defaults/defaults/main.yaml | 36 +++++++++---------- .../kubernetes-services-endpoint.yml.j2 | 12 ++----- .../packet_almalinux8-calico-ha-ebpf.yml | 1 - 6 files changed, 24 insertions(+), 45 deletions(-) diff --git a/docs/calico.md b/docs/calico.md index 433d4ce1b27..937cc05d39a 100644 --- a/docs/calico.md +++ b/docs/calico.md @@ -376,12 +376,7 @@ Calico node, typha and kube-controllers need to be able to talk to the kubernete Kubespray sets up the `kubernetes-services-endpoint` configmap based on the contents of the `loadbalancer_apiserver` inventory variable documented in [HA Mode](/docs/ha-mode.md). -If no external loadbalancer is used, Calico eBPF can also use the localhost loadbalancer option. In this case Calico Automatic Host Endpoints need to be enabled to allow services like `coredns` and `metrics-server` to communicate with the kubernetes host endpoint. See [this blog post](https://www.projectcalico.org/securing-kubernetes-nodes-with-calico-automatic-host-endpoints/) on enabling automatic host endpoints. - -```yaml -loadbalancer_apiserver_localhost: true -use_localhost_as_kubeapi_loadbalancer: true -``` +If no external loadbalancer is used, Calico eBPF can also use the localhost loadbalancer option. We are able to do so only if you use the same port for the localhost apiserver loadbalancer and the kube-apiserver. In this case Calico Automatic Host Endpoints need to be enabled to allow services like `coredns` and `metrics-server` to communicate with the kubernetes host endpoint. See [this blog post](https://www.projectcalico.org/securing-kubernetes-nodes-with-calico-automatic-host-endpoints/) on enabling automatic host endpoints. ### Tunneled versus Direct Server Return diff --git a/docs/cilium.md b/docs/cilium.md index 033ea6a2922..4141e383e60 100644 --- a/docs/cilium.md +++ b/docs/cilium.md @@ -6,11 +6,9 @@ Cilium can run without kube-proxy by setting `cilium_kube_proxy_replacement` to `strict`. Without kube-proxy, cilium needs to know the address of the kube-apiserver -and this must be set globally for all cilium components (agents and operators). -Hence, in this configuration in Kubespray, Cilium will always contact -the external loadbalancer (even from a node in the control plane) -and if there is no external load balancer It will ignore any local load -balancer deployed by Kubespray and **only contacts the first master**. +and this must be set globally for all Cilium components (agents and operators). +We can only use the localhost apiserver loadbalancer in this mode +whenever it uses the same port as the kube-apiserver (by default it does). ## Cilium Operator diff --git a/inventory/sample/group_vars/all/all.yml b/inventory/sample/group_vars/all/all.yml index b9639a8a8df..d8a13dead5f 100644 --- a/inventory/sample/group_vars/all/all.yml +++ b/inventory/sample/group_vars/all/all.yml @@ -21,11 +21,6 @@ bin_dir: /usr/local/bin # valid options are "nginx" or "haproxy" # loadbalancer_apiserver_type: nginx # valid values "nginx" or "haproxy" -## If the cilium is going to be used in strict mode, we can use the -## localhost connection and not use the external LB. If this parameter is -## not specified, the first node to connect to kubeapi will be used. -# use_localhost_as_kubeapi_loadbalancer: true - ## Local loadbalancer should use this port ## And must be set port 6443 loadbalancer_apiserver_port: 6443 diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 00cdecd5236..f3638119c83 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -547,24 +547,24 @@ loadbalancer_apiserver_localhost: "{{ loadbalancer_apiserver is not defined }}" loadbalancer_apiserver_type: "nginx" # applied if only external loadbalancer_apiserver is defined, otherwise ignored apiserver_loadbalancer_domain_name: "lb-apiserver.kubernetes.local" -# kube_apiserver_global_endpoint: |- -# {% if loadbalancer_apiserver is defined -%} -# https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port|default(kube_apiserver_port) }} -# {%- elif use_localhost_as_kubeapi_loadbalancer|default(False)|bool -%} -# https://127.0.0.1:{{ kube_apiserver_port }} -# {%- else -%} -# https://{{ first_kube_control_plane_address }}:{{ kube_apiserver_port }} -# {%- endif %} -# kube_apiserver_endpoint: |- -# {% if loadbalancer_apiserver is defined -%} -# https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port|default(kube_apiserver_port) }} -# {%- elif not is_kube_master and loadbalancer_apiserver_localhost -%} -# https://localhost:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }} -# {%- elif is_kube_master -%} -# https://{{ kube_apiserver_bind_address | regex_replace('0\.0\.0\.0','127.0.0.1') }}:{{ kube_apiserver_port }} -# {%- else -%} -# https://{{ first_kube_control_plane_address }}:{{ kube_apiserver_port }} -# {%- endif %} +kube_apiserver_global_endpoint: |- + {% if loadbalancer_apiserver is defined -%} + https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port|default(kube_apiserver_port) }} + {%- elif loadbalancer_apiserver_localhost and (loadbalancer_apiserver_port is not defined or loadbalancer_apiserver_port == kube_apiserver_port) -%} + https://localhost:{{ kube_apiserver_port }} + {%- else -%} + https://{{ first_kube_control_plane_address }}:{{ kube_apiserver_port }} + {%- endif %} +kube_apiserver_endpoint: |- + {% if loadbalancer_apiserver is defined -%} + https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port|default(kube_apiserver_port) }} + {%- elif not is_kube_master and loadbalancer_apiserver_localhost -%} + https://localhost:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }} + {%- elif is_kube_master -%} + https://{{ kube_apiserver_bind_address | regex_replace('0\.0\.0\.0','127.0.0.1') }}:{{ kube_apiserver_port }} + {%- else -%} + https://{{ first_kube_control_plane_address }}:{{ kube_apiserver_port }} + {%- endif %} kube_apiserver_client_cert: "{{ kube_cert_dir }}/ca.crt" kube_apiserver_client_key: "{{ kube_cert_dir }}/ca.key" diff --git a/roles/network_plugin/calico/templates/kubernetes-services-endpoint.yml.j2 b/roles/network_plugin/calico/templates/kubernetes-services-endpoint.yml.j2 index 7ececd4f5a5..f1e81776edb 100644 --- a/roles/network_plugin/calico/templates/kubernetes-services-endpoint.yml.j2 +++ b/roles/network_plugin/calico/templates/kubernetes-services-endpoint.yml.j2 @@ -6,14 +6,6 @@ metadata: name: kubernetes-services-endpoint data: {% if calico_bpf_enabled %} -{% if loadbalancer_apiserver is defined %} - KUBERNETES_SERVICE_HOST: "{{ apiserver_loadbalancer_domain_name }}" - KUBERNETES_SERVICE_PORT: "{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}" -{%- elif use_localhost_as_kubeapi_loadbalancer|default(False)|bool %} - KUBERNETES_SERVICE_HOST: "127.0.0.1" - KUBERNETES_SERVICE_PORT: "{{ kube_apiserver_port }}" -{%- else %} - KUBERNETES_SERVICE_HOST: "{{ first_kube_control_plane_address }}" - KUBERNETES_SERVICE_PORT: "{{ kube_apiserver_port }}" -{%- endif %} + KUBERNETES_SERVICE_HOST: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}" + KUBERNETES_SERVICE_PORT: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}" {% endif %} diff --git a/tests/files/packet_almalinux8-calico-ha-ebpf.yml b/tests/files/packet_almalinux8-calico-ha-ebpf.yml index ee443f1d068..e4f4bb65833 100644 --- a/tests/files/packet_almalinux8-calico-ha-ebpf.yml +++ b/tests/files/packet_almalinux8-calico-ha-ebpf.yml @@ -7,5 +7,4 @@ vm_memory: 3072Mi # Kubespray settings calico_bpf_enabled: true loadbalancer_apiserver_localhost: true -use_localhost_as_kubeapi_loadbalancer: true auto_renew_certificates: true