calico: cilium: use localhost lb by default on kube-proxy replacement (…

…kubernetes-sigs#9718)

This commit removes the variable `use_localhost_as_kubeapi_loadbalancer`
and rather detects that we are in a situation where we can use the
localhost apiserver loadbalancer (meaning that we use the localhost load
balancer and that the same ports are used for both the load balancer and
the kube-apiserver).

This also cleanups the calico code to use `kube_apiserver_global_endpoint`
rather than implementing the same logic all over again.

Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>

docs/calico.md

@@ -376,12 +376,7 @@ Calico node, typha and kube-controllers need to be able to talk to the kubernete
 
 Kubespray sets up the `kubernetes-services-endpoint` configmap based on the contents of the `loadbalancer_apiserver` inventory variable documented in [HA Mode](/docs/ha-mode.md).
 
-If no external loadbalancer is used, Calico eBPF can also use the localhost loadbalancer option. In this case Calico Automatic Host Endpoints need to be enabled to allow services like `coredns` and `metrics-server` to communicate with the kubernetes host endpoint. See [this blog post](https://www.projectcalico.org/securing-kubernetes-nodes-with-calico-automatic-host-endpoints/) on enabling automatic host endpoints.
-
-```yaml
-loadbalancer_apiserver_localhost: true
-use_localhost_as_kubeapi_loadbalancer: true
-```
+If no external loadbalancer is used, Calico eBPF can also use the localhost loadbalancer option. We are able to do so only if you use the same port for the localhost apiserver loadbalancer and the kube-apiserver. In this case Calico Automatic Host Endpoints need to be enabled to allow services like `coredns` and `metrics-server` to communicate with the kubernetes host endpoint. See [this blog post](https://www.projectcalico.org/securing-kubernetes-nodes-with-calico-automatic-host-endpoints/) on enabling automatic host endpoints.
 
 ### Tunneled versus Direct Server Return
 

docs/cilium.md

@@ -6,11 +6,9 @@ Cilium can run without kube-proxy by setting `cilium_kube_proxy_replacement`
 to `strict`.
 
 Without kube-proxy, cilium needs to know the address of the kube-apiserver
-and this must be set globally for all cilium components (agents and operators).
-Hence, in this configuration in Kubespray, Cilium will always contact
-the external loadbalancer (even from a node in the control plane)
-and if there is no external load balancer It will ignore any local load
-balancer deployed by Kubespray and **only contacts the first master**.
+and this must be set globally for all Cilium components (agents and operators).
+We can only use the localhost apiserver loadbalancer in this mode
+whenever it uses the same port as the kube-apiserver (by default it does).
 
 ## Cilium Operator
 

inventory/sample/group_vars/all/all.yml

@@ -21,11 +21,6 @@ bin_dir: /usr/local/bin
 # valid options are "nginx" or "haproxy"
 # loadbalancer_apiserver_type: nginx  # valid values "nginx" or "haproxy"
 
-## If the cilium is going to be used in strict mode, we can use the
-## localhost connection and not use the external LB. If this parameter is
-## not specified, the first node to connect to kubeapi will be used.
-# use_localhost_as_kubeapi_loadbalancer: true
-
 ## Local loadbalancer should use this port
 ## And must be set port 6443
 loadbalancer_apiserver_port: 6443

roles/kubespray-defaults/defaults/main.yaml

@@ -547,24 +547,24 @@ loadbalancer_apiserver_localhost: "{{ loadbalancer_apiserver is not defined }}"
 loadbalancer_apiserver_type: "nginx"
 # applied if only external loadbalancer_apiserver is defined, otherwise ignored
 apiserver_loadbalancer_domain_name: "lb-apiserver.kubernetes.local"
-# kube_apiserver_global_endpoint: |-
-#   {% if loadbalancer_apiserver is defined -%}
-#       https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port|default(kube_apiserver_port) }}
-#   {%- elif use_localhost_as_kubeapi_loadbalancer|default(False)|bool -%}
-#       https://127.0.0.1:{{ kube_apiserver_port }}
-#   {%- else -%}
-#       https://{{ first_kube_control_plane_address }}:{{ kube_apiserver_port }}
-#   {%- endif %}
-# kube_apiserver_endpoint: |-
-#   {% if loadbalancer_apiserver is defined -%}
-#       https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port|default(kube_apiserver_port) }}
-#   {%- elif not is_kube_master and loadbalancer_apiserver_localhost -%}
-#       https://localhost:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }}
-#   {%- elif is_kube_master -%}
-#       https://{{ kube_apiserver_bind_address | regex_replace('0\.0\.0\.0','127.0.0.1') }}:{{ kube_apiserver_port }}
-#   {%- else -%}
-#       https://{{ first_kube_control_plane_address }}:{{ kube_apiserver_port }}
-#   {%- endif %}
+kube_apiserver_global_endpoint: |-
+  {% if loadbalancer_apiserver is defined -%}
+      https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port|default(kube_apiserver_port) }}
+  {%- elif loadbalancer_apiserver_localhost and (loadbalancer_apiserver_port is not defined or loadbalancer_apiserver_port == kube_apiserver_port) -%}
+      https://localhost:{{ kube_apiserver_port }}
+  {%- else -%}
+      https://{{ first_kube_control_plane_address }}:{{ kube_apiserver_port }}
+  {%- endif %}
+kube_apiserver_endpoint: |-
+  {% if loadbalancer_apiserver is defined -%}
+      https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port|default(kube_apiserver_port) }}
+  {%- elif not is_kube_master and loadbalancer_apiserver_localhost -%}
+      https://localhost:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }}
+  {%- elif is_kube_master -%}
+      https://{{ kube_apiserver_bind_address | regex_replace('0\.0\.0\.0','127.0.0.1') }}:{{ kube_apiserver_port }}
+  {%- else -%}
+      https://{{ first_kube_control_plane_address }}:{{ kube_apiserver_port }}
+  {%- endif %}
 kube_apiserver_client_cert: "{{ kube_cert_dir }}/ca.crt"
 kube_apiserver_client_key: "{{ kube_cert_dir }}/ca.key"
 

roles/network_plugin/calico/templates/kubernetes-services-endpoint.yml.j2

@@ -6,14 +6,6 @@ metadata:
   name: kubernetes-services-endpoint
 data:
 {% if calico_bpf_enabled %}
-{% if loadbalancer_apiserver is defined %}
-  KUBERNETES_SERVICE_HOST: "{{ apiserver_loadbalancer_domain_name }}"
-  KUBERNETES_SERVICE_PORT: "{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}"
-{%- elif use_localhost_as_kubeapi_loadbalancer|default(False)|bool %}
-  KUBERNETES_SERVICE_HOST: "127.0.0.1"
-  KUBERNETES_SERVICE_PORT: "{{ kube_apiserver_port }}"
-{%- else %}
-  KUBERNETES_SERVICE_HOST: "{{ first_kube_control_plane_address }}"
-  KUBERNETES_SERVICE_PORT: "{{ kube_apiserver_port }}"
-{%- endif %}
+  KUBERNETES_SERVICE_HOST: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}"
+  KUBERNETES_SERVICE_PORT: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}"
 {% endif %}

tests/files/packet_almalinux8-calico-ha-ebpf.yml

@@ -7,5 +7,4 @@ vm_memory: 3072Mi
 # Kubespray settings
 calico_bpf_enabled: true
 loadbalancer_apiserver_localhost: true
-use_localhost_as_kubeapi_loadbalancer: true
 auto_renew_certificates: true