Merge pull request #8593 from AMacedoP/handleSingleIPLBRR

[release-v3.27] Cherry pick #8358: Fix announcement of Single LB IPs with Route Reflectors.
projectcalico · Mar 7, 2024 · 414b586 · 414b586
2 parents bb66bf8 + 5f8c2dd
commit 414b586
Show file tree

Hide file tree

Showing 7 changed files with 250 additions and 2 deletions.
diff --git a/confd/etc/calico/confd/conf.d/bird6_ipam.toml b/confd/etc/calico/confd/conf.d/bird6_ipam.toml
@@ -4,6 +4,8 @@ dest = "/etc/calico/confd/config/bird6_ipam.cfg"
 prefix = "/calico"
 keys = [
     "/v1/ipam/v6/pool",
+    "/bgp/v1/host//NODENAME",
+    "/bgp/v1/global/svc_loadbalancer_ips",
     "/staticroutesv6",
     "/rejectcidrsv6",
 ]

diff --git a/confd/etc/calico/confd/conf.d/bird_ipam.toml b/confd/etc/calico/confd/conf.d/bird_ipam.toml
@@ -5,6 +5,7 @@ prefix = "/calico"
 keys = [
     "/v1/ipam/v4/pool",
     "/bgp/v1/host//NODENAME",
+    "/bgp/v1/global/svc_loadbalancer_ips",
     "/staticroutes",
     "/rejectcidrs",
 ]

diff --git a/confd/etc/calico/confd/templates/bird6_ipam.cfg.template b/confd/etc/calico/confd/templates/bird6_ipam.cfg.template
@@ -38,6 +38,20 @@ function calico_export_to_bgp_peers(bool internal_peer) {
   if ( net ~ {{$cidr}} ) then { accept; }
   {{- end}}
 {{- end}}
+{{- $rr_cluster_id_key := printf "/bgp/v1/host/%s/rr_cluster_id" (getenv "NODENAME")}}
+{{- $lb_ips := "/bgp/v1/global/svc_loadbalancer_ips"}}
+{{- if exists $rr_cluster_id_key}}{{$rr_cluster_id := getv $rr_cluster_id_key}}
+{{- if and (not (eq $rr_cluster_id "")) (exists $lb_ips)}}
+
+  # Configured as a RR - accept any routes within configured LB service IP ranges
+  {{- range split (getv $lb_ips) ","}}
+    {{- $cidr := .}}
+    {{- if contains $cidr ":"}}
+  if ( net ~ {{$cidr}} ) then { accept; }
+    {{- end}}
+  {{- end}}
+{{- end}}
+{{- end}}
 {{range ls "/v1/ipam/v6/pool"}}{{$data := json (getv (printf "/v1/ipam/v6/pool/%s" .))}}
 {{- if $data.disableBGPExport}}
   # Skip {{$data.cidr}} as BGP export is disabled for it

diff --git a/confd/etc/calico/confd/templates/bird_ipam.cfg.template b/confd/etc/calico/confd/templates/bird_ipam.cfg.template
@@ -38,6 +38,20 @@ function calico_export_to_bgp_peers(bool internal_peer) {
   if ( net ~ {{$cidr}} ) then { accept; }
   {{- end}}
 {{- end}}
+{{- $rr_cluster_id_key := printf "/bgp/v1/host/%s/rr_cluster_id" (getenv "NODENAME")}}
+{{- $lb_ips := "/bgp/v1/global/svc_loadbalancer_ips"}}
+{{- if exists $rr_cluster_id_key}}{{$rr_cluster_id := getv $rr_cluster_id_key}}
+{{- if and (not (eq $rr_cluster_id "")) (exists $lb_ips)}}
+
+  # Configured as a RR - accept any routes within configured LB service IP ranges
+  {{- range split (getv $lb_ips) ","}}
+    {{- $cidr := .}}
+    {{- if not (contains $cidr ":")}}
+  if ( net ~ {{$cidr}} ) then { accept; }
+    {{- end}}
+  {{- end}}
+{{- end}}
+{{- end}}
 {{range ls "/v1/ipam/v4/pool"}}{{$data := json (getv (printf "/v1/ipam/v4/pool/%s" .))}}
 {{- if $data.disableBGPExport}}
   # Skip {{$data.cidr}} as BGP export is disabled for it

diff --git a/node/tests/k8st/infra/metallb-config.yaml b/node/tests/k8st/infra/metallb-config.yaml
@@ -10,3 +10,4 @@ data:
       protocol: bgp
       addresses:
       - 80.15.0.0/24
+      - fdff::/64
diff --git a/node/tests/k8st/tests/test_bgp_advert.py b/node/tests/k8st/tests/test_bgp_advert.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2018 Tigera, Inc. All rights reserved.
+# Copyright (c) 2018-2024 Tigera, Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -905,3 +905,110 @@ def test_rr(self):
         external_ip = svc_dict['spec']['externalIPs'][0]
         retry_until_success(lambda: self.assertIn(cluster_ip, self.get_routes()))
         retry_until_success(lambda: self.assertIn(external_ip, self.get_routes()))
+
+    def test_single_ip_lb_rr(self):
+        """
+        Tests a /32 LB service with externalTrafficPolicy=Local using a RR
+        """
+        # Create an LB ExternalTrafficPolicy Local service with one endpoint
+        # on node-1
+        kubectl("""apply -f - << EOF
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-rr
+  namespace: bgp-test
+  labels:
+    app: nginx
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: nginx
+      run: nginx-rr
+  template:
+    metadata:
+      labels:
+        app: nginx
+        run: nginx-rr
+    spec:
+      containers:
+      - name: nginx-rr
+        image: %s
+        ports:
+        - containerPort: 80
+      nodeSelector:
+        kubernetes.io/os: linux
+        kubernetes.io/hostname: %s
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx-rr
+  namespace: bgp-test
+  labels:
+    app: nginx
+    run: nginx-rr
+  annotations:
+    metallb.universe.tf/loadBalancerIPs: 80.15.0.100
+spec:
+  ports:
+  - port: 80
+    targetPort: 80
+  selector:
+    app: nginx
+    run: nginx-rr
+  type: LoadBalancer
+  loadBalancerIP: 80.15.0.100
+  externalTrafficPolicy: Local
+EOF
+""" % (NGINX_IMAGE, self.nodes[1]))
+
+        calicoctl("get nodes -o yaml")
+        calicoctl("get bgppeers -o yaml")
+        calicoctl("get bgpconfigs -o yaml")
+
+        # Update the node-2 to behave as a route-reflector
+        json_str = calicoctl("get node %s -o json" % self.nodes[2])
+        node_dict = json.loads(json_str)
+        node_dict['metadata']['labels']['i-am-a-route-reflector'] = 'true'
+        node_dict['spec']['bgp']['routeReflectorClusterID'] = '224.0.0.1'
+        calicoctl("""apply -f - << EOF
+%s
+EOF
+""" % json.dumps(node_dict))
+
+        # Disable node-to-node mesh, add cluster and external IP CIDRs to
+        # advertise, and configure BGP peering between the cluster nodes and the
+        # RR.  (The BGP peering from the external node to the RR is included in
+        # bird_conf_rr above.)
+        calicoctl("""apply -f - << EOF
+apiVersion: projectcalico.org/v3
+kind: BGPConfiguration
+metadata:
+  name: default
+spec:
+  nodeToNodeMeshEnabled: false
+  asNumber: 64512
+  serviceClusterIPs:
+  - cidr: 10.96.0.0/12
+  serviceLoadBalancerIPs:
+  - cidr: 80.15.0.100/32
+EOF
+""")
+
+        calicoctl("""apply -f - << EOF
+apiVersion: projectcalico.org/v3
+kind: BGPPeer
+metadata: {name: peer-with-rr}
+spec:
+  peerIP: %s
+  asNumber: 64512
+EOF
+""" % self.ips[2])
+        svc_json = kubectl("get svc nginx-rr -n bgp-test -o json")
+        svc_dict = json.loads(svc_json)
+        cluster_ip = svc_dict['spec']['clusterIP']
+        load_balancer_ip = svc_dict['spec']['loadBalancerIP']
+        retry_until_success(lambda: self.assertIn(cluster_ip, self.get_routes()))
+        retry_until_success(lambda: self.assertIn(load_balancer_ip, self.get_routes()))
diff --git a/node/tests/k8st/tests/test_bgp_advert_v6.py b/node/tests/k8st/tests/test_bgp_advert_v6.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2020 Tigera, Inc. All rights reserved.
+# Copyright (c) 2020-2024 Tigera, Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -640,3 +640,112 @@ def test_rr(self):
         external_ip = svc_dict['spec']['externalIPs'][0]
         retry_until_success(lambda: self.assertIn(cluster_ip, self.get_routes()))
         retry_until_success(lambda: self.assertIn(external_ip, self.get_routes()))
+
+
+    def test_single_ip_lb_rr(self):
+        """
+        Tests a /128 LB service with externalTrafficPolicy=Local using a RR
+        """
+        # Create ExternalTrafficPolicy Local service with one endpoint on node-1
+        kubectl("""apply -f - << EOF
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-rr
+  namespace: bgp-test
+  labels:
+    app: nginx
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: nginx
+      run: nginx-rr
+  template:
+    metadata:
+      labels:
+        app: nginx
+        run: nginx-rr
+    spec:
+      containers:
+      - name: nginx-rr
+        image: %s
+        ports:
+        - containerPort: 80
+      nodeSelector:
+        kubernetes.io/os: linux
+        kubernetes.io/hostname: %s
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx-rr
+  namespace: bgp-test
+  labels:
+    app: nginx
+    run: nginx-rr
+  annotations:
+    metallb.universe.tf/loadBalancerIPs: fdff::96
+spec:
+  ipFamilies:
+  - IPv6
+  ports:
+  - port: 80
+    targetPort: 80
+  selector:
+    app: nginx
+    run: nginx-rr
+  type: LoadBalancer
+  loadBalancerIP: fdff::96
+  externalTrafficPolicy: Local
+EOF
+""" % (NGINX_IMAGE, self.nodes[1]))
+
+        calicoctl("get nodes -o yaml")
+        calicoctl("get bgppeers -o yaml")
+        calicoctl("get bgpconfigs -o yaml")
+
+        # Update the node-2 to behave as a route-reflector
+        json_str = calicoctl("get node %s -o json" % self.nodes[2])
+        node_dict = json.loads(json_str)
+        node_dict['metadata']['labels']['i-am-a-route-reflector'] = 'true'
+        node_dict['spec']['bgp']['routeReflectorClusterID'] = '224.0.0.1'
+        calicoctl("""apply -f - << EOF
+%s
+EOF
+""" % json.dumps(node_dict))
+
+        # Disable node-to-node mesh, add cluster and external IP CIDRs to
+        # advertise, and configure BGP peering between the cluster nodes and the
+        # RR.  (The BGP peering from the external node to the RR is included in
+        # get_bird_conf() above.)
+        calicoctl("""apply -f - << EOF
+apiVersion: projectcalico.org/v3
+kind: BGPConfiguration
+metadata:
+  name: default
+spec:
+  nodeToNodeMeshEnabled: false
+  asNumber: 64512
+  serviceClusterIPs:
+  - cidr: fd00:10:96::/112
+  serviceLoadBalancerIPs:
+  - cidr: fdff::96/128
+EOF
+""")
+
+        calicoctl("""apply -f - << EOF
+apiVersion: projectcalico.org/v3
+kind: BGPPeer
+metadata: {name: peer-with-rr}
+spec:
+  peerIP: %s
+  asNumber: 64512
+EOF
+""" % self.ipv6s[2])
+        svc_json = kubectl("get svc nginx-rr -n bgp-test -o json")
+        svc_dict = json.loads(svc_json)
+        cluster_ip = svc_dict['spec']['clusterIP']
+        load_balancer_ip = svc_dict['spec']['loadBalancerIP']
+        retry_until_success(lambda: self.assertIn(cluster_ip, self.get_routes()))
+        retry_until_success(lambda: self.assertIn(load_balancer_ip, self.get_routes()))