feature: make the node's label "node.kubernetes.io/exclude-from-exter…

…nal-load-balancers" valuable

The label specifies that the node should not be considered as a target for external load-balancers which use nodes as a second hop.

Signed-off-by: cyclinder <qifeng.guo@daocloud.io>

e2etest/pkg/frr/validation.go

@@ -60,9 +60,23 @@ func RoutesMatchNodes(nodes []v1.Node, route Route, ipFamily ipfamily.Family, vr
 
 		delete(nodesIPs, h.String())
 	}
+
+	excludesNodeIPs, err := k8s.NodeIPsForExcludeBalancers(nodes, ipFamily, vrfName)
+	if err != nil {
+		return err
+	}
+
+	// If the nodeIPs include IPs of nodes that should be excluded, we should ignore them
+	for _, enIP := range excludesNodeIPs {
+		if _, ok := nodesIPs[enIP]; ok {
+			delete(nodesIPs, enIP)
+		}
+	}
+
 	if len(nodesIPs) != 0 { // some leftover, meaning more nodes than routes
 		return fmt.Errorf("IP %v found in nodes but not in next hops", nodesIPs)
 	}
+
 	return nil
 }
 

e2etest/pkg/k8s/nodes.go

@@ -59,6 +59,51 @@ func NodeIPsForFamily(nodes []v1.Node, family ipfamily.Family, vrfName string) (
 	return res, nil
 }
 
+// NodeIPsForExcludeBalancers return the IP addresses of all nodes with the label
+// 'node.kubernetes.io/exclude-from-external-load-balancers'
+func NodeIPsForExcludeBalancers(nodes []v1.Node, family ipfamily.Family, vrfName string) ([]string, error) {
+	res := []string{}
+	for _, n := range nodes {
+		// If a node has label 'node.kubernetes.io/exclude-from-external-load-balancers',
+		// we should not consider it as the next hop for the LoadBalancer service.
+		if _, ok := n.Labels[corev1.LabelNodeExcludeBalancers]; ok {
+			// If the peer is supposed to be connected via a VRF, taking the node address is not enough.
+			// We need to find the ip associated to the interface inside the VRF
+			if vrfName != "" {
+				exec := executor.ForContainer(n.Name)
+				dev, err := netdev.WithMaster(exec, vrfName)
+				if err != nil {
+					return nil, err
+				}
+				addr, err := netdev.AddressesForDevice(exec, dev)
+				if err != nil {
+					return nil, err
+				}
+				switch family {
+				case ipfamily.IPv4:
+					res = append(res, addr.IPV4Address)
+				case ipfamily.IPv6:
+					res = append(res, addr.IPV6Address)
+				case ipfamily.DualStack:
+					res = append(res, addr.IPV4Address)
+					res = append(res, addr.IPV6Address)
+				}
+				continue
+			}
+
+			for _, a := range n.Status.Addresses {
+				if a.Type == v1.NodeInternalIP {
+					if family != ipfamily.DualStack && ipfamily.ForAddress(net.ParseIP(a.Address)) != family {
+						continue
+					}
+					res = append(res, a.Address)
+				}
+			}
+		}
+	}
+	return res, nil
+}
+
 func SelectorsForNodes(nodes []v1.Node) []metav1.LabelSelector {
 	selectors := []metav1.LabelSelector{}
 	if len(nodes) == 0 {

e2etest/pkg/routes/routes.go

@@ -9,8 +9,8 @@ import (
 	"strings"
 
 	"go.universe.tf/e2etest/pkg/executor"
-	"go.universe.tf/e2etest/pkg/k8s"
 	"go.universe.tf/e2etest/pkg/ipfamily"
+	"go.universe.tf/e2etest/pkg/k8s"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/kubernetes/test/e2e/framework"
 )
@@ -86,6 +86,19 @@ func MatchNodes(nodes []v1.Node, ips []net.IP, ipFamily ipfamily.Family, vrfName
 		}
 		delete(nodesIPs, ip.String())
 	}
+
+	excludesNodeIPs, err := k8s.NodeIPsForExcludeBalancers(nodes, ipFamily, vrfName)
+	if err != nil {
+		return err
+	}
+
+	// If the nodeIPs include IPs of nodes that should be excluded, we should ignore them
+	for _, enIP := range excludesNodeIPs {
+		if _, ok := nodesIPs[enIP]; ok {
+			delete(nodesIPs, enIP)
+		}
+	}
+
 	if len(nodesIPs) != 0 { // some leftover, meaning more nodes than routes
 		return fmt.Errorf("IP %v found in nodes but not in routes. Routes %v", nodesIPs, ips)
 	}

internal/config/config.go

@@ -1140,6 +1140,11 @@ func selectedNodes(nodes []corev1.Node, selectors []metav1.LabelSelector) (map[s
 	res := make(map[string]bool)
 OUTER:
 	for _, node := range nodes {
+		_, ok := node.Labels[corev1.LabelNodeExcludeBalancers]
+		if ok {
+			continue
+		}
+
 		if len(labelSelectors) == 0 { // no selector mean all nodes are valid
 			res[node.Name] = true
 		}

website/content/release-notes/_index.md

@@ -9,6 +9,7 @@ Chores:
 
 - Enforce adding a release notes entry on each PR ([PR 2191](https://github.com/metallb/metallb/pull/2191))
 - Dev-env: don't run tests against VRFs by default ([PR 2201](https://github.com/metallb/metallb/pull/2201), [ISSUE 2197](https://github.com/metallb/metallb/issues/2197))
+- make the label "node.kubernetes.io/exclude-from-external-load-balancers" valuable ([PR 2073](https://github.com/metallb/metallb/pull/2073), [ISSUE 2021](https://github.com/metallb/metallb/issues/2021))
 
 ## Version 0.13.12
 
@@ -238,7 +239,7 @@ Bug Fixes:
 
 - A race condition happening when the speaker container was slower than the frr one was fixed ([PR 1463](https://github.com/metallb/metallb/pull/1463))
 
-This release includes contributions from Andrea Panattoni, Carlos Goncalves, Federico Paolinelli, jay vyas, Joshua Carnes, liornoy, Mani Kanth, manu, Mateusz Gozdek, Mathieu Parent, Matt Layher, mkeppel@solvinity.com, Mohamed Mahmoud, Ori Braunshtein, Periyasamy Palanisamy, Rodrigo Campos, Sabina Aledort, Scott Laird, Stefan Coetzee, Tyler Auerbeck, zhoujiao. Thank you!
+This release includes contributions from Andrea Panattoni, Carlos Goncalves, Federico Paolinelli, jay vyas, Joshua Carnes, liornoy, Mani Kanth, manu, Mateusz Gozdek, Mathieu Parent, Matt Layher, <mkeppel@solvinity.com>, Mohamed Mahmoud, Ori Braunshtein, Periyasamy Palanisamy, Rodrigo Campos, Sabina Aledort, Scott Laird, Stefan Coetzee, Tyler Auerbeck, zhoujiao. Thank you!
 
 ## Version 0.12.1