Update L3 gateway address dynamically

When a node gets a new IP addresses, node loadbalancers
should be updated to serve NodePort services on the new
address.

Update `k8s.ovn.org/l3-gateway-config` IPAddresses field
in addressManager.

Add End2End test to control-plane suite.

Signed-off-by: Andrea Panattoni <apanatto@redhat.com>

go-controller/pkg/node/node_ip_handler_linux.go

@@ -4,6 +4,7 @@
 package node
 
 import (
+	"fmt"
 	"net"
 	"sync"
 	"time"
@@ -154,11 +155,9 @@ func (c *addressManager) Run(stopChan <-chan struct{}, doneWg *sync.WaitGroup) {
 				}
 
 				if addrChanged || !c.doesNodeHostAddressesMatch() {
-					if err := util.SetNodeHostAddresses(c.nodeAnnotator, c.addresses); err != nil {
-						klog.Errorf("Failed to set node annotations: %v", err)
-						continue
-					}
-					if err := c.nodeAnnotator.Run(); err != nil {
+					klog.Infof("Changing addresses")
+					err := c.setAddressesOnNodeAnnotations()
+					if err != nil {
 						klog.Errorf("Failed to set node annotations: %v", err)
 					}
 					c.OnChanged()
@@ -266,10 +265,52 @@ func (c *addressManager) sync() {
 	addrChanged := c.assignAddresses(currAddresses)
 	if addrChanged || !c.doesNodeHostAddressesMatch() {
 		klog.Infof("Node address annotation being set to: %v addrChanged: %v", currAddresses, addrChanged)
-		if err := util.SetNodeHostAddresses(c.nodeAnnotator, c.addresses); err != nil {
-			klog.Errorf("Failed to set node annotations: %v", err)
-		} else if err := c.nodeAnnotator.Run(); err != nil {
+		err := c.setAddressesOnNodeAnnotations()
+		if err != nil {
 			klog.Errorf("Failed to set node annotations: %v", err)
 		}
 	}
 }
+
+func (c *addressManager) setAddressesOnNodeAnnotations() error {
+	err := util.SetNodeHostAddresses(c.nodeAnnotator, c.addresses)
+	if err != nil {
+		return fmt.Errorf("can't set host address annotation: %w", err)
+	}
+
+	node, err := c.watchFactory.GetNode(c.nodeName)
+	if err != nil {
+		return fmt.Errorf("can't get node: %w", err)
+	}
+	l3GatewayConfig, err := util.ParseNodeL3GatewayAnnotation(node)
+	if err != nil {
+		return fmt.Errorf("failed to parse l3 gateway annotation for node: %w", err)
+	}
+
+	l3GatewayConfig.IPAddresses = []*net.IPNet{}
+	for _, address := range c.addresses.List() {
+		ip := net.ParseIP(address)
+		if ip == nil {
+			return fmt.Errorf("bad IP address: %s", address)
+		}
+		m := net.CIDRMask(net.IPv6len, net.IPv6len)
+		if ip.To4() != nil {
+			m = net.CIDRMask(net.IPv4len, net.IPv4len)
+		}
+
+		ipAndMask := &net.IPNet{IP: ip, Mask: m}
+		l3GatewayConfig.IPAddresses = append(l3GatewayConfig.IPAddresses, ipAndMask)
+	}
+
+	err = util.SetL3GatewayConfig(c.nodeAnnotator, l3GatewayConfig)
+	if err != nil {
+		return fmt.Errorf("can't set host address annotation: %w", err)
+	}
+
+	err = c.nodeAnnotator.Run()
+	if err != nil {
+		return fmt.Errorf("can't run annotator on node: %w", err)
+	}
+
+	return nil
+}

test/e2e/e2e.go

@@ -1776,7 +1776,8 @@ var _ = ginkgo.Describe("e2e ingress traffic validation", func() {
 			}
 		})
 	})
-	ginkgo.Context("Validating ExternalIP ingress traffic to manually added node IPs", func() {
+
+	ginkgo.Context("Validating ingress traffic to manually added node IPs", func() {
 		ginkgo.BeforeEach(func() {
 			endPoints = make([]*v1.Pod, 0)
 			nodesHostnames = sets.NewString()
@@ -1846,6 +1847,7 @@ var _ = ginkgo.Describe("e2e ingress traffic validation", func() {
 				}
 			}
 		})
+
 		// This test validates ingress traffic to externalservices after a new node Ip is added.
 		// It creates a service on both udp and tcp and assigns the new node IPs as
 		// external Addresses. Then, creates a backend pod on each node.
@@ -1891,6 +1893,38 @@ var _ = ginkgo.Describe("e2e ingress traffic validation", func() {
 				}
 			}
 		})
+
+		// This test verifies a NodePort service is reachable on manually added IP addresses.
+		ginkgo.It("for NodePort services", func() {
+			serviceName := "nodeportservice"
+
+			ginkgo.By("Creating NodePort service")
+			svcSpec := nodePortServiceSpecFrom(serviceName, v1.IPFamilyPolicyPreferDualStack, endpointHTTPPort, endpointUDPPort, clusterHTTPPort, clusterUDPPort, endpointsSelector, v1.ServiceExternalTrafficPolicyTypeLocal)
+			svcSpec, err := f.ClientSet.CoreV1().Services(f.Namespace.Name).Create(context.Background(), svcSpec, metav1.CreateOptions{})
+			framework.ExpectNoError(err)
+
+			ginkgo.By("Waiting for the endpoints to pop up")
+			err = framework.WaitForServiceEndpointsNum(f.ClientSet, f.Namespace.Name, serviceName, len(endPoints), time.Second, wait.ForeverTestTimeout)
+			framework.ExpectNoError(err, "failed to validate endpoints for service %s in namespace: %s", serviceName, f.Namespace.Name)
+
+			tcpNodePort, udpNodePort := nodePortsFromService(svcSpec)
+
+			for _, protocol := range []string{"http", "udp"} {
+				toCurlPort := int32(tcpNodePort)
+				if protocol == "udp" {
+					toCurlPort = int32(udpNodePort)
+				}
+
+				for _, newAddress := range newNodeAddresses {
+					ginkgo.By("Hitting the service on " + newAddress + " via " + protocol)
+					gomega.Eventually(func() bool {
+						epHostname := pokeEndpoint("", clientContainerName, protocol, newAddress, toCurlPort, "hostname")
+						// Expect to receive a valid hostname
+						return nodesHostnames.Has(epHostname)
+					}, "10s", "1s").Should(gomega.BeTrue())
+				}
+			}
+		})
 	})
 })