fix(FoU): make more robust

FoU implementation now properly handles a whole host of things: * It now actually handles IPv6 by changing the encapsulation protocol to GUE instead of generic FoU. I worked with generic FoU tunnels for several days and could get it to support IPv4 and IPv6 at all even when placing using it with the IPv6 proto and with iproute2 in IPv6 mode (-6) * It now handles converting between the two tunnel types seemlessly and without leaving legacy tunnel artifacts behind. Previously, you could change the encap type but it wouldn't change the tunnels * Abstracted constants
cloudnativelabs · Oct 7, 2023 · 944ab91 · 944ab91
1 parent bac4ae6
commit 944ab91
Show file tree

Hide file tree

Showing 3 changed files with 149 additions and 29 deletions.
diff --git a/pkg/controllers/routing/network_routes_controller.go b/pkg/controllers/routing/network_routes_controller.go
@@ -77,6 +77,9 @@ const (
 	encapTypeFOU  = "fou"
 	encapTypeIPIP = "ipip"
 
+	ipipModev4 = "ipip"
+	ipipModev6 = "ip6ip6"
+
 	maxPort = uint16(65535)
 	minPort = uint16(1024)
 )
@@ -656,7 +659,7 @@ func (nrc *NetworkRoutingController) injectRoute(path *gobgpapi.Path) error {
 	// if the user has disabled overlays, don't create tunnels. If we're not creating a tunnel, check to see if there is
 	// any cleanup that needs to happen.
 	if shouldCreateTunnel() {
-		link, err = nrc.setupOverlayTunnel(tunnelName, nextHop)
+		link, err = nrc.setupOverlayTunnel(tunnelName, nextHop, dst)
 		if err != nil {
 			return err
 		}
@@ -741,60 +744,104 @@ func (nrc *NetworkRoutingController) cleanupTunnel(destinationSubnet *net.IPNet,
 }
 
 // setupOverlayTunnel attempts to create a tunnel link and corresponding routes for IPIP based overlay networks
-func (nrc *NetworkRoutingController) setupOverlayTunnel(tunnelName string, nextHop net.IP) (netlink.Link, error) {
+func (nrc *NetworkRoutingController) setupOverlayTunnel(tunnelName string, nextHop net.IP,
+	nextHopSubnet *net.IPNet) (netlink.Link, error) {
 	var out []byte
 	link, err := netlink.LinkByName(tunnelName)
 
 	var bestIPForFamily net.IP
-	var ipipMode string
-	var ipProto string
+	var ipipMode, fouLinkType string
+	isIPv6 := false
 	ipBase := make([]string, 0)
+	strFormattedEncapPort := strconv.FormatInt(int64(nrc.overlayEncapPort), 10)
+
 	if nextHop.To4() != nil {
 		bestIPForFamily = utils.FindBestIPv4NodeAddress(nrc.primaryIP, nrc.nodeIPv4Addrs)
-		ipipMode = "ipip"
-		ipProto = "4"
+		ipipMode = encapTypeIPIP
+		fouLinkType = ipipModev4
 	} else {
 		// Need to activate the ip command in IPv6 mode
 		ipBase = append(ipBase, "-6")
 		bestIPForFamily = utils.FindBestIPv6NodeAddress(nrc.primaryIP, nrc.nodeIPv6Addrs)
-		ipipMode = "ip6ip6"
-		ipProto = "6"
+		ipipMode = ipipModev6
+		fouLinkType = "ip6tnl"
+		isIPv6 = true
 	}
 	if nil == bestIPForFamily {
 		return nil, fmt.Errorf("not able to find an appropriate configured IP address on node for destination "+
 			"IP family: %s", nextHop.String())
 	}
 
+	// This indicated that the tunnel already exists, so it's possible that there might be nothing more needed. However,
+	// it is also possible that the user changed the encap type, so we need to make sure that the encap type matches
+	// and if it doesn't, create it
+	recreate := false
+	if err == nil {
+		klog.V(1).Infof("Tunnel interface: %s with encap type %s for the node %s already exists.",
+			tunnelName, link.Attrs().EncapType, nextHop.String())
+
+		switch nrc.overlayEncap {
+		case encapTypeIPIP:
+			if linkFOUEnabled(tunnelName) {
+				klog.Infof("Was configured to use ipip tunnels, but found existing fou tunnels in place, cleaning up")
+				recreate = true
+
+				// Even though we are setup for IPIP tunels we have existing tunnels that are FoU tunnels, remove them
+				// so that we can recreate them as IPIP
+				nrc.cleanupTunnel(nextHopSubnet, tunnelName)
+
+				// If we are transitioning from FoU to IPIP we also need to clean up the old FoU port if it exists
+				if fouPortAndProtoExist(nrc.overlayEncapPort, isIPv6) {
+					fouArgs := ipBase
+					fouArgs = append(fouArgs, "fou", "del", "port", strFormattedEncapPort)
+					out, err := exec.Command("ip", fouArgs...).CombinedOutput()
+					if err != nil {
+						klog.Warningf("failed to clean up previous FoU tunnel port (this is only a warning because it "+
+							"won't stop kube-router from working for now, but still shouldn't have happened) - error: "+
+							"%v, output %s", err, out)
+					}
+				}
+			}
+		case encapTypeFOU:
+			if !linkFOUEnabled(tunnelName) {
+				klog.Infof("Was configured to use fou tunnels, but found existing ipip tunnels in place, cleaning up")
+				recreate = true
+				// Even though we are setup for FoU tunels we have existing tunnels that are IPIP tunnels, remove them
+				// so that we can recreate them as IPIP
+				nrc.cleanupTunnel(nextHopSubnet, tunnelName)
+			}
+		}
+	}
+
 	// an error here indicates that the tunnel didn't exist, so we need to create it, if it already exists there's
 	// nothing to do here
-	if err != nil {
+	if err != nil || recreate {
+		klog.Infof("Creating tunnel %s of type %s with encap %s for destination %s",
+			tunnelName, fouLinkType, nrc.overlayEncap, nextHop.String())
 		cmdArgs := ipBase
 		switch nrc.overlayEncap {
-		case "ipip":
+		case encapTypeIPIP:
 			// Plain IPIP tunnel without any encapsulation
 			cmdArgs = append(cmdArgs, "tunnel", "add", tunnelName, "mode", ipipMode, "local", bestIPForFamily.String(),
 				"remote", nextHop.String())
-		case "fou":
-			strFormattedEncapPort := strconv.FormatInt(int64(nrc.overlayEncapPort), 10)
 
+		case encapTypeFOU:
 			// Ensure that the FOU tunnel port is set correctly
-			cmdArgs = append(cmdArgs, "fou", "show")
-			out, err := exec.Command("ip", cmdArgs...).CombinedOutput()
-			if err != nil || !strings.Contains(string(out), strFormattedEncapPort) {
-				//nolint:gocritic // we understand that we are appending to a new slice
-				cmdArgs = append(ipBase, "fou", "add", "port", strFormattedEncapPort, "ipproto", ipProto)
-				out, err := exec.Command("ip", cmdArgs...).CombinedOutput()
+			if !fouPortAndProtoExist(nrc.overlayEncapPort, isIPv6) {
+				fouArgs := ipBase
+				fouArgs = append(fouArgs, "fou", "add", "port", strFormattedEncapPort, "gue")
+				out, err := exec.Command("ip", fouArgs...).CombinedOutput()
 				if err != nil {
 					return nil, fmt.Errorf("route not injected for the route advertised by the node %s "+
 						"Failed to set FoU tunnel port - error: %s, output: %s", tunnelName, err, string(out))
 				}
 			}
 
 			// Prep IPIP tunnel for FOU encapsulation
-			//nolint:gocritic // we understand that we are appending to a new slice
-			cmdArgs = append(ipBase, "link", "add", "name", tunnelName, "type", "ipip", "remote", nextHop.String(),
-				"local", bestIPForFamily.String(), "ttl", "225", "encap", "fou", "encap-sport", "auto", "encap-dport",
+			cmdArgs = append(cmdArgs, "link", "add", "name", tunnelName, "type", fouLinkType, "remote", nextHop.String(),
+				"local", bestIPForFamily.String(), "ttl", "225", "encap", "gue", "encap-sport", "auto", "encap-dport",
 				strFormattedEncapPort, "mode", ipipMode)
+
 		default:
 			return nil, fmt.Errorf("unknown tunnel encapsulation was passed: %s, unable to continue with overlay "+
 				"setup", nrc.overlayEncap)
@@ -821,9 +868,6 @@ func (nrc *NetworkRoutingController) setupOverlayTunnel(tunnelName string, nextH
 		if err = netlink.LinkSetUp(link); err != nil {
 			return nil, errors.New("Failed to bring tunnel interface " + tunnelName + " up due to: " + err.Error())
 		}
-	} else {
-		klog.V(1).Infof(
-			"Tunnel interface: " + tunnelName + " for the node " + nextHop.String() + " already exists.")
 	}
 
 	// Now that the tunnel link exists, we need to add a route to it, so the node knows where to send traffic bound for

diff --git a/pkg/controllers/routing/network_routes_controller_test.go b/pkg/controllers/routing/network_routes_controller_test.go
@@ -1996,11 +1996,7 @@ func Test_generateTunnelName(t *testing.T) {
 			tunnelName := generateTunnelName(testcase.nodeIP)
 			assert.Lessf(t, len(tunnelName), 16, "the maximum length of the tunnel name should never exceed"+
 				"15 characters as 16 characters is the maximum length of a Unix interface name")
-			if tunnelName != testcase.tunnelName {
-				t.Logf("actual tunnel interface name: %s", tunnelName)
-				t.Logf("expected tunnel interface name: %s", testcase.tunnelName)
-				t.Error("did not get expected tunnel interface name")
-			}
+			assert.Equal(t, testcase.tunnelName, tunnelName, "did not get expected tunnel interface name")
 		})
 	}
 }

diff --git a/pkg/controllers/routing/utils.go b/pkg/controllers/routing/utils.go
@@ -1,11 +1,13 @@
 package routing
 
 import (
+	"bufio"
 	"crypto/sha256"
 	"encoding/base64"
 	"errors"
 	"fmt"
 	"net"
+	"os/exec"
 	"regexp"
 	"strconv"
 	"strings"
@@ -324,3 +326,81 @@ func (nrc *NetworkRoutingController) getBGPRouteInfoForVIP(vip string) (subnet u
 	err = fmt.Errorf("could not convert IP to IPv4 or IPv6, unable to find subnet for: %s", vip)
 	return
 }
+
+// fouPortAndProtoExist checks to see if the given FoU port is already configured on the system via iproute2
+// tooling for the given protocol
+//
+// fou show, shows both IPv4 and IPv6 ports in the same show command, they look like:
+// port 5556 gue
+// port 5556 gue -6
+// where the only thing that distinguishes them is the -6 or not on the end
+// WARNING we're parsing a CLI tool here not an API, this may break at some point in the future
+func fouPortAndProtoExist(port uint16, isIPv6 bool) bool {
+	const ipRoute2IPv6Prefix = "-6"
+	strPort := strconv.FormatInt(int64(port), 10)
+	fouArgs := make([]string, 0)
+	klog.V(2).Infof("Checking FOU Port and Proto... %s - %t", strPort, isIPv6)
+
+	if isIPv6 {
+		fouArgs = append(fouArgs, ipRoute2IPv6Prefix)
+	}
+	fouArgs = append(fouArgs, "fou", "show")
+
+	out, err := exec.Command("ip", fouArgs...).CombinedOutput()
+	// iproute2 returns an error if no fou configuration exists
+	if err != nil {
+		return false
+	}
+
+	strOut := string(out)
+	klog.V(2).Infof("Combined output of ip fou show: %s", strOut)
+	scanner := bufio.NewScanner(strings.NewReader(strOut))
+
+	// loop over all lines of output
+	for scanner.Scan() {
+		scannedLine := scanner.Text()
+		// if the output doesn't contain our port at all, then continue
+		if !strings.Contains(scannedLine, strPort) {
+			continue
+		}
+
+		// if this is IPv6 port and it has the correct IPv6 suffix (see example above) then return true
+		if isIPv6 && strings.HasSuffix(scannedLine, ipRoute2IPv6Prefix) {
+			return true
+		}
+
+		// if this is not IPv6 and it does not have an IPv6 suffix (see example above) then return true
+		if !isIPv6 && !strings.HasSuffix(scannedLine, ipRoute2IPv6Prefix) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// linkFOUEnabled checks to see whether the given link has FoU (Foo over Ethernet) enabled on it, specifically since
+// kube-router only works with GUE (Generic UDP Encapsulation) we look for that and not just FoU in general. If the
+// linkName is enabled with FoU GUE then we return true, otherwise false
+//
+// Output for a FoU Enabled GUE tunnel looks like:
+// ipip ipip remote <ip> local <ip> dev <dev> ttl 225 pmtudisc encap gue encap-sport auto encap-dport 5555 ...
+// Output for a normal IPIP tunnel looks like:
+// ipip ipip remote <ip> local <ip> dev <dev> ttl inherit ...
+func linkFOUEnabled(linkName string) bool {
+	const fouEncapEnabled = "encap gue"
+	cmdArgs := []string{"-details", "link", "show", linkName}
+
+	out, err := exec.Command("ip", cmdArgs...).CombinedOutput()
+
+	if err != nil {
+		klog.Warning("recevied an error while trying to look at the link details of %s, this shouldn't have happened",
+			linkName)
+		return false
+	}
+
+	if strings.Contains(string(out), fouEncapEnabled) {
+		return true
+	}
+
+	return false
+}