From 6ce37e616738f7e5c41351d1a8b3b37ff6024a42 Mon Sep 17 00:00:00 2001 From: Kartik Raval Date: Tue, 31 Jan 2023 08:03:49 +0000 Subject: [PATCH] Support for FoU encapsulation for IPIP tunnel --- docs/user-guide.md | 2 + .../routing/network_routes_controller.go | 42 ++++++++++++++++--- pkg/options/options.go | 14 +++++-- 3 files changed, 50 insertions(+), 8 deletions(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index 1ae60727c..30f2f77d4 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -79,6 +79,8 @@ Usage of kube-router: --metrics-port uint16 Prometheus metrics port, (Default 0, Disabled) --nodeport-bindon-all-ip For service of NodePort type create IPVS service that listens on all IP's of the node. --nodes-full-mesh Each node in the cluster will setup BGP peering with rest of the nodes. (default true) + --overlay-encap string Valid encapsulation types are "fou" - If set to "fou", the udp port can be specified via "overlay-encap-port" + --overlay-encap-port uint16 Overlay tunnel encapsulation port (default 5555) --overlay-type string Possible values: subnet,full - When set to "subnet", the default, default "--enable-overlay=true" behavior is used. When set to "full", it changes "--enable-overlay=true" default behavior so that IP-in-IP tunneling is used for pod-to-pod networking across nodes regardless of the subnet the nodes are in. (default "subnet") --override-nexthop Override the next-hop in bgp routes sent to peers with the local ip. --peer-router-asns uints ASN numbers of the BGP peer to which cluster nodes will advertise cluster ip and node's pod cidr. (default []) diff --git a/pkg/controllers/routing/network_routes_controller.go b/pkg/controllers/routing/network_routes_controller.go index d8d0ceeb0..abee78bd0 100644 --- a/pkg/controllers/routing/network_routes_controller.go +++ b/pkg/controllers/routing/network_routes_controller.go @@ -115,6 +115,8 @@ type NetworkRoutingController struct { iptablesCmdHandlers map[v1core.IPFamily]utils.IPTablesHandler enableOverlays bool overlayType string + overlayEncap string + overlayEncapPort uint16 peerMultihopTTL uint8 MetricsEnabled bool bgpServerStarted bool @@ -178,14 +180,23 @@ func (nrc *NetworkRoutingController) Run(healthChan chan<- *healthcheck.Controll // Handle ipip tunnel overlay if nrc.enableOverlays { - klog.V(1).Info("IPIP Tunnel Overlay enabled in configuration.") + klog.V(1).Info("Tunnel Overlay enabled in configuration.") klog.V(1).Info("Setting up overlay networking.") err = nrc.enablePolicyBasedRouting() if err != nil { klog.Errorf("Failed to enable required policy based routing: %s", err.Error()) } + if nrc.overlayEncap == "fou" { + // enable FoU module for the overlay tunnel + if _, err := exec.Command("modprobe", "fou").CombinedOutput(); err != nil { + klog.Errorf("Failed to enable FoU for tunnel overlay: %s", err.Error()) + } + if _, err := exec.Command("modprobe", "fou6").CombinedOutput(); err != nil { + klog.Errorf("Failed to enable FoU6 for tunnel overlay: %s", err.Error()) + } + } } else { - klog.V(1).Info("IPIP Tunnel Overlay disabled in configuration.") + klog.V(1).Info("Tunnel Overlay disabled in configuration.") klog.V(1).Info("Cleaning up old overlay networking if needed.") err = nrc.disablePolicyBasedRouting() if err != nil { @@ -748,9 +759,28 @@ func (nrc *NetworkRoutingController) setupOverlayTunnel(tunnelName string, nextH // an error here indicates that the tunnel didn't exist, so we need to create it, if it already exists there's // nothing to do here if err != nil { - //nolint:gocritic // we understand that we are appending to a new slice - cmdArgs := append(ipBase, "tunnel", "add", tunnelName, "mode", ipipMode, "local", bestIPForFamily.String(), - "remote", nextHop.String()) + cmdArgs := ipBase + if nrc.overlayEncap == "" { + // Plain IPIP tunnel without any encapsulation + cmdArgs = append(cmdArgs, "tunnel", "add", tunnelName, "mode", ipipMode, "local", bestIPForFamily.String(), + "remote", nextHop.String()) + + } else if nrc.overlayEncap == "fou" { + + cmdArgs = append(cmdArgs, "fou", "add", "port", strconv.FormatInt(int64(nrc.overlayEncapPort), 10), "ipproto", "4") + out, err := exec.Command("ip", cmdArgs...).CombinedOutput() + if err != nil { + return nil, fmt.Errorf("route not injected for the route advertised by the node %s "+ + "Failed to set FoU tunnel port - error: %s, output: %s", tunnelName, err, string(out)) + } + cmdArgs = ipBase + cmdArgs = append(cmdArgs, + "link", "add", "name", tunnelName, "type", "ipip", + "remote", nextHop.String(), "local", bestIPForFamily.String(), + "ttl", "225", "encap", "fou", "encap-sport", "auto", "encap-dport", + strconv.FormatInt(int64(nrc.overlayEncapPort), 10), + "mode", ipipMode) + } // need to skip binding device if nrc.nodeInterface is loopback, otherwise packets never leave // from egress interface to the tunnel peer. if nrc.nodeInterface != "lo" { @@ -1480,6 +1510,8 @@ func NewNetworkRoutingController(clientset kubernetes.Interface, nrc.autoMTU = kubeRouterConfig.AutoMTU nrc.enableOverlays = kubeRouterConfig.EnableOverlay nrc.overlayType = kubeRouterConfig.OverlayType + nrc.overlayEncap = kubeRouterConfig.OverlayEncap + nrc.overlayEncapPort = kubeRouterConfig.OverlayEncapPort nrc.CNIFirewallSetup = sync.NewCond(&sync.Mutex{}) nrc.bgpPort = kubeRouterConfig.BGPPort diff --git a/pkg/options/options.go b/pkg/options/options.go index d2fd5505a..324fd0e04 100644 --- a/pkg/options/options.go +++ b/pkg/options/options.go @@ -9,9 +9,10 @@ import ( ) const ( - DefaultBgpPort = 179 - DefaultBgpHoldTime = 90 * time.Second - defaultHealthCheckPort = 20244 + DefaultBgpPort = 179 + DefaultBgpHoldTime = 90 * time.Second + defaultHealthCheckPort = 20244 + defaultOverlayTunnelEncapPort = 5555 ) type KubeRouterConfig struct { @@ -59,6 +60,8 @@ type KubeRouterConfig struct { NodePortBindOnAllIP bool NodePortRange string OverlayType string + OverlayEncap string + OverlayEncapPort uint16 OverrideNextHop bool PeerASNs []uint PeerMultihopTTL uint8 @@ -180,6 +183,11 @@ func (s *KubeRouterConfig) AddFlags(fs *pflag.FlagSet) { "For service of NodePort type create IPVS service that listens on all IP's of the node.") fs.BoolVar(&s.FullMeshMode, "nodes-full-mesh", true, "Each node in the cluster will setup BGP peering with rest of the nodes.") + fs.StringVar(&s.OverlayEncap, "overlay-encap", s.OverlayEncap, + "Valid encapsulation types are \"fou\" - "+ + "If set to \"fou\", the udp port can be specified via \"overlay-encap-port\"") + fs.Uint16Var(&s.OverlayEncapPort, "overlay-encap-port", uint16(defaultOverlayTunnelEncapPort), + "Overlay tunnel encapsulation port") fs.StringVar(&s.OverlayType, "overlay-type", s.OverlayType, "Possible values: subnet,full - "+ "When set to \"subnet\", the default, default \"--enable-overlay=true\" behavior is used. "+