Delete IPv6 routes to peer gateway if necessary

Check whether the route to peer gateway is replaced or not when
adding routes and delete the route if it is no longer required. This
can happen when the traffic encryption mode or traffic encapsulation
mode changes.

Fixes #3437

Signed-off-by: Xu Liu <xliu2@vmware.com>

pkg/agent/route/route_linux.go

@@ -836,51 +836,74 @@ func (c *Client) AddRoutes(podCIDR *net.IPNet, nodeName string, nodeIP, nodeGwIP
 		return err
 	}
 	// Install routes to this Node.
-	route := &netlink.Route{
+	podCIDRRoute := &netlink.Route{
 		Dst: podCIDR,
 	}
 	var routes []*netlink.Route
+	var nodeGwIPNetv6 *net.IPNet
+	if utilnet.IsIPv6(nodeGwIP) {
+		nodeGwIPNetv6 = &net.IPNet{IP: nodeGwIP, Mask: net.IPMask{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}}
+	}
 	// If WireGuard is enabled, create a route via WireGuard device regardless of the traffic encapsulation modes.
 	if c.networkConfig.TrafficEncryptionMode == config.TrafficEncryptionModeWireGuard {
-		route.LinkIndex = c.nodeConfig.WireGuardConfig.LinkIndex
-		route.Scope = netlink.SCOPE_LINK
+		podCIDRRoute.LinkIndex = c.nodeConfig.WireGuardConfig.LinkIndex
+		podCIDRRoute.Scope = netlink.SCOPE_LINK
 		if podCIDR.IP.To4() != nil {
-			route.Src = c.nodeConfig.GatewayConfig.IPv4
+			podCIDRRoute.Src = c.nodeConfig.GatewayConfig.IPv4
 		} else {
-			route.Src = c.nodeConfig.GatewayConfig.IPv6
+			podCIDRRoute.Src = c.nodeConfig.GatewayConfig.IPv6
 		}
+		routes = append(routes, podCIDRRoute)
 	} else if c.networkConfig.NeedsTunnelToPeer(nodeIP, nodeTransportIPAddr) {
 		if podCIDR.IP.To4() == nil {
 			// "on-link" is not identified in IPv6 route entries, so split the configuration into 2 entries.
-			routes = []*netlink.Route{
-				{
-					Dst:       &net.IPNet{IP: nodeGwIP, Mask: net.IPMask{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}},
-					LinkIndex: c.nodeConfig.GatewayConfig.LinkIndex,
-				},
-			}
+			// TODO: Kernel >= 4.16 supports adding IPv6 route with onlink flag. Delete this route after Kernel version
+			//       requirement bump in future.
+			routes = append(routes, &netlink.Route{
+				Dst:       nodeGwIPNetv6,
+				LinkIndex: c.nodeConfig.GatewayConfig.LinkIndex,
+			})
 		} else {
-			route.Flags = int(netlink.FLAG_ONLINK)
+			podCIDRRoute.Flags = int(netlink.FLAG_ONLINK)
+			podCIDRRoute.LinkIndex = c.nodeConfig.GatewayConfig.LinkIndex
+			podCIDRRoute.Gw = nodeGwIP
+			routes = append(routes, podCIDRRoute)
 		}
-		route.LinkIndex = c.nodeConfig.GatewayConfig.LinkIndex
-		route.Gw = nodeGwIP
 	} else if c.networkConfig.NeedsDirectRoutingToPeer(nodeIP, nodeTransportIPAddr) {
 		// NoEncap traffic to Node on the same subnet.
 		// Set the peerNodeIP as next hop.
-		route.Gw = nodeIP
+		podCIDRRoute.Gw = nodeIP
+		routes = append(routes, podCIDRRoute)
 	} else {
 		// NetworkPolicyOnly mode or NoEncap traffic to a Node on a different subnet.
 		// Routing should be handled by a route which is already present on the host.
-		return nil
+		klog.InfoS("Skip adding routes to peer", "node", nodeName, "ip", nodeIP, "podCIDR", podCIDR)
 	}
-	routes = append(routes, route)
-
+	routeToNodeGwIPv6Replaced := false
+	routeToPodCIDRReplaced := false
 	for _, route := range routes {
 		if err := netlink.RouteReplace(route); err != nil {
 			return fmt.Errorf("failed to install route to peer %s (%s) with netlink. Route config: %s. Error: %v", nodeName, nodeIP, route.String(), err)
 		}
+		if route.Dst == nodeGwIPNetv6 {
+			routeToNodeGwIPv6Replaced = true
+		}
+		if route.Dst == podCIDR {
+			routeToPodCIDRReplaced = true
+		}
 	}
-
-	if podCIDR.IP.To4() == nil {
+	if !routeToNodeGwIPv6Replaced && utilnet.IsIPv6(nodeGwIP) {
+		routeToNodeGwIPNetv6 := &netlink.Route{
+			Dst: nodeGwIPNetv6,
+		}
+		if err := netlink.RouteDel(routeToNodeGwIPNetv6); err == nil {
+			klog.Info("Deleted route to peer gateway", "node", nodeName, "ip", nodeIP, "route", routeToNodeGwIPNetv6)
+		} else if err != unix.ESRCH {
+			return fmt.Errorf("failed to delete route to peer gateway %s on Node %s (%s) with netlink. Error: %v",
+				nodeGwIPNetv6, nodeName, nodeIP, err)
+		}
+	}
+	if routeToPodCIDRReplaced && podCIDR.IP.To4() == nil {
 		// Add IPv6 neighbor if the given podCIDR is using IPv6 address.
 		neigh := &netlink.Neigh{
 			LinkIndex:    c.nodeConfig.GatewayConfig.LinkIndex,