add ovn0 default route (#4127)

Signed-off-by: 马洪贞 <hzma@alauda.io>

pkg/daemon/controller_linux.go

@@ -196,7 +196,8 @@ func (c *Controller) reconcileRouters(event *subnetEvent) error {
 	joinCIDR := make([]string, 0, 2)
 	cidrs := make([]string, 0, len(subnets)*2)
 	for _, subnet := range subnets {
-		if (subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway) || subnet.Spec.Vpc != c.config.ClusterRouter || !subnet.Status.IsReady() {
+		// The route for overlay subnet cidr via ovn0 should not be deleted even though subnet.Status has changed to not ready
+		if (subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway) || subnet.Spec.Vpc != c.config.ClusterRouter {
 			continue
 		}
 

pkg/daemon/controller_windows.go

@@ -53,9 +53,9 @@ func (c *Controller) reconcileRouters(_ *subnetEvent) error {
 	gwIPv4, gwIPv6 := util.SplitStringIP(gateway)
 	v4Cidrs, v6Cidrs := make([]string, 0, len(subnets)), make([]string, 0, len(subnets))
 	for _, subnet := range subnets {
+		// The route for overlay subnet cidr via ovn0 should not be deleted even though subnet.Status has changed to not ready
 		if (subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway) ||
-			subnet.Spec.Vpc != c.config.ClusterRouter ||
-			!subnet.Status.IsReady() {
+			subnet.Spec.Vpc != c.config.ClusterRouter {
 			continue
 		}
 

pkg/daemon/init.go

@@ -82,7 +82,7 @@ func InitNodeGateway(config *Configuration) error {
 	}
 
 	ipAddr = util.GetIPAddrWithMask(ip, cidr)
-	return configureNodeNic(portName, ipAddr, gw, mac, config.MTU)
+	return configureNodeNic(portName, ipAddr, gw, cidr, mac, config.MTU)
 }
 
 func InitMirror(config *Configuration) error {

pkg/daemon/ovs_linux.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net"
 	"os"
@@ -12,6 +13,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/containernetworking/plugins/pkg/ns"
@@ -404,7 +406,7 @@ func waitNetworkReady(nic, ipAddr, gateway string, underlayGateway, verbose bool
 	return nil
 }
 
-func configureNodeNic(portName, ip, gw string, macAddr net.HardwareAddr, mtu int) error {
+func configureNodeNic(portName, ip, gw, joinCIDR string, macAddr net.HardwareAddr, mtu int) error {
 	ipStr := util.GetIPWithoutMask(ip)
 	raw, err := ovs.Exec(ovs.MayExist, "add-port", "br-int", util.NodeNic, "--",
 		"set", "interface", util.NodeNic, "type=internal", "--",
@@ -437,6 +439,41 @@ func configureNodeNic(portName, ip, gw string, macAddr net.HardwareAddr, mtu int
 		return fmt.Errorf("can not set host nic %s qlen: %v", util.NodeNic, err)
 	}
 
+	// check and add default route for ovn0 in case of can not add automatically
+	nodeNicRoutes, err := getNicExistRoutes(hostLink, gw)
+	if err != nil {
+		klog.Error(err)
+		return err
+	}
+
+	var toAdd []netlink.Route
+	for _, c := range strings.Split(joinCIDR, ",") {
+		found := false
+		for _, r := range nodeNicRoutes {
+			if r.Dst.String() == c {
+				found = true
+				break
+			}
+		}
+		if !found {
+			_, cidr, _ := net.ParseCIDR(c)
+			toAdd = append(toAdd, netlink.Route{
+				Dst:   cidr,
+				Scope: netlink.SCOPE_UNIVERSE,
+			})
+		}
+	}
+	if len(toAdd) > 0 {
+		klog.Infof("route to add for nic %s, %v", util.NodeNic, toAdd)
+	}
+
+	for _, r := range toAdd {
+		r.LinkIndex = hostLink.Attrs().Index
+		if err = netlink.RouteReplace(&r); err != nil && !errors.Is(err, syscall.EEXIST) {
+			klog.Errorf("failed to replace route %v: %v", r, err)
+		}
+	}
+
 	// ping ovn0 gw to activate the flow
 	klog.Infof("wait ovn0 gw ready")
 	if err := waitNetworkReady(util.NodeNic, ip, gw, false, true, gatewayCheckMaxRetry); err != nil {

pkg/daemon/ovs_windows.go

@@ -256,7 +256,7 @@ func waitNetworkReady(nic, ipAddr, gateway string, underlayGateway, verbose bool
 	return nil
 }
 
-func configureNodeNic(portName, ip, gw string, macAddr net.HardwareAddr, mtu int) error {
+func configureNodeNic(portName, ip, gw, joinCIDR string, macAddr net.HardwareAddr, mtu int) error {
 	ipStr := util.GetIPWithoutMask(ip)
 	raw, err := ovs.Exec(ovs.MayExist, "add-port", "br-int", util.NodeNic, "--",
 		"set", "interface", util.NodeNic, "type=internal", "--",
@@ -271,6 +271,52 @@ func configureNodeNic(portName, ip, gw string, macAddr net.HardwareAddr, mtu int
 		return err
 	}
 
+	// check and add default route for ovn0 in case of can not add automatically
+	// IPv4: 100.64.0.0/16 dev ovn0 proto kernel scope link src 100.64.0.2
+	// IPv6: fd00:100:64::/112 dev ovn0 proto kernel metric 256 pref medium
+	adapter, err := util.GetNetAdapter(util.NodeNic, false)
+	if err != nil {
+		klog.Errorf("failed to get network adapter %s: %v", util.NodeNic, err)
+		return err
+	}
+	routes, err := util.GetNetRoute(adapter.InterfaceIndex)
+	if err != nil {
+		klog.Errorf("failed to get NetIPRoute with index %d: %v", adapter.InterfaceIndex, err)
+		return err
+	}
+
+	var toAddV4, toAddV6 []string
+	for _, cidr := range strings.Split(joinCIDR, ",") {
+		found := false
+		for _, route := range routes {
+			if route.DestinationPrefix == cidr {
+				found = true
+				break
+			}
+			if !found {
+				if util.CheckProtocol(cidr) == kubeovnv1.ProtocolIPv4 {
+					toAddV4 = append(toAddV4, cidr)
+				} else {
+					toAddV6 = append(toAddV6, cidr)
+				}
+			}
+		}
+	}
+	if len(toAddV4) > 0 || len(toAddV6) > 0 {
+		klog.Infof("route to add for nic %s, ipv4 %v, ipv6 %v", util.NodeNic, toAddV4, toAddV6)
+	}
+
+	for _, r := range toAddV4 {
+		if err = util.NewNetRoute(adapter.InterfaceIndex, r, "0.0.0.0"); err != nil {
+			klog.Errorf("failed to add ipv4 route %s: %v", r, err)
+		}
+	}
+	for _, r := range toAddV6 {
+		if err = util.NewNetRoute(adapter.InterfaceIndex, r, "::"); err != nil {
+			klog.Errorf("failed to add ipv6 route %s: %v", r, err)
+		}
+	}
+
 	// ping ovn0 gw to activate the flow
 	klog.Infof("wait ovn0 gw ready")
 	if err := waitNetworkReady(util.NodeNic, ip, gw, false, true, gatewayCheckMaxRetry); err != nil {