add ovn0 default route (#4127)

Signed-off-by: 马洪贞 <hzma@alauda.io>

pkg/daemon/controller_linux.go

@@ -189,7 +189,8 @@ func (c *Controller) reconcileRouters(event *subnetEvent) error {
 
 	cidrs := make([]string, 0, len(subnets)*2)
 	for _, subnet := range subnets {
-		if (subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway) || subnet.Spec.Vpc != util.DefaultVpc || !subnet.Status.IsReady() {
+		//The route for overlay subnet cidr via ovn0 should not be deleted even though subnet.Status has changed to not ready
+		if (subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway) || subnet.Spec.Vpc != util.DefaultVpc {
 			continue
 		}
 

pkg/daemon/controller_windows.go

@@ -53,9 +53,9 @@ func (c *Controller) reconcileRouters(_ *subnetEvent) error {
 	gwIPv4, gwIPv6 := util.SplitStringIP(gateway)
 	v4Cidrs, v6Cidrs := make([]string, 0, len(subnets)), make([]string, 0, len(subnets))
 	for _, subnet := range subnets {
+		// The route for overlay subnet cidr via ovn0 should not be deleted even though subnet.Status has changed to not ready
 		if (subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway) ||
-			subnet.Spec.Vpc != util.DefaultVpc ||
-			!subnet.Status.IsReady() {
+			subnet.Spec.Vpc != util.DefaultVpc {
 			continue
 		}
 

pkg/daemon/init.go

@@ -88,7 +88,7 @@ func InitNodeGateway(config *Configuration) error {
 		klog.Errorf("failed to get ip %s with mask %s, %v", ip, cidr, err)
 		return err
 	}
-	return configureNodeNic(portName, ipAddr, gw, mac, config.MTU)
+	return configureNodeNic(portName, ipAddr, gw, cidr, mac, config.MTU)
 }
 
 func InitMirror(config *Configuration) error {

pkg/daemon/ovs_linux.go

@@ -1,6 +1,7 @@
 package daemon
 
 import (
+	"errors"
 	"fmt"
 	"net"
 	"os"
@@ -9,8 +10,11 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
+	"syscall"
 	"time"
 
+	"strconv"
+
 	"github.com/Mellanox/sriovnet"
 	sriovutilfs "github.com/Mellanox/sriovnet/pkg/utils/filesystem"
 	"github.com/containernetworking/plugins/pkg/ns"
@@ -19,7 +23,6 @@ import (
 	"golang.org/x/sys/unix"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/klog/v2"
-	"strconv"
 
 	kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
 	"github.com/kubeovn/kube-ovn/pkg/ovs"
@@ -607,7 +610,7 @@ func waitNetworkReady(nic, ipAddr, gateway string, underlayGateway, verbose bool
 	return nil
 }
 
-func configureNodeNic(portName, ip, gw string, macAddr net.HardwareAddr, mtu int) error {
+func configureNodeNic(portName, ip, gw, joinCIDR string, macAddr net.HardwareAddr, mtu int) error {
 	ipStr := util.GetIpWithoutMask(ip)
 	raw, err := ovs.Exec(ovs.MayExist, "add-port", "br-int", util.NodeNic, "--",
 		"set", "interface", util.NodeNic, "type=internal", "--",
@@ -631,6 +634,41 @@ func configureNodeNic(portName, ip, gw string, macAddr net.HardwareAddr, mtu int
 		return fmt.Errorf("can not set host nic %s qlen: %v", util.NodeNic, err)
 	}
 
+	// check and add default route for ovn0 in case of can not add automatically
+	nodeNicRoutes, err := getNicExistRoutes(hostLink, gw)
+	if err != nil {
+		klog.Error(err)
+		return err
+	}
+
+	var toAdd []netlink.Route
+	for _, c := range strings.Split(joinCIDR, ",") {
+		found := false
+		for _, r := range nodeNicRoutes {
+			if r.Dst.String() == c {
+				found = true
+				break
+			}
+		}
+		if !found {
+			_, cidr, _ := net.ParseCIDR(c)
+			toAdd = append(toAdd, netlink.Route{
+				Dst:   cidr,
+				Scope: netlink.SCOPE_UNIVERSE,
+			})
+		}
+	}
+	if len(toAdd) > 0 {
+		klog.Infof("route to add for nic %s, %v", util.NodeNic, toAdd)
+	}
+
+	for _, r := range toAdd {
+		r.LinkIndex = hostLink.Attrs().Index
+		if err = netlink.RouteReplace(&r); err != nil && !errors.Is(err, syscall.EEXIST) {
+			klog.Errorf("failed to replace route %v: %v", r, err)
+		}
+	}
+
 	// ping ovn0 gw to activate the flow
 	klog.Infof("wait ovn0 gw ready")
 	if err := waitNetworkReady(util.NodeNic, ip, gw, false, true, gatewayCheckMaxRetry); err != nil {

pkg/daemon/ovs_windows.go

@@ -258,7 +258,7 @@ func waitNetworkReady(nic, ipAddr, gateway string, underlayGateway, verbose bool
 	return nil
 }
 
-func configureNodeNic(portName, ip, gw string, macAddr net.HardwareAddr, mtu int) error {
+func configureNodeNic(portName, ip, gw, joinCIDR string, macAddr net.HardwareAddr, mtu int) error {
 	ipStr := util.GetIpWithoutMask(ip)
 	raw, err := ovs.Exec(ovs.MayExist, "add-port", "br-int", util.NodeNic, "--",
 		"set", "interface", util.NodeNic, "type=internal", "--",
@@ -274,6 +274,52 @@ func configureNodeNic(portName, ip, gw string, macAddr net.HardwareAddr, mtu int
 		return err
 	}
 
+	// check and add default route for ovn0 in case of can not add automatically
+	// IPv4: 100.64.0.0/16 dev ovn0 proto kernel scope link src 100.64.0.2
+	// IPv6: fd00:100:64::/112 dev ovn0 proto kernel metric 256 pref medium
+	adapter, err := util.GetNetAdapter(util.NodeNic, false)
+	if err != nil {
+		klog.Errorf("failed to get network adapter %s: %v", util.NodeNic, err)
+		return err
+	}
+	routes, err := util.GetNetRoute(adapter.InterfaceIndex)
+	if err != nil {
+		klog.Errorf("failed to get NetIPRoute with index %d: %v", adapter.InterfaceIndex, err)
+		return err
+	}
+
+	var toAddV4, toAddV6 []string
+	for _, cidr := range strings.Split(joinCIDR, ",") {
+		found := false
+		for _, route := range routes {
+			if route.DestinationPrefix == cidr {
+				found = true
+				break
+			}
+			if !found {
+				if util.CheckProtocol(cidr) == kubeovnv1.ProtocolIPv4 {
+					toAddV4 = append(toAddV4, cidr)
+				} else {
+					toAddV6 = append(toAddV6, cidr)
+				}
+			}
+		}
+	}
+	if len(toAddV4) > 0 || len(toAddV6) > 0 {
+		klog.Infof("route to add for nic %s, ipv4 %v, ipv6 %v", util.NodeNic, toAddV4, toAddV6)
+	}
+
+	for _, r := range toAddV4 {
+		if err = util.NewNetRoute(adapter.InterfaceIndex, r, "0.0.0.0"); err != nil {
+			klog.Errorf("failed to add ipv4 route %s: %v", r, err)
+		}
+	}
+	for _, r := range toAddV6 {
+		if err = util.NewNetRoute(adapter.InterfaceIndex, r, "::"); err != nil {
+			klog.Errorf("failed to add ipv6 route %s: %v", r, err)
+		}
+	}
+
 	// ping ovn0 gw to activate the flow
 	klog.Infof("wait ovn0 gw ready")
 	if err := waitNetworkReady(util.NodeNic, ip, gw, false, true, gatewayCheckMaxRetry); err != nil {