manage ovn lr static route with libovsdb

pkg/controller/controller.go

@@ -9,7 +9,6 @@ import (
 
 	"golang.org/x/time/rate"
 	corev1 "k8s.io/api/core/v1"
-	k8sv1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -64,7 +63,7 @@ type Controller struct {
 	podsSynced             cache.InformerSynced
 	addOrUpdatePodQueue    workqueue.RateLimitingInterface
 	deletePodQueue         workqueue.RateLimitingInterface
-	deletingPodObjMap      map[string]*k8sv1.Pod
+	deletingPodObjMap      map[string]*corev1.Pod
 	updatePodSecurityQueue workqueue.RateLimitingInterface
 	podKeyMutex            keymutex.KeyMutex
 
@@ -396,7 +395,7 @@ func Run(ctx context.Context, config *Configuration) {
 			workqueue.NewNamedDelayingQueue("DeletePod"),
 			workqueue.DefaultControllerRateLimiter(),
 		),
-		deletingPodObjMap:      make(map[string]*k8sv1.Pod),
+		deletingPodObjMap:      make(map[string]*corev1.Pod),
 		updatePodSecurityQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "UpdatePodSecurity"),
 		podKeyMutex:            keymutex.NewHashed(numKeyLocks),
 

pkg/controller/gc.go

@@ -615,7 +615,7 @@ func (c *Controller) gcPortGroup() error {
 
 func (c *Controller) gcStaticRoute() error {
 	klog.Infof("start to gc static routes")
-	routes, err := c.ovnLegacyClient.GetStaticRouteList(c.config.ClusterRouter)
+	routes, err := c.ovnClient.ListLogicalRouterStaticRoutes(c.config.ClusterRouter, nil, nil, "", nil)
 	if err != nil {
 		klog.Errorf("failed to list static route %v", err)
 		return err
@@ -629,23 +629,23 @@ func (c *Controller) gcStaticRoute() error {
 	for _, route := range routes {
 		keepStaticRoute = false
 		for _, item := range defaultVpc.Spec.StaticRoutes {
-			if route.CIDR == item.CIDR && route.NextHop == item.NextHopIP && route.RouteTable == item.RouteTable {
+			if route.IPPrefix == item.CIDR && route.Nexthop == item.NextHopIP && route.RouteTable == item.RouteTable {
 				keepStaticRoute = true
 				break
 			}
 		}
 		if keepStaticRoute {
 			continue
 		}
-		if route.CIDR != "0.0.0.0/0" && route.CIDR != "::/0" && c.ipam.ContainAddress(route.CIDR) {
-			exist, err := c.ovnLegacyClient.NatRuleExists(route.CIDR)
+		if route.IPPrefix != "0.0.0.0/0" && route.IPPrefix != "::/0" && c.ipam.ContainAddress(route.IPPrefix) {
+			exist, err := c.ovnLegacyClient.NatRuleExists(route.IPPrefix)
 			if exist || err != nil {
-				klog.Errorf("failed to get NatRule by LogicalIP %s, %v", route.CIDR, err)
+				klog.Errorf("failed to get NatRule by LogicalIP %s, %v", route.IPPrefix, err)
 				continue
 			}
-			klog.Infof("gc static route %s %s %s %s", route.RouteTable, route.Policy, route.CIDR, route.NextHop)
-			if err := c.ovnLegacyClient.DeleteStaticRoute(route.CIDR, c.config.ClusterRouter, route.RouteTable); err != nil {
-				klog.Errorf("failed to delete stale route %s, %v", route.NextHop, err)
+			klog.Infof("gc static route %s %v %s %s", route.RouteTable, route.Policy, route.IPPrefix, route.Nexthop)
+			if err = c.ovnClient.DeleteLogicalRouterStaticRoute(c.config.ClusterRouter, &route.RouteTable, route.Policy, route.IPPrefix, route.Nexthop); err != nil {
+				klog.Errorf("failed to delete stale route %s %v %s %s: %v", route.RouteTable, route.Policy, route.IPPrefix, route.Nexthop, err)
 			}
 		}
 	}

pkg/controller/init.go

@@ -745,29 +745,24 @@ func (c *Controller) initSyncCrdVlans() error {
 
 func (c *Controller) migrateNodeRoute(af int, node, ip, nexthop string) error {
 	match := fmt.Sprintf("ip%d.dst == %s", af, ip)
+	action := ovnnb.LogicalRouterPolicyActionReroute
 	externalIDs := map[string]string{
 		"vendor": util.CniTypeName,
 		"node":   node,
 	}
 	klog.V(3).Infof("add policy route for router: %s, priority: %d, match %s, action %s, nexthop %s, extrenalID %v",
-		c.config.ClusterRouter, util.NodeRouterPolicyPriority, match, "reroute", nexthop, externalIDs)
-	if err := c.ovnClient.AddLogicalRouterPolicy(c.config.ClusterRouter, util.NodeRouterPolicyPriority, match, "reroute", []string{nexthop}, externalIDs); err != nil {
+		c.config.ClusterRouter, util.NodeRouterPolicyPriority, match, action, nexthop, externalIDs)
+	if err := c.ovnClient.AddLogicalRouterPolicy(c.config.ClusterRouter, util.NodeRouterPolicyPriority, match, action, []string{nexthop}, externalIDs); err != nil {
 		klog.Errorf("failed to add logical router policy for node %s: %v", node, err)
 		return err
 	}
 
-	routeTables, err := c.ovnLegacyClient.GetRouteTables(c.config.ClusterRouter)
-	if err != nil {
+	routeTable := util.MainRouteTable
+	if err := c.ovnClient.DeleteLogicalRouterStaticRoute(c.config.ClusterRouter, &routeTable, nil, ip, ""); err != nil {
+		klog.Errorf("failed to delete obsolete static route for node %s: %v", node, err)
 		return err
 	}
 
-	for rtb := range routeTables {
-		if err := c.ovnLegacyClient.DeleteStaticRoute(ip, c.config.ClusterRouter, rtb); err != nil {
-			klog.Errorf("failed to delete obsolete static route for node %s: %v", node, err)
-			return err
-		}
-	}
-
 	asName := nodeUnderlayAddressSetName(node, af)
 	obsoleteMatch := fmt.Sprintf("ip%d.dst == %s && ip%d.src != $%s", af, ip, af, asName)
 	klog.V(3).Infof("delete policy route for router: %s, priority: %d, match %s", c.config.ClusterRouter, util.NodeRouterPolicyPriority, obsoleteMatch)

pkg/controller/node.go

@@ -23,6 +23,7 @@ import (
 
 	kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
 	"github.com/kubeovn/kube-ovn/pkg/ovs"
+	"github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb"
 	"github.com/kubeovn/kube-ovn/pkg/util"
 )
 
@@ -272,13 +273,14 @@ func (c *Controller) handleAddNode(key string) error {
 		}
 		if nodeIP != "" {
 			match := fmt.Sprintf("ip%d.dst == %s", af, nodeIP)
+			action := ovnnb.LogicalRouterPolicyActionReroute
 			externalIDs := map[string]string{
 				"vendor":         util.CniTypeName,
 				"node":           node.Name,
 				"address-family": strconv.Itoa(af),
 			}
-			klog.Infof("add policy route for router: %s, match %s, action %s, nexthop %s, externalID %v", c.config.ClusterRouter, match, "reroute", ip, externalIDs)
-			if err = c.ovnClient.AddLogicalRouterPolicy(c.config.ClusterRouter, util.NodeRouterPolicyPriority, match, "reroute", []string{ip}, externalIDs); err != nil {
+			klog.Infof("add policy route for router: %s, match %s, action %s, nexthop %s, externalID %v", c.config.ClusterRouter, match, action, ip, externalIDs)
+			if err = c.ovnClient.AddLogicalRouterPolicy(c.config.ClusterRouter, util.NodeRouterPolicyPriority, match, action, []string{ip}, externalIDs); err != nil {
 				klog.Errorf("failed to add logical router policy for node %s: %v", node.Name, err)
 				return err
 			}
@@ -858,26 +860,6 @@ func (c *Controller) checkGatewayReady() error {
 	return nil
 }
 
-func (c *Controller) checkRouteExist(nextHop, cidrBlock, routePolicy, routeTable string) (bool, error) {
-	routes, err := c.ovnLegacyClient.GetStaticRouteList(c.config.ClusterRouter)
-	if err != nil {
-		klog.Errorf("failed to list static route %v", err)
-		return false, err
-	}
-
-	for _, route := range routes {
-		if route.Policy != routePolicy {
-			continue
-		}
-
-		if route.CIDR == cidrBlock && route.NextHop == nextHop && route.RouteTable == routeTable {
-			klog.V(3).Infof("static route exists for cidr %s, nexthop %v", cidrBlock, nextHop)
-			return true, nil
-		}
-	}
-	return false, nil
-}
-
 func (c *Controller) checkChassisDupl(node *v1.Node) error {
 	// notice that multiple chassises may arise and we are not prepared
 	chassisAdd, err := c.ovnLegacyClient.GetChassis(node.Name)
@@ -1049,7 +1031,7 @@ func (c *Controller) validateChassis(node *v1.Node) error {
 func (c *Controller) addNodeGwStaticRoute() error {
 	// If user not manage static route for default vpc, just add route about ovn-default to join
 	if vpc, err := c.vpcsLister.Get(c.config.ClusterRouter); err != nil || vpc.Spec.StaticRoutes != nil {
-		existRoute, err := c.ovnLegacyClient.GetStaticRouteList(c.config.ClusterRouter)
+		existRoute, err := c.ovnClient.ListLogicalRouterStaticRoutes(c.config.ClusterRouter, nil, nil, "", nil)
 		if err != nil {
 			klog.Errorf("failed to get vpc %s static route list, %v", c.config.ClusterRouter, err)
 		}
@@ -1064,21 +1046,10 @@ func (c *Controller) addNodeGwStaticRoute() error {
 			if util.CheckProtocol(cidrBlock) != util.CheckProtocol(nextHop) {
 				continue
 			}
-			exist, err := c.checkRouteExist(nextHop, cidrBlock, ovs.PolicyDstIP, util.MainRouteTable)
-			if err != nil {
-				klog.Errorf("get static route for node gw error %v", err)
+			if err := c.ovnClient.AddLogicalRouterStaticRoute(c.config.ClusterRouter, util.MainRouteTable, ovnnb.LogicalRouterStaticRoutePolicyDstIP, cidrBlock, nextHop); err != nil {
+				klog.Errorf("failed to add static route for node gw: %v", err)
 				return err
 			}
-
-			if !exist {
-				klog.Infof("add static route for node gw")
-				if err := c.ovnLegacyClient.AddStaticRoute(
-					"", cidrBlock, nextHop, "", "",
-					c.config.ClusterRouter, util.MainRouteTable, util.NormalRouteType); err != nil {
-					klog.Errorf("failed to add static route for node gw: %v", err)
-					return err
-				}
-			}
 		}
 	}
 	return nil
@@ -1256,8 +1227,9 @@ func (c *Controller) addPolicyRouteForLocalDnsCacheOnNode(nodePortName, nodeIP,
 
 	pgAs := strings.Replace(fmt.Sprintf("%s_ip%d", nodePortName, af), "-", ".", -1)
 	match := fmt.Sprintf("ip%d.src == $%s && ip%d.dst == %s", af, pgAs, af, c.config.NodeLocalDnsIP)
-	klog.Infof("add node local dns cache policy route for router: %s, match %s, action %s, nexthop %s, externalID %v", c.config.ClusterRouter, match, "reroute", nodeIP, externalIDs)
-	if err := c.ovnClient.AddLogicalRouterPolicy(c.config.ClusterRouter, util.NodeLocalDnsPolicyPriority, match, "reroute", []string{nodeIP}, externalIDs); err != nil {
+	action := ovnnb.LogicalRouterPolicyActionReroute
+	klog.Infof("add node local dns cache policy route for router: %s, match %s, action %s, nexthop %s, externalID %v", c.config.ClusterRouter, match, action, nodeIP, externalIDs)
+	if err := c.ovnClient.AddLogicalRouterPolicy(c.config.ClusterRouter, util.NodeLocalDnsPolicyPriority, match, action, []string{nodeIP}, externalIDs); err != nil {
 		klog.Errorf("failed to add logical router policy for node %s: %v", nodeName, err)
 		return err
 	}

pkg/controller/ovn-ic.go

@@ -347,54 +347,26 @@ func (c *Controller) waitTsReady() error {
 }
 
 func (c *Controller) delLearnedRoute() error {
-	originalPorts, err := c.ovnLegacyClient.CustomFindEntity("Logical_Router_Static_Route", []string{"_uuid", "ip_prefix"})
+	lrList, err := c.ovnClient.ListLogicalRouter(false, nil)
 	if err != nil {
-		klog.Errorf("failed to list static routes of logical router, %v", err)
+		klog.Errorf("failed to list logical routers: %v", err)
 		return err
 	}
-	filteredPorts, err := c.ovnLegacyClient.CustomFindEntity("Logical_Router_Static_Route", []string{"_uuid", "ip_prefix"}, "external_ids:ic-learned-route{<=}1")
-	if err != nil {
-		klog.Errorf("failed to filter static routes of logical router, %v", err)
-		return err
-	}
-	learnedPorts := []map[string][]string{}
-	for _, aOriPort := range originalPorts {
-		isFiltered := false
-		for _, aFtPort := range filteredPorts {
-			if aFtPort["_uuid"][0] == aOriPort["_uuid"][0] {
-				isFiltered = true
-			}
-		}
-		if !isFiltered {
-			learnedPorts = append(learnedPorts, aOriPort)
+	for _, lr := range lrList {
+		routeList, err := c.ovnClient.ListLogicalRouterStaticRoutes(lr.Name, nil, nil, "", map[string]string{"ic-learned-route": ""})
+		if err != nil {
+			klog.Errorf("failed to list learned static routes on logical router %s: %v", lr.Name, err)
+			return err
 		}
-	}
-	if len(learnedPorts) != 0 {
-		for _, aLdPort := range learnedPorts {
-			itsRouter, err := c.ovnLegacyClient.CustomFindEntity("Logical_Router", []string{"name"}, fmt.Sprintf("static_routes{>}%s", aLdPort["_uuid"][0]))
-			if err != nil {
-				klog.Errorf("failed to list logical router of static route %s, %v", aLdPort["_uuid"][0], err)
-				return err
-			} else if len(itsRouter) != 1 {
-				klog.Errorf("number wrong of logical router for static route %s, %v", aLdPort["_uuid"][0], itsRouter)
-				return nil
-			}
-
-			rtbs, err := c.ovnLegacyClient.GetRouteTables(itsRouter[0]["name"][0])
-			if err != nil {
-				klog.Errorf("failed to list route tables of logical router %s, %v", itsRouter[0]["name"][0], err)
+		for _, r := range routeList {
+			if err = c.ovnClient.DeleteLogicalRouterStaticRoute(lr.Name, &r.RouteTable, r.Policy, r.IPPrefix, r.Nexthop); err != nil {
+				klog.Errorf("failed to delete learned static route %#v on logical router %s: %v", r, lr.Name, err)
 				return err
 			}
-
-			for rtb := range rtbs {
-				if err := c.ovnLegacyClient.DeleteStaticRoute(aLdPort["ip_prefix"][0], itsRouter[0]["name"][0], rtb); err != nil {
-					klog.Errorf("failed to delete static route %s, %v", aLdPort["ip_prefix"][0], err)
-					return err
-				}
-			}
 		}
-		klog.V(5).Infof("finish removing learned routes")
 	}
+
+	klog.V(5).Infof("finish removing learned routes")
 	return nil
 }
 
@@ -459,7 +431,7 @@ func (c *Controller) syncOneRouteToPolicy(key, value string) {
 		klog.Errorf("logical router does not exist %v at %v", err, time.Now())
 		return
 	}
-	lrRouteList, err := c.ovnClient.ListLogicalRouterStaticRoutesByOption(lr.Name, key, value)
+	lrRouteList, err := c.ovnClient.ListLogicalRouterStaticRoutesByOption(lr.Name, util.MainRouteTable, key, value)
 	if err != nil {
 		klog.Errorf("failed to list lr ovn-ic route %v", err)
 		return

pkg/controller/pod.go

@@ -27,6 +27,7 @@ import (
 	kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
 	"github.com/kubeovn/kube-ovn/pkg/ipam"
 	"github.com/kubeovn/kube-ovn/pkg/ovs"
+	"github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb"
 	"github.com/kubeovn/kube-ovn/pkg/util"
 )
 
@@ -769,9 +770,9 @@ func (c *Controller) reconcileRouteSubnets(cachedPod, pod *v1.Pod, needRoutePodN
 					nextHop = strings.Split(nextHop, "/")[0]
 				}
 
-				if err := c.ovnLegacyClient.AddStaticRoute(
-					ovs.PolicySrcIP, podIP, nextHop, "", "",
-					c.config.ClusterRouter, subnet.Spec.RouteTable, util.NormalRouteType); err != nil {
+				if err := c.ovnClient.AddLogicalRouterStaticRoute(
+					c.config.ClusterRouter, subnet.Spec.RouteTable, ovnnb.LogicalRouterStaticRoutePolicySrcIP, podIP, nextHop,
+				); err != nil {
 					klog.Errorf("failed to add static route, %v", err)
 					return err
 				}
@@ -812,15 +813,16 @@ func (c *Controller) reconcileRouteSubnets(cachedPod, pod *v1.Pod, needRoutePodN
 				}
 
 				if pod.Annotations[util.NorthGatewayAnnotation] != "" {
-					if err := c.ovnLegacyClient.AddStaticRoute(
-						ovs.PolicySrcIP, podIP, pod.Annotations[util.NorthGatewayAnnotation], "", "",
-						c.config.ClusterRouter, subnet.Spec.RouteTable, util.NormalRouteType); err != nil {
+					if err := c.ovnClient.AddLogicalRouterStaticRoute(
+						c.config.ClusterRouter, subnet.Spec.RouteTable, ovnnb.LogicalRouterStaticRoutePolicySrcIP, podIP, pod.Annotations[util.NorthGatewayAnnotation],
+					); err != nil {
 						klog.Errorf("failed to add static route, %v", err)
 						return err
 					}
 				} else if c.config.EnableEipSnat {
-					if err := c.ovnLegacyClient.DeleteStaticRoute(
-						podIP, c.config.ClusterRouter, subnet.Spec.RouteTable); err != nil {
+					if err = c.ovnClient.DeleteLogicalRouterStaticRoute(
+						c.config.ClusterRouter, &subnet.Spec.RouteTable, nil, podIP, "",
+					); err != nil {
 						return err
 					}
 				}
@@ -903,8 +905,7 @@ func (c *Controller) handleDeletePod(key string) error {
 			}
 			// If pod has snat or eip, also need delete staticRoute when delete pod
 			if vpc.Name == c.config.ClusterRouter {
-				if err := c.ovnLegacyClient.DeleteStaticRoute(
-					address.Ip, vpc.Name, subnet.Spec.RouteTable); err != nil {
+				if err = c.ovnClient.DeleteLogicalRouterStaticRoute(vpc.Name, &subnet.Spec.RouteTable, nil, address.Ip, ""); err != nil {
 					return err
 				}
 			}