fix: change ovn-ic static route to policy (#1670)

pkg/controller/controller.go

@@ -574,6 +574,10 @@ func (c *Controller) startWorkers(stopCh <-chan struct{}) {
 		c.resyncInterConnection()
 	}, time.Second, stopCh)
 
+	go wait.Until(func() {
+		c.SynRouteToPolicy()
+	}, 5*time.Second, stopCh)
+
 	go wait.Until(func() {
 		c.resyncExternalGateway()
 	}, time.Second, stopCh)

pkg/controller/ovn-ic.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb"
 	"os"
 	"os/exec"
 	"reflect"
@@ -331,9 +332,12 @@ func (c *Controller) delLearnedRoute() error {
 	if len(learnedPorts) != 0 {
 		for _, aLdPort := range learnedPorts {
 			itsRouter, err := c.ovnLegacyClient.CustomFindEntity("Logical_Router", []string{"name"}, fmt.Sprintf("static_routes{>}%s", aLdPort["_uuid"][0]))
-			if err != nil || len(itsRouter) != 1 {
+			if err != nil {
 				klog.Errorf("failed to list logical router of static route %s, %v", aLdPort["_uuid"][0], err)
 				return err
+			} else if len(itsRouter) != 1 {
+				klog.Errorf("number wrong of logical router for static route %s, %v", aLdPort["_uuid"][0], itsRouter)
+				return nil
 			}
 			if err := c.ovnLegacyClient.DeleteStaticRoute(aLdPort["ip_prefix"][0], itsRouter[0]["name"][0]); err != nil {
 				klog.Errorf("failed to delete stale route %s, %v", aLdPort["ip_prefix"][0], err)
@@ -361,3 +365,73 @@ func genHostAddress(host string, port string) (hostaddress string) {
 	}
 	return hostaddress
 }
+
+func (c *Controller) SynRouteToPolicy() {
+
+	lr, err := c.ovnClient.GetLogicalRouter(util.DefaultVpc, false)
+	if err != nil {
+		klog.Errorf("logical router does not exist %v at %v", err, time.Now())
+		return
+	}
+	lrRouteList, err := c.ovnClient.GetLogicalRouterRouteByOpts(util.OvnICKey, util.OvnICValue)
+	if err != nil {
+		klog.Errorf("failed to list lr ovn-ic route %v", err)
+		return
+	}
+	if len(lrRouteList) == 0 {
+		klog.V(5).Info(" lr ovn-ic route does not exist")
+		lrPolicyList, err := c.ovnClient.GetLogicalRouterPoliciesByExtID(util.OvnICKey, util.OvnICValue)
+		if err != nil {
+			klog.Errorf("failed to list ovn-ic lr policy ", err)
+			return
+		}
+		for _, lrPolicy := range lrPolicyList {
+			if err := c.ovnClient.DeleteRouterPolicy(lr, lrPolicy.UUID); err != nil {
+				klog.Errorf("deleting router policy failed %v", err)
+			}
+		}
+		return
+	}
+
+	policyMap := map[string]string{}
+	lrPolicyList, err := c.ovnClient.GetLogicalRouterPoliciesByExtID(util.OvnICKey, util.OvnICValue)
+	if err != nil {
+		klog.Errorf("failed to list ovn-ic lr policy ", err)
+		return
+	}
+	for _, lrPolicy := range lrPolicyList {
+		match, err := stripPrefix(lrPolicy.Match)
+		if err != nil {
+			klog.Errorf("policy match abnormal ", err)
+			continue
+		}
+		policyMap[match] = lrPolicy.UUID
+	}
+	for _, lrRoute := range lrRouteList {
+		if _, ok := policyMap[lrRoute.IPPrefix]; ok {
+			delete(policyMap, lrRoute.IPPrefix)
+		} else {
+			matchFiled := util.MatchV4Dst + " == " + lrRoute.IPPrefix
+			if err := c.ovnClient.AddRouterPolicy(lr, matchFiled, ovnnb.LogicalRouterPolicyActionAllow,
+				map[string]string{},
+				map[string]string{util.OvnICKey: util.OvnICValue, "vendor": util.CniTypeName},
+				util.OvnICPolicyPriority); err != nil {
+				klog.Errorf("adding router policy failed %v", err)
+			}
+		}
+	}
+	for _, uuid := range policyMap {
+		if err := c.ovnClient.DeleteRouterPolicy(lr, uuid); err != nil {
+			klog.Errorf("deleting router policy failed %v", err)
+		}
+	}
+}
+
+func stripPrefix(policyMatch string) (string, error) {
+	matches := strings.Split(policyMatch, "==")
+	if strings.Trim(matches[0], " ") == util.MatchV4Dst {
+		return strings.Trim(matches[1], " "), nil
+	} else {
+		return "", fmt.Errorf("policy %s is mismatched", policyMatch)
+	}
+}

pkg/ovs/ovn-nb-logical_router_policy.go

@@ -0,0 +1,77 @@
+package ovs
+
+import (
+	"fmt"
+	ovsclient "github.com/kubeovn/kube-ovn/pkg/ovsdb/client"
+	"github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb"
+	"github.com/ovn-org/libovsdb/model"
+	"github.com/ovn-org/libovsdb/ovsdb"
+)
+
+func (c OvnClient) AddRouterPolicy(lr *ovnnb.LogicalRouter, matchfield string, action ovnnb.LogicalRouterPolicyAction,
+	opts map[string]string, extIDs map[string]string, priority int) error {
+	lrPolicy := &ovnnb.LogicalRouterPolicy{
+		Action:      action,
+		Match:       matchfield,
+		Options:     opts,
+		Priority:    priority,
+		ExternalIDs: extIDs,
+		UUID:        ovsclient.NamedUUID(),
+		Nexthop:     nil,
+		Nexthops:    nil,
+	}
+
+	var ops []ovsdb.Operation
+
+	waitOp := ConstructWaitForUniqueOperation("Logical_Router_Policy", "match", matchfield)
+	ops = append(ops, waitOp)
+
+	createOps, err := c.ovnNbClient.Create(lrPolicy)
+	if err != nil {
+		return err
+	}
+	ops = append(ops, createOps...)
+	mutationOps, err := c.ovnNbClient.Where(lr).Mutate(lr, model.Mutation{
+		Field:   &lr.Policies,
+		Mutator: ovsdb.MutateOperationInsert,
+		Value:   []string{lrPolicy.UUID},
+	})
+	if err != nil {
+		return fmt.Errorf("failed to generate create operations for router policy %s: %v", matchfield, err)
+	}
+	ops = append(ops, mutationOps...)
+
+	if err = Transact(c.ovnNbClient, "lr-policy-add", ops, c.ovnNbClient.Timeout); err != nil {
+		return fmt.Errorf("failed to create route policy %s: %v", matchfield, err)
+	}
+	return nil
+}
+
+func (c OvnClient) DeleteRouterPolicy(lr *ovnnb.LogicalRouter, uuid string) error {
+
+	var ops []ovsdb.Operation
+
+	delOps, err := c.ovnNbClient.Where(lr).Mutate(lr, model.Mutation{
+		Field:   &lr.Policies,
+		Mutator: ovsdb.MutateOperationDelete,
+		Value:   []string{uuid},
+	})
+	if err != nil {
+		return fmt.Errorf("failed to generate delete operations for router %s: %v", uuid, err)
+	}
+	ops = append(ops, delOps...)
+
+	lrPolicy := &ovnnb.LogicalRouterPolicy{
+		UUID: uuid,
+	}
+	deleteOps, err := c.ovnNbClient.Where(lrPolicy).Delete()
+	if err != nil {
+		return fmt.Errorf("failed to generate delete operations for router policy %s: %v", uuid, err)
+	}
+	ops = append(ops, deleteOps...)
+
+	if err = Transact(c.ovnNbClient, "lr-policy-delete", ops, c.ovnNbClient.Timeout); err != nil {
+		return fmt.Errorf("failed to delete route policy %s: %v", uuid, err)
+	}
+	return nil
+}

pkg/ovs/ovn-nb-logical_router_route.go

@@ -0,0 +1,35 @@
+package ovs
+
+import (
+	"context"
+	"github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb"
+	"github.com/ovn-org/libovsdb/client"
+)
+
+func (c OvnClient) GetLogicalRouterRouteByOpts(key, value string) ([]ovnnb.LogicalRouterStaticRoute, error) {
+
+	var lrRouteList []ovnnb.LogicalRouterStaticRoute
+	err := c.ovnNbClient.WhereCache(
+		func(lrroute *ovnnb.LogicalRouterStaticRoute) bool {
+			return lrroute.Options[key] == value
+		}).List(context.TODO(), &lrRouteList)
+	if err != nil && err != client.ErrNotFound {
+		return nil, err
+	}
+
+	return lrRouteList, nil
+}
+
+func (c OvnClient) GetLogicalRouterPoliciesByExtID(key, value string) ([]ovnnb.LogicalRouterPolicy, error) {
+
+	var lrPolicyList []ovnnb.LogicalRouterPolicy
+	err := c.ovnNbClient.WhereCache(
+		func(lrPoliy *ovnnb.LogicalRouterPolicy) bool {
+			return lrPoliy.ExternalIDs[key] == value
+		}).List(context.TODO(), &lrPolicyList)
+	if err != nil && err != client.ErrNotFound {
+		return nil, err
+	}
+
+	return lrPolicyList, nil
+}

pkg/ovsdb/client/client.go

@@ -88,6 +88,8 @@ func NewNbClient(addr string, timeout int) (client.Client, error) {
 		client.WithTable(&ovnnb.LogicalRouterPolicy{}),
 		client.WithTable(&ovnnb.LogicalSwitchPort{}),
 		client.WithTable(&ovnnb.PortGroup{}),
+		client.WithTable(&ovnnb.LogicalRouterStaticRoute{}),
+		client.WithTable(&ovnnb.LogicalRouterPolicy{}),
 	}
 	if _, err = c.Monitor(context.TODO(), c.NewMonitor(monitorOpts...)); err != nil {
 		klog.Errorf("failed to monitor database on OVN NB server %s: %v", addr, err)

pkg/util/const.go

@@ -128,6 +128,7 @@ const (
 	GatewayRouterPolicyPriority = 29000
 	NodeRouterPolicyPriority    = 30000
 	SubnetRouterPolicyPriority  = 31000
+	OvnICPolicyPriority         = 29500
 
 	VethType     = "veth-pair"
 	OffloadType  = "offload-port"
@@ -164,4 +165,17 @@ const (
 	NetemQosLatencyAnnotationTemplate = "%s.kubernetes.io/latency"
 	NetemQosLimitAnnotationTemplate   = "%s.kubernetes.io/limit"
 	NetemQosLossAnnotationTemplate    = "%s.kubernetes.io/loss"
+
+	POD_IP             = "POD_IP"
+	ContentType        = "application/vnd.kubernetes.protobuf"
+	AcceptContentTypes = "application/vnd.kubernetes.protobuf,application/json"
+
+	AttachmentProvider = "ovn.kubernetes.io/attchmentprovider"
+	LbSvcPodImg        = "ovn.kubernetes.io/lb_svc_img"
+
+	OvnICKey   = "origin"
+	OvnICValue = "connected"
+
+	MatchV4Src = "ip4.src"
+	MatchV4Dst = "ip4.dst"
 )