Skip to content

Commit

Permalink
manage ovn lr static route with libovsdb
Browse files Browse the repository at this point in the history
  • Loading branch information
@zhangzujian
zhangzujian committed May 15, 2023
1 parent 74221a6 commit 6acff8e
Show file tree
Hide file tree
Showing 15 changed files with 291 additions and 655 deletions.
90 changes: 50 additions & 40 deletions mocks/pkg/ovs/interface.go
View file

Large diffs are not rendered by default.

5 changes: 2 additions & 3 deletions pkg/controller/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (

"golang.org/x/time/rate"
corev1 "k8s.io/api/core/v1"
k8sv1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
Expand Down Expand Up @@ -64,7 +63,7 @@ type Controller struct {
podsSynced cache.InformerSynced
addOrUpdatePodQueue workqueue.RateLimitingInterface
deletePodQueue workqueue.RateLimitingInterface
deletingPodObjMap map[string]*k8sv1.Pod
deletingPodObjMap map[string]*corev1.Pod
updatePodSecurityQueue workqueue.RateLimitingInterface
podKeyMutex keymutex.KeyMutex

Expand Down Expand Up @@ -396,7 +395,7 @@ func Run(ctx context.Context, config *Configuration) {
workqueue.NewNamedDelayingQueue("DeletePod"),
workqueue.DefaultControllerRateLimiter(),
),
deletingPodObjMap: make(map[string]*k8sv1.Pod),
deletingPodObjMap: make(map[string]*corev1.Pod),
updatePodSecurityQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "UpdatePodSecurity"),
podKeyMutex: keymutex.NewHashed(numKeyLocks),

Expand Down
16 changes: 8 additions & 8 deletions pkg/controller/gc.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -615,7 +615,7 @@ func (c *Controller) gcPortGroup() error {

func (c *Controller) gcStaticRoute() error {
klog.Infof("start to gc static routes")
routes, err := c.ovnLegacyClient.GetStaticRouteList(c.config.ClusterRouter)
routes, err := c.ovnClient.ListLogicalRouterStaticRoutes(c.config.ClusterRouter, nil, nil, "", nil)
if err != nil {
klog.Errorf("failed to list static route %v", err)
return err
Expand All @@ -629,23 +629,23 @@ func (c *Controller) gcStaticRoute() error {
for _, route := range routes {
keepStaticRoute = false
for _, item := range defaultVpc.Spec.StaticRoutes {
if route.CIDR == item.CIDR && route.NextHop == item.NextHopIP && route.RouteTable == item.RouteTable {
if route.IPPrefix == item.CIDR && route.Nexthop == item.NextHopIP && route.RouteTable == item.RouteTable {
keepStaticRoute = true
break
}
}
if keepStaticRoute {
continue
}
if route.CIDR != "0.0.0.0/0" && route.CIDR != "::/0" && c.ipam.ContainAddress(route.CIDR) {
exist, err := c.ovnLegacyClient.NatRuleExists(route.CIDR)
if route.IPPrefix != "0.0.0.0/0" && route.IPPrefix != "::/0" && c.ipam.ContainAddress(route.IPPrefix) {
exist, err := c.ovnLegacyClient.NatRuleExists(route.IPPrefix)
if exist || err != nil {
klog.Errorf("failed to get NatRule by LogicalIP %s, %v", route.CIDR, err)
klog.Errorf("failed to get NatRule by LogicalIP %s, %v", route.IPPrefix, err)
continue
}
klog.Infof("gc static route %s %s %s %s", route.RouteTable, route.Policy, route.CIDR, route.NextHop)
if err := c.ovnLegacyClient.DeleteStaticRoute(route.CIDR, c.config.ClusterRouter, route.RouteTable); err != nil {
klog.Errorf("failed to delete stale route %s, %v", route.NextHop, err)
klog.Infof("gc static route %s %v %s %s", route.RouteTable, route.Policy, route.IPPrefix, route.Nexthop)
if err = c.ovnClient.DeleteLogicalRouterStaticRoute(c.config.ClusterRouter, &route.RouteTable, route.Policy, route.IPPrefix, route.Nexthop); err != nil {
klog.Errorf("failed to delete stale route %s %v %s %s: %v", route.RouteTable, route.Policy, route.IPPrefix, route.Nexthop, err)
}
}
}
Expand Down
12 changes: 3 additions & 9 deletions pkg/controller/init.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -756,18 +756,12 @@ func (c *Controller) migrateNodeRoute(af int, node, ip, nexthop string) error {
return err
}

routeTables, err := c.ovnLegacyClient.GetRouteTables(c.config.ClusterRouter)
if err != nil {
routeTable := util.MainRouteTable
if err := c.ovnClient.DeleteLogicalRouterStaticRoute(c.config.ClusterRouter, &routeTable, nil, ip, ""); err != nil {
klog.Errorf("failed to delete obsolete static route for node %s: %v", node, err)
return err
}

for rtb := range routeTables {
if err := c.ovnLegacyClient.DeleteStaticRoute(ip, c.config.ClusterRouter, rtb); err != nil {
klog.Errorf("failed to delete obsolete static route for node %s: %v", node, err)
return err
}
}

asName := nodeUnderlayAddressSetName(node, af)
obsoleteMatch := fmt.Sprintf("ip%d.dst == %s && ip%d.src != $%s", af, ip, af, asName)
klog.V(3).Infof("delete policy route for router: %s, priority: %d, match %s", c.config.ClusterRouter, util.NodeRouterPolicyPriority, obsoleteMatch)
Expand Down
38 changes: 4 additions & 34 deletions pkg/controller/node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (

kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
"github.com/kubeovn/kube-ovn/pkg/ovs"
"github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb"
"github.com/kubeovn/kube-ovn/pkg/util"
)

Expand Down Expand Up @@ -858,26 +859,6 @@ func (c *Controller) checkGatewayReady() error {
return nil
}

func (c *Controller) checkRouteExist(nextHop, cidrBlock, routePolicy, routeTable string) (bool, error) {
routes, err := c.ovnLegacyClient.GetStaticRouteList(c.config.ClusterRouter)
if err != nil {
klog.Errorf("failed to list static route %v", err)
return false, err
}

for _, route := range routes {
if route.Policy != routePolicy {
continue
}

if route.CIDR == cidrBlock && route.NextHop == nextHop && route.RouteTable == routeTable {
klog.V(3).Infof("static route exists for cidr %s, nexthop %v", cidrBlock, nextHop)
return true, nil
}
}
return false, nil
}

func (c *Controller) checkChassisDupl(node *v1.Node) error {
// notice that multiple chassises may arise and we are not prepared
chassisAdd, err := c.ovnLegacyClient.GetChassis(node.Name)
Expand Down Expand Up @@ -1049,7 +1030,7 @@ func (c *Controller) validateChassis(node *v1.Node) error {
func (c *Controller) addNodeGwStaticRoute() error {
// If user not manage static route for default vpc, just add route about ovn-default to join
if vpc, err := c.vpcsLister.Get(c.config.ClusterRouter); err != nil || vpc.Spec.StaticRoutes != nil {
existRoute, err := c.ovnLegacyClient.GetStaticRouteList(c.config.ClusterRouter)
existRoute, err := c.ovnClient.ListLogicalRouterStaticRoutes(c.config.ClusterRouter, nil, nil, "", nil)
if err != nil {
klog.Errorf("failed to get vpc %s static route list, %v", c.config.ClusterRouter, err)
}
Expand All @@ -1064,21 +1045,10 @@ func (c *Controller) addNodeGwStaticRoute() error {
if util.CheckProtocol(cidrBlock) != util.CheckProtocol(nextHop) {
continue
}
exist, err := c.checkRouteExist(nextHop, cidrBlock, ovs.PolicyDstIP, util.MainRouteTable)
if err != nil {
klog.Errorf("get static route for node gw error %v", err)
if err := c.ovnClient.AddLogicalRouterStaticRoute(c.config.ClusterRouter, util.MainRouteTable, ovnnb.LogicalRouterStaticRoutePolicyDstIP, cidrBlock, nextHop); err != nil {
klog.Errorf("failed to add static route for node gw: %v", err)
return err
}

if !exist {
klog.Infof("add static route for node gw")
if err := c.ovnLegacyClient.AddStaticRoute(
"", cidrBlock, nextHop, "", "",
c.config.ClusterRouter, util.MainRouteTable, util.NormalRouteType); err != nil {
klog.Errorf("failed to add static route for node gw: %v", err)
return err
}
}
}
}
return nil
Expand Down
54 changes: 13 additions & 41 deletions pkg/controller/ovn-ic.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -347,54 +347,26 @@ func (c *Controller) waitTsReady() error {
}

func (c *Controller) delLearnedRoute() error {
originalPorts, err := c.ovnLegacyClient.CustomFindEntity("Logical_Router_Static_Route", []string{"_uuid", "ip_prefix"})
lrList, err := c.ovnClient.ListLogicalRouter(false, nil)
if err != nil {
klog.Errorf("failed to list static routes of logical router, %v", err)
klog.Errorf("failed to list logical routers: %v", err)
return err
}
filteredPorts, err := c.ovnLegacyClient.CustomFindEntity("Logical_Router_Static_Route", []string{"_uuid", "ip_prefix"}, "external_ids:ic-learned-route{<=}1")
if err != nil {
klog.Errorf("failed to filter static routes of logical router, %v", err)
return err
}
learnedPorts := []map[string][]string{}
for _, aOriPort := range originalPorts {
isFiltered := false
for _, aFtPort := range filteredPorts {
if aFtPort["_uuid"][0] == aOriPort["_uuid"][0] {
isFiltered = true
}
}
if !isFiltered {
learnedPorts = append(learnedPorts, aOriPort)
for _, lr := range lrList {
routeList, err := c.ovnClient.ListLogicalRouterStaticRoutes(lr.Name, nil, nil, "", map[string]string{"ic-learned-route": ""})
if err != nil {
klog.Errorf("failed to list learned static routes on logical router %s: %v", lr.Name, err)
return err
}
}
if len(learnedPorts) != 0 {
for _, aLdPort := range learnedPorts {
itsRouter, err := c.ovnLegacyClient.CustomFindEntity("Logical_Router", []string{"name"}, fmt.Sprintf("static_routes{>}%s", aLdPort["_uuid"][0]))
if err != nil {
klog.Errorf("failed to list logical router of static route %s, %v", aLdPort["_uuid"][0], err)
return err
} else if len(itsRouter) != 1 {
klog.Errorf("number wrong of logical router for static route %s, %v", aLdPort["_uuid"][0], itsRouter)
return nil
}

rtbs, err := c.ovnLegacyClient.GetRouteTables(itsRouter[0]["name"][0])
if err != nil {
klog.Errorf("failed to list route tables of logical router %s, %v", itsRouter[0]["name"][0], err)
for _, r := range routeList {
if err = c.ovnClient.DeleteLogicalRouterStaticRoute(lr.Name, &r.RouteTable, r.Policy, r.IPPrefix, r.Nexthop); err != nil {
klog.Errorf("failed to delete learned static route %#v on logical router %s: %v", r, lr.Name, err)
return err
}

for rtb := range rtbs {
if err := c.ovnLegacyClient.DeleteStaticRoute(aLdPort["ip_prefix"][0], itsRouter[0]["name"][0], rtb); err != nil {
klog.Errorf("failed to delete static route %s, %v", aLdPort["ip_prefix"][0], err)
return err
}
}
}
klog.V(5).Infof("finish removing learned routes")
}

klog.V(5).Infof("finish removing learned routes")
return nil
}

Expand Down Expand Up @@ -459,7 +431,7 @@ func (c *Controller) syncOneRouteToPolicy(key, value string) {
klog.Errorf("logical router does not exist %v at %v", err, time.Now())
return
}
lrRouteList, err := c.ovnClient.ListLogicalRouterStaticRoutesByOption(lr.Name, key, value)
lrRouteList, err := c.ovnClient.ListLogicalRouterStaticRoutesByOption(lr.Name, util.MainRouteTable, key, value)
if err != nil {
klog.Errorf("failed to list lr ovn-ic route %v", err)
return
Expand Down
21 changes: 11 additions & 10 deletions pkg/controller/pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
"github.com/kubeovn/kube-ovn/pkg/ipam"
"github.com/kubeovn/kube-ovn/pkg/ovs"
"github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb"
"github.com/kubeovn/kube-ovn/pkg/util"
)

Expand Down Expand Up @@ -769,9 +770,9 @@ func (c *Controller) reconcileRouteSubnets(cachedPod, pod *v1.Pod, needRoutePodN
nextHop = strings.Split(nextHop, "/")[0]
}

if err := c.ovnLegacyClient.AddStaticRoute(
ovs.PolicySrcIP, podIP, nextHop, "", "",
c.config.ClusterRouter, subnet.Spec.RouteTable, util.NormalRouteType); err != nil {
if err := c.ovnClient.AddLogicalRouterStaticRoute(
c.config.ClusterRouter, subnet.Spec.RouteTable, ovnnb.LogicalRouterStaticRoutePolicySrcIP, podIP, nextHop,
); err != nil {
klog.Errorf("failed to add static route, %v", err)
return err
}
Expand Down Expand Up @@ -812,15 +813,16 @@ func (c *Controller) reconcileRouteSubnets(cachedPod, pod *v1.Pod, needRoutePodN
}

if pod.Annotations[util.NorthGatewayAnnotation] != "" {
if err := c.ovnLegacyClient.AddStaticRoute(
ovs.PolicySrcIP, podIP, pod.Annotations[util.NorthGatewayAnnotation], "", "",
c.config.ClusterRouter, subnet.Spec.RouteTable, util.NormalRouteType); err != nil {
if err := c.ovnClient.AddLogicalRouterStaticRoute(
c.config.ClusterRouter, subnet.Spec.RouteTable, ovnnb.LogicalRouterStaticRoutePolicySrcIP, podIP, pod.Annotations[util.NorthGatewayAnnotation],
); err != nil {
klog.Errorf("failed to add static route, %v", err)
return err
}
} else if c.config.EnableEipSnat {
if err := c.ovnLegacyClient.DeleteStaticRoute(
podIP, c.config.ClusterRouter, subnet.Spec.RouteTable); err != nil {
if err = c.ovnClient.DeleteLogicalRouterStaticRoute(
c.config.ClusterRouter, &subnet.Spec.RouteTable, nil, podIP, "",
); err != nil {
return err
}
}
Expand Down Expand Up @@ -903,8 +905,7 @@ func (c *Controller) handleDeletePod(key string) error {
}
// If pod has snat or eip, also need delete staticRoute when delete pod
if vpc.Name == c.config.ClusterRouter {
if err := c.ovnLegacyClient.DeleteStaticRoute(
address.Ip, vpc.Name, subnet.Spec.RouteTable); err != nil {
if err = c.ovnClient.DeleteLogicalRouterStaticRoute(vpc.Name, &subnet.Spec.RouteTable, nil, address.Ip, ""); err != nil {
return err
}
}
Expand Down
25 changes: 5 additions & 20 deletions pkg/controller/subnet.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1437,25 +1437,10 @@ func (c *Controller) reconcileOvnDefaultVpcRoute(subnet *kubeovnv1.Subnet) error

if pod.Annotations[util.NorthGatewayAnnotation] != "" {
nextHop := pod.Annotations[util.NorthGatewayAnnotation]
exist, err := c.checkRouteExist(
nextHop,
pod.Annotations[fmt.Sprintf(util.IpAddressAnnotationTemplate, podNet.ProviderName)],
ovs.PolicySrcIP,
subnet.Spec.RouteTable,
)
if err != nil {
klog.Errorf("failed to get static route for subnet %v, error %v", subnet.Name, err)
return err
}
if exist {
continue
}

if err := c.ovnLegacyClient.AddStaticRoute(
ovs.PolicySrcIP,
pod.Annotations[fmt.Sprintf(util.IpAddressAnnotationTemplate, podNet.ProviderName)],
nextHop, "", "", c.config.ClusterRouter,
subnet.Spec.RouteTable, util.NormalRouteType); err != nil {
if err := c.ovnClient.AddLogicalRouterStaticRoute(
c.config.ClusterRouter, util.MainRouteTable, ovnnb.LogicalRouterStaticRoutePolicySrcIP,
pod.Annotations[fmt.Sprintf(util.IpAddressAnnotationTemplate, podNet.ProviderName)], nextHop,
); err != nil {
klog.Errorf("add static route failed, %v", err)
return err
}
Expand Down Expand Up @@ -1687,7 +1672,7 @@ func (c *Controller) reconcileOvnCustomVpcRoute(subnet *kubeovnv1.Subnet) error

func (c *Controller) deleteStaticRoute(ip, router, routeTable string) error {
for _, ipStr := range strings.Split(ip, ",") {
if err := c.ovnLegacyClient.DeleteStaticRoute(ipStr, router, routeTable); err != nil {
if err := c.ovnClient.DeleteLogicalRouterStaticRoute(router, &routeTable, nil, ipStr, ""); err != nil {
klog.Errorf("failed to delete static route %s, %v", ipStr, err)
return err
}
Expand Down

0 comments on commit 6acff8e

Please sign in to comment.