Skip to content

Commit

Permalink
refactor logical router routes (#1500)
Browse files Browse the repository at this point in the history
  • Loading branch information
@zhangzujian
zhangzujian committed May 10, 2022
1 parent 5160362 commit ecc8268
Show file tree
Hide file tree
Showing 10 changed files with 172 additions and 329 deletions.
20 changes: 4 additions & 16 deletions pkg/controller/gc.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -599,22 +599,10 @@ func (c *Controller) gcStaticRoute() error {
return err
}
for _, route := range routes {
if route.Policy == ovs.PolicyDstIP || route.Policy == "" {
if !c.ipam.ContainAddress(route.NextHop) {
klog.Infof("gc static route %s %s %s", route.Policy, route.CIDR, route.NextHop)
if err := c.ovnClient.DeleteStaticRouteByNextHop(route.NextHop); err != nil {
klog.Errorf("failed to delete stale nexthop route %s, %v", route.NextHop, err)
}
}
} else {
if strings.Contains(route.CIDR, "/") {
continue
}
if !c.ipam.ContainAddress(route.CIDR) {
klog.Infof("gc static route %s %s %s", route.Policy, route.CIDR, route.NextHop)
if err := c.ovnClient.DeleteStaticRoute(route.CIDR, c.config.ClusterRouter); err != nil {
klog.Errorf("failed to delete stale route %s, %v", route.NextHop, err)
}
if route.CIDR != "0.0.0.0/0" && route.CIDR != "::/0" && c.ipam.ContainAddress(route.CIDR) {
klog.Infof("gc static route %s %s %s", route.Policy, route.CIDR, route.NextHop)
if err := c.ovnClient.DeleteStaticRoute(route.CIDR, c.config.ClusterRouter); err != nil {
klog.Errorf("failed to delete stale route %s, %v", route.NextHop, err)
}
}
}
Expand Down
50 changes: 17 additions & 33 deletions pkg/controller/init.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,11 +58,6 @@ func (c *Controller) InitOVN() error {
return err
}

if err := c.createOverlaySubnetsAddressSet(); err != nil {
klog.Errorf("failed to create overlay subnets address-set, %v", err)
return err
}

return nil
}

Expand Down Expand Up @@ -656,20 +651,30 @@ func (c *Controller) initSyncCrdVlans() error {
return nil
}

func (c *Controller) migrateNodeRoute(af int, node, ip, nexthop string, cidrs []string) error {
func (c *Controller) migrateNodeRoute(af int, node, ip, nexthop string) error {
if err := c.ovnClient.DeleteStaticRoute(ip, c.config.ClusterRouter); err != nil {
klog.Errorf("failed to delete obsolete static route for node %s: %v", node, err)
return err
}

asName := nodeUnderlayAddressSetName(node, af)
if err := c.ovnClient.CreateAddressSetWithAddresses(asName, cidrs...); err != nil {
klog.Errorf("failed to create address set %s for node %s: %v", asName, node, err)
obsoleteMatch := fmt.Sprintf("ip%d.dst == %s && ip%d.src != $%s", af, ip, af, asName)
if err := c.ovnClient.DeletePolicyRoute(c.config.ClusterRouter, util.NodeRouterPolicyPriority, obsoleteMatch); err != nil {
klog.Errorf("failed to delete obsolete logical router policy for node %s: %v", node, err)
return err
}

if err := c.ovnClient.DeleteAddressSet(asName); err != nil {
klog.Errorf("failed to delete obsolete address set %s for node %s: %v", asName, node, err)
return err
}

match := fmt.Sprintf("ip%d.dst == %s && ip%d.src != $%s", af, ip, af, asName)
if err := c.ovnClient.AddPolicyRoute(c.config.ClusterRouter, util.NodeRouterPolicyPriority, match, "reroute", nexthop); err != nil {
match := fmt.Sprintf("ip%d.dst == %s", af, ip)
externalIDs := map[string]string{
"vendor": util.CniTypeName,
"node": node,
}
if err := c.ovnClient.AddPolicyRoute(c.config.ClusterRouter, util.NodeRouterPolicyPriority, match, "reroute", nexthop, externalIDs); err != nil {
klog.Errorf("failed to add logical router policy for node %s: %v", node, err)
return err
}
Expand All @@ -678,42 +683,21 @@ func (c *Controller) migrateNodeRoute(af int, node, ip, nexthop string, cidrs []
}

func (c *Controller) initNodeRoutes() error {
subnets, err := c.subnetsLister.List(labels.Everything())
if err != nil {
klog.Errorf("failed to list subnets: %v", err)
return err
}
nodes, err := c.nodesLister.List(labels.Everything())
if err != nil {
klog.Errorf("failed to list nodes: %v", err)
return err
}
for _, node := range nodes {
nodeIPv4, nodeIPv6 := util.GetNodeInternalIP(*node)

var v4CIDRs, v6CIDRs []string
for _, subnet := range subnets {
if subnet.Spec.Vlan == "" || !subnet.Spec.LogicalGateway || subnet.Spec.Vpc != util.DefaultVpc {
continue
}

v4, v6 := util.SplitStringIP(subnet.Spec.CIDRBlock)
if util.CIDRContainIP(v4, nodeIPv4) {
v4CIDRs = append(v4CIDRs, v4)
}
if util.CIDRContainIP(v6, nodeIPv6) {
v6CIDRs = append(v6CIDRs, v6)
}
}

joinAddrV4, joinAddrV6 := util.SplitStringIP(node.Annotations[util.IpAddressAnnotation])
if nodeIPv4 != "" && joinAddrV4 != "" {
if err = c.migrateNodeRoute(4, node.Name, nodeIPv4, joinAddrV4, v4CIDRs); err != nil {
if err = c.migrateNodeRoute(4, node.Name, nodeIPv4, joinAddrV4); err != nil {
klog.Errorf("failed to migrate IPv4 route for node %s: %v", node.Name, err)
}
}
if nodeIPv6 != "" && joinAddrV6 != "" {
if err = c.migrateNodeRoute(6, node.Name, nodeIPv6, joinAddrV6, v6CIDRs); err != nil {
if err = c.migrateNodeRoute(6, node.Name, nodeIPv6, joinAddrV6); err != nil {
klog.Errorf("failed to migrate IPv6 route for node %s: %v", node.Name, err)
}
}
Expand Down
46 changes: 12 additions & 34 deletions pkg/controller/node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"reflect"
"strconv"
"strings"
"time"

Expand Down Expand Up @@ -210,46 +211,21 @@ func (c *Controller) handleAddNode(key string) error {
return err
}

var v4CIDRs, v6CIDRs []string
nodeIPv4, nodeIPv6 := util.GetNodeInternalIP(*node)
for _, subnet := range subnets {
if subnet.Spec.Vpc != util.DefaultVpc {
continue
}

var conflict bool
v4, v6 := util.SplitStringIP(subnet.Spec.CIDRBlock)
if util.CIDRContainIP(v4, nodeIPv4) {
if subnet.Spec.Vlan == "" {
conflict = true
} else if subnet.Spec.LogicalGateway {
v4CIDRs = append(v4CIDRs, v4)
}
}
if util.CIDRContainIP(v6, nodeIPv6) {
if subnet.Spec.Vlan == "" {
conflict = true
} else if subnet.Spec.LogicalGateway {
v6CIDRs = append(v6CIDRs, v6)
}
}

if conflict {
if subnet.Spec.Vlan == "" && (util.CIDRContainIP(v4, nodeIPv4) || util.CIDRContainIP(v6, nodeIPv6)) {
msg := fmt.Sprintf("internal IP address of node %s is in CIDR of subnet %s, this may result in network issues", node.Name, subnet.Name)
klog.Warning(msg)
c.recorder.Eventf(&v1.Node{ObjectMeta: metav1.ObjectMeta{Name: node.Name, UID: types.UID(node.Name)}}, v1.EventTypeWarning, "NodeAddressConflictWithSubnet", msg)
break
}
}

if err = c.ovnClient.CreateAddressSetWithAddresses(nodeUnderlayAddressSetName(node.Name, 4), v4CIDRs...); err != nil {
klog.Errorf("failed to create address set for node %s: %v", node.Name, err)
return err
}
if err = c.ovnClient.CreateAddressSetWithAddresses(nodeUnderlayAddressSetName(node.Name, 6), v6CIDRs...); err != nil {
klog.Errorf("failed to create address set for node %s: %v", node.Name, err)
return err
}
if err = c.handleNodeAnnotationsForProviderNetworks(node); err != nil {
klog.Errorf("failed to handle annotations of node %s for provider networks: %v", node.Name, err)
return err
Expand Down Expand Up @@ -294,8 +270,13 @@ func (c *Controller) handleAddNode(key string) error {
nodeIP, af = nodeIPv6, 6
}
if nodeIP != "" {
match := fmt.Sprintf("ip%d.dst == %s && ip%d.src != $%s", af, nodeIP, af, nodeUnderlayAddressSetName(node.Name, af))
if err = c.ovnClient.AddPolicyRoute(c.config.ClusterRouter, util.NodeRouterPolicyPriority, match, "reroute", ip); err != nil {
match := fmt.Sprintf("ip%d.dst == %s", af, nodeIP)
externalIDs := map[string]string{
"vendor": util.CniTypeName,
"node": node.Name,
"address-family": strconv.Itoa(af),
}
if err = c.ovnClient.AddPolicyRoute(c.config.ClusterRouter, util.NodeRouterPolicyPriority, match, "reroute", ip, externalIDs); err != nil {
klog.Errorf("failed to add logical router policy for node %s: %v", node.Name, err)
return err
}
Expand Down Expand Up @@ -591,7 +572,7 @@ func (c *Controller) handleUpdateNode(key string) error {
for _, orisubnet := range subnets {
subnet := orisubnet.DeepCopy()
if util.GatewayContains(subnet.Spec.GatewayNode, node.Name) {
if err := c.reconcileGateway(subnet); err != nil {
if err := c.reconcileOvnRoute(subnet); err != nil {
return err
}
}
Expand Down Expand Up @@ -1076,14 +1057,11 @@ func (c *Controller) addNodeGwStaticRoute() error {

func (c *Controller) getPolicyRouteParas(cidr string) ([]string, map[string]string, error) {
ipSuffix := "ip4"
subnetAsName := getOverlaySubnetsAddressSetName(c.config.ClusterRouter, kubeovnv1.ProtocolIPv4)
if util.CheckProtocol(cidr) == kubeovnv1.ProtocolIPv6 {
ipSuffix = "ip6"
subnetAsName = getOverlaySubnetsAddressSetName(c.config.ClusterRouter, kubeovnv1.ProtocolIPv6)
}
match := fmt.Sprintf("%s.src == %s && %s.dst != $%s", ipSuffix, cidr, ipSuffix, subnetAsName)

nextHops, nameIpMap, err := c.ovnClient.GetPolicyRouteParas(util.CentralSubnetPriority, match)
match := fmt.Sprintf("%s.src == %s", ipSuffix, cidr)
nextHops, nameIpMap, err := c.ovnClient.GetPolicyRouteParas(util.GatewayRouterPolicyPriority, match)
if err != nil {
klog.Errorf("failed to get policy route paras, %v", err)
return nextHops, nameIpMap, err
Expand Down
13 changes: 8 additions & 5 deletions pkg/controller/pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -872,20 +872,23 @@ func (c *Controller) handleUpdatePod(key string) error {
}

ipSuffix := "ip4"
subnetAsName := getOverlaySubnetsAddressSetName(c.config.ClusterRouter, kubeovnv1.ProtocolIPv4)
if util.CheckProtocol(nodeAddr.String()) == kubeovnv1.ProtocolIPv6 {
ipSuffix = "ip6"
subnetAsName = getOverlaySubnetsAddressSetName(c.config.ClusterRouter, kubeovnv1.ProtocolIPv6)
}
pgAs := fmt.Sprintf("%s_%s", pgName, ipSuffix)
match := fmt.Sprintf("%s.src == $%s && %s.dst != $%s", ipSuffix, pgAs, ipSuffix, subnetAsName)
match := fmt.Sprintf("%s.src == $%s", ipSuffix, pgAs)

exist, err := c.ovnClient.PolicyRouteExists(util.PodRouterPolicyPriority, match)
exist, err := c.ovnClient.PolicyRouteExists(util.GatewayRouterPolicyPriority, match)
if err != nil {
return err
}
if !exist {
if err = c.ovnClient.AddPolicyRoute(c.config.ClusterRouter, util.PodRouterPolicyPriority, match, "reroute", nodeAddr.String()); err != nil {
externalIDs := map[string]string{
"vendor": util.CniTypeName,
"subnet": subnet.Name,
"node": node.Name,
}
if err = c.ovnClient.AddPolicyRoute(c.config.ClusterRouter, util.GatewayRouterPolicyPriority, match, "reroute", nodeAddr.String(), externalIDs); err != nil {
klog.Errorf("failed to add logical router policy for port-group address-set %s: %v", pgAs, err)
return err
}
Expand Down
Loading

0 comments on commit ecc8268

Please sign in to comment.