Skip to content

Commit

Permalink
fix adding OVN routes in dual stack Kubernetes
Browse files Browse the repository at this point in the history
  • Loading branch information
@zhangzujian
zhangzujian committed Sep 8, 2021
1 parent c114778 commit 7c5fed6
Show file tree
Hide file tree
Showing 3 changed files with 66 additions and 42 deletions.
42 changes: 26 additions & 16 deletions pkg/controller/subnet.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -945,19 +945,23 @@ func (c *Controller) reconcileGateway(subnet *kubeovnv1.Subnet) error {
klog.Errorf("gateway node %v has no ip annotation", node.Name)
continue
}
nodeIPs = append(nodeIPs, nodeTunlIP)
nodeIPs = append(nodeIPs, strings.Split(nodeTunlIP, ",")...)
}
}
nodeIPs, err = c.filterRepeatEcmpRoutes(nodeIPs, subnet.Spec.CIDRBlock)
if err != nil {
klog.Errorf("filter ecmp static route for subnet %v, error %v", subnet.Name, err)
}
klog.Infof("subnet %s adds centralized gw %v", subnet.Name, nodeIPs)

for _, nextHop := range nodeIPs {
if err := c.ovnClient.AddStaticRoute(ovs.PolicySrcIP, subnet.Spec.CIDRBlock, nextHop, c.config.ClusterRouter, util.EcmpRouteType); err != nil {
klog.Errorf("failed to add static route, %v", err)
return err
for _, cidr := range strings.Split(subnet.Spec.CIDRBlock, ",") {
nextHops, err := c.filterRepeatEcmpRoutes(nodeIPs, cidr)
if err != nil {
klog.Errorf("failed to filter ecmp static route for CIDR %s of subnet %s: %v", cidr, subnet.Name, err)
continue
}
klog.Infof("subnet %s adds centralized gw %v", subnet.Name, nextHops)

for _, nextHop := range nextHops {
if err = c.ovnClient.AddStaticRoute(ovs.PolicySrcIP, cidr, nextHop, c.config.ClusterRouter, util.EcmpRouteType); err != nil {
klog.Errorf("failed to add static route: %v", err)
return err
}
}
}

Expand Down Expand Up @@ -1197,26 +1201,31 @@ func filterRepeatIPRange(mapIps map[string]ipam.IPRange) map[string]ipam.IPRange
return mapIps
}

func (c *Controller) filterRepeatEcmpRoutes(nodeIps []string, cidrBlock string) ([]string, error) {
func (c *Controller) filterRepeatEcmpRoutes(nodeIps []string, cidr string) ([]string, error) {
var nextHops []string
routes, err := c.ovnClient.GetStaticRouteList(c.config.ClusterRouter)
if err != nil {
klog.Errorf("failed to list static route %v", err)
klog.Errorf("failed to list static route: %v", err)
return nextHops, err
}
if len(nodeIps) == 0 {
return nextHops, fmt.Errorf("nexthop is nil for add ecmp static route")
}

protocol := util.CheckProtocol(cidr)
for _, nodeIp := range nodeIps {
found := false
if util.CheckProtocol(nodeIp) != protocol {
continue
}

var found bool
for _, route := range routes {
if route.Policy != ovs.PolicySrcIP || route.CIDR != cidrBlock {
if route.Policy != ovs.PolicySrcIP || route.CIDR != cidr {
continue
}

if route.NextHop == nodeIp {
klog.Infof("src-ip static route exist for cidr %s, nexthop %v", cidrBlock, nodeIp)
klog.Infof("src-ip static route exist for cidr %s, nexthop %s", cidr, nodeIp)
found = true
break
}
Expand All @@ -1225,7 +1234,8 @@ func (c *Controller) filterRepeatEcmpRoutes(nodeIps []string, cidrBlock string)
nextHops = append(nextHops, nodeIp)
}
}
klog.Infof("ecmp static route to add, cidr %s, nexthop %v", cidrBlock, nextHops)

klog.Infof("ecmp static route to add, cidr %s, nexthop %v", cidr, nextHops)
return nextHops, nil
}

Expand Down
20 changes: 6 additions & 14 deletions test/e2e/e2e_suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,16 @@ import (
"strconv"
"testing"

kubeovn "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
"github.com/kubeovn/kube-ovn/test/e2e/framework"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"

kubeovn "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
"github.com/kubeovn/kube-ovn/pkg/util"
"github.com/kubeovn/kube-ovn/test/e2e/framework"

// tests to run
_ "github.com/kubeovn/kube-ovn/test/e2e/ip"
_ "github.com/kubeovn/kube-ovn/test/e2e/kubectl-ko"
Expand Down Expand Up @@ -162,20 +164,10 @@ var _ = SynchronizedBeforeSuite(func() []byte {
}

for i := range nodes.Items {
var nodeIP string
for _, addr := range nodes.Items[i].Status.Addresses {
if addr.Type == corev1.NodeInternalIP {
nodeIP = addr.Address
break
}
}
if nodeIP == "" {
Fail("failed to get IP of node " + nodes.Items[i].Name)
}

var cniPod *corev1.Pod
nodeIPv4, nodeIPv6 := util.GetNodeInternalIP(nodes.Items[i])
for _, pod := range cniPods.Items {
if pod.Status.HostIP == nodeIP {
if pod.Status.HostIP == nodeIPv4 || pod.Status.HostIP == nodeIPv6 {
cniPod = &pod
break
}
Expand Down
46 changes: 34 additions & 12 deletions test/e2e/subnet/normal.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -311,18 +311,32 @@ var _ = Describe("[Subnet]", func() {
}

name := f.GetName()
af, cidr, nodeAddrPrefix := 4, "11.15.0.0/16", 16
priority, tableID := uint32(1001), uint32(1002)

af, cidr := 4, "11.15.0.0/16"
if isIPv6 {
af, cidr, nodeAddrPrefix = 6, "fd00:11:15::/112", 64
af, cidr = 6, "fd00:11:15::/112"
}

var egw string
nodeIPv4, nodeIPv6 := util.GetNodeInternalIP(nodes.Items[0])
if isIPv6 {
egw, _ = util.FirstIP(fmt.Sprintf("%s/%d", nodeIPv6, 64))
} else {
egw, _ = util.FirstIP(fmt.Sprintf("%s/%d", nodeIPv4, 16))
}
egw, _ := util.FirstIP(fmt.Sprintf("%s/%d", nodes.Items[0].Status.Addresses[0].Address, nodeAddrPrefix))
priority, tableID := uint32(1001), uint32(1002)

gatewayNodes := make([]string, 0, 2)
nodeIPs := make(map[string]string)
for i := 0; i < 2 && i < len(nodes.Items); i++ {
gatewayNodes = append(gatewayNodes, nodes.Items[i].Name)
nodeIPs[nodes.Items[i].Status.Addresses[0].Address] = gatewayNodes[i]
nodeIPv4, nodeIPv6 := util.GetNodeInternalIP(nodes.Items[i])
if nodeIPv4 != "" {
nodeIPs[nodeIPv4] = gatewayNodes[i]
}
if nodeIPv6 != "" {
nodeIPs[nodeIPv6] = gatewayNodes[i]
}
}

By("create subnet")
Expand Down Expand Up @@ -435,12 +449,12 @@ var _ = Describe("[Subnet]", func() {
Expect(err).NotTo(HaveOccurred())

name := f.GetName()
af, cidr, nodeAddrPrefix := 4, "11.16.0.0/16", 16
priority, tableID := uint32(1003), uint32(1004)

af, cidr := 4, "11.16.0.0/16"
if isIPv6 {
af, cidr, nodeAddrPrefix = 6, "fd00:11:16::/112", 64
af, cidr = 6, "fd00:11:16::/112"
}
egw, _ := util.FirstIP(fmt.Sprintf("%s/%d", nodes.Items[0].Status.Addresses[0].Address, nodeAddrPrefix))
priority, tableID := uint32(1003), uint32(1004)

var selectedNode *corev1.Node
for i, node := range nodes.Items {
Expand All @@ -462,6 +476,14 @@ var _ = Describe("[Subnet]", func() {
}
Expect(selectedNode).NotTo(BeNil())

var egw string
nodeIPv4, nodeIPv6 := util.GetNodeInternalIP(*selectedNode)
if isIPv6 {
egw, _ = util.FirstIP(fmt.Sprintf("%s/%d", nodeIPv6, 64))
} else {
egw, _ = util.FirstIP(fmt.Sprintf("%s/%d", nodeIPv4, 16))
}

By("create subnet")
s := kubeovn.Subnet{
ObjectMeta: metav1.ObjectMeta{
Expand Down Expand Up @@ -530,7 +552,7 @@ var _ = Describe("[Subnet]", func() {
routePrefix := fmt.Sprintf("default via %s ", egw)

for _, ovsPod := range ovsPods.Items {
if ovsPod.Status.HostIP != selectedNode.Status.Addresses[0].Address {
if ovsPod.Status.HostIP != nodeIPv4 && ovsPod.Status.HostIP != nodeIPv6 {
continue
}

Expand Down Expand Up @@ -561,7 +583,7 @@ var _ = Describe("[Subnet]", func() {
time.Sleep(1 * time.Second)

for _, ovsPod := range ovsPods.Items {
if ovsPod.Status.HostIP != selectedNode.Status.Addresses[0].Address {
if ovsPod.Status.HostIP != nodeIPv4 && ovsPod.Status.HostIP != nodeIPv6 {
continue
}

Expand Down Expand Up @@ -589,7 +611,7 @@ var _ = Describe("[Subnet]", func() {
time.Sleep(5 * time.Second)

for _, ovsPod := range ovsPods.Items {
if ovsPod.Status.HostIP != selectedNode.Status.Addresses[0].Address {
if ovsPod.Status.HostIP != nodeIPv4 && ovsPod.Status.HostIP != nodeIPv6 {
continue
}

Expand Down

0 comments on commit 7c5fed6

Please sign in to comment.