Skip to content

Commit

Permalink
add back centralized subnet active-standby mode
Browse files Browse the repository at this point in the history
  • Loading branch information
@hongzhen-ma
hongzhen-ma committed Jan 14, 2022
1 parent 9bb50e7 commit f210b93
Show file tree
Hide file tree
Showing 3 changed files with 133 additions and 26 deletions.
3 changes: 3 additions & 0 deletions pkg/controller/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ type Configuration struct {
EnableLb bool
EnableNP bool
EnableExternalVpc bool
EnableEcmp bool

ExternalGatewayConfigNS string
ExternalGatewayNet string
Expand Down Expand Up @@ -115,6 +116,7 @@ func ParseFlags() (*Configuration, error) {
argEnableLb = pflag.Bool("enable-lb", true, "Enable load balancer")
argEnableNP = pflag.Bool("enable-np", true, "Enable network policy support")
argEnableExternalVpc = pflag.Bool("enable-external-vpc", true, "Enable external vpc support")
argEnableEcmp = pflag.Bool("enable-ecmp", false, "Enable ecmp route for centralized subnet")

argExternalGatewayConfigNS = pflag.String("external-gateway-config-ns", "kube-system", "The namespace of configmap external-gateway-config, default: kube-system")
argExternalGatewayNet = pflag.String("external-gateway-net", "external", "The namespace of configmap external-gateway-config, default: external")
Expand Down Expand Up @@ -175,6 +177,7 @@ func ParseFlags() (*Configuration, error) {
ExternalGatewayConfigNS: *argExternalGatewayConfigNS,
ExternalGatewayNet: *argExternalGatewayNet,
ExternalGatewayVlanID: *argExternalGatewayVlanID,
EnableEcmp: *argEnableEcmp,
}

if config.NetworkType == util.NetworkTypeVlan && config.DefaultHostInterface == "" {
Expand Down
3 changes: 3 additions & 0 deletions pkg/controller/node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -578,6 +578,9 @@ func (c *Controller) CheckGatewayReady() {
}

func (c *Controller) checkGatewayReady() error {
if !c.config.EnableEcmp {
return nil
}
klog.V(3).Infoln("start to check gateway status")
subnetList, err := c.subnetsLister.List(labels.Everything())
if err != nil {
Expand Down
153 changes: 127 additions & 26 deletions pkg/controller/subnet.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -957,43 +957,100 @@ func (c *Controller) reconcileGateway(subnet *kubeovnv1.Subnet) error {
return fmt.Errorf("failed to add ecmp static route, no gateway node exists")
}

nodeIPs := make([]string, 0, len(strings.Split(subnet.Spec.GatewayNode, ",")))
for _, gw := range strings.Split(subnet.Spec.GatewayNode, ",") {
// the format of gatewayNodeStr can be like 'kube-ovn-worker:172.18.0.2, kube-ovn-control-plane:172.18.0.3', which consists of node name and designative egress ip
if strings.Contains(gw, ":") {
gw = strings.TrimSpace(strings.Split(gw, ":")[0])
} else {
gw = strings.TrimSpace(gw)
if c.config.EnableEcmp {
nodeIPs := make([]string, 0, len(strings.Split(subnet.Spec.GatewayNode, ",")))
for _, gw := range strings.Split(subnet.Spec.GatewayNode, ",") {
// the format of gatewayNodeStr can be like 'kube-ovn-worker:172.18.0.2, kube-ovn-control-plane:172.18.0.3', which consists of node name and designative egress ip
if strings.Contains(gw, ":") {
gw = strings.TrimSpace(strings.Split(gw, ":")[0])
} else {
gw = strings.TrimSpace(gw)
}

node, err := c.nodesLister.Get(gw)
if err == nil && nodeReady(node) {
nodeTunlIP := strings.TrimSpace(node.Annotations[util.IpAddressAnnotation])
if nodeTunlIP == "" {
klog.Errorf("gateway node %v has no ip annotation", node.Name)
continue
}
nodeIPs = append(nodeIPs, strings.Split(nodeTunlIP, ",")...)
} else {
klog.Errorf("gateway node %v is not ready", gw)
}
}
klog.Infof("subnet %s configure gateway node, nodeIPs %v", subnet.Name, nodeIPs)

node, err := c.nodesLister.Get(gw)
if err == nil && nodeReady(node) {
nodeTunlIP := strings.TrimSpace(node.Annotations[util.IpAddressAnnotation])
if nodeTunlIP == "" {
klog.Errorf("gateway node %v has no ip annotation", node.Name)
for _, cidr := range strings.Split(subnet.Spec.CIDRBlock, ",") {
nextHops, err := c.filterRepeatEcmpRoutes(nodeIPs, cidr)
if err != nil {
klog.Errorf("failed to filter ecmp static route for CIDR %s of subnet %s: %v", cidr, subnet.Name, err)
continue
}
nodeIPs = append(nodeIPs, strings.Split(nodeTunlIP, ",")...)
} else {
klog.Errorf("gateway node %v is not ready", gw)
klog.Infof("subnet %s adds centralized gw %v", subnet.Name, nextHops)

for _, nextHop := range nextHops {
if err = c.ovnClient.AddStaticRoute(ovs.PolicySrcIP, cidr, nextHop, c.config.ClusterRouter, util.EcmpRouteType); err != nil {
klog.Errorf("failed to add static route: %v", err)
return err
}
}
}
} else {
if err := c.deleteEcmpRouteForNode(subnet); err != nil {
klog.Errorf("failed to delete ecmp route for subnet %s", subnet.Name)
return err
}
}
klog.Infof("subnet %s configure gateway node, nodeIPs %v", subnet.Name, nodeIPs)

for _, cidr := range strings.Split(subnet.Spec.CIDRBlock, ",") {
nextHops, err := c.filterRepeatEcmpRoutes(nodeIPs, cidr)
if err != nil {
klog.Errorf("failed to filter ecmp static route for CIDR %s of subnet %s: %v", cidr, subnet.Name, err)
continue
// check if activateGateway still ready
if subnet.Status.ActivateGateway != "" {
node, err := c.nodesLister.Get(subnet.Status.ActivateGateway)
if err == nil && nodeReady(node) {
klog.Infof("subnet %s uses the old activate gw %s", subnet.Name, node.Name)
return nil
}
}
klog.Infof("subnet %s adds centralized gw %v", subnet.Name, nextHops)

for _, nextHop := range nextHops {
if err = c.ovnClient.AddStaticRoute(ovs.PolicySrcIP, cidr, nextHop, c.config.ClusterRouter, util.EcmpRouteType); err != nil {
klog.Errorf("failed to add static route: %v", err)
klog.Info("find a new activate node")
// need a new activate gateway
newActivateNode := ""
var nodeTunlIPAddr []net.IP
for _, gw := range strings.Split(subnet.Spec.GatewayNode, ",") {
// the format of gatewayNodeStr can be like 'kube-ovn-worker:172.18.0.2, kube-ovn-control-plane:172.18.0.3', which consists of node name and designative egress ip
if strings.Contains(gw, ":") {
gw = strings.TrimSpace(strings.Split(gw, ":")[0])
} else {
gw = strings.TrimSpace(gw)
}
node, err := c.nodesLister.Get(gw)
if err == nil && nodeReady(node) {
newActivateNode = node.Name
nodeTunlIPAddr, err = getNodeTunlIP(node)
if err != nil {
return err
}
klog.Infof("subnet %s uses a new activate gw %s", subnet.Name, node.Name)
break
}
}
if newActivateNode == "" {
klog.Warningf("all subnet %s gws are not ready", subnet.Name)
subnet.Status.ActivateGateway = newActivateNode
subnet.Status.NotReady("NoReadyGateway", "")
bytes, err := subnet.Status.Bytes()
if err != nil {
return err
}
_, err = c.config.KubeOvnClient.KubeovnV1().Subnets().Patch(context.Background(), subnet.Name, types.MergePatchType, bytes, metav1.PatchOptions{}, "status")
return err
}

nextHop := getNextHopByTunnelIP(nodeTunlIPAddr)
if err := c.ovnClient.AddStaticRoute(ovs.PolicySrcIP, subnet.Spec.CIDRBlock, nextHop, c.config.ClusterRouter, util.NormalRouteType); err != nil {
klog.Errorf("failed to add static route, %v", err)
return err
}
subnet.Status.ActivateGateway = newActivateNode
}

for _, pod := range pods {
Expand Down Expand Up @@ -1296,3 +1353,47 @@ func (c *Controller) checkGwNodeExists(gatewayNode string) bool {
}
return found
}

func (c *Controller) deleteEcmpRouteForNode(subnet *kubeovnv1.Subnet) error {
for _, gw := range strings.Split(subnet.Spec.GatewayNode, ",") {
// the format of gatewayNodeStr can be like 'kube-ovn-worker:172.18.0.2, kube-ovn-control-plane:172.18.0.3', which consists of node name and designative egress ip
if strings.Contains(gw, ":") {
gw = strings.TrimSpace(strings.Split(gw, ":")[0])
} else {
gw = strings.TrimSpace(gw)
}
node, err := c.nodesLister.Get(gw)
if err != nil {
continue
}

ipStr := node.Annotations[util.IpAddressAnnotation]
for _, ip := range strings.Split(ipStr, ",") {
var cidrBlock string
for _, cidrBlock = range strings.Split(subnet.Spec.CIDRBlock, ",") {
if util.CheckProtocol(cidrBlock) != util.CheckProtocol(ip) {
continue
}

exist, err := c.checkNodeEcmpRouteExist(ip, cidrBlock)
if err != nil {
klog.Errorf("get ecmp static route for subnet %v, error %v", subnet.Name, err)
break
}

if exist {
if subnet.Status.ActivateGateway != "" && subnet.Status.ActivateGateway == gw {
continue
}

klog.Infof("subnet %v changed to active-standby mode, delete ecmp route for node %s, ip %v", subnet.Name, node.Name, ip)
if err := c.ovnClient.DeleteMatchedStaticRoute(cidrBlock, ip, c.config.ClusterRouter); err != nil {
klog.Errorf("failed to delete static route %s for node %s, %v", ip, node.Name, err)
return err
}
}
}
}
}
return nil
}

0 comments on commit f210b93

Please sign in to comment.