feat: gw switch from overlay to underlay

kubeovn · Aug 9, 2020 · 9535c26 · 9535c26
1 parent 4b09558
commit 9535c26
Show file tree

Hide file tree

Showing 8 changed files with 147 additions and 107 deletions.
diff --git a/dist/images/install-pre-1.16.sh b/dist/images/install-pre-1.16.sh
@@ -10,7 +10,7 @@ JOIN_CIDR="100.64.0.0/16"              # Do NOT overlap with NODE/POD/SVC CIDR
 LABEL="node-role.kubernetes.io/master" # The node label to deploy OVN DB
 IFACE=""                               # The nic to support container network, if empty will use the nic that the default route use
 NETWORK_TYPE="geneve"                  # geneve or vlan
-VERSION="v1.3.0"
+VERSION="v1.4.0"
 IMAGE_PULL_POLICY="IfNotPresent"
 
 # VLAN Config only take effect when NETWORK_TYPE is vlan

diff --git a/dist/images/install.sh b/dist/images/install.sh
@@ -22,7 +22,7 @@ EXCLUDE_IPS=""                         # EXCLUDE_IPS for default subnet
 LABEL="node-role.kubernetes.io/master" # The node label to deploy OVN DB
 IFACE=""                               # The nic to support container network, if empty will use the nic that the default route use
 NETWORK_TYPE="geneve"                  # geneve or vlan
-VERSION="v1.3.0"
+VERSION="v1.4.0"
 IMAGE_PULL_POLICY="IfNotPresent"
 HW_OFFLOAD="false"
 

diff --git a/pkg/controller/gc.go b/pkg/controller/gc.go
@@ -139,7 +139,7 @@ func (c *Controller) markAndCleanLSP() error {
 				noPodLSP[lsp] = true
 			} else {
 				klog.Infof("gc logical switch port %s", lsp)
-				if err := c.ovnClient.DeletePort(lsp); err != nil {
+				if err := c.ovnClient.DeleteLogicalSwitchPort(lsp); err != nil {
 					klog.Errorf("failed to delete lsp %s, %v", lsp, err)
 					return err
 				}

diff --git a/pkg/controller/node.go b/pkg/controller/node.go
@@ -312,7 +312,7 @@ func (c *Controller) handleAddNode(key string) error {
 
 func (c *Controller) handleDeleteNode(key string) error {
 	portName := fmt.Sprintf("node-%s", key)
-	if err := c.ovnClient.DeletePort(portName); err != nil {
+	if err := c.ovnClient.DeleteLogicalSwitchPort(portName); err != nil {
 		klog.Errorf("failed to delete node switch port node-%s %v", key, err)
 		return err
 	}

diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go
@@ -423,7 +423,7 @@ func (c *Controller) handleDeletePod(key string) error {
 		}
 	}
 
-	if err := c.ovnClient.DeletePort(ovs.PodNameToPortName(name, namespace)); err != nil {
+	if err := c.ovnClient.DeleteLogicalSwitchPort(ovs.PodNameToPortName(name, namespace)); err != nil {
 		klog.Errorf("failed to delete lsp %s, %v", ovs.PodNameToPortName(name, namespace), err)
 		return err
 	}
@@ -458,18 +458,17 @@ func (c *Controller) handleUpdatePod(key string) error {
 	klog.Infof("update pod %s/%s", namespace, name)
 	podIP := pod.Annotations[util.IpAddressAnnotation]
 
-	if pod.Annotations[util.NorthGatewayAnnotation] != "" {
-		if err := c.ovnClient.AddStaticRoute(ovs.PolicySrcIP, podIP, pod.Annotations[util.NorthGatewayAnnotation], c.config.ClusterRouter); err != nil {
-			return errors.Annotate(err, "add static route failed")
-		}
-	} else {
-		subnet, err := c.getPodDefaultSubnet(pod)
-		if err != nil {
-			klog.Errorf("failed to get subnet %v", err)
-			return err
-		}
-
-		if !subnet.Spec.UnderlayGateway {
+	subnet, err := c.getPodDefaultSubnet(pod)
+	if err != nil {
+		klog.Errorf("failed to get subnet %v", err)
+		return err
+	}
+	if !subnet.Spec.UnderlayGateway {
+		if pod.Annotations[util.NorthGatewayAnnotation] != "" {
+			if err := c.ovnClient.AddStaticRoute(ovs.PolicySrcIP, podIP, pod.Annotations[util.NorthGatewayAnnotation], c.config.ClusterRouter); err != nil {
+				return errors.Annotate(err, "add static route failed")
+			}
+		} else {
 			if subnet.Spec.GatewayType == kubeovnv1.GWDistributedType {
 				node, err := c.nodesLister.Get(pod.Spec.NodeName)
 				if err != nil {

diff --git a/pkg/controller/subnet.go b/pkg/controller/subnet.go
@@ -5,7 +5,6 @@ import (
 	kubeovnv1 "github.com/alauda/kube-ovn/pkg/apis/kubeovn/v1"
 	"github.com/alauda/kube-ovn/pkg/ovs"
 	"github.com/alauda/kube-ovn/pkg/util"
-	"github.com/juju/errors"
 	"net"
 	"reflect"
 	"strconv"
@@ -639,116 +638,147 @@ func (c *Controller) reconcileGateway(subnet *kubeovnv1.Subnet) error {
 		return err
 	}
 
-	// if gw is distributed remove activateGateway field
-	if subnet.Spec.GatewayType == kubeovnv1.GWDistributedType {
-		if subnet.Status.ActivateGateway == "" {
-			return nil
+	if subnet.Spec.UnderlayGateway {
+		for _, pod := range pods {
+			if pod.Annotations[util.LogicalSwitchAnnotation] == subnet.Name && pod.Annotations[util.IpAddressAnnotation] != "" {
+				if err := c.ovnClient.DeleteStaticRoute(pod.Annotations[util.IpAddressAnnotation], c.config.ClusterRouter); err != nil {
+					klog.Errorf("failed to delete route %s, %v", pod.Annotations[util.IpAddressAnnotation], err)
+					return err
+				}
+			}
 		}
-		subnet.Status.ActivateGateway = ""
-		bytes, err := subnet.Status.Bytes()
-		if err != nil {
+		if err := c.ovnClient.DeleteStaticRoute(subnet.Spec.CIDRBlock, c.config.ClusterRouter); err != nil {
+			klog.Errorf("failed to delete route %s, %v", subnet.Spec.CIDRBlock, err)
 			return err
 		}
-		_, err = c.config.KubeOvnClient.KubeovnV1().Subnets().Patch(subnet.Name, types.MergePatchType, bytes, "status")
-		if err != nil {
+
+		if err := c.ovnClient.DeleteLogicalSwitchPort(fmt.Sprintf("%s-%s", subnet.Name, c.config.ClusterRouter)); err != nil {
+			klog.Errorf("failed to delete lsp %s-%s, %v", subnet.Name, c.config.ClusterRouter, err)
 			return err
 		}
-
-		for _, pod := range pods {
-			if !isPodAlive(pod) || pod.Annotations[util.IpAddressAnnotation] == "" || pod.Annotations[util.LogicalSwitchAnnotation] != subnet.Name {
-				continue
+		if err := c.ovnClient.DeleteLogicalRouterPort(fmt.Sprintf("%s-%s", c.config.ClusterRouter, subnet.Name)); err != nil {
+			klog.Errorf("failed to delete lrp %s-%s, %v", c.config.ClusterRouter, subnet.Name, err)
+			return err
+		}
+	} else {
+		// if gw is distributed remove activateGateway field
+		if subnet.Spec.GatewayType == kubeovnv1.GWDistributedType {
+			if subnet.Status.ActivateGateway == "" {
+				return nil
 			}
-
-			node, err := c.nodesLister.Get(pod.Spec.NodeName)
+			subnet.Status.ActivateGateway = ""
+			bytes, err := subnet.Status.Bytes()
+			if err != nil {
+				return err
+			}
+			_, err = c.config.KubeOvnClient.KubeovnV1().Subnets().Patch(subnet.Name, types.MergePatchType, bytes, "status")
 			if err != nil {
-				if k8serrors.IsNotFound(err) {
+				return err
+			}
+
+			for _, pod := range pods {
+				if !isPodAlive(pod) || pod.Annotations[util.IpAddressAnnotation] == "" || pod.Annotations[util.LogicalSwitchAnnotation] != subnet.Name {
 					continue
-				} else {
-					klog.Errorf("failed to get node %s, %v", pod.Spec.NodeName, err)
+				}
+
+				node, err := c.nodesLister.Get(pod.Spec.NodeName)
+				if err != nil {
+					if k8serrors.IsNotFound(err) {
+						continue
+					} else {
+						klog.Errorf("failed to get node %s, %v", pod.Spec.NodeName, err)
+						return err
+					}
+				}
+				gw, err := getNodeTunlIP(node)
+				if err != nil {
+					klog.Errorf("failed to get node %s tunl ip, %v", node.Name, err)
+					return err
+				}
+				nextHop := gw.String()
+
+				if pod.Annotations[util.NorthGatewayAnnotation] != "" {
+					nextHop = pod.Annotations[util.NorthGatewayAnnotation]
+				}
+
+				if err := c.ovnClient.AddStaticRoute(ovs.PolicySrcIP, pod.Annotations[util.IpAddressAnnotation], nextHop, c.config.ClusterRouter); err != nil {
+					klog.Errorf("add static route failed, %v", err)
 					return err
 				}
 			}
-			gw, err := getNodeTunlIP(node)
-			if err != nil {
-				klog.Errorf("failed to get node %s tunl ip, %v", node.Name, err)
+			if err := c.ovnClient.DeleteStaticRoute(subnet.Spec.CIDRBlock, c.config.ClusterRouter); err != nil {
+				klog.Errorf("failed to delete static route %s, %v", subnet.Spec.CIDRBlock, err)
 				return err
 			}
-			nextHop := gw.String()
+			return nil
+		} else {
+			klog.Infof("start to init centralized gateway for subnet %s", subnet.Name)
+
+			// check if activateGateway still ready
+			if subnet.Status.ActivateGateway != "" {
+				node, err := c.nodesLister.Get(subnet.Status.ActivateGateway)
+				if err == nil && nodeReady(node) {
+					klog.Infof("subnet %s uses the old activate gw %s", subnet.Name, node.Name)
+					return nil
+				}
+			}
 
-			if pod.Annotations[util.NorthGatewayAnnotation] != "" {
-				nextHop = pod.Annotations[util.NorthGatewayAnnotation]
+			klog.Info("find a new activate node")
+			// need a new activate gateway
+			newActivateNode := ""
+			var nodeTunlIPAddr net.IP
+			for _, gw := range strings.Split(subnet.Spec.GatewayNode, ",") {
+				gw = strings.TrimSpace(gw)
+				node, err := c.nodesLister.Get(gw)
+				if err == nil && nodeReady(node) {
+					newActivateNode = node.Name
+					nodeTunlIPAddr, err = getNodeTunlIP(node)
+					if err != nil {
+						return err
+					}
+					klog.Infof("subnet %s uses a new activate gw %s", subnet.Name, node.Name)
+					break
+				}
+			}
+			if newActivateNode == "" {
+				klog.Warningf("all subnet %s gws are not ready", subnet.Name)
+				subnet.Status.ActivateGateway = newActivateNode
+				subnet.Status.NotReady("NoReadyGateway", "")
+				bytes, err := subnet.Status.Bytes()
+				if err != nil {
+					return err
+				}
+				_, err = c.config.KubeOvnClient.KubeovnV1().Subnets().Patch(subnet.Name, types.MergePatchType, bytes, "status")
+				return err
 			}
 
-			if err := c.ovnClient.AddStaticRoute(ovs.PolicySrcIP, pod.Annotations[util.IpAddressAnnotation], nextHop, c.config.ClusterRouter); err != nil {
-				return errors.Annotate(err, "add static route failed")
+			if !subnet.Spec.UnderlayGateway {
+				if err := c.ovnClient.AddStaticRoute(ovs.PolicySrcIP, subnet.Spec.CIDRBlock, nodeTunlIPAddr.String(), c.config.ClusterRouter); err != nil {
+					klog.Errorf("failed to add static route, %v", err)
+					return err
+				}
 			}
-		}
-		if err := c.ovnClient.DeleteStaticRoute(subnet.Spec.CIDRBlock, c.config.ClusterRouter); err != nil {
-			klog.Errorf("failed to delete static route %s, %v", subnet.Spec.CIDRBlock, err)
-			return err
-		}
-		return nil
-	}
-	klog.Infof("start to init centralized gateway for subnet %s", subnet.Name)
 
-	// check if activateGateway still ready
-	if subnet.Status.ActivateGateway != "" {
-		node, err := c.nodesLister.Get(subnet.Status.ActivateGateway)
-		if err == nil && nodeReady(node) {
-			klog.Infof("subnet %s uses the old activate gw %s", subnet.Name, node.Name)
-			return nil
-		}
-	}
+			for _, pod := range pods {
+				if isPodAlive(pod) && pod.Annotations[util.IpAddressAnnotation] != "" && pod.Annotations[util.LogicalSwitchAnnotation] == subnet.Name && pod.Annotations[util.NorthGatewayAnnotation] == "" {
+					if err := c.ovnClient.DeleteStaticRoute(pod.Annotations[util.IpAddressAnnotation], c.config.ClusterRouter); err != nil {
+						klog.Errorf("failed to delete static route, %v", err)
+						return err
+					}
+				}
+			}
 
-	klog.Info("find a new activate node")
-	// need a new activate gateway
-	newActivateNode := ""
-	var nodeTunlIPAddr net.IP
-	for _, gw := range strings.Split(subnet.Spec.GatewayNode, ",") {
-		gw = strings.TrimSpace(gw)
-		node, err := c.nodesLister.Get(gw)
-		if err == nil && nodeReady(node) {
-			newActivateNode = node.Name
-			nodeTunlIPAddr, err = getNodeTunlIP(node)
+			subnet.Status.ActivateGateway = newActivateNode
+			bytes, err := subnet.Status.Bytes()
+			subnet.Status.Ready("ReconcileCentralizedGatewaySuccess", "")
 			if err != nil {
 				return err
 			}
-			klog.Infof("subnet %s uses a new activate gw %s", subnet.Name, node.Name)
-			break
-		}
-	}
-	if newActivateNode == "" {
-		klog.Warningf("all subnet %s gws are not ready", subnet.Name)
-		subnet.Status.ActivateGateway = newActivateNode
-		subnet.Status.NotReady("NoReadyGateway", "")
-		bytes, err := subnet.Status.Bytes()
-		if err != nil {
+			_, err = c.config.KubeOvnClient.KubeovnV1().Subnets().Patch(subnet.Name, types.MergePatchType, bytes, "status")
 			return err
 		}
-		_, err = c.config.KubeOvnClient.KubeovnV1().Subnets().Patch(subnet.Name, types.MergePatchType, bytes, "status")
-		return err
 	}
-
-	if err := c.ovnClient.AddStaticRoute(ovs.PolicySrcIP, subnet.Spec.CIDRBlock, nodeTunlIPAddr.String(), c.config.ClusterRouter); err != nil {
-		return errors.Annotate(err, "add static route failed")
-	}
-	for _, pod := range pods {
-		if isPodAlive(pod) && pod.Annotations[util.IpAddressAnnotation] != "" && pod.Annotations[util.LogicalSwitchAnnotation] == subnet.Name && pod.Annotations[util.NorthGatewayAnnotation] == "" {
-			if err := c.ovnClient.DeleteStaticRoute(pod.Annotations[util.IpAddressAnnotation], c.config.ClusterRouter); err != nil {
-				klog.Errorf("failed to delete static route, %v", err)
-				return err
-			}
-		}
-	}
-
-	subnet.Status.ActivateGateway = newActivateNode
-	bytes, err := subnet.Status.Bytes()
-	subnet.Status.Ready("ReconcileCentralizedGatewaySuccess", "")
-	if err != nil {
-		return err
-	}
-	_, err = c.config.KubeOvnClient.KubeovnV1().Subnets().Patch(subnet.Name, types.MergePatchType, bytes, "status")
-	return err
+	return nil
 }
 
 func (c *Controller) reconcileVlan(subnet *kubeovnv1.Subnet) error {

diff --git a/pkg/controller/vlan.go b/pkg/controller/vlan.go
@@ -331,7 +331,7 @@ func (c *Controller) addLocalnet(subnet *kubeovnv1.Subnet) error {
 func (c *Controller) delLocalnet(key string) error {
 	localnetPort := ovs.PodNameToLocalnetName(key)
 
-	if err := c.ovnClient.DeletePort(localnetPort); err != nil {
+	if err := c.ovnClient.DeleteLogicalSwitchPort(localnetPort); err != nil {
 		klog.Errorf("failed to delete localnet port %s, %v", localnetPort, err)
 		return err
 	}

diff --git a/pkg/ovs/ovn-nbctl.go b/pkg/ovs/ovn-nbctl.go
@@ -26,10 +26,18 @@ func (c Client) ovnNbCommand(cmdArgs ...string) (string, error) {
 	return trimCommandOutput(raw), nil
 }
 
-// DeletePort delete logical switch port in ovn
-func (c Client) DeletePort(port string) error {
+// DeleteLogicalSwitchPort delete logical switch port in ovn
+func (c Client) DeleteLogicalSwitchPort(port string) error {
 	if _, err := c.ovnNbCommand(IfExists, "lsp-del", port); err != nil {
-		return fmt.Errorf("failed to delete port %s, %v", port, err)
+		return fmt.Errorf("failed to delete logical switch port %s, %v", port, err)
+	}
+	return nil
+}
+
+// DeleteLogicalRouterPort delete logical switch port in ovn
+func (c Client) DeleteLogicalRouterPort(port string) error {
+	if _, err := c.ovnNbCommand(IfExists, "lrp-del", port); err != nil {
+		return fmt.Errorf("failed to delete logical router port %s, %v", port, err)
 	}
 	return nil
 }
@@ -274,6 +282,9 @@ func (c Client) AddStaticRoute(policy, cidr, nextHop, router string) error {
 
 // DeleteStaticRoute delete a static route rule in ovn
 func (c Client) DeleteStaticRoute(cidr, router string) error {
+	if cidr == "" {
+		return nil
+	}
 	_, err := c.ovnNbCommand(IfExists, "lr-route-del", router, cidr)
 	return err
 }