feat: distributed eip

.gitignore

@@ -11,6 +11,7 @@ dist/images/kube-ovn-webhook
 dist/images/kube-ovn-pinger
 dist/images/kube-ovn-speaker
 dist/images/kube-ovn-monitor
+dist/images/kube-ovn-cmd
 kube-ovn.yaml
 kube-ovn-crd.yaml
 ovn.yaml

docs/snat-and-eip.md

@@ -19,6 +19,7 @@ metadata:
   namespace: kube-system
 data:
   enable-external-gw: "true"
+  type: "centralized"                   # centralized or distributed, when centralized external-gw-nodes config below will take effect. When distributed, every node in cluster must have a same nic and eip function will perform in distributed way.
   external-gw-nodes: "kube-ovn-worker"  # NodeName in kubernetes which will act the overlay to underlay gateway functions
   external-gw-nic: "eth1"               # The nic that will be bridged into ovs and act as overlay to underlay gateway
   nic-ip: "172.56.0.1/16"               # The ip and mask of the underlay physical gateway

pkg/controller/external-gw.go

@@ -6,6 +6,7 @@ import (
 	"github.com/alauda/kube-ovn/pkg/util"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/klog"
 	"reflect"
@@ -48,6 +49,7 @@ func (c *Controller) resyncExternalGateway() {
 		}
 		exGwEnabled = "true"
 		lastExGwCM = cm.Data
+		c.ovnClient.ExternalGatewayType = cm.Data["type"]
 		klog.Info("finish establishing ovn external gw")
 	}
 }
@@ -89,7 +91,18 @@ func (c *Controller) removeExternalGateway() error {
 
 func (c *Controller) establishExternalGateway(config map[string]string) error {
 	chassises := []string{}
-	gwNodes := strings.Split(config["external-gw-nodes"], ",")
+	nodes, err := c.nodesLister.List(labels.Everything())
+	if err != nil {
+		klog.Errorf("failed to list nodes, %v", err)
+		return err
+	}
+	gwNodes := make([]string, 0, len(nodes))
+	for _, node := range nodes {
+		gwNodes = append(gwNodes, node.Name)
+	}
+	if config["type"] != "distributed" {
+		gwNodes = strings.Split(config["external-gw-nodes"], ",")
+	}
 	for _, gw := range gwNodes {
 		gw = strings.TrimSpace(gw)
 		node, err := c.nodesLister.Get(gw)

pkg/controller/pod.go

@@ -563,12 +563,12 @@ func (c *Controller) handleUpdatePod(key string) error {
 			}
 
 			for _, ipStr := range strings.Split(podIP, ",") {
-				if err := c.ovnClient.UpdateNatRule("dnat_and_snat", ipStr, pod.Annotations[util.EipAnnotation], c.config.ClusterRouter); err != nil {
+				if err := c.ovnClient.UpdateNatRule("dnat_and_snat", ipStr, pod.Annotations[util.EipAnnotation], c.config.ClusterRouter, pod.Annotations[util.MacAddressAnnotation], fmt.Sprintf("%s.%s", pod.Name, pod.Namespace)); err != nil {
 					klog.Errorf("failed to add nat rules, %v", err)
 					return err
 				}
 
-				if err := c.ovnClient.UpdateNatRule("snat", ipStr, pod.Annotations[util.SnatAnnotation], c.config.ClusterRouter); err != nil {
+				if err := c.ovnClient.UpdateNatRule("snat", ipStr, pod.Annotations[util.SnatAnnotation], c.config.ClusterRouter, "", ""); err != nil {
 					klog.Errorf("failed to add nat rules, %v", err)
 					return err
 				}

pkg/ovs/ovn-nbctl.go

@@ -507,7 +507,7 @@ func parseLrRouteListOutput(output string) (routeList []*StaticRoute, err error)
 	return routeList, nil
 }
 
-func (c Client) UpdateNatRule(policy, logicalIP, externalIP, router string) error {
+func (c Client) UpdateNatRule(policy, logicalIP, externalIP, router, logicalMac, port string) error {
 	if policy == "snat" {
 		if externalIP == "" {
 			_, err := c.ovnNbCommand(IfExists, "lr-nat-del", router, "snat", logicalIP)
@@ -534,8 +534,13 @@ func (c Client) UpdateNatRule(policy, logicalIP, externalIP, router string) erro
 			}
 		}
 		if externalIP != "" {
-			_, err = c.ovnNbCommand(MayExist, "lr-nat-add", router, policy, externalIP, logicalIP)
-			return err
+			if c.ExternalGatewayType == "distributed" {
+				_, err = c.ovnNbCommand(MayExist, "--stateless", "lr-nat-add", router, policy, externalIP, logicalIP, port, logicalMac)
+				return err
+			} else {
+				_, err = c.ovnNbCommand(MayExist, "lr-nat-add", router, policy, externalIP, logicalIP)
+				return err
+			}
 		}
 	}
 	return nil

pkg/ovs/ovn.go

@@ -22,6 +22,7 @@ type Client struct {
 	ClusterUdpSessionLoadBalancer string
 	NodeSwitch                    string
 	NodeSwitchCIDR                string
+	ExternalGatewayType           string
 }
 
 const (