From ddda633204486877d8176476d1bd0470a84c3ecc Mon Sep 17 00:00:00 2001 From: Mengxin Liu Date: Wed, 9 Dec 2020 18:31:42 +0800 Subject: [PATCH] feat: distributed eip --- .gitignore | 1 + docs/snat-and-eip.md | 1 + pkg/controller/external-gw.go | 15 ++++++++++++++- pkg/controller/pod.go | 4 ++-- pkg/ovs/ovn-nbctl.go | 11 ++++++++--- pkg/ovs/ovn.go | 1 + 6 files changed, 27 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 92c330c49c1..3472a614eab 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ dist/images/kube-ovn-webhook dist/images/kube-ovn-pinger dist/images/kube-ovn-speaker dist/images/kube-ovn-monitor +dist/images/kube-ovn-cmd kube-ovn.yaml kube-ovn-crd.yaml ovn.yaml diff --git a/docs/snat-and-eip.md b/docs/snat-and-eip.md index adf2e5087cf..1cb4f27f27c 100644 --- a/docs/snat-and-eip.md +++ b/docs/snat-and-eip.md @@ -19,6 +19,7 @@ metadata: namespace: kube-system data: enable-external-gw: "true" + type: "centralized" # centralized or distributed, when centralized external-gw-nodes config below will take effect. When distributed, every node in cluster must have a same nic and eip function will perform in distributed way. external-gw-nodes: "kube-ovn-worker" # NodeName in kubernetes which will act the overlay to underlay gateway functions external-gw-nic: "eth1" # The nic that will be bridged into ovs and act as overlay to underlay gateway nic-ip: "172.56.0.1/16" # The ip and mask of the underlay physical gateway diff --git a/pkg/controller/external-gw.go b/pkg/controller/external-gw.go index a37f85abb67..819d68dfedd 100644 --- a/pkg/controller/external-gw.go +++ b/pkg/controller/external-gw.go @@ -6,6 +6,7 @@ import ( "github.com/alauda/kube-ovn/pkg/util" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/klog" "reflect" @@ -48,6 +49,7 @@ func (c *Controller) resyncExternalGateway() { } exGwEnabled = "true" lastExGwCM = cm.Data + c.ovnClient.ExternalGatewayType = cm.Data["type"] klog.Info("finish establishing ovn external gw") } } @@ -89,7 +91,18 @@ func (c *Controller) removeExternalGateway() error { func (c *Controller) establishExternalGateway(config map[string]string) error { chassises := []string{} - gwNodes := strings.Split(config["external-gw-nodes"], ",") + nodes, err := c.nodesLister.List(labels.Everything()) + if err != nil { + klog.Errorf("failed to list nodes, %v", err) + return err + } + gwNodes := make([]string, 0, len(nodes)) + for _, node := range nodes { + gwNodes = append(gwNodes, node.Name) + } + if config["type"] != "distributed" { + gwNodes = strings.Split(config["external-gw-nodes"], ",") + } for _, gw := range gwNodes { gw = strings.TrimSpace(gw) node, err := c.nodesLister.Get(gw) diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index 101e273237f..1985d24bf88 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -563,12 +563,12 @@ func (c *Controller) handleUpdatePod(key string) error { } for _, ipStr := range strings.Split(podIP, ",") { - if err := c.ovnClient.UpdateNatRule("dnat_and_snat", ipStr, pod.Annotations[util.EipAnnotation], c.config.ClusterRouter); err != nil { + if err := c.ovnClient.UpdateNatRule("dnat_and_snat", ipStr, pod.Annotations[util.EipAnnotation], c.config.ClusterRouter, pod.Annotations[util.MacAddressAnnotation], fmt.Sprintf("%s.%s", pod.Name, pod.Namespace)); err != nil { klog.Errorf("failed to add nat rules, %v", err) return err } - if err := c.ovnClient.UpdateNatRule("snat", ipStr, pod.Annotations[util.SnatAnnotation], c.config.ClusterRouter); err != nil { + if err := c.ovnClient.UpdateNatRule("snat", ipStr, pod.Annotations[util.SnatAnnotation], c.config.ClusterRouter, "", ""); err != nil { klog.Errorf("failed to add nat rules, %v", err) return err } diff --git a/pkg/ovs/ovn-nbctl.go b/pkg/ovs/ovn-nbctl.go index ffed8b9fa66..54e2f1eb72f 100644 --- a/pkg/ovs/ovn-nbctl.go +++ b/pkg/ovs/ovn-nbctl.go @@ -507,7 +507,7 @@ func parseLrRouteListOutput(output string) (routeList []*StaticRoute, err error) return routeList, nil } -func (c Client) UpdateNatRule(policy, logicalIP, externalIP, router string) error { +func (c Client) UpdateNatRule(policy, logicalIP, externalIP, router, logicalMac, port string) error { if policy == "snat" { if externalIP == "" { _, err := c.ovnNbCommand(IfExists, "lr-nat-del", router, "snat", logicalIP) @@ -534,8 +534,13 @@ func (c Client) UpdateNatRule(policy, logicalIP, externalIP, router string) erro } } if externalIP != "" { - _, err = c.ovnNbCommand(MayExist, "lr-nat-add", router, policy, externalIP, logicalIP) - return err + if c.ExternalGatewayType == "distributed" { + _, err = c.ovnNbCommand(MayExist, "--stateless", "lr-nat-add", router, policy, externalIP, logicalIP, port, logicalMac) + return err + } else { + _, err = c.ovnNbCommand(MayExist, "lr-nat-add", router, policy, externalIP, logicalIP) + return err + } } } return nil diff --git a/pkg/ovs/ovn.go b/pkg/ovs/ovn.go index 50334823faa..69c25d3639e 100644 --- a/pkg/ovs/ovn.go +++ b/pkg/ovs/ovn.go @@ -22,6 +22,7 @@ type Client struct { ClusterUdpSessionLoadBalancer string NodeSwitch string NodeSwitchCIDR string + ExternalGatewayType string } const (