Implement centralized gateway.

Outgoing traffic can use the given node as the gateway.
kubeovn · Apr 16, 2019 · 24cda41 · 24cda41
1 parent e1ff5c7
commit 24cda41
Show file tree

Hide file tree

Showing 4 changed files with 108 additions and 43 deletions.
diff --git a/docs/gateway.md b/docs/gateway.md
@@ -0,0 +1,54 @@
+# Outgoing traffic
+
+A gateway is been used to offer public network access fro ovn networked pods.
+kube-ovn offers two kinds of the gateway, distributed gateway, and centralized gateway.
+
+For a distributed gateway, outgoing traffic from ovn networked containers to destinations outside of ovn cidr or cluster ip range will be masqueraded with pod's host.
+
+For a centralized gateway, outgoing traffic from ovn networked containers to destinations outside of ovn cidr or cluster ip range will be masqueraded with pod namespace given host.
+
+## Example
+
+Create the following namespace.
+
+```yaml
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: testns
+  annotations:
+    ovn.kubernetes.io/gateway_type: centralized // or distributed by default
+    ovn.kubernetes.io/gateway_node: node1 // if using centralied gateway, a gateway_node is needed
+```
+
+Creat the demo pods.
+
+```yaml
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: app1
+  namespace: testns
+  labels:
+    app: app1
+spec:
+  selector:
+    matchLabels:
+      name: app1
+  template:
+    metadata:
+      labels:
+        name: app1
+    spec:
+      containers:
+      - name: toolbox
+        image: halfcrazy/toolbox
+```
+
+then open two terminal, one on master
+
+`kubectl -n testns exec -it app1-xxxx ping 114.114.114.114`
+
+and one on node1
+
+`tcpdump -n -i eth0 icmp and host 114.114.114.114`
diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go
@@ -356,26 +356,15 @@ func (c *Controller) handleDeletePod(key string) error {
 		utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key))
 		return nil
 	}
-	ns, err := c.namespacesLister.Get(namespace)
+	portAddr, err := c.ovnClient.GetPortAddr(ovs.PodNameToPortName(name, namespace))
 	if err != nil {
-		klog.Errorf("get namespace %s failed %v", namespace, err)
-		return err
-	}
-	nsGWType := ns.Annotations[util.GWTypeAnnotation]
-	switch nsGWType {
-	case "", util.GWDistributedMode:
-		portAddr, err := c.ovnClient.GetPortAddr(ovs.PodNameToPortName(name, namespace))
-		if err != nil {
-			if strings.Contains(err.Error(), "no row") {
-				break
-			}
+		if !strings.Contains(err.Error(), "no row") {
 			return err
 		}
+	} else {
 		if err := c.ovnClient.DeleteStaticRouter(portAddr[1], c.config.ClusterRouter); err != nil {
 			return err
 		}
-	case util.GWCentralizedMode:
-		// TODO:
 	}
 	pod, err := c.podsLister.Pods(namespace).Get(name)
 	if err != nil {
@@ -427,29 +416,35 @@ func (c *Controller) handleUpdatePod(key string) error {
 		klog.Errorf("get namespace %s failed %v", namespace, err)
 		return err
 	}
-
-	node, err := c.nodesLister.Get(pod.Spec.NodeName)
-	if err != nil {
-		klog.Errorf("get node %s failed %v", pod.Spec.NodeName, err)
-		return err
-	}
-	nodeTunlIP := node.Annotations[util.IpAddressAnnotation]
-	if nodeTunlIP == "" {
-		klog.Errorf("node %s has no tunl ip annotation", pod.Spec.NodeName)
-		return nil
-	}
-	nodeTunlIPAddr, _, err := net.ParseCIDR(nodeTunlIP)
-	if err != nil {
-		return errors.Annotatef(err, "parse node tunl ip %s faield", nodeTunlIP)
-	}
 	nsGWType := ns.Annotations[util.GWTypeAnnotation]
 	switch nsGWType {
 	case "", util.GWDistributedMode:
+		node, err := c.nodesLister.Get(pod.Spec.NodeName)
+		if err != nil {
+			klog.Errorf("get node %s failed %v", pod.Spec.NodeName, err)
+			return err
+		}
+		nodeTunlIPAddr, err := getNodeTunlIP(node)
+		if err != nil {
+			return err
+		}
 		if err := c.ovnClient.AddStaticRouter(ovs.PolicySrcIP, pod.Status.PodIP, nodeTunlIPAddr.String(), c.config.ClusterRouter); err != nil {
 			return errors.Annotate(err, "add static route failed")
 		}
 	case util.GWCentralizedMode:
-		// TODO:
+		gatewayNode := ns.Annotations[util.GWNode]
+		node, err := c.nodesLister.Get(gatewayNode)
+		if err != nil {
+			klog.Errorf("get node %s failed %v", pod.Spec.NodeName, err)
+			return err
+		}
+		nodeTunlIPAddr, err := getNodeTunlIP(node)
+		if err != nil {
+			return err
+		}
+		if err := c.ovnClient.AddStaticRouter(ovs.PolicySrcIP, pod.Status.PodIP, nodeTunlIPAddr.String(), c.config.ClusterRouter); err != nil {
+			return errors.Annotate(err, "add static route failed")
+		}
 	}
 	return nil
 }
@@ -462,3 +457,15 @@ func isStatefulSetPod(pod *v1.Pod) bool {
 	}
 	return false
 }
+
+func getNodeTunlIP(node *v1.Node) (net.IP, error) {
+	nodeTunlIP := node.Annotations[util.IpAddressAnnotation]
+	if nodeTunlIP == "" {
+		return nil, errors.New("node has no tunl ip annotation")
+	}
+	nodeTunlIPAddr, _, err := net.ParseCIDR(nodeTunlIP)
+	if err != nil {
+		return nil, errors.Annotatef(err, "parse node tunl ip %s faield", nodeTunlIP)
+	}
+	return nodeTunlIPAddr, nil
+}
diff --git a/pkg/daemon/gateway.go b/pkg/daemon/gateway.go
@@ -120,18 +120,24 @@ func (c *Controller) getLocalPodIPsNeedNAT() ([]string, error) {
 		return nil, err
 	}
 	for _, pod := range allPods {
-		if pod.Spec.NodeName == hostname && pod.Spec.HostNetwork != true && pod.Status.PodIP != "" {
-			ns, err := c.namespacesLister.Get(pod.Namespace)
-			if err != nil {
-				klog.Errorf("get ns %s failed, %+v", pod.Namespace, err)
-				continue
+		if pod.Spec.HostNetwork == true || pod.Status.PodIP == "" {
+			continue
+		}
+		ns, err := c.namespacesLister.Get(pod.Namespace)
+		if err != nil {
+			klog.Errorf("get ns %s failed, %+v", pod.Namespace, err)
+			continue
+		}
+		nsGWType := ns.Annotations[util.GWTypeAnnotation]
+		switch nsGWType {
+		case "", util.GWDistributedMode:
+			if pod.Spec.NodeName == hostname {
+				localPodIPs = append(localPodIPs, pod.Status.PodIP)
 			}
-			nsGWType := ns.Annotations[util.GWTypeAnnotation]
-			switch nsGWType {
-			case "", util.GWDistributedMode:
+		case util.GWCentralizedMode:
+			gwNode := ns.Annotations[util.GWNode]
+			if gwNode == hostname {
 				localPodIPs = append(localPodIPs, pod.Status.PodIP)
-			case util.GWCentralizedMode:
-				// TODO:
 			}
 		}
 	}

diff --git a/pkg/util/const.go b/pkg/util/const.go
@@ -29,10 +29,8 @@ const (
 	NodeAllowPriority   = "1001"
 	SubnetAllowPriority = "1002"
 
-	GWSouth = "south"
-	GWNorth = "north"
-
 	GWTypeAnnotation  = "ovn.kubernetes.io/gateway_type"
 	GWDistributedMode = "distributed"
 	GWCentralizedMode = "centralized"
+	GWNode            = "ovn.kubernetes.io/gateway_node"
 )