do not snat packets only for subnets with distributed gateway when ex…

…ternal traffic policy is set to local (#1616)
kubeovn · Jul 8, 2022 · 8cb6e36 · 8cb6e36
1 parent 86ecfd2
commit 8cb6e36
Show file tree

Hide file tree

Showing 4 changed files with 58 additions and 10 deletions.
diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml
@@ -432,7 +432,15 @@ jobs:
           max_attempts: 3
           shell: bash
           command: |
+            for node in $(sudo kubectl get no -o jsonpath='{.items[*].metadata.name}'); do
+              taint=$(sudo kubectl get no $node -o jsonpath='{.spec.taints[?(@.key=="node-role.kubernetes.io/master")]}')
+              if [ -n "$taint" ]; then
+                sudo kubectl taint node $node node-role.kubernetes.io/master:NoSchedule-
+              fi
+            done
             sudo make kind-install
+            sudo kubectl patch subnet ovn-default --type merge \
+              -p '{"spec":{"gatewayType": "centralized", "gatewayNode": "kube-ovn-control-plane"}}'
 
       - name: Set up Go 1.x
         uses: actions/setup-go@v3

diff --git a/dist/images/uninstall.sh b/dist/images/uninstall.sh
@@ -3,7 +3,8 @@
 ovs-dpctl del-dp ovs-system
 
 iptables -t nat -D POSTROUTING -m mark --mark 0x4000/0x4000 -j MASQUERADE
-iptables -t nat -D POSTROUTING -m mark --mark 0x80000/0x80000 -j RETURN
+iptables -t nat -D POSTROUTING -m mark --mark 0x80000/0x80000 -m set --match-set ovn40subnets-distributed-gw dst -j RETURN
+iptables -t nat -D POSTROUTING -m mark --mark 0x80000/0x80000 -j MASQUERADE
 iptables -t nat -D POSTROUTING -p tcp --tcp-flags SYN NONE -m conntrack --ctstate NEW -j RETURN
 iptables -t nat -D POSTROUTING -m set ! --match-set ovn40subnets src -m set ! --match-set ovn40other-node src -m set --match-set ovn40subnets-nat dst -j RETURN
 iptables -t nat -D POSTROUTING -m set --match-set ovn40subnets-nat src -m set ! --match-set ovn40subnets dst -j MASQUERADE
@@ -24,12 +25,14 @@ sleep 1
 
 ipset destroy ovn40subnets-nat
 ipset destroy ovn40subnets
+ipset destroy ovn40subnets-distributed-gw
 ipset destroy ovn40local-pod-ip-nat
 ipset destroy ovn40other-node
 ipset destroy ovn40services
 
 ip6tables -t nat -D POSTROUTING -m mark --mark 0x4000/0x4000 -j MASQUERADE
-ip6tables -t nat -D POSTROUTING -m mark --mark 0x80000/0x80000 -j RETURN
+ip6tables -t nat -D POSTROUTING -m mark --mark 0x80000/0x80000 -m set --match-set ovn60subnets-distributed-gw dst -j RETURN
+ip6tables -t nat -D POSTROUTING -m mark --mark 0x80000/0x80000 -j MASQUERADE
 ip6tables -t nat -D POSTROUTING -p tcp --tcp-flags SYN NONE -m conntrack --ctstate NEW -j RETURN
 ip6tables -t nat -D POSTROUTING -m set ! --match-set ovn60subnets src -m set ! --match-set ovn60other-node src -m set --match-set ovn60subnets-nat dst -j RETURN
 ip6tables -t nat -D POSTROUTING -m set --match-set ovn60subnets-nat src -m set ! --match-set ovn60subnets dst -j MASQUERADE
@@ -50,6 +53,7 @@ sleep 1
 
 ipset destroy ovn6subnets-nat
 ipset destroy ovn60subnets
+ipset destroy ovn60subnets-distributed-gw
 ipset destroy ovn60local-pod-ip-nat
 ipset destroy ovn60other-node
 ipset destroy ovn60services

diff --git a/pkg/daemon/gateway.go b/pkg/daemon/gateway.go
@@ -105,6 +105,27 @@ func (c *Controller) getSubnetsNeedNAT(protocol string) ([]string, error) {
 	return subnetsNeedNat, nil
 }
 
+func (c *Controller) getSubnetsDistributedGateway(protocol string) ([]string, error) {
+	subnets, err := c.subnetsLister.List(labels.Everything())
+	if err != nil {
+		klog.Errorf("failed to list subnets: %v", err)
+		return nil, err
+	}
+
+	var result []string
+	for _, subnet := range subnets {
+		if subnet.DeletionTimestamp == nil &&
+			subnet.Spec.Vlan == "" &&
+			subnet.Spec.Vpc == util.DefaultVpc &&
+			subnet.Spec.GatewayType == kubeovnv1.GWDistributedType &&
+			(subnet.Spec.Protocol == kubeovnv1.ProtocolDual || subnet.Spec.Protocol == protocol) {
+			cidrBlock := getCidrByProtocol(subnet.Spec.CIDRBlock, protocol)
+			result = append(result, cidrBlock)
+		}
+	}
+	return result, nil
+}
+
 func (c *Controller) getServicesCIDR(protocol string) []string {
 	ret := make([]string, 0)
 	for _, cidr := range strings.Split(c.config.ServiceClusterIPRange, ",") {

diff --git a/pkg/daemon/gateway_linux.go b/pkg/daemon/gateway_linux.go
@@ -23,12 +23,13 @@ import (
 )
 
 const (
-	ServiceSet   = "services"
-	SubnetSet    = "subnets"
-	SubnetNatSet = "subnets-nat"
-	LocalPodSet  = "local-pod-ip-nat"
-	OtherNodeSet = "other-node"
-	IPSetPrefix  = "ovn"
+	ServiceSet             = "services"
+	SubnetSet              = "subnets"
+	SubnetNatSet           = "subnets-nat"
+	SubnetDistributedGwSet = "subnets-distributed-gw"
+	LocalPodSet            = "local-pod-ip-nat"
+	OtherNodeSet           = "other-node"
+	IPSetPrefix            = "ovn"
 )
 
 type policyRouteMeta struct {
@@ -63,6 +64,11 @@ func (c *Controller) setIPSet() error {
 			klog.Errorf("get need nat subnets failed, %+v", err)
 			return err
 		}
+		subnetsDistributedGateway, err := c.getSubnetsDistributedGateway(protocol)
+		if err != nil {
+			klog.Errorf("failed to get subnets with centralized gateway: %v", err)
+			return err
+		}
 		otherNode, err := c.getOtherNodes(protocol)
 		if err != nil {
 			klog.Errorf("failed to get node, %+v", err)
@@ -88,6 +94,11 @@ func (c *Controller) setIPSet() error {
 			SetID:   SubnetNatSet,
 			Type:    ipsets.IPSetTypeHashNet,
 		}, subnetsNeedNat)
+		c.ipsets[protocol].AddOrReplaceIPSet(ipsets.IPSetMetadata{
+			MaxSize: 1048576,
+			SetID:   SubnetDistributedGwSet,
+			Type:    ipsets.IPSetTypeHashNet,
+		}, subnetsDistributedGateway)
 		c.ipsets[protocol].AddOrReplaceIPSet(ipsets.IPSetMetadata{
 			MaxSize: 1048576,
 			SetID:   OtherNodeSet,
@@ -313,7 +324,9 @@ func (c *Controller) setIptables() error {
 			// nat packets marked by kube-proxy or kube-ovn
 			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m mark --mark 0x4000/0x4000 -j MASQUERADE`)},
 			// do not nat node port service traffic with external traffic policy set to local
-			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -j RETURN`)},
+			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -m set --match-set ovn40subnets-distributed-gw dst -j RETURN`)},
+			// nat node port service traffic with external traffic policy set to local for subnets with centralized gateway
+			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -j MASQUERADE`)},
 			// do not nat reply packets in direct routing
 			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-p tcp --tcp-flags SYN NONE -m conntrack --ctstate NEW -j RETURN`)},
 			// do not nat route traffic
@@ -339,7 +352,9 @@ func (c *Controller) setIptables() error {
 			// nat packets marked by kube-proxy or kube-ovn
 			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m mark --mark 0x4000/0x4000 -j MASQUERADE`)},
 			// do not nat node port service traffic with external traffic policy set to local
-			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -j RETURN`)},
+			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -m set --match-set ovn60subnets-distributed-gw dst -j RETURN`)},
+			// nat node port service traffic with external traffic policy set to local for subnets with centralized gateway
+			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -j MASQUERADE`)},
 			// do not nat reply packets in direct routing
 			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-p tcp --tcp-flags SYN NONE -m conntrack --ctstate NEW -j RETURN`)},
 			// do not nat route traffic