fix iptables

kubeovn · Jul 27, 2021 · a9bee80 · a9bee80
1 parent 5cd1b14
commit a9bee80
Show file tree

Hide file tree

Showing 3 changed files with 510 additions and 122 deletions.
diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml
@@ -123,6 +123,53 @@ jobs:
           sudo chmod 666 /home/runner/.kube/config
           make e2e
 
+  single-iptables-e2e:
+    needs: build
+    name: 1-master-iptables-e2e
+    runs-on: ubuntu-18.04
+    timeout-minutes: 30
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Install Kind
+        env:
+          KIND_VERSION: v0.11.1
+        run: |
+          curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
+          chmod +x ./kind
+          sudo mv kind /usr/local/bin
+
+      - name: Init Kind
+        run: |
+          pip install j2cli --user
+          pip install "j2cli[yaml]" --user
+          sudo PATH=~/.local/bin:$PATH make kind-init-iptables
+
+      - name: Download image
+        uses: actions/download-artifact@v2
+        with:
+          name: image
+
+      - name: Install Kube-OVN
+        run: |
+          docker load --input image.tar
+          sudo make kind-install
+
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v2
+        with:
+          go-version: ^1.16
+        id: go
+
+      - name: Run E2E
+        run: |
+          go get -u github.com/onsi/ginkgo/ginkgo
+          go get -u github.com/onsi/gomega/...
+          sudo kubectl cluster-info
+          sudo chmod 666 /home/runner/.kube/config
+          make e2e
+
   single-vlan-e2e-single-nic:
     needs: build
     name: 1-master-vlan-e2e-single-nic
@@ -338,14 +385,115 @@ jobs:
           sudo chmod 666 /home/runner/.kube/config
           make e2e-vlan-single-nic
 
+  no-lb-e2e:
+    needs: build
+    name: disable-loadbalancer-e2e
+    runs-on: ubuntu-18.04
+    timeout-minutes: 30
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Install Kind
+        env:
+          KIND_VERSION: v0.11.1
+        run: |
+          curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
+          chmod +x ./kind
+          sudo mv kind /usr/local/bin
+
+      - name: Init Kind
+        run: |
+          pip install j2cli --user
+          pip install "j2cli[yaml]" --user
+          sudo PATH=~/.local/bin:$PATH make kind-init
+
+      - name: Download image
+        uses: actions/download-artifact@v2
+        with:
+          name: image
+
+      - name: Install Kube-OVN without LoadBalancer
+        env:
+          ENABLE_LB: "false"
+        run: |
+          docker load --input image.tar
+          sudo make kind-install
+
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v2
+        with:
+          go-version: ^1.16
+        id: go
+
+      - name: Run E2E
+        run: |
+          go get -u github.com/onsi/ginkgo/ginkgo
+          go get -u github.com/onsi/gomega/...
+          sudo kubectl cluster-info
+          sudo chmod 666 /home/runner/.kube/config
+          make e2e
+
+  no-lb-iptables-e2e:
+    needs: build
+    name: disable-loadbalancer-iptables-e2e
+    runs-on: ubuntu-18.04
+    timeout-minutes: 30
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Install Kind
+        env:
+          KIND_VERSION: v0.11.1
+        run: |
+          curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
+          chmod +x ./kind
+          sudo mv kind /usr/local/bin
+
+      - name: Init Kind
+        run: |
+          pip install j2cli --user
+          pip install "j2cli[yaml]" --user
+          sudo PATH=~/.local/bin:$PATH make kind-init-iptables
+
+      - name: Download image
+        uses: actions/download-artifact@v2
+        with:
+          name: image
+
+      - name: Install Kube-OVN without LoadBalancer
+        env:
+          ENABLE_LB: "false"
+        run: |
+          docker load --input image.tar
+          sudo make kind-install
+
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v2
+        with:
+          go-version: ^1.16
+        id: go
+
+      - name: Run E2E
+        run: |
+          go get -u github.com/onsi/ginkgo/ginkgo
+          go get -u github.com/onsi/gomega/...
+          sudo kubectl cluster-info
+          sudo chmod 666 /home/runner/.kube/config
+          make e2e
+
   push:
     needs:
       - single-e2e
+      - single-iptables-e2e
       - single-vlan-e2e-single-nic
       - single-node-e2e
       - ha-e2e
       - ipv6-e2e
       - ipv6-vlan-e2e-single-nic
+      - no-lb-e2e
+      - no-lb-iptables-e2e
     name: push
     runs-on: ubuntu-18.04
     steps:

diff --git a/pkg/daemon/gateway.go b/pkg/daemon/gateway.go
@@ -23,6 +23,7 @@ import (
 )
 
 const (
+	ServiceSet   = "services"
 	SubnetSet    = "subnets"
 	SubnetNatSet = "subnets-nat"
 	LocalPodSet  = "local-pod-ip-nat"
@@ -75,6 +76,7 @@ func (c *Controller) setIPSet() error {
 		if c.ipset[protocol] == nil {
 			continue
 		}
+		services := c.getServicesCIDR(protocol)
 		subnets, err := c.getOverlaySubnetsCIDR(protocol)
 		if err != nil {
 			klog.Errorf("get subnets failed, %+v", err)
@@ -95,6 +97,11 @@ func (c *Controller) setIPSet() error {
 			klog.Errorf("failed to get node, %+v", err)
 			return err
 		}
+		c.ipset[protocol].AddOrReplaceIPSet(ipsets.IPSetMetadata{
+			MaxSize: 1048576,
+			SetID:   ServiceSet,
+			Type:    ipsets.IPSetTypeHashNet,
+		}, services)
 		c.ipset[protocol].AddOrReplaceIPSet(ipsets.IPSetMetadata{
 			MaxSize: 1048576,
 			SetID:   SubnetSet,
@@ -389,37 +396,58 @@ func (c *Controller) setIptables() error {
 	hostIP := util.GetNodeInternalIP(*node)
 
 	var (
-		v4Rules = []util.IPTableRule{
-			// Prevent performing Masquerade on external traffic which arrives from a Node that owns the Pod/Subnet IP
-			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Split(`-m set ! --match-set ovn40subnets src -m set ! --match-set ovn40other-node src -m set --match-set ovn40local-pod-ip-nat dst -j RETURN`, " ")},
-			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Split(`-m set ! --match-set ovn40subnets src -m set ! --match-set ovn40other-node src -m set --match-set ovn40subnets-nat dst -j RETURN`, " ")},
-			// NAT if pod/subnet to external address
-			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Split(`-m set --match-set ovn40local-pod-ip-nat src -m set ! --match-set ovn40subnets dst -j MASQUERADE`, " ")},
-			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Split(`-m set --match-set ovn40subnets-nat src -m set ! --match-set ovn40subnets dst -j MASQUERADE`, " ")},
-			// masq traffic from hostport/nodeport
+		v4AbandonedRules = []util.IPTableRule{
+			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m set ! --match-set ovn40subnets src -m set ! --match-set ovn40other-node src -m set --match-set ovn40local-pod-ip-nat dst -j RETURN`)},
+			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m set ! --match-set ovn40subnets src -m set ! --match-set ovn40other-node src -m set --match-set ovn40subnets-nat dst -j RETURN`)},
+			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(fmt.Sprintf(`-o ovn0 ! -s %s -m mark --mark 0x4000/0x4000 -j MASQUERADE`, hostIP))},
+		}
+		v6AbandonedRules = []util.IPTableRule{
+			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Split(`-m set ! --match-set ovn60subnets src -m set ! --match-set ovn60other-node src -m set --match-set ovn60local-pod-ip-nat dst -j RETURN`, " ")},
+			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Split(`-m set ! --match-set ovn60subnets src -m set ! --match-set ovn60other-node src -m set --match-set ovn60subnets-nat dst -j RETURN`, " ")},
 			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Split(fmt.Sprintf(`-o ovn0 ! -s %s -m mark --mark 0x4000/0x4000 -j MASQUERADE`, hostIP), " ")},
+		}
+
+		v4Rules = []util.IPTableRule{
+			// nat outgoing
+			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m set --match-set ovn40subnets-nat src -m set ! --match-set ovn40subnets dst -j MASQUERADE`)},
+			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m set --match-set ovn40local-pod-ip-nat src -m set ! --match-set ovn40subnets dst -j MASQUERADE`)},
+			// external traffic to overlay pod or to service
+			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(fmt.Sprintf(`! -s %s -m set --match-set ovn40subnets dst -j MASQUERADE`, hostIP))},
+			// masq traffic from overlay pod to service
+			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m mark --mark 0x40000/0x40000 -j MASQUERADE`)},
+			// mark traffic from overlay pod to service
+			{Table: "mangle", Chain: "PREROUTING", Rule: strings.Fields(`-i ovn0 -m set --match-set ovn40subnets src -m set --match-set ovn40services dst -j MARK --set-xmark 0x40000/0x40000`)},
 			// Input Accept
-			{Table: "filter", Chain: "FORWARD", Rule: strings.Split(`-m set --match-set ovn40subnets src -j ACCEPT`, " ")},
-			{Table: "filter", Chain: "FORWARD", Rule: strings.Split(`-m set --match-set ovn40subnets dst -j ACCEPT`, " ")},
+			{Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn40subnets src -j ACCEPT`)},
+			{Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn40subnets dst -j ACCEPT`)},
+			{Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn40services src -j ACCEPT`)},
+			{Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn40services dst -j ACCEPT`)},
 			// Forward Accept
-			{Table: "filter", Chain: "INPUT", Rule: strings.Split(`-m set --match-set ovn40subnets src -j ACCEPT`, " ")},
-			{Table: "filter", Chain: "INPUT", Rule: strings.Split(`-m set --match-set ovn40subnets dst -j ACCEPT`, " ")},
+			{Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn40subnets src -j ACCEPT`)},
+			{Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn40subnets dst -j ACCEPT`)},
+			{Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn40services src -j ACCEPT`)},
+			{Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn40services dst -j ACCEPT`)},
 		}
 		v6Rules = []util.IPTableRule{
-			// Prevent performing Masquerade on external traffic which arrives from a Node that owns the Pod/Subnet IP
-			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Split(`-m set ! --match-set ovn60subnets src -m set ! --match-set ovn60other-node src -m set --match-set ovn60local-pod-ip-nat dst -j RETURN`, " ")},
-			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Split(`-m set ! --match-set ovn60subnets src -m set ! --match-set ovn60other-node src -m set --match-set ovn60subnets-nat dst -j RETURN`, " ")},
-			// NAT if pod/subnet to external address
-			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Split(`-m set --match-set ovn60local-pod-ip-nat src -m set ! --match-set ovn60subnets dst -j MASQUERADE`, " ")},
-			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Split(`-m set --match-set ovn60subnets-nat src -m set ! --match-set ovn60subnets dst -j MASQUERADE`, " ")},
-			// masq traffic from hostport/nodeport
-			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Split(fmt.Sprintf(`-o ovn0 ! -s %s -m mark --mark 0x4000/0x4000 -j MASQUERADE`, hostIP), " ")},
+			// nat outgoing
+			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m set --match-set ovn60subnets-nat src -m set ! --match-set ovn60subnets dst -j MASQUERADE`)},
+			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m set --match-set ovn60local-pod-ip-nat src -m set ! --match-set ovn60subnets dst -j MASQUERADE`)},
+			// external traffic to overlay pod or to service
+			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(fmt.Sprintf(`! -s %s -m set --match-set ovn60subnets dst -j MASQUERADE`, hostIP))},
+			// masq traffic from overlay pod to service
+			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m mark --mark 0x40000/0x40000 -j MASQUERADE`)},
+			// mark traffic from overlay pod to service
+			{Table: "mangle", Chain: "PREROUTING", Rule: strings.Fields(`-i ovn0 -m set --match-set ovn60subnets src -m set --match-set ovn60services dst -j MARK --set-xmark 0x40000/0x40000`)},
 			// Input Accept
-			{Table: "filter", Chain: "FORWARD", Rule: strings.Split(`-m set --match-set ovn60subnets src -j ACCEPT`, " ")},
-			{Table: "filter", Chain: "FORWARD", Rule: strings.Split(`-m set --match-set ovn60subnets dst -j ACCEPT`, " ")},
+			{Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn60subnets src -j ACCEPT`)},
+			{Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn60subnets dst -j ACCEPT`)},
+			{Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn60services src -j ACCEPT`)},
+			{Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn60services dst -j ACCEPT`)},
 			// Forward Accept
-			{Table: "filter", Chain: "INPUT", Rule: strings.Split(`-m set --match-set ovn60subnets src -j ACCEPT`, " ")},
-			{Table: "filter", Chain: "INPUT", Rule: strings.Split(`-m set --match-set ovn60subnets dst -j ACCEPT`, " ")},
+			{Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn60subnets src -j ACCEPT`)},
+			{Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn60subnets dst -j ACCEPT`)},
+			{Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn60services src -j ACCEPT`)},
+			{Table: "filter", Chain: "FORWARD", Rule: strings.Fields(`-m set --match-set ovn60services dst -j ACCEPT`)},
 		}
 	)
 	protocols := make([]string, 2)
@@ -434,16 +462,43 @@ func (c *Controller) setIptables() error {
 		if c.iptable[protocol] == nil {
 			continue
 		}
-		var iptableRules []util.IPTableRule
+
+		var abandonedRules, iptableRules []util.IPTableRule
 		if protocol == kubeovnv1.ProtocolIPv4 {
-			iptableRules = v4Rules
+			abandonedRules, iptableRules = v4AbandonedRules, v4Rules
 		} else {
-			iptableRules = v6Rules
+			abandonedRules, iptableRules = v6AbandonedRules, v6Rules
+		}
+
+		// delete abandoned iptables rules
+		for _, iptRule := range abandonedRules {
+			exists, err := c.iptable[protocol].Exists(iptRule.Table, iptRule.Chain, iptRule.Rule...)
+			if err != nil {
+				klog.Errorf("failed to check existence of iptables rule: %v", err)
+				return err
+			}
+			if exists {
+				klog.Infof("deleting abandoned iptables rule: %s", strings.Join(iptRule.Rule, " "))
+				if err := c.iptable[protocol].Delete(iptRule.Table, iptRule.Chain, iptRule.Rule...); err != nil {
+					klog.Errorf("failed to delete iptables rule %s: %v", strings.Join(iptRule.Rule, " "), err)
+					return err
+				}
+			}
+		}
+
+		// reverse rules in nat table
+		var idx int
+		for idx = range iptableRules {
+			if iptableRules[idx].Table != "nat" {
+				break
+			}
 		}
-		iptableRules[0], iptableRules[1], iptableRules[3], iptableRules[4] =
-			iptableRules[4], iptableRules[3], iptableRules[1], iptableRules[0]
+		for i := 0; i < idx/2; i++ {
+			iptableRules[i], iptableRules[idx-1-i] = iptableRules[idx-1-i], iptableRules[i]
+		}
+
 		for _, iptRule := range iptableRules {
-			if strings.Contains(strings.Join(iptRule.Rule, " "), "ovn0") && protocol != util.CheckProtocol(hostIP) {
+			if util.ContainsString(iptRule.Rule, "ovn0") && protocol != util.CheckProtocol(hostIP) {
 				klog.V(3).Infof("ignore check iptable rule, protocol %v, hostIP %v", protocol, hostIP)
 				continue
 			}
@@ -706,22 +761,27 @@ func (c *Controller) getSubnetsNeedPR(protocol string) (map[policyRouteMeta]stri
 	return subnetsNeedPR, nil
 }
 
+func (c *Controller) getServicesCIDR(protocol string) []string {
+	ret := make([]string, 0)
+	for _, cidr := range strings.Split(c.config.ServiceClusterIPRange, ",") {
+		if util.CheckProtocol(cidr) == protocol {
+			ret = append(ret, cidr)
+		}
+	}
+	return ret
+}
+
 func (c *Controller) getOverlaySubnetsCIDR(protocol string) ([]string, error) {
 	subnets, err := c.subnetsLister.List(labels.Everything())
 	if err != nil {
 		klog.Error("failed to list subnets")
 		return nil, err
 	}
 
-	ret := make([]string, 0, len(subnets)+3)
+	ret := make([]string, 0, len(subnets)+1)
 	if c.config.NodeLocalDNSIP != "" && net.ParseIP(c.config.NodeLocalDNSIP) != nil && util.CheckProtocol(c.config.NodeLocalDNSIP) == protocol {
 		ret = append(ret, c.config.NodeLocalDNSIP)
 	}
-	for _, sip := range strings.Split(c.config.ServiceClusterIPRange, ",") {
-		if util.CheckProtocol(sip) == protocol {
-			ret = append(ret, sip)
-		}
-	}
 	for _, subnet := range subnets {
 		if subnet.Spec.Vpc == util.DefaultVpc && subnet.Spec.Vlan == "" {
 			cidrBlock := getCidrByProtocol(subnet.Spec.CIDRBlock, protocol)
@@ -766,7 +826,7 @@ func (c *Controller) appendMssRule() {
 		MssMangleRule := util.IPTableRule{
 			Table: "mangle",
 			Chain: "POSTROUTING",
-			Rule:  strings.Split(rule, " "),
+			Rule:  strings.Fields(rule),
 		}
 
 		switch c.protocol {