From dff950b18c790820354f1b22c52aeda5d4cf8929 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E7=A5=96=E5=BB=BA?= Date: Thu, 8 Jun 2023 19:22:10 +0800 Subject: [PATCH] cni-server: clear iptables mark before doing masquerade (#2919) --- pkg/daemon/gateway_linux.go | 51 ++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/pkg/daemon/gateway_linux.go b/pkg/daemon/gateway_linux.go index c7dcdddcda3..79df0e94d12 100644 --- a/pkg/daemon/gateway_linux.go +++ b/pkg/daemon/gateway_linux.go @@ -43,6 +43,7 @@ const ( Postrouting = "POSTROUTING" OvnPrerouting = "OVN-PREROUTING" OvnPostrouting = "OVN-POSTROUTING" + OvnMasquerade = "OVN-MASQUERADE" OvnNatOutGoingPolicy = "OVN-NAT-POLICY" OvnNatOutGoingPolicySubnet = "OVN-NAT-PSUBNET-" ) @@ -513,24 +514,27 @@ func (c *Controller) setIptables() error { // mark packets from pod to service {Table: NAT, Chain: OvnPrerouting, Rule: strings.Fields(`-i ovn0 -m set --match-set ovn40subnets src -m set --match-set ovn40services dst -j MARK --set-xmark 0x4000/0x4000`)}, // nat packets marked by kube-proxy or kube-ovn - {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x4000/0x4000 -j MASQUERADE`)}, + {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x4000/0x4000 -j ` + OvnMasquerade)}, // nat service traffic - {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m set --match-set ovn40subnets src -m set --match-set ovn40subnets dst -j MASQUERADE`)}, + {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m set --match-set ovn40subnets src -m set --match-set ovn40subnets dst -j ` + OvnMasquerade)}, // do not nat node port service traffic with external traffic policy set to local {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -m set --match-set ovn40subnets-distributed-gw dst -j RETURN`)}, // nat node port service traffic with external traffic policy set to local for subnets with centralized gateway - {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -j MASQUERADE`)}, + {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -j ` + OvnMasquerade)}, // do not nat reply packets in direct routing {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-p tcp -m tcp --tcp-flags SYN NONE -m conntrack --ctstate NEW -j RETURN`)}, // do not nat route traffic {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m set ! --match-set ovn40subnets src -m set ! --match-set ovn40other-node src -m set --match-set ovn40subnets-nat dst -j RETURN`)}, - // nat outgoing // nat outgoing policy rules {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(fmt.Sprintf(`-m set --match-set ovn40subnets-nat-policy src -m set ! --match-set ovn40subnets dst -j %s`, OvnNatOutGoingPolicy))}, - {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(fmt.Sprintf(`-m mark --mark %s -j MASQUERADE`, OnOutGoingNatMark))}, + {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(fmt.Sprintf(`-m mark --mark %s -j %s`, OnOutGoingNatMark, OvnMasquerade))}, {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(fmt.Sprintf(`-m mark --mark %s -j RETURN`, OnOutGoingForwardMark))}, // default nat outgoing rules - {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m set --match-set ovn40subnets-nat src -m set ! --match-set ovn40subnets dst -j MASQUERADE`)}, + {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m set --match-set ovn40subnets-nat src -m set ! --match-set ovn40subnets dst -j ` + OvnMasquerade)}, + // clear mark + {Table: NAT, Chain: OvnMasquerade, Rule: strings.Fields(`-j MARK --set-xmark 0x0/0xffffffff`)}, + // do masquerade + {Table: NAT, Chain: OvnMasquerade, Rule: strings.Fields(`-j MASQUERADE`)}, // Input Accept {Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn40subnets src -j ACCEPT`)}, {Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn40subnets dst -j ACCEPT`)}, @@ -548,23 +552,26 @@ func (c *Controller) setIptables() error { // mark packets from pod to service {Table: NAT, Chain: OvnPrerouting, Rule: strings.Fields(`-i ovn0 -m set --match-set ovn60subnets src -m set --match-set ovn60services dst -j MARK --set-xmark 0x4000/0x4000`)}, // nat packets marked by kube-proxy or kube-ovn - {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x4000/0x4000 -j MASQUERADE`)}, + {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x4000/0x4000 -j ` + OvnMasquerade)}, // nat service traffic - {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m set --match-set ovn60subnets src -m set --match-set ovn60subnets dst -j MASQUERADE`)}, + {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m set --match-set ovn60subnets src -m set --match-set ovn60subnets dst -j ` + OvnMasquerade)}, // do not nat node port service traffic with external traffic policy set to local {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -m set --match-set ovn60subnets-distributed-gw dst -j RETURN`)}, // nat node port service traffic with external traffic policy set to local for subnets with centralized gateway - {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -j MASQUERADE`)}, + {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -j ` + OvnMasquerade)}, // do not nat reply packets in direct routing {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-p tcp -m tcp --tcp-flags SYN NONE -m conntrack --ctstate NEW -j RETURN`)}, // do not nat route traffic {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m set ! --match-set ovn60subnets src -m set ! --match-set ovn60other-node src -m set --match-set ovn60subnets-nat dst -j RETURN`)}, - // nat outgoing // nat outgoing policy rules {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(fmt.Sprintf(`-m set --match-set ovn60subnets-nat-policy src -m set ! --match-set ovn60subnets dst -j %s`, OvnNatOutGoingPolicy))}, - {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(fmt.Sprintf(`-m mark --mark %s -j MASQUERADE`, OnOutGoingNatMark))}, + {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(fmt.Sprintf(`-m mark --mark %s -j %s`, OnOutGoingNatMark, OvnMasquerade))}, {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(fmt.Sprintf(`-m mark --mark %s -j RETURN`, OnOutGoingForwardMark))}, - {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m set --match-set ovn60subnets-nat src -m set ! --match-set ovn60subnets dst -j MASQUERADE`)}, + {Table: NAT, Chain: OvnPostrouting, Rule: strings.Fields(`-m set --match-set ovn60subnets-nat src -m set ! --match-set ovn60subnets dst -j ` + OvnMasquerade)}, + // clear mark + {Table: NAT, Chain: OvnMasquerade, Rule: strings.Fields(`-j MARK --set-xmark 0x0/0xffffffff`)}, + // do masquerade + {Table: NAT, Chain: OvnMasquerade, Rule: strings.Fields(`-j MASQUERADE`)}, // Input Accept {Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn60subnets src -j ACCEPT`)}, {Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn60subnets dst -j ACCEPT`)}, @@ -682,23 +689,23 @@ func (c *Controller) setIptables() error { } } - var natPreroutingRules, natPostroutingRules []util.IPTableRule + var natPreroutingRules, natPostroutingRules, ovnMasqueradeRules []util.IPTableRule for _, rule := range iptablesRules { if rule.Table == NAT { + if c.k8siptables[protocol].HasRandomFully() && rule.Rule[len(rule.Rule)-1] == "MASQUERADE" { + rule.Rule = append(rule.Rule, "--random-fully") + } + switch rule.Chain { case OvnPrerouting: natPreroutingRules = append(natPreroutingRules, rule) continue case OvnPostrouting: - if util.ContainsString(rule.Rule, "MASQUERADE") && c.k8siptables[protocol].HasRandomFully() { - // https://github.com/kubeovn/kube-ovn/issues/2641 - // Work around Linux kernel bug that sometimes causes multiple flows to - // get mapped to the same IP:PORT and consequently some suffer packet - // drops. - rule.Rule = append(rule.Rule, "--random-fully") - } natPostroutingRules = append(natPostroutingRules, rule) continue + case OvnMasquerade: + ovnMasqueradeRules = append(ovnMasqueradeRules, rule) + continue } } @@ -733,6 +740,10 @@ func (c *Controller) setIptables() error { klog.Errorf("failed to update chain %s/%s: %v", NAT, OvnPrerouting) return err } + if err = c.updateIptablesChain(ipt, NAT, OvnMasquerade, "", ovnMasqueradeRules); err != nil { + klog.Errorf("failed to update chain %s/%s: %v", NAT, OvnMasquerade) + return err + } if err = c.updateIptablesChain(ipt, NAT, OvnPostrouting, Postrouting, natPostroutingRules); err != nil { klog.Errorf("failed to update chain %s/%s: %v", NAT, OvnPostrouting) return err