From 236574c750edad001626c02170c21a280e50ca2c Mon Sep 17 00:00:00 2001 From: changluyi <47097611+changluyi@users.noreply.github.com> Date: Mon, 24 Apr 2023 09:10:51 +0800 Subject: [PATCH] Add random fully when nat (#2681) * add random fully option in snat --- dist/images/kubectl-ko | 18 ++++++++++++++---- pkg/daemon/controller.go | 4 ++++ pkg/daemon/controller_linux.go | 5 +++++ pkg/daemon/gateway_linux.go | 9 ++++++++- 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/dist/images/kubectl-ko b/dist/images/kubectl-ko index 8ba5114d02f..a14cfcd23f5 100755 --- a/dist/images/kubectl-ko +++ b/dist/images/kubectl-ko @@ -982,15 +982,25 @@ log_linux(){ if [[ "$sub_component_param" == "dmesg" ]]; then kubectl exec $pod -n kube-system -- dmesg -T > ./kubectl-ko-log/$nodeName/$component_param/$sub_component_param.log || : elif [[ "$sub_component_param" == "iptables-legacy" ]]; then - echo "******************legacy filter ************************" > ./kubectl-ko-log/$nodeName/$component_param/$sub_component_param.log + kubectl exec $pod -n kube-system -- /usr/sbin/iptables-legacy -V > ./kubectl-ko-log/$nodeName/$component_param/$sub_component_param.log || : + echo "******************legacy filter v4 ************************" >> ./kubectl-ko-log/$nodeName/$component_param/$sub_component_param.log kubectl exec $pod -n kube-system -- /usr/sbin/iptables-legacy -S >> ./kubectl-ko-log/$nodeName/$component_param/$sub_component_param.log || : - echo "****************** legacy nat ************************" >> ./kubectl-ko-log/$nodeName/$component_param/$sub_component_param.log + echo "****************** legacy nat v4 ************************" >> ./kubectl-ko-log/$nodeName/$component_param/$sub_component_param.log kubectl exec $pod -n kube-system -- /usr/sbin/iptables-legacy -S -t nat >> ./kubectl-ko-log/$nodeName/$component_param/$sub_component_param.log || : + echo "******************legacy filter v6 ************************" >> ./kubectl-ko-log/$nodeName/$component_param/$sub_component_param.log + kubectl exec $pod -n kube-system -- /usr/sbin/ip6tables-legacy -S >> ./kubectl-ko-log/$nodeName/$component_param/$sub_component_param.log || : + echo "****************** legacy nat v6 ************************" >> ./kubectl-ko-log/$nodeName/$component_param/$sub_component_param.log + kubectl exec $pod -n kube-system -- /usr/sbin/ip6tables-legacy -S -t nat >> ./kubectl-ko-log/$nodeName/$component_param/$sub_component_param.log || : elif [[ "$sub_component_param" == "iptables-nft" ]]; then - echo "*********************nft filter ************************" > ./kubectl-ko-log/$nodeName/$component_param/$sub_component_param.log + kubectl exec $pod -n kube-system -- /usr/sbin/iptables-nft -V > ./kubectl-ko-log/$nodeName/$component_param/$sub_component_param.log 2>/dev/null || : + echo "*********************nft filter v4 ************************" >> ./kubectl-ko-log/$nodeName/$component_param/$sub_component_param.log kubectl exec $pod -n kube-system -- /usr/sbin/iptables-nft -S >> ./kubectl-ko-log/$nodeName/$component_param/$sub_component_param.log 2>/dev/null || : - echo "********************* nft nat ************************" >> ./kubectl-ko-log/$nodeName/$component_param/$sub_component_param.log + echo "********************* nft nat v4 ************************" >> ./kubectl-ko-log/$nodeName/$component_param/$sub_component_param.log kubectl exec $pod -n kube-system -- /usr/sbin/iptables-nft -S -t nat >> ./kubectl-ko-log/$nodeName/$component_param/$sub_component_param.log 2>/dev/null || : + echo "*********************nft filter v6 ************************" >> ./kubectl-ko-log/$nodeName/$component_param/$sub_component_param.log + kubectl exec $pod -n kube-system -- /usr/sbin/ip6tables-nft -S >> ./kubectl-ko-log/$nodeName/$component_param/$sub_component_param.log 2>/dev/null || : + echo "********************* nft nat v6 ************************" >> ./kubectl-ko-log/$nodeName/$component_param/$sub_component_param.log + kubectl exec $pod -n kube-system -- /usr/sbin/ip6tables-nft -S -t nat >> ./kubectl-ko-log/$nodeName/$component_param/$sub_component_param.log 2>/dev/null || : elif [[ "$sub_component_param" == "route" ]]; then kubectl exec $pod -n kube-system -- ip route show > ./kubectl-ko-log/$nodeName/$component_param/$sub_component_param.log || : kubectl exec $pod -n kube-system -- ip -6 route show >> ./kubectl-ko-log/$nodeName/$component_param/$sub_component_param.log || : diff --git a/pkg/daemon/controller.go b/pkg/daemon/controller.go index 64744b35e2a..459a804fdf4 100644 --- a/pkg/daemon/controller.go +++ b/pkg/daemon/controller.go @@ -22,6 +22,7 @@ import ( "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" "k8s.io/klog/v2" + k8sexec "k8s.io/utils/exec" kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" kubeovninformer "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions" @@ -60,6 +61,8 @@ type Controller struct { ControllerRuntime localPodName string localNamespace string + + k8sExec k8sexec.Interface } // NewController init a daemon controller @@ -98,6 +101,7 @@ func NewController(config *Configuration, podInformerFactory informers.SharedInf nodesSynced: nodeInformer.Informer().HasSynced, recorder: recorder, + k8sExec: k8sexec.New(), } node, err := config.KubeClient.CoreV1().Nodes().Get(context.Background(), config.NodeName, metav1.GetOptions{}) diff --git a/pkg/daemon/controller_linux.go b/pkg/daemon/controller_linux.go index 7fafcbdad43..9bc956cfaa4 100644 --- a/pkg/daemon/controller_linux.go +++ b/pkg/daemon/controller_linux.go @@ -22,6 +22,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" + k8siptables "k8s.io/kubernetes/pkg/util/iptables" kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" "github.com/kubeovn/kube-ovn/pkg/ovs" @@ -32,6 +33,7 @@ import ( type ControllerRuntime struct { iptables map[string]*iptables.IPTables iptablesObsolete map[string]*iptables.IPTables + k8siptables map[string]k8siptables.Interface ipsets map[string]*ipsets.IPSets gwCounters map[string]*util.GwIPtableCounters } @@ -75,6 +77,7 @@ func (c *Controller) initRuntime() error { c.iptables = make(map[string]*iptables.IPTables) c.ipsets = make(map[string]*ipsets.IPSets) c.gwCounters = make(map[string]*util.GwIPtableCounters) + c.k8siptables = make(map[string]k8siptables.Interface) if c.protocol == kubeovnv1.ProtocolIPv4 || c.protocol == kubeovnv1.ProtocolDual { ipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv4) @@ -89,6 +92,7 @@ func (c *Controller) initRuntime() error { c.iptablesObsolete[kubeovnv1.ProtocolIPv4] = ipt } c.ipsets[kubeovnv1.ProtocolIPv4] = ipsets.NewIPSets(ipsets.NewIPVersionConfig(ipsets.IPFamilyV4, IPSetPrefix, nil, nil)) + c.k8siptables[kubeovnv1.ProtocolIPv4] = k8siptables.New(c.k8sExec, k8siptables.ProtocolIPv4) } if c.protocol == kubeovnv1.ProtocolIPv6 || c.protocol == kubeovnv1.ProtocolDual { ipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv6) @@ -103,6 +107,7 @@ func (c *Controller) initRuntime() error { c.iptablesObsolete[kubeovnv1.ProtocolIPv6] = ipt } c.ipsets[kubeovnv1.ProtocolIPv6] = ipsets.NewIPSets(ipsets.NewIPVersionConfig(ipsets.IPFamilyV6, IPSetPrefix, nil, nil)) + c.k8siptables[kubeovnv1.ProtocolIPv6] = k8siptables.New(c.k8sExec, k8siptables.ProtocolIPv6) } return nil diff --git a/pkg/daemon/gateway_linux.go b/pkg/daemon/gateway_linux.go index d309c383a14..923a092ada4 100644 --- a/pkg/daemon/gateway_linux.go +++ b/pkg/daemon/gateway_linux.go @@ -379,7 +379,7 @@ func (c *Controller) updateIptablesChain(ipt *iptables.IPTables, table, chain, p added++ } for i := len(existingRules) - 1; i >= len(rules)-added; i-- { - if err = ipt.Delete(table, chain, strconv.Itoa(i+added)); err != nil { + if err = ipt.Delete(table, chain, strconv.Itoa(i+added+1)); err != nil { klog.Errorf(`failed to delete iptables rule %v: %v`, existingRules[i], err) return err } @@ -579,6 +579,13 @@ func (c *Controller) setIptables() error { natPreroutingRules = append(natPreroutingRules, rule) continue case OvnPostrouting: + if util.ContainsString(rule.Rule, "MASQUERADE") && c.k8siptables[protocol].HasRandomFully() { + // https://github.com/kubeovn/kube-ovn/issues/2641 + // Work around Linux kernel bug that sometimes causes multiple flows to + // get mapped to the same IP:PORT and consequently some suffer packet + // drops. + rule.Rule = append(rule.Rule, "--random-fully") + } natPostroutingRules = append(natPostroutingRules, rule) continue }