Skip to content

Commit

Permalink
avoid frequent ipset update
Browse files Browse the repository at this point in the history
  • Loading branch information
@zhangzujian
zhangzujian committed Mar 28, 2022
1 parent b44bbc5 commit bf167a6
Show file tree
Hide file tree
Showing 4 changed files with 13 additions and 119 deletions.
2 changes: 0 additions & 2 deletions dist/images/uninstall.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ if [ -n "$1" ]; then
fi

iptables -t nat -D POSTROUTING -m set --match-set ovn40subnets-nat src -m set ! --match-set ovn40subnets dst -j MASQUERADE
iptables -t nat -D POSTROUTING -m set --match-set ovn40local-pod-ip-nat src -m set ! --match-set ovn40subnets dst -j MASQUERADE
iptables -t nat -D POSTROUTING -m mark --mark 0x40000/0x40000 -j MASQUERADE
iptables -t mangle -D PREROUTING -i ovn0 -m set --match-set ovn40subnets src -m set --match-set ovn40services dst -j MARK --set-xmark 0x40000/0x40000
iptables -t filter -D INPUT -m set --match-set ovn40subnets dst -j ACCEPT
Expand All @@ -48,7 +47,6 @@ ipset destroy ovn40other-node
ipset destroy ovn40services

ip6tables -t nat -D POSTROUTING -m set --match-set ovn60subnets-nat src -m set ! --match-set ovn60subnets dst -j MASQUERADE
ip6tables -t nat -D POSTROUTING -m set --match-set ovn60local-pod-ip-nat src -m set ! --match-set ovn60subnets dst -j MASQUERADE
ip6tables -t nat -D POSTROUTING -m mark --mark 0x40000/0x40000 -j MASQUERADE
ip6tables -t mangle -D PREROUTING -i ovn0 -m set --match-set ovn60subnets src -m set --match-set ovn60services dst -j MARK --set-xmark 0x40000/0x40000
ip6tables -t filter -D INPUT -m set --match-set ovn60subnets dst -j ACCEPT
Expand Down
1 change: 1 addition & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ require (
)

replace (
github.com/alauda/felix => github.com/kubeovn/felix v0.0.0-20220325073257-c8a0f705d139
github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2
github.com/greenpau/ovsdb => github.com/alauda/ovsdb v0.0.0-20210113100339-040cf3e76c28
github.com/openshift/api => github.com/openshift/api v0.0.0-20210428205234-a8389931bee7
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,8 +79,6 @@ github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWX
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
github.com/alauda/felix v3.6.6-0.20201207121355-187332daf314+incompatible h1:f8w9fTcjS9/Zdw57Law9JsFxh/sttnBor4TGkWQpSf8=
github.com/alauda/felix v3.6.6-0.20201207121355-187332daf314+incompatible/go.mod h1:NeFJ7/fvKQ8X9styGJvh7sHmZoUprV5ZKzE2Ey82BEA=
github.com/alauda/ovsdb v0.0.0-20210113100339-040cf3e76c28 h1:FW5M3SAwSGBdtTboeV5sI7kEY6zraApSZQxTUfZ7LQY=
github.com/alauda/ovsdb v0.0.0-20210113100339-040cf3e76c28/go.mod h1:dXpg+IAC2yp2IZQlEVmnmEc1rqEmSZzgNfu6+ai38J4=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
Expand Down Expand Up @@ -522,6 +520,8 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
github.com/kr/text v0.0.0-20160504234017-7cafcd837844/go.mod h1:sjUstKUATFIcff4qlB53Kml0wQPtJVc/3fWrmuUmcfA=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kubeovn/felix v0.0.0-20220325073257-c8a0f705d139 h1:MaLC8/dohKHU8nkfglfE2oikefB6urJG75yZDOcKTRU=
github.com/kubeovn/felix v0.0.0-20220325073257-c8a0f705d139/go.mod h1:ulxnUH9cbIOtCH+exhJPeV2mleh+bDv67WKsl/MVU/g=
github.com/kubernetes-csi/csi-lib-utils v0.7.0/go.mod h1:bze+2G9+cmoHxN6+WyG1qT4MDxgZJMLGwc7V4acPNm0=
github.com/kubernetes-csi/csi-test v2.0.0+incompatible/go.mod h1:YxJ4UiuPWIhMBkxUKY5c267DyA0uDZ/MtAimhx/2TA0=
github.com/kubernetes-csi/external-snapshotter/v2 v2.1.1 h1:t5bmB3Y8nCaLA4aFrIpX0zjHEF/HUkJp6f5rm7BsVzM=
Expand Down
125 changes: 10 additions & 115 deletions pkg/daemon/gateway.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,11 +86,6 @@ func (c *Controller) setIPSet() error {
klog.Errorf("get subnets failed, %+v", err)
return err
}
localPodIPs, err := c.getLocalPodIPsNeedNAT(protocol)
if err != nil {
klog.Errorf("get local pod ips failed, %+v", err)
return err
}
subnetsNeedNat, err := c.getSubnetsNeedNAT(protocol)
if err != nil {
klog.Errorf("get need nat subnets failed, %+v", err)
Expand All @@ -115,7 +110,7 @@ func (c *Controller) setIPSet() error {
MaxSize: 1048576,
SetID: LocalPodSet,
Type: ipsets.IPSetTypeHashIP,
}, localPodIPs)
}, nil)
c.ipset[protocol].AddOrReplaceIPSet(ipsets.IPSetMetadata{
MaxSize: 1048576,
SetID: SubnetNatSet,
Expand Down Expand Up @@ -186,13 +181,9 @@ func (c *Controller) addEgressConfig(subnet *kubeovnv1.Subnet, ip string) error
return nil
}

podIPs := strings.Split(ip, ",")
protocol := util.CheckProtocol(ip)
if subnet.Spec.NatOutgoing {
c.addIPSetMembers(LocalPodSet, protocol, podIPs)
return nil
}
if subnet.Spec.ExternalEgressGateway != "" {
if !subnet.Spec.NatOutgoing && subnet.Spec.ExternalEgressGateway != "" {
podIPs := strings.Split(ip, ",")
protocol := util.CheckProtocol(ip)
return c.addPodPolicyRouting(protocol, subnet.Spec.ExternalEgressGateway, subnet.Spec.PolicyRoutingPriority, subnet.Spec.PolicyRoutingTableID, podIPs)
}

Expand All @@ -218,57 +209,15 @@ func (c *Controller) removeEgressConfig(subnet, ip string) error {
return nil
}

podIPs := strings.Split(ip, ",")
protocol := util.CheckProtocol(ip)
if podSubnet.Spec.NatOutgoing {
c.removeIPSetMembers(LocalPodSet, protocol, podIPs)
return nil
}
if podSubnet.Spec.ExternalEgressGateway != "" {
if !podSubnet.Spec.NatOutgoing && podSubnet.Spec.ExternalEgressGateway != "" {
podIPs := strings.Split(ip, ",")
protocol := util.CheckProtocol(ip)
return c.deletePodPolicyRouting(protocol, podSubnet.Spec.ExternalEgressGateway, podSubnet.Spec.PolicyRoutingPriority, podSubnet.Spec.PolicyRoutingTableID, podIPs)
}

return nil
}

func (c *Controller) addIPSetMembers(setID, protocol string, ips []string) {
c.ipsetLock.Lock()
defer c.ipsetLock.Unlock()

if protocol == kubeovnv1.ProtocolDual {
if c.ipset[kubeovnv1.ProtocolIPv4] != nil {
c.ipset[kubeovnv1.ProtocolIPv4].AddMembers(setID, ips[:1])
c.ipset[kubeovnv1.ProtocolIPv4].ApplyUpdates()
}
if c.ipset[kubeovnv1.ProtocolIPv6] != nil {
c.ipset[kubeovnv1.ProtocolIPv6].AddMembers(setID, ips[1:])
c.ipset[kubeovnv1.ProtocolIPv6].ApplyUpdates()
}
} else if c.ipset[protocol] != nil {
c.ipset[protocol].AddMembers(setID, ips[:1])
c.ipset[protocol].ApplyUpdates()
}
}

func (c *Controller) removeIPSetMembers(setID, protocol string, ips []string) {
c.ipsetLock.Lock()
defer c.ipsetLock.Unlock()

if protocol == kubeovnv1.ProtocolDual {
if c.ipset[kubeovnv1.ProtocolIPv4] != nil {
c.ipset[kubeovnv1.ProtocolIPv4].RemoveMembers(setID, ips[:1])
c.ipset[kubeovnv1.ProtocolIPv4].ApplyUpdates()
}
if c.ipset[kubeovnv1.ProtocolIPv6] != nil {
c.ipset[kubeovnv1.ProtocolIPv6].RemoveMembers(setID, ips[1:])
c.ipset[kubeovnv1.ProtocolIPv6].ApplyUpdates()
}
} else if c.ipset[protocol] != nil {
c.ipset[protocol].RemoveMembers(setID, ips[:1])
c.ipset[protocol].ApplyUpdates()
}
}

func (c *Controller) addPodPolicyRouting(podProtocol, externalEgressGateway string, priority, tableID uint32, ips []string) error {
egw := strings.Split(externalEgressGateway, ",")
prMetas := make([]policyRouteMeta, 0, 2)
Expand Down Expand Up @@ -437,7 +386,6 @@ func (c *Controller) setIptables() error {
v4Rules = []util.IPTableRule{
// nat outgoing
{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m set --match-set ovn40subnets-nat src -m set ! --match-set ovn40subnets dst -j MASQUERADE`)},
{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m set --match-set ovn40local-pod-ip-nat src -m set ! --match-set ovn40subnets dst -j MASQUERADE`)},
// external traffic to overlay pod or to service
// {Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(fmt.Sprintf(`! -s %s -m set --match-set ovn40subnets dst -j MASQUERADE`, nodeIPv4))},
// masq traffic from overlay pod to service
Expand All @@ -460,7 +408,6 @@ func (c *Controller) setIptables() error {
v6Rules = []util.IPTableRule{
// nat outgoing
{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m set --match-set ovn60subnets-nat src -m set ! --match-set ovn60subnets dst -j MASQUERADE`)},
{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m set --match-set ovn60local-pod-ip-nat src -m set ! --match-set ovn60subnets dst -j MASQUERADE`)},
// external traffic to overlay pod or to service
// {Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(fmt.Sprintf(`! -s %s -m set --match-set ovn60subnets dst -j MASQUERADE`, nodeIPv6))},
// masq traffic from overlay pod to service
Expand Down Expand Up @@ -513,9 +460,9 @@ func (c *Controller) setIptables() error {
abandonedRules = append(abandonedRules, util.IPTableRule{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(fmt.Sprintf(`-o ovn0 ! -s %s -m mark --mark 0x4000/0x4000 -j MASQUERADE`, nodeIP))})

rules := make([]util.IPTableRule, len(iptableRules)+1)
copy(rules[:2], iptableRules[:2])
rules[2] = util.IPTableRule{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(fmt.Sprintf(`! -s %s -m set --match-set %s dst -j MASQUERADE`, nodeIP, matchset))}
copy(rules[3:], iptableRules[2:])
copy(rules[:1], iptableRules[:1])
rules[1] = util.IPTableRule{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(fmt.Sprintf(`! -s %s -m set --match-set %s dst -j MASQUERADE`, nodeIP, matchset))}
copy(rules[2:], iptableRules[1:])
iptableRules = rules
}

Expand Down Expand Up @@ -671,56 +618,6 @@ func (c *Controller) setExGateway() error {
return nil
}

func (c *Controller) getLocalPodIPsNeedNAT(protocol string) ([]string, error) {
var localPodIPs []string
nodeName := os.Getenv(util.HostnameEnv)
allPods, err := c.podsLister.List(labels.Everything())
if err != nil {
klog.Errorf("list pods failed, %+v", err)
return nil, err
}
for _, pod := range allPods {
if pod.Spec.HostNetwork ||
pod.DeletionTimestamp != nil ||
pod.Spec.NodeName != nodeName ||
pod.Annotations[util.LogicalSwitchAnnotation] == "" ||
pod.Annotations[util.IpAddressAnnotation] == "" {
continue
}
subnet, err := c.subnetsLister.Get(pod.Annotations[util.LogicalSwitchAnnotation])
if err != nil {
klog.Errorf("get subnet %s failed, %+v", pod.Annotations[util.LogicalSwitchAnnotation], err)
continue
}

if !subnet.Spec.NatOutgoing ||
subnet.Spec.Vlan != "" ||
subnet.Spec.Vpc != util.DefaultVpc ||
subnet.Spec.GatewayType != kubeovnv1.GWDistributedType {
continue
}

if len(pod.Status.PodIPs) != 0 {
if len(pod.Status.PodIPs) == 2 && protocol == kubeovnv1.ProtocolIPv6 {
localPodIPs = append(localPodIPs, pod.Status.PodIPs[1].IP)
} else if util.CheckProtocol(pod.Status.PodIP) == protocol {
localPodIPs = append(localPodIPs, pod.Status.PodIP)
}
} else {
ipv4, ipv6 := util.SplitStringIP(pod.Annotations[util.IpAddressAnnotation])
if ipv4 != "" && protocol == kubeovnv1.ProtocolIPv4 {
localPodIPs = append(localPodIPs, ipv4)
}
if ipv6 != "" && protocol == kubeovnv1.ProtocolIPv6 {
localPodIPs = append(localPodIPs, ipv6)
}
}
}

klog.V(3).Infof("local pod ips %v", localPodIPs)
return localPodIPs, nil
}

func (c *Controller) getLocalPodIPsNeedPR(protocol string) (map[policyRouteMeta][]string, error) {
allPods, err := c.podsLister.List(labels.Everything())
if err != nil {
Expand Down Expand Up @@ -805,8 +702,6 @@ func (c *Controller) getSubnetsNeedNAT(protocol string) ([]string, error) {
if subnet.DeletionTimestamp == nil &&
subnet.Spec.NatOutgoing &&
subnet.Spec.Vlan == "" &&
subnet.Spec.GatewayType == kubeovnv1.GWCentralizedType &&
util.GatewayContains(subnet.Spec.GatewayNode, c.config.NodeName) &&
subnet.Spec.Vpc == util.DefaultVpc &&
(subnet.Spec.Protocol == kubeovnv1.ProtocolDual || subnet.Spec.Protocol == protocol) {
cidrBlock := getCidrByProtocol(subnet.Spec.CIDRBlock, protocol)
Expand Down

0 comments on commit bf167a6

Please sign in to comment.