Skip to content

Commit

Permalink
avoid frequent ipset update
Browse files Browse the repository at this point in the history
  • Loading branch information
@zhangzujian
zhangzujian committed Mar 28, 2022
1 parent f475736 commit ff695aa
Show file tree
Hide file tree
Showing 5 changed files with 14 additions and 164 deletions.
4 changes: 0 additions & 4 deletions dist/images/uninstall.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,8 @@ if [ -n "$1" ]; then
fi
fi

iptables -t nat -D POSTROUTING -m set ! --match-set ovn40subnets src -m set ! --match-set ovn40other-node src -m set --match-set ovn40local-pod-ip-nat dst -j RETURN
iptables -t nat -D POSTROUTING -m set ! --match-set ovn40subnets src -m set ! --match-set ovn40other-node src -m set --match-set ovn40subnets-nat dst -j RETURN
iptables -t nat -D POSTROUTING -m set --match-set ovn40subnets-nat src -m set ! --match-set ovn40subnets dst -j MASQUERADE
iptables -t nat -D POSTROUTING -m set --match-set ovn40local-pod-ip-nat src -m set ! --match-set ovn40subnets dst -j MASQUERADE
iptables -t filter -D INPUT -m set --match-set ovn40subnets dst -j ACCEPT
iptables -t filter -D INPUT -m set --match-set ovn40subnets src -j ACCEPT
iptables -t filter -D INPUT -m set --match-set ovn40services dst -j ACCEPT
Expand All @@ -48,10 +46,8 @@ ipset destroy ovn40local-pod-ip-nat
ipset destroy ovn40other-node
ipset destroy ovn40services

ip6tables -t nat -D POSTROUTING -m set ! --match-set ovn60subnets src -m set ! --match-set ovn60other-node src -m set --match-set ovn60local-pod-ip-nat dst -j RETURN
ip6tables -t nat -D POSTROUTING -m set ! --match-set ovn60subnets src -m set ! --match-set ovn60other-node src -m set --match-set ovn60subnets-nat dst -j RETURN
ip6tables -t nat -D POSTROUTING -m set --match-set ovn60subnets-nat src -m set ! --match-set ovn60subnets dst -j MASQUERADE
ip6tables -t nat -D POSTROUTING -m set --match-set ovn60local-pod-ip-nat src -m set ! --match-set ovn60subnets dst -j MASQUERADE
ip6tables -t filter -D INPUT -m set --match-set ovn60subnets dst -j ACCEPT
ip6tables -t filter -D INPUT -m set --match-set ovn60subnets src -j ACCEPT
ip6tables -t filter -D INPUT -m set --match-set ovn60services dst -j ACCEPT
Expand Down
1 change: 1 addition & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@ require (
)

replace (
github.com/alauda/felix => github.com/kubeovn/felix v0.0.0-20220325073257-c8a0f705d139
github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2
github.com/greenpau/ovsdb => github.com/alauda/ovsdb v0.0.0-20210113100339-040cf3e76c28
github.com/openshift/api => github.com/openshift/api v0.0.0-20210428205234-a8389931bee7
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,6 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
github.com/alauda/felix v3.6.6-0.20201207121355-187332daf314+incompatible h1:f8w9fTcjS9/Zdw57Law9JsFxh/sttnBor4TGkWQpSf8=
github.com/alauda/felix v3.6.6-0.20201207121355-187332daf314+incompatible/go.mod h1:NeFJ7/fvKQ8X9styGJvh7sHmZoUprV5ZKzE2Ey82BEA=
github.com/alauda/ovsdb v0.0.0-20210113100339-040cf3e76c28 h1:FW5M3SAwSGBdtTboeV5sI7kEY6zraApSZQxTUfZ7LQY=
github.com/alauda/ovsdb v0.0.0-20210113100339-040cf3e76c28/go.mod h1:dXpg+IAC2yp2IZQlEVmnmEc1rqEmSZzgNfu6+ai38J4=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
Expand Down Expand Up @@ -687,6 +685,8 @@ github.com/kr/text v0.0.0-20160504234017-7cafcd837844/go.mod h1:sjUstKUATFIcff4q
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kubeovn/felix v0.0.0-20220325073257-c8a0f705d139 h1:MaLC8/dohKHU8nkfglfE2oikefB6urJG75yZDOcKTRU=
github.com/kubeovn/felix v0.0.0-20220325073257-c8a0f705d139/go.mod h1:ulxnUH9cbIOtCH+exhJPeV2mleh+bDv67WKsl/MVU/g=
github.com/kubernetes-csi/csi-lib-utils v0.7.0/go.mod h1:bze+2G9+cmoHxN6+WyG1qT4MDxgZJMLGwc7V4acPNm0=
github.com/kubernetes-csi/csi-test v2.0.0+incompatible/go.mod h1:YxJ4UiuPWIhMBkxUKY5c267DyA0uDZ/MtAimhx/2TA0=
github.com/kubernetes-csi/external-snapshotter/v2 v2.1.1 h1:t5bmB3Y8nCaLA4aFrIpX0zjHEF/HUkJp6f5rm7BsVzM=
Expand Down
167 changes: 10 additions & 157 deletions pkg/daemon/gateway.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,11 +86,6 @@ func (c *Controller) setIPSet() error {
klog.Errorf("get subnets failed, %+v", err)
return err
}
localPodIPs, err := c.getLocalPodIPsNeedNAT(protocol)
if err != nil {
klog.Errorf("get local pod ips failed, %+v", err)
return err
}
subnetsNeedNat, err := c.getSubnetsNeedNAT(protocol)
if err != nil {
klog.Errorf("get need nat subnets failed, %+v", err)
Expand All @@ -115,7 +110,7 @@ func (c *Controller) setIPSet() error {
MaxSize: 1048576,
SetID: LocalPodSet,
Type: ipsets.IPSetTypeHashIP,
}, localPodIPs)
}, nil)
c.ipsets[protocol].AddOrReplaceIPSet(ipsets.IPSetMetadata{
MaxSize: 1048576,
SetID: SubnetNatSet,
Expand Down Expand Up @@ -186,13 +181,9 @@ func (c *Controller) addEgressConfig(subnet *kubeovnv1.Subnet, ip string) error
return nil
}

podIPs := strings.Split(ip, ",")
protocol := util.CheckProtocol(ip)
if subnet.Spec.NatOutgoing {
c.addIPSetMembers(LocalPodSet, protocol, podIPs)
return nil
}
if subnet.Spec.ExternalEgressGateway != "" {
if !subnet.Spec.NatOutgoing && subnet.Spec.ExternalEgressGateway != "" {
podIPs := strings.Split(ip, ",")
protocol := util.CheckProtocol(ip)
return c.addPodPolicyRouting(protocol, subnet.Spec.ExternalEgressGateway, subnet.Spec.PolicyRoutingPriority, subnet.Spec.PolicyRoutingTableID, podIPs)
}

Expand All @@ -218,57 +209,15 @@ func (c *Controller) removeEgressConfig(subnet, ip string) error {
return nil
}

podIPs := strings.Split(ip, ",")
protocol := util.CheckProtocol(ip)
if podSubnet.Spec.NatOutgoing {
c.removeIPSetMembers(LocalPodSet, protocol, podIPs)
return nil
}
if podSubnet.Spec.ExternalEgressGateway != "" {
if !podSubnet.Spec.NatOutgoing && podSubnet.Spec.ExternalEgressGateway != "" {
podIPs := strings.Split(ip, ",")
protocol := util.CheckProtocol(ip)
return c.deletePodPolicyRouting(protocol, podSubnet.Spec.ExternalEgressGateway, podSubnet.Spec.PolicyRoutingPriority, podSubnet.Spec.PolicyRoutingTableID, podIPs)
}

return nil
}

func (c *Controller) addIPSetMembers(setID, protocol string, ips []string) {
c.ipsetLock.Lock()
defer c.ipsetLock.Unlock()

if protocol == kubeovnv1.ProtocolDual {
if c.ipsets[kubeovnv1.ProtocolIPv4] != nil {
c.ipsets[kubeovnv1.ProtocolIPv4].AddMembers(setID, ips[:1])
c.ipsets[kubeovnv1.ProtocolIPv4].ApplyUpdates()
}
if c.ipsets[kubeovnv1.ProtocolIPv6] != nil {
c.ipsets[kubeovnv1.ProtocolIPv6].AddMembers(setID, ips[1:])
c.ipsets[kubeovnv1.ProtocolIPv6].ApplyUpdates()
}
} else if c.ipsets[protocol] != nil {
c.ipsets[protocol].AddMembers(setID, ips[:1])
c.ipsets[protocol].ApplyUpdates()
}
}

func (c *Controller) removeIPSetMembers(setID, protocol string, ips []string) {
c.ipsetLock.Lock()
defer c.ipsetLock.Unlock()

if protocol == kubeovnv1.ProtocolDual {
if c.ipsets[kubeovnv1.ProtocolIPv4] != nil {
c.ipsets[kubeovnv1.ProtocolIPv4].RemoveMembers(setID, ips[:1])
c.ipsets[kubeovnv1.ProtocolIPv4].ApplyUpdates()
}
if c.ipsets[kubeovnv1.ProtocolIPv6] != nil {
c.ipsets[kubeovnv1.ProtocolIPv6].RemoveMembers(setID, ips[1:])
c.ipsets[kubeovnv1.ProtocolIPv6].ApplyUpdates()
}
} else if c.ipsets[protocol] != nil {
c.ipsets[protocol].RemoveMembers(setID, ips[:1])
c.ipsets[protocol].ApplyUpdates()
}
}

func (c *Controller) addPodPolicyRouting(podProtocol, externalEgressGateway string, priority, tableID uint32, ips []string) error {
egw := strings.Split(externalEgressGateway, ",")
prMetas := make([]policyRouteMeta, 0, 2)
Expand Down Expand Up @@ -434,11 +383,9 @@ func (c *Controller) setIptables() error {

v4Rules = []util.IPTableRule{
// do not nat route traffic
{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m set ! --match-set ovn40subnets src -m set ! --match-set ovn40other-node src -m set --match-set ovn40local-pod-ip-nat dst -j RETURN`)},
{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m set ! --match-set ovn40subnets src -m set ! --match-set ovn40other-node src -m set --match-set ovn40subnets-nat dst -j RETURN`)},
// nat outgoing
{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m set --match-set ovn40subnets-nat src -m set ! --match-set ovn40subnets dst -j MASQUERADE`)},
{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m set --match-set ovn40local-pod-ip-nat src -m set ! --match-set ovn40subnets dst -j MASQUERADE`)},
// Input Accept
{Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn40subnets src -j ACCEPT`)},
{Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn40subnets dst -j ACCEPT`)},
Expand All @@ -454,11 +401,9 @@ func (c *Controller) setIptables() error {
}
v6Rules = []util.IPTableRule{
// do not nat route traffic
{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m set ! --match-set ovn60subnets src -m set ! --match-set ovn60other-node src -m set --match-set ovn60local-pod-ip-nat dst -j RETURN`)},
{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m set ! --match-set ovn60subnets src -m set ! --match-set ovn60other-node src -m set --match-set ovn60subnets-nat dst -j RETURN`)},
// nat outgoing
{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m set --match-set ovn60subnets-nat src -m set ! --match-set ovn60subnets dst -j MASQUERADE`)},
{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m set --match-set ovn60local-pod-ip-nat src -m set ! --match-set ovn60subnets dst -j MASQUERADE`)},
// Input Accept
{Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn60subnets src -j ACCEPT`)},
{Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn60subnets dst -j ACCEPT`)},
Expand Down Expand Up @@ -507,10 +452,10 @@ func (c *Controller) setIptables() error {
)

rules := make([]util.IPTableRule, len(iptablesRules)+2)
copy(rules[1:5], iptablesRules[:4])
copy(rules[1:3], iptablesRules[:2])
rules[0] = util.IPTableRule{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(fmt.Sprintf(`! -s %s -m mark --mark 0x4000/0x4000 -j MASQUERADE`, nodeIP))}
rules[5] = util.IPTableRule{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(fmt.Sprintf(`! -s %s -m set ! --match-set %s src -m set --match-set %s dst -j MASQUERADE`, nodeIP, matchset, matchset))}
copy(rules[6:], iptablesRules[4:])
rules[3] = util.IPTableRule{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(fmt.Sprintf(`! -s %s -m set ! --match-set %s src -m set --match-set %s dst -j MASQUERADE`, nodeIP, matchset, matchset))}
copy(rules[4:], iptablesRules[2:])
iptablesRules = rules
}

Expand Down Expand Up @@ -672,62 +617,6 @@ func (c *Controller) setExGateway() error {
return nil
}

func (c *Controller) getLocalPodIPsNeedNAT(protocol string) ([]string, error) {
var localPodIPs []string
nodeName := os.Getenv(util.HostnameEnv)
allPods, err := c.podsLister.List(labels.Everything())
if err != nil {
klog.Errorf("list pods failed, %+v", err)
return nil, err
}
for _, pod := range allPods {
if pod.Spec.HostNetwork ||
pod.DeletionTimestamp != nil ||
pod.Spec.NodeName != nodeName ||
pod.Annotations[util.LogicalSwitchAnnotation] == "" ||
pod.Annotations[util.IpAddressAnnotation] == "" {
continue
}
subnet, err := c.subnetsLister.Get(pod.Annotations[util.LogicalSwitchAnnotation])
if err != nil {
klog.Errorf("get subnet %s failed, %+v", pod.Annotations[util.LogicalSwitchAnnotation], err)
continue
}

if !subnet.Spec.NatOutgoing ||
subnet.Spec.Vpc != util.DefaultVpc ||
subnet.Spec.GatewayType != kubeovnv1.GWDistributedType {
continue
}
if subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway {
continue
}

if len(pod.Status.PodIPs) != 0 {
if len(pod.Status.PodIPs) == 2 && protocol == kubeovnv1.ProtocolIPv6 {
localPodIPs = append(localPodIPs, pod.Status.PodIPs[1].IP)
} else if util.CheckProtocol(pod.Status.PodIP) == protocol {
localPodIPs = append(localPodIPs, pod.Status.PodIP)
}
} else {
ipv4, ipv6 := util.SplitStringIP(pod.Annotations[util.IpAddressAnnotation])
if ipv4 != "" && protocol == kubeovnv1.ProtocolIPv4 {
localPodIPs = append(localPodIPs, ipv4)
}
if ipv6 != "" && protocol == kubeovnv1.ProtocolIPv6 {
localPodIPs = append(localPodIPs, ipv6)
}
}
attachIps, err := c.getAttachmentLocalPodIPsNeedNAT(pod, nodeName, protocol)
if len(attachIps) != 0 && err == nil {
localPodIPs = append(localPodIPs, attachIps...)
}
}

klog.V(3).Infof("local pod ips %v", localPodIPs)
return localPodIPs, nil
}

func (c *Controller) getLocalPodIPsNeedPR(protocol string) (map[policyRouteMeta][]string, error) {
allPods, err := c.podsLister.List(labels.Everything())
if err != nil {
Expand Down Expand Up @@ -814,8 +703,6 @@ func (c *Controller) getSubnetsNeedNAT(protocol string) ([]string, error) {
if subnet.DeletionTimestamp == nil &&
subnet.Spec.NatOutgoing &&
(subnet.Spec.Vlan == "" || subnet.Spec.LogicalGateway) &&
subnet.Spec.GatewayType == kubeovnv1.GWCentralizedType &&
util.GatewayContains(subnet.Spec.GatewayNode, c.config.NodeName) &&
subnet.Spec.Vpc == util.DefaultVpc &&
(subnet.Spec.Protocol == kubeovnv1.ProtocolDual || subnet.Spec.Protocol == protocol) {
cidrBlock := getCidrByProtocol(subnet.Spec.CIDRBlock, protocol)
Expand Down Expand Up @@ -1071,37 +958,3 @@ func getIptablesRuleNum(table, chain, rule, dstNatIp string) (string, error) {
}
return num, nil
}

func (c *Controller) getAttachmentLocalPodIPsNeedNAT(pod *v1.Pod, nodeName, protocol string) ([]string, error) {
var attachPodIPs []string

attachNets, err := util.ParsePodNetworkAnnotation(pod.Annotations[util.AttachmentNetworkAnnotation], pod.Namespace)
if err != nil {
klog.Errorf("failed to parse attach net for pod '%s', %v", pod.Name, err)
return attachPodIPs, err
}
for _, multiNet := range attachNets {
provider := fmt.Sprintf("%s.%s.ovn", multiNet.Name, multiNet.Namespace)
if pod.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, provider)] == "true" {
subnet, err := c.subnetsLister.Get(pod.Annotations[fmt.Sprintf(util.LogicalSwitchAnnotationTemplate, provider)])
if err != nil {
klog.Errorf("get subnet %s failed, %+v", pod.Annotations[fmt.Sprintf(util.LogicalSwitchAnnotationTemplate, provider)], err)
continue
}

if subnet.Spec.NatOutgoing &&
subnet.Spec.Vpc == util.DefaultVpc &&
subnet.Spec.GatewayType == kubeovnv1.GWDistributedType &&
pod.Spec.NodeName == nodeName {
ipv4, ipv6 := util.SplitStringIP(pod.Annotations[fmt.Sprintf(util.IpAddressAnnotationTemplate, provider)])
if ipv4 != "" && protocol == kubeovnv1.ProtocolIPv4 {
attachPodIPs = append(attachPodIPs, ipv4)
}
if ipv6 != "" && protocol == kubeovnv1.ProtocolIPv6 {
attachPodIPs = append(attachPodIPs, ipv6)
}
}
}
}
return attachPodIPs, nil
}
2 changes: 1 addition & 1 deletion test/e2e/service/service.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -302,7 +302,7 @@ var _ = Describe("[Service]", func() {
for _, node := range nodes.Items {
hasEndpoint := hasEndpoint(node.Name, localEtpHostEndpoints)
for _, pod := range containerPods.Items {
shoudSucceed := hasEndpoint || (!proxyIpvsMode && pod.Spec.NodeName == node.Name)
shoudSucceed := hasEndpoint || !proxyIpvsMode
for _, nodeIP := range nodeIPs(node) {
checkService(shoudSucceed, "kubectl", strings.Fields(kubectlArgs(pod.Name, nodeIP, port))...)
}
Expand Down

0 comments on commit ff695aa

Please sign in to comment.