fix SNAT on pod startup

(cherry picked from commit aad8154)
kubeovn · Apr 23, 2021 · 14de53e · 14de53e
1 parent 2f42118
commit 14de53e
Show file tree

Hide file tree

Showing 2 changed files with 74 additions and 0 deletions.
diff --git a/pkg/daemon/gateway.go b/pkg/daemon/gateway.go
@@ -107,6 +107,60 @@ func (c *Controller) setIPSet() error {
 	return nil
 }
 
+func (c *Controller) addIPSetMembers(setID, subnet, ip string) error {
+	podSubnet, err := c.subnetsLister.Get(subnet)
+	if err != nil {
+		klog.Errorf("get subnet %s failed, %+v", subnet, err)
+		return err
+	}
+
+	if !podSubnet.Spec.NatOutgoing ||
+		podSubnet.Spec.Vpc != util.DefaultVpc ||
+		podSubnet.Spec.GatewayType != kubeovnv1.GWDistributedType {
+		return nil
+	}
+
+	podIPs := strings.Split(ip, ",")
+	if protocol := util.CheckProtocol(ip); protocol == kubeovnv1.ProtocolDual {
+		c.ipset[kubeovnv1.ProtocolIPv4].AddMembers(setID, []string{podIPs[0]})
+		c.ipset[kubeovnv1.ProtocolIPv6].AddMembers(setID, []string{podIPs[1]})
+		c.ipset[kubeovnv1.ProtocolIPv4].ApplyUpdates()
+		c.ipset[kubeovnv1.ProtocolIPv6].ApplyUpdates()
+	} else {
+		c.ipset[protocol].AddMembers(setID, []string{podIPs[0]})
+		c.ipset[protocol].ApplyUpdates()
+	}
+
+	return nil
+}
+
+func (c *Controller) removeIPSetMembers(setID, subnet, ip string) error {
+	podSubnet, err := c.subnetsLister.Get(subnet)
+	if err != nil {
+		klog.Errorf("get subnet %s failed, %+v", subnet, err)
+		return err
+	}
+
+	if !podSubnet.Spec.NatOutgoing ||
+		podSubnet.Spec.Vpc != util.DefaultVpc ||
+		podSubnet.Spec.GatewayType != kubeovnv1.GWDistributedType {
+		return nil
+	}
+
+	podIPs := strings.Split(ip, ",")
+	if protocol := util.CheckProtocol(ip); protocol == kubeovnv1.ProtocolDual {
+		c.ipset[kubeovnv1.ProtocolIPv4].RemoveMembers(setID, []string{podIPs[0]})
+		c.ipset[kubeovnv1.ProtocolIPv6].RemoveMembers(setID, []string{podIPs[1]})
+		c.ipset[kubeovnv1.ProtocolIPv4].ApplyUpdates()
+		c.ipset[kubeovnv1.ProtocolIPv6].ApplyUpdates()
+	} else {
+		c.ipset[protocol].RemoveMembers(setID, []string{podIPs[0]})
+		c.ipset[protocol].ApplyUpdates()
+	}
+
+	return nil
+}
+
 func (c *Controller) setIptables() error {
 	klog.V(3).Infoln("start to set up iptables")
 	node, err := c.nodesLister.Get(c.config.NodeName)

diff --git a/pkg/daemon/handler.go b/pkg/daemon/handler.go
@@ -115,6 +115,15 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
 			}
 			return
 		}
+
+		if err = csh.Controller.addIPSetMembers(LocalPodSet, subnet, ip); err != nil {
+			errMsg := fmt.Errorf("add ipset members failed %v", err)
+			klog.Error(errMsg)
+			if err = resp.WriteHeaderAndEntity(http.StatusInternalServerError, request.CniResponse{Err: errMsg.Error()}); err != nil {
+				klog.Errorf("failed to write response, %v", err)
+			}
+			return
+		}
 	}
 
 	if err := resp.WriteHeaderAndEntity(http.StatusOK, request.CniResponse{Protocol: util.CheckProtocol(cidr), IpAddress: ip, MacAddress: macAddr, CIDR: cidr, Gateway: gw}); err != nil {
@@ -208,6 +217,17 @@ func (csh cniServerHandler) handleDel(req *restful.Request, resp *restful.Respon
 
 	klog.Infof("delete port request %v", podRequest)
 	if podRequest.Provider == util.OvnProvider {
+		subnet := pod.Annotations[fmt.Sprintf(util.LogicalSwitchAnnotationTemplate, podRequest.Provider)]
+		ip := pod.Annotations[fmt.Sprintf(util.IpAddressAnnotationTemplate, podRequest.Provider)]
+		if err = csh.Controller.removeIPSetMembers(LocalPodSet, subnet, ip); err != nil {
+			errMsg := fmt.Errorf("remove ipset members failed %v", err)
+			klog.Error(errMsg)
+			if err = resp.WriteHeaderAndEntity(http.StatusInternalServerError, request.CniResponse{Err: errMsg.Error()}); err != nil {
+				klog.Errorf("failed to write response, %v", err)
+			}
+			return
+		}
+
 		err = csh.deleteNic(podRequest.PodName, podRequest.PodNamespace, podRequest.ContainerID, podRequest.DeviceID)
 		if err != nil {
 			errMsg := fmt.Errorf("del nic failed %v", err)