modify pod's process of update for use multus cni as default cni

kubeovn · Dec 30, 2021 · 8611de8 · 8611de8
1 parent b59f522
commit 8611de8
Show file tree

Hide file tree

Showing 5 changed files with 52 additions and 39 deletions.
diff --git a/pkg/controller/init.go b/pkg/controller/init.go
@@ -297,20 +297,23 @@ func (c *Controller) InitIPAM() error {
 		return err
 	}
 	for _, pod := range pods {
-		if isPodAlive(pod) && pod.Annotations[util.AllocatedAnnotation] == "true" {
-			podNets, err := c.getPodKubeovnNets(pod)
-			if err != nil {
-				klog.Errorf("failed to get pod kubeovn nets %s.%s address %s: %v", pod.Name, pod.Namespace, pod.Annotations[util.IpAddressAnnotation], err)
+		podNets, err := c.getPodKubeovnNets(pod)
+		if err != nil {
+			klog.Errorf("failed to get pod kubeovn nets %s.%s address %s: %v", pod.Name, pod.Namespace, pod.Annotations[util.IpAddressAnnotation], err)
+		}
+		for _, podNet := range podNets {
+			if !isOvnSubnet(podNet.Subnet) {
+				continue
 			}
-			for _, podNet := range podNets {
+			if isPodAlive(pod) && pod.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, podNet.ProviderName)] == "true" {
 				_, _, _, err := c.ipam.GetStaticAddress(
 					fmt.Sprintf("%s/%s", pod.Namespace, pod.Name),
 					ovs.PodNameToPortName(pod.Name, pod.Namespace, podNet.ProviderName),
 					pod.Annotations[fmt.Sprintf(util.IpAddressAnnotationTemplate, podNet.ProviderName)],
 					pod.Annotations[fmt.Sprintf(util.MacAddressAnnotationTemplate, podNet.ProviderName)],
 					pod.Annotations[fmt.Sprintf(util.LogicalSwitchAnnotationTemplate, podNet.ProviderName)], false)
 				if err != nil {
-					klog.Errorf("failed to init pod %s.%s address %s: %v", pod.Name, pod.Namespace, pod.Annotations[util.IpAddressAnnotation], err)
+					klog.Errorf("failed to init pod %s.%s address %s: %v", pod.Name, pod.Namespace, pod.Annotations[fmt.Sprintf(util.IpAddressAnnotationTemplate, podNet.ProviderName)], err)
 				}
 			}
 			if err = c.initAppendPodExternalIds(pod); err != nil {

diff --git a/pkg/controller/node.go b/pkg/controller/node.go
@@ -741,8 +741,20 @@ func (c *Controller) fetchPodsOnNode(nodeName string) ([]string, error) {
 			continue
 		}
 
-		if pod.Annotations[util.AllocatedAnnotation] == "true" {
-			ports = append(ports, fmt.Sprintf("%s.%s", pod.Name, pod.Namespace))
+		podNets, err := c.getPodKubeovnNets(pod)
+		if err != nil {
+			klog.Errorf("failed to get pod nets %v", err)
+			return nil, err
+		}
+
+		for _, podNet := range podNets {
+			if !isOvnSubnet(podNet.Subnet) {
+				continue
+			}
+
+			if pod.Annotations != nil && pod.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, podNet.ProviderName)] == "true" {
+				ports = append(ports, ovs.PodNameToPortName(pod.Name, pod.Namespace, podNet.ProviderName))
+			}
 		}
 	}
 	return ports, nil

diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go
@@ -94,24 +94,23 @@ func (c *Controller) enqueueAddPod(obj interface{}) {
 		return
 	}
 	// In case update event might lost during leader election
-	if p.Annotations != nil &&
-		p.Annotations[util.AllocatedAnnotation] == "true" &&
-		p.Status.HostIP != "" && p.Status.PodIP != "" {
-		for _, podNet := range podNets {
-			if !isOvnSubnet(podNet.Subnet) {
-				continue
-			}
+	for _, podNet := range podNets {
+		if !isOvnSubnet(podNet.Subnet) {
+			continue
+		}
 
+		if p.Annotations != nil &&
+			p.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, podNet.ProviderName)] == "true" &&
+			p.Status.HostIP != "" && p.Status.PodIP != "" {
 			if p.Annotations[fmt.Sprintf(util.RoutedAnnotationTemplate, podNet.ProviderName)] != "true" {
 				c.updatePodQueue.Add(key)
-				break
+				return
 			}
 		}
-		return
-	}
 
-	if p.Annotations != nil && p.Annotations[util.AllocatedAnnotation] == "true" {
-		return
+		if p.Annotations != nil && p.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, podNet.ProviderName)] == "true" {
+			return
+		}
 	}
 
 	klog.V(3).Infof("enqueue add pod %s", key)
@@ -227,13 +226,12 @@ func (c *Controller) enqueueUpdatePod(oldObj, newObj interface{}) {
 	}
 
 	// pod assigned an ip
-	if newPod.Annotations[util.AllocatedAnnotation] == "true" &&
-		newPod.Spec.NodeName != "" {
-		for _, podNet := range podNets {
-			if !isOvnSubnet(podNet.Subnet) {
-				continue
-			}
+	for _, podNet := range podNets {
+		if !isOvnSubnet(podNet.Subnet) {
+			continue
+		}
 
+		if newPod.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, podNet.ProviderName)] == "true" && newPod.Spec.NodeName != "" {
 			if newPod.Annotations[fmt.Sprintf(util.RoutedAnnotationTemplate, podNet.ProviderName)] != "true" {
 				klog.V(3).Infof("enqueue update pod %s", key)
 				c.updatePodQueue.Add(key)
@@ -487,8 +485,8 @@ func (c *Controller) handleAddPod(key string) error {
 		pod.Annotations[fmt.Sprintf(util.GatewayAnnotationTemplate, podNet.ProviderName)] = subnet.Spec.Gateway
 		pod.Annotations[fmt.Sprintf(util.LogicalSwitchAnnotationTemplate, podNet.ProviderName)] = subnet.Name
 		pod.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, podNet.ProviderName)] = "true"
-		if pod.Annotations[util.PodNicAnnotation] == "" {
-			pod.Annotations[util.PodNicAnnotation] = c.config.PodNicType
+		if pod.Annotations[fmt.Sprintf(util.PodNicAnnotationTemplate, podNet.ProviderName)] == "" {
+			pod.Annotations[fmt.Sprintf(util.PodNicAnnotationTemplate, podNet.ProviderName)] = c.config.PodNicType
 		}
 
 		if err := util.ValidatePodCidr(podNet.Subnet.Spec.CIDRBlock, ipStr); err != nil {
@@ -710,11 +708,6 @@ func (c *Controller) handleUpdatePod(key string) error {
 	}
 	pod := oripod.DeepCopy()
 
-	// in case update handler overlap the annotation when cache is not in sync
-	if pod.Annotations[util.AllocatedAnnotation] == "" {
-		return fmt.Errorf("no address has been allocated to %s/%s", namespace, name)
-	}
-
 	klog.Infof("update pod %s/%s", namespace, name)
 
 	var podIP string
@@ -729,6 +722,11 @@ func (c *Controller) handleUpdatePod(key string) error {
 		if !isOvnSubnet(podNet.Subnet) {
 			continue
 		}
+		// in case update handler overlap the annotation when cache is not in sync
+		if pod.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, podNet.ProviderName)] == "" {
+			return fmt.Errorf("no address has been allocated to %s/%s", namespace, name)
+		}
+
 		podIP = pod.Annotations[fmt.Sprintf(util.IpAddressAnnotationTemplate, podNet.ProviderName)]
 		subnet = podNet.Subnet
 

diff --git a/pkg/daemon/handler.go b/pkg/daemon/handler.go
@@ -118,7 +118,7 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
 		if podRequest.DeviceID != "" {
 			nicType = util.OffloadType
 		} else {
-			nicType = pod.Annotations[util.PodNicAnnotation]
+			nicType = pod.Annotations[fmt.Sprintf(util.PodNicAnnotationTemplate, podRequest.Provider)]
 		}
 
 		switch pod.Annotations[fmt.Sprintf(util.DefaultRouteAnnotationTemplate, podRequest.Provider)] {
@@ -341,7 +341,7 @@ func (csh cniServerHandler) handleDel(req *restful.Request, resp *restful.Respon
 		if podRequest.DeviceID != "" {
 			nicType = util.OffloadType
 		} else {
-			nicType = pod.Annotations[util.PodNicAnnotation]
+			nicType = pod.Annotations[fmt.Sprintf(util.PodNicAnnotationTemplate, podRequest.Provider)]
 		}
 		err = csh.deleteNic(podRequest.PodName, podRequest.PodNamespace, podRequest.ContainerID, podRequest.DeviceID, podRequest.IfName, nicType)
 		if err != nil {

diff --git a/pkg/util/const.go b/pkg/util/const.go
@@ -59,6 +59,7 @@ const (
 	ProviderNetworkInterfaceTemplate = "%s.provider-network.kubernetes.io/interface"
 	ProviderNetworkMtuTemplate       = "%s.provider-network.kubernetes.io/mtu"
 	MirrorControlAnnotationTemplate  = "%s.kubernetes.io/mirror"
+	PodNicAnnotationTemplate         = "%s.kubernetes.io/pod_nic_type"
 
 	ExcludeIpsAnnotation = "ovn.kubernetes.io/exclude_ips"
 
@@ -123,10 +124,9 @@ const (
 
 	NodeRouterPolicyPriority = 30000
 
-	PodNicAnnotation = "ovn.kubernetes.io/pod_nic_type"
-	VethType         = "veth-pair"
-	OffloadType      = "offload-port"
-	InternalType     = "internal-port"
+	VethType     = "veth-pair"
+	OffloadType  = "offload-port"
+	InternalType = "internal-port"
 
 	ChassisLoc     = "/etc/openvswitch/system-id.conf"
 	HostnameEnv    = "KUBE_NODE_NAME"