support recreate a backup pod with full annotation (#3144)

* Support full ready annotation but no ip , pod handle add * cni wait for backup pod ip to be created * no need to make sure node on ip
kubeovn · Aug 16, 2023 · a7af897 · a7af897
1 parent 515bdb7
commit a7af897
Show file tree

Hide file tree

Showing 2 changed files with 37 additions and 18 deletions.
diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go
@@ -215,12 +215,12 @@ func (c *Controller) enqueueAddPod(obj interface{}) {
 		return
 	}
 
-	exist, err := c.podNeedSync(p)
+	need, err := c.podNeedSync(p)
 	if err != nil {
 		klog.Errorf("invalid pod net: %v", err)
 		return
 	}
-	if exist {
+	if need {
 		klog.Infof("enqueue add pod %s", key)
 		c.addOrUpdatePodQueue.Add(key)
 	}
@@ -1266,6 +1266,13 @@ func (c *Controller) podNeedSync(pod *v1.Pod) (bool, error) {
 		if pod.Annotations[fmt.Sprintf(util.RoutedAnnotationTemplate, n.ProviderName)] != "true" {
 			return true, nil
 		}
+		ipName := ovs.PodNameToPortName(pod.Name, pod.Namespace, n.ProviderName)
+		if _, err = c.ipsLister.Get(ipName); err != nil {
+			err = fmt.Errorf("pod has no ip %s: %v", ipName, err)
+			// need to sync to create ip
+			klog.Error(err)
+			return true, nil
+		}
 	}
 	return false, nil
 }

diff --git a/pkg/daemon/handler.go b/pkg/daemon/handler.go
@@ -331,24 +331,36 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
 
 func (csh cniServerHandler) UpdateIPCr(podRequest request.CniRequest, subnet, ip string) error {
 	ipCrName := ovs.PodNameToPortName(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider)
-	oriIpCr, err := csh.KubeOvnClient.KubeovnV1().IPs().Get(context.Background(), ipCrName, metav1.GetOptions{})
-	if err != nil {
-		errMsg := fmt.Errorf("failed to get ip crd for %s, %v", ip, err)
-		klog.Error(errMsg)
-		return errMsg
-	} else {
-		ipCr := oriIpCr.DeepCopy()
-		ipCr.Spec.NodeName = csh.Config.NodeName
-		ipCr.Spec.AttachIPs = []string{}
-		ipCr.Labels[subnet] = ""
-		ipCr.Spec.AttachSubnets = []string{}
-		ipCr.Spec.AttachMacs = []string{}
-		if _, err := csh.KubeOvnClient.KubeovnV1().IPs().Update(context.Background(), ipCr, metav1.UpdateOptions{}); err != nil {
-			errMsg := fmt.Errorf("failed to update ip crd for %s, %v", ip, err)
-			klog.Error(errMsg)
-			return errMsg
+	for i := 0; i < 20; i++ {
+		oriIpCr, err := csh.KubeOvnClient.KubeovnV1().IPs().Get(context.Background(), ipCrName, metav1.GetOptions{})
+		if err != nil {
+			err = fmt.Errorf("failed to get ip crd for %s, %v", ip, err)
+			// maybe create a backup pod with previous annotations
+			klog.Error(err)
+		} else {
+			if oriIpCr.Spec.NodeName != csh.Config.NodeName {
+				ipCr := oriIpCr.DeepCopy()
+				ipCr.Spec.NodeName = csh.Config.NodeName
+				ipCr.Spec.AttachIPs = []string{}
+				ipCr.Labels[subnet] = ""
+				ipCr.Spec.AttachSubnets = []string{}
+				ipCr.Spec.AttachMacs = []string{}
+				if _, err := csh.KubeOvnClient.KubeovnV1().IPs().Update(context.Background(), ipCr, metav1.UpdateOptions{}); err != nil {
+					err = fmt.Errorf("failed to update ip crd for %s, %v", ip, err)
+					klog.Error(err)
+				} else {
+					return nil
+				}
+			}
+		}
+		if err != nil {
+			klog.Warning("wait pod ip %s to be ready", ipCrName)
+			time.Sleep(1 * time.Second)
+		} else {
+			return nil
 		}
 	}
+	// update ip spec node is not that necessary, so we just log the error
 	return nil
 }