adapt ippool annotation (#2678)

* adapt ippool annotation * append e2e for ippool separated by comma
kubeovn · Apr 20, 2023 · 951f89c · 951f89c
1 parent 96e8be6
commit 951f89c
Show file tree

Hide file tree

Showing 2 changed files with 104 additions and 31 deletions.
diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go
@@ -1438,49 +1438,54 @@ func (c *Controller) acquireAddress(pod *v1.Pod, podNet *kubeovnNet) (string, st
 	}
 
 	// IPPool allocate
-	ipPool := strings.Split(pod.Annotations[fmt.Sprintf(util.IpPoolAnnotationTemplate, podNet.ProviderName)], ";")
-	for i, ip := range ipPool {
-		ipPool[i] = strings.TrimSpace(ip)
-	}
+	if pod.Annotations[fmt.Sprintf(util.IpPoolAnnotationTemplate, podNet.ProviderName)] != "" {
+		var ipPool []string
+		if strings.Contains(pod.Annotations[fmt.Sprintf(util.IpPoolAnnotationTemplate, podNet.ProviderName)], ";") {
+			ipPool = strings.Split(pod.Annotations[fmt.Sprintf(util.IpPoolAnnotationTemplate, podNet.ProviderName)], ";")
+		} else {
+			ipPool = strings.Split(pod.Annotations[fmt.Sprintf(util.IpPoolAnnotationTemplate, podNet.ProviderName)], ",")
+		}
+		for i, ip := range ipPool {
+			ipPool[i] = strings.TrimSpace(ip)
+		}
 
-	if !isStsPod {
-		for _, net := range nsNets {
-			for _, staticIPs := range ipPool {
-				ipProtocol := util.CheckProtocol(staticIPs)
-				for _, staticIP := range strings.Split(staticIPs, ",") {
-					if assignedPod, ok := c.ipam.IsIPAssignedToOtherPod(staticIP, net.Subnet.Name, key); ok {
+		if !isStsPod {
+			for _, net := range nsNets {
+				for _, staticIP := range ipPool {
+					var checkIP string
+					ipProtocol := util.CheckProtocol(staticIP)
+					if ipProtocol == kubeovnv1.ProtocolDual {
+						checkIP = strings.Split(staticIP, ",")[0]
+					} else {
+						checkIP = staticIP
+					}
+
+					if assignedPod, ok := c.ipam.IsIPAssignedToOtherPod(checkIP, net.Subnet.Name, key); ok {
 						klog.Errorf("static address %s for %s has been assigned to %s", staticIP, key, assignedPod)
 						continue
 					}
 
-					if ipProtocol != kubeovnv1.ProtocolDual {
-						v4IP, v6IP, mac, err = c.acquireStaticAddress(key, portName, staticIP, macStr, net.Subnet.Name, net.AllowLiveMigration)
-						if err == nil {
-							return v4IP, v6IP, mac, net.Subnet, nil
-						}
-					}
-				}
-				if ipProtocol == kubeovnv1.ProtocolDual {
-					v4IP, v6IP, mac, err = c.acquireStaticAddress(key, portName, staticIPs, macStr, net.Subnet.Name, net.AllowLiveMigration)
+					v4IP, v6IP, mac, err = c.acquireStaticAddress(key, portName, staticIP, macStr, net.Subnet.Name, net.AllowLiveMigration)
 					if err == nil {
 						return v4IP, v6IP, mac, net.Subnet, nil
 					}
 				}
 			}
-		}
-		klog.Errorf("acquire address %s for %s failed, %v", pod.Annotations[fmt.Sprintf(util.IpPoolAnnotationTemplate, podNet.ProviderName)], key, err)
-	} else {
-		tempStrs := strings.Split(pod.Name, "-")
-		numStr := tempStrs[len(tempStrs)-1]
-		index, _ := strconv.Atoi(numStr)
-		if index < len(ipPool) {
-			for _, net := range nsNets {
-				v4IP, v6IP, mac, err = c.acquireStaticAddress(key, portName, ipPool[index], macStr, net.Subnet.Name, net.AllowLiveMigration)
-				if err == nil {
-					return v4IP, v6IP, mac, net.Subnet, nil
+			klog.Errorf("acquire address %s for %s failed, %v", pod.Annotations[fmt.Sprintf(util.IpPoolAnnotationTemplate, podNet.ProviderName)], key, err)
+		} else {
+			tempStrs := strings.Split(pod.Name, "-")
+			numStr := tempStrs[len(tempStrs)-1]
+			index, _ := strconv.Atoi(numStr)
+
+			if index < len(ipPool) {
+				for _, net := range nsNets {
+					v4IP, v6IP, mac, err = c.acquireStaticAddress(key, portName, ipPool[index], macStr, net.Subnet.Name, net.AllowLiveMigration)
+					if err == nil {
+						return v4IP, v6IP, mac, net.Subnet, nil
+					}
 				}
+				klog.Errorf("acquire address %s for %s failed, %v", ipPool[index], key, err)
 			}
-			klog.Errorf("acquire address %s for %s failed, %v", ipPool[index], key, err)
 		}
 	}
 	klog.Errorf("alloc address for %s failed, return NoAvailableAddress", key)

diff --git a/test/e2e/kube-ovn/ipam/ipam.go b/test/e2e/kube-ovn/ipam/ipam.go
@@ -327,4 +327,72 @@ var _ = framework.Describe("[group:ipam]", func() {
 		err = cs.AppsV1().StatefulSets(namespaceName).Delete(context.TODO(), name, metav1.DeleteOptions{})
 		framework.ExpectNoError(err, "failed to delete statefulset "+name)
 	})
+
+	// separate ippool annotation by comma
+	framework.ConformanceIt("should allocate static ip for statefulset with ippool separated by comma", func() {
+		if f.ClusterIpFamily == "dual" {
+			ginkgo.Skip("Comma separated ippool is not supported for dual stack")
+		}
+
+		ippoolSep := ","
+		replicas := 3
+		ippool := framework.RandomIPPool(cidr, ippoolSep, replicas)
+		labels := map[string]string{"app": stsName}
+
+		ginkgo.By("Creating statefulset " + stsName + " with ippool " + ippool)
+		sts := framework.MakeStatefulSet(stsName, stsName, int32(replicas), labels, framework.PauseImage)
+		sts.Spec.Template.Annotations = map[string]string{util.IpPoolAnnotation: ippool}
+		sts = stsClient.CreateSync(sts)
+
+		ginkgo.By("Getting pods for statefulset " + stsName)
+		pods := stsClient.GetPods(sts)
+		framework.ExpectHaveLen(pods.Items, replicas)
+
+		ips := make([]string, 0, replicas)
+		for _, pod := range pods.Items {
+			framework.ExpectHaveKeyWithValue(pod.Annotations, util.AllocatedAnnotation, "true")
+			framework.ExpectHaveKeyWithValue(pod.Annotations, util.CidrAnnotation, subnet.Spec.CIDRBlock)
+			framework.ExpectHaveKeyWithValue(pod.Annotations, util.GatewayAnnotation, subnet.Spec.Gateway)
+			framework.ExpectHaveKeyWithValue(pod.Annotations, util.IpPoolAnnotation, ippool)
+			framework.ExpectHaveKeyWithValue(pod.Annotations, util.LogicalSwitchAnnotation, subnet.Name)
+			framework.ExpectMAC(pod.Annotations[util.MacAddressAnnotation])
+			framework.ExpectHaveKeyWithValue(pod.Annotations, util.RoutedAnnotation, "true")
+
+			podIPs := make([]string, 0, len(pod.Status.PodIPs))
+			for _, podIP := range pod.Status.PodIPs {
+				podIPs = append(podIPs, podIP.IP)
+			}
+			framework.ExpectConsistOf(podIPs, strings.Split(pod.Annotations[util.IpAddressAnnotation], ","))
+			ips = append(ips, pod.Annotations[util.IpAddressAnnotation])
+		}
+		framework.ExpectConsistOf(ips, strings.Split(ippool, ippoolSep))
+
+		ginkgo.By("Deleting pods for statefulset " + stsName)
+		for _, pod := range pods.Items {
+			err := podClient.Delete(context.TODO(), pod.Name, metav1.DeleteOptions{})
+			framework.ExpectNoError(err, "failed to delete pod "+pod.Name)
+		}
+		stsClient.WaitForRunningAndReady(sts)
+
+		ginkgo.By("Getting pods for statefulset " + stsName)
+		pods = stsClient.GetPods(sts)
+		framework.ExpectHaveLen(pods.Items, replicas)
+
+		for i, pod := range pods.Items {
+			framework.ExpectHaveKeyWithValue(pod.Annotations, util.AllocatedAnnotation, "true")
+			framework.ExpectHaveKeyWithValue(pod.Annotations, util.CidrAnnotation, subnet.Spec.CIDRBlock)
+			framework.ExpectHaveKeyWithValue(pod.Annotations, util.GatewayAnnotation, subnet.Spec.Gateway)
+			framework.ExpectHaveKeyWithValue(pod.Annotations, util.IpPoolAnnotation, ippool)
+			framework.ExpectHaveKeyWithValue(pod.Annotations, util.IpAddressAnnotation, ips[i])
+			framework.ExpectHaveKeyWithValue(pod.Annotations, util.LogicalSwitchAnnotation, subnet.Name)
+			framework.ExpectMAC(pod.Annotations[util.MacAddressAnnotation])
+			framework.ExpectHaveKeyWithValue(pod.Annotations, util.RoutedAnnotation, "true")
+
+			podIPs := make([]string, 0, len(pod.Status.PodIPs))
+			for _, podIP := range pod.Status.PodIPs {
+				podIPs = append(podIPs, podIP.IP)
+			}
+			framework.ExpectConsistOf(podIPs, strings.Split(pod.Annotations[util.IpAddressAnnotation], ","))
+		}
+	})
 })