diff --git a/dist/images/install.sh b/dist/images/install.sh index 696da640ffe..cbeea559897 100755 --- a/dist/images/install.sh +++ b/dist/images/install.sh @@ -1354,6 +1354,14 @@ spec: type: string u2oInterconnectionIP: type: string + v4usingIPrange: + type: string + v4availableIPrange: + type: string + v6usingIPrange: + type: string + v6availableIPrange: + type: string conditions: type: array items: diff --git a/kubeovn-helm/templates/kube-ovn-crd.yaml b/kubeovn-helm/templates/kube-ovn-crd.yaml index 2723330d562..f318ac6ee25 100644 --- a/kubeovn-helm/templates/kube-ovn-crd.yaml +++ b/kubeovn-helm/templates/kube-ovn-crd.yaml @@ -1138,6 +1138,14 @@ spec: type: string u2oInterconnectionIP: type: string + v4usingIPrange: + type: string + v4availableIPrange: + type: string + v6usingIPrange: + type: string + v6availableIPrange: + type: string conditions: type: array items: diff --git a/pkg/apis/kubeovn/v1/types.go b/pkg/apis/kubeovn/v1/types.go index 8843193749d..96105d9d1ab 100644 --- a/pkg/apis/kubeovn/v1/types.go +++ b/pkg/apis/kubeovn/v1/types.go @@ -181,9 +181,13 @@ type SubnetStatus struct { Conditions []SubnetCondition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` V4AvailableIPs float64 `json:"v4availableIPs"` + V4AvailableIPRange string `json:"v4availableIPrange"` V4UsingIPs float64 `json:"v4usingIPs"` + V4UsingIPRange string `json:"v4usingIPrange"` V6AvailableIPs float64 `json:"v6availableIPs"` + V6AvailableIPRange string `json:"v6availableIPrange"` V6UsingIPs float64 `json:"v6usingIPs"` + V6UsingIPRange string `json:"v6usingIPrange"` ActivateGateway string `json:"activateGateway"` DHCPv4OptionsUUID string `json:"dhcpV4OptionsUUID"` DHCPv6OptionsUUID string `json:"dhcpV6OptionsUUID"` diff --git a/pkg/controller/subnet.go b/pkg/controller/subnet.go index 88d50e4f96f..3d40d6a5711 100644 --- a/pkg/controller/subnet.go +++ b/pkg/controller/subnet.go @@ -1568,10 +1568,16 @@ func calcDualSubnetStatusIP(subnet *kubeovnv1.Subnet, c *Controller) error { v6availableIPs = 0 } + v4UsingIPStr, v6UsingIPStr, v4AvailableIPStr, v6AvailableIPStr := c.ipam.GetSubnetIPRangeString(subnet.Name) + if subnet.Status.V4AvailableIPs == v4availableIPs && subnet.Status.V6AvailableIPs == v6availableIPs && subnet.Status.V4UsingIPs == usingIPs && - subnet.Status.V6UsingIPs == usingIPs { + subnet.Status.V6UsingIPs == usingIPs && + subnet.Status.V4UsingIPRange == v4UsingIPStr && + subnet.Status.V6UsingIPRange == v6UsingIPStr && + subnet.Status.V4AvailableIPRange == v4AvailableIPStr && + subnet.Status.V6AvailableIPRange == v6AvailableIPStr { return nil } @@ -1579,6 +1585,11 @@ func calcDualSubnetStatusIP(subnet *kubeovnv1.Subnet, c *Controller) error { subnet.Status.V6AvailableIPs = v6availableIPs subnet.Status.V4UsingIPs = usingIPs subnet.Status.V6UsingIPs = usingIPs + subnet.Status.V4UsingIPRange = v4UsingIPStr + subnet.Status.V6UsingIPRange = v6UsingIPStr + subnet.Status.V4AvailableIPRange = v4AvailableIPStr + subnet.Status.V6AvailableIPRange = v6AvailableIPStr + bytes, err := subnet.Status.Bytes() if err != nil { return err @@ -1626,28 +1637,45 @@ func calcSubnetStatusIP(subnet *kubeovnv1.Subnet, c *Controller) error { availableIPs = 0 } - cachedFields := [4]float64{ + v4UsingIPStr, v6UsingIPStr, v4AvailableIPStr, v6AvailableIPStr := c.ipam.GetSubnetIPRangeString(subnet.Name) + cachedFloatFields := [4]float64{ subnet.Status.V4AvailableIPs, subnet.Status.V4UsingIPs, subnet.Status.V6AvailableIPs, subnet.Status.V6UsingIPs, } + cachedStringFields := [4]string{ + subnet.Status.V4UsingIPRange, + subnet.Status.V4AvailableIPRange, + subnet.Status.V6UsingIPRange, + subnet.Status.V6AvailableIPRange, + } + if subnet.Spec.Protocol == kubeovnv1.ProtocolIPv4 { subnet.Status.V4AvailableIPs = availableIPs subnet.Status.V4UsingIPs = usingIPs + subnet.Status.V4UsingIPRange = v4UsingIPStr + subnet.Status.V4AvailableIPRange = v4AvailableIPStr subnet.Status.V6AvailableIPs = 0 subnet.Status.V6UsingIPs = 0 } else { subnet.Status.V6AvailableIPs = availableIPs subnet.Status.V6UsingIPs = usingIPs + subnet.Status.V6UsingIPRange = v6UsingIPStr + subnet.Status.V6AvailableIPRange = v6AvailableIPStr subnet.Status.V4AvailableIPs = 0 subnet.Status.V4UsingIPs = 0 } - if cachedFields == [4]float64{ + if cachedFloatFields == [4]float64{ subnet.Status.V4AvailableIPs, subnet.Status.V4UsingIPs, subnet.Status.V6AvailableIPs, subnet.Status.V6UsingIPs, + } && cachedStringFields == [4]string{ + subnet.Status.V4UsingIPRange, + subnet.Status.V4AvailableIPRange, + subnet.Status.V6UsingIPRange, + subnet.Status.V6AvailableIPRange, } { return nil } diff --git a/pkg/ipam/ip.go b/pkg/ipam/ip.go index 6f3f63831e4..92a03e96cc6 100644 --- a/pkg/ipam/ip.go +++ b/pkg/ipam/ip.go @@ -1,6 +1,7 @@ package ipam import ( + "fmt" "math/big" "strings" @@ -50,6 +51,18 @@ func (iprl IPRangeList) Contains(ip IP) bool { return false } +func (iprl IPRangeList) IpRangetoString() string { + var ipRangeString []string + for _, ipr := range iprl { + if ipr.Start.Equal(ipr.End) { + ipRangeString = append(ipRangeString, string(ipr.Start)) + } else { + ipRangeString = append(ipRangeString, fmt.Sprintf("%s-%s", ipr.Start, ipr.End)) + } + } + return strings.Join(ipRangeString, ",") +} + func splitIPRangeList(iprl IPRangeList, ip IP) (bool, IPRangeList) { newIPRangeList := []*IPRange{} split := false diff --git a/pkg/ipam/ipam.go b/pkg/ipam/ipam.go index e119f9aaf98..e0db2da2eb6 100644 --- a/pkg/ipam/ipam.go +++ b/pkg/ipam/ipam.go @@ -261,3 +261,20 @@ func (ipam *IPAM) GetSubnetV4Mask(subnetName string) (string, error) { return "", ErrNoAvailable } } + +func (ipam *IPAM) GetSubnetIPRangeString(subnetName string) (string, string, string, string) { + ipam.mutex.RLock() + defer ipam.mutex.RUnlock() + + var v4UsingIPStr, v6UsingIPStr, v4AvailableIPStr, v6AvailableIPStr string + + if subnet, ok := ipam.Subnets[subnetName]; ok { + + v4UsingIPStr = subnet.V4UsingIPList.IpRangetoString() + v6UsingIPStr = subnet.V6UsingIPList.IpRangetoString() + v4AvailableIPStr = subnet.V4AvailIPList.IpRangetoString() + v6AvailableIPStr = subnet.V6AvailIPList.IpRangetoString() + } + + return v4UsingIPStr, v6UsingIPStr, v4AvailableIPStr, v6AvailableIPStr +} diff --git a/pkg/ipam/subnet.go b/pkg/ipam/subnet.go index 3d6d978067b..7aa271c3293 100644 --- a/pkg/ipam/subnet.go +++ b/pkg/ipam/subnet.go @@ -20,12 +20,16 @@ type Subnet struct { V4FreeIPList IPRangeList V4ReleasedIPList IPRangeList V4ReservedIPList IPRangeList + V4AvailIPList IPRangeList + V4UsingIPList IPRangeList V4NicToIP map[string]IP V4IPToPod map[IP]string V6CIDR *net.IPNet V6FreeIPList IPRangeList V6ReleasedIPList IPRangeList V6ReservedIPList IPRangeList + V6AvailIPList IPRangeList + V6UsingIPList IPRangeList V6NicToIP map[string]IP V6IPToPod map[IP]string NicToMac map[string]string @@ -64,6 +68,8 @@ func NewSubnet(name, cidrStr string, excludeIps []string) (*Subnet, error) { V4FreeIPList: IPRangeList{&IPRange{Start: IP(firstIP), End: IP(lastIP)}}, V4ReleasedIPList: IPRangeList{}, V4ReservedIPList: convertExcludeIps(v4ExcludeIps), + V4AvailIPList: IPRangeList{&IPRange{Start: IP(firstIP), End: IP(lastIP)}}, + V4UsingIPList: IPRangeList{}, V4NicToIP: map[string]IP{}, V4IPToPod: map[IP]string{}, V6NicToIP: map[string]IP{}, @@ -85,6 +91,8 @@ func NewSubnet(name, cidrStr string, excludeIps []string) (*Subnet, error) { V6FreeIPList: IPRangeList{&IPRange{Start: IP(firstIP), End: IP(lastIP)}}, V6ReleasedIPList: IPRangeList{}, V6ReservedIPList: convertExcludeIps(v6ExcludeIps), + V6AvailIPList: IPRangeList{&IPRange{Start: IP(firstIP), End: IP(lastIP)}}, + V6UsingIPList: IPRangeList{}, V4NicToIP: map[string]IP{}, V4IPToPod: map[IP]string{}, V6NicToIP: map[string]IP{}, @@ -109,12 +117,16 @@ func NewSubnet(name, cidrStr string, excludeIps []string) (*Subnet, error) { V4FreeIPList: IPRangeList{&IPRange{Start: IP(v4FirstIP), End: IP(v4LastIP)}}, V4ReleasedIPList: IPRangeList{}, V4ReservedIPList: convertExcludeIps(v4ExcludeIps), + V4AvailIPList: IPRangeList{&IPRange{Start: IP(v4FirstIP), End: IP(v4LastIP)}}, + V4UsingIPList: IPRangeList{}, V4NicToIP: map[string]IP{}, V4IPToPod: map[IP]string{}, V6CIDR: cidrs[1], V6FreeIPList: IPRangeList{&IPRange{Start: IP(v6FirstIP), End: IP(v6LastIP)}}, V6ReleasedIPList: IPRangeList{}, V6ReservedIPList: convertExcludeIps(v6ExcludeIps), + V6AvailIPList: IPRangeList{&IPRange{Start: IP(v6FirstIP), End: IP(v6LastIP)}}, + V6UsingIPList: IPRangeList{}, V6NicToIP: map[string]IP{}, V6IPToPod: map[IP]string{}, MacToPod: map[string]string{}, @@ -241,6 +253,13 @@ func (subnet *Subnet) getV4RandomAddress(podName, nicName string, mac string, sk if !part2.Start.GreaterThan(part2.End) { subnet.V4FreeIPList = append(subnet.V4FreeIPList, part2) } + if split, NewV4AvailIPRangeList := splitIPRangeList(subnet.V4AvailIPList, ip); split { + subnet.V4AvailIPList = NewV4AvailIPRangeList + } + + if merged, NewV4UsingIPRangeList := mergeIPRangeList(subnet.V4UsingIPList, ip); merged { + subnet.V4UsingIPList = NewV4UsingIPRangeList + } subnet.V4NicToIP[nicName] = ip subnet.V4IPToPod[ip] = podName @@ -302,6 +321,13 @@ func (subnet *Subnet) getV6RandomAddress(podName, nicName string, mac string, sk if !part2.Start.GreaterThan(part2.End) { subnet.V6FreeIPList = append(subnet.V6FreeIPList, part2) } + if split, NewV6AvailIPRangeList := splitIPRangeList(subnet.V6AvailIPList, ip); split { + subnet.V6AvailIPList = NewV6AvailIPRangeList + } + + if merged, NewV6UsingIPRangeList := mergeIPRangeList(subnet.V6UsingIPList, ip); merged { + subnet.V6UsingIPList = NewV6UsingIPRangeList + } subnet.V6NicToIP[nicName] = ip subnet.V6IPToPod[ip] = podName @@ -317,13 +343,35 @@ func (subnet *Subnet) getV6RandomAddress(podName, nicName string, mac string, sk } func (subnet *Subnet) GetStaticAddress(podName, nicName string, ip IP, mac string, force bool, checkConflict bool) (IP, string, error) { + var v4, v6 bool + isAllocated := false subnet.mutex.Lock() defer func() { subnet.pushPodNic(podName, nicName) + if isAllocated { + if v4 { + if split, NewV4AvailIPRangeList := splitIPRangeList(subnet.V4AvailIPList, ip); split { + subnet.V4AvailIPList = NewV4AvailIPRangeList + } + + if merged, NewV4UsingIPRangeList := mergeIPRangeList(subnet.V4UsingIPList, ip); merged { + subnet.V4UsingIPList = NewV4UsingIPRangeList + } + } + + if v6 { + if split, NewV6AvailIPRangeList := splitIPRangeList(subnet.V6AvailIPList, ip); split { + subnet.V6AvailIPList = NewV6AvailIPRangeList + } + + if merged, NewV6UsingIPRangeList := mergeIPRangeList(subnet.V6UsingIPList, ip); merged { + subnet.V6UsingIPList = NewV6UsingIPRangeList + } + } + } subnet.mutex.Unlock() }() - var v4, v6 bool if net.ParseIP(string(ip)).To4() != nil { v4 = subnet.V4CIDR != nil } else { @@ -379,12 +427,14 @@ func (subnet *Subnet) GetStaticAddress(podName, nicName string, ip IP, mac strin subnet.V4FreeIPList = newFreeList subnet.V4NicToIP[nicName] = ip subnet.V4IPToPod[ip] = podName + isAllocated = true return ip, mac, nil } else { if split, newReleasedList := splitIPRangeList(subnet.V4ReleasedIPList, ip); split { subnet.V4ReleasedIPList = newReleasedList subnet.V4NicToIP[nicName] = ip subnet.V4IPToPod[ip] = podName + isAllocated = true return ip, mac, nil } } @@ -414,12 +464,14 @@ func (subnet *Subnet) GetStaticAddress(podName, nicName string, ip IP, mac strin subnet.V6FreeIPList = newFreeList subnet.V6NicToIP[nicName] = ip subnet.V6IPToPod[ip] = podName + isAllocated = true return ip, mac, nil } else { if split, newReleasedList := splitIPRangeList(subnet.V6ReleasedIPList, ip); split { subnet.V6ReleasedIPList = newReleasedList subnet.V6NicToIP[nicName] = ip subnet.V6IPToPod[ip] = podName + isAllocated = true return ip, mac, nil } } @@ -459,6 +511,14 @@ func (subnet *Subnet) releaseAddr(podName, nicName string) { subnet.V4ReleasedIPList = newReleasedList klog.Infof("release v4 %s mac %s for %s, add ip to released list", ip, mac, podName) } + + if merged, NewV4AvailIPRangeList := mergeIPRangeList(subnet.V4AvailIPList, ip); merged { + subnet.V4AvailIPList = NewV4AvailIPRangeList + } + + if split, NewV4UsingIPList := splitIPRangeList(subnet.V4UsingIPList, ip); split { + subnet.V4UsingIPList = NewV4UsingIPList + } } } if ip, ok = subnet.V6NicToIP[nicName]; ok { @@ -489,6 +549,14 @@ func (subnet *Subnet) releaseAddr(podName, nicName string) { subnet.V6ReleasedIPList = newReleasedList klog.Infof("release v6 %s mac %s for %s, add ip to released list", ip, mac, podName) } + + if merged, NewV6AvailIPRangeList := mergeIPRangeList(subnet.V6AvailIPList, ip); merged { + subnet.V6AvailIPList = NewV6AvailIPRangeList + } + + if split, NewV6UsingIPList := splitIPRangeList(subnet.V6UsingIPList, ip); split { + subnet.V6UsingIPList = NewV6UsingIPList + } } } } @@ -525,6 +593,14 @@ func (subnet *Subnet) joinFreeWithReserve() { } } subnet.V4FreeIPList = newFreeList + + newAvailableList := IPRangeList{} + for _, availIpr := range subnet.V4AvailIPList { + if iprl := splitRange(availIpr, reserveIpr); iprl != nil { + newAvailableList = append(newAvailableList, iprl...) + } + } + subnet.V4AvailIPList = newAvailableList } } if protocol == kubeovnv1.ProtocolDual || protocol == kubeovnv1.ProtocolIPv6 { @@ -536,6 +612,14 @@ func (subnet *Subnet) joinFreeWithReserve() { } } subnet.V6FreeIPList = newFreeList + + newAvailableList := IPRangeList{} + for _, availIpr := range subnet.V6AvailIPList { + if iprl := splitRange(availIpr, reserveIpr); iprl != nil { + newAvailableList = append(newAvailableList, iprl...) + } + } + subnet.V6AvailIPList = newAvailableList } } } diff --git a/test/e2e/kube-ovn/subnet/subnet.go b/test/e2e/kube-ovn/subnet/subnet.go index f995e144809..4e01fe83406 100644 --- a/test/e2e/kube-ovn/subnet/subnet.go +++ b/test/e2e/kube-ovn/subnet/subnet.go @@ -1,17 +1,22 @@ package subnet import ( + "context" "fmt" + "math/big" "math/rand" "net" + "os/exec" "strconv" "strings" + "time" + "github.com/onsi/ginkgo/v2" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework/deployment" e2enode "k8s.io/kubernetes/test/e2e/framework/node" - "github.com/onsi/ginkgo/v2" - apiv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" "github.com/kubeovn/kube-ovn/pkg/util" "github.com/kubeovn/kube-ovn/test/e2e/framework" @@ -28,8 +33,10 @@ var _ = framework.Describe("[group:subnet]", func() { var podClient *framework.PodClient var subnetClient *framework.SubnetClient var namespaceName, subnetName string - var cidr, cidrV4, cidrV6, firstIPv4, firstIPv6 string + var cidr, cidrV4, cidrV6, firstIPv4, lastIPv4, firstIPv6, lastIPv6 string var gateways []string + var podCount int + var podNamePre, deployName string ginkgo.BeforeEach(func() { cs = f.ClientSet @@ -40,20 +47,40 @@ var _ = framework.Describe("[group:subnet]", func() { cidr = framework.RandomCIDR(f.ClusterIpFamily) cidrV4, cidrV6 = util.SplitStringIP(cidr) gateways = nil + podCount = 0 + podNamePre = "" + deployName = "" if cidrV4 == "" { firstIPv4 = "" + lastIPv4 = "" } else { firstIPv4, _ = util.FirstIP(cidrV4) + lastIPv4, _ = util.LastIP(cidrV4) gateways = append(gateways, firstIPv4) } if cidrV6 == "" { firstIPv6 = "" + lastIPv6 = "" } else { firstIPv6, _ = util.FirstIP(cidrV6) + lastIPv6, _ = util.LastIP(cidrV6) gateways = append(gateways, firstIPv6) } }) ginkgo.AfterEach(func() { + ginkgo.By("Deleting deployment ") + if deployName != "" { + err := cs.AppsV1().Deployments(namespaceName).Delete(context.TODO(), deployName, metav1.DeleteOptions{}) + fmt.Printf("delete deployment %s failed with err %v", deployName, err) + } + + ginkgo.By("Deleting pods ") + if podCount != 0 { + for i := 1; i <= podCount; i++ { + podClient.DeleteSync(fmt.Sprintf("%s%d", podNamePre, i)) + } + } + ginkgo.By("Deleting subnet " + subnetName) subnetClient.DeleteSync(subnetName) }) @@ -522,4 +549,286 @@ var _ = framework.Describe("[group:subnet]", func() { framework.ExpectConsistOf(nexthops, gateways) } }) + + framework.ConformanceIt("should support subnet AvailableIPRange and UsingIPRange creating pod no specify ip", func() { + f.SkipVersionPriorTo(1, 12, "Support for display AvailableIPRange and UsingIPRange in v1.12") + podCount = 5 + podNamePre = "sample" + var startIPv4, startIPv6 string + if firstIPv4 != "" { + startIPv4 = util.BigInt2Ip(big.NewInt(0).Add(util.Ip2BigInt(firstIPv4), big.NewInt(1))) + } + if firstIPv6 != "" { + startIPv6 = util.BigInt2Ip(big.NewInt(0).Add(util.Ip2BigInt(firstIPv6), big.NewInt(1))) + } + + ginkgo.By("Creating subnet " + subnetName) + subnet = framework.MakeSubnet(subnetName, "", cidr, "", nil, nil, nil) + subnet = subnetClient.CreateSync(subnet) + + ginkgo.By("Creating pod with no specify pod ip ") + annotations := map[string]string{ + util.LogicalSwitchAnnotation: subnetName, + } + for i := 1; i <= podCount; i++ { + pod := framework.MakePod("", fmt.Sprintf("%s%d", podNamePre, i), nil, annotations, "", nil, nil) + podClient.CreateSync(pod) + } + + subnet = subnetClient.Get(subnetName) + if cidrV4 != "" { + v4UsingIPEnd := util.BigInt2Ip(big.NewInt(0).Add(util.Ip2BigInt(startIPv4), big.NewInt(int64(podCount-1)))) + v4AvailableIPStart := util.BigInt2Ip(big.NewInt(0).Add(util.Ip2BigInt(v4UsingIPEnd), big.NewInt(1))) + framework.ExpectEqual(subnet.Status.V4UsingIPRange, fmt.Sprintf("%s-%s", startIPv4, v4UsingIPEnd)) + framework.ExpectEqual(subnet.Status.V4AvailableIPRange, fmt.Sprintf("%s-%s", v4AvailableIPStart, lastIPv4)) + } + + if cidrV6 != "" { + v6UsingIPEnd := util.BigInt2Ip(big.NewInt(0).Add(util.Ip2BigInt(startIPv6), big.NewInt(int64(podCount-1)))) + v6AvailableIPStart := util.BigInt2Ip(big.NewInt(0).Add(util.Ip2BigInt(v6UsingIPEnd), big.NewInt(1))) + framework.ExpectEqual(subnet.Status.V6UsingIPRange, fmt.Sprintf("%s-%s", startIPv6, v6UsingIPEnd)) + framework.ExpectEqual(subnet.Status.V6AvailableIPRange, fmt.Sprintf("%s-%s", v6AvailableIPStart, lastIPv6)) + } + + ginkgo.By("delete all pods") + for i := 1; i <= podCount; i++ { + podClient.DeleteSync(fmt.Sprintf("%s%d", podNamePre, i)) + } + + subnet = subnetClient.Get(subnetName) + if cidrV4 != "" { + framework.ExpectEqual(subnet.Status.V4UsingIPRange, "") + framework.ExpectEqual(subnet.Status.V4AvailableIPRange, fmt.Sprintf("%s-%s", startIPv4, lastIPv4)) + } + + if cidrV6 != "" { + framework.ExpectEqual(subnet.Status.V6UsingIPRange, "") + framework.ExpectEqual(subnet.Status.V6AvailableIPRange, fmt.Sprintf("%s-%s", startIPv6, lastIPv6)) + } + }) + + framework.ConformanceIt("should support subnet AvailableIPRange and UsingIPRange creating pod specify ip", func() { + f.SkipVersionPriorTo(1, 12, "Support for display AvailableIPRange and UsingIPRange in v1.12") + podCount = 5 + podNamePre = "sample" + var startIPv4, startIPv6, usingIPv4Str, availableIPv4Str, usingIPv6Str, availableIPv6Str string + + if firstIPv4 != "" { + startIPv4 = util.BigInt2Ip(big.NewInt(0).Add(util.Ip2BigInt(firstIPv4), big.NewInt(1))) + } + if firstIPv6 != "" { + startIPv6 = util.BigInt2Ip(big.NewInt(0).Add(util.Ip2BigInt(firstIPv6), big.NewInt(1))) + } + + ginkgo.By("Creating subnet " + subnetName) + subnet = framework.MakeSubnet(subnetName, "", cidr, "", nil, nil, nil) + subnet = subnetClient.CreateSync(subnet) + + ginkgo.By("Creating pod with specify pod ip ") + podIPv4s, podIPv6s := createPodsByRandomIPs(podClient, subnetClient, subnetName, podNamePre, podCount, startIPv4, startIPv6) + subnet = subnetClient.Get(subnetName) + + if podIPv4s != nil { + usingIPv4Str, availableIPv4Str = calcuIPRangeListStr(podIPv4s, startIPv4, lastIPv4) + framework.ExpectEqual(subnet.Status.V4UsingIPRange, usingIPv4Str) + framework.ExpectEqual(subnet.Status.V4AvailableIPRange, availableIPv4Str) + } + + if podIPv6s != nil { + usingIPv6Str, availableIPv6Str = calcuIPRangeListStr(podIPv6s, startIPv6, lastIPv6) + framework.ExpectEqual(subnet.Status.V6UsingIPRange, usingIPv6Str) + framework.ExpectEqual(subnet.Status.V6AvailableIPRange, availableIPv6Str) + } + + ginkgo.By("delete all pods") + for i := 1; i <= podCount; i++ { + podClient.DeleteSync(fmt.Sprintf("%s%d", podNamePre, i)) + } + + subnet = subnetClient.Get(subnetName) + if cidrV4 != "" { + framework.ExpectEqual(subnet.Status.V4UsingIPRange, "") + framework.ExpectEqual(subnet.Status.V4AvailableIPRange, fmt.Sprintf("%s-%s", startIPv4, lastIPv4)) + } + + if cidrV6 != "" { + framework.ExpectEqual(subnet.Status.V6UsingIPRange, "") + framework.ExpectEqual(subnet.Status.V6AvailableIPRange, fmt.Sprintf("%s-%s", startIPv6, lastIPv6)) + } + }) + + framework.ConformanceIt("should support subnet AvailableIPRange and UsingIPRange is correct when restart deployment", func() { + f.SkipVersionPriorTo(1, 12, "Support for display AvailableIPRange and UsingIPRange in v1.12") + + var startIPv4, startIPv6 string + if firstIPv4 != "" { + startIPv4 = util.BigInt2Ip(big.NewInt(0).Add(util.Ip2BigInt(firstIPv4), big.NewInt(1))) + } + if firstIPv6 != "" { + startIPv6 = util.BigInt2Ip(big.NewInt(0).Add(util.Ip2BigInt(firstIPv6), big.NewInt(1))) + } + + ginkgo.By("Creating subnet " + subnetName) + subnet = framework.MakeSubnet(subnetName, "", cidr, "", nil, nil, nil) + subnet = subnetClient.CreateSync(subnet) + + replicas := int64(5) + deployName = "deployment-" + framework.RandomSuffix() + labels := map[string]string{"app": deployName} + deploy := deployment.NewDeployment(deployName, int32(replicas), labels, "pause", framework.PauseImage, "") + deploy.Spec.Template.Annotations = map[string]string{util.LogicalSwitchAnnotation: subnetName} + deploy, err := cs.AppsV1().Deployments(namespaceName).Create(context.TODO(), deploy, metav1.CreateOptions{}) + framework.ExpectNoError(err, "failed to to create deployment") + err = deployment.WaitForDeploymentComplete(cs, deploy) + framework.ExpectNoError(err, "deployment failed to complete") + + checkFunc := func(usingIPRange, availableIPRange, startIP, lastIP string, count int64) { + usingIPEnd := util.BigInt2Ip(big.NewInt(0).Add(util.Ip2BigInt(startIP), big.NewInt(count-1))) + availableIPStart := util.BigInt2Ip(big.NewInt(0).Add(util.Ip2BigInt(usingIPEnd), big.NewInt(1))) + framework.ExpectEqual(usingIPRange, fmt.Sprintf("%s-%s", startIP, usingIPEnd)) + framework.ExpectEqual(availableIPRange, fmt.Sprintf("%s-%s", availableIPStart, lastIP)) + } + + subnet = subnetClient.Get(subnetName) + if cidrV4 != "" { + checkFunc(subnet.Status.V4UsingIPRange, subnet.Status.V4AvailableIPRange, startIPv4, lastIPv4, replicas) + } + + if cidrV6 != "" { + checkFunc(subnet.Status.V6UsingIPRange, subnet.Status.V6AvailableIPRange, startIPv6, lastIPv6, replicas) + } + + ginkgo.By("restart deployment ") + restartCmd := fmt.Sprintf("kubectl rollout restart deployment %s -n %s", deployName, namespaceName) + _, err = exec.Command("bash", "-c", restartCmd).CombinedOutput() + framework.ExpectNoError(err, "restart deployment failed") + err = deployment.WaitForDeploymentComplete(cs, deploy) + framework.ExpectNoError(err, "deployment failed to complete") + + checkFunc2 := func(usingIPRange, availableIPRange, startIP, lastIP string, count int64, isFrameworkCheck bool) bool { + expectAvailIPRangeStr := fmt.Sprintf("%s-%s,%s-%s", + startIP, + util.BigInt2Ip(big.NewInt(0).Add(util.Ip2BigInt(startIP), big.NewInt(count-1))), + util.BigInt2Ip(big.NewInt(0).Add(util.Ip2BigInt(startIP), big.NewInt(2*count))), + lastIP, + ) + + expectUsingIPRangeStr := fmt.Sprintf("%s-%s", + util.BigInt2Ip(big.NewInt(0).Add(util.Ip2BigInt(startIP), big.NewInt(count))), + util.BigInt2Ip(big.NewInt(0).Add(util.Ip2BigInt(startIP), big.NewInt(2*count-1))), + ) + + fmt.Printf("subnet status usingIPRange %s expect %s \n", usingIPRange, expectUsingIPRangeStr) + fmt.Printf("subnet status availableIPRange %s expect %s \n", availableIPRange, expectAvailIPRangeStr) + + if isFrameworkCheck { + framework.ExpectEqual(usingIPRange, expectUsingIPRangeStr) + framework.ExpectEqual(availableIPRange, expectAvailIPRangeStr) + return false + } else { + return usingIPRange == expectUsingIPRangeStr && availableIPRange == expectAvailIPRangeStr + } + } + + checkTimes := 0 + isSuccess := false + maxRetryTimes := 30 + for { + time.Sleep(1 * time.Second) + if checkTimes > maxRetryTimes || isSuccess { + break + } + subnet = subnetClient.Get(subnetName) + if cidrV4 != "" { + isSuccess = checkFunc2(subnet.Status.V4UsingIPRange, subnet.Status.V4AvailableIPRange, startIPv4, lastIPv4, replicas, false) + } + + if cidrV6 != "" { + isSuccess = checkFunc2(subnet.Status.V6UsingIPRange, subnet.Status.V6AvailableIPRange, startIPv6, lastIPv6, replicas, false) + } + checkTimes++ + } + + if cidrV4 != "" { + checkFunc2(subnet.Status.V4UsingIPRange, subnet.Status.V4AvailableIPRange, startIPv4, lastIPv4, replicas, true) + } + + if cidrV6 != "" { + checkFunc2(subnet.Status.V6UsingIPRange, subnet.Status.V6AvailableIPRange, startIPv6, lastIPv6, replicas, true) + } + }) }) + +func createPodsByRandomIPs(podClient *framework.PodClient, subnetClient *framework.SubnetClient, subnetName, podNamePre string, podCount int, startIPv4, startIPv6 string) ([]string, []string) { + var allocIP string + var podIPv4s, podIPv6s []string + podv4IP := startIPv4 + podv6IP := startIPv6 + + subnet := subnetClient.Get(subnetName) + for i := 1; i <= podCount; i++ { + step := rand.Int63()%10 + 2 + if subnet.Spec.Protocol == apiv1.ProtocolIPv4 { + podv4IP = util.BigInt2Ip(big.NewInt(0).Add(util.Ip2BigInt(podv4IP), big.NewInt(step))) + allocIP = podv4IP + } else if subnet.Spec.Protocol == apiv1.ProtocolIPv6 { + podv6IP = util.BigInt2Ip(big.NewInt(0).Add(util.Ip2BigInt(podv6IP), big.NewInt(step))) + allocIP = podv6IP + } else { + podv4IP = util.BigInt2Ip(big.NewInt(0).Add(util.Ip2BigInt(podv4IP), big.NewInt(step))) + podv6IP = util.BigInt2Ip(big.NewInt(0).Add(util.Ip2BigInt(podv6IP), big.NewInt(step))) + allocIP = fmt.Sprintf("%s,%s", podv4IP, podv6IP) + } + + annotations := map[string]string{ + util.LogicalSwitchAnnotation: subnetName, + fmt.Sprintf(util.IpAddressAnnotationTemplate, subnet.Spec.Provider): allocIP, + } + + podName := fmt.Sprintf("%s%d", podNamePre, i) + pod := framework.MakePod("", podName, nil, annotations, "", nil, nil) + podClient.CreateSync(pod) + + if podv4IP != "" { + podIPv4s = append(podIPv4s, podv4IP) + } + if podv6IP != "" { + podIPv6s = append(podIPv6s, podv6IP) + } + } + + return podIPv4s, podIPv6s +} + +func calcuIPRangeListStr(podIPs []string, startIP, lastIP string) (string, string) { + var usingIPs, availableIPs []string + var usingIPStr, availableIPStr, prePodIP string + + for index, podIP := range podIPs { + usingIPs = append(usingIPs, podIP) + if index == 0 { + availableIPs = append(availableIPs, fmt.Sprintf("%s-%s", startIP, util.BigInt2Ip(big.NewInt(0).Add(util.Ip2BigInt(podIP), big.NewInt(-1))))) + } else { + preIP := prePodIP + start := util.BigInt2Ip(big.NewInt(0).Add(util.Ip2BigInt(preIP), big.NewInt(1))) + end := util.BigInt2Ip(big.NewInt(0).Add(util.Ip2BigInt(podIP), big.NewInt(-1))) + + if start == end { + availableIPs = append(availableIPs, start) + } else { + availableIPs = append(availableIPs, fmt.Sprintf("%s-%s", start, end)) + } + } + prePodIP = podIP + } + + if prePodIP != "" { + availableIPs = append(availableIPs, fmt.Sprintf("%s-%s", util.BigInt2Ip(big.NewInt(0).Add(util.Ip2BigInt(prePodIP), big.NewInt(1))), lastIP)) + } + + usingIPStr = strings.Join(usingIPs, ",") + availableIPStr = strings.Join(availableIPs, ",") + fmt.Printf("usingIPStr is %s ", usingIPStr) + fmt.Printf("availableIPStr is %s ", availableIPStr) + return usingIPStr, availableIPStr +} diff --git a/yamls/crd.yaml b/yamls/crd.yaml index 78cc618d46f..3735dd1a57f 100644 --- a/yamls/crd.yaml +++ b/yamls/crd.yaml @@ -731,6 +731,14 @@ spec: type: string u2oInterconnectionIP: type: string + v4usingIPrange: + type: string + v4availableIPrange: + type: string + v6usingIPrange: + type: string + v6availableIPrange: + type: string conditions: type: array items: