diff --git a/Makefile.e2e b/Makefile.e2e index 5e05340ea64..b75c402d7fe 100644 --- a/Makefile.e2e +++ b/Makefile.e2e @@ -110,7 +110,7 @@ kube-ovn-conformance-e2e: E2E_BRANCH=$(E2E_BRANCH) \ E2E_IP_FAMILY=$(E2E_IP_FAMILY) \ E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \ - ginkgo $(GINKGO_PARALLEL_OPT) --randomize-all -v \ + ginkgo $(GINKGO_PARALLEL_OPT) --randomize-all -v --timeout=25m \ --focus=CNI:Kube-OVN ./test/e2e/kube-ovn/kube-ovn.test .PHONY: kube-ovn-ic-conformance-e2e diff --git a/pkg/controller/subnet.go b/pkg/controller/subnet.go index 80af215d5d7..5c43cdf96ff 100644 --- a/pkg/controller/subnet.go +++ b/pkg/controller/subnet.go @@ -823,6 +823,19 @@ func (c *Controller) handleAddOrUpdateSubnet(key string) error { } c.updateVpcStatusQueue.Add(subnet.Spec.Vpc) + + ippools, err := c.ippoolLister.List(labels.Everything()) + if err != nil { + klog.Errorf("failed to list ippools: %v", err) + return err + } + + for _, p := range ippools { + if p.Spec.Subnet == subnet.Name { + c.addOrUpdateIPPoolQueue.Add(p.Name) + } + } + return nil } diff --git a/pkg/ipam/ip.go b/pkg/ipam/ip.go index bef5b48bc67..7de50ff8464 100644 --- a/pkg/ipam/ip.go +++ b/pkg/ipam/ip.go @@ -22,6 +22,12 @@ func NewIP(s string) (IP, error) { return IP(ip), nil } +func (a IP) Clone() IP { + v := make(IP, len(a)) + copy(v, a) + return v +} + func (a IP) To4() net.IP { return net.IP(a).To4() } @@ -42,28 +48,23 @@ func (a IP) GreaterThan(b IP) bool { return big.NewInt(0).SetBytes([]byte(a)).Cmp(big.NewInt(0).SetBytes([]byte(b))) > 0 } -func (a IP) Add(n int64) IP { - buff := big.NewInt(0).Add(big.NewInt(0).SetBytes([]byte(a)), big.NewInt(n)).Bytes() - if len(buff) < len(a) { - tmp := make([]byte, len(a)) +func bytes2IP(buff []byte, length int) IP { + if len(buff) < length { + tmp := make([]byte, length) copy(tmp[len(tmp)-len(buff):], buff) buff = tmp - } else if len(buff) > len(a) { - buff = buff[len(buff)-len(a):] + } else if len(buff) > length { + buff = buff[len(buff)-length:] } return IP(buff) } +func (a IP) Add(n int64) IP { + return bytes2IP(big.NewInt(0).Add(big.NewInt(0).SetBytes([]byte(a)), big.NewInt(n)).Bytes(), len(a)) +} + func (a IP) Sub(n int64) IP { - buff := big.NewInt(0).Sub(big.NewInt(0).SetBytes([]byte(a)), big.NewInt(n)).Bytes() - if len(buff) < len(a) { - tmp := make([]byte, len(a)) - copy(tmp[len(tmp)-len(buff):], buff) - buff = tmp - } else if len(buff) > len(a) { - buff = buff[len(buff)-len(a):] - } - return IP(buff) + return bytes2IP(big.NewInt(0).Sub(big.NewInt(0).SetBytes([]byte(a)), big.NewInt(n)).Bytes(), len(a)) } func (a IP) String() string { diff --git a/pkg/ipam/ip_range.go b/pkg/ipam/ip_range.go index fea58d9857d..8e221ff4fb3 100644 --- a/pkg/ipam/ip_range.go +++ b/pkg/ipam/ip_range.go @@ -1,8 +1,10 @@ package ipam import ( + "crypto/rand" "fmt" "math/big" + "net" "github.com/kubeovn/kube-ovn/pkg/internal" ) @@ -16,8 +18,18 @@ func NewIPRange(start, end IP) *IPRange { return &IPRange{start, end} } +func NewIPRangeFromCIDR(cidr net.IPNet) *IPRange { + start, _ := NewIP(cidr.IP.Mask(cidr.Mask).String()) + end := make(IP, len(start)) + for i := 0; i < len(end); i++ { + end[i] = start[i] | ^cidr.Mask[i] + } + + return &IPRange{start, end} +} + func (r *IPRange) Clone() *IPRange { - return NewIPRange(r.start, r.end) + return NewIPRange(r.start.Clone(), r.end.Clone()) } func (r *IPRange) Start() IP { @@ -41,6 +53,13 @@ func (r *IPRange) Count() internal.BigInt { return internal.BigInt{Int: *n.Add(n, big.NewInt(1))} } +func (r *IPRange) Random() IP { + x := big.NewInt(0).SetBytes([]byte(r.start)) + y := big.NewInt(0).SetBytes([]byte(r.end)) + n, _ := rand.Int(rand.Reader, big.NewInt(0).Add(big.NewInt(0).Sub(y, x), big.NewInt(1))) + return bytes2IP(big.NewInt(0).Add(x, n).Bytes(), len(r.start)) +} + func (r *IPRange) Contains(ip IP) bool { return !r.start.GreaterThan(ip) && !r.end.LessThan(ip) } diff --git a/pkg/ipam/ip_range_list.go b/pkg/ipam/ip_range_list.go index e4fe2a17151..adec935e6a5 100644 --- a/pkg/ipam/ip_range_list.go +++ b/pkg/ipam/ip_range_list.go @@ -2,6 +2,7 @@ package ipam import ( "fmt" + "net" "sort" "strings" @@ -31,21 +32,32 @@ func NewIPRangeList(ips ...IP) (*IPRangeList, error) { func NewIPRangeListFrom(x ...string) (*IPRangeList, error) { ret := &IPRangeList{} for _, s := range x { - ips := strings.Split(s, "..") - start, err := NewIP(ips[0]) - if err != nil { - return nil, err - } var r *IPRange - if len(ips) == 1 { - r = NewIPRange(start, start) - } else { + if strings.Contains(s, "..") { + ips := strings.Split(s, "..") + start, err := NewIP(ips[0]) + if err != nil { + return nil, err + } end, err := NewIP(ips[1]) if err != nil { return nil, err } r = NewIPRange(start, end) + } else if strings.ContainsRune(s, '/') { + _, cidr, err := net.ParseCIDR(s) + if err != nil { + return nil, err + } + r = NewIPRangeFromCIDR(*cidr) + } else { + start, err := NewIP(s) + if err != nil { + return nil, err + } + r = NewIPRange(start, start.Clone()) } + ret = ret.Merge(&IPRangeList{[]*IPRange{r}}) } return ret, nil @@ -256,6 +268,10 @@ func (r *IPRangeList) Merge(x *IPRangeList) *IPRangeList { return ret.Clone() } +func (r *IPRangeList) MergeRange(x *IPRange) *IPRangeList { + return r.Merge(&IPRangeList{ranges: []*IPRange{x}}).Clone() +} + // Intersect returns a new list which contains items which are in both `r` and `x` func (r *IPRangeList) Intersect(x *IPRangeList) *IPRangeList { r1, r2 := r.Separate(x), x.Separate(r) diff --git a/pkg/util/net.go b/pkg/util/net.go index 78d50b56464..27d0fa53e9c 100644 --- a/pkg/util/net.go +++ b/pkg/util/net.go @@ -274,7 +274,7 @@ func SplitIpsByProtocol(excludeIps []string) ([]string, []string) { var v4ExcludeIps, v6ExcludeIps []string for _, ex := range excludeIps { ips := strings.Split(ex, "..") - if net.ParseIP(ips[0]).To4() != nil { + if CheckProtocol(ips[0]) == kubeovnv1.ProtocolIPv4 { v4ExcludeIps = append(v4ExcludeIps, ex) } else { v6ExcludeIps = append(v6ExcludeIps, ex) diff --git a/test/e2e/framework/expect.go b/test/e2e/framework/expect.go index 633e0d22c3b..7e72c60b5f4 100644 --- a/test/e2e/framework/expect.go +++ b/test/e2e/framework/expect.go @@ -98,6 +98,16 @@ func ExpectNotContainElement(actual interface{}, extra interface{}, explain ...i gomega.ExpectWithOffset(1, actual).NotTo(gomega.ContainElement(extra), explain...) } +// ExpectContainSubstring expects actual contains the passed-in substring. +func ExpectContainSubstring(actual, substr string, explain ...interface{}) { + gomega.ExpectWithOffset(1, actual).To(gomega.ContainSubstring(substr), explain...) +} + +// ExpectNotContainSubstring expects actual does not contain the passed-in substring. +func ExpectNotContainSubstring(actual, substr string, explain ...interface{}) { + gomega.ExpectWithOffset(1, actual).NotTo(gomega.ContainSubstring(substr), explain...) +} + // ExpectHaveKey expects the actual map has the key in the keyset func ExpectHaveKey(actual interface{}, key interface{}, explain ...interface{}) { gomega.ExpectWithOffset(1, actual).To(gomega.HaveKey(key), explain...) diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index b658f0d719b..52357e197ea 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -16,6 +16,12 @@ import ( kubeovncs "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned" ) +const ( + IPv4 = "ipv4" + IPv6 = "ipv6" + Dual = "dual" +) + const ( // poll is how often to Poll resources. poll = 2 * time.Second @@ -103,8 +109,24 @@ func NewFrameworkWithContext(baseName, kubeContext string) *Framework { return f } -func (f *Framework) IPv6() bool { - return f.ClusterIpFamily == "ipv6" +func (f *Framework) IsIPv4() bool { + return f.ClusterIpFamily == IPv4 +} + +func (f *Framework) IsIPv6() bool { + return f.ClusterIpFamily == IPv6 +} + +func (f *Framework) IsDual() bool { + return f.ClusterIpFamily == Dual +} + +func (f *Framework) HasIPv4() bool { + return !f.IsIPv6() +} + +func (f *Framework) HasIPv6() bool { + return !f.IsIPv4() } // BeforeEach gets a kube-ovn client diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index 75bf75343d9..4cc40ab9d91 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -2,16 +2,15 @@ package framework import ( "fmt" - "math/big" "math/rand" "net" "sort" "strings" - "time" "github.com/onsi/ginkgo/v2" "github.com/scylladb/go-set/strset" + "github.com/kubeovn/kube-ovn/pkg/ipam" "github.com/kubeovn/kube-ovn/pkg/util" ) @@ -48,11 +47,11 @@ func RandomCIDR(family string) string { } switch family { - case "ipv4": + case IPv4: return fnIPv4() - case "ipv6": + case IPv6: return fnIPv6() - case "dual": + case Dual: return fnIPv4() + "," + fnIPv6() default: Failf("invalid ip family: %q", family) @@ -62,7 +61,11 @@ func RandomCIDR(family string) string { func sortIPs(ips []string) { sort.Slice(ips, func(i, j int) bool { - return util.Ip2BigInt(ips[i]).Cmp(util.Ip2BigInt(ips[j])) < 0 + x, err := ipam.NewIP(ips[i]) + ExpectNoError(err) + y, err := ipam.NewIP(ips[j]) + ExpectNoError(err) + return x.LessThan(y) }) } @@ -72,9 +75,11 @@ func RandomExcludeIPs(cidr string, count int) []string { return nil } + ExpectNotContainSubstring(cidr, ",") + ExpectNotContainSubstring(cidr, ";") + rangeCount := rand.Intn(count + 1) - ips := strings.Split(RandomIPPool(cidr, ";", rangeCount*2+count-rangeCount), ";") - sortIPs(ips) + ips := randomSortedIPs(cidr, rangeCount*2+count-rangeCount, true) var idx int rangeLeft := rangeCount @@ -93,36 +98,38 @@ func RandomExcludeIPs(cidr string, count int) []string { return ret } -func RandomIPPool(cidr, sep string, count int) string { - fn := func(cidr string) []string { - if cidr == "" { - return nil - } +// ipv4/ipv6 only +func randomSortedIPs(cidr string, count int, sort bool) []string { + if cidr == "" { + return nil + } - firstIP, _ := util.FirstIP(cidr) - _, ipnet, _ := net.ParseCIDR(cidr) + _, ipNet, err := net.ParseCIDR(cidr) + ExpectNoError(err) - base := util.Ip2BigInt(firstIP) - base.Add(base, big.NewInt(1)) - prefix, size := ipnet.Mask.Size() - rnd := rand.New(rand.NewSource(time.Now().UnixNano())) - max := big.NewInt(0).Exp(big.NewInt(2), big.NewInt(int64(size-prefix)), nil) - max.Sub(max, big.NewInt(3)) + set := strset.NewWithSize(count) + r := ipam.NewIPRangeFromCIDR(*ipNet) + r = ipam.NewIPRange(r.Start().Add(2), r.End().Sub(1)) + for set.Size() != count { + set.Add(r.Random().String()) + } - ips := strset.NewWithSize(count) - for ips.Size() != count { - n := big.NewInt(0).Rand(rnd, max) - ips.Add(util.BigInt2Ip(n.Add(n, base))) - } - return ips.List() + ips := set.List() + if sort { + sortIPs(ips) } + return ips +} + +func RandomIPs(cidr, sep string, count int) string { cidrV4, cidrV6 := util.SplitStringIP(cidr) - ipsV4, ipsV6 := fn(cidrV4), fn(cidrV6) + ipsV4 := randomSortedIPs(cidrV4, count, false) + ipsV6 := randomSortedIPs(cidrV6, count, false) dual := make([]string, 0, count) for i := 0; i < count; i++ { - var ips []string + ips := make([]string, 0, 2) if i < len(ipsV4) { ips = append(ips, ipsV4[i]) } @@ -135,12 +142,96 @@ func RandomIPPool(cidr, sep string, count int) string { return strings.Join(dual, sep) } +// ipv4/ipv6 only +func randomPool(cidr string, count int) []string { + if cidr == "" || count == 0 { + return nil + } + + _, ipNet, err := net.ParseCIDR(cidr) + ExpectNoError(err) + + r := ipam.NewIPRangeFromCIDR(*ipNet) + r = ipam.NewIPRange(r.Start().Add(2), r.End().Sub(1)) + + ones, bits := ipNet.Mask.Size() + rl := ipam.NewEmptyIPRangeList() + set := strset.NewWithSize(count) + for set.Size() != count/4 { + prefix := (ones+bits)/2 + rand.Intn((bits-ones)/2+1) + _, ipNet, err = net.ParseCIDR(fmt.Sprintf("%s/%d", r.Random(), prefix)) + ExpectNoError(err) + + v := ipam.NewIPRangeFromCIDR(*ipNet) + start, end := v.Start(), v.End() + if !r.Contains(start) || !r.Contains(end) || rl.Contains(start) || rl.Contains(end) { + continue + } + + rl = rl.MergeRange(ipam.NewIPRange(start, end)) + set.Add(ipNet.String()) + } + + count -= set.Size() + m := count / 3 // .. + n := count - m // + ips := make([]ipam.IP, 0, m*2+n) + ipSet := strset.NewWithSize(cap(ips)) + for len(ips) != cap(ips) { + ip := r.Random() + if rl.Contains(ip) { + continue + } + + s := ip.String() + if ipSet.Has(s) { + continue + } + ips = append(ips, ip) + ipSet.Add(s) + } + sort.Slice(ips, func(i, j int) bool { return ips[i].LessThan(ips[j]) }) + + var i, j int + k := rand.Intn(len(ips)) + for i != m || j != n { + if i != m { + x, y := k%len(ips), (k+1)%len(ips) + n1, _ := rl.Find(ips[x]) + n2, _ := rl.Find(ips[y]) + if n1 == n2 { + set.Add(fmt.Sprintf("%s..%s", ips[x].String(), ips[y].String())) + i, k = i+1, k+2 + continue + } + } + + if j != n { + set.Add(ips[k%len(ips)].String()) + j, k = j+1, k+1 + } + } + + return set.List() +} + +func RandomIPPool(cidr string, count int) []string { + cidrV4, cidrV6 := util.SplitStringIP(cidr) + ipsV4, ipsV6 := randomPool(cidrV4, count), randomPool(cidrV6, count) + set := strset.NewWithSize(len(cidrV4) + len(cidrV6)) + set.Add(ipsV4...) + set.Add(ipsV6...) + return set.List() +} + func PrevIP(ip string) string { - n := util.Ip2BigInt(ip) - return util.BigInt2Ip(n.Add(n, big.NewInt(-1))) + v, err := ipam.NewIP(ip) + ExpectNoError(err) + return v.Sub(1).String() } func NextIP(ip string) string { - n := util.Ip2BigInt(ip) - return util.BigInt2Ip(n.Add(n, big.NewInt(1))) + v, err := ipam.NewIP(ip) + ExpectNoError(err) + return v.Add(1).String() } diff --git a/test/e2e/iptables-vpc-nat-gw/e2e_test.go b/test/e2e/iptables-vpc-nat-gw/e2e_test.go index 75bf8c70fa0..91e3a11f799 100644 --- a/test/e2e/iptables-vpc-nat-gw/e2e_test.go +++ b/test/e2e/iptables-vpc-nat-gw/e2e_test.go @@ -103,12 +103,12 @@ func setupNetworkAttachmentDefinition( for _, config := range dockerExtNetNetwork.IPAM.Config { switch util.CheckProtocol(config.Subnet) { case apiv1.ProtocolIPv4: - if f.ClusterIpFamily != "ipv6" { + if f.HasIPv4() { cidr = append(cidr, config.Subnet) gateway = append(gateway, config.Gateway) } case apiv1.ProtocolIPv6: - if f.ClusterIpFamily != "ipv4" { + if f.HasIPv6() { cidr = append(cidr, config.Subnet) gateway = append(gateway, config.Gateway) } @@ -116,10 +116,10 @@ func setupNetworkAttachmentDefinition( } excludeIPs := make([]string, 0, len(network.Containers)*2) for _, container := range network.Containers { - if container.IPv4Address != "" && f.ClusterIpFamily != "ipv6" { + if container.IPv4Address != "" && f.HasIPv4() { excludeIPs = append(excludeIPs, strings.Split(container.IPv4Address, "/")[0]) } - if container.IPv6Address != "" && f.ClusterIpFamily != "ipv4" { + if container.IPv6Address != "" && f.HasIPv6() { excludeIPs = append(excludeIPs, strings.Split(container.IPv6Address, "/")[0]) } } diff --git a/test/e2e/kube-ovn/ipam/ipam.go b/test/e2e/kube-ovn/ipam/ipam.go index 6d8e5f49b4e..13c2753c83e 100644 --- a/test/e2e/kube-ovn/ipam/ipam.go +++ b/test/e2e/kube-ovn/ipam/ipam.go @@ -3,7 +3,6 @@ package ipam import ( "context" "fmt" - "sort" "strings" "time" @@ -71,7 +70,7 @@ var _ = framework.Describe("[group:ipam]", func() { framework.ConformanceIt("should allocate static ipv4 and mac for pod", func() { mac := util.GenerateMac() - ip := framework.RandomIPPool(cidr, ";", 1) + ip := framework.RandomIPs(cidr, ";", 1) ginkgo.By("Creating pod " + podName + " with ip " + ip + " and mac " + mac) annotations := map[string]string{ @@ -97,11 +96,11 @@ var _ = framework.Describe("[group:ipam]", func() { }) framework.ConformanceIt("should allocate static ip for pod with comma separated ippool", func() { - if f.ClusterIpFamily == "dual" { + if f.IsDual() { ginkgo.Skip("Comma separated ippool is not supported for dual stack") } - pool := framework.RandomIPPool(cidr, ",", 3) + pool := framework.RandomIPs(cidr, ",", 3) ginkgo.By("Creating pod " + podName + " with ippool " + pool) annotations := map[string]string{util.IpPoolAnnotation: pool} pod := framework.MakePod(namespaceName, podName, nil, annotations, "", nil, nil) @@ -120,16 +119,15 @@ var _ = framework.Describe("[group:ipam]", func() { framework.ConformanceIt("should allocate static ip for deployment with ippool", func() { ippoolSep := ";" - if f.ClusterVersionMajor < 1 || - (f.ClusterVersionMajor == 1 && f.ClusterVersionMinor < 11) { - if f.ClusterIpFamily == "dual" { + if f.VersionPriorTo(1, 11) { + if f.IsDual() { ginkgo.Skip("Support for dual stack ippool was introduced in v1.11") } ippoolSep = "," } replicas := 3 - ippool := framework.RandomIPPool(cidr, ippoolSep, replicas) + ippool := framework.RandomIPs(cidr, ippoolSep, replicas) ginkgo.By("Creating deployment " + deployName + " with ippool " + ippool) labels := map[string]string{"app": deployName} @@ -247,15 +245,15 @@ var _ = framework.Describe("[group:ipam]", func() { framework.ConformanceIt("should allocate static ip for statefulset with ippool", func() { ippoolSep := ";" - if f.ClusterVersionMajor <= 1 && f.ClusterVersionMinor < 11 { - if f.ClusterIpFamily == "dual" { + if f.VersionPriorTo(1, 11) { + if f.IsDual() { ginkgo.Skip("Support for dual stack ippool was introduced in v1.11") } ippoolSep = "," } replicas := 3 - ippool := framework.RandomIPPool(cidr, ippoolSep, replicas) + ippool := framework.RandomIPs(cidr, ippoolSep, replicas) labels := map[string]string{"app": stsName} ginkgo.By("Creating statefulset " + stsName + " with ippool " + ippool) @@ -317,13 +315,13 @@ var _ = framework.Describe("[group:ipam]", func() { // separate ippool annotation by comma framework.ConformanceIt("should allocate static ip for statefulset with ippool separated by comma", func() { - if f.ClusterIpFamily == "dual" { + if f.IsDual() { ginkgo.Skip("Comma separated ippool is not supported for dual stack") } ippoolSep := "," replicas := 3 - ippool := framework.RandomIPPool(cidr, ippoolSep, replicas) + ippool := framework.RandomIPs(cidr, ippoolSep, replicas) labels := map[string]string{"app": stsName} ginkgo.By("Creating statefulset " + stsName + " with ippool " + ippool) @@ -386,60 +384,70 @@ var _ = framework.Describe("[group:ipam]", func() { framework.ConformanceIt("should support IPPool feature", func() { f.SkipVersionPriorTo(1, 12, "Support for IPPool feature was introduced in v1.12") - ipsCount := 3 - randomIPs := strings.Split(framework.RandomIPPool(cidr, ";", ipsCount), ";") - ips := make([]string, 0, ipsCount*2) - for _, s := range randomIPs { - ips = append(ips, strings.Split(s, ",")...) - } + ipsCount := 12 + ips := framework.RandomIPPool(cidr, ipsCount) ipv4, ipv6 := util.SplitIpsByProtocol(ips) - if len(ipv4) != 0 { + if f.HasIPv4() { framework.ExpectHaveLen(ipv4, ipsCount) } - if len(ipv6) != 0 { + if f.HasIPv6() { framework.ExpectHaveLen(ipv6, ipsCount) } - sort.Slice(ipv4, func(i, j int) bool { - ip1, _ := ipam.NewIP(ipv4[i]) - ip2, _ := ipam.NewIP(ipv4[j]) - return ip1.LessThan(ip2) - }) - sort.Slice(ipv6, func(i, j int) bool { - ip1, _ := ipam.NewIP(ipv6[i]) - ip2, _ := ipam.NewIP(ipv6[j]) - return ip1.LessThan(ip2) - }) - - var err error - ips = make([]string, 0, ipsCount*2) - ipv4Range, ipv6Range := ipam.NewEmptyIPRangeList(), ipam.NewEmptyIPRangeList() - if len(ipv4) != 0 { - tmp := []string{ipv4[0], fmt.Sprintf("%s..%s", ipv4[1], ipv4[2])} - ipv4Range, err = ipam.NewIPRangeListFrom(tmp...) - framework.ExpectNoError(err) - ips = append(ips, tmp...) - } - if len(ipv6) != 0 { - tmp := []string{ipv6[0], fmt.Sprintf("%s..%s", ipv6[1], ipv6[2])} - ipv6Range, err = ipam.NewIPRangeListFrom(tmp...) - framework.ExpectNoError(err) - ips = append(ips, tmp...) - } + ipv4Range, err := ipam.NewIPRangeListFrom(ipv4...) + framework.ExpectNoError(err) + ipv6Range, err := ipam.NewIPRangeListFrom(ipv6...) + framework.ExpectNoError(err) + + excludeV4, excludeV6 := util.SplitIpsByProtocol(subnet.Spec.ExcludeIps) + excludeV4Range, err := ipam.NewIPRangeListFrom(excludeV4...) + framework.ExpectNoError(err) + excludeV6Range, err := ipam.NewIPRangeListFrom(excludeV6...) + framework.ExpectNoError(err) + + ipv4Range = ipv4Range.Separate(excludeV4Range) + ipv6Range = ipv6Range.Separate(excludeV6Range) ginkgo.By(fmt.Sprintf("Creating ippool %s with ips %v", ippoolName, ips)) ippool := framework.MakeIPPool(ippoolName, subnetName, ips, nil) ippool = ippoolClient.CreateSync(ippool) ginkgo.By("Validating ippool status") - framework.ExpectTrue(ippool.Status.V4UsingIPs.EqualInt64(0)) - framework.ExpectTrue(ippool.Status.V6UsingIPs.EqualInt64(0)) - framework.ExpectEmpty(ippool.Status.V4UsingIPRange) - framework.ExpectEmpty(ippool.Status.V6UsingIPRange) - framework.ExpectTrue(ippool.Status.V4AvailableIPs.Equal(ipv4Range.Count())) - framework.ExpectTrue(ippool.Status.V6AvailableIPs.Equal(ipv6Range.Count())) - framework.ExpectEqual(ippool.Status.V4AvailableIPRange, ipv4Range.String()) - framework.ExpectEqual(ippool.Status.V6AvailableIPRange, ipv6Range.String()) + framework.WaitUntil(2*time.Second, 30*time.Second, func(_ context.Context) (bool, error) { + if !ippool.Status.V4UsingIPs.EqualInt64(0) { + framework.Logf("unexpected .status.v4UsingIPs: %s", ippool.Status.V4UsingIPs) + return false, nil + } + if !ippool.Status.V6UsingIPs.EqualInt64(0) { + framework.Logf("unexpected .status.v6UsingIPs: %s", ippool.Status.V6UsingIPs) + return false, nil + } + if ippool.Status.V4UsingIPRange != "" { + framework.Logf("unexpected .status.v4UsingIPRange: %s", ippool.Status.V4UsingIPRange) + return false, nil + } + if ippool.Status.V6UsingIPRange != "" { + framework.Logf("unexpected .status.v6UsingIPRange: %s", ippool.Status.V6UsingIPRange) + return false, nil + } + if !ippool.Status.V4AvailableIPs.Equal(ipv4Range.Count()) { + framework.Logf(".status.v4AvailableIPs mismatch: expect %s, actual %s", ipv4Range.Count(), ippool.Status.V4AvailableIPs) + return false, nil + } + if !ippool.Status.V6AvailableIPs.Equal(ipv6Range.Count()) { + framework.Logf(".status.v6AvailableIPs mismatch: expect %s, actual %s", ipv6Range.Count(), ippool.Status.V6AvailableIPs) + return false, nil + } + if ippool.Status.V4AvailableIPRange != ipv4Range.String() { + framework.Logf(".status.v4AvailableIPRange mismatch: expect %s, actual %s", ipv4Range, ippool.Status.V4AvailableIPRange) + return false, nil + } + if ippool.Status.V6AvailableIPRange != ipv6Range.String() { + framework.Logf(".status.v6AvailableIPRange mismatch: expect %s, actual %s", ipv6Range, ippool.Status.V6AvailableIPRange) + return false, nil + } + return true, nil + }, "") ginkgo.By("Creating deployment " + deployName + " within ippool " + ippoolName) replicas := 3 diff --git a/test/e2e/kube-ovn/kubectl-ko/kubectl-ko.go b/test/e2e/kube-ovn/kubectl-ko/kubectl-ko.go index dc4eed2dbe7..556d56f8648 100644 --- a/test/e2e/kube-ovn/kubectl-ko/kubectl-ko.go +++ b/test/e2e/kube-ovn/kubectl-ko/kubectl-ko.go @@ -110,7 +110,7 @@ var _ = framework.Describe("[group:kubectl-ko]", func() { framework.ConformanceIt(`should support "kubectl ko tcpdump -c1"`, func() { ping, target := "ping", targetIPv4 - if f.IPv6() { + if f.IsIPv6() { ping, target = "ping6", targetIPv6 } diff --git a/test/e2e/kube-ovn/service/service.go b/test/e2e/kube-ovn/service/service.go index 81f7c6f8de0..4abd66de56a 100644 --- a/test/e2e/kube-ovn/service/service.go +++ b/test/e2e/kube-ovn/service/service.go @@ -121,7 +121,7 @@ var _ = framework.Describe("[group:service]", func() { }) framework.ConformanceIt("should ovn nb change vip when dual-stack service removes the cluster ip ", func() { - if f.ClusterIpFamily != "dual" { + if !f.IsDual() { ginkgo.Skip("this case only support dual mode") } f.SkipVersionPriorTo(1, 11, "This case is support in v1.11") diff --git a/test/e2e/kube-ovn/subnet/subnet.go b/test/e2e/kube-ovn/subnet/subnet.go index bab32ecd674..54a8378905a 100644 --- a/test/e2e/kube-ovn/subnet/subnet.go +++ b/test/e2e/kube-ovn/subnet/subnet.go @@ -198,7 +198,7 @@ var _ = framework.Describe("[group:subnet]", func() { return "" } _, ipnet, _ := net.ParseCIDR(cidr) - ipnet.IP = net.ParseIP(framework.RandomIPPool(cidr, ";", 1)) + ipnet.IP = net.ParseIP(framework.RandomIPs(cidr, ";", 1)) return ipnet.String() } @@ -911,9 +911,9 @@ var _ = framework.Describe("[group:subnet]", func() { ginkgo.By("Restarting deployment " + deployName) _ = deployClient.RestartSync(deploy) - checkFunc2 := func(usingIPRange, availableIPRange, startIP, lastIP string, count int64) { + checkFunc2 := func(usingIPRange, availableIPRange, startIP, lastIP string, count int64) bool { if startIP == "" { - return + return true } expectAvailIPRangeStr := fmt.Sprintf("%s-%s,%s-%s", @@ -922,22 +922,28 @@ var _ = framework.Describe("[group:subnet]", func() { util.BigInt2Ip(big.NewInt(0).Add(util.Ip2BigInt(startIP), big.NewInt(2*count))), lastIP, ) - expectUsingIPRangeStr := fmt.Sprintf("%s-%s", util.BigInt2Ip(big.NewInt(0).Add(util.Ip2BigInt(startIP), big.NewInt(count))), util.BigInt2Ip(big.NewInt(0).Add(util.Ip2BigInt(startIP), big.NewInt(2*count-1))), ) framework.Logf("subnet status usingIPRange %q expect %q", usingIPRange, expectUsingIPRangeStr) - framework.ExpectEqual(usingIPRange, expectUsingIPRangeStr) + if usingIPRange != expectUsingIPRangeStr { + return false + } framework.Logf("subnet status availableIPRange %q expect %q", availableIPRange, expectAvailIPRangeStr) - framework.ExpectEqual(availableIPRange, expectAvailIPRangeStr) + return availableIPRange == expectAvailIPRangeStr } ginkgo.By("Checking subnet status") subnet = subnetClient.Get(subnetName) - checkFunc2(subnet.Status.V4UsingIPRange, subnet.Status.V4AvailableIPRange, startIPv4, lastIPv4, replicas) - checkFunc2(subnet.Status.V6UsingIPRange, subnet.Status.V6AvailableIPRange, startIPv6, lastIPv6, replicas) + framework.WaitUntil(2*time.Second, 30*time.Second, func(_ context.Context) (bool, error) { + subnet = subnetClient.Get(subnetName) + if !checkFunc2(subnet.Status.V4UsingIPRange, subnet.Status.V4AvailableIPRange, startIPv4, lastIPv4, replicas) { + return false, nil + } + return checkFunc2(subnet.Status.V6UsingIPRange, subnet.Status.V6AvailableIPRange, startIPv6, lastIPv6, replicas), nil + }, "") }) framework.ConformanceIt("create subnet with enableLb option", func() { diff --git a/test/e2e/kube-ovn/underlay/underlay.go b/test/e2e/kube-ovn/underlay/underlay.go index 05829b56733..ca7a71ce704 100644 --- a/test/e2e/kube-ovn/underlay/underlay.go +++ b/test/e2e/kube-ovn/underlay/underlay.go @@ -363,12 +363,12 @@ var _ = framework.SerialDescribe("[group:underlay]", func() { for _, config := range dockerNetwork.IPAM.Config { switch util.CheckProtocol(config.Subnet) { case apiv1.ProtocolIPv4: - if f.ClusterIpFamily != "ipv6" { + if f.HasIPv4() { cidr = append(cidr, config.Subnet) gateway = append(gateway, config.Gateway) } case apiv1.ProtocolIPv6: - if f.ClusterIpFamily != "ipv4" { + if f.HasIPv6() { cidr = append(cidr, config.Subnet) gateway = append(gateway, config.Gateway) } @@ -376,10 +376,10 @@ var _ = framework.SerialDescribe("[group:underlay]", func() { } excludeIPs := make([]string, 0, len(network.Containers)*2) for _, container := range network.Containers { - if container.IPv4Address != "" && f.ClusterIpFamily != "ipv6" { + if container.IPv4Address != "" && f.HasIPv4() { excludeIPs = append(excludeIPs, strings.Split(container.IPv4Address, "/")[0]) } - if container.IPv6Address != "" && f.ClusterIpFamily != "ipv4" { + if container.IPv6Address != "" && f.HasIPv6() { excludeIPs = append(excludeIPs, strings.Split(container.IPv6Address, "/")[0]) } } @@ -401,7 +401,7 @@ var _ = framework.SerialDescribe("[group:underlay]", func() { }) framework.ConformanceIt("should be able to detect IPv4 address conflict", func() { - if f.ClusterIpFamily != "ipv4" { + if !f.IsIPv4() { ginkgo.Skip("Address conflict detection only supports IPv4") } f.SkipVersionPriorTo(1, 9, "Address conflict detection was introduced in v1.9") @@ -487,12 +487,12 @@ var _ = framework.SerialDescribe("[group:underlay]", func() { for _, config := range dockerNetwork.IPAM.Config { switch util.CheckProtocol(config.Subnet) { case apiv1.ProtocolIPv4: - if f.ClusterIpFamily != "ipv6" { + if f.HasIPv4() { underlayCidr = append(underlayCidr, config.Subnet) gateway = append(gateway, config.Gateway) } case apiv1.ProtocolIPv6: - if f.ClusterIpFamily != "ipv4" { + if f.HasIPv6() { underlayCidr = append(underlayCidr, config.Subnet) gateway = append(gateway, config.Gateway) } @@ -501,10 +501,10 @@ var _ = framework.SerialDescribe("[group:underlay]", func() { excludeIPs := make([]string, 0, len(network.Containers)*2) for _, container := range network.Containers { - if container.IPv4Address != "" && f.ClusterIpFamily != "ipv6" { + if container.IPv4Address != "" && f.HasIPv4() { excludeIPs = append(excludeIPs, strings.Split(container.IPv4Address, "/")[0]) } - if container.IPv6Address != "" && f.ClusterIpFamily != "ipv4" { + if container.IPv6Address != "" && f.HasIPv6() { excludeIPs = append(excludeIPs, strings.Split(container.IPv6Address, "/")[0]) } } diff --git a/test/e2e/ovn-vpc-nat-gw/e2e_test.go b/test/e2e/ovn-vpc-nat-gw/e2e_test.go index 2bfff4ebb41..b2f8bf3d074 100644 --- a/test/e2e/ovn-vpc-nat-gw/e2e_test.go +++ b/test/e2e/ovn-vpc-nat-gw/e2e_test.go @@ -333,12 +333,12 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { for _, config := range dockerNetwork.IPAM.Config { switch util.CheckProtocol(config.Subnet) { case apiv1.ProtocolIPv4: - if f.ClusterIpFamily != "ipv6" { + if f.HasIPv4() { cidr = append(cidr, config.Subnet) gateway = append(gateway, config.Gateway) } case apiv1.ProtocolIPv6: - if f.ClusterIpFamily != "ipv4" { + if f.HasIPv6() { cidr = append(cidr, config.Subnet) gateway = append(gateway, config.Gateway) } @@ -346,10 +346,10 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { } excludeIPs := make([]string, 0, len(network.Containers)*2) for _, container := range network.Containers { - if container.IPv4Address != "" && f.ClusterIpFamily != "ipv6" { + if container.IPv4Address != "" && f.HasIPv4() { excludeIPs = append(excludeIPs, strings.Split(container.IPv4Address, "/")[0]) } - if container.IPv6Address != "" && f.ClusterIpFamily != "ipv4" { + if container.IPv6Address != "" && f.HasIPv6() { excludeIPs = append(excludeIPs, strings.Split(container.IPv6Address, "/")[0]) } } diff --git a/test/unittest/ipam/ip.go b/test/unittest/ipam/ip.go index e78901d0193..44c9d6c8bfd 100644 --- a/test/unittest/ipam/ip.go +++ b/test/unittest/ipam/ip.go @@ -15,6 +15,10 @@ func uint32ToIPv4(n uint32) string { return fmt.Sprintf("%d.%d.%d.%d", n>>24, n&0xff0000>>16, n&0xff00>>8, n&0xff) } +func ipv4ToUint32(ip net.IP) uint32 { + return uint32(ip[0])<<24 | uint32(ip[1])<<16 | uint32(ip[2])<<8 | uint32(ip[3]) +} + func uint32ToIPv6(n [4]uint32) string { return fmt.Sprintf( "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x", diff --git a/test/unittest/ipam/ip_range.go b/test/unittest/ipam/ip_range.go index 20485f5a5c7..0dc22ee3197 100644 --- a/test/unittest/ipam/ip_range.go +++ b/test/unittest/ipam/ip_range.go @@ -2,6 +2,7 @@ package ipam import ( "fmt" + "math" "math/big" "math/rand" "net" @@ -54,6 +55,34 @@ var _ = ginkgo.Context("[group:IPAM]", func() { gomega.Expect(r.Contains(start.Add(1))).To(gomega.BeTrue()) gomega.Expect(r.Contains(end.Sub(1))).To(gomega.BeTrue()) } + + for i := uint32(0); i < 100 && i <= n2-n1; i++ { + gomega.Expect(r.Contains(r.Random())).To(gomega.BeTrue()) + } + + prefix := rand.Intn(net.IPv4len*8 + 1) + _, cidr, err := net.ParseCIDR(fmt.Sprintf("%s/%d", startStr, prefix)) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(cidr).NotTo(gomega.BeNil()) + + r = ipam.NewIPRangeFromCIDR(*cidr) + gomega.Expect(r.Contains(start)).To(gomega.BeTrue()) + + start, err = ipam.NewIP(cidr.IP.String()) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(r.Contains(start)).To(gomega.BeTrue()) + if !cidr.IP.Equal(net.IPv4zero) { + gomega.Expect(r.Contains(start.Sub(1))).To(gomega.BeFalse()) + } + if prefix != net.IPv4len*8 { + gomega.Expect(r.Contains(start.Add(1))).To(gomega.BeTrue()) + } + + c = r.Count() + gomega.Expect(c.Int.Cmp(big.NewInt(int64(math.Exp2(float64(net.IPv4len*8 - prefix)))))).To(gomega.Equal(0)) + for i := int64(0); i < 100 && i <= c.Int64(); i++ { + gomega.Expect(r.Contains(r.Random())).To(gomega.BeTrue()) + } }) ginkgo.It("IPv6", func() { @@ -111,6 +140,36 @@ var _ = ginkgo.Context("[group:IPAM]", func() { gomega.Expect(r.Contains(start.Add(1))).To(gomega.BeTrue()) gomega.Expect(r.Contains(end.Sub(1))).To(gomega.BeTrue()) } + + for i := 0; i < 100; i++ { + gomega.Expect(r.Contains(r.Random())).To(gomega.BeTrue()) + } + + prefix := rand.Intn(net.IPv6len*8 + 1) + _, cidr, err := net.ParseCIDR(fmt.Sprintf("%s/%d", startStr, prefix)) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(cidr).NotTo(gomega.BeNil()) + + r = ipam.NewIPRangeFromCIDR(*cidr) + gomega.Expect(r.Contains(start)).To(gomega.BeTrue()) + + start, err = ipam.NewIP(cidr.IP.String()) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(r.Contains(start)).To(gomega.BeTrue()) + if !cidr.IP.Equal(net.IPv6zero) { + gomega.Expect(r.Contains(start.Sub(1))).To(gomega.BeFalse()) + } + if prefix != net.IPv6len*8 { + gomega.Expect(r.Contains(start.Add(1))).To(gomega.BeTrue()) + } + + for i := 0; i < 100; i++ { + gomega.Expect(r.Contains(r.Random())).To(gomega.BeTrue()) + } + + count = r.Count() + expectedCount = big.NewInt(0).Exp(big.NewInt(2), big.NewInt(int64(net.IPv6len*8-prefix)), nil) + gomega.Expect(count.Int.Cmp(expectedCount)).To(gomega.Equal(0)) }) }) }) diff --git a/test/unittest/ipam/ip_range_list.go b/test/unittest/ipam/ip_range_list.go index 7732fcb18c8..19f721391c7 100644 --- a/test/unittest/ipam/ip_range_list.go +++ b/test/unittest/ipam/ip_range_list.go @@ -3,6 +3,7 @@ package ipam import ( "fmt" "math/rand" + "net" "sort" "strings" @@ -201,53 +202,117 @@ var _ = ginkgo.Context("[group:IPAM]", func() { }) ginkgo.It("NewIPRangeListFrom", func() { - n := 100 + rand.Intn(50) - set := u32set.NewWithSize(n) + n := 40 + rand.Intn(20) + cidrList := make([]*net.IPNet, 0, n) + cidrSet := u32set.NewWithSize(n * 2) + for len(cidrList) != cap(cidrList) { + _, cidr, err := net.ParseCIDR(fmt.Sprintf("%s/%d", uint32ToIPv4(rand.Uint32()), 16+rand.Intn(16))) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + var invalid bool + for _, c := range cidrList { + if c.Contains(cidr.IP) || cidr.Contains(c.IP) { + invalid = true + break + } + } + if !invalid { + cidrList = append(cidrList, cidr) + cidrSet.Add(ipv4ToUint32(cidr.IP)) + bcast := make(net.IP, len(cidr.IP)) + for i := 0; i < len(bcast); i++ { + bcast[i] = cidr.IP[i] | ^cidr.Mask[i] + } + cidrSet.Add(ipv4ToUint32(bcast)) + } + } + + n = 80 + rand.Intn(40) + set := u32set.NewWithSize(cidrSet.Size() + n) for set.Size() != n { - set.Add(rand.Uint32()) + v := rand.Uint32() + ip := net.ParseIP(uint32ToIPv4(v)) + var invalid bool + for _, cidr := range cidrList { + if cidr.Contains(ip) { + invalid = true + break + } + } + if !invalid { + set.Add(v) + } } + set.Merge(cidrSet) ints := set.List() sort.Slice(ints, func(i, j int) bool { return ints[i] < ints[j] }) - var ips, mergedIPs []string + ips := make([]string, 0, len(cidrList)+set.Size()) + mergedInts := make([]uint32, 0, set.Size()*2) var expectedCount uint32 for i := 0; i < len(ints); i++ { + if cidrSet.Has(ints[i]) { + expectedCount += ints[i+1] - ints[i] + 1 + if i != 0 && ints[i] == ints[i-1]+1 { + mergedInts[len(mergedInts)-1] = ints[i+1] + } else { + mergedInts = append(mergedInts, ints[i], ints[i+1]) + } + i++ + continue + } + start := uint32ToIPv4(ints[i]) - var merged string - if rand.Int()%2 == 0 && i+1 != len(ints) { - end := uint32ToIPv4(ints[i+1]) - ips = append(ips, fmt.Sprintf("%s..%s", start, end)) + if cidrSet.Has(ints[i]) || (rand.Int()%2 == 0 && i+1 != len(ints) && !cidrSet.Has(ints[i+1])) { + if !cidrSet.Has(ints[i]) { + end := uint32ToIPv4(ints[i+1]) + ips = append(ips, fmt.Sprintf("%s..%s", start, end)) + } if i != 0 && ints[i] == ints[i-1]+1 { - merged = fmt.Sprintf("%s..%s", strings.Split(mergedIPs[len(mergedIPs)-1], "..")[0], end) + mergedInts[len(mergedInts)-1] = ints[i+1] + } else { + mergedInts = append(mergedInts, ints[i], ints[i+1]) } expectedCount += ints[i+1] - ints[i] + 1 i++ } else { + if rand.Int()%8 == 0 { + start += "/32" + } ips = append(ips, start) if i != 0 && ints[i] == ints[i-1]+1 { - merged = fmt.Sprintf("%s..%s", strings.Split(mergedIPs[len(mergedIPs)-1], "..")[0], start) + mergedInts[len(mergedInts)-1] = ints[i] + } else { + mergedInts = append(mergedInts, ints[i], ints[i]) } expectedCount++ } + } + + for _, cidr := range cidrList { + ips = append(ips, cidr.String()) + } - if merged != "" { - mergedIPs[len(mergedIPs)-1] = merged + mergedIPs := make([]string, len(mergedInts)/2) + for i := 0; i < len(mergedInts)/2; i++ { + if mergedInts[i*2] == mergedInts[i*2+1] { + mergedIPs[i] = uint32ToIPv4(mergedInts[i*2]) } else { - mergedIPs = append(mergedIPs, ips[len(ips)-1]) + mergedIPs[i] = fmt.Sprintf("%s-%s", uint32ToIPv4(mergedInts[i*2]), uint32ToIPv4(mergedInts[i*2+1])) } } list, err := ipam.NewIPRangeListFrom(strset.New(ips...).List()...) gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(list.Len()).To(gomega.Equal(len(mergedIPs))) - gomega.Expect(list.String()).To(gomega.Equal(strings.ReplaceAll(strings.Join(mergedIPs, ","), "..", "-"))) + gomega.Expect(list.String()).To(gomega.Equal(strings.Join(mergedIPs, ","))) count := list.Count() gomega.Expect(count.Int64()).To(gomega.Equal(int64(expectedCount))) for _, s := range mergedIPs { - fields := strings.Split(s, "..") + fields := strings.Split(s, "-") start, err := ipam.NewIP(fields[0]) gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(list.Contains(start)).To(gomega.BeTrue())