operator: Add core logic for IPAM pools

This commit adds the core logic for allocating pod CIDRs from specific
pools to nodes. The logic is not yet hooked up with the CiliumNode
watcher, that will follow in a subsequent commit.

The main idea behind the `PoolAllocator` is that it tracks both the
available IP pools, but also which node has allocated which CIDR from
which pool. Each pool is implemented as a slice of `CIDRAllocators`
(based on the `github.com/cilium/ipam/cidrset` package), which means it
manages the free-lists for each pool, but does not keep track of who
allocated the CIDR. This is done separately to easily be able to obtain
a list of CIDRs allocated to a node, without having to iterate over all
pools.

As with other IPAM allocators, a main concern is state restoration after
the operator has been restarted. The approach we take here is similar to
the one used by clusterpool (v1): In an initial phase (indicated by
`PoolAllocator.ready=false`), the watcher passes in all known
CiliumNodes to `AllocateToNode`. This will cause the `PoolAllocator` to
mark any previously (before the operator restart) allocated CIDR as
occupied. Once all CiliumNodes have been observed, the flip is switched
via `RestoreFinished`, at which point the watcher will have to invoke
`AllocateToNode` again for every node, allowing new CIDRs now to be
allocated to that node if necessary. This ensures that we make sure we
know what CIDRs have been previously allocated before we hand out new
CIDRs.

Co-authored-by: Tobias Klauser <tobias@cilium.io>
Signed-off-by: Tobias Klauser <tobias@cilium.io>
Signed-off-by: Sebastian Wicki <sebastian@isovalent.com>

go.mod

@@ -91,6 +91,7 @@ require (
 	go.uber.org/goleak v1.2.1
 	go.uber.org/multierr v1.11.0
 	go.universe.tf/metallb v0.11.0
+	go4.org/netipx v0.0.0-20230303233057-f1b76eb4bb35
 	golang.org/x/crypto v0.9.0
 	golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea
 	golang.org/x/mod v0.10.0

pkg/ipam/allocator/multipool/pool.go

@@ -0,0 +1,112 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright Authors of Cilium
+
+package multipool
+
+import (
+	"errors"
+	"fmt"
+	"net/netip"
+
+	"go4.org/netipx"
+
+	"github.com/cilium/cilium/pkg/ip"
+	"github.com/cilium/cilium/pkg/ipam"
+	"github.com/cilium/cilium/pkg/ipam/allocator/clusterpool/cidralloc"
+)
+
+var (
+	errPoolEmpty = errors.New("pool empty")
+)
+
+func allocFirstFreeCIDR(allocators []cidralloc.CIDRAllocator) (netip.Prefix, error) {
+	for _, alloc := range allocators {
+		if alloc.IsFull() {
+			continue
+		}
+
+		ipnet, err := alloc.AllocateNext()
+		if err != nil {
+			return netip.Prefix{}, err
+		}
+
+		prefix, ok := netipx.FromStdIPNet(ipnet)
+		if !ok {
+			return netip.Prefix{}, fmt.Errorf("invalid cidr %s allocated", ipnet)
+		}
+		return prefix, nil
+	}
+
+	return netip.Prefix{}, errPoolEmpty
+}
+
+func occupyCIDR(allocators []cidralloc.CIDRAllocator, cidr netip.Prefix) error {
+	ipnet := ip.PrefixToIPNet(cidr)
+	for _, alloc := range allocators {
+		if !alloc.InRange(ipnet) {
+			continue
+		}
+		if alloc.IsFull() {
+			return errPoolEmpty
+		}
+		allocated, err := alloc.IsAllocated(ipnet)
+		if err != nil {
+			return err
+		}
+		if allocated {
+			return fmt.Errorf("cidr %s has already been allocated", cidr)
+		}
+
+		return alloc.Occupy(ipnet)
+	}
+
+	return fmt.Errorf("cidr %s is not part of the requested pool", cidr)
+}
+
+func releaseCIDR(allocators []cidralloc.CIDRAllocator, cidr netip.Prefix) error {
+	ipnet := ip.PrefixToIPNet(cidr)
+	for _, alloc := range allocators {
+		if !alloc.InRange(ipnet) {
+			continue
+		}
+
+		allocated, err := alloc.IsAllocated(ipnet)
+		if err != nil {
+			return err
+		}
+		if !allocated {
+			return nil // not an error to release a cidr twice
+		}
+
+		return alloc.Release(ipnet)
+	}
+
+	return fmt.Errorf("released cidr %s was not part the pool", cidr)
+}
+
+func (c *cidrPool) allocCIDR(family ipam.Family) (netip.Prefix, error) {
+	switch family {
+	case ipam.IPv4:
+		return allocFirstFreeCIDR(c.v4)
+	case ipam.IPv6:
+		return allocFirstFreeCIDR(c.v6)
+	default:
+		return netip.Prefix{}, fmt.Errorf("invalid cidr family: %s", family)
+	}
+}
+
+func (c *cidrPool) occupyCIDR(cidr netip.Prefix) error {
+	if cidr.Addr().Is4() {
+		return occupyCIDR(c.v4, cidr)
+	} else {
+		return occupyCIDR(c.v6, cidr)
+	}
+}
+
+func (c *cidrPool) releaseCIDR(cidr netip.Prefix) error {
+	if cidr.Addr().Is4() {
+		return releaseCIDR(c.v4, cidr)
+	} else {
+		return releaseCIDR(c.v6, cidr)
+	}
+}