ipam: add metrics to track per node capacity

Documentation/observability/metrics.rst

@@ -508,6 +508,9 @@ Name                                     Labels
 ``ipam_resync_total``                                                                                      Enabled    Number of synchronization operations with external IPAM API
 ``ipam_api_duration_seconds``            ``operation``, ``response_code``                                  Enabled    Duration of interactions with external IPAM API.
 ``ipam_api_rate_limit_duration_seconds`` ``operation``                                                     Enabled    Duration of rate limiting while accessing external IPAM API
+``ipam_available_ips``                   ``target_node``                                                   Enabled    Number of available IPs on a node (taking into account plugin specific NIC/Address limits).
+``ipam_used_ips``                        ``target_node``                                                   Enabled    Number of currently used IPs on a node.
+``ipam_needed_ips``                      ``target_node``                                                   Enabled    Number of IPs needed to satisfy allocation on a node.
 ======================================== ================================================================= ========== ========================================================
 
 Hubble

Documentation/operations/upgrade.rst

@@ -357,6 +357,9 @@ Added Metrics
 * ``cilium_operator_ces_sync_total``
 * ``cilium_policy_change_total``
 * ``go_sched_latencies_seconds``
+* ``cilium_operator_ipam_available_ips``
+* ``cilium_operator_ipam_used_ips``
+* ``cilium_operator_ipam_needed_ips``
 
 Deprecated Metrics
 ~~~~~~~~~~~~~~~~~~
@@ -365,6 +368,7 @@ Deprecated Metrics
 * ``cilium_policy_import_errors_total`` is deprecated. Please use
   ``cilium_policy_change_total``, which counts all policy changes (Add, Update, Delete)
   based on outcome ("success" or "failure").
+* ``cilium_operator_ipam_ips`` is deprecated. Use ``cilium_operator_ipam_{available,used,needed}_ips`` instead.
 
 Changed Metrics
 ~~~~~~~~~~~~~~~

pkg/alibabacloud/eni/node.go

@@ -14,6 +14,7 @@ import (
 	"github.com/cilium/cilium/pkg/alibabacloud/utils"
 	"github.com/cilium/cilium/pkg/defaults"
 	"github.com/cilium/cilium/pkg/ipam"
+	"github.com/cilium/cilium/pkg/ipam/stats"
 	ipamTypes "github.com/cilium/cilium/pkg/ipam/types"
 	v2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2"
 	"github.com/cilium/cilium/pkg/lock"
@@ -41,9 +42,13 @@ const (
 	maxENIPerNode = 50
 )
 
+type ipamNodeActions interface {
+	InstanceID() string
+}
+
 type Node struct {
 	// node contains the general purpose fields of a node
-	node *ipam.Node
+	node ipamNodeActions
 
 	// mutex protects members below this field
 	mutex lock.RWMutex
@@ -183,11 +188,16 @@ func (n *Node) CreateInterface(ctx context.Context, allocation *ipam.AllocationA
 
 // ResyncInterfacesAndIPs is called to retrieve and ENIs and IPs as known to
 // the AlibabaCloud API and return them
-func (n *Node) ResyncInterfacesAndIPs(ctx context.Context, scopedLog *logrus.Entry) (available ipamTypes.AllocationMap, remainAvailableENIsCount int, err error) {
+func (n *Node) ResyncInterfacesAndIPs(ctx context.Context, scopedLog *logrus.Entry) (available ipamTypes.AllocationMap, stats stats.InterfaceStats, err error) {
 	limits, limitsAvailable := n.getLimits()
 	if !limitsAvailable {
-		return nil, -1, fmt.Errorf(errUnableToDetermineLimits)
+		return nil, stats, fmt.Errorf(errUnableToDetermineLimits)
 	}
+
+	// During preparation of IP allocations, the primary NIC is not considered
+	// for allocation, so we don't need to consider it for capacity calculation.
+	stats.NodeCapacity = limits.IPv4 * (limits.Adapters - 1)
+
 	instanceID := n.node.InstanceID()
 	available = ipamTypes.AllocationMap{}
 
@@ -207,9 +217,18 @@ func (n *Node) ResyncInterfacesAndIPs(ctx context.Context, scopedLog *logrus.Ent
 				return nil
 			}
 
+			// We exclude all "primary" IPs from the capacity.
+			primaryAllocated := 0
+			for _, ip := range e.PrivateIPSets {
+				if ip.Primary {
+					primaryAllocated++
+				}
+			}
+			stats.NodeCapacity -= primaryAllocated
+
 			availableOnENI := math.IntMax(limits.IPv4-len(e.PrivateIPSets), 0)
 			if availableOnENI > 0 {
-				remainAvailableENIsCount++
+				stats.RemainingAvailableInterfaceCount++
 			}
 
 			for _, ip := range e.PrivateIPSets {
@@ -222,11 +241,11 @@ func (n *Node) ResyncInterfacesAndIPs(ctx context.Context, scopedLog *logrus.Ent
 	// An ECS instance has at least one ENI attached, no ENI found implies instance not found.
 	if enis == 0 {
 		scopedLog.Warning("Instance not found! Please delete corresponding ciliumnode if instance has already been deleted.")
-		return nil, -1, fmt.Errorf("unable to retrieve ENIs")
+		return nil, stats, fmt.Errorf("unable to retrieve ENIs")
 	}
 
-	remainAvailableENIsCount += limits.Adapters - len(n.enis)
-	return available, remainAvailableENIsCount, nil
+	stats.RemainingAvailableInterfaceCount += limits.Adapters - len(n.enis)
+	return available, stats, nil
 }
 
 // PrepareIPAllocation returns the number of ENI IPs and interfaces that can be
@@ -251,6 +270,7 @@ func (n *Node) PrepareIPAllocation(scopedLog *logrus.Entry) (*ipam.AllocationAct
 			"allocated": len(e.PrivateIPSets),
 		}).Debug("Considering ENI for allocation")
 
+		// limit
 		availableOnENI := math.IntMax(l.IPv4-len(e.PrivateIPSets), 0)
 		if availableOnENI <= 0 {
 			continue

pkg/alibabacloud/eni/node_stat_test.go

@@ -0,0 +1,62 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright Authors of Cilium
+
+package eni
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/cilium/cilium/pkg/alibabacloud/eni/limits"
+	eniTypes "github.com/cilium/cilium/pkg/alibabacloud/eni/types"
+	ipamTypes "github.com/cilium/cilium/pkg/ipam/types"
+	v2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2"
+)
+
+func TestENIIPAMCapacityAccounting(t *testing.T) {
+	assert := assert.New(t)
+	limits.Update(map[string]ipamTypes.Limits{
+		"ecs.g6.large": {
+			IPv4:     10,
+			Adapters: 3,
+		},
+	})
+	m := ipamTypes.NewInstanceMap()
+	resource := &eniTypes.ENI{
+		NetworkInterfaceID: "eni-1",
+		PrivateIPSets: []eniTypes.PrivateIPSet{
+			{
+				Primary: true, // one primary IP
+			},
+		},
+	}
+	m.Update("vm1", ipamTypes.InterfaceRevision{
+		Resource: resource.DeepCopy(),
+	})
+
+	n := &Node{
+		node: mockIPAMNode("vm1"),
+		manager: &InstancesManager{
+			instances: m,
+		},
+		k8sObj: &v2.CiliumNode{
+			Spec: v2.NodeSpec{
+				AlibabaCloud: eniTypes.Spec{
+					InstanceType: "ecs.g6.large",
+				},
+			},
+		},
+	}
+	_, stats, err := n.ResyncInterfacesAndIPs(context.Background(), log)
+	assert.NoError(err)
+	// 3 ENIs, 10 IPs per ENI, 1 primary IP and one ENI is primary.
+	assert.Equal(19, stats.NodeCapacity)
+}
+
+type mockIPAMNode string
+
+func (m mockIPAMNode) InstanceID() string {
+	return string(m)
+}

pkg/aws/eni/instances.go

@@ -222,8 +222,9 @@ func (m *InstancesManager) UpdateENI(instanceID string, eni *eniTypes.ENI) {
 	m.mutex.Unlock()
 }
 
-// ForeachInstance will iterate over each instance inside `instances`, and call
-// `fn`. This function is read-locked for the entire execution.
+// ForeachInstance will iterate over each interface for a particular instance inside `instances`
+// and call `fn`.
+// This function is read-locked for the entire execution.
 func (m *InstancesManager) ForeachInstance(instanceID string, fn ipamTypes.InterfaceIterator) {
 	// This is a safety net in case the InstanceID is not known for some
 	// reason. If we don't know the instanceID, we also can't derive the

pkg/aws/eni/node.go

@@ -25,6 +25,7 @@ import (
 	"github.com/cilium/cilium/pkg/ip"
 	"github.com/cilium/cilium/pkg/ipam"
 	"github.com/cilium/cilium/pkg/ipam/option"
+	"github.com/cilium/cilium/pkg/ipam/stats"
 	ipamTypes "github.com/cilium/cilium/pkg/ipam/types"
 	v2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2"
 	"github.com/cilium/cilium/pkg/lock"
@@ -40,11 +41,19 @@ const (
 		" this could lead to ip allocation overflows if the max-allocate flag is not set"
 )
 
+type ipamNodeActions interface {
+	IsPrefixDelegationEnabled() bool
+	InstanceID() string
+	Ops() ipam.NodeOperations
+	SetRunning(bool)
+	UpdatedResource(*v2.CiliumNode) bool
+}
+
 // Node represents a Kubernetes node running Cilium with an associated
 // CiliumNode custom resource
 type Node struct {
 	// node contains the general purpose fields of a node
-	node *ipam.Node
+	node ipamNodeActions
 
 	// mutex protects members below this field
 	mutex lock.RWMutex
@@ -221,7 +230,7 @@ func (n *Node) PrepareIPAllocation(scopedLog *logrus.Entry) (a *ipam.AllocationA
 			continue
 		}
 
-		effectiveLimits := n.getEffectiveIPLimits(&e, limits.IPv4)
+		_, effectiveLimits := n.getEffectiveIPLimits(&e, limits.IPv4)
 		availableOnENI := math.IntMax(effectiveLimits-len(e.Addresses), 0)
 		if availableOnENI <= 0 {
 			continue
@@ -531,11 +540,15 @@ func (n *Node) CreateInterface(ctx context.Context, allocation *ipam.AllocationA
 
 // ResyncInterfacesAndIPs is called to retrieve and ENIs and IPs as known to
 // the EC2 API and return them
-func (n *Node) ResyncInterfacesAndIPs(ctx context.Context, scopedLog *logrus.Entry) (available ipamTypes.AllocationMap, remainAvailableENIsCount int, err error) {
+func (n *Node) ResyncInterfacesAndIPs(ctx context.Context, scopedLog *logrus.Entry) (
+	available ipamTypes.AllocationMap,
+	stats stats.InterfaceStats,
+	err error) {
 	limits, limitsAvailable := n.getLimits()
 	if !limitsAvailable {
-		return nil, -1, fmt.Errorf(errUnableToDetermineLimits)
+		return nil, stats, fmt.Errorf(errUnableToDetermineLimits)
 	}
+
 	// n.node does not need to be protected by n.mutex as it is only written to
 	// upon creation of `n`
 	instanceID := n.node.InstanceID()
@@ -544,6 +557,19 @@ func (n *Node) ResyncInterfacesAndIPs(ctx context.Context, scopedLog *logrus.Ent
 	n.mutex.Lock()
 	n.enis = map[string]eniTypes.ENI{}
 
+	// 1. This calculates the base interface effective limit on this Node, given:
+	// 		* IPAM Prefix Delegation
+	// 		* Node Spec usePrimaryAddress being enabled
+	//
+	_, stats.NodeCapacity = n.getEffectiveIPLimits(nil, limits.IPv4)
+
+	// 2. The base node limit is the number of adapters multiplied by the instances IP limit.
+	//
+	// Note: This may be modified in step(s) 3, where:
+	// * Any leftover additional prefix delegated room will be added to this total.
+	// * Any excluded interfaces will be subtracted from this total.
+	stats.NodeCapacity *= limits.Adapters
+
 	n.manager.ForeachInstance(instanceID,
 		func(instanceID, interfaceID string, rev ipamTypes.InterfaceRevision) error {
 			e, ok := rev.Resource.(*eniTypes.ENI)
@@ -552,14 +578,22 @@ func (n *Node) ResyncInterfacesAndIPs(ctx context.Context, scopedLog *logrus.Ent
 			}
 
 			n.enis[e.ID] = *e
+
+			// 3. Finally, we iterate any already existing interfaces and add on any extra
+			//		capacity to account for leftover prefix delegated /28 ip slots.
+			leftoverPrefixCapcity, effectiveLimits := n.getEffectiveIPLimits(e, limits.IPv4)
 			if e.IsExcludedBySpec(n.k8sObj.Spec.ENI) {
+				// If this ENI is excluded by the CN Spec, we remove it from the total
+				// capacity.
+				stats.NodeCapacity -= effectiveLimits
 				return nil
+			} else {
+				stats.NodeCapacity += leftoverPrefixCapcity
 			}
 
-			effectiveLimits := n.getEffectiveIPLimits(e, limits.IPv4)
 			availableOnENI := math.IntMax(effectiveLimits-len(e.Addresses), 0)
 			if availableOnENI > 0 {
-				remainAvailableENIsCount++
+				stats.RemainingAvailableInterfaceCount++
 			}
 
 			for _, ip := range e.Addresses {
@@ -573,11 +607,11 @@ func (n *Node) ResyncInterfacesAndIPs(ctx context.Context, scopedLog *logrus.Ent
 	// An ec2 instance has at least one ENI attached, no ENI found implies instance not found.
 	if enis == 0 {
 		scopedLog.Warning("Instance not found! Please delete corresponding ciliumnode if instance has already been deleted.")
-		return nil, -1, fmt.Errorf("unable to retrieve ENIs")
+		return nil, stats, fmt.Errorf("unable to retrieve ENIs")
 	}
 
-	remainAvailableENIsCount += limits.Adapters - len(n.enis)
-	return available, remainAvailableENIsCount, nil
+	stats.RemainingAvailableInterfaceCount += limits.Adapters - len(n.enis)
+	return available, stats, nil
 }
 
 // GetMaximumAllocatableIPv4 returns the maximum amount of IPv4 addresses
@@ -694,11 +728,18 @@ func (n *Node) GetMinimumAllocatableIPv4() int {
 	return math.IntMin(minimum, (limits.Adapters-index)*maxPerInterface)
 }
 
+func (n *Node) isPrefixDelegationEnabled() bool {
+	if n.node == nil {
+		return false
+	}
+	return n.node.IsPrefixDelegationEnabled()
+}
+
 // IsPrefixDelegated indicates whether prefix delegation can be enabled on a node.
 // Currently, mixed usage of secondary IPs and prefixes is not supported. n.mutex
 // read lock must be held before calling this method.
 func (n *Node) IsPrefixDelegated() bool {
-	if !n.node.IsPrefixDelegationEnabled() {
+	if !n.isPrefixDelegationEnabled() {
 		return false
 	}
 	// Verify if this node is nitro based
@@ -780,8 +821,9 @@ func (n *Node) GetUsedIPWithPrefixes() int {
 }
 
 // getEffectiveIPLimits computing the effective number of available addresses on the ENI
-// based on limits
-func (n *Node) getEffectiveIPLimits(eni *eniTypes.ENI, limits int) (effectiveLimits int) {
+// based on limits (which includes any left over prefix delegation capacity), as well as
+// just the left over prefix delegation capacity.
+func (n *Node) getEffectiveIPLimits(eni *eniTypes.ENI, limits int) (leftoverPrefixCapacity, effectiveLimits int) {
 	// The limits include the primary IP, so we need to take it into account
 	// when computing the effective number of available addresses on the ENI.
 	effectiveLimits = limits - 1
@@ -790,13 +832,15 @@ func (n *Node) getEffectiveIPLimits(eni *eniTypes.ENI, limits int) (effectiveLim
 	if n.k8sObj.Spec.ENI.UsePrimaryAddress != nil && *n.k8sObj.Spec.ENI.UsePrimaryAddress {
 		effectiveLimits++
 	}
-	if n.node.Ops().IsPrefixDelegated() {
+
+	if n.IsPrefixDelegated() {
 		effectiveLimits = effectiveLimits * option.ENIPDBlockSizeIPv4
-	} else if len(eni.Prefixes) > 0 {
+	} else if eni != nil && len(eni.Prefixes) > 0 {
 		// If prefix delegation was previously enabled on this node, account for IPs from prefixes
-		effectiveLimits += len(eni.Prefixes) * (option.ENIPDBlockSizeIPv4 - 1)
+		leftoverPrefixCapacity = len(eni.Prefixes) * (option.ENIPDBlockSizeIPv4 - 1)
+		effectiveLimits += leftoverPrefixCapacity
 	}
-	return effectiveLimits
+	return leftoverPrefixCapacity, effectiveLimits
 }
 
 // findSuitableSubnet attempts to find a subnet to allocate an ENI in according to the following heuristic.