feat: v2 swift ipampool

cns/api.go

@@ -50,6 +50,7 @@ type HTTPService interface {
 	GetPodIPConfigState() map[string]IPConfigurationStatus
 	MarkIPAsPendingRelease(numberToMark int) (map[string]IPConfigurationStatus, error)
 	AttachIPConfigsHandlerMiddleware(IPConfigsHandlerMiddleware)
+	MarkNIPsPendingRelease(n int) (map[string]IPConfigurationStatus, error)
 }
 
 // IPConfigsHandlerFunc
@@ -281,13 +282,13 @@ type NodeConfiguration struct {
 	NodeSubnet Subnet
 }
 
+// IpamPoolMonitorStateSnapshot struct to expose state values for IPAMPoolMonitor struct
 type IPAMPoolMonitor interface {
 	Start(ctx context.Context) error
 	Update(nnc *v1alpha.NodeNetworkConfig) error
 	GetStateSnapshot() IpamPoolMonitorStateSnapshot
 }
 
-// IpamPoolMonitorStateSnapshot struct to expose state values for IPAMPoolMonitor struct
 type IpamPoolMonitorStateSnapshot struct {
 	MinimumFreeIps           int64
 	MaximumFreeIps           int64

cns/configuration/configuration.go

@@ -26,14 +26,26 @@ const (
 )
 
 type CNSConfig struct {
+	AZRSettings                 AZRSettings
+	AsyncPodDeletePath          string
+	CNIConflistFilepath         string
+	CNIConflistScenario         string
 	ChannelMode                 string
+	EnableAsyncPodDelete        bool
+	EnableCNIConflistGeneration bool
+	EnableIPAMv2                bool
 	EnablePprof                 bool
 	EnableSubnetScarcity        bool
 	EnableSwiftV2               bool
-	SWIFTV2Mode                 SWIFTV2Mode
 	InitializeFromCNI           bool
+	KeyVaultSettings            KeyVaultSettings
+	MSISettings                 MSISettings
+	ManageEndpointState         bool
 	ManagedSettings             ManagedSettings
+	MellanoxMonitorIntervalSecs int
 	MetricsBindAddress          string
+	ProgramSNATIPTables         bool
+	SWIFTV2Mode                 SWIFTV2Mode
 	SyncHostNCTimeoutMs         int
 	SyncHostNCVersionIntervalMs int
 	TLSCertificatePath          string
@@ -42,19 +54,8 @@ type CNSConfig struct {
 	TLSSubjectName              string
 	TelemetrySettings           TelemetrySettings
 	UseHTTPS                    bool
+	WatchPods                   bool `json:"-"`
 	WireserverIP                string
-	KeyVaultSettings            KeyVaultSettings
-	MSISettings                 MSISettings
-	ProgramSNATIPTables         bool
-	ManageEndpointState         bool
-	CNIConflistScenario         string
-	EnableCNIConflistGeneration bool
-	CNIConflistFilepath         string
-	MellanoxMonitorIntervalSecs int
-	AZRSettings                 AZRSettings
-	WatchPods                   bool
-	EnableAsyncPodDelete        bool
-	AsyncPodDeletePath          string
 }
 
 type TelemetrySettings struct {
@@ -219,4 +220,5 @@ func SetCNSConfigDefaults(config *CNSConfig) {
 	if config.AsyncPodDeletePath == "" {
 		config.AsyncPodDeletePath = "/var/run/azure-vnet/deleteIDs"
 	}
+	config.WatchPods = config.EnableIPAMv2 || config.EnableSwiftV2
 }

cns/fakes/cnsfake.go

@@ -115,6 +115,11 @@ func (ipm *IPStateManager) ReleaseIPConfig(ipconfigID string) (cns.IPConfigurati
 	return ipm.AvailableIPConfigState[ipconfigID], nil
 }
 
+func (ipm *IPStateManager) MarkNIPsPendingRelease(n int) (map[string]cns.IPConfigurationStatus, error) {
+	// MarkIPASPendingRelease actually already errors if it is unable to release all N IPs
+	return ipm.MarkIPAsPendingRelease(n)
+}
+
 func (ipm *IPStateManager) MarkIPAsPendingRelease(numberOfIPsToMark int) (map[string]cns.IPConfigurationStatus, error) {
 	ipm.Lock()
 	defer ipm.Unlock()
@@ -256,6 +261,10 @@ func (fake *HTTPServiceFake) GetPodIPConfigState() map[string]cns.IPConfiguratio
 	return ipconfigs
 }
 
+func (fake *HTTPServiceFake) MarkNIPsPendingRelease(n int) (map[string]cns.IPConfigurationStatus, error) {
+	return fake.IPStateManager.MarkIPAsPendingRelease(n)
+}
+
 // TODO: Populate on scale down
 func (fake *HTTPServiceFake) MarkIPAsPendingRelease(numberToMark int) (map[string]cns.IPConfigurationStatus, error) {
 	return fake.IPStateManager.MarkIPAsPendingRelease(numberToMark)

cns/ipampool/metrics.go

@@ -12,160 +12,160 @@ const (
 	customerMetricLabel        = "customer_metric"
 	customerMetricLabelValue   = "customer metric"
 	subnetExhaustionStateLabel = "subnet_exhaustion_state"
-	subnetIPExhausted          = 1
-	subnetIPNotExhausted       = 0
+	SubnetIPExhausted          = 1
+	SubnetIPNotExhausted       = 0
 )
 
 var (
-	ipamAllocatedIPCount = prometheus.NewGaugeVec(
+	IpamAllocatedIPCount = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name:        "cx_ipam_pod_allocated_ips",
 			Help:        "IPs currently in use by Pods on this CNS Node.",
 			ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
 		},
 		[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
 	)
-	ipamAvailableIPCount = prometheus.NewGaugeVec(
+	IpamAvailableIPCount = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name:        "cx_ipam_available_ips",
 			Help:        "IPs available on this CNS Node for use by a Pod.",
 			ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
 		},
 		[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
 	)
-	ipamBatchSize = prometheus.NewGaugeVec(
+	IpamBatchSize = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name:        "cx_ipam_batch_size",
 			Help:        "IPAM IP pool scaling batch size.",
 			ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
 		},
 		[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
 	)
-	ipamCurrentAvailableIPcount = prometheus.NewGaugeVec(
+	IpamCurrentAvailableIPcount = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name:        "cx_ipam_current_available_ips",
 			Help:        "Current available IP count.",
 			ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
 		},
 		[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
 	)
-	ipamExpectedAvailableIPCount = prometheus.NewGaugeVec(
+	IpamExpectedAvailableIPCount = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name:        "cx_ipam_expect_available_ips",
 			Help:        "Expected future available IP count assuming the Requested IP count is honored.",
 			ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
 		},
 		[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
 	)
-	ipamMaxIPCount = prometheus.NewGaugeVec(
+	IpamMaxIPCount = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name:        "cx_ipam_max_ips",
 			Help:        "Maximum Secondary IPs allowed on this Node.",
 			ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
 		},
 		[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
 	)
-	ipamPendingProgramIPCount = prometheus.NewGaugeVec(
+	IpamPendingProgramIPCount = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name:        "cx_ipam_pending_programming_ips",
 			Help:        "IPs reserved but not yet available (Pending Programming).",
 			ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
 		},
 		[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
 	)
-	ipamPendingReleaseIPCount = prometheus.NewGaugeVec(
+	IpamPendingReleaseIPCount = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name:        "cx_ipam_pending_release_ips",
 			Help:        "IPs reserved but not available anymore (Pending Release).",
 			ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
 		},
 		[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
 	)
-	ipamPrimaryIPCount = prometheus.NewGaugeVec(
+	IpamPrimaryIPCount = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name:        "cx_ipam_primary_ips",
 			Help:        "NC Primary IP count (reserved from Pod Subnet for DNS and IMDS SNAT).",
 			ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
 		},
 		[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
 	)
-	ipamRequestedIPConfigCount = prometheus.NewGaugeVec(
+	IpamRequestedIPConfigCount = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name:        "cx_ipam_requested_ips",
 			Help:        "Secondary Pod Subnet IPs requested by this CNS Node (for Pods).",
 			ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
 		},
 		[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
 	)
-	ipamSecondaryIPCount = prometheus.NewGaugeVec(
+	IpamSecondaryIPCount = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name:        "cx_ipam_secondary_ips",
 			Help:        "Node NC Secondary IP count (reserved usable by Pods).",
 			ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
 		},
 		[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
 	)
-	ipamSubnetExhaustionCount = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "cx_ipam_subnet_exhaustion_state_count_total",
-			Help: "Count of the number of times the ipam pool monitor sees subnet exhaustion",
-		},
-		[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel, subnetExhaustionStateLabel},
-	)
-	ipamSubnetExhaustionState = prometheus.NewGaugeVec(
+	IpamTotalIPCount = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name:        "cx_ipam_subnet_exhaustion_state",
-			Help:        "CNS view of subnet exhaustion state",
+			Name:        "cx_ipam_total_ips",
+			Help:        "Count of total IP pool size allocated to CNS by DNC.",
 			ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
 		},
 		[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
 	)
-	ipamTotalIPCount = prometheus.NewGaugeVec(
+	IpamSubnetExhaustionState = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name:        "cx_ipam_total_ips",
-			Help:        "Total IPs reserved from the Pod Subnet by this Node.",
+			Name:        "cx_ipam_subnet_exhaustion_state",
+			Help:        "IPAM view of subnet exhaustion state",
 			ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
 		},
 		[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
 	)
+	IpamSubnetExhaustionCount = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "cx_ipam_subnet_exhaustion_state_count_total",
+			Help: "Count of the number of times the ipam pool monitor sees subnet exhaustion",
+		},
+		[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel, subnetExhaustionStateLabel},
+	)
 )
 
 func init() {
 	metrics.Registry.MustRegister(
-		ipamAllocatedIPCount,
-		ipamAvailableIPCount,
-		ipamBatchSize,
-		ipamCurrentAvailableIPcount,
-		ipamExpectedAvailableIPCount,
-		ipamMaxIPCount,
-		ipamPendingProgramIPCount,
-		ipamPendingReleaseIPCount,
-		ipamPrimaryIPCount,
-		ipamRequestedIPConfigCount,
-		ipamSecondaryIPCount,
-		ipamSubnetExhaustionCount,
-		ipamSubnetExhaustionState,
-		ipamTotalIPCount,
+		IpamAllocatedIPCount,
+		IpamAvailableIPCount,
+		IpamBatchSize,
+		IpamCurrentAvailableIPcount,
+		IpamExpectedAvailableIPCount,
+		IpamMaxIPCount,
+		IpamPendingProgramIPCount,
+		IpamPendingReleaseIPCount,
+		IpamPrimaryIPCount,
+		IpamSecondaryIPCount,
+		IpamRequestedIPConfigCount,
+		IpamTotalIPCount,
+		IpamSubnetExhaustionState,
+		IpamSubnetExhaustionCount,
 	)
 }
 
 func observeIPPoolState(state ipPoolState, meta metaState) {
 	labels := []string{meta.subnet, meta.subnetCIDR, meta.subnetARMID}
-	ipamAllocatedIPCount.WithLabelValues(labels...).Set(float64(state.allocatedToPods))
-	ipamAvailableIPCount.WithLabelValues(labels...).Set(float64(state.available))
-	ipamBatchSize.WithLabelValues(labels...).Set(float64(meta.batch))
-	ipamCurrentAvailableIPcount.WithLabelValues(labels...).Set(float64(state.currentAvailableIPs))
-	ipamExpectedAvailableIPCount.WithLabelValues(labels...).Set(float64(state.expectedAvailableIPs))
-	ipamMaxIPCount.WithLabelValues(labels...).Set(float64(meta.max))
-	ipamPendingProgramIPCount.WithLabelValues(labels...).Set(float64(state.pendingProgramming))
-	ipamPendingReleaseIPCount.WithLabelValues(labels...).Set(float64(state.pendingRelease))
-	ipamPrimaryIPCount.WithLabelValues(labels...).Set(float64(len(meta.primaryIPAddresses)))
-	ipamRequestedIPConfigCount.WithLabelValues(labels...).Set(float64(state.requestedIPs))
-	ipamSecondaryIPCount.WithLabelValues(labels...).Set(float64(state.secondaryIPs))
-	ipamTotalIPCount.WithLabelValues(labels...).Set(float64(state.secondaryIPs + int64(len(meta.primaryIPAddresses))))
+	IpamAllocatedIPCount.WithLabelValues(labels...).Set(float64(state.allocatedToPods))
+	IpamAvailableIPCount.WithLabelValues(labels...).Set(float64(state.available))
+	IpamBatchSize.WithLabelValues(labels...).Set(float64(meta.batch))
+	IpamCurrentAvailableIPcount.WithLabelValues(labels...).Set(float64(state.currentAvailableIPs))
+	IpamExpectedAvailableIPCount.WithLabelValues(labels...).Set(float64(state.expectedAvailableIPs))
+	IpamMaxIPCount.WithLabelValues(labels...).Set(float64(meta.max))
+	IpamPendingProgramIPCount.WithLabelValues(labels...).Set(float64(state.pendingProgramming))
+	IpamPendingReleaseIPCount.WithLabelValues(labels...).Set(float64(state.pendingRelease))
+	IpamPrimaryIPCount.WithLabelValues(labels...).Set(float64(len(meta.primaryIPAddresses)))
+	IpamRequestedIPConfigCount.WithLabelValues(labels...).Set(float64(state.requestedIPs))
+	IpamSecondaryIPCount.WithLabelValues(labels...).Set(float64(state.secondaryIPs))
+	IpamTotalIPCount.WithLabelValues(labels...).Set(float64(state.secondaryIPs + int64(len(meta.primaryIPAddresses))))
 	if meta.exhausted {
-		ipamSubnetExhaustionState.WithLabelValues(labels...).Set(float64(subnetIPExhausted))
+		IpamSubnetExhaustionState.WithLabelValues(labels...).Set(float64(SubnetIPExhausted))
 	} else {
-		ipamSubnetExhaustionState.WithLabelValues(labels...).Set(float64(subnetIPNotExhausted))
+		IpamSubnetExhaustionState.WithLabelValues(labels...).Set(float64(SubnetIPNotExhausted))
 	}
 }

cns/ipampool/monitor.go

@@ -105,7 +105,7 @@ func (pm *Monitor) Start(ctx context.Context) error {
 		case css := <-pm.cssSource: // received an updated ClusterSubnetState
 			pm.metastate.exhausted = css.Status.Exhausted
 			logger.Printf("subnet exhausted status = %t", pm.metastate.exhausted)
-			ipamSubnetExhaustionCount.With(prometheus.Labels{
+			IpamSubnetExhaustionCount.With(prometheus.Labels{
 				subnetLabel: pm.metastate.subnet, subnetCIDRLabel: pm.metastate.subnetCIDR,
 				podnetARMIDLabel: pm.metastate.subnetARMID, subnetExhaustionStateLabel: strconv.FormatBool(pm.metastate.exhausted),
 			}).Inc()

cns/ipampool/v2/adapter.go

@@ -0,0 +1,36 @@
+package v2
+
+import (
+	"github.com/Azure/azure-container-networking/cns"
+	"github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha"
+	v1 "k8s.io/api/core/v1"
+)
+
+var _ cns.IPAMPoolMonitor = (*adapter)(nil)
+
+type adapter struct {
+	nncSink chan<- v1alpha.NodeNetworkConfig
+	*Monitor
+}
+
+func (m *Monitor) AsV1(nncSink chan<- v1alpha.NodeNetworkConfig) cns.IPAMPoolMonitor {
+	return &adapter{
+		nncSink: nncSink,
+		Monitor: m,
+	}
+}
+
+func (m *adapter) Update(nnc *v1alpha.NodeNetworkConfig) error {
+	m.nncSink <- *nnc
+	return nil
+}
+
+func (m *adapter) GetStateSnapshot() cns.IpamPoolMonitorStateSnapshot {
+	return cns.IpamPoolMonitorStateSnapshot{}
+}
+
+func PodIPDemandListener(ch chan<- int) func([]v1.Pod) {
+	return func(pods []v1.Pod) {
+		ch <- len(pods)
+	}
+}