Skip to content

Commit 1b01e20

Browse files
rbtrrbtr
committed
feat: v2 ipampool with idempotent scaling math
1 parent 7169036 commit 1b01e20

File tree

13 files changed

+928
-140
lines changed

13 files changed

+928
-140
lines changed

‎cns/api.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ type HTTPService interface {
5050
GetPodIPConfigState() map[string]IPConfigurationStatus
5151
MarkIPAsPendingRelease(numberToMark int) (map[string]IPConfigurationStatus, error)
5252
AttachSWIFTv2Middleware(middleware SWIFTv2Middleware)
53+
MarkNIPsPendingRelease(n int) (map[string]IPConfigurationStatus, error)
5354
}
5455

5556
// Middleware interface for testing later on
@@ -282,13 +283,13 @@ type NodeConfiguration struct {
282283
NodeSubnet Subnet
283284
}
284285

286+
// IpamPoolMonitorStateSnapshot struct to expose state values for IPAMPoolMonitor struct
285287
type IPAMPoolMonitor interface {
286288
Start(ctx context.Context) error
287289
Update(nnc *v1alpha.NodeNetworkConfig) error
288290
GetStateSnapshot() IpamPoolMonitorStateSnapshot
289291
}
290292

291-
// IpamPoolMonitorStateSnapshot struct to expose state values for IPAMPoolMonitor struct
292293
type IpamPoolMonitorStateSnapshot struct {
293294
MinimumFreeIps int64
294295
MaximumFreeIps int64

‎cns/configuration/configuration.go‎

Lines changed: 31 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -20,34 +20,36 @@ const (
2020
)
2121

2222
type CNSConfig struct {
23-
ChannelMode string
24-
EnablePprof bool
25-
EnableSubnetScarcity bool
26-
EnableSwiftV2 bool
27-
InitializeFromCNI bool
28-
ManagedSettings ManagedSettings
29-
MetricsBindAddress string
30-
SyncHostNCTimeoutMs int
31-
SyncHostNCVersionIntervalMs int
32-
TLSCertificatePath string
33-
TLSEndpoint string
34-
TLSPort string
35-
TLSSubjectName string
36-
TelemetrySettings TelemetrySettings
37-
UseHTTPS bool
38-
WireserverIP string
39-
KeyVaultSettings KeyVaultSettings
40-
MSISettings MSISettings
41-
ProgramSNATIPTables bool
42-
ManageEndpointState bool
43-
CNIConflistScenario string
44-
EnableCNIConflistGeneration bool
45-
CNIConflistFilepath string
46-
MellanoxMonitorIntervalSecs int
47-
AZRSettings AZRSettings
48-
WatchPods bool
49-
EnableAsyncPodDelete bool
50-
AsyncPodDeletePath string
23+
AZRSettings AZRSettings
24+
AsyncPodDeletePath string
25+
CNIConflistFilepath string
26+
CNIConflistScenario string
27+
ChannelMode string
28+
EnableAsyncPodDelete bool
29+
EnableCNIConflistGeneration bool
30+
EnablePprof bool
31+
EnableSubnetScarcity bool
32+
EnableSwiftV2 bool
33+
EnableIPAMv2 bool
34+
InitializeFromCNI bool
35+
KeyVaultSettings KeyVaultSettings
36+
MSISettings MSISettings
37+
ManageEndpointState bool
38+
ManagedSettings ManagedSettings
39+
MellanoxMonitorIntervalSecs int
40+
MetricsBindAddress string
41+
PopulateHomeAzCacheRetryIntervalSecs int
42+
ProgramSNATIPTables bool
43+
SyncHostNCTimeoutMs int
44+
SyncHostNCVersionIntervalMs int
45+
TLSCertificatePath string
46+
TLSEndpoint string
47+
TLSPort string
48+
TLSSubjectName string
49+
TelemetrySettings TelemetrySettings
50+
UseHTTPS bool
51+
WatchPods bool `json:"-"`
52+
WireserverIP string
5153
}
5254

5355
type TelemetrySettings struct {
@@ -212,4 +214,5 @@ func SetCNSConfigDefaults(config *CNSConfig) {
212214
if config.AsyncPodDeletePath == "" {
213215
config.AsyncPodDeletePath = "/var/run/azure-vnet/deleteIDs"
214216
}
217+
config.WatchPods = config.EnableIPAMv2 || config.EnableSwiftV2
215218
}

‎cns/fakes/cnsfake.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,11 @@ func (ipm *IPStateManager) ReleaseIPConfig(ipconfigID string) (cns.IPConfigurati
115115
return ipm.AvailableIPConfigState[ipconfigID], nil
116116
}
117117

118+
func (ipm *IPStateManager) MarkNIPsPendingRelease(n int) (map[string]cns.IPConfigurationStatus, error) {
119+
// MarkIPASPendingRelease actually already errors if it is unable to release all N IPs
120+
return ipm.MarkIPAsPendingRelease(n)
121+
}
122+
118123
func (ipm *IPStateManager) MarkIPAsPendingRelease(numberOfIPsToMark int) (map[string]cns.IPConfigurationStatus, error) {
119124
ipm.Lock()
120125
defer ipm.Unlock()
@@ -256,6 +261,10 @@ func (fake *HTTPServiceFake) GetPodIPConfigState() map[string]cns.IPConfiguratio
256261
return ipconfigs
257262
}
258263

264+
func (fake *HTTPServiceFake) MarkNIPsPendingRelease(n int) (map[string]cns.IPConfigurationStatus, error) {
265+
return fake.IPStateManager.MarkIPAsPendingRelease(n)
266+
}
267+
259268
// TODO: Populate on scale down
260269
func (fake *HTTPServiceFake) MarkIPAsPendingRelease(numberToMark int) (map[string]cns.IPConfigurationStatus, error) {
261270
return fake.IPStateManager.MarkIPAsPendingRelease(numberToMark)

‎cns/ipampool/metrics.go‎

Lines changed: 54 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -12,160 +12,160 @@ const (
1212
customerMetricLabel = "customer_metric"
1313
customerMetricLabelValue = "customer metric"
1414
subnetExhaustionStateLabel = "subnet_exhaustion_state"
15-
subnetIPExhausted = 1
16-
subnetIPNotExhausted = 0
15+
SubnetIPExhausted = 1
16+
SubnetIPNotExhausted = 0
1717
)
1818

1919
var (
20-
ipamAllocatedIPCount = prometheus.NewGaugeVec(
20+
IpamAllocatedIPCount = prometheus.NewGaugeVec(
2121
prometheus.GaugeOpts{
2222
Name: "cx_ipam_pod_allocated_ips",
2323
Help: "IPs currently in use by Pods on this CNS Node.",
2424
ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
2525
},
2626
[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
2727
)
28-
ipamAvailableIPCount = prometheus.NewGaugeVec(
28+
IpamAvailableIPCount = prometheus.NewGaugeVec(
2929
prometheus.GaugeOpts{
3030
Name: "cx_ipam_available_ips",
3131
Help: "IPs available on this CNS Node for use by a Pod.",
3232
ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
3333
},
3434
[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
3535
)
36-
ipamBatchSize = prometheus.NewGaugeVec(
36+
IpamBatchSize = prometheus.NewGaugeVec(
3737
prometheus.GaugeOpts{
3838
Name: "cx_ipam_batch_size",
3939
Help: "IPAM IP pool scaling batch size.",
4040
ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
4141
},
4242
[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
4343
)
44-
ipamCurrentAvailableIPcount = prometheus.NewGaugeVec(
44+
IpamCurrentAvailableIPcount = prometheus.NewGaugeVec(
4545
prometheus.GaugeOpts{
4646
Name: "cx_ipam_current_available_ips",
4747
Help: "Current available IP count.",
4848
ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
4949
},
5050
[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
5151
)
52-
ipamExpectedAvailableIPCount = prometheus.NewGaugeVec(
52+
IpamExpectedAvailableIPCount = prometheus.NewGaugeVec(
5353
prometheus.GaugeOpts{
5454
Name: "cx_ipam_expect_available_ips",
5555
Help: "Expected future available IP count assuming the Requested IP count is honored.",
5656
ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
5757
},
5858
[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
5959
)
60-
ipamMaxIPCount = prometheus.NewGaugeVec(
60+
IpamMaxIPCount = prometheus.NewGaugeVec(
6161
prometheus.GaugeOpts{
6262
Name: "cx_ipam_max_ips",
6363
Help: "Maximum Secondary IPs allowed on this Node.",
6464
ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
6565
},
6666
[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
6767
)
68-
ipamPendingProgramIPCount = prometheus.NewGaugeVec(
68+
IpamPendingProgramIPCount = prometheus.NewGaugeVec(
6969
prometheus.GaugeOpts{
7070
Name: "cx_ipam_pending_programming_ips",
7171
Help: "IPs reserved but not yet available (Pending Programming).",
7272
ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
7373
},
7474
[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
7575
)
76-
ipamPendingReleaseIPCount = prometheus.NewGaugeVec(
76+
IpamPendingReleaseIPCount = prometheus.NewGaugeVec(
7777
prometheus.GaugeOpts{
7878
Name: "cx_ipam_pending_release_ips",
7979
Help: "IPs reserved but not available anymore (Pending Release).",
8080
ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
8181
},
8282
[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
8383
)
84-
ipamPrimaryIPCount = prometheus.NewGaugeVec(
84+
IpamPrimaryIPCount = prometheus.NewGaugeVec(
8585
prometheus.GaugeOpts{
8686
Name: "cx_ipam_primary_ips",
8787
Help: "NC Primary IP count (reserved from Pod Subnet for DNS and IMDS SNAT).",
8888
ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
8989
},
9090
[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
9191
)
92-
ipamRequestedIPConfigCount = prometheus.NewGaugeVec(
92+
IpamRequestedIPConfigCount = prometheus.NewGaugeVec(
9393
prometheus.GaugeOpts{
9494
Name: "cx_ipam_requested_ips",
9595
Help: "Secondary Pod Subnet IPs requested by this CNS Node (for Pods).",
9696
ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
9797
},
9898
[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
9999
)
100-
ipamSecondaryIPCount = prometheus.NewGaugeVec(
100+
IpamSecondaryIPCount = prometheus.NewGaugeVec(
101101
prometheus.GaugeOpts{
102102
Name: "cx_ipam_secondary_ips",
103103
Help: "Node NC Secondary IP count (reserved usable by Pods).",
104104
ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
105105
},
106106
[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
107107
)
108-
ipamSubnetExhaustionCount = prometheus.NewCounterVec(
109-
prometheus.CounterOpts{
110-
Name: "cx_ipam_subnet_exhaustion_state_count_total",
111-
Help: "Count of the number of times the ipam pool monitor sees subnet exhaustion",
112-
},
113-
[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel, subnetExhaustionStateLabel},
114-
)
115-
ipamSubnetExhaustionState = prometheus.NewGaugeVec(
108+
IpamTotalIPCount = prometheus.NewGaugeVec(
116109
prometheus.GaugeOpts{
117-
Name: "cx_ipam_subnet_exhaustion_state",
118-
Help: "CNS view of subnet exhaustion state",
110+
Name: "cx_ipam_total_ips",
111+
Help: "Count of total IP pool size allocated to CNS by DNC.",
119112
ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
120113
},
121114
[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
122115
)
123-
ipamTotalIPCount = prometheus.NewGaugeVec(
116+
IpamSubnetExhaustionState = prometheus.NewGaugeVec(
124117
prometheus.GaugeOpts{
125-
Name: "cx_ipam_total_ips",
126-
Help: "Total IPs reserved from the Pod Subnet by this Node.",
118+
Name: "cx_ipam_subnet_exhaustion_state",
119+
Help: "IPAM view of subnet exhaustion state",
127120
ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue},
128121
},
129122
[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel},
130123
)
124+
IpamSubnetExhaustionCount = prometheus.NewCounterVec(
125+
prometheus.CounterOpts{
126+
Name: "cx_ipam_subnet_exhaustion_state_count_total",
127+
Help: "Count of the number of times the ipam pool monitor sees subnet exhaustion",
128+
},
129+
[]string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel, subnetExhaustionStateLabel},
130+
)
131131
)
132132

133133
func init() {
134134
metrics.Registry.MustRegister(
135-
ipamAllocatedIPCount,
136-
ipamAvailableIPCount,
137-
ipamBatchSize,
138-
ipamCurrentAvailableIPcount,
139-
ipamExpectedAvailableIPCount,
140-
ipamMaxIPCount,
141-
ipamPendingProgramIPCount,
142-
ipamPendingReleaseIPCount,
143-
ipamPrimaryIPCount,
144-
ipamRequestedIPConfigCount,
145-
ipamSecondaryIPCount,
146-
ipamSubnetExhaustionCount,
147-
ipamSubnetExhaustionState,
148-
ipamTotalIPCount,
135+
IpamAllocatedIPCount,
136+
IpamAvailableIPCount,
137+
IpamBatchSize,
138+
IpamCurrentAvailableIPcount,
139+
IpamExpectedAvailableIPCount,
140+
IpamMaxIPCount,
141+
IpamPendingProgramIPCount,
142+
IpamPendingReleaseIPCount,
143+
IpamPrimaryIPCount,
144+
IpamSecondaryIPCount,
145+
IpamRequestedIPConfigCount,
146+
IpamTotalIPCount,
147+
IpamSubnetExhaustionState,
148+
IpamSubnetExhaustionCount,
149149
)
150150
}
151151

152152
func observeIPPoolState(state ipPoolState, meta metaState) {
153153
labels := []string{meta.subnet, meta.subnetCIDR, meta.subnetARMID}
154-
ipamAllocatedIPCount.WithLabelValues(labels...).Set(float64(state.allocatedToPods))
155-
ipamAvailableIPCount.WithLabelValues(labels...).Set(float64(state.available))
156-
ipamBatchSize.WithLabelValues(labels...).Set(float64(meta.batch))
157-
ipamCurrentAvailableIPcount.WithLabelValues(labels...).Set(float64(state.currentAvailableIPs))
158-
ipamExpectedAvailableIPCount.WithLabelValues(labels...).Set(float64(state.expectedAvailableIPs))
159-
ipamMaxIPCount.WithLabelValues(labels...).Set(float64(meta.max))
160-
ipamPendingProgramIPCount.WithLabelValues(labels...).Set(float64(state.pendingProgramming))
161-
ipamPendingReleaseIPCount.WithLabelValues(labels...).Set(float64(state.pendingRelease))
162-
ipamPrimaryIPCount.WithLabelValues(labels...).Set(float64(len(meta.primaryIPAddresses)))
163-
ipamRequestedIPConfigCount.WithLabelValues(labels...).Set(float64(state.requestedIPs))
164-
ipamSecondaryIPCount.WithLabelValues(labels...).Set(float64(state.secondaryIPs))
165-
ipamTotalIPCount.WithLabelValues(labels...).Set(float64(state.secondaryIPs + int64(len(meta.primaryIPAddresses))))
154+
IpamAllocatedIPCount.WithLabelValues(labels...).Set(float64(state.allocatedToPods))
155+
IpamAvailableIPCount.WithLabelValues(labels...).Set(float64(state.available))
156+
IpamBatchSize.WithLabelValues(labels...).Set(float64(meta.batch))
157+
IpamCurrentAvailableIPcount.WithLabelValues(labels...).Set(float64(state.currentAvailableIPs))
158+
IpamExpectedAvailableIPCount.WithLabelValues(labels...).Set(float64(state.expectedAvailableIPs))
159+
IpamMaxIPCount.WithLabelValues(labels...).Set(float64(meta.max))
160+
IpamPendingProgramIPCount.WithLabelValues(labels...).Set(float64(state.pendingProgramming))
161+
IpamPendingReleaseIPCount.WithLabelValues(labels...).Set(float64(state.pendingRelease))
162+
IpamPrimaryIPCount.WithLabelValues(labels...).Set(float64(len(meta.primaryIPAddresses)))
163+
IpamRequestedIPConfigCount.WithLabelValues(labels...).Set(float64(state.requestedIPs))
164+
IpamSecondaryIPCount.WithLabelValues(labels...).Set(float64(state.secondaryIPs))
165+
IpamTotalIPCount.WithLabelValues(labels...).Set(float64(state.secondaryIPs + int64(len(meta.primaryIPAddresses))))
166166
if meta.exhausted {
167-
ipamSubnetExhaustionState.WithLabelValues(labels...).Set(float64(subnetIPExhausted))
167+
IpamSubnetExhaustionState.WithLabelValues(labels...).Set(float64(SubnetIPExhausted))
168168
} else {
169-
ipamSubnetExhaustionState.WithLabelValues(labels...).Set(float64(subnetIPNotExhausted))
169+
IpamSubnetExhaustionState.WithLabelValues(labels...).Set(float64(SubnetIPNotExhausted))
170170
}
171171
}

‎cns/ipampool/monitor.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ func (pm *Monitor) Start(ctx context.Context) error {
105105
case css := <-pm.cssSource: // received an updated ClusterSubnetState
106106
pm.metastate.exhausted = css.Status.Exhausted
107107
logger.Printf("subnet exhausted status = %t", pm.metastate.exhausted)
108-
ipamSubnetExhaustionCount.With(prometheus.Labels{
108+
IpamSubnetExhaustionCount.With(prometheus.Labels{
109109
subnetLabel: pm.metastate.subnet, subnetCIDRLabel: pm.metastate.subnetCIDR,
110110
podnetARMIDLabel: pm.metastate.subnetARMID, subnetExhaustionStateLabel: strconv.FormatBool(pm.metastate.exhausted),
111111
}).Inc()

‎cns/ipampool/v2/adapter.go‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
package v2
2+
3+
import (
4+
"github.com/Azure/azure-container-networking/cns"
5+
"github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha"
6+
v1 "k8s.io/api/core/v1"
7+
)
8+
9+
var _ cns.IPAMPoolMonitor = (*adapter)(nil)
10+
11+
type adapter struct {
12+
nncSink chan<- v1alpha.NodeNetworkConfig
13+
*Monitor
14+
}
15+
16+
func (m *Monitor) AsV1(nncSink chan<- v1alpha.NodeNetworkConfig) cns.IPAMPoolMonitor {
17+
return &adapter{
18+
nncSink: nncSink,
19+
Monitor: m,
20+
}
21+
}
22+
23+
func (m *adapter) Update(nnc *v1alpha.NodeNetworkConfig) error {
24+
m.nncSink <- *nnc
25+
return nil
26+
}
27+
28+
func (m *adapter) GetStateSnapshot() cns.IpamPoolMonitorStateSnapshot {
29+
return cns.IpamPoolMonitorStateSnapshot{}
30+
}
31+
32+
func PodIPDemandListener(ch chan<- int) func([]v1.Pod) {
33+
return func(pods []v1.Pod) {
34+
ch <- len(pods)
35+
}
36+
}

0 commit comments

Comments
 (0)