From feba3d72160cd41ce0e0fc35497baef3bcb5d27a Mon Sep 17 00:00:00 2001 From: rsagasthya Date: Mon, 28 Feb 2022 18:13:22 +0000 Subject: [PATCH 1/9] Added the Pod Net ARM ID as a Label. Added Pod Net Arm Id as labels to the CNS Metrics. --- cns/ipampool/metrics.go | 25 +++++++++++++------------ cns/ipampool/monitor.go | 11 +++++++---- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/cns/ipampool/metrics.go b/cns/ipampool/metrics.go index 77059401e5..e87e10dbab 100644 --- a/cns/ipampool/metrics.go +++ b/cns/ipampool/metrics.go @@ -6,8 +6,9 @@ import ( ) const ( - subnetLabel = "subnet" - subnetCIDRLabel = "subnet_cidr" + subnetLabel = "subnet" + subnetCIDRLabel = "subnet_cidr" + podnetARMIDLabel = "podnet_arm_id" ) var ( @@ -16,70 +17,70 @@ var ( Name: "ipam_pod_allocated_ips", Help: "Count of IPs CNS has allocated to Pods.", }, - []string{subnetLabel, subnetCIDRLabel}, + []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamAvailableIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "ipam_available_ips", Help: "Available IP count.", }, - []string{subnetLabel, subnetCIDRLabel}, + []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamBatchSize = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "ipam_batch_size", Help: "IPAM IP pool batch size.", }, - []string{subnetLabel, subnetCIDRLabel}, + []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamCurrentAvailableIPcount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "ipam_current_available_ips", Help: "Current available IP count.", }, - []string{subnetLabel, subnetCIDRLabel}, + []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamExpectedAvailableIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "ipam_expect_available_ips", Help: "Expected future available IP count assuming the Requested IP count is honored.", }, - []string{subnetLabel, subnetCIDRLabel}, + []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamMaxIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "ipam_max_ips", Help: "Maximum IP count.", }, - []string{subnetLabel, subnetCIDRLabel}, + []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamPendingProgramIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "ipam_pending_programming_ips", Help: "Pending programming IP count.", }, - []string{subnetLabel, subnetCIDRLabel}, + []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamPendingReleaseIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "ipam_pending_release_ips", Help: "Pending release IP count.", }, - []string{subnetLabel, subnetCIDRLabel}, + []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamRequestedIPConfigCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "ipam_requested_ips", Help: "Requested IP count.", }, - []string{subnetLabel, subnetCIDRLabel}, + []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamTotalIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "ipam_total_ips", Help: "Count of total IP pool size allocated to CNS by DNC.", }, - []string{subnetLabel, subnetCIDRLabel}, + []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ) diff --git a/cns/ipampool/monitor.go b/cns/ipampool/monitor.go index e8a1b7ac85..7a708f1cf1 100644 --- a/cns/ipampool/monitor.go +++ b/cns/ipampool/monitor.go @@ -18,6 +18,8 @@ const ( DefaultRefreshDelay = 1 * time.Second // DefaultMaxIPs default maximum allocatable IPs DefaultMaxIPs = 250 + // Pod Network ARM ID Key for NNC Labels + PodSubnetARMIDLabel = "kubernetes.azure.com/podnetwork-arm-id" ) type nodeNetworkConfigSpecUpdater interface { @@ -49,8 +51,8 @@ type Monitor struct { once sync.Once } -// Global Variables for Subnet and SubnetAddressSpace -var subnet, subnetCIDR string +// Global Variables for Subnet, Subnet Address Space and Pod Network ARM ID +var subnet, subnetCIDR, subnetARMID string func NewMonitor(httpService cns.HTTPService, nnccli nodeNetworkConfigSpecUpdater, opts *Options) *Monitor { if opts.RefreshDelay < 1 { @@ -94,9 +96,10 @@ func (pm *Monitor) Start(ctx context.Context) error { pm.spec = nnc.Spec scaler := nnc.Status.Scaler - // Set SubnetName and SubnetAddressSpace values to the global subnet and subnetCIDR variables. + // Set SubnetName, SubnetAddressSpace and Pod Network ARM ID values to the global subnet, subnetCIDR and subnetARM variables. subnet = nnc.Status.NetworkContainers[0].SubnetName subnetCIDR = nnc.Status.NetworkContainers[0].SubnetAddressSpace + subnetARMID = nnc.Labels[PodSubnetARMIDLabel] pm.metastate.batch = scaler.BatchSize pm.metastate.max = scaler.MaxIPCount @@ -157,7 +160,7 @@ func (pm *Monitor) reconcile(ctx context.Context) error { allocatedIPs := pm.httpService.GetPodIPConfigState() state := buildIPPoolState(allocatedIPs, pm.spec) logger.Printf("ipam-pool-monitor state %+v", state) - observeIPPoolState(state, pm.metastate, []string{subnet, subnetCIDR}) + observeIPPoolState(state, pm.metastate, []string{subnet, subnetCIDR, subnetARMID}) switch { // pod count is increasing From e817c95ff9278c1acfb350c655cff7a91dca647b Mon Sep 17 00:00:00 2001 From: Rahul Agasthya Date: Mon, 7 Mar 2022 15:04:33 -0800 Subject: [PATCH 2/9] CNS Metrics: ARM ID as Label Necessary changes to the read the ARM ID metadata coming from DNC. --- cns/ipampool/monitor.go | 21 ++++++++++++++++--- .../api/v1alpha/nodenetworkconfig.go | 4 ++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/cns/ipampool/monitor.go b/cns/ipampool/monitor.go index 7a708f1cf1..7d2ed32ee1 100644 --- a/cns/ipampool/monitor.go +++ b/cns/ipampool/monitor.go @@ -2,6 +2,7 @@ package ipampool import ( "context" + "fmt" "sync" "time" @@ -18,8 +19,8 @@ const ( DefaultRefreshDelay = 1 * time.Second // DefaultMaxIPs default maximum allocatable IPs DefaultMaxIPs = 250 - // Pod Network ARM ID Key for NNC Labels - PodSubnetARMIDLabel = "kubernetes.azure.com/podnetwork-arm-id" + // Subnet ARM ID /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/$(subnet) + subnetARMIDTemplate = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s/subnets/%s" ) type nodeNetworkConfigSpecUpdater interface { @@ -99,7 +100,7 @@ func (pm *Monitor) Start(ctx context.Context) error { // Set SubnetName, SubnetAddressSpace and Pod Network ARM ID values to the global subnet, subnetCIDR and subnetARM variables. subnet = nnc.Status.NetworkContainers[0].SubnetName subnetCIDR = nnc.Status.NetworkContainers[0].SubnetAddressSpace - subnetARMID = nnc.Labels[PodSubnetARMIDLabel] + subnetARMID = GenerateARMID(nnc) pm.metastate.batch = scaler.BatchSize pm.metastate.max = scaler.MaxIPCount @@ -346,6 +347,20 @@ func (pm *Monitor) GetStateSnapshot() cns.IpamPoolMonitorStateSnapshot { } } +// GenerateARMID uses the Subnet ARM ID format to populate the ARM ID with the metadata. +// If either of the metadata attributes are empty, then the ARM ID will be an empty string. +func GenerateARMID(nnc v1alpha.NodeNetworkConfig) string { + subscription := nnc.Status.NetworkContainers[0].SubscriptionID + resourceGroup := nnc.Status.NetworkContainers[0].ResourceGroupID + vnetId := nnc.Status.NetworkContainers[0].VNETID + subnetId := nnc.Status.NetworkContainers[0].SubnetId + + if subscription == "" || resourceGroup == "" || vnetId == "" || subnetId == "" { + return "" + } + return fmt.Sprintf(subnetARMIDTemplate, subscription, resourceGroup, vnetId, subnetId) +} + // Update ingests a NodeNetworkConfig, clamping some values to ensure they are legal and then // pushing it to the Monitor's source channel. // If the Monitor has been Started but is blocking until it receives an NNC, this will start diff --git a/crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go b/crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go index afdcc0662f..0badc6eb3d 100644 --- a/crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go +++ b/crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go @@ -81,6 +81,10 @@ type NetworkContainer struct { SubnetAddressSpace string `json:"subnetAddressSpace,omitempty"` Version int64 `json:"version,omitempty"` NodeIP string `json:"nodeIP,omitempty"` + SubscriptionID string `json:"subcriptionId,omitempty"` + ResourceGroupID string `json:"resourceGroupId,omitempty"` + VNETID string `json:"vnetId,omitempty"` + SubnetId string `json:"subnetId,omitempty"` } // IPAssignment groups an IP address and Name. Name is a UUID set by the the IP address assigner. From 028d2ddca83c08f0ff48f101090175699ca7ec8c Mon Sep 17 00:00:00 2001 From: Rahul Agasthya Date: Wed, 9 Mar 2022 08:55:13 -0800 Subject: [PATCH 3/9] Subtle changes in NNC Variable names had to be modified to meet lint standards. --- crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go b/crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go index 0badc6eb3d..eb4f9ab538 100644 --- a/crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go +++ b/crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go @@ -83,8 +83,8 @@ type NetworkContainer struct { NodeIP string `json:"nodeIP,omitempty"` SubscriptionID string `json:"subcriptionId,omitempty"` ResourceGroupID string `json:"resourceGroupId,omitempty"` - VNETID string `json:"vnetId,omitempty"` - SubnetId string `json:"subnetId,omitempty"` + VNETID string `json:"vnetID,omitempty"` + SubnetID string `json:"subnetID,omitempty"` } // IPAssignment groups an IP address and Name. Name is a UUID set by the the IP address assigner. From 9c37d0ef7aa9f20c171e8bece52832d44b60c34c Mon Sep 17 00:00:00 2001 From: Rahul Agasthya Date: Wed, 9 Mar 2022 08:55:13 -0800 Subject: [PATCH 4/9] Subtle changes in NNC Variable names had to be modified to meet lint standards. --- cns/ipampool/monitor.go | 2 +- crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cns/ipampool/monitor.go b/cns/ipampool/monitor.go index 7d2ed32ee1..726617982e 100644 --- a/cns/ipampool/monitor.go +++ b/cns/ipampool/monitor.go @@ -353,7 +353,7 @@ func GenerateARMID(nnc v1alpha.NodeNetworkConfig) string { subscription := nnc.Status.NetworkContainers[0].SubscriptionID resourceGroup := nnc.Status.NetworkContainers[0].ResourceGroupID vnetId := nnc.Status.NetworkContainers[0].VNETID - subnetId := nnc.Status.NetworkContainers[0].SubnetId + subnetId := nnc.Status.NetworkContainers[0].SubnetID if subscription == "" || resourceGroup == "" || vnetId == "" || subnetId == "" { return "" diff --git a/crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go b/crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go index 0badc6eb3d..eb4f9ab538 100644 --- a/crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go +++ b/crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go @@ -83,8 +83,8 @@ type NetworkContainer struct { NodeIP string `json:"nodeIP,omitempty"` SubscriptionID string `json:"subcriptionId,omitempty"` ResourceGroupID string `json:"resourceGroupId,omitempty"` - VNETID string `json:"vnetId,omitempty"` - SubnetId string `json:"subnetId,omitempty"` + VNETID string `json:"vnetID,omitempty"` + SubnetID string `json:"subnetID,omitempty"` } // IPAssignment groups an IP address and Name. Name is a UUID set by the the IP address assigner. From d914bc5668c7ae89a28e94dedf4f79f5b7b27e9e Mon Sep 17 00:00:00 2001 From: Rahul Agasthya Date: Wed, 9 Mar 2022 09:46:57 -0800 Subject: [PATCH 5/9] Made NNC arg a Pointer NNC that was passed to create the ARM ID was too heavy. Converted that argument to a pointer and passed the reference of the NNC in that method. --- cns/ipampool/monitor.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cns/ipampool/monitor.go b/cns/ipampool/monitor.go index 726617982e..963d39ce7a 100644 --- a/cns/ipampool/monitor.go +++ b/cns/ipampool/monitor.go @@ -100,7 +100,7 @@ func (pm *Monitor) Start(ctx context.Context) error { // Set SubnetName, SubnetAddressSpace and Pod Network ARM ID values to the global subnet, subnetCIDR and subnetARM variables. subnet = nnc.Status.NetworkContainers[0].SubnetName subnetCIDR = nnc.Status.NetworkContainers[0].SubnetAddressSpace - subnetARMID = GenerateARMID(nnc) + subnetARMID = GenerateARMID(&nnc) pm.metastate.batch = scaler.BatchSize pm.metastate.max = scaler.MaxIPCount @@ -349,16 +349,16 @@ func (pm *Monitor) GetStateSnapshot() cns.IpamPoolMonitorStateSnapshot { // GenerateARMID uses the Subnet ARM ID format to populate the ARM ID with the metadata. // If either of the metadata attributes are empty, then the ARM ID will be an empty string. -func GenerateARMID(nnc v1alpha.NodeNetworkConfig) string { +func GenerateARMID(nnc *v1alpha.NodeNetworkConfig) string { subscription := nnc.Status.NetworkContainers[0].SubscriptionID resourceGroup := nnc.Status.NetworkContainers[0].ResourceGroupID - vnetId := nnc.Status.NetworkContainers[0].VNETID - subnetId := nnc.Status.NetworkContainers[0].SubnetID + vnetID := nnc.Status.NetworkContainers[0].VNETID + subnetID := nnc.Status.NetworkContainers[0].SubnetID - if subscription == "" || resourceGroup == "" || vnetId == "" || subnetId == "" { + if subscription == "" || resourceGroup == "" || vnetID == "" || subnetID == "" { return "" } - return fmt.Sprintf(subnetARMIDTemplate, subscription, resourceGroup, vnetId, subnetId) + return fmt.Sprintf(subnetARMIDTemplate, subscription, resourceGroup, vnetID, subnetID) } // Update ingests a NodeNetworkConfig, clamping some values to ensure they are legal and then From 7d75ebdc188eb495b37ddbf7f2c2e739f637c13b Mon Sep 17 00:00:00 2001 From: Rahul Agasthya Date: Fri, 11 Mar 2022 11:54:38 -0800 Subject: [PATCH 6/9] [CNS] Metrics Prefix Added the "cx_" prefix to prometheus metrics of CNS. --- cns/ipampool/metrics.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/cns/ipampool/metrics.go b/cns/ipampool/metrics.go index e87e10dbab..2f30dc7578 100644 --- a/cns/ipampool/metrics.go +++ b/cns/ipampool/metrics.go @@ -14,70 +14,70 @@ const ( var ( ipamAllocatedIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ipam_pod_allocated_ips", + Name: "cx_ipam_pod_allocated_ips", Help: "Count of IPs CNS has allocated to Pods.", }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamAvailableIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ipam_available_ips", + Name: "cx_ipam_available_ips", Help: "Available IP count.", }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamBatchSize = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ipam_batch_size", + Name: "cx_ipam_batch_size", Help: "IPAM IP pool batch size.", }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamCurrentAvailableIPcount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ipam_current_available_ips", + Name: "cx_ipam_current_available_ips", Help: "Current available IP count.", }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamExpectedAvailableIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ipam_expect_available_ips", + Name: "cx_ipam_expect_available_ips", Help: "Expected future available IP count assuming the Requested IP count is honored.", }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamMaxIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ipam_max_ips", + Name: "cx_ipam_max_ips", Help: "Maximum IP count.", }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamPendingProgramIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ipam_pending_programming_ips", + Name: "cx_ipam_pending_programming_ips", Help: "Pending programming IP count.", }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamPendingReleaseIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ipam_pending_release_ips", + Name: "cx_ipam_pending_release_ips", Help: "Pending release IP count.", }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamRequestedIPConfigCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ipam_requested_ips", + Name: "cx_ipam_requested_ips", Help: "Requested IP count.", }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamTotalIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ipam_total_ips", + Name: "cx_ipam_total_ips", Help: "Count of total IP pool size allocated to CNS by DNC.", }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, From d3c6534c5f1fc3a787d1ba04ee3d8de5e6e8740c Mon Sep 17 00:00:00 2001 From: Rahul Agasthya Date: Tue, 15 Mar 2022 11:21:15 -0700 Subject: [PATCH 7/9] Minor Changes Minor edits with the Metric Names and Documentation. --- cns/ipampool/metrics.go | 50 ++++++++++++++++++++++++----------------- cns/ipampool/monitor.go | 16 ++++++------- 2 files changed, 38 insertions(+), 28 deletions(-) diff --git a/cns/ipampool/metrics.go b/cns/ipampool/metrics.go index 2f30dc7578..2a22dd539d 100644 --- a/cns/ipampool/metrics.go +++ b/cns/ipampool/metrics.go @@ -14,71 +14,81 @@ const ( var ( ipamAllocatedIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "cx_ipam_pod_allocated_ips", - Help: "Count of IPs CNS has allocated to Pods.", + Name: "ipam_pod_allocated_ips", + Help: "Count of IPs CNS has allocated to Pods.", + ConstLabels: prometheus.Labels{"customer_metric": "cx_ipam_pod_allocated_ips"}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamAvailableIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "cx_ipam_available_ips", - Help: "Available IP count.", + Name: "ipam_available_ips", + Help: "Available IP count.", + ConstLabels: prometheus.Labels{"customer_metric": "cx_ipam_available_ips"}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamBatchSize = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "cx_ipam_batch_size", - Help: "IPAM IP pool batch size.", + Name: "ipam_batch_size", + Help: "IPAM IP pool batch size.", + ConstLabels: prometheus.Labels{"customer_metric": "cx_ipam_batch_size"}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamCurrentAvailableIPcount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "cx_ipam_current_available_ips", - Help: "Current available IP count.", + Name: "ipam_current_available_ips", + Help: "Current available IP count.", + ConstLabels: prometheus.Labels{"customer_metric": "cx_ipam_current_available_ips"}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamExpectedAvailableIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "cx_ipam_expect_available_ips", - Help: "Expected future available IP count assuming the Requested IP count is honored.", + Name: "ipam_expect_available_ips", + Help: "Expected future available IP count assuming the Requested IP count is honored.", + ConstLabels: prometheus.Labels{"customer_metric": "cx_ipam_expect_available_ips"}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamMaxIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "cx_ipam_max_ips", - Help: "Maximum IP count.", + Name: "ipam_max_ips", + Help: "Maximum IP count.", + ConstLabels: prometheus.Labels{"customer_metric": "cx_ipam_max_ips"}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamPendingProgramIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "cx_ipam_pending_programming_ips", - Help: "Pending programming IP count.", + Name: "ipam_pending_programming_ips", + Help: "Pending programming IP count.", + ConstLabels: prometheus.Labels{"customer_metric": "cx_ipam_pending_programming_ips"}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamPendingReleaseIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "cx_ipam_pending_release_ips", - Help: "Pending release IP count.", + Name: "ipam_pending_release_ips", + Help: "Pending release IP count.", + ConstLabels: prometheus.Labels{"customer_metric": "cx_ipam_pending_release_ips"}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamRequestedIPConfigCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "cx_ipam_requested_ips", - Help: "Requested IP count.", + Name: "ipam_requested_ips", + Help: "Requested IP count.", + ConstLabels: prometheus.Labels{"customer_metric": "cx_ipam_requested_ips"}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamTotalIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "cx_ipam_total_ips", - Help: "Count of total IP pool size allocated to CNS by DNC.", + Name: "ipam_total_ips", + Help: "Count of total IP pool size allocated to CNS by DNC.", + ConstLabels: prometheus.Labels{"customer_metric": "cx_ipam_total_ips"}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) diff --git a/cns/ipampool/monitor.go b/cns/ipampool/monitor.go index 963d39ce7a..9e280ad67d 100644 --- a/cns/ipampool/monitor.go +++ b/cns/ipampool/monitor.go @@ -19,7 +19,7 @@ const ( DefaultRefreshDelay = 1 * time.Second // DefaultMaxIPs default maximum allocatable IPs DefaultMaxIPs = 250 - // Subnet ARM ID /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/$(subnet) + // Subnet ARM ID /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/$(SUBNET) subnetARMIDTemplate = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s/subnets/%s" ) @@ -52,7 +52,7 @@ type Monitor struct { once sync.Once } -// Global Variables for Subnet, Subnet Address Space and Pod Network ARM ID +// Global Variables for Subnet, Subnet Address Space and Subnet ARM ID var subnet, subnetCIDR, subnetARMID string func NewMonitor(httpService cns.HTTPService, nnccli nodeNetworkConfigSpecUpdater, opts *Options) *Monitor { @@ -100,7 +100,7 @@ func (pm *Monitor) Start(ctx context.Context) error { // Set SubnetName, SubnetAddressSpace and Pod Network ARM ID values to the global subnet, subnetCIDR and subnetARM variables. subnet = nnc.Status.NetworkContainers[0].SubnetName subnetCIDR = nnc.Status.NetworkContainers[0].SubnetAddressSpace - subnetARMID = GenerateARMID(&nnc) + subnetARMID = GenerateARMID(&nnc.Status.NetworkContainers[0]) pm.metastate.batch = scaler.BatchSize pm.metastate.max = scaler.MaxIPCount @@ -349,11 +349,11 @@ func (pm *Monitor) GetStateSnapshot() cns.IpamPoolMonitorStateSnapshot { // GenerateARMID uses the Subnet ARM ID format to populate the ARM ID with the metadata. // If either of the metadata attributes are empty, then the ARM ID will be an empty string. -func GenerateARMID(nnc *v1alpha.NodeNetworkConfig) string { - subscription := nnc.Status.NetworkContainers[0].SubscriptionID - resourceGroup := nnc.Status.NetworkContainers[0].ResourceGroupID - vnetID := nnc.Status.NetworkContainers[0].VNETID - subnetID := nnc.Status.NetworkContainers[0].SubnetID +func GenerateARMID(nc *v1alpha.NetworkContainer) string { + subscription := nc.SubscriptionID + resourceGroup := nc.ResourceGroupID + vnetID := nc.VNETID + subnetID := nc.SubnetID if subscription == "" || resourceGroup == "" || vnetID == "" || subnetID == "" { return "" From af443d3a16f634de2fbab23a3cf5a59bba98cce1 Mon Sep 17 00:00:00 2001 From: Rahul Agasthya Date: Tue, 15 Mar 2022 13:57:57 -0700 Subject: [PATCH 8/9] ConstLabels in Metrics Made changes to the ConstLabels in the CNS Metrics. --- cns/ipampool/metrics.go | 48 +++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/cns/ipampool/metrics.go b/cns/ipampool/metrics.go index 2a22dd539d..7c2737a137 100644 --- a/cns/ipampool/metrics.go +++ b/cns/ipampool/metrics.go @@ -6,89 +6,91 @@ import ( ) const ( - subnetLabel = "subnet" - subnetCIDRLabel = "subnet_cidr" - podnetARMIDLabel = "podnet_arm_id" + subnetLabel = "subnet" + subnetCIDRLabel = "subnet_cidr" + podnetARMIDLabel = "podnet_arm_id" + customerMetricLabel = "customer_metric" + customerMetricLabelValue = "customer metric" ) var ( ipamAllocatedIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ipam_pod_allocated_ips", + Name: "cx_ipam_pod_allocated_ips", Help: "Count of IPs CNS has allocated to Pods.", - ConstLabels: prometheus.Labels{"customer_metric": "cx_ipam_pod_allocated_ips"}, + ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamAvailableIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ipam_available_ips", + Name: "cx_ipam_available_ips", Help: "Available IP count.", - ConstLabels: prometheus.Labels{"customer_metric": "cx_ipam_available_ips"}, + ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamBatchSize = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ipam_batch_size", + Name: "cx_ipam_batch_size", Help: "IPAM IP pool batch size.", - ConstLabels: prometheus.Labels{"customer_metric": "cx_ipam_batch_size"}, + ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamCurrentAvailableIPcount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ipam_current_available_ips", + Name: "cx_ipam_current_available_ips", Help: "Current available IP count.", - ConstLabels: prometheus.Labels{"customer_metric": "cx_ipam_current_available_ips"}, + ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamExpectedAvailableIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ipam_expect_available_ips", + Name: "cx_ipam_expect_available_ips", Help: "Expected future available IP count assuming the Requested IP count is honored.", - ConstLabels: prometheus.Labels{"customer_metric": "cx_ipam_expect_available_ips"}, + ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamMaxIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ipam_max_ips", + Name: "cx_ipam_max_ips", Help: "Maximum IP count.", - ConstLabels: prometheus.Labels{"customer_metric": "cx_ipam_max_ips"}, + ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamPendingProgramIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ipam_pending_programming_ips", + Name: "cx_ipam_pending_programming_ips", Help: "Pending programming IP count.", - ConstLabels: prometheus.Labels{"customer_metric": "cx_ipam_pending_programming_ips"}, + ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamPendingReleaseIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ipam_pending_release_ips", + Name: "cx_ipam_pending_release_ips", Help: "Pending release IP count.", - ConstLabels: prometheus.Labels{"customer_metric": "cx_ipam_pending_release_ips"}, + ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamRequestedIPConfigCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ipam_requested_ips", + Name: "cx_ipam_requested_ips", Help: "Requested IP count.", - ConstLabels: prometheus.Labels{"customer_metric": "cx_ipam_requested_ips"}, + ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamTotalIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ipam_total_ips", + Name: "cx_ipam_total_ips", Help: "Count of total IP pool size allocated to CNS by DNC.", - ConstLabels: prometheus.Labels{"customer_metric": "cx_ipam_total_ips"}, + ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) From 3fc144e88e015e494ec7bac5a6c7248f303ade62 Mon Sep 17 00:00:00 2001 From: Rahul Agasthya Date: Tue, 15 Mar 2022 13:57:57 -0700 Subject: [PATCH 9/9] ConstLabels in Metrics Made changes to the ConstLabels in the CNS Metrics. --- cns/ipampool/metrics.go | 48 ++++++++++--------- .../api/v1alpha/nodenetworkconfig.go | 4 +- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/cns/ipampool/metrics.go b/cns/ipampool/metrics.go index 2a22dd539d..7c2737a137 100644 --- a/cns/ipampool/metrics.go +++ b/cns/ipampool/metrics.go @@ -6,89 +6,91 @@ import ( ) const ( - subnetLabel = "subnet" - subnetCIDRLabel = "subnet_cidr" - podnetARMIDLabel = "podnet_arm_id" + subnetLabel = "subnet" + subnetCIDRLabel = "subnet_cidr" + podnetARMIDLabel = "podnet_arm_id" + customerMetricLabel = "customer_metric" + customerMetricLabelValue = "customer metric" ) var ( ipamAllocatedIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ipam_pod_allocated_ips", + Name: "cx_ipam_pod_allocated_ips", Help: "Count of IPs CNS has allocated to Pods.", - ConstLabels: prometheus.Labels{"customer_metric": "cx_ipam_pod_allocated_ips"}, + ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamAvailableIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ipam_available_ips", + Name: "cx_ipam_available_ips", Help: "Available IP count.", - ConstLabels: prometheus.Labels{"customer_metric": "cx_ipam_available_ips"}, + ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamBatchSize = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ipam_batch_size", + Name: "cx_ipam_batch_size", Help: "IPAM IP pool batch size.", - ConstLabels: prometheus.Labels{"customer_metric": "cx_ipam_batch_size"}, + ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamCurrentAvailableIPcount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ipam_current_available_ips", + Name: "cx_ipam_current_available_ips", Help: "Current available IP count.", - ConstLabels: prometheus.Labels{"customer_metric": "cx_ipam_current_available_ips"}, + ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamExpectedAvailableIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ipam_expect_available_ips", + Name: "cx_ipam_expect_available_ips", Help: "Expected future available IP count assuming the Requested IP count is honored.", - ConstLabels: prometheus.Labels{"customer_metric": "cx_ipam_expect_available_ips"}, + ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamMaxIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ipam_max_ips", + Name: "cx_ipam_max_ips", Help: "Maximum IP count.", - ConstLabels: prometheus.Labels{"customer_metric": "cx_ipam_max_ips"}, + ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamPendingProgramIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ipam_pending_programming_ips", + Name: "cx_ipam_pending_programming_ips", Help: "Pending programming IP count.", - ConstLabels: prometheus.Labels{"customer_metric": "cx_ipam_pending_programming_ips"}, + ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamPendingReleaseIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ipam_pending_release_ips", + Name: "cx_ipam_pending_release_ips", Help: "Pending release IP count.", - ConstLabels: prometheus.Labels{"customer_metric": "cx_ipam_pending_release_ips"}, + ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamRequestedIPConfigCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ipam_requested_ips", + Name: "cx_ipam_requested_ips", Help: "Requested IP count.", - ConstLabels: prometheus.Labels{"customer_metric": "cx_ipam_requested_ips"}, + ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) ipamTotalIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "ipam_total_ips", + Name: "cx_ipam_total_ips", Help: "Count of total IP pool size allocated to CNS by DNC.", - ConstLabels: prometheus.Labels{"customer_metric": "cx_ipam_total_ips"}, + ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) diff --git a/crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go b/crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go index eb4f9ab538..3ceb7cdb7b 100644 --- a/crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go +++ b/crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go @@ -81,8 +81,8 @@ type NetworkContainer struct { SubnetAddressSpace string `json:"subnetAddressSpace,omitempty"` Version int64 `json:"version,omitempty"` NodeIP string `json:"nodeIP,omitempty"` - SubscriptionID string `json:"subcriptionId,omitempty"` - ResourceGroupID string `json:"resourceGroupId,omitempty"` + SubscriptionID string `json:"subcriptionID,omitempty"` + ResourceGroupID string `json:"resourceGroupID,omitempty"` VNETID string `json:"vnetID,omitempty"` SubnetID string `json:"subnetID,omitempty"` }