From f16bd594ae2aa33d8e897f7c5322fb3f4262c77e Mon Sep 17 00:00:00 2001 From: Ashley Kasim Date: Wed, 18 Sep 2019 13:53:06 -0700 Subject: [PATCH] add hpa stats for current utilization and average value --- docs/horizontalpodautoscaler-metrics.md | 4 +- internal/store/hpa.go | 54 ++++++++++++++++++++++++- internal/store/hpa_test.go | 46 +++++++++++++++------ 3 files changed, 88 insertions(+), 16 deletions(-) diff --git a/docs/horizontalpodautoscaler-metrics.md b/docs/horizontalpodautoscaler-metrics.md index 0b4c1cc773..3aa4cda7f6 100644 --- a/docs/horizontalpodautoscaler-metrics.md +++ b/docs/horizontalpodautoscaler-metrics.md @@ -7,5 +7,7 @@ | kube_hpa_spec_min_replicas | Gauge | `hpa`=<hpa-name>
`namespace`=<hpa-namespace> | STABLE | | kube_hpa_status_current_replicas | Gauge | `hpa`=<hpa-name>
`namespace`=<hpa-namespace> | STABLE | | kube_hpa_status_desired_replicas | Gauge | `hpa`=<hpa-name>
`namespace`=<hpa-namespace> | STABLE | -| kube_hpa_status_condition | Gauge | `hpa`=<hpa-name>
`namespace`=<hpa-namespace>
`condition`=<hpa-condition>
`status`=<true\|false\|unknown> | STABLE | +| kube_hpa_status_condition | Gauge | `hpa`=<hpa-name>
`namespace`=<hpa-namespace>
`condition`=<hpa-condition>
`status`=<true\|false\|unknown>
`reason`=<reason> | STABLE | | kube_hpa_labels | Gauge | `hpa`=<hpa-name>
`namespace`=<hpa-namespace> | STABLE | +| kube_hpa_status_average_value | Gauge | `hpa`=<hpa-name>
`namespace`=<hpa-namespace> | STABLE | +| kube_hpa_status_average_utilization | Gauge | `hpa`=<hpa-name>
`namespace`=<hpa-namespace> | STABLE | diff --git a/internal/store/hpa.go b/internal/store/hpa.go index 08a949c29b..71f7990516 100644 --- a/internal/store/hpa.go +++ b/internal/store/hpa.go @@ -132,8 +132,8 @@ var ( for j, m := range metrics { metric := m - metric.LabelKeys = []string{"condition", "status"} - metric.LabelValues = append([]string{string(c.Type)}, metric.LabelValues...) + metric.LabelKeys = []string{"condition", "status", "reason"} + metric.LabelValues = append(metric.LabelValues, string(c.Type), c.Reason) ms[i*len(conditionStatuses)+j] = metric } } @@ -143,6 +143,56 @@ var ( } }), }, + { + Name: "kube_hpa_status_average_value", + Type: metric.Gauge, + Help: "Average metric value observed by the autoscaler.", + GenerateFunc: wrapHPAFunc(func(a *autoscaling.HorizontalPodAutoscaler) *metric.Family { + ms := make([]*metric.Metric, len(a.Status.CurrentMetrics)) + for i, c := range a.Status.CurrentMetrics { + var value int64 + if c.Type == "Resource" { + value, _ = c.Resource.CurrentAverageValue.AsInt64() + } else if c.Type == "Pods" { + value, _ = c.Pods.CurrentAverageValue.AsInt64() + } else if c.Type == "Object" { + value, _ = c.Object.AverageValue.AsInt64() + } else if c.Type == "External" { + value, _ = c.External.CurrentAverageValue.AsInt64() + } else { + // Skip unsupported metric type + continue + } + ms[i] = &metric.Metric{ + Value: float64(value), + } + } + return &metric.Family{ + Metrics: ms, + } + }), + }, + { + Name: "kube_hpa_status_average_utilization", + Type: metric.Gauge, + Help: "Average metric utilization observed by the autoscaler.", + GenerateFunc: wrapHPAFunc(func(a *autoscaling.HorizontalPodAutoscaler) *metric.Family { + ms := make([]*metric.Metric, len(a.Status.CurrentMetrics)) + for i, c := range a.Status.CurrentMetrics { + if c.Type == "Resource" { + ms[i] = &metric.Metric{ + Value: float64(*c.Resource.CurrentAverageUtilization), + } + } else { + // Skip unsupported metric type + continue + } + } + return &metric.Family{ + Metrics: ms, + } + }), + }, } ) diff --git a/internal/store/hpa_test.go b/internal/store/hpa_test.go index fcea8f65c4..4c0bcf35f8 100644 --- a/internal/store/hpa_test.go +++ b/internal/store/hpa_test.go @@ -21,6 +21,7 @@ import ( autoscaling "k8s.io/api/autoscaling/v2beta1" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kube-state-metrics/pkg/metric" @@ -44,10 +45,14 @@ func TestHPAStore(t *testing.T) { # TYPE kube_hpa_status_current_replicas gauge # HELP kube_hpa_status_desired_replicas Desired number of replicas of pods managed by this autoscaler. # TYPE kube_hpa_status_desired_replicas gauge - # HELP kube_hpa_status_condition The condition of this autoscaler. - # TYPE kube_hpa_status_condition gauge - # HELP kube_hpa_labels Kubernetes labels converted to Prometheus labels. - # TYPE kube_hpa_labels gauge + # HELP kube_hpa_status_condition The condition of this autoscaler. + # TYPE kube_hpa_status_condition gauge + # HELP kube_hpa_labels Kubernetes labels converted to Prometheus labels. + # TYPE kube_hpa_labels gauge + # HELP kube_hpa_status_average_value Average metric value observed by the autoscaler. + # TYPE kube_hpa_status_average_value gauge + # HELP kube_hpa_status_average_utilization Average metric utilization observed by the autoscaler. + # TYPE kube_hpa_status_average_utilization gauge ` cases := []generateMetricsTestCase{ { @@ -77,20 +82,33 @@ func TestHPAStore(t *testing.T) { { Type: autoscaling.AbleToScale, Status: v1.ConditionTrue, + Reason: "reason", + }, + }, + CurrentMetrics: []autoscaling.MetricStatus{ + { + Type: "Resource", + Resource: &autoscaling.ResourceMetricStatus{ + Name: "cpu", + CurrentAverageUtilization: new(int32), + CurrentAverageValue: resource.MustParse("10"), + }, }, }, }, }, Want: metadata + ` - kube_hpa_labels{hpa="hpa1",label_app="foobar",namespace="ns1"} 1 - kube_hpa_metadata_generation{hpa="hpa1",namespace="ns1"} 2 - kube_hpa_spec_max_replicas{hpa="hpa1",namespace="ns1"} 4 - kube_hpa_spec_min_replicas{hpa="hpa1",namespace="ns1"} 2 - kube_hpa_status_condition{condition="AbleToScale",hpa="hpa1",namespace="ns1",status="false"} 0 - kube_hpa_status_condition{condition="AbleToScale",hpa="hpa1",namespace="ns1",status="true"} 1 - kube_hpa_status_condition{condition="AbleToScale",hpa="hpa1",namespace="ns1",status="unknown"} 0 - kube_hpa_status_current_replicas{hpa="hpa1",namespace="ns1"} 2 - kube_hpa_status_desired_replicas{hpa="hpa1",namespace="ns1"} 2 + kube_hpa_labels{hpa="hpa1",label_app="foobar",namespace="ns1"} 1 + kube_hpa_metadata_generation{hpa="hpa1",namespace="ns1"} 2 + kube_hpa_spec_max_replicas{hpa="hpa1",namespace="ns1"} 4 + kube_hpa_spec_min_replicas{hpa="hpa1",namespace="ns1"} 2 + kube_hpa_status_condition{condition="false",hpa="hpa1",namespace="ns1",reason="reason",status="AbleToScale"} 0 + kube_hpa_status_condition{condition="true",hpa="hpa1",namespace="ns1",reason="reason",status="AbleToScale"} 1 + kube_hpa_status_condition{condition="unknown",hpa="hpa1",namespace="ns1",reason="reason",status="AbleToScale"} 0 + kube_hpa_status_current_replicas{hpa="hpa1",namespace="ns1"} 2 + kube_hpa_status_desired_replicas{hpa="hpa1",namespace="ns1"} 2 + kube_hpa_status_average_value{hpa="hpa1",namespace="ns1"} 10 + kube_hpa_status_average_utilization{hpa="hpa1",namespace="ns1"} 0 `, MetricNames: []string{ "kube_hpa_metadata_generation", @@ -100,6 +118,8 @@ func TestHPAStore(t *testing.T) { "kube_hpa_status_desired_replicas", "kube_hpa_status_condition", "kube_hpa_labels", + "kube_hpa_status_average_value", + "kube_hpa_status_average_utilization", }, }, }