From a0c63d592010d7499f2ad4d57598c6a51ccd4423 Mon Sep 17 00:00:00 2001 From: Jan Zantinge Date: Fri, 1 Nov 2019 14:04:19 -0400 Subject: [PATCH 1/2] Added a kube_hpa_target_metric metric that publishes all of a HPA's target metrics. --- docs/horizontalpodautoscaler-metrics.md | 1 + internal/store/hpa.go | 69 ++++++++++++++++++++ internal/store/hpa_test.go | 87 +++++++++++++++++++++---- 3 files changed, 146 insertions(+), 11 deletions(-) diff --git a/docs/horizontalpodautoscaler-metrics.md b/docs/horizontalpodautoscaler-metrics.md index 9531864edb..4be0ea9574 100644 --- a/docs/horizontalpodautoscaler-metrics.md +++ b/docs/horizontalpodautoscaler-metrics.md @@ -6,6 +6,7 @@ | kube_hpa_metadata_generation | Gauge | `hpa`=<hpa-name>
`namespace`=<hpa-namespace> | STABLE | | kube_hpa_spec_max_replicas | Gauge | `hpa`=<hpa-name>
`namespace`=<hpa-namespace> | STABLE | | kube_hpa_spec_min_replicas | Gauge | `hpa`=<hpa-name>
`namespace`=<hpa-namespace> | STABLE | +| kube_hpa_spec_target_metric | Gauge | `hpa`=<hpa-name>
`namespace`=<hpa-namespace>
`metric_name`=<metric-name>
`metric_target_type`=<value\|utilization\|average> | EXPERIMENTAL | | kube_hpa_status_condition | Gauge | `hpa`=<hpa-name>
`namespace`=<hpa-namespace>
`condition`=<hpa-condition>
`status`=<true\|false\|unknown> | STABLE | | kube_hpa_status_current_replicas | Gauge | `hpa`=<hpa-name>
`namespace`=<hpa-namespace> | STABLE | | kube_hpa_status_currentmetrics_average_utilization | Gauge | `hpa`=<hpa-name>
`namespace`=<hpa-namespace> | EXPERIMENTAL | diff --git a/internal/store/hpa.go b/internal/store/hpa.go index 251debc6ab..8f08b9efb5 100644 --- a/internal/store/hpa.go +++ b/internal/store/hpa.go @@ -33,6 +33,8 @@ var ( descHorizontalPodAutoscalerLabelsHelp = "Kubernetes labels converted to Prometheus labels." descHorizontalPodAutoscalerLabelsDefaultLabels = []string{"namespace", "hpa"} + targetMetricLabels = []string{"metric_name", "metric_target_type"} + hpaMetricFamilies = []metric.FamilyGenerator{ { Name: "kube_hpa_metadata_generation", @@ -76,6 +78,73 @@ var ( } }), }, + { + Name: "kube_hpa_spec_target_metric", + Type: metric.Gauge, + Help: "The metric specifications used by this autoscaler when calculating the desired replica count.", + GenerateFunc: wrapHPAFunc(func(a *autoscaling.HorizontalPodAutoscaler) *metric.Family { + targets := make([]*metric.Metric, 0, len(a.Spec.Metrics)) + for _, m := range a.Spec.Metrics { + var metricName string + var value, average int64 + valueLabel := "value" + var valueOk, averageOk bool + + switch m.Type { + case autoscaling.ObjectMetricSourceType: + metricName = m.Object.MetricName + + value, valueOk = m.Object.TargetValue.AsInt64() + if m.Object.AverageValue != nil { + average, averageOk = m.Object.AverageValue.AsInt64() + } + case autoscaling.PodsMetricSourceType: + metricName = m.Pods.MetricName + + average, averageOk = m.Pods.TargetAverageValue.AsInt64() + case autoscaling.ResourceMetricSourceType: + metricName = string(m.Resource.Name) + + value = int64(*m.Resource.TargetAverageUtilization) + valueOk = true + valueLabel = "utilization" + + if m.Resource.TargetAverageValue != nil { + average, averageOk = m.Resource.TargetAverageValue.AsInt64() + } + case autoscaling.ExternalMetricSourceType: + metricName = m.External.MetricName + + // The TargetValue and TargetAverageValue are mutually exclusive + if m.External.TargetValue != nil { + value, valueOk = m.External.TargetValue.AsInt64() + } + if m.External.TargetAverageValue != nil { + average, averageOk = m.External.TargetAverageValue.AsInt64() + } + default: + // Skip unsupported metric type + continue + } + + if valueOk { + targets = append(targets, &metric.Metric{ + LabelKeys: targetMetricLabels, + LabelValues: []string{metricName, valueLabel}, + Value: float64(value), + }) + } + if averageOk { + targets = append(targets, &metric.Metric{ + LabelKeys: targetMetricLabels, + LabelValues: []string{metricName, "average"}, + Value: float64(average), + }) + } + } + return &metric.Family{Metrics: targets} + }), + }, { Name: "kube_hpa_status_current_replicas", Type: metric.Gauge, diff --git a/internal/store/hpa_test.go b/internal/store/hpa_test.go index 22d242539f..6cddf2ece3 100644 --- a/internal/store/hpa_test.go +++ b/internal/store/hpa_test.go @@ -41,6 +41,8 @@ func TestHPAStore(t *testing.T) { # TYPE kube_hpa_spec_max_replicas gauge # HELP kube_hpa_spec_min_replicas Lower limit for the number of pods that can be set by the autoscaler, default 1. # TYPE kube_hpa_spec_min_replicas gauge + # HELP kube_hpa_spec_target_metric The metric specifications used by this autoscaler when calculating the desired replica count. + # TYPE kube_hpa_spec_target_metric gauge # HELP kube_hpa_status_current_replicas Current number of replicas of pods managed by this autoscaler. # TYPE kube_hpa_status_current_replicas gauge # HELP kube_hpa_status_desired_replicas Desired number of replicas of pods managed by this autoscaler. @@ -69,6 +71,52 @@ func TestHPAStore(t *testing.T) { Spec: autoscaling.HorizontalPodAutoscalerSpec{ MaxReplicas: 4, MinReplicas: &hpa1MinReplicas, + Metrics: []autoscaling.MetricSpec{ + { + Type: autoscaling.ObjectMetricSourceType, + Object: &autoscaling.ObjectMetricSource{ + MetricName: "hits", + TargetValue: resource.MustParse("10"), + AverageValue: resourcePtr(resource.MustParse("12")), + }, + }, + { + Type: autoscaling.PodsMetricSourceType, + Pods: &autoscaling.PodsMetricSource{ + MetricName: "transactions_processed", + TargetAverageValue: resource.MustParse("33"), + }, + }, + { + Type: autoscaling.ResourceMetricSourceType, + Resource: &autoscaling.ResourceMetricSource{ + Name: "cpu", + TargetAverageUtilization: int32ptr(80), + }, + }, + { + Type: autoscaling.ResourceMetricSourceType, + Resource: &autoscaling.ResourceMetricSource{ + Name: "memory", + TargetAverageUtilization: int32ptr(80), + TargetAverageValue: resourcePtr(resource.MustParse("800Ki")), + }, + }, + { + Type: autoscaling.ExternalMetricSourceType, + External: &autoscaling.ExternalMetricSource{ + MetricName: "sqs_jobs", + TargetValue: resourcePtr(resource.MustParse("30")), + }, + }, + { + Type: autoscaling.ExternalMetricSourceType, + External: &autoscaling.ExternalMetricSource{ + MetricName: "events", + TargetAverageValue: resourcePtr(resource.MustParse("30")), + }, + }, + }, ScaleTargetRef: autoscaling.CrossVersionObjectReference{ APIVersion: "apps/v1", Kind: "Deployment", @@ -98,22 +146,31 @@ func TestHPAStore(t *testing.T) { }, }, Want: metadata + ` - kube_hpa_labels{hpa="hpa1",label_app="foobar",namespace="ns1"} 1 - kube_hpa_metadata_generation{hpa="hpa1",namespace="ns1"} 2 - kube_hpa_spec_max_replicas{hpa="hpa1",namespace="ns1"} 4 - kube_hpa_spec_min_replicas{hpa="hpa1",namespace="ns1"} 2 - kube_hpa_status_condition{condition="AbleToScale",hpa="hpa1",namespace="ns1",status="false"} 0 - kube_hpa_status_condition{condition="AbleToScale",hpa="hpa1",namespace="ns1",status="true"} 1 - kube_hpa_status_condition{condition="AbleToScale",hpa="hpa1",namespace="ns1",status="unknown"} 0 - kube_hpa_status_current_replicas{hpa="hpa1",namespace="ns1"} 2 - kube_hpa_status_desired_replicas{hpa="hpa1",namespace="ns1"} 2 - kube_hpa_status_currentmetrics_average_value{hpa="hpa1",namespace="ns1"} 10 - kube_hpa_status_currentmetrics_average_utilization{hpa="hpa1",namespace="ns1"} 0 + kube_hpa_labels{hpa="hpa1",label_app="foobar",namespace="ns1"} 1 + kube_hpa_metadata_generation{hpa="hpa1",namespace="ns1"} 2 + kube_hpa_spec_max_replicas{hpa="hpa1",namespace="ns1"} 4 + kube_hpa_spec_min_replicas{hpa="hpa1",namespace="ns1"} 2 + kube_hpa_spec_target_metric{hpa="hpa1",namespace="ns1",metric_name="hits",metric_target_type="value"} 10 + kube_hpa_spec_target_metric{hpa="hpa1",namespace="ns1",metric_name="hits",metric_target_type="average"} 12 + kube_hpa_spec_target_metric{hpa="hpa1",namespace="ns1",metric_name="transactions_processed",metric_target_type="average"} 33 + kube_hpa_spec_target_metric{hpa="hpa1",namespace="ns1",metric_name="cpu",metric_target_type="utilization"} 80 + kube_hpa_spec_target_metric{hpa="hpa1",namespace="ns1",metric_name="memory",metric_target_type="utilization"} 80 + kube_hpa_spec_target_metric{hpa="hpa1",namespace="ns1",metric_name="memory",metric_target_type="average"} 819200 + kube_hpa_spec_target_metric{hpa="hpa1",namespace="ns1",metric_name="sqs_jobs",metric_target_type="value"} 30 + kube_hpa_spec_target_metric{hpa="hpa1",namespace="ns1",metric_name="events",metric_target_type="average"} 30 + kube_hpa_status_condition{condition="AbleToScale",hpa="hpa1",namespace="ns1",status="false"} 0 + kube_hpa_status_condition{condition="AbleToScale",hpa="hpa1",namespace="ns1",status="true"} 1 + kube_hpa_status_condition{condition="AbleToScale",hpa="hpa1",namespace="ns1",status="unknown"} 0 + kube_hpa_status_current_replicas{hpa="hpa1",namespace="ns1"} 2 + kube_hpa_status_desired_replicas{hpa="hpa1",namespace="ns1"} 2 + kube_hpa_status_currentmetrics_average_value{hpa="hpa1",namespace="ns1"} 10 + kube_hpa_status_currentmetrics_average_utilization{hpa="hpa1",namespace="ns1"} 0 `, MetricNames: []string{ "kube_hpa_metadata_generation", "kube_hpa_spec_max_replicas", "kube_hpa_spec_min_replicas", + "kube_hpa_spec_target_metric", "kube_hpa_status_current_replicas", "kube_hpa_status_desired_replicas", "kube_hpa_status_condition", @@ -131,3 +188,11 @@ func TestHPAStore(t *testing.T) { } } } + +func int32ptr(value int32) *int32 { + return &value +} + +func resourcePtr(quantity resource.Quantity) *resource.Quantity { + return &quantity +} From cef3baba67265556d64fbf955e804318f1a97332 Mon Sep 17 00:00:00 2001 From: Jan Zantinge Date: Thu, 14 Nov 2019 13:56:14 -0500 Subject: [PATCH 2/2] Added an enumeration to represent the different MetricTargetTypes (value, average, and utilization). --- internal/store/hpa.go | 63 ++++++++++++++++++++++---------------- internal/store/hpa_test.go | 7 +++++ 2 files changed, 43 insertions(+), 27 deletions(-) diff --git a/internal/store/hpa.go b/internal/store/hpa.go index 8f08b9efb5..b04b247d9e 100644 --- a/internal/store/hpa.go +++ b/internal/store/hpa.go @@ -28,6 +28,20 @@ import ( "k8s.io/client-go/tools/cache" ) +type MetricTargetType int + +const ( + Value MetricTargetType = iota + Utilization + Average + + MetricTargetTypeCount // Used as a length argument to arrays +) + +func (m MetricTargetType) String() string { + return [...]string{"value", "utilization", "average"}[m] +} + var ( descHorizontalPodAutoscalerLabelsName = "kube_hpa_labels" descHorizontalPodAutoscalerLabelsHelp = "Kubernetes labels converted to Prometheus labels." @@ -83,66 +97,61 @@ var ( Type: metric.Gauge, Help: "The metric specifications used by this autoscaler when calculating the desired replica count.", GenerateFunc: wrapHPAFunc(func(a *autoscaling.HorizontalPodAutoscaler) *metric.Family { - targets := make([]*metric.Metric, 0, len(a.Spec.Metrics)) + ms := make([]*metric.Metric, 0, len(a.Spec.Metrics)) for _, m := range a.Spec.Metrics { var metricName string - var value, average int64 - valueLabel := "value" - var valueOk, averageOk bool + + var v [MetricTargetTypeCount]int64 + var ok [MetricTargetTypeCount]bool switch m.Type { case autoscaling.ObjectMetricSourceType: metricName = m.Object.MetricName - value, valueOk = m.Object.TargetValue.AsInt64() + v[Value], ok[Value] = m.Object.TargetValue.AsInt64() if m.Object.AverageValue != nil { - average, averageOk = m.Object.AverageValue.AsInt64() + v[Average], ok[Average] = m.Object.AverageValue.AsInt64() } case autoscaling.PodsMetricSourceType: metricName = m.Pods.MetricName - average, averageOk = m.Pods.TargetAverageValue.AsInt64() + v[Average], ok[Average] = m.Pods.TargetAverageValue.AsInt64() case autoscaling.ResourceMetricSourceType: metricName = string(m.Resource.Name) - value = int64(*m.Resource.TargetAverageUtilization) - valueOk = true - valueLabel = "utilization" + if ok[Utilization] = (m.Resource.TargetAverageUtilization != nil); ok[Utilization] { + v[Utilization] = int64(*m.Resource.TargetAverageUtilization) + } if m.Resource.TargetAverageValue != nil { - average, averageOk = m.Resource.TargetAverageValue.AsInt64() + v[Average], ok[Average] = m.Resource.TargetAverageValue.AsInt64() } case autoscaling.ExternalMetricSourceType: metricName = m.External.MetricName // The TargetValue and TargetAverageValue are mutually exclusive if m.External.TargetValue != nil { - value, valueOk = m.External.TargetValue.AsInt64() + v[Value], ok[Value] = m.External.TargetValue.AsInt64() } if m.External.TargetAverageValue != nil { - average, averageOk = m.External.TargetAverageValue.AsInt64() + v[Average], ok[Average] = m.External.TargetAverageValue.AsInt64() } default: // Skip unsupported metric type continue } - if valueOk { - targets = append(targets, &metric.Metric{ - LabelKeys: targetMetricLabels, - LabelValues: []string{metricName, valueLabel}, - Value: float64(value), - }) - } - if averageOk { - targets = append(targets, &metric.Metric{ - LabelKeys: targetMetricLabels, - LabelValues: []string{metricName, "average"}, - Value: float64(average), - }) + for i := range ok { + if ok[i] { + ms = append(ms, &metric.Metric{ + LabelKeys: targetMetricLabels, + LabelValues: []string{metricName, MetricTargetType(i).String()}, + Value: float64(v[i]), + }) + } } } - return &metric.Family{Metrics: targets} + return &metric.Family{Metrics: ms} }), }, { diff --git a/internal/store/hpa_test.go b/internal/store/hpa_test.go index 6cddf2ece3..d9fb62a247 100644 --- a/internal/store/hpa_test.go +++ b/internal/store/hpa_test.go @@ -102,6 +102,13 @@ func TestHPAStore(t *testing.T) { TargetAverageValue: resourcePtr(resource.MustParse("800Ki")), }, }, + // No targets, this metric should be ignored + { + Type: autoscaling.ResourceMetricSourceType, + Resource: &autoscaling.ResourceMetricSource{ + Name: "disk", + }, + }, { Type: autoscaling.ExternalMetricSourceType, External: &autoscaling.ExternalMetricSource{