Skip to content

Commit

Permalink
Added a kube_hpa_target_metric metric that publishes all of a HPA's t…
Browse files Browse the repository at this point in the history 
…arget metrics.
  • Loading branch information
@janzantinge
janzantinge committed Nov 4, 2019
1 parent e51fc8f commit 32e4797
Show file tree
Hide file tree
Showing 3 changed files with 164 additions and 11 deletions.
1 change: 1 addition & 0 deletions docs/horizontalpodautoscaler-metrics.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
| kube_hpa_metadata_generation | Gauge | `hpa`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt; | STABLE |
| kube_hpa_spec_max_replicas | Gauge | `hpa`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt; | STABLE |
| kube_hpa_spec_min_replicas | Gauge | `hpa`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt; | STABLE |
| kube_hpa_spec_target_metric | Gauge | `hpa`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt; <br> `metric_name`=&lt;metric-name&gt; <br> `metric_target_type`=&lt;value\|utilization\|average&gt; | EXPERIMENTAL |
| kube_hpa_status_condition | Gauge | `hpa`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt; <br> `condition`=&lt;hpa-condition&gt; <br> `status`=&lt;true\|false\|unknown&gt; | STABLE |
| kube_hpa_status_current_replicas | Gauge | `hpa`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt; | STABLE |
| kube_hpa_status_currentmetrics_average_utilization | Gauge | `hpa`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt; | EXPERIMENTAL |
Expand Down
87 changes: 87 additions & 0 deletions internal/store/hpa.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ var (
descHorizontalPodAutoscalerLabelsHelp = "Kubernetes labels converted to Prometheus labels."
descHorizontalPodAutoscalerLabelsDefaultLabels = []string{"namespace", "hpa"}

targetMetricLabels = []string{"metric_name", "metric_target_type"}

hpaMetricFamilies = []metric.FamilyGenerator{
{
Name: "kube_hpa_metadata_generation",
Expand Down Expand Up @@ -76,6 +78,91 @@ var (
}
}),
},
{
Name: "kube_hpa_spec_target_metric",
Type: metric.Gauge,
Help: "The metric specifications used by this autoscaler when calculating the desired replica count.",
GenerateFunc: wrapHPAFunc(func(a *autoscaling.HorizontalPodAutoscaler) *metric.Family {
targets := make([]*metric.Metric, 0, len(a.Spec.Metrics))
for _, m := range a.Spec.Metrics {
switch m.Type {
case autoscaling.ObjectMetricSourceType:
targetValue, ok := m.Object.TargetValue.AsInt64()
if ok {
targets = append(targets, &metric.Metric{
LabelKeys: targetMetricLabels,
LabelValues: []string{m.Object.MetricName, "value"},
Value: float64(targetValue),
})
}
if m.Object.AverageValue != nil {
averageValue, ok := m.Object.AverageValue.AsInt64()
if ok {
targets = append(targets, &metric.Metric{
LabelKeys: targetMetricLabels,
LabelValues: []string{m.Object.MetricName, "average"},
Value: float64(averageValue),
})
}
}
case autoscaling.PodsMetricSourceType:
value, ok := m.Pods.TargetAverageValue.AsInt64()
if !ok {
continue
}
targets = append(targets, &metric.Metric{
LabelKeys: targetMetricLabels,
LabelValues: []string{m.Pods.MetricName, "average"},
Value: float64(value),
})
case autoscaling.ResourceMetricSourceType:
targets = append(targets, &metric.Metric{
LabelKeys: targetMetricLabels,
LabelValues: []string{string(m.Resource.Name), "utilization"},
Value: float64(*m.Resource.TargetAverageUtilization),
})

if m.Resource.TargetAverageValue != nil {
value, ok := m.Resource.TargetAverageValue.AsInt64()
if !ok {
continue
}
targets = append(targets, &metric.Metric{
LabelKeys: targetMetricLabels,
LabelValues: []string{string(m.Resource.Name), "value"},
Value: float64(value),
})
}
case autoscaling.ExternalMetricSourceType:
// The TargetValue and TargetAverageValue are mutually exclusive
if m.External.TargetValue != nil {
value, ok := m.External.TargetValue.AsInt64()
if ok {
targets = append(targets, &metric.Metric{
LabelKeys: targetMetricLabels,
LabelValues: []string{m.External.MetricName, "value"},
Value: float64(value),
})
}
}
if m.External.TargetAverageValue != nil {
value, ok := m.External.TargetAverageValue.AsInt64()
if ok {
targets = append(targets, &metric.Metric{
LabelKeys: targetMetricLabels,
LabelValues: []string{m.External.MetricName, "average"},
Value: float64(value),
})
}
}
default:
// Skip unsupported metric type
continue
}
}
return &metric.Family{Metrics: targets}
}),
},
{
Name: "kube_hpa_status_current_replicas",
Type: metric.Gauge,
Expand Down
87 changes: 76 additions & 11 deletions internal/store/hpa_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ func TestHPAStore(t *testing.T) {
# TYPE kube_hpa_spec_max_replicas gauge
# HELP kube_hpa_spec_min_replicas Lower limit for the number of pods that can be set by the autoscaler, default 1.
# TYPE kube_hpa_spec_min_replicas gauge
# HELP kube_hpa_spec_target_metric The metric specifications used by this autoscaler when calculating the desired replica count.
# TYPE kube_hpa_spec_target_metric gauge
# HELP kube_hpa_status_current_replicas Current number of replicas of pods managed by this autoscaler.
# TYPE kube_hpa_status_current_replicas gauge
# HELP kube_hpa_status_desired_replicas Desired number of replicas of pods managed by this autoscaler.
Expand Down Expand Up @@ -69,6 +71,52 @@ func TestHPAStore(t *testing.T) {
Spec: autoscaling.HorizontalPodAutoscalerSpec{
MaxReplicas: 4,
MinReplicas: &hpa1MinReplicas,
Metrics: []autoscaling.MetricSpec{
{
Type: autoscaling.ObjectMetricSourceType,
Object: &autoscaling.ObjectMetricSource{
MetricName: "hits",
TargetValue: resource.MustParse("10"),
AverageValue: resourcePtr(resource.MustParse("12")),
},
},
{
Type: autoscaling.PodsMetricSourceType,
Pods: &autoscaling.PodsMetricSource{
MetricName: "transactions_processed",
TargetAverageValue: resource.MustParse("33"),
},
},
{
Type: autoscaling.ResourceMetricSourceType,
Resource: &autoscaling.ResourceMetricSource{
Name: "cpu",
TargetAverageUtilization: int32ptr(80),
},
},
{
Type: autoscaling.ResourceMetricSourceType,
Resource: &autoscaling.ResourceMetricSource{
Name: "memory",
TargetAverageUtilization: int32ptr(80),
TargetAverageValue: resourcePtr(resource.MustParse("800Ki")),
},
},
{
Type: autoscaling.ExternalMetricSourceType,
External: &autoscaling.ExternalMetricSource{
MetricName: "sqs_jobs",
TargetValue: resourcePtr(resource.MustParse("30")),
},
},
{
Type: autoscaling.ExternalMetricSourceType,
External: &autoscaling.ExternalMetricSource{
MetricName: "events",
TargetAverageValue: resourcePtr(resource.MustParse("30")),
},
},
},
ScaleTargetRef: autoscaling.CrossVersionObjectReference{
APIVersion: "apps/v1",
Kind: "Deployment",
Expand Down Expand Up @@ -98,22 +146,31 @@ func TestHPAStore(t *testing.T) {
},
},
Want: metadata + `
kube_hpa_labels{hpa="hpa1",label_app="foobar",namespace="ns1"} 1
kube_hpa_metadata_generation{hpa="hpa1",namespace="ns1"} 2
kube_hpa_spec_max_replicas{hpa="hpa1",namespace="ns1"} 4
kube_hpa_spec_min_replicas{hpa="hpa1",namespace="ns1"} 2
kube_hpa_status_condition{condition="AbleToScale",hpa="hpa1",namespace="ns1",status="false"} 0
kube_hpa_status_condition{condition="AbleToScale",hpa="hpa1",namespace="ns1",status="true"} 1
kube_hpa_status_condition{condition="AbleToScale",hpa="hpa1",namespace="ns1",status="unknown"} 0
kube_hpa_status_current_replicas{hpa="hpa1",namespace="ns1"} 2
kube_hpa_status_desired_replicas{hpa="hpa1",namespace="ns1"} 2
kube_hpa_status_currentmetrics_average_value{hpa="hpa1",namespace="ns1"} 10
kube_hpa_status_currentmetrics_average_utilization{hpa="hpa1",namespace="ns1"} 0
kube_hpa_labels{hpa="hpa1",label_app="foobar",namespace="ns1"} 1
kube_hpa_metadata_generation{hpa="hpa1",namespace="ns1"} 2
kube_hpa_spec_max_replicas{hpa="hpa1",namespace="ns1"} 4
kube_hpa_spec_min_replicas{hpa="hpa1",namespace="ns1"} 2
kube_hpa_spec_target_metric{hpa="hpa1",namespace="ns1",metric_name="hits",metric_target_type="value"} 10
kube_hpa_spec_target_metric{hpa="hpa1",namespace="ns1",metric_name="hits",metric_target_type="average"} 12
kube_hpa_spec_target_metric{hpa="hpa1",namespace="ns1",metric_name="transactions_processed",metric_target_type="average"} 33
kube_hpa_spec_target_metric{hpa="hpa1",namespace="ns1",metric_name="cpu",metric_target_type="utilization"} 80
kube_hpa_spec_target_metric{hpa="hpa1",namespace="ns1",metric_name="memory",metric_target_type="utilization"} 80
kube_hpa_spec_target_metric{hpa="hpa1",namespace="ns1",metric_name="memory",metric_target_type="value"} 819200
kube_hpa_spec_target_metric{hpa="hpa1",namespace="ns1",metric_name="sqs_jobs",metric_target_type="value"} 30
kube_hpa_spec_target_metric{hpa="hpa1",namespace="ns1",metric_name="events",metric_target_type="average"} 30
kube_hpa_status_condition{condition="AbleToScale",hpa="hpa1",namespace="ns1",status="false"} 0
kube_hpa_status_condition{condition="AbleToScale",hpa="hpa1",namespace="ns1",status="true"} 1
kube_hpa_status_condition{condition="AbleToScale",hpa="hpa1",namespace="ns1",status="unknown"} 0
kube_hpa_status_current_replicas{hpa="hpa1",namespace="ns1"} 2
kube_hpa_status_desired_replicas{hpa="hpa1",namespace="ns1"} 2
kube_hpa_status_currentmetrics_average_value{hpa="hpa1",namespace="ns1"} 10
kube_hpa_status_currentmetrics_average_utilization{hpa="hpa1",namespace="ns1"} 0
`,
MetricNames: []string{
"kube_hpa_metadata_generation",
"kube_hpa_spec_max_replicas",
"kube_hpa_spec_min_replicas",
"kube_hpa_spec_target_metric",
"kube_hpa_status_current_replicas",
"kube_hpa_status_desired_replicas",
"kube_hpa_status_condition",
Expand All @@ -131,3 +188,11 @@ func TestHPAStore(t *testing.T) {
}
}
}

func int32ptr(value int32) *int32 {
return &value
}

func resourcePtr(quantity resource.Quantity) *resource.Quantity {
return &quantity
}

0 comments on commit 32e4797

Please sign in to comment.