Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Added a kube_hpa_target_metric metric that publishes all of a HPA's target metrics. #966

Merged
merged 2 commits into from
Nov 14, 2019
Merged

Added a kube_hpa_target_metric metric that publishes all of a HPA's target metrics. #966

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a0c63d5
Added a kube_hpa_target_metric metric that publishes all of a HPA's t…
janzantinge Nov 1, 2019
cef3bab
Added an enumeration to represent the different MetricTargetTypes
janzantinge Nov 14, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/horizontalpodautoscaler-metrics.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
| kube_hpa_metadata_generation | Gauge | `hpa`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt; | STABLE |
| kube_hpa_spec_max_replicas | Gauge | `hpa`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt; | STABLE |
| kube_hpa_spec_min_replicas | Gauge | `hpa`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt; | STABLE |
| kube_hpa_spec_target_metric | Gauge | `hpa`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt; <br> `metric_name`=&lt;metric-name&gt; <br> `metric_target_type`=&lt;value\|utilization\|average&gt; | EXPERIMENTAL |
| kube_hpa_status_condition | Gauge | `hpa`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt; <br> `condition`=&lt;hpa-condition&gt; <br> `status`=&lt;true\|false\|unknown&gt; | STABLE |
| kube_hpa_status_current_replicas | Gauge | `hpa`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt; | STABLE |
| kube_hpa_status_currentmetrics_average_utilization | Gauge | `hpa`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt; | EXPERIMENTAL |
Expand Down
78 changes: 78 additions & 0 deletions internal/store/hpa.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,27 @@ import (
"k8s.io/client-go/tools/cache"
)

type MetricTargetType int

const (
Value MetricTargetType = iota
Utilization
Average

MetricTargetTypeCount // Used as a length argument to arrays
)

func (m MetricTargetType) String() string {
return [...]string{"value", "utilization", "average"}[m]
}

var (
descHorizontalPodAutoscalerLabelsName = "kube_hpa_labels"
descHorizontalPodAutoscalerLabelsHelp = "Kubernetes labels converted to Prometheus labels."
descHorizontalPodAutoscalerLabelsDefaultLabels = []string{"namespace", "hpa"}

targetMetricLabels = []string{"metric_name", "metric_target_type"}

hpaMetricFamilies = []metric.FamilyGenerator{
{
Name: "kube_hpa_metadata_generation",
Expand Down Expand Up @@ -76,6 +92,68 @@ var (
}
}),
},
{
Name: "kube_hpa_spec_target_metric",
Type: metric.Gauge,
Help: "The metric specifications used by this autoscaler when calculating the desired replica count.",
GenerateFunc: wrapHPAFunc(func(a *autoscaling.HorizontalPodAutoscaler) *metric.Family {
ms := make([]*metric.Metric, 0, len(a.Spec.Metrics))
for _, m := range a.Spec.Metrics {
var metricName string

var v [MetricTargetTypeCount]int64
var ok [MetricTargetTypeCount]bool

switch m.Type {
case autoscaling.ObjectMetricSourceType:
metricName = m.Object.MetricName

v[Value], ok[Value] = m.Object.TargetValue.AsInt64()
if m.Object.AverageValue != nil {
v[Average], ok[Average] = m.Object.AverageValue.AsInt64()
}
case autoscaling.PodsMetricSourceType:
tariq1890 marked this conversation as resolved.
Show resolved Hide resolved
metricName = m.Pods.MetricName

v[Average], ok[Average] = m.Pods.TargetAverageValue.AsInt64()
case autoscaling.ResourceMetricSourceType:
metricName = string(m.Resource.Name)

if ok[Utilization] = (m.Resource.TargetAverageUtilization != nil); ok[Utilization] {
v[Utilization] = int64(*m.Resource.TargetAverageUtilization)
}

if m.Resource.TargetAverageValue != nil {
v[Average], ok[Average] = m.Resource.TargetAverageValue.AsInt64()
}
case autoscaling.ExternalMetricSourceType:
metricName = m.External.MetricName

// The TargetValue and TargetAverageValue are mutually exclusive
if m.External.TargetValue != nil {
v[Value], ok[Value] = m.External.TargetValue.AsInt64()
}
if m.External.TargetAverageValue != nil {
v[Average], ok[Average] = m.External.TargetAverageValue.AsInt64()
}
default:
// Skip unsupported metric type
continue
}

for i := range ok {
if ok[i] {
ms = append(ms, &metric.Metric{
LabelKeys: targetMetricLabels,
LabelValues: []string{metricName, MetricTargetType(i).String()},
Value: float64(v[i]),
})
}
}
}
return &metric.Family{Metrics: ms}
}),
},
{
Name: "kube_hpa_status_current_replicas",
Type: metric.Gauge,
Expand Down
94 changes: 83 additions & 11 deletions internal/store/hpa_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ func TestHPAStore(t *testing.T) {
# TYPE kube_hpa_spec_max_replicas gauge
# HELP kube_hpa_spec_min_replicas Lower limit for the number of pods that can be set by the autoscaler, default 1.
# TYPE kube_hpa_spec_min_replicas gauge
# HELP kube_hpa_spec_target_metric The metric specifications used by this autoscaler when calculating the desired replica count.
# TYPE kube_hpa_spec_target_metric gauge
# HELP kube_hpa_status_current_replicas Current number of replicas of pods managed by this autoscaler.
# TYPE kube_hpa_status_current_replicas gauge
# HELP kube_hpa_status_desired_replicas Desired number of replicas of pods managed by this autoscaler.
Expand Down Expand Up @@ -69,6 +71,59 @@ func TestHPAStore(t *testing.T) {
Spec: autoscaling.HorizontalPodAutoscalerSpec{
MaxReplicas: 4,
MinReplicas: &hpa1MinReplicas,
Metrics: []autoscaling.MetricSpec{
{
Type: autoscaling.ObjectMetricSourceType,
Object: &autoscaling.ObjectMetricSource{
MetricName: "hits",
TargetValue: resource.MustParse("10"),
AverageValue: resourcePtr(resource.MustParse("12")),
},
},
{
Type: autoscaling.PodsMetricSourceType,
Pods: &autoscaling.PodsMetricSource{
MetricName: "transactions_processed",
TargetAverageValue: resource.MustParse("33"),
},
},
{
Type: autoscaling.ResourceMetricSourceType,
Resource: &autoscaling.ResourceMetricSource{
Name: "cpu",
TargetAverageUtilization: int32ptr(80),
},
},
{
Type: autoscaling.ResourceMetricSourceType,
Resource: &autoscaling.ResourceMetricSource{
Name: "memory",
TargetAverageUtilization: int32ptr(80),
TargetAverageValue: resourcePtr(resource.MustParse("800Ki")),
},
},
// No targets, this metric should be ignored
{
Type: autoscaling.ResourceMetricSourceType,
Resource: &autoscaling.ResourceMetricSource{
Name: "disk",
},
},
{
Type: autoscaling.ExternalMetricSourceType,
External: &autoscaling.ExternalMetricSource{
MetricName: "sqs_jobs",
TargetValue: resourcePtr(resource.MustParse("30")),
},
},
{
Type: autoscaling.ExternalMetricSourceType,
External: &autoscaling.ExternalMetricSource{
MetricName: "events",
TargetAverageValue: resourcePtr(resource.MustParse("30")),
},
},
},
ScaleTargetRef: autoscaling.CrossVersionObjectReference{
APIVersion: "apps/v1",
Kind: "Deployment",
Expand Down Expand Up @@ -98,22 +153,31 @@ func TestHPAStore(t *testing.T) {
},
},
Want: metadata + `
kube_hpa_labels{hpa="hpa1",label_app="foobar",namespace="ns1"} 1
kube_hpa_metadata_generation{hpa="hpa1",namespace="ns1"} 2
kube_hpa_spec_max_replicas{hpa="hpa1",namespace="ns1"} 4
kube_hpa_spec_min_replicas{hpa="hpa1",namespace="ns1"} 2
kube_hpa_status_condition{condition="AbleToScale",hpa="hpa1",namespace="ns1",status="false"} 0
kube_hpa_status_condition{condition="AbleToScale",hpa="hpa1",namespace="ns1",status="true"} 1
kube_hpa_status_condition{condition="AbleToScale",hpa="hpa1",namespace="ns1",status="unknown"} 0
kube_hpa_status_current_replicas{hpa="hpa1",namespace="ns1"} 2
kube_hpa_status_desired_replicas{hpa="hpa1",namespace="ns1"} 2
kube_hpa_status_currentmetrics_average_value{hpa="hpa1",namespace="ns1"} 10
kube_hpa_status_currentmetrics_average_utilization{hpa="hpa1",namespace="ns1"} 0
kube_hpa_labels{hpa="hpa1",label_app="foobar",namespace="ns1"} 1
kube_hpa_metadata_generation{hpa="hpa1",namespace="ns1"} 2
kube_hpa_spec_max_replicas{hpa="hpa1",namespace="ns1"} 4
kube_hpa_spec_min_replicas{hpa="hpa1",namespace="ns1"} 2
kube_hpa_spec_target_metric{hpa="hpa1",namespace="ns1",metric_name="hits",metric_target_type="value"} 10
kube_hpa_spec_target_metric{hpa="hpa1",namespace="ns1",metric_name="hits",metric_target_type="average"} 12
kube_hpa_spec_target_metric{hpa="hpa1",namespace="ns1",metric_name="transactions_processed",metric_target_type="average"} 33
kube_hpa_spec_target_metric{hpa="hpa1",namespace="ns1",metric_name="cpu",metric_target_type="utilization"} 80
kube_hpa_spec_target_metric{hpa="hpa1",namespace="ns1",metric_name="memory",metric_target_type="utilization"} 80
kube_hpa_spec_target_metric{hpa="hpa1",namespace="ns1",metric_name="memory",metric_target_type="average"} 819200
kube_hpa_spec_target_metric{hpa="hpa1",namespace="ns1",metric_name="sqs_jobs",metric_target_type="value"} 30
kube_hpa_spec_target_metric{hpa="hpa1",namespace="ns1",metric_name="events",metric_target_type="average"} 30
kube_hpa_status_condition{condition="AbleToScale",hpa="hpa1",namespace="ns1",status="false"} 0
kube_hpa_status_condition{condition="AbleToScale",hpa="hpa1",namespace="ns1",status="true"} 1
kube_hpa_status_condition{condition="AbleToScale",hpa="hpa1",namespace="ns1",status="unknown"} 0
kube_hpa_status_current_replicas{hpa="hpa1",namespace="ns1"} 2
kube_hpa_status_desired_replicas{hpa="hpa1",namespace="ns1"} 2
kube_hpa_status_currentmetrics_average_value{hpa="hpa1",namespace="ns1"} 10
kube_hpa_status_currentmetrics_average_utilization{hpa="hpa1",namespace="ns1"} 0
`,
MetricNames: []string{
"kube_hpa_metadata_generation",
"kube_hpa_spec_max_replicas",
"kube_hpa_spec_min_replicas",
"kube_hpa_spec_target_metric",
"kube_hpa_status_current_replicas",
"kube_hpa_status_desired_replicas",
"kube_hpa_status_condition",
Expand All @@ -131,3 +195,11 @@ func TestHPAStore(t *testing.T) {
}
}
}

func int32ptr(value int32) *int32 {
return &value
}

func resourcePtr(quantity resource.Quantity) *resource.Quantity {
return &quantity
}
tariq1890 marked this conversation as resolved.
Show resolved Hide resolved