Skip to content

Commit

Permalink
Revert "Add a namespace label to admission metrics and expand histogr…
Browse files Browse the repository at this point in the history 
…am range to 0-10s"
  • Loading branch information
@s-urbaniak @rphillips
s-urbaniak authored and rphillips committed Jul 30, 2021
1 parent ff834e7 commit 9ffddde
Show file tree
Hide file tree
Showing 4 changed files with 20 additions and 26 deletions.
22 changes: 11 additions & 11 deletions staging/src/k8s.io/apiserver/pkg/admission/metrics/metrics.go
View file
Expand Up @@ -45,8 +45,8 @@ const (
)

var (
// Use buckets ranging from 5 ms to 10 seconds (admission webhooks timeout at 30 seconds by default).
latencyBuckets = []float64{0.005, 0.025, 0.1, 0.5, 2.5, 5.0, 10.0}
// Use buckets ranging from 5 ms to 2.5 seconds (admission webhooks timeout at 30 seconds by default).
latencyBuckets = []float64{0.005, 0.025, 0.1, 0.5, 2.5}
latencySummaryMaxAge = 5 * time.Hour

// Metrics provides access to all admission metrics.
Expand Down Expand Up @@ -126,17 +126,17 @@ func newAdmissionMetrics() *AdmissionMetrics {
// Admission metrics for a step of the admission flow. The entire admission flow is broken down into a series of steps
// Each step is identified by a distinct type label value.
step := newMetricSet("step",
[]string{"type", "operation", "rejected", "namespace"},
[]string{"type", "operation", "rejected"},
"Admission sub-step %s, broken out for each operation and API resource and step type (validate or admit).", true)

// Built-in admission controller metrics. Each admission controller is identified by name.
controller := newMetricSet("controller",
[]string{"name", "type", "operation", "rejected", "namespace"},
[]string{"name", "type", "operation", "rejected"},
"Admission controller %s, identified by name and broken out for each operation and API resource and type (validate or admit).", false)

// Admission webhook metrics. Each webhook is identified by name.
webhook := newMetricSet("webhook",
[]string{"name", "type", "operation", "rejected", "namespace"},
[]string{"name", "type", "operation", "rejected"},
"Admission webhook %s, identified by name and broken out for each operation and API resource and type (validate or admit).", false)

webhookRejection := metrics.NewCounterVec(
Expand All @@ -147,7 +147,7 @@ func newAdmissionMetrics() *AdmissionMetrics {
Help: "Admission webhook rejection count, identified by name and broken out for each admission type (validating or admit) and operation. Additional labels specify an error type (calling_webhook_error or apiserver_internal_error if an error occurred; no_error otherwise) and optionally a non-zero rejection code if the webhook rejects the request with an HTTP status code (honored by the apiserver when the code is greater or equal to 400). Codes greater than 600 are truncated to 600, to keep the metrics cardinality bounded.",
StabilityLevel: metrics.ALPHA,
},
[]string{"name", "type", "operation", "error_type", "rejection_code", "namespace"})
[]string{"name", "type", "operation", "error_type", "rejection_code"})

step.mustRegister()
controller.mustRegister()
Expand All @@ -164,27 +164,27 @@ func (m *AdmissionMetrics) reset() {

// ObserveAdmissionStep records admission related metrics for a admission step, identified by step type.
func (m *AdmissionMetrics) ObserveAdmissionStep(ctx context.Context, elapsed time.Duration, rejected bool, attr admission.Attributes, stepType string, extraLabels ...string) {
m.step.observe(ctx, elapsed, append(extraLabels, stepType, string(attr.GetOperation()), strconv.FormatBool(rejected), attr.GetNamespace())...)
m.step.observe(ctx, elapsed, append(extraLabels, stepType, string(attr.GetOperation()), strconv.FormatBool(rejected))...)
}

// ObserveAdmissionController records admission related metrics for a built-in admission controller, identified by it's plugin handler name.
func (m *AdmissionMetrics) ObserveAdmissionController(ctx context.Context, elapsed time.Duration, rejected bool, attr admission.Attributes, stepType string, extraLabels ...string) {
m.controller.observe(ctx, elapsed, append(extraLabels, stepType, string(attr.GetOperation()), strconv.FormatBool(rejected), attr.GetNamespace())...)
m.controller.observe(ctx, elapsed, append(extraLabels, stepType, string(attr.GetOperation()), strconv.FormatBool(rejected))...)
}

// ObserveWebhook records admission related metrics for a admission webhook.
func (m *AdmissionMetrics) ObserveWebhook(ctx context.Context, elapsed time.Duration, rejected bool, attr admission.Attributes, stepType string, extraLabels ...string) {
m.webhook.observe(ctx, elapsed, append(extraLabels, stepType, string(attr.GetOperation()), strconv.FormatBool(rejected), attr.GetNamespace())...)
m.webhook.observe(ctx, elapsed, append(extraLabels, stepType, string(attr.GetOperation()), strconv.FormatBool(rejected))...)
}

// ObserveWebhookRejection records admission related metrics for an admission webhook rejection.
func (m *AdmissionMetrics) ObserveWebhookRejection(ctx context.Context, name, stepType string, attr admission.Attributes, errorType WebhookRejectionErrorType, rejectionCode int) {
func (m *AdmissionMetrics) ObserveWebhookRejection(ctx context.Context, name, stepType, operation string, errorType WebhookRejectionErrorType, rejectionCode int) {
// We truncate codes greater than 600 to keep the cardinality bounded.
// This should be rarely done by a malfunctioning webhook server.
if rejectionCode > 600 {
rejectionCode = 600
}
m.webhookRejection.WithContext(ctx).WithLabelValues(name, stepType, string(attr.GetOperation()), string(errorType), strconv.Itoa(rejectionCode), attr.GetNamespace()).Inc()
m.webhookRejection.WithContext(ctx).WithLabelValues(name, stepType, operation, string(errorType), strconv.Itoa(rejectionCode)).Inc()
}

type metricSet struct {
Expand Down
12 changes: 3 additions & 9 deletions staging/src/k8s.io/apiserver/pkg/admission/metrics/metrics_test.go
View file
Expand Up @@ -49,7 +49,6 @@ func TestObserveAdmissionStep(t *testing.T) {
"operation": string(admission.Create),
"type": "admit",
"rejected": "false",
"namespace": "ns",
}
expectHistogramCountTotal(t, "apiserver_admission_step_admission_duration_seconds", wantLabels, 1)
expectFindMetric(t, "apiserver_admission_step_admission_duration_seconds_summary", wantLabels)
Expand All @@ -74,7 +73,6 @@ func TestObserveAdmissionController(t *testing.T) {
"operation": string(admission.Create),
"type": "admit",
"rejected": "false",
"namespace": "ns",
}
expectHistogramCountTotal(t, "apiserver_admission_controller_admission_duration_seconds", wantLabels, 1)

Expand All @@ -91,39 +89,35 @@ func TestObserveWebhook(t *testing.T) {
"operation": string(admission.Create),
"type": "admit",
"rejected": "false",
"namespace": "ns",
}
expectHistogramCountTotal(t, "apiserver_admission_webhook_admission_duration_seconds", wantLabels, 1)
}

func TestObserveWebhookRejection(t *testing.T) {
Metrics.reset()
Metrics.ObserveWebhookRejection(context.TODO(), "x", stepAdmit, attr, WebhookRejectionNoError, 500)
Metrics.ObserveWebhookRejection(context.TODO(), "x", stepAdmit, attr, WebhookRejectionNoError, 654)
Metrics.ObserveWebhookRejection(context.TODO(), "x", stepValidate, admission.NewAttributesRecord(nil, nil, kind, "ns", "name", resource, "subresource", admission.Update, &metav1.UpdateOptions{}, false, nil), WebhookRejectionCallingWebhookError, 0)
Metrics.ObserveWebhookRejection(context.TODO(), "x", stepAdmit, string(admission.Create), WebhookRejectionNoError, 500)
Metrics.ObserveWebhookRejection(context.TODO(), "x", stepAdmit, string(admission.Create), WebhookRejectionNoError, 654)
Metrics.ObserveWebhookRejection(context.TODO(), "x", stepValidate, string(admission.Update), WebhookRejectionCallingWebhookError, 0)
wantLabels := map[string]string{
"name": "x",
"operation": string(admission.Create),
"type": "admit",
"error_type": "no_error",
"rejection_code": "500",
"namespace": "ns",
}
wantLabels600 := map[string]string{
"name": "x",
"operation": string(admission.Create),
"type": "admit",
"error_type": "no_error",
"rejection_code": "600",
"namespace": "ns",
}
wantLabelsCallingWebhookError := map[string]string{
"name": "x",
"operation": string(admission.Update),
"type": "validate",
"error_type": "calling_webhook_error",
"rejection_code": "0",
"namespace": "ns",
}
expectCounterValue(t, "apiserver_admission_webhook_rejection_count", wantLabels, 1)
expectCounterValue(t, "apiserver_admission_webhook_rejection_count", wantLabels600, 1)
Expand Down
6 changes: 3 additions & 3 deletions staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/mutating/dispatcher.go
View file
Expand Up @@ -148,14 +148,14 @@ func (a *mutatingDispatcher) Dispatch(ctx context.Context, attr admission.Attrib
case *webhookutil.ErrCallingWebhook:
if !ignoreClientCallFailures {
rejected = true
admissionmetrics.Metrics.ObserveWebhookRejection(ctx, hook.Name, "admit", versionedAttr.Attributes, admissionmetrics.WebhookRejectionCallingWebhookError, 0)
admissionmetrics.Metrics.ObserveWebhookRejection(ctx, hook.Name, "admit", string(versionedAttr.Attributes.GetOperation()), admissionmetrics.WebhookRejectionCallingWebhookError, 0)
}
case *webhookutil.ErrWebhookRejection:
rejected = true
admissionmetrics.Metrics.ObserveWebhookRejection(ctx, hook.Name, "admit", versionedAttr.Attributes, admissionmetrics.WebhookRejectionNoError, int(err.Status.ErrStatus.Code))
admissionmetrics.Metrics.ObserveWebhookRejection(ctx, hook.Name, "admit", string(versionedAttr.Attributes.GetOperation()), admissionmetrics.WebhookRejectionNoError, int(err.Status.ErrStatus.Code))
default:
rejected = true
admissionmetrics.Metrics.ObserveWebhookRejection(ctx, hook.Name, "admit", versionedAttr.Attributes, admissionmetrics.WebhookRejectionAPIServerInternalError, 0)
admissionmetrics.Metrics.ObserveWebhookRejection(ctx, hook.Name, "admit", string(versionedAttr.Attributes.GetOperation()), admissionmetrics.WebhookRejectionAPIServerInternalError, 0)
}
}
admissionmetrics.Metrics.ObserveWebhook(ctx, time.Since(t), rejected, versionedAttr.Attributes, "admit", hook.Name)
Expand Down
6 changes: 3 additions & 3 deletions staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/validating/dispatcher.go
View file
Expand Up @@ -119,14 +119,14 @@ func (d *validatingDispatcher) Dispatch(ctx context.Context, attr admission.Attr
case *webhookutil.ErrCallingWebhook:
if !ignoreClientCallFailures {
rejected = true
admissionmetrics.Metrics.ObserveWebhookRejection(ctx, hook.Name, "validating", versionedAttr.Attributes, admissionmetrics.WebhookRejectionCallingWebhookError, 0)
admissionmetrics.Metrics.ObserveWebhookRejection(ctx, hook.Name, "validating", string(versionedAttr.Attributes.GetOperation()), admissionmetrics.WebhookRejectionCallingWebhookError, 0)
}
case *webhookutil.ErrWebhookRejection:
rejected = true
admissionmetrics.Metrics.ObserveWebhookRejection(ctx, hook.Name, "validating", versionedAttr.Attributes, admissionmetrics.WebhookRejectionNoError, int(err.Status.ErrStatus.Code))
admissionmetrics.Metrics.ObserveWebhookRejection(ctx, hook.Name, "validating", string(versionedAttr.Attributes.GetOperation()), admissionmetrics.WebhookRejectionNoError, int(err.Status.ErrStatus.Code))
default:
rejected = true
admissionmetrics.Metrics.ObserveWebhookRejection(ctx, hook.Name, "validating", versionedAttr.Attributes, admissionmetrics.WebhookRejectionAPIServerInternalError, 0)
admissionmetrics.Metrics.ObserveWebhookRejection(ctx, hook.Name, "validating", string(versionedAttr.Attributes.GetOperation()), admissionmetrics.WebhookRejectionAPIServerInternalError, 0)
}
}
admissionmetrics.Metrics.ObserveWebhook(ctx, time.Since(t), rejected, versionedAttr.Attributes, "validating", hook.Name)
Expand Down

0 comments on commit 9ffddde

Please sign in to comment.