Skip to content
Permalink
Browse files

migrate kube-apiserver metrics to stability framework

  • Loading branch information...
logicalhan committed Aug 19, 2019
1 parent b149157 commit 115d4f973d0430a2bdde71c4ccb31a6111138310
Showing with 291 additions and 217 deletions.
  1. +1 −2 cmd/kube-apiserver/BUILD
  2. +2 −2 cmd/kube-apiserver/apiserver.go
  3. +2 −2 cmd/kube-apiserver/app/BUILD
  4. +2 −2 cmd/kube-apiserver/app/server.go
  5. +2 −1 pkg/ssh/BUILD
  6. +14 −11 pkg/ssh/ssh.go
  7. +2 −0 staging/publishing/import-restrictions.yaml
  8. +2 −1 staging/src/k8s.io/apiserver/pkg/admission/metrics/BUILD
  9. +23 −21 staging/src/k8s.io/apiserver/pkg/admission/metrics/metrics.go
  10. +2 −1 staging/src/k8s.io/apiserver/pkg/audit/BUILD
  11. +24 −19 staging/src/k8s.io/apiserver/pkg/audit/metrics.go
  12. +2 −1 staging/src/k8s.io/apiserver/pkg/authentication/request/x509/BUILD
  13. +6 −5 staging/src/k8s.io/apiserver/pkg/authentication/request/x509/x509.go
  14. +2 −1 staging/src/k8s.io/apiserver/pkg/endpoints/filters/BUILD
  15. +9 −8 staging/src/k8s.io/apiserver/pkg/endpoints/filters/authentication.go
  16. +2 −0 staging/src/k8s.io/apiserver/pkg/endpoints/metrics/BUILD
  17. +63 −49 staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go
  18. +1 −1 staging/src/k8s.io/apiserver/pkg/server/routes/BUILD
  19. +3 −4 staging/src/k8s.io/apiserver/pkg/server/routes/metrics.go
  20. +4 −1 staging/src/k8s.io/apiserver/pkg/storage/etcd3/metrics/BUILD
  21. +20 −16 staging/src/k8s.io/apiserver/pkg/storage/etcd3/metrics/metrics.go
  22. +1 −0 staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/BUILD
  23. +5 −0 staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go
  24. +2 −0 staging/src/k8s.io/apiserver/pkg/storage/value/BUILD
  25. +63 −53 staging/src/k8s.io/apiserver/pkg/storage/value/metrics.go
  26. +5 −5 staging/src/k8s.io/apiserver/pkg/storage/value/metrics_test.go
  27. +5 −0 staging/src/k8s.io/component-base/metrics/metric.go
  28. +2 −0 staging/src/k8s.io/legacy-cloud-providers/gce/BUILD
  29. +14 −11 staging/src/k8s.io/legacy-cloud-providers/gce/token_source.go
  30. +1 −0 staging/src/k8s.io/legacy-cloud-providers/go.sum
  31. +5 −0 vendor/modules.txt
@@ -20,9 +20,8 @@ go_library(
importpath = "k8s.io/kubernetes/cmd/kube-apiserver",
deps = [
"//cmd/kube-apiserver/app:go_default_library",
"//pkg/util/prometheusclientgo:go_default_library",
"//pkg/version/prometheus:go_default_library",
"//staging/src/k8s.io/component-base/logs:go_default_library",
"//staging/src/k8s.io/component-base/metrics/prometheus/clientgo:go_default_library",
],
)

@@ -24,12 +24,12 @@ import (
"time"

"k8s.io/component-base/logs"
_ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugins
"k8s.io/kubernetes/cmd/kube-apiserver/app"
_ "k8s.io/kubernetes/pkg/util/prometheusclientgo" // load all the prometheus client-go plugins
_ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration
)

func main() {

rand.Seed(time.Now().UnixNano())

command := app.NewAPIServerCommand()
@@ -30,8 +30,6 @@ go_library(
"//pkg/registry/rbac/rest:go_default_library",
"//pkg/serviceaccount:go_default_library",
"//pkg/util/flag:go_default_library",
"//pkg/util/workqueue/prometheus:go_default_library",
"//pkg/version:go_default_library",
"//pkg/version/verflag:go_default_library",
"//plugin/pkg/auth/authenticator/token/bootstrap:go_default_library",
"//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
@@ -66,6 +64,8 @@ go_library(
"//staging/src/k8s.io/cloud-provider:go_default_library",
"//staging/src/k8s.io/component-base/cli/flag:go_default_library",
"//staging/src/k8s.io/component-base/cli/globalflag:go_default_library",
"//staging/src/k8s.io/component-base/metrics/prometheus/workqueue:go_default_library",
"//staging/src/k8s.io/component-base/version:go_default_library",
"//staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1:go_default_library",
"//staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helper:go_default_library",
"//staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1:go_default_library",
@@ -58,6 +58,8 @@ import (
cloudprovider "k8s.io/cloud-provider"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/cli/globalflag"
_ "k8s.io/component-base/metrics/prometheus/workqueue" // for workqueue metric registration
"k8s.io/component-base/version"
"k8s.io/klog"
aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver"
aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
@@ -80,8 +82,6 @@ import (
rbacrest "k8s.io/kubernetes/pkg/registry/rbac/rest"
"k8s.io/kubernetes/pkg/serviceaccount"
utilflag "k8s.io/kubernetes/pkg/util/flag"
_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus" // for workqueue metric registration
"k8s.io/kubernetes/pkg/version"
"k8s.io/kubernetes/pkg/version/verflag"
"k8s.io/kubernetes/plugin/pkg/auth/authenticator/token/bootstrap"
)
@@ -25,7 +25,8 @@ go_library(
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
"//staging/src/k8s.io/component-base/metrics:go_default_library",
"//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library",
"//vendor/golang.org/x/crypto/ssh:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
],
@@ -36,33 +36,36 @@ import (
"sync"
"time"

"github.com/prometheus/client_golang/prometheus"
"golang.org/x/crypto/ssh"

utilnet "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
"k8s.io/klog"
)

var (
tunnelOpenCounter = prometheus.NewCounter(
prometheus.CounterOpts{
Name: "ssh_tunnel_open_count",
Help: "Counter of ssh tunnel total open attempts",
tunnelOpenCounter = metrics.NewCounter(
&metrics.CounterOpts{
Name: "ssh_tunnel_open_count",
Help: "Counter of ssh tunnel total open attempts",
StabilityLevel: metrics.ALPHA,
},
)
tunnelOpenFailCounter = prometheus.NewCounter(
prometheus.CounterOpts{
Name: "ssh_tunnel_open_fail_count",
Help: "Counter of ssh tunnel failed open attempts",
tunnelOpenFailCounter = metrics.NewCounter(
&metrics.CounterOpts{
Name: "ssh_tunnel_open_fail_count",
Help: "Counter of ssh tunnel failed open attempts",
StabilityLevel: metrics.ALPHA,
},
)
)

func init() {
prometheus.MustRegister(tunnelOpenCounter)
prometheus.MustRegister(tunnelOpenFailCounter)
legacyregistry.MustRegister(tunnelOpenCounter)
legacyregistry.MustRegister(tunnelOpenFailCounter)
}

// TODO: Unit tests for this code, we can spin up a test SSH server with instructions here:
@@ -233,6 +233,8 @@
- k8s.io/utils
- k8s.io/apiserver/pkg/util/feature
- k8s.io/component-base/featuregate
- k8s.io/component-base/metrics
- k8s.io/component-base/metrics/legacyregistry

- baseImportPath: "./vendor/k8s.io/node-api/"
allowedImports:
@@ -8,7 +8,8 @@ go_library(
visibility = ["//visibility:public"],
deps = [
"//staging/src/k8s.io/apiserver/pkg/admission:go_default_library",
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
"//staging/src/k8s.io/component-base/metrics:go_default_library",
"//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library",
],
)

@@ -21,9 +21,9 @@ import (
"strconv"
"time"

"github.com/prometheus/client_golang/prometheus"

"k8s.io/apiserver/pkg/admission"
"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
)

const (
@@ -153,33 +153,35 @@ func (m *AdmissionMetrics) ObserveWebhook(elapsed time.Duration, rejected bool,
}

type metricSet struct {
latencies *prometheus.HistogramVec
latenciesSummary *prometheus.SummaryVec
latencies *metrics.HistogramVec
latenciesSummary *metrics.SummaryVec
}

func newMetricSet(name string, labels []string, helpTemplate string, hasSummary bool) *metricSet {
var summary *prometheus.SummaryVec
var summary *metrics.SummaryVec
if hasSummary {
summary = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: fmt.Sprintf("%s_admission_duration_seconds_summary", name),
Help: fmt.Sprintf(helpTemplate, "latency summary in seconds"),
MaxAge: latencySummaryMaxAge,
summary = metrics.NewSummaryVec(
&metrics.SummaryOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: fmt.Sprintf("%s_admission_duration_seconds_summary", name),
Help: fmt.Sprintf(helpTemplate, "latency summary in seconds"),
MaxAge: latencySummaryMaxAge,
StabilityLevel: metrics.ALPHA,
},
labels,
)
}

return &metricSet{
latencies: prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: fmt.Sprintf("%s_admission_duration_seconds", name),
Help: fmt.Sprintf(helpTemplate, "latency histogram in seconds"),
Buckets: latencyBuckets,
latencies: metrics.NewHistogramVec(
&metrics.HistogramOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: fmt.Sprintf("%s_admission_duration_seconds", name),
Help: fmt.Sprintf(helpTemplate, "latency histogram in seconds"),
Buckets: latencyBuckets,
StabilityLevel: metrics.ALPHA,
},
labels,
),
@@ -190,9 +192,9 @@ func newMetricSet(name string, labels []string, helpTemplate string, hasSummary

// MustRegister registers all the prometheus metrics in the metricSet.
func (m *metricSet) mustRegister() {
prometheus.MustRegister(m.latencies)
legacyregistry.MustRegister(m.latencies)
if m.latenciesSummary != nil {
prometheus.MustRegister(m.latenciesSummary)
legacyregistry.MustRegister(m.latenciesSummary)
}
}

@@ -34,8 +34,9 @@ go_library(
"//staging/src/k8s.io/apiserver/pkg/apis/audit/v1beta1:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library",
"//staging/src/k8s.io/component-base/metrics:go_default_library",
"//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library",
"//vendor/github.com/pborman/uuid:go_default_library",
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
],
)
@@ -19,8 +19,9 @@ package audit
import (
"fmt"

"github.com/prometheus/client_golang/prometheus"
auditinternal "k8s.io/apiserver/pkg/apis/audit"
"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
"k8s.io/klog"
)

@@ -29,45 +30,49 @@ const (
)

var (
eventCounter = prometheus.NewCounter(
prometheus.CounterOpts{
Subsystem: subsystem,
Name: "event_total",
Help: "Counter of audit events generated and sent to the audit backend.",
eventCounter = metrics.NewCounter(
&metrics.CounterOpts{
Subsystem: subsystem,
Name: "event_total",
Help: "Counter of audit events generated and sent to the audit backend.",
StabilityLevel: metrics.ALPHA,
})
errorCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{
errorCounter = metrics.NewCounterVec(
&metrics.CounterOpts{
Subsystem: subsystem,
Name: "error_total",
Help: "Counter of audit events that failed to be audited properly. " +
"Plugin identifies the plugin affected by the error.",
StabilityLevel: metrics.ALPHA,
},
[]string{"plugin"},
)
levelCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{
Subsystem: subsystem,
Name: "level_total",
Help: "Counter of policy levels for audit events (1 per request).",
levelCounter = metrics.NewCounterVec(
&metrics.CounterOpts{
Subsystem: subsystem,
Name: "level_total",
Help: "Counter of policy levels for audit events (1 per request).",
StabilityLevel: metrics.ALPHA,
},
[]string{"level"},
)

ApiserverAuditDroppedCounter = prometheus.NewCounter(
prometheus.CounterOpts{
ApiserverAuditDroppedCounter = metrics.NewCounter(
&metrics.CounterOpts{
Subsystem: subsystem,
Name: "requests_rejected_total",
Help: "Counter of apiserver requests rejected due to an error " +
"in audit logging backend.",
StabilityLevel: metrics.ALPHA,
},
)
)

func init() {
prometheus.MustRegister(eventCounter)
prometheus.MustRegister(errorCounter)
prometheus.MustRegister(levelCounter)
prometheus.MustRegister(ApiserverAuditDroppedCounter)
legacyregistry.MustRegister(eventCounter)
legacyregistry.MustRegister(errorCounter)
legacyregistry.MustRegister(levelCounter)
legacyregistry.MustRegister(ApiserverAuditDroppedCounter)
}

// ObserveEvent updates the relevant prometheus metrics for the generated audit event.
@@ -36,7 +36,8 @@ go_library(
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library",
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
"//staging/src/k8s.io/component-base/metrics:go_default_library",
"//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library",
],
)

@@ -23,16 +23,16 @@ import (
"net/http"
"time"

"github.com/prometheus/client_golang/prometheus"

utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apiserver/pkg/authentication/authenticator"
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
)

var clientCertificateExpirationHistogram = prometheus.NewHistogram(
prometheus.HistogramOpts{
var clientCertificateExpirationHistogram = metrics.NewHistogram(
&metrics.HistogramOpts{
Namespace: "apiserver",
Subsystem: "client",
Name: "certificate_expiration_seconds",
@@ -53,11 +53,12 @@ var clientCertificateExpirationHistogram = prometheus.NewHistogram(
(6 * 30 * 24 * time.Hour).Seconds(),
(12 * 30 * 24 * time.Hour).Seconds(),
},
StabilityLevel: metrics.ALPHA,
},
)

func init() {
prometheus.MustRegister(clientCertificateExpirationHistogram)
legacyregistry.MustRegister(clientCertificateExpirationHistogram)
}

// UserConversion defines an interface for extracting user info from a client certificate chain
@@ -68,7 +68,8 @@ go_library(
"//staging/src/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/server/httplog:go_default_library",
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
"//staging/src/k8s.io/component-base/metrics:go_default_library",
"//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
],
)
@@ -21,29 +21,30 @@ import (
"net/http"
"strings"

"github.com/prometheus/client_golang/prometheus"
"k8s.io/klog"

apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/authentication/authenticator"
"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
"k8s.io/klog"
)

var (
authenticatedUserCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "authenticated_user_requests",
Help: "Counter of authenticated requests broken out by username.",
authenticatedUserCounter = metrics.NewCounterVec(
&metrics.CounterOpts{
Name: "authenticated_user_requests",
Help: "Counter of authenticated requests broken out by username.",
StabilityLevel: metrics.ALPHA,
},
[]string{"username"},
)
)

func init() {
prometheus.MustRegister(authenticatedUserCounter)
legacyregistry.MustRegister(authenticatedUserCounter)
}

// WithAuthentication creates an http handler that tries to authenticate the given request as a user, and then

0 comments on commit 115d4f9

Please sign in to comment.
You can’t perform that action at this time.