From 5bde1cdb1efc48fa372ae7ceae9f74fbe22b43c1 Mon Sep 17 00:00:00 2001 From: ricoberger Date: Fri, 6 Aug 2021 16:49:47 +0200 Subject: [PATCH] Add Prometheus metrics for API requests We are adding a new middleware, which is used to generate Prometheus metrics for all API requests. The middleware generates two metrics for each status, method and path: kobs_chi_requests_total and kobs_chi_request_duration_milliseconds. These metrics can be used to monitor the performance of all kobs API requests. We also add an option to the kobs Helm chart to create a Service Monitor for the Prometheus Operator, which can be used to collect these metrics. --- deploy/helm/kobs/Chart.yaml | 2 +- deploy/helm/kobs/templates/_helpers.tpl | 9 ++++ .../helm/kobs/templates/servicemonitor.yaml | 34 +++++++++++++++ deploy/helm/kobs/values.yaml | 41 +++++++++++++++++++ docs/installation/helm.md | 7 ++++ pkg/api/api.go | 2 + pkg/api/middleware/metrics/metrics.go | 39 ++++++++++++++++++ 7 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 deploy/helm/kobs/templates/servicemonitor.yaml create mode 100644 pkg/api/middleware/metrics/metrics.go diff --git a/deploy/helm/kobs/Chart.yaml b/deploy/helm/kobs/Chart.yaml index beeae6785..9c86ab7fa 100644 --- a/deploy/helm/kobs/Chart.yaml +++ b/deploy/helm/kobs/Chart.yaml @@ -4,5 +4,5 @@ description: Kubernetes Observability Platform type: application home: https://kobs.io icon: https://kobs.io/assets/images/logo.svg -version: 0.5.3 +version: 0.5.4 appVersion: v0.5.0 diff --git a/deploy/helm/kobs/templates/_helpers.tpl b/deploy/helm/kobs/templates/_helpers.tpl index 96a629a4d..8d39a053c 100644 --- a/deploy/helm/kobs/templates/_helpers.tpl +++ b/deploy/helm/kobs/templates/_helpers.tpl @@ -108,3 +108,12 @@ Additional labels for the Service {{- toYaml .Values.service.labels }} {{- end }} {{- end }} + +{{/* +Additional labels for the Service Monitor +*/}} +{{- define "kobs.serviceMonitorLabels" -}} +{{- if .Values.serviceMonitor.labels }} +{{- toYaml .Values.serviceMonitor.labels }} +{{- end }} +{{- end }} diff --git a/deploy/helm/kobs/templates/servicemonitor.yaml b/deploy/helm/kobs/templates/servicemonitor.yaml new file mode 100644 index 000000000..95ce1ed13 --- /dev/null +++ b/deploy/helm/kobs/templates/servicemonitor.yaml @@ -0,0 +1,34 @@ +{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "kobs.fullname" . }} + labels: + {{- include "kobs.labels" . | nindent 4 }} + {{- include "kobs.serviceMonitorLabels" . | nindent 4 }} +spec: + endpoints: + - port: http-metrics + {{- if .Values.serviceMonitor.interval }} + interval: {{ .Values.serviceMonitor.interval }} + {{- end }} + {{- if .Values.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.serviceMonitor.scrapeTimeout }} + {{- end }} + path: /metrics + honorLabels: {{ .Values.serviceMonitor.honorLabels }} + {{- if .Values.serviceMonitor.metricRelabelings }} + metricRelabelings: + {{ toYaml .Values.serviceMonitor.metricRelabelings | nindent 6 }} + {{- end }} + {{- if .Values.serviceMonitor.relabelings }} + relabelings: + {{ toYaml .Values.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + selector: + matchLabels: + {{- include "kobs.selectorLabels" . | nindent 6 }} +{{- end -}} diff --git a/deploy/helm/kobs/values.yaml b/deploy/helm/kobs/values.yaml index 8cbe5e1d4..18f447132 100644 --- a/deploy/helm/kobs/values.yaml +++ b/deploy/helm/kobs/values.yaml @@ -167,3 +167,44 @@ ingress: # - secretName: chart-example-tls # hosts: # - chart-example.local + +## Create a Service Monitor for the Prometheus Operator. +## See: https://github.com/coreos/prometheus-operator +## +serviceMonitor: + enabled: false + + ## Interval at which metrics should be scraped. Fallback to the Prometheus default unless specified. + ## + # interval: 10s + + ## Timeout after which the scrape is ended. Fallback to the Prometheus default unless specified. + ## + # scrapeTimeout: 30s + + ## Additional labels that are used by the Prometheus installed in your cluster to select Service Monitors to work with + ## See: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#prometheusspec + ## + labels: {} + + ## HonorLabels chooses the metric's labels on collisions with target labels. + ## + honorLabels: true + + ## MetricRelabelConfigs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping. Prometheus Operator automatically adds relabelings for a few + ## standard Kubernetes fields and replaces original scrape job name with __tmp_prometheus_job_name. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace diff --git a/docs/installation/helm.md b/docs/installation/helm.md index c7367db93..9be50a3e5 100644 --- a/docs/installation/helm.md +++ b/docs/installation/helm.md @@ -85,3 +85,10 @@ helm upgrade kobs kobs/kobs | `ingress.annotations` | Annotations to add to the ingress. | `{}` | | `ingress.hosts` | Hosts to use for the ingress. | `[]` | | `ingress.tls` | TLS configuration for the ingress. | `[]` | +| `serviceMonitor.enabled` | Create a Service Monitor for kobs. | `false` | +| `serviceMonitor.interval` | Interval at which metrics should be scraped. Fallback to the Prometheus default unless specified. | | +| `serviceMonitor.scrapeTimeout` | Timeout after which the scrape is ended. Fallback to the Prometheus default unless specified. | | +| `serviceMonitor.labels` | Additional labels for the the Service Monitor. | `{}` | +| `serviceMonitor.honorLabels` | Chooses the metric's labels on collisions with target labels. | `false` | +| `serviceMonitor.metricRelabelings` | Metric relabel config. | `[]` | +| `serviceMonitor.relabelings` | Relabel config. | `[]` | diff --git a/pkg/api/api.go b/pkg/api/api.go index d7b4e6ca3..548d146a8 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -9,6 +9,7 @@ import ( "github.com/kobsio/kobs/pkg/api/clusters" "github.com/kobsio/kobs/pkg/api/middleware/auth" "github.com/kobsio/kobs/pkg/api/middleware/httplog" + "github.com/kobsio/kobs/pkg/api/middleware/metrics" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" @@ -72,6 +73,7 @@ func New(loadedClusters *clusters.Clusters, pluginsRouter chi.Router, isDevelopm router.Use(middleware.RequestID) router.Use(middleware.Recoverer) router.Use(middleware.URLFormat) + router.Use(metrics.Metrics) router.Use(auth.Auth) router.Use(httplog.NewStructuredLogger(log.Logger)) router.Use(render.SetContentType(render.ContentTypeJSON)) diff --git a/pkg/api/middleware/metrics/metrics.go b/pkg/api/middleware/metrics/metrics.go new file mode 100644 index 000000000..31d82ad0d --- /dev/null +++ b/pkg/api/middleware/metrics/metrics.go @@ -0,0 +1,39 @@ +package metrics + +import ( + "net/http" + "time" + + "github.com/go-chi/chi/v5/middleware" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + reqs = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "kobs", + Name: "chi_requests_total", + Help: "Number of HTTP requests processed, partitioned by status code, method and path.", + }, []string{"code", "method", "path"}) + + latency = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "kobs", + Name: "chi_request_duration_milliseconds", + Help: "Latency of HTTP requests processed, partitioned by status code, method and path.", + Buckets: []float64{100, 500, 1000, 5000}, + }, []string{"code", "method", "path"}) +) + +// Metrics is a middleware that handles the Prometheus metrics for kobs and chi. +func Metrics(next http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + wrw := middleware.NewWrapResponseWriter(w, r.ProtoMajor) + next.ServeHTTP(wrw, r) + + reqs.WithLabelValues(http.StatusText(wrw.Status()), r.Method, r.URL.Path).Inc() + latency.WithLabelValues(http.StatusText(wrw.Status()), r.Method, r.URL.Path).Observe(float64(time.Since(start).Nanoseconds()) / 1000000) + } + + return http.HandlerFunc(fn) +}