Add Prometheus metrics for API requests

deploy/helm/kobs/Chart.yaml

@@ -4,5 +4,5 @@ description: Kubernetes Observability Platform
 type: application
 home: https://kobs.io
 icon: https://kobs.io/assets/images/logo.svg
-version: 0.5.3
+version: 0.5.4
 appVersion: v0.5.0

deploy/helm/kobs/templates/_helpers.tpl

@@ -108,3 +108,12 @@ Additional labels for the Service
 {{- toYaml .Values.service.labels }}
 {{- end }}
 {{- end }}
+
+{{/*
+Additional labels for the Service Monitor
+*/}}
+{{- define "kobs.serviceMonitorLabels" -}}
+{{- if .Values.serviceMonitor.labels }}
+{{- toYaml .Values.serviceMonitor.labels }}
+{{- end }}
+{{- end }}

deploy/helm/kobs/templates/servicemonitor.yaml

@@ -0,0 +1,34 @@
+{{- if .Values.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "kobs.fullname" . }}
+  labels:
+    {{- include "kobs.labels" . | nindent 4 }}
+    {{- include "kobs.serviceMonitorLabels" . | nindent 4 }}
+spec:
+  endpoints:
+    - port: http-metrics
+      {{- if .Values.serviceMonitor.interval }}
+      interval: {{ .Values.serviceMonitor.interval }}
+      {{- end }}
+      {{- if .Values.serviceMonitor.scrapeTimeout }}
+      scrapeTimeout: {{ .Values.serviceMonitor.scrapeTimeout }}
+      {{- end }}
+      path: /metrics
+      honorLabels: {{ .Values.serviceMonitor.honorLabels }}
+      {{- if .Values.serviceMonitor.metricRelabelings }}
+      metricRelabelings:
+      {{ toYaml .Values.serviceMonitor.metricRelabelings | nindent 6 }}
+      {{- end }}
+      {{- if .Values.serviceMonitor.relabelings }}
+      relabelings:
+      {{ toYaml .Values.serviceMonitor.relabelings | nindent 6 }}
+      {{- end }}
+  namespaceSelector:
+    matchNames:
+      - {{ .Release.Namespace }}
+  selector:
+    matchLabels:
+      {{- include "kobs.selectorLabels" . | nindent 6 }}
+{{- end -}}

deploy/helm/kobs/values.yaml

@@ -167,3 +167,44 @@ ingress:
   #  - secretName: chart-example-tls
   #    hosts:
   #      - chart-example.local
+
+## Create a Service Monitor for the Prometheus Operator.
+## See: https://github.com/coreos/prometheus-operator
+##
+serviceMonitor:
+  enabled: false
+
+  ## Interval at which metrics should be scraped. Fallback to the Prometheus default unless specified.
+  ##
+  # interval: 10s
+
+  ## Timeout after which the scrape is ended. Fallback to the Prometheus default unless specified.
+  ##
+  # scrapeTimeout: 30s
+
+  ## Additional labels that are used by the Prometheus installed in your cluster to select Service Monitors to work with
+  ## See: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#prometheusspec
+  ##
+  labels: {}
+
+  ## HonorLabels chooses the metric's labels on collisions with target labels.
+  ##
+  honorLabels: true
+
+  ## MetricRelabelConfigs to apply to samples before ingestion.
+  ##
+  metricRelabelings: []
+    # - action: keep
+    #   regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+'
+    #   sourceLabels: [__name__]
+
+  ## RelabelConfigs to apply to samples before scraping. Prometheus Operator automatically adds relabelings for a few
+  ## standard Kubernetes fields and replaces original scrape job name with __tmp_prometheus_job_name.
+  ##
+  relabelings: []
+    # - sourceLabels: [__meta_kubernetes_pod_node_name]
+    #   separator: ;
+    #   regex: ^(.*)$
+    #   targetLabel: nodename
+    #   replacement: $1
+    #   action: replace

docs/installation/helm.md

@@ -85,3 +85,10 @@ helm upgrade kobs kobs/kobs
 | `ingress.annotations` | Annotations to add to the ingress. | `{}` |
 | `ingress.hosts` | Hosts to use for the ingress. | `[]` |
 | `ingress.tls` | TLS configuration for the ingress. | `[]` |
+| `serviceMonitor.enabled` | Create a Service Monitor for kobs. | `false` |
+| `serviceMonitor.interval` | Interval at which metrics should be scraped. Fallback to the Prometheus default unless specified. | |
+| `serviceMonitor.scrapeTimeout` | Timeout after which the scrape is ended. Fallback to the Prometheus default unless specified. | |
+| `serviceMonitor.labels` | Additional labels for the the Service Monitor. | `{}` |
+| `serviceMonitor.honorLabels` | Chooses the metric's labels on collisions with target labels. | `false` |
+| `serviceMonitor.metricRelabelings` | Metric relabel config. | `[]` |
+| `serviceMonitor.relabelings` | Relabel config. | `[]` |

pkg/api/api.go

@@ -9,6 +9,7 @@ import (
 	"github.com/kobsio/kobs/pkg/api/clusters"
 	"github.com/kobsio/kobs/pkg/api/middleware/auth"
 	"github.com/kobsio/kobs/pkg/api/middleware/httplog"
+	"github.com/kobsio/kobs/pkg/api/middleware/metrics"
 
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/chi/v5/middleware"
@@ -72,6 +73,7 @@ func New(loadedClusters *clusters.Clusters, pluginsRouter chi.Router, isDevelopm
 	router.Use(middleware.RequestID)
 	router.Use(middleware.Recoverer)
 	router.Use(middleware.URLFormat)
+	router.Use(metrics.Metrics)
 	router.Use(auth.Auth)
 	router.Use(httplog.NewStructuredLogger(log.Logger))
 	router.Use(render.SetContentType(render.ContentTypeJSON))

pkg/api/middleware/metrics/metrics.go

@@ -0,0 +1,39 @@
+package metrics
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/go-chi/chi/v5/middleware"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	reqs = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "kobs",
+		Name:      "chi_requests_total",
+		Help:      "Number of HTTP requests processed, partitioned by status code, method and path.",
+	}, []string{"code", "method", "path"})
+
+	latency = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "kobs",
+		Name:      "chi_request_duration_milliseconds",
+		Help:      "Latency of HTTP requests processed, partitioned by status code, method and path.",
+		Buckets:   []float64{100, 500, 1000, 5000},
+	}, []string{"code", "method", "path"})
+)
+
+// Metrics is a middleware that handles the Prometheus metrics for kobs and chi.
+func Metrics(next http.Handler) http.Handler {
+	fn := func(w http.ResponseWriter, r *http.Request) {
+		start := time.Now()
+		wrw := middleware.NewWrapResponseWriter(w, r.ProtoMajor)
+		next.ServeHTTP(wrw, r)
+
+		reqs.WithLabelValues(http.StatusText(wrw.Status()), r.Method, r.URL.Path).Inc()
+		latency.WithLabelValues(http.StatusText(wrw.Status()), r.Method, r.URL.Path).Observe(float64(time.Since(start).Nanoseconds()) / 1000000)
+	}
+
+	return http.HandlerFunc(fn)
+}