From 28a241d55567f7a727ff6ac6e0cf1e9ca025b639 Mon Sep 17 00:00:00 2001 From: ricoberger Date: Wed, 29 Sep 2021 21:26:33 +0200 Subject: [PATCH] Improve HTTP metrics and add ClickHouse metric Improve the exported HTTP metrics, by using summary instead of histograms for the request latency. Add a new ClickHouse metric, to determine how often a field is used in queries. With the help of this metric we can then add more fields to the logs table to improve the query time for the fields which are used most of the time. --- CHANGELOG.md | 1 + pkg/api/middleware/metrics/metrics.go | 21 +++++++++++---------- plugins/clickhouse/pkg/instance/logs.go | 15 +++++++++++++++ 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b167877cc..ed5c3a62e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,6 +68,7 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan - [#152](https://github.com/kobsio/kobs/pull/152): Improve performance for large dashboards and open Application page in gallery view. - [#155](https://github.com/kobsio/kobs/pull/155): Allow users to get all Applications from all namespaces, by allowing an empty namespace list. - [#157](https://github.com/kobsio/kobs/pull/157): Imporve query performance for ClickHouse plugin. +- [#158](https://github.com/kobsio/kobs/pull/158): Improve Pormetheus HTTP metrics and create a metric to find the most used fields in queries. ## [v0.5.0](https://github.com/kobsio/kobs/releases/tag/v0.5.0) (2021-08-03) diff --git a/pkg/api/middleware/metrics/metrics.go b/pkg/api/middleware/metrics/metrics.go index 31d82ad0d..d2dde4b5f 100644 --- a/pkg/api/middleware/metrics/metrics.go +++ b/pkg/api/middleware/metrics/metrics.go @@ -2,6 +2,7 @@ package metrics import ( "net/http" + "strconv" "time" "github.com/go-chi/chi/v5/middleware" @@ -10,18 +11,18 @@ import ( ) var ( - reqs = promauto.NewCounterVec(prometheus.CounterOpts{ + reqMetric = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "kobs", Name: "chi_requests_total", Help: "Number of HTTP requests processed, partitioned by status code, method and path.", - }, []string{"code", "method", "path"}) + }, []string{"response_code", "request_method", "request_path"}) - latency = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "kobs", - Name: "chi_request_duration_milliseconds", - Help: "Latency of HTTP requests processed, partitioned by status code, method and path.", - Buckets: []float64{100, 500, 1000, 5000}, - }, []string{"code", "method", "path"}) + sumMetric = promauto.NewSummaryVec(prometheus.SummaryOpts{ + Namespace: "kobs", + Name: "chi_request_duration_milliseconds", + Help: "Latency of HTTP requests processed, partitioned by status code, method and path.", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.95: 0.005, 0.99: 0.001}, + }, []string{"response_code", "request_method", "request_path"}) ) // Metrics is a middleware that handles the Prometheus metrics for kobs and chi. @@ -31,8 +32,8 @@ func Metrics(next http.Handler) http.Handler { wrw := middleware.NewWrapResponseWriter(w, r.ProtoMajor) next.ServeHTTP(wrw, r) - reqs.WithLabelValues(http.StatusText(wrw.Status()), r.Method, r.URL.Path).Inc() - latency.WithLabelValues(http.StatusText(wrw.Status()), r.Method, r.URL.Path).Observe(float64(time.Since(start).Nanoseconds()) / 1000000) + reqMetric.WithLabelValues(strconv.Itoa(wrw.Status()), r.Method, r.URL.Path).Inc() + sumMetric.WithLabelValues(strconv.Itoa(wrw.Status()), r.Method, r.URL.Path).Observe(float64(time.Since(start).Nanoseconds()) / 1000000) } return http.HandlerFunc(fn) diff --git a/plugins/clickhouse/pkg/instance/logs.go b/plugins/clickhouse/pkg/instance/logs.go index 813908a72..a099509fa 100644 --- a/plugins/clickhouse/pkg/instance/logs.go +++ b/plugins/clickhouse/pkg/instance/logs.go @@ -3,11 +3,20 @@ package instance import ( "fmt" "strings" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" ) var ( defaultFields = []string{"timestamp", "cluster", "namespace", "app", "pod_name", "container_name", "host", "log"} defaultColumns = "timestamp, cluster, namespace, app, pod_name, container_name, host, fields_string.key, fields_string.value, fields_number.key, fields_number.value, log" + + fieldsMetric = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "kobs", + Name: "clickhouse_fields_total", + Help: "Number how often a field was used in a query.", + }, []string{"field"}) ) // parseLogsQuery parses the given query string and return the conditions for the where statement in the sql query. We @@ -123,6 +132,12 @@ func handleConditionParts(key, value, operator string) (string, error) { key = strings.TrimSpace(key) value = strings.TrimSpace(value) + // The kobs_clickhouse_fields_total metric can be used to determine how often a field is used. This information can + // then be used to create an additional column for this field via the following SQL commands: + // ALTER TABLE logs.logs ON CLUSTER '{cluster}' ADD COLUMN String DEFAULT fields_string.value[indexOf(fields_string.key, '')]; + // ALTER TABLE logs.logs ON CLUSTER '{cluster}' ADD COLUMN String DEFAULT fields_number.value[indexOf(fields_number.key, '')]; + fieldsMetric.WithLabelValues(key).Inc() + if contains(defaultFields, key) { if operator == "=~" { return fmt.Sprintf("%s ILIKE %s", key, value), nil