From 5bd2bf478fbb2338f475b22fb23be558c8925862 Mon Sep 17 00:00:00 2001 From: gotjosh Date: Mon, 1 Mar 2021 17:22:35 +0000 Subject: [PATCH 1/2] Instrumentation: Add histogram for request duration on gRPC client to Ingesters Signed-off-by: gotjosh --- pkg/ingester/client/client.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pkg/ingester/client/client.go b/pkg/ingester/client/client.go index 0f6259893ad5..36457b2f2f2c 100644 --- a/pkg/ingester/client/client.go +++ b/pkg/ingester/client/client.go @@ -9,13 +9,22 @@ import ( "github.com/cortexproject/cortex/pkg/util/grpcclient" "github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc" "github.com/opentracing/opentracing-go" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "github.com/weaveworks/common/middleware" "google.golang.org/grpc" "google.golang.org/grpc/health/grpc_health_v1" + cortex_middleware "github.com/cortexproject/cortex/pkg/util/middleware" "github.com/grafana/loki/pkg/logproto" ) +var ingesterClientRequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "loki_ingester_client_request_duration_seconds", + Help: "Time spent doing Ingester requests.", + Buckets: prometheus.ExponentialBuckets(0.001, 4, 6), +}, []string{"operation", "status_code"}) + type HealthAndIngesterClient interface { logproto.IngesterClient grpc_health_v1.HealthClient @@ -76,8 +85,10 @@ func instrumentation() ([]grpc.UnaryClientInterceptor, []grpc.StreamClientInterc return []grpc.UnaryClientInterceptor{ otgrpc.OpenTracingClientInterceptor(opentracing.GlobalTracer()), middleware.ClientUserHeaderInterceptor, + cortex_middleware.PrometheusGRPCUnaryInstrumentation(ingesterClientRequestDuration), }, []grpc.StreamClientInterceptor{ otgrpc.OpenTracingStreamClientInterceptor(opentracing.GlobalTracer()), middleware.StreamClientUserHeaderInterceptor, + cortex_middleware.PrometheusGRPCStreamInstrumentation(ingesterClientRequestDuration), } } From 8beda9a991c46d5004096a5d334e118e5834f637 Mon Sep 17 00:00:00 2001 From: Edward Welch Date: Mon, 1 Mar 2021 15:49:08 -0500 Subject: [PATCH 2/2] fix lint --- pkg/ingester/client/client.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/ingester/client/client.go b/pkg/ingester/client/client.go index 36457b2f2f2c..9ff7d3ffb611 100644 --- a/pkg/ingester/client/client.go +++ b/pkg/ingester/client/client.go @@ -16,6 +16,7 @@ import ( "google.golang.org/grpc/health/grpc_health_v1" cortex_middleware "github.com/cortexproject/cortex/pkg/util/middleware" + "github.com/grafana/loki/pkg/logproto" )