Add metrics for backend_retries_total

middlewares/metrics.go

@@ -1,22 +1,29 @@
 package middlewares
 
 import (
-	"github.com/go-kit/kit/metrics"
 	"net/http"
 	"strconv"
 	"time"
+
+	"github.com/go-kit/kit/metrics"
 )
 
 // Metrics is an Interface that must be satisfied by any system that
-// wants to expose and monitor metrics
+// wants to expose and monitor Metrics.
 type Metrics interface {
 	getReqsCounter() metrics.Counter
-	getLatencyHistogram() metrics.Histogram
-	handler() http.Handler
+	getReqDurationHistogram() metrics.Histogram
+	RetryMetrics
+}
+
+// RetryMetrics must be satisfied by any system that wants to collect and
+// expose retry specific Metrics.
+type RetryMetrics interface {
+	getRetryCounter() metrics.Counter
 }
 
 // MetricsWrapper is a Negroni compatible Handler which relies on a
-// given Metrics implementation to expose and monitor Traefik metrics
+// given Metrics implementation to expose and monitor Traefik Metrics.
 type MetricsWrapper struct {
 	Impl Metrics
 }
@@ -35,17 +42,25 @@ func (m *MetricsWrapper) ServeHTTP(rw http.ResponseWriter, r *http.Request, next
 	start := time.Now()
 	prw := &responseRecorder{rw, http.StatusOK}
 	next(prw, r)
-	labels := []string{"code", strconv.Itoa(prw.StatusCode()), "method", r.Method}
+	labels := []string{"code", strconv.Itoa(prw.statusCode), "method", r.Method}
 	m.Impl.getReqsCounter().With(labels...).Add(1)
-	m.Impl.getLatencyHistogram().Observe(float64(time.Since(start).Seconds()))
+	m.Impl.getReqDurationHistogram().Observe(float64(time.Since(start).Seconds()))
+}
+
+// MetricsRetryListener is an implementation of the RetryListener interface to
+// record Metrics about retry attempts.
+type MetricsRetryListener struct {
+	retryMetrics RetryMetrics
 }
 
-func (rw *responseRecorder) StatusCode() int {
-	return rw.statusCode
+// Retried tracks the retry in the Metrics implementation.
+func (m *MetricsRetryListener) Retried(attempt int) {
+	if m.retryMetrics != nil {
+		m.retryMetrics.getRetryCounter().Add(1)
+	}
 }
 
-// Handler is the chance for the Metrics implementation
-// to expose its metrics on a server endpoint
-func (m *MetricsWrapper) Handler() http.Handler {
-	return m.Impl.handler()
+// NewMetricsRetryListener instantiates a MetricsRetryListener with the given RetryMetrics.
+func NewMetricsRetryListener(retryMetrics RetryMetrics) RetryListener {
+	return &MetricsRetryListener{retryMetrics: retryMetrics}
 }

middlewares/metrics_test.go

@@ -0,0 +1,48 @@
+package middlewares
+
+import (
+	"testing"
+
+	"github.com/go-kit/kit/metrics"
+)
+
+func TestMetricsRetryListener(t *testing.T) {
+	// nil implementation, nothing should fail
+	retryListener := NewMetricsRetryListener(nil)
+	retryListener.Retried(1)
+
+	retryMetrics := newCollectingMetrics()
+	retryListener = NewMetricsRetryListener(retryMetrics)
+	retryListener.Retried(1)
+	retryListener.Retried(2)
+
+	wantCounterValue := float64(2)
+	if retryMetrics.retryCounter.counterValue != wantCounterValue {
+		t.Errorf("got counter value of %d, want %d", retryMetrics.retryCounter.counterValue, wantCounterValue)
+	}
+}
+
+// collectingRetryMetrics is an implementation of the RetryMetrics interface that can be used inside tests to collect the times Add() was called.
+type collectingRetryMetrics struct {
+	retryCounter *collectingCounter
+}
+
+func newCollectingMetrics() collectingRetryMetrics {
+	return collectingRetryMetrics{retryCounter: &collectingCounter{}}
+}
+
+func (metrics collectingRetryMetrics) getRetryCounter() metrics.Counter {
+	return metrics.retryCounter
+}
+
+type collectingCounter struct {
+	counterValue float64
+}
+
+func (c *collectingCounter) With(labelValues ...string) metrics.Counter {
+	panic("collectingCounter.With not implemented!")
+}
+
+func (c *collectingCounter) Add(delta float64) {
+	c.counterValue += delta
+}

middlewares/prometheus.go

@@ -1,89 +1,129 @@
 package middlewares
 
 import (
+	"fmt"
+
 	"github.com/containous/traefik/types"
 	"github.com/go-kit/kit/metrics"
 	"github.com/go-kit/kit/metrics/prometheus"
 	stdprometheus "github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"net/http"
 )
 
 const (
-	reqsName    = "traefik_requests_total"
-	latencyName = "traefik_request_duration_seconds"
+	reqsTotalName    = "traefik_requests_total"
+	reqDurationName  = "traefik_request_duration_seconds"
+	retriesTotalName = "traefik_backend_retries_total"
 )
 
-// Prometheus is an Implementation for Metrics that exposes prometheus metrics for the latency
-// and the number of requests partitioned by status code and method.
+// Prometheus is an Implementation for Metrics that exposes the following Prometheus metrics:
+// - number of requests partitioned by status code and method
+// - request durations
+// - amount of retries happened
 type Prometheus struct {
-	reqsCounter      metrics.Counter
-	latencyHistogram metrics.Histogram
+	reqsCounter          metrics.Counter
+	reqDurationHistogram metrics.Histogram
+	retryCounter         metrics.Counter
 }
 
 func (p *Prometheus) getReqsCounter() metrics.Counter {
 	return p.reqsCounter
 }
 
-func (p *Prometheus) getLatencyHistogram() metrics.Histogram {
-	return p.latencyHistogram
+func (p *Prometheus) getReqDurationHistogram() metrics.Histogram {
+	return p.reqDurationHistogram
+}
+
+func (p *Prometheus) getRetryCounter() metrics.Counter {
+	return p.retryCounter
 }
 
-// NewPrometheus returns a new prometheus Metrics implementation.
-func NewPrometheus(name string, config *types.Prometheus) *Prometheus {
-	var m Prometheus
+// NewPrometheus returns a new Prometheus Metrics implementation.
+// With the returned collectors you have the possibility to clean up the internal Prometheus state by unsubscribing the collectors.
+// This is for example useful while testing the Prometheus implementation.
+// If any of the Prometheus Metrics can not be registered an error will be returned and the returned Metrics implementation will be nil.
+func NewPrometheus(name string, config *types.Prometheus) (*Prometheus, []stdprometheus.Collector, error) {
+	var prom Prometheus
+	var collectors []stdprometheus.Collector
 
 	cv := stdprometheus.NewCounterVec(
 		stdprometheus.CounterOpts{
-			Name:        reqsName,
+			Name:        reqsTotalName,
 			Help:        "How many HTTP requests processed, partitioned by status code and method.",
 			ConstLabels: stdprometheus.Labels{"service": name},
 		},
 		[]string{"code", "method"},
 	)
-
-	err := stdprometheus.Register(cv)
+	cv, err := registerCounterVec(cv)
 	if err != nil {
-		e, ok := err.(stdprometheus.AlreadyRegisteredError)
-		if !ok {
-			panic(err)
-		}
-		m.reqsCounter = prometheus.NewCounter(e.ExistingCollector.(*stdprometheus.CounterVec))
-	} else {
-		m.reqsCounter = prometheus.NewCounter(cv)
+		return nil, collectors, err
 	}
+	prom.reqsCounter = prometheus.NewCounter(cv)
+	collectors = append(collectors, cv)
 
 	var buckets []float64
 	if config.Buckets != nil {
 		buckets = config.Buckets
 	} else {
 		buckets = []float64{0.1, 0.3, 1.2, 5}
 	}
-
 	hv := stdprometheus.NewHistogramVec(
 		stdprometheus.HistogramOpts{
-			Name:        latencyName,
+			Name:        reqDurationName,
 			Help:        "How long it took to process the request.",
 			ConstLabels: stdprometheus.Labels{"service": name},
 			Buckets:     buckets,
 		},
 		[]string{},
 	)
+	hv, err = registerHistogramVec(hv)
+	if err != nil {
+		return nil, collectors, err
+	}
+	prom.reqDurationHistogram = prometheus.NewHistogram(hv)
+	collectors = append(collectors, hv)
+
+	cv = stdprometheus.NewCounterVec(
+		stdprometheus.CounterOpts{
+			Name:        retriesTotalName,
+			Help:        "How many request retries happened in total.",
+			ConstLabels: stdprometheus.Labels{"service": name},
+		},
+		[]string{},
+	)
+	cv, err = registerCounterVec(cv)
+	if err != nil {
+		return nil, collectors, err
+	}
+	prom.retryCounter = prometheus.NewCounter(cv)
+	collectors = append(collectors, cv)
+
+	return &prom, collectors, nil
+}
+
+func registerCounterVec(cv *stdprometheus.CounterVec) (*stdprometheus.CounterVec, error) {
+	err := stdprometheus.Register(cv)
 
-	err = stdprometheus.Register(hv)
 	if err != nil {
 		e, ok := err.(stdprometheus.AlreadyRegisteredError)
 		if !ok {
-			panic(err)
+			return nil, fmt.Errorf("error registering CounterVec: %s", e)
 		}
-		m.latencyHistogram = prometheus.NewHistogram(e.ExistingCollector.(*stdprometheus.HistogramVec))
-	} else {
-		m.latencyHistogram = prometheus.NewHistogram(hv)
+		cv = e.ExistingCollector.(*stdprometheus.CounterVec)
 	}
 
-	return &m
+	return cv, nil
 }
 
-func (p *Prometheus) handler() http.Handler {
-	return promhttp.Handler()
+func registerHistogramVec(hv *stdprometheus.HistogramVec) (*stdprometheus.HistogramVec, error) {
+	err := stdprometheus.Register(hv)
+
+	if err != nil {
+		e, ok := err.(stdprometheus.AlreadyRegisteredError)
+		if !ok {
+			return nil, fmt.Errorf("error registering HistogramVec: %s", e)
+		}
+		hv = e.ExistingCollector.(*stdprometheus.HistogramVec)
+	}
+
+	return hv, nil
 }