-
Notifications
You must be signed in to change notification settings - Fork 20
/
retries.go
88 lines (76 loc) · 2.29 KB
/
retries.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
package metrics
import (
"sync"
"time"
"github.com/prometheus/client_golang/prometheus"
)
var (
// Retries observes finished retried operations.
Retries RetriesMetric = &retriesMetric{}
)
func init() {
Retries.(*retriesMetric).init()
}
// RetriesMetric observes finished retried operations.
type RetriesMetric interface {
// Observe performs a single observation of a finished retry loop.
// codeLocation is a string representation of the code location
// of the retry loop.
// retryCount is the number of retries performed, where the very
// first attempt is not counted as retry.
// latency is the elapsed time from the start of the retry loop
// until is has been finished.
Observe(codeLocation string, retryCount uint64, latency time.Duration)
}
type retriesMetric struct {
initOnlyOnce sync.Once
countMetric *prometheus.HistogramVec
latencyMetric *prometheus.HistogramVec
}
func (m *retriesMetric) init() {
m.initOnlyOnce.Do(func() {
countBuckets := func() []float64 {
return append(
prometheus.LinearBuckets(1, 1, 9),
prometheus.ExponentialBuckets(10, 2, 11)...,
)
}
m.countMetric = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Subsystem: Subsystem,
Name: "retries_retrycount",
Help: "Generic metric for retry loops collecting the number of retries performed for retried operations.",
Buckets: countBuckets(),
},
[]string{
"location",
},
)
Registerer().MustRegister(m.countMetric)
latencyBuckets := func() []float64 {
list := make([]float64, 0, 18)
for i := 1e-3; i <= 1e+5; i *= 10.0 {
list = append(list, i, i*5.0)
}
return list
}
m.latencyMetric = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Subsystem: Subsystem,
Name: "retries_latency_seconds",
Help: "Generic metric for retry loops collecting the latency (in seconds) caused by retrying operations.",
Buckets: latencyBuckets(),
},
[]string{
"location",
},
)
Registerer().MustRegister(m.latencyMetric)
})
}
func (m *retriesMetric) Observe(codeLocation string, retryCount uint64, latency time.Duration) {
if retryCount > 0 {
m.countMetric.WithLabelValues(codeLocation).Observe(float64(retryCount))
m.latencyMetric.WithLabelValues(codeLocation).Observe(float64(latency.Seconds()))
}
}