-
Notifications
You must be signed in to change notification settings - Fork 73
/
hooks.go
149 lines (133 loc) · 4.86 KB
/
hooks.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
package redisbp
import (
"context"
"errors"
"fmt"
"time"
"github.com/go-redis/redis/v8"
"github.com/opentracing/opentracing-go"
"github.com/prometheus/client_golang/prometheus"
"github.com/reddit/baseplate.go/internal/prometheusbpint"
"github.com/reddit/baseplate.go/prometheusbp"
"github.com/reddit/baseplate.go/redis/internal/redisprom"
"github.com/reddit/baseplate.go/tracing"
)
type promCtxKeyType struct{}
var promCtxKey promCtxKeyType
type promCtx struct {
command string
start time.Time
}
// SpanHook is a redis.Hook for wrapping Redis commands and pipelines
// in Client Spans and metrics.
type SpanHook struct {
ClientName string
Type string
Deployment string
Database string
promActive *prometheusbpint.HighWatermarkGauge
}
var _ redis.Hook = SpanHook{}
// BeforeProcess starts a client Span before processing a Redis command and
// starts a timer to record how long the command took.
func (h SpanHook) BeforeProcess(ctx context.Context, cmd redis.Cmder) (context.Context, error) {
return h.startChildSpan(ctx, cmd.Name()), nil
}
// AfterProcess ends the client Span started by BeforeProcess, publishes the
// time the Redis command took to complete, and a metric indicating whether the
// command was a "success" or "fail"
func (h SpanHook) AfterProcess(ctx context.Context, cmd redis.Cmder) error {
h.endChildSpan(ctx, cmd.Err())
// NOTE: returning non-nil error from the hook changes the error the caller gets.
// for this particular case if we return cmd.Err(), it will not change the client error,
// but anyway it's not necessary
// see: https://github.com/go-redis/redis/blob/v8.10.0/redis.go#L60
return nil
}
// BeforeProcessPipeline starts a client span before processing a Redis pipeline
// and starts a timer to record how long the pipeline took.
func (h SpanHook) BeforeProcessPipeline(ctx context.Context, cmds []redis.Cmder) (context.Context, error) {
return h.startChildSpan(ctx, "pipeline"), nil
}
// AfterProcessPipeline ends the client span started by BeforeProcessPipeline,
// publishes the time the Redis pipeline took to complete, and a metric
// indicating whether the pipeline was a "success" or "fail"
func (h SpanHook) AfterProcessPipeline(ctx context.Context, cmds []redis.Cmder) error {
errs := make([]error, 0, len(cmds))
for _, cmd := range cmds {
if err := cmd.Err(); !errors.Is(err, redis.Nil) {
errs = append(errs, err)
}
}
h.endChildSpan(ctx, errors.Join(errs...))
// NOTE: returning non-nil error from the hook changes the error the caller gets, and that's something we want to avoid.
// see: https://github.com/go-redis/redis/blob/v8.10.0/redis.go#L101
return nil
}
func (h SpanHook) startChildSpan(ctx context.Context, cmdName string) context.Context {
name := fmt.Sprintf("%s.%s", h.ClientName, cmdName)
_, ctx = opentracing.StartSpanFromContext(
ctx,
name,
tracing.SpanTypeOption{Type: tracing.SpanTypeClient},
)
if h.promActive != nil {
h.promActive.Inc()
}
redisprom.ActiveRequests.With(prometheus.Labels{
redisprom.ClientNameLabel: h.ClientName,
redisprom.TypeLabel: h.Type,
redisprom.CommandLabel: cmdName,
redisprom.DeploymentLabel: h.Deployment,
redisprom.DatabaseLabel: h.Database,
}).Inc()
return context.WithValue(ctx, promCtxKey, &promCtx{
command: cmdName,
start: time.Now(),
})
}
func (h SpanHook) endChildSpan(ctx context.Context, err error) {
command := "unknown"
if v, _ := ctx.Value(promCtxKey).(*promCtx); v != nil {
command = v.command
durationSeconds := time.Since(v.start).Seconds()
latencyTimer.With(prometheus.Labels{
nameLabel: h.ClientName,
commandLabel: v.command,
successLabel: prometheusbp.BoolString(err == nil),
}).Observe(durationSeconds)
redisprom.LatencySeconds.With(prometheus.Labels{
redisprom.ClientNameLabel: h.ClientName,
redisprom.TypeLabel: h.Type,
redisprom.CommandLabel: command,
redisprom.DeploymentLabel: h.Deployment,
redisprom.SuccessLabel: prometheusbp.BoolString(err == nil),
redisprom.DatabaseLabel: h.Database,
}).Observe(durationSeconds)
}
// Outside of the context casting because we always want this to work.
redisprom.RequestsTotal.With(prometheus.Labels{
redisprom.ClientNameLabel: h.ClientName,
redisprom.TypeLabel: h.Type,
redisprom.CommandLabel: command,
redisprom.DeploymentLabel: h.Deployment,
redisprom.SuccessLabel: prometheusbp.BoolString(err == nil),
redisprom.DatabaseLabel: h.Database,
}).Inc()
redisprom.ActiveRequests.With(prometheus.Labels{
redisprom.ClientNameLabel: h.ClientName,
redisprom.TypeLabel: h.Type,
redisprom.CommandLabel: command,
redisprom.DeploymentLabel: h.Deployment,
redisprom.DatabaseLabel: h.Database,
}).Dec()
if h.promActive != nil {
h.promActive.Dec()
}
if span := opentracing.SpanFromContext(ctx); span != nil {
span.FinishWithOptions(tracing.FinishOptions{
Ctx: ctx,
Err: err,
}.Convert())
}
}