/
prom.go
83 lines (70 loc) · 1.78 KB
/
prom.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
package actor
import (
"fmt"
"reflect"
"strings"
"time"
prom "github.com/prometheus/client_golang/prometheus"
"github.com/determined-ai/determined/master/internal/config"
)
const (
promNamespace = "determined"
promSubsystem = "actors"
)
var (
receiveLabels = []string{"from", "to", "msg"}
receiveHistogram = prom.NewHistogramVec(prom.HistogramOpts{
Namespace: promNamespace,
Subsystem: promSubsystem,
Name: "receive",
Help: "timings for actor receive calls",
Buckets: prom.DefBuckets,
}, receiveLabels)
receiveErrors = prom.NewCounterVec(prom.CounterOpts{
Namespace: promNamespace,
Subsystem: promSubsystem,
Name: "receive_errors",
Help: "errors from actor receive calls",
}, receiveLabels)
)
func init() {
prom.MustRegister(receiveHistogram)
prom.MustRegister(receiveErrors)
}
func (r *Ref) time(ctx *Context) (end func()) {
if !config.GetMasterConfig().Observability.EnablePrometheus {
return func() {}
}
start := time.Now()
return func() {
receiveHistogram.WithLabelValues(r.promLabels(ctx)...).Observe(time.Since(start).Seconds())
}
}
func (r *Ref) recordErr(ctx *Context) func(error) {
if !config.GetMasterConfig().Observability.EnablePrometheus {
return func(error) {}
}
return func(err error) {
if err == nil {
return
}
receiveErrors.WithLabelValues(r.promLabels(ctx)...).Inc()
}
}
func (r *Ref) promLabels(ctx *Context) []string {
from := "system"
if ctx != nil && ctx.sender != nil {
from = actorToGenericName(ctx.sender.actor)
}
to := actorToGenericName(r.actor)
msg := "PreStart"
if ctx != nil {
msg = reflect.TypeOf(ctx.message).String()
}
return []string{from, to, msg}
}
func actorToGenericName(actor Actor) string {
out := fmt.Sprintf("%T", actor)
out = strings.TrimPrefix(out, "*")
return out
}