-
Notifications
You must be signed in to change notification settings - Fork 664
/
metrics.go
242 lines (206 loc) · 6.81 KB
/
metrics.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.
package snowman
import (
"time"
"github.com/prometheus/client_golang/prometheus"
"go.uber.org/zap"
"github.com/ava-labs/avalanchego/ids"
"github.com/ava-labs/avalanchego/snow/choices"
"github.com/ava-labs/avalanchego/utils/linked"
"github.com/ava-labs/avalanchego/utils/logging"
"github.com/ava-labs/avalanchego/utils/metric"
"github.com/ava-labs/avalanchego/utils/wrappers"
)
type processingStart struct {
time time.Time
pollNumber uint64
}
type metrics struct {
log logging.Logger
currentMaxVerifiedHeight uint64
maxVerifiedHeight prometheus.Gauge
lastAcceptedHeight prometheus.Gauge
lastAcceptedTimestamp prometheus.Gauge
// processingBlocks keeps track of the [processingStart] that each block was
// issued into the consensus instance. This is used to calculate the amount
// of time to accept or reject the block.
processingBlocks *linked.Hashmap[ids.ID, processingStart]
// numProcessing keeps track of the number of processing blocks
numProcessing prometheus.Gauge
blockSizeAcceptedSum prometheus.Gauge
// pollsAccepted tracks the number of polls that a block was in processing
// for before being accepted
pollsAccepted metric.Averager
// latAccepted tracks the number of nanoseconds that a block was processing
// before being accepted
latAccepted metric.Averager
buildLatencyAccepted prometheus.Gauge
blockSizeRejectedSum prometheus.Gauge
// pollsRejected tracks the number of polls that a block was in processing
// for before being rejected
pollsRejected metric.Averager
// latRejected tracks the number of nanoseconds that a block was processing
// before being rejected
latRejected metric.Averager
// numFailedPolls keeps track of the number of polls that failed
numFailedPolls prometheus.Counter
// numSuccessfulPolls keeps track of the number of polls that succeeded
numSuccessfulPolls prometheus.Counter
}
func newMetrics(
log logging.Logger,
reg prometheus.Registerer,
lastAcceptedHeight uint64,
lastAcceptedTime time.Time,
) (*metrics, error) {
errs := wrappers.Errs{}
m := &metrics{
log: log,
currentMaxVerifiedHeight: lastAcceptedHeight,
maxVerifiedHeight: prometheus.NewGauge(prometheus.GaugeOpts{
Name: "max_verified_height",
Help: "highest verified height",
}),
lastAcceptedHeight: prometheus.NewGauge(prometheus.GaugeOpts{
Name: "last_accepted_height",
Help: "last height accepted",
}),
lastAcceptedTimestamp: prometheus.NewGauge(prometheus.GaugeOpts{
Name: "last_accepted_timestamp",
Help: "timestamp of the last accepted block in unix seconds",
}),
processingBlocks: linked.NewHashmap[ids.ID, processingStart](),
numProcessing: prometheus.NewGauge(prometheus.GaugeOpts{
Name: "blks_processing",
Help: "number of currently processing blocks",
}),
blockSizeAcceptedSum: prometheus.NewGauge(prometheus.GaugeOpts{
Name: "blks_accepted_container_size_sum",
Help: "cumulative size of all accepted blocks",
}),
pollsAccepted: metric.NewAveragerWithErrs(
"blks_polls_accepted",
"number of polls from the issuance of a block to its acceptance",
reg,
&errs,
),
latAccepted: metric.NewAveragerWithErrs(
"blks_accepted",
"time (in ns) from the issuance of a block to its acceptance",
reg,
&errs,
),
buildLatencyAccepted: prometheus.NewGauge(prometheus.GaugeOpts{
Name: "blks_build_accept_latency",
Help: "time (in ns) from the timestamp of a block to the time it was accepted",
}),
blockSizeRejectedSum: prometheus.NewGauge(prometheus.GaugeOpts{
Name: "blks_rejected_container_size_sum",
Help: "cumulative size of all rejected blocks",
}),
pollsRejected: metric.NewAveragerWithErrs(
"blks_polls_rejected",
"number of polls from the issuance of a block to its rejection",
reg,
&errs,
),
latRejected: metric.NewAveragerWithErrs(
"blks_rejected",
"time (in ns) from the issuance of a block to its rejection",
reg,
&errs,
),
numSuccessfulPolls: prometheus.NewCounter(prometheus.CounterOpts{
Name: "polls_successful",
Help: "number of successful polls",
}),
numFailedPolls: prometheus.NewCounter(prometheus.CounterOpts{
Name: "polls_failed",
Help: "number of failed polls",
}),
}
// Initially set the metrics for the last accepted block.
m.maxVerifiedHeight.Set(float64(lastAcceptedHeight))
m.lastAcceptedHeight.Set(float64(lastAcceptedHeight))
m.lastAcceptedTimestamp.Set(float64(lastAcceptedTime.Unix()))
errs.Add(
reg.Register(m.maxVerifiedHeight),
reg.Register(m.lastAcceptedHeight),
reg.Register(m.lastAcceptedTimestamp),
reg.Register(m.numProcessing),
reg.Register(m.blockSizeAcceptedSum),
reg.Register(m.buildLatencyAccepted),
reg.Register(m.blockSizeRejectedSum),
reg.Register(m.numSuccessfulPolls),
reg.Register(m.numFailedPolls),
)
return m, errs.Err
}
func (m *metrics) Issued(blkID ids.ID, pollNumber uint64) {
m.processingBlocks.Put(blkID, processingStart{
time: time.Now(),
pollNumber: pollNumber,
})
m.numProcessing.Inc()
}
func (m *metrics) Verified(height uint64) {
m.currentMaxVerifiedHeight = max(m.currentMaxVerifiedHeight, height)
m.maxVerifiedHeight.Set(float64(m.currentMaxVerifiedHeight))
}
func (m *metrics) Accepted(
blkID ids.ID,
height uint64,
timestamp time.Time,
pollNumber uint64,
blockSize int,
) {
start, ok := m.processingBlocks.Get(blkID)
if !ok {
m.log.Error("unable to measure latency",
zap.Stringer("blkID", blkID),
zap.Stringer("status", choices.Accepted),
)
return
}
m.lastAcceptedHeight.Set(float64(height))
m.lastAcceptedTimestamp.Set(float64(timestamp.Unix()))
m.processingBlocks.Delete(blkID)
m.numProcessing.Dec()
m.blockSizeAcceptedSum.Add(float64(blockSize))
m.pollsAccepted.Observe(float64(pollNumber - start.pollNumber))
now := time.Now()
processingDuration := now.Sub(start.time)
m.latAccepted.Observe(float64(processingDuration))
builtDuration := now.Sub(timestamp)
m.buildLatencyAccepted.Add(float64(builtDuration))
}
func (m *metrics) Rejected(blkID ids.ID, pollNumber uint64, blockSize int) {
start, ok := m.processingBlocks.Get(blkID)
if !ok {
m.log.Error("unable to measure latency",
zap.Stringer("blkID", blkID),
zap.Stringer("status", choices.Rejected),
)
return
}
m.processingBlocks.Delete(blkID)
m.numProcessing.Dec()
m.blockSizeRejectedSum.Add(float64(blockSize))
m.pollsRejected.Observe(float64(pollNumber - start.pollNumber))
duration := time.Since(start.time)
m.latRejected.Observe(float64(duration))
}
func (m *metrics) MeasureAndGetOldestDuration() time.Duration {
_, oldestOp, exists := m.processingBlocks.Oldest()
if !exists {
return 0
}
return time.Since(oldestOp.time)
}
func (m *metrics) SuccessfulPoll() {
m.numSuccessfulPolls.Inc()
}
func (m *metrics) FailedPoll() {
m.numFailedPolls.Inc()
}