-
Notifications
You must be signed in to change notification settings - Fork 175
/
herocache.go
273 lines (221 loc) · 10.9 KB
/
herocache.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
package metrics
import (
"fmt"
"github.com/prometheus/client_golang/prometheus"
"github.com/onflow/flow-go/module"
)
const subsystemHeroCache = "hero_cache"
var _ module.HeroCacheMetrics = (*HeroCacheCollector)(nil)
type HeroCacheCollector struct {
histogramNormalizedBucketSlotAvailable prometheus.Histogram
countKeyGetSuccess prometheus.Counter
countKeyGetFailure prometheus.Counter
countKeyPutSuccess prometheus.Counter
countKeyPutDrop prometheus.Counter
countKeyPutDeduplicated prometheus.Counter
countKeyPutAttempt prometheus.Counter
countKeyRemoved prometheus.Counter
size prometheus.Gauge
countKeyEjectionDueToFullCapacity prometheus.Counter
countKeyEjectionDueToEmergency prometheus.Counter
}
type HeroCacheMetricsRegistrationFunc func(uint64) module.HeroCacheMetrics
func NetworkReceiveCacheMetricsFactory(registrar prometheus.Registerer) *HeroCacheCollector {
return NewHeroCacheCollector(namespaceNetwork, ResourceNetworkingReceiveCache, registrar)
}
func PublicNetworkReceiveCacheMetricsFactory(registrar prometheus.Registerer) *HeroCacheCollector {
return NewHeroCacheCollector(namespaceNetwork, ResourcePublicNetworkingReceiveCache, registrar)
}
func NetworkDnsTxtCacheMetricsFactory(registrar prometheus.Registerer) *HeroCacheCollector {
return NewHeroCacheCollector(namespaceNetwork, ResourceNetworkingDnsTxtCache, registrar)
}
func NetworkDnsIpCacheMetricsFactory(registrar prometheus.Registerer) *HeroCacheCollector {
return NewHeroCacheCollector(namespaceNetwork, ResourceNetworkingDnsIpCache, registrar)
}
func ChunkDataPackRequestQueueMetricsFactory(registrar prometheus.Registerer) *HeroCacheCollector {
return NewHeroCacheCollector(namespaceExecution, ResourceChunkDataPackRequests, registrar)
}
func ReceiptRequestsQueueMetricFactory(registrar prometheus.Registerer) *HeroCacheCollector {
return NewHeroCacheCollector(namespaceExecution, ResourceReceipt, registrar)
}
func CollectionRequestsQueueMetricFactory(registrar prometheus.Registerer) *HeroCacheCollector {
return NewHeroCacheCollector(namespaceCollection, ResourceCollection, registrar)
}
func CollectionNodeTransactionsCacheMetrics(registrar prometheus.Registerer, epoch uint64) *HeroCacheCollector {
return NewHeroCacheCollector(namespaceCollection, fmt.Sprintf("%s_%d", ResourceTransaction, epoch), registrar)
}
func NewHeroCacheCollector(nameSpace string, cacheName string, registrar prometheus.Registerer) *HeroCacheCollector {
histogramNormalizedBucketSlotAvailable := prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: nameSpace,
Subsystem: subsystemHeroCache,
// Note that the notion of "bucket" in HeroCache differs from Prometheus.
// A HeroCache "bucket" is used to group the keys of the entities.
// A Prometheus "bucket" is used to group collected data points within a range.
// This metric represents the histogram of normalized available slots in buckets, where
// a data point set to 1 represents a bucket with all slots available (i.e., a fully empty bucket),
// and a data point set to 0 means a bucket with no available slots (i.e., a completely full bucket).
//
// We generally set total slots of a bucket in HeroCache to 16. Hence:
// Prometheus bucket 1 represents total number of HeroCache buckets with at most 16 available slots.
// Prometheus bucket 0.75 represents total number of HeroCache buckets with at most 12 available slots.
// Prometheus bucket 0.5 represents total number of HeroCache buckets with at most 8 available slots.
// Prometheus bucket 0.25 represents total number of HeroCache buckets with at most 4 available slots.
// Prometheus bucket 0.1 represents total number of HeroCache buckets with at most 1 available slots.
// Prometheus bucket 0 represents total number of HeroCache buckets with no (i.e., zero) available slots.
Buckets: []float64{0, 0.1, 0.25, 0.5, 0.75, 1},
Name: cacheName + "_" + "normalized_bucket_available_slot_count",
Help: "normalized histogram of available slots across all buckets",
})
size := prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: nameSpace,
Subsystem: subsystemHeroCache,
Name: cacheName + "_" + "items_total",
Help: "total number of items in the cache",
})
countKeyGetSuccess := prometheus.NewCounter(prometheus.CounterOpts{
Namespace: nameSpace,
Subsystem: subsystemHeroCache,
Name: cacheName + "_" + "successful_read_count_total",
Help: "total number of successful read queries",
})
countKeyGetFailure := prometheus.NewCounter(prometheus.CounterOpts{
Namespace: nameSpace,
Subsystem: subsystemHeroCache,
Name: cacheName + "_" + "unsuccessful_read_count_total",
Help: "total number of unsuccessful read queries",
})
countKeyPutAttempt := prometheus.NewCounter(prometheus.CounterOpts{
Namespace: nameSpace,
Subsystem: subsystemHeroCache,
Name: cacheName + "_" + "write_attempt_count_total",
Help: "total number of put queries",
})
countKeyPutDrop := prometheus.NewCounter(prometheus.CounterOpts{
Namespace: nameSpace,
Subsystem: subsystemHeroCache,
Name: cacheName + "_" + "write_drop_count_total",
Help: "total number of put queries dropped due to full capacity",
})
countKeyPutSuccess := prometheus.NewCounter(prometheus.CounterOpts{
Namespace: nameSpace,
Subsystem: subsystemHeroCache,
Name: cacheName + "_" + "successful_write_count_total",
Help: "total number successful write queries",
})
countKeyPutDeduplicated := prometheus.NewCounter(prometheus.CounterOpts{
Namespace: nameSpace,
Subsystem: subsystemHeroCache,
Name: cacheName + "_" + "unsuccessful_write_count_total",
Help: "total number of queries writing an already existing (duplicate) entity to the cache",
})
countKeyRemoved := prometheus.NewCounter(prometheus.CounterOpts{
Namespace: nameSpace,
Subsystem: subsystemHeroCache,
Name: cacheName + "_" + "removed_count_total",
Help: "total number of entities removed from the cache",
})
countKeyEjectionDueToFullCapacity := prometheus.NewCounter(prometheus.CounterOpts{
Namespace: nameSpace,
Subsystem: subsystemHeroCache,
Name: cacheName + "_" + "full_capacity_entity_ejection_total",
Help: "total number of entities ejected when writing new entities at full capacity",
})
countKeyEjectionDueToEmergency := prometheus.NewCounter(prometheus.CounterOpts{
Namespace: nameSpace,
Subsystem: subsystemHeroCache,
Name: cacheName + "_" + "emergency_key_ejection_total",
Help: "total number of emergency key ejections at bucket level",
})
registrar.MustRegister(
// available slot distribution
histogramNormalizedBucketSlotAvailable,
// size
size,
// read
countKeyGetSuccess,
countKeyGetFailure,
// write
countKeyPutSuccess,
countKeyPutDeduplicated,
countKeyPutDrop,
countKeyPutAttempt,
// remove
countKeyRemoved,
// ejection
countKeyEjectionDueToFullCapacity,
countKeyEjectionDueToEmergency)
return &HeroCacheCollector{
histogramNormalizedBucketSlotAvailable: histogramNormalizedBucketSlotAvailable,
size: size,
countKeyGetSuccess: countKeyGetSuccess,
countKeyGetFailure: countKeyGetFailure,
countKeyPutAttempt: countKeyPutAttempt,
countKeyPutSuccess: countKeyPutSuccess,
countKeyPutDeduplicated: countKeyPutDeduplicated,
countKeyPutDrop: countKeyPutDrop,
countKeyRemoved: countKeyRemoved,
countKeyEjectionDueToFullCapacity: countKeyEjectionDueToFullCapacity,
countKeyEjectionDueToEmergency: countKeyEjectionDueToEmergency,
}
}
// BucketAvailableSlots keeps track of number of available slots in buckets of cache.
func (h *HeroCacheCollector) BucketAvailableSlots(availableSlots uint64, totalSlots uint64) {
normalizedAvailableSlots := float64(availableSlots) / float64(totalSlots)
h.histogramNormalizedBucketSlotAvailable.Observe(normalizedAvailableSlots)
}
// OnKeyPutSuccess is called whenever a new (key, entity) pair is successfully added to the cache.
// size parameter is the current size of the cache post insertion.
func (h *HeroCacheCollector) OnKeyPutSuccess(size uint32) {
h.countKeyPutSuccess.Inc()
h.size.Set(float64(size))
}
// OnKeyPutDeduplicated is tracking the total number of unsuccessful writes caused by adding a duplicate key to the cache.
// A duplicate key is dropped by the cache when it is written to the cache.
// Note: in context of HeroCache, the key corresponds to the identifier of its entity. Hence, a duplicate key corresponds to
// a duplicate entity.
func (h *HeroCacheCollector) OnKeyPutDeduplicated() {
h.countKeyPutDeduplicated.Inc()
}
// OnKeyGetSuccess tracks total number of successful read queries.
// A read query is successful if the entity corresponding to its key is available in the cache.
// Note: in context of HeroCache, the key corresponds to the identifier of its entity.
func (h *HeroCacheCollector) OnKeyGetSuccess() {
h.countKeyGetSuccess.Inc()
}
// OnKeyGetFailure tracks total number of unsuccessful read queries.
// A read query is unsuccessful if the entity corresponding to its key is not available in the cache.
// Note: in context of HeroCache, the key corresponds to the identifier of its entity.
func (h *HeroCacheCollector) OnKeyGetFailure() {
h.countKeyGetFailure.Inc()
}
// OnKeyPutAttempt is called whenever a new (key, value) pair is attempted to be put in cache.
// It does not reflect whether the put was successful or not.
// A (key, value) pair put attempt may fail if the cache is full, or the key already exists.
// size parameter is the current size of the cache prior to the put attempt.
func (h *HeroCacheCollector) OnKeyPutAttempt(size uint32) {
h.countKeyPutAttempt.Inc()
h.size.Set(float64(size))
}
// OnKeyPutDrop is called whenever a new (key, entity) pair is dropped from the cache due to full cache.
func (h *HeroCacheCollector) OnKeyPutDrop() {
h.countKeyPutDrop.Inc()
}
// OnKeyRemoved is called whenever a (key, entity) pair is removed from the cache.
// size parameter is the current size of the cache.
func (h *HeroCacheCollector) OnKeyRemoved(size uint32) {
h.countKeyRemoved.Inc()
h.size.Set(float64(size))
}
// OnEntityEjectionDueToFullCapacity is called whenever adding a new (key, entity) to the cache results in ejection of another (key', entity') pair.
// This normally happens -- and is expected -- when the cache is full.
// Note: in context of HeroCache, the key corresponds to the identifier of its entity.
func (h *HeroCacheCollector) OnEntityEjectionDueToFullCapacity() {
h.countKeyEjectionDueToFullCapacity.Inc()
}
// OnEntityEjectionDueToEmergency is called whenever a bucket is found full and all of its keys are valid, i.e.,
// each key belongs to an existing (key, entity) pair.
// Hence, adding a new key to that bucket will replace the oldest valid key inside that bucket.
// Note: in context of HeroCache, the key corresponds to the identifier of its entity.
func (h *HeroCacheCollector) OnEntityEjectionDueToEmergency() {
h.countKeyEjectionDueToEmergency.Inc()
}