/
ems.go
764 lines (666 loc) · 21.7 KB
/
ems.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
package ems
import (
"fmt"
"github.com/netapp/harvest/v2/cmd/collectors"
rest2 "github.com/netapp/harvest/v2/cmd/collectors/rest"
"github.com/netapp/harvest/v2/cmd/poller/collector"
"github.com/netapp/harvest/v2/cmd/poller/plugin"
"github.com/netapp/harvest/v2/cmd/tools/rest"
"github.com/netapp/harvest/v2/pkg/errs"
"github.com/netapp/harvest/v2/pkg/matrix"
"github.com/netapp/harvest/v2/pkg/set"
"github.com/netapp/harvest/v2/pkg/tree/node"
"github.com/netapp/harvest/v2/pkg/util"
"github.com/tidwall/gjson"
"strconv"
"strings"
"time"
)
const defaultDataPollDuration = 3 * time.Minute
const maxURLSize = 8_000 // bytes
const severityFilterPrefix = "message.severity="
const defaultSeverityFilter = "alert|emergency|error|informational|notice"
const MaxBookendInstances = 1000
const DefaultBookendResolutionDuration = 28 * 24 * time.Hour // 28 days == 672 hours
const Hyphen = "-"
const AutoResolved = "autoresolved"
type Ems struct {
*rest2.Rest // provides: AbstractCollector, Client, Object, Query, TemplateFn, TemplateType
Query string
TemplatePath string
emsProp map[string][]*emsProp // array is used here to handle same ems written with different ops, matches or exports. Example: arw.volume.state ems with op as disabled or dry-run
Filter []string
Fields []string
ReturnTimeOut *int
lastFilterTime int64
maxURLSize int
DefaultLabels []string
severityFilter string
eventNames []string // consist of all ems events supported
bookendEmsMap map[string]*set.Set // This is reverse bookend ems map, [Resolving ems]:[Set of Issuing ems]. Using Set here to ensure that it has slice of unique issuing ems
resolveAfter map[string]time.Duration // This is resolve after map, [Issuing ems]:[Duration]. After this duration, ems got auto resolved.
}
type Metric struct {
Label string
Name string
MetricType string
Exportable bool
}
type Matches struct {
Name string
value string
}
type emsProp struct {
Name string
InstanceKeys []string
InstanceLabels map[string]string
Metrics map[string]*Metric
Plugins []plugin.Plugin // built-in or custom plugins
Matches []*Matches
Labels map[string]string
}
func init() {
plugin.RegisterModule(&Ems{})
}
func (e *Ems) HarvestModule() plugin.ModuleInfo {
return plugin.ModuleInfo{
ID: "harvest.collector.ems",
New: func() plugin.Module { return new(Ems) },
}
}
func (e *Ems) InitEmsProp() {
e.emsProp = make(map[string][]*emsProp)
}
func (e *Ems) Init(a *collector.AbstractCollector) error {
var err error
e.Rest = &rest2.Rest{AbstractCollector: a}
e.Fields = []string{"*"}
e.maxURLSize = maxURLSize
e.severityFilter = severityFilterPrefix + defaultSeverityFilter
// init Rest props
e.InitProp()
// init ems props
e.InitEmsProp()
e.bookendEmsMap = make(map[string]*set.Set)
e.resolveAfter = make(map[string]time.Duration)
if err := e.InitClient(); err != nil {
return err
}
if e.TemplatePath, err = e.LoadTemplate(); err != nil {
return err
}
if err := collector.Init(e); err != nil {
return err
}
if err := e.InitCache(); err != nil {
return err
}
return e.InitMatrix()
}
func (e *Ems) InitMatrix() error {
mat := e.Matrix[e.Object]
// overwrite from abstract collector
mat.Object = e.Object
// Add system (cluster) name
mat.SetGlobalLabel("cluster", e.Client.Cluster().Name)
mat.SetGlobalLabel("cluster_uuid", e.Client.Cluster().UUID)
if e.Params.HasChildS("labels") {
for _, l := range e.Params.GetChildS("labels").GetChildren() {
mat.SetGlobalLabel(l.GetNameS(), l.GetContentS())
}
}
return nil
}
func (e *Ems) LoadPlugin(kind string, _ *plugin.AbstractPlugin) plugin.Plugin {
e.Logger.Warn().Str("kind", kind).Msg("no ems plugin found")
return nil
}
func (e *Ems) InitCache() error {
var (
events *node.Node
)
if x := e.Params.GetChildContentS("object"); x != "" {
e.Prop.Object = x
} else {
e.Prop.Object = strings.ToLower(e.Object)
}
if b := e.Params.GetChildContentS("max_url_size"); b != "" {
if s, err := strconv.Atoi(b); err == nil {
e.maxURLSize = s
}
}
e.Logger.Debug().Int("max_url_size", e.maxURLSize).Msgf("")
if s := e.Params.GetChildContentS("severity"); s != "" {
e.severityFilter = severityFilterPrefix + s
}
e.Logger.Debug().Str("severityFilter", e.severityFilter).Msgf("")
if export := e.Params.GetChildS("export_options"); export != nil {
e.Matrix[e.Object].SetExportOptions(export)
}
if e.Query = e.Params.GetChildContentS("query"); e.Query == "" {
return errs.New(errs.ErrMissingParam, "query")
}
// Used for autosupport
e.Prop.Query = e.Query
if exports := e.Params.GetChildS("exports"); exports != nil {
for _, line := range exports.GetChildren() {
if line != nil {
e.DefaultLabels = append(e.DefaultLabels, line.GetContentS())
}
}
}
if events = e.Params.GetChildS("events"); events == nil || len(events.GetChildren()) == 0 {
return errs.New(errs.ErrMissingParam, "events")
}
// default value for ONTAP is 15 sec
if returnTimeout := e.Params.GetChildContentS("return_timeout"); returnTimeout != "" {
iReturnTimeout, err := strconv.Atoi(returnTimeout)
if err != nil {
e.Logger.Warn().Str("returnTimeout", returnTimeout).Msg("Invalid value of returnTimeout")
} else {
e.ReturnTimeOut = &iReturnTimeout
}
}
// init plugins
if e.Plugins == nil {
e.Plugins = make(map[string][]plugin.Plugin)
}
for _, line := range events.GetChildren() {
prop := emsProp{}
prop.InstanceKeys = make([]string, 0)
prop.InstanceLabels = make(map[string]string)
prop.Metrics = make(map[string]*Metric)
// check if name is present in template
if line.GetChildContentS("name") == "" {
e.Logger.Error().Msg("Missing event name")
continue
}
// populate prop counter for asup
eventName := line.GetChildContentS("name")
e.Prop.Counters[eventName] = eventName
e.ParseDefaults(&prop)
for _, line1 := range line.GetChildren() {
if line1.GetNameS() == "name" {
prop.Name = line1.GetContentS()
}
if line1.GetNameS() == "exports" {
e.ParseExports(line1, &prop)
}
if line1.GetNameS() == "matches" {
e.ParseMatches(line1, &prop)
}
if line1.GetNameS() == "labels" {
e.ParseLabels(line1, &prop)
}
if line1.GetNameS() == "plugins" {
if err := e.LoadPlugins(line1, e, prop.Name); err != nil {
e.Logger.Error().Stack().Err(err).Msg("Failed to load plugin")
}
}
if line1.GetNameS() == "resolve_when_ems" {
e.ParseResolveEms(line1, prop)
}
}
e.emsProp[prop.Name] = append(e.emsProp[prop.Name], &prop)
}
// add severity filter
e.Filter = append(e.Filter, e.severityFilter)
return nil
}
// returns time filter (clustertime - polldata duration)
func (e *Ems) getTimeStampFilter(clusterTime time.Time) string {
fromTime := e.lastFilterTime
// check if this is the first request
if e.lastFilterTime == 0 {
// if first request fetch cluster time
dataDuration, err := collectors.GetDataInterval(e.GetParams(), defaultDataPollDuration)
if err != nil {
e.Logger.Warn().Err(err).
Str("defaultDataPollDuration", defaultDataPollDuration.String()).
Msg("Failed to parse duration. using default")
}
fromTime = clusterTime.Add(-dataDuration).Unix()
}
return fmt.Sprintf("time=>=%d", fromTime)
}
func (e *Ems) fetchEMSData(href string) ([]gjson.Result, error) {
var (
records []gjson.Result
err error
)
if records, err = e.GetRestData(href); err != nil {
return nil, err
}
return records, nil
}
// PollInstance queries the cluster's EMS catalog and intersects that catalog with the EMS template.
// This is required because ONTAP EMS Rest endpoint fails when queried for an EMS message that does not exist.
func (e *Ems) PollInstance() (map[string]*matrix.Matrix, error) {
var (
err error
records []gjson.Result
bookendCacheSize int
)
query := "api/support/ems/messages"
fields := []string{"name"}
href := rest.NewHrefBuilder().
APIPath(query).
Fields(fields).
ReturnTimeout(e.ReturnTimeOut).
Build()
apiT := time.Now()
if records, err = e.GetRestData(href); err != nil {
return nil, err
}
apiD := time.Since(apiT)
parseT := time.Now()
if len(records) == 0 {
return nil, errs.New(errs.ErrNoInstance, e.Object+" no ems message found on cluster")
}
var emsEventCatalogue []string
for _, instanceData := range records {
name := instanceData.Get("name")
if name.Exists() {
emsEventCatalogue = append(emsEventCatalogue, name.String())
}
}
// collect all event names
names := make([]string, 0, len(e.emsProp))
for key := range e.emsProp {
names = append(names, key)
}
// Filter out names which exist on the cluster.
// ONTAP rest ems throws error for a message.name filter if that event is not supported by that cluster
filteredNames, _ := util.Intersection(names, emsEventCatalogue)
_, missingNames := util.Intersection(filteredNames, names)
e.Logger.Debug().Strs("skipped events", missingNames).Send()
e.eventNames = filteredNames
// warning when total instance in cache > 1000 instance
for _, issuingEmsList := range e.bookendEmsMap {
for _, issuingEms := range issuingEmsList.Slice() {
if mx := e.Matrix[issuingEms]; mx != nil {
bookendCacheSize += len(mx.GetInstances())
}
}
}
e.Logger.Info().Int("total instances", bookendCacheSize).Send()
// warning when total instance in cache > 1000 instance
if bookendCacheSize > MaxBookendInstances {
e.Logger.Warn().Int("total instances", bookendCacheSize).Msg("cache has more than 1000 instances")
}
// update metadata for collector logs
_ = e.Metadata.LazySetValueInt64("api_time", "instance", apiD.Microseconds())
_ = e.Metadata.LazySetValueInt64("parse_time", "instance", time.Since(parseT).Microseconds())
_ = e.Metadata.LazySetValueUint64("instances", "instance", uint64(bookendCacheSize))
return nil, nil
}
func (e *Ems) PollData() (map[string]*matrix.Matrix, error) {
var (
count, instanceCount uint64
apiD, parseD time.Duration
startTime time.Time
err error
records []gjson.Result
)
// Update cache for bookend ems
e.updateMatrix(time.Now())
startTime = time.Now()
// add time filter
clusterTime, err := collectors.GetClusterTime(e.Client, e.ReturnTimeOut, e.Logger)
if err != nil {
return nil, err
}
toTime := clusterTime.Unix()
timeFilter := e.getTimeStampFilter(clusterTime)
filter := e.Filter
filter = append(filter, timeFilter)
// build hrefs up to maxURLSize
var hrefs []string
start := 0
for end := 0; end < len(e.eventNames); end++ {
h := e.getHref(e.eventNames[start:end], filter)
if len(h) > e.maxURLSize {
if end == 0 {
return nil, fmt.Errorf("maxURLSize=%d is too small to form queries. Increase it to at least %d",
e.maxURLSize, len(h))
}
end--
h = e.getHref(e.eventNames[start:end], filter)
hrefs = append(hrefs, h)
start = end
} else if end == len(e.eventNames)-1 {
end = len(e.eventNames)
h = e.getHref(e.eventNames[start:end], filter)
hrefs = append(hrefs, h)
}
}
for _, h := range hrefs {
r, err := e.fetchEMSData(h)
if err != nil {
return nil, err
}
records = append(records, r...)
}
apiD = time.Since(startTime)
startTime = time.Now()
_, count, instanceCount = e.HandleResults(records, e.emsProp)
parseD = time.Since(startTime)
_ = e.Metadata.LazySetValueInt64("api_time", "data", apiD.Microseconds())
_ = e.Metadata.LazySetValueInt64("parse_time", "data", parseD.Microseconds())
_ = e.Metadata.LazySetValueUint64("metrics", "data", count)
_ = e.Metadata.LazySetValueUint64("instances", "data", instanceCount)
e.AddCollectCount(count)
// update lastFilterTime to current cluster time
e.lastFilterTime = toTime
return e.Matrix, nil
}
func (e *Ems) getHref(names []string, filter []string) string {
nameFilter := "message.name=" + strings.Join(names, ",")
filter = append(filter, nameFilter)
// If both issuing ems and resolving ems would come together in same poll, This index ordering would make sure that latest ems would process last. So, if resolving ems would be latest, it will resolve the issue.
// add filter as order by index in ascending order
orderByIndexFilter := "order_by=" + "index%20asc"
filter = append(filter, orderByIndexFilter)
href := rest.NewHrefBuilder().
APIPath(e.Query).
Fields(e.Fields).
Filter(filter).
ReturnTimeout(e.ReturnTimeOut).
Build()
return href
}
func parseProperties(instanceData gjson.Result, property string) gjson.Result {
if !strings.HasPrefix(property, "parameters.") {
// if prefix is not parameters.
value := gjson.Get(instanceData.String(), property)
return value
}
// strip parameters. from property name
_, after, found := strings.Cut(property, "parameters.")
if found {
property = after
}
// process parameter search
t := gjson.Get(instanceData.String(), "parameters.#.name")
for _, name := range t.Array() {
if name.String() == property {
value := gjson.Get(instanceData.String(), "parameters.#(name="+property+").value")
return value
}
}
return gjson.Result{}
}
// HandleResults function is used for handling the rest response for parent as well as endpoints calls,
func (e *Ems) HandleResults(result []gjson.Result, prop map[string][]*emsProp) (map[string]*matrix.Matrix, uint64, uint64) {
var (
err error
count, instanceCount uint64
mx *matrix.Matrix
)
var m = e.Matrix
for _, instanceData := range result {
var (
instanceKey string
)
if !instanceData.IsObject() {
e.Logger.Warn().Str("type", instanceData.Type.String()).Msg("Instance data is not object, skipping")
continue
}
messageName := instanceData.Get("message.name")
// verify if message name exists in ONTAP response
if !messageName.Exists() {
e.Logger.Error().Msg("skip instance, missing message name")
continue
}
msgName := messageName.String()
if issuingEmsList, ok := e.bookendEmsMap[msgName]; ok {
props := prop[msgName]
if len(props) == 0 {
e.Logger.Warn().Str("resolving ems", msgName).
Msg("Ems properties not found")
continue
}
// resolving ems would only have 1 prop record
p := props[0]
bookendKey := e.getInstanceKeys(p, instanceData)
emsResolved := false
/* Below logic will evaluate this way:
case 1: For Bookend ems (one to one): [LUN.offline - LUN.offline]
- loop would iterate once
- if issuing ems exist then resolve else log warning as unable to find matching ems in cache
case 2: For Bookend with same resoling ems (many to one): [monitor.fan.critical, monitor.fan.failed, monitor.fan.warning - monitor.fan.ok]
- loop would iterate for all possible issuing ems
- if one or more issuing ems exist then resolve all matching ems else log warning as unable to find matching ems in cache
*/
for _, issuingEms := range issuingEmsList.Slice() {
if mx = m[issuingEms]; mx != nil {
metr, exist := mx.GetMetrics()["events"]
if !exist {
e.Logger.Warn().
Str("name", "events").
Msg("failed to get metric")
continue
}
// get all active instances by issuingems-bookendkey
if instances := mx.GetInstancesBySuffix(issuingEms + bookendKey); len(instances) != 0 {
for _, instance := range instances {
if err = metr.SetValueFloat64(instance, 0); err != nil {
e.Logger.Error().Err(err).Str("key", "events").
Msg("Unable to set float key on metric")
continue
}
instance.SetExportable(true)
}
emsResolved = true
}
}
}
if !emsResolved {
e.Logger.Warn().Str("resolving ems", msgName).Str("issue ems", strings.Join(issuingEmsList.Slice(), ",")).
Msg("Unable to find matching issue ems in cache")
}
} else {
if _, ok := m[msgName]; !ok {
// create matrix if not exists for the ems event
mx = matrix.New(msgName, e.Prop.Object, msgName)
mx.SetGlobalLabels(e.Matrix[e.Object].GetGlobalLabels())
m[msgName] = mx
} else {
mx = m[msgName]
}
// Check matches at all same name ems
var isMatch bool
// Check matches at each ems
var isMatchPs bool
// Check instance count at all same name ems
instanceLabelCount := uint64(0)
// Check instance count at each ems
var instanceLabelCountPs uint64
// parse ems properties for the instance
if ps, ok := prop[msgName]; ok {
for _, p := range ps {
isMatchPs = false
instanceLabelCountPs = 0
instanceKey = e.getInstanceKeys(p, instanceData)
instance := mx.GetInstance(instanceKey)
if instance == nil {
if instance, err = mx.NewInstance(instanceKey); err != nil {
e.Logger.Error().Err(err).Str("Instance key", instanceKey).Send()
continue
}
}
// explicitly set to true. for bookend case, it was set as false for the same instance[when multiple ems received]
instance.SetExportable(true)
for label, display := range p.InstanceLabels {
if label == AutoResolved {
instance.SetLabel(display, "")
continue
}
value := parseProperties(instanceData, label)
if value.Exists() {
if value.IsArray() {
var labelArray []string
for _, r := range value.Array() {
labelString := r.String()
labelArray = append(labelArray, labelString)
}
instance.SetLabel(display, strings.Join(labelArray, ","))
} else {
instance.SetLabel(display, value.String())
}
instanceLabelCountPs++
} else {
e.Logger.Warn().Str("Instance key", instanceKey).Str("label", label).Msg("Missing label value")
}
}
// set labels
for k, v := range p.Labels {
instance.SetLabel(k, v)
}
// matches filtering
if len(p.Matches) == 0 {
isMatchPs = true
} else {
for _, match := range p.Matches {
if value := instance.GetLabel(match.Name); value != "" {
if value == match.value {
isMatchPs = true
break
}
} else {
// value not found
e.Logger.Warn().
Str("Instance key", instanceKey).
Str("name", match.Name).
Str("value", match.value).
Msg("label is not found")
}
}
}
if !isMatchPs {
continue
}
for _, metric := range p.Metrics {
metr, ok := mx.GetMetrics()[metric.Name]
if !ok {
if metr, err = mx.NewMetricFloat64(metric.Name); err != nil {
e.Logger.Error().Err(err).
Str("name", metric.Name).
Msg("failed to get metric")
continue
}
metr.SetExportable(metric.Exportable)
}
switch {
case metric.Name == "events":
if err = metr.SetValueFloat64(instance, 1); err != nil {
e.Logger.Error().Err(err).Str("key", metric.Name).Str("metric", metric.Label).
Msg("Unable to set float key on metric")
}
case metric.Name == "timestamp":
if err = metr.SetValueFloat64(instance, float64(time.Now().UnixMicro())); err != nil {
e.Logger.Error().Err(err).Str("key", metric.Name).Str("metric", metric.Label).
Msg("Unable to set timestamp on metric")
}
default:
e.Logger.Warn().Str("key", metric.Name).Str("metric", metric.Label).
Msg("Unable to find metric")
}
}
instanceLabelCount += instanceLabelCountPs
isMatch = true
}
}
if !isMatch {
mx.RemoveInstance(instanceKey)
continue
}
count += instanceLabelCount
}
}
for _, v := range e.Matrix {
for _, i := range v.GetInstances() {
if i.IsExportable() {
instanceCount++
}
}
}
return m, count, instanceCount
}
func (e *Ems) getInstanceKeys(p *emsProp, instanceData gjson.Result) string {
var instanceKey string
// extract instance key(s)
for _, k := range p.InstanceKeys {
value := parseProperties(instanceData, k)
if value.Exists() {
instanceKey += Hyphen + value.String()
} else {
e.Logger.Error().Str("key", k).Msg("skip instance, missing key")
break
}
}
return instanceKey
}
func (e *Ems) updateMatrix(begin time.Time) {
tempMap := make(map[string]*matrix.Matrix)
// store the bookend ems metric in tempMap
for _, issuingEmsList := range e.bookendEmsMap {
for _, issuingEms := range issuingEmsList.Slice() {
if mx, exist := e.Matrix[issuingEms]; exist {
tempMap[issuingEms] = mx
}
}
}
// remove all ems matrix except parent object
mat := e.Matrix[e.Object]
e.Matrix = make(map[string]*matrix.Matrix)
e.Matrix[e.Object] = mat
for issuingEms, mx := range tempMap {
eventMetric, ok := mx.GetMetrics()["events"]
if !ok {
e.Logger.Error().
Str("issuingEms", issuingEms).
Str("name", "events").
Msg("failed to get metric")
continue
}
timestampMetric, ok := mx.GetMetrics()["timestamp"]
if !ok {
e.Logger.Error().
Str("issuingEms", issuingEms).
Str("name", "timestamp").
Msg("failed to get metric")
continue
}
for instanceKey, instance := range mx.GetInstances() {
// set export to false
instance.SetExportable(false)
if val, exist := eventMetric.GetValueFloat64(instance); exist && val == 0 {
mx.RemoveInstance(instanceKey)
continue
}
// check instance timestamp and remove it after given resolve_after duration
if metricTimestamp, ok := timestampMetric.GetValueFloat64(instance); ok {
if collectors.IsTimestampOlderThanDuration(begin, metricTimestamp, e.resolveAfter[issuingEms]) {
// Set events metric value as 0 and export instance to true with label autoresolved as true.
if err := eventMetric.SetValueFloat64(instance, 0); err != nil {
e.Logger.Error().Err(err).Str("key", "events").
Msg("Unable to set float key on metric")
continue
}
instance.SetExportable(true)
instance.SetLabel(AutoResolved, "true")
}
}
}
if instances := mx.GetInstances(); len(instances) == 0 {
delete(e.Matrix, issuingEms)
continue
}
e.Matrix[issuingEms] = mx
}
}
// Interface guards
var (
_ collector.Collector = (*Ems)(nil)
)