/
metrics.go
77 lines (69 loc) · 2.3 KB
/
metrics.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
package worker
import (
"time"
"github.com/moira-alert/moira"
"github.com/patrickmn/go-cache"
)
func (worker *Checker) newMetricsHandler(metricEventsChannel <-chan *moira.MetricEvent) error {
for {
metricEvent, ok := <-metricEventsChannel
if !ok {
return nil
}
pattern := metricEvent.Pattern
if worker.needHandlePattern(pattern) {
if err := worker.handleMetricEvent(pattern); err != nil {
worker.Logger.Errorf("Failed to handle metricEvent: %s", err.Error())
}
}
}
}
func (worker *Checker) needHandlePattern(pattern string) bool {
err := worker.PatternCache.Add(pattern, true, cache.DefaultExpiration)
return err == nil
}
func (worker *Checker) handleMetricEvent(pattern string) error {
start := time.Now()
defer worker.Metrics.MetricEventsHandleTime.UpdateSince(start)
worker.lastData = time.Now().UTC().Unix()
triggerIds, err := worker.Database.GetPatternTriggerIDs(pattern)
if err != nil {
return err
}
// Cleanup pattern and its metrics if this pattern doesn't match to any trigger
if len(triggerIds) == 0 {
if err := worker.Database.RemovePatternWithMetrics(pattern); err != nil {
return err
}
}
worker.addTriggerIDsIfNeeded(triggerIds)
return nil
}
func (worker *Checker) addTriggerIDsIfNeeded(triggerIDs []string) {
needToCheckTriggerIDs := worker.getTriggerIDsToCheck(triggerIDs)
if len(needToCheckTriggerIDs) > 0 {
worker.Database.AddLocalTriggersToCheck(needToCheckTriggerIDs)
}
}
func (worker *Checker) addRemoteTriggerIDsIfNeeded(triggerIDs []string) {
needToCheckRemoteTriggerIDs := worker.getTriggerIDsToCheck(triggerIDs)
if len(needToCheckRemoteTriggerIDs) > 0 {
worker.Database.AddRemoteTriggersToCheck(needToCheckRemoteTriggerIDs)
}
}
func (worker *Checker) getTriggerIDsToCheck(triggerIDs []string) []string {
lazyTriggerIDs := worker.lazyTriggerIDs.Load().(map[string]bool)
triggerIDsToCheck := make([]string, len(triggerIDs))
for _, triggerID := range triggerIDs {
if _, ok := lazyTriggerIDs[triggerID]; ok {
randomDuration := worker.getRandomLazyCacheDuration()
if err := worker.LazyTriggersCache.Add(triggerID, true, randomDuration); err != nil {
continue
}
}
if err := worker.TriggerCache.Add(triggerID, true, cache.DefaultExpiration); err == nil {
triggerIDsToCheck = append(triggerIDsToCheck, triggerID)
}
}
return triggerIDsToCheck
}