-
Notifications
You must be signed in to change notification settings - Fork 29
/
status.go
127 lines (110 loc) · 3.59 KB
/
status.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
package canary
import (
"fmt"
"strings"
"time"
v1 "github.com/flanksource/canary-checker/api/v1"
"github.com/flanksource/canary-checker/pkg"
"github.com/flanksource/canary-checker/pkg/db"
"github.com/flanksource/canary-checker/pkg/metrics"
"github.com/flanksource/canary-checker/pkg/utils"
"github.com/flanksource/duty/context"
"github.com/flanksource/duty/query"
dutyTypes "github.com/flanksource/duty/types"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
)
func UpdateCanaryStatusAndEvent(ctx context.Context, canary v1.Canary, results []*pkg.CheckResult) {
if CanaryStatusChannel == nil {
return
}
// Skip function if canary is not sourced from Kubernetes CRD
if !strings.HasPrefix(canary.Annotations["source"], "kubernetes") {
return
}
var checkStatus = make(map[string]*v1.CheckStatus)
var duration int64
var messages, errors []string
var failEvents []string
var msg, errorMsg string
var pass = true
var lastTransitionedTime *metav1.Time
var highestLatency float64
var uptimeAgg dutyTypes.Uptime
transitioned := false
for _, result := range results {
// Increment duration
duration += result.Duration
// Set uptime and latency
uptime, latency := metrics.Record(ctx, canary, result)
checkID := canary.Status.Checks[result.Check.GetName()]
checkStatus[checkID] = &v1.CheckStatus{
Uptime1H: uptime.String(),
Latency1H: latency.String(),
}
// Increment aggregate uptime
uptimeAgg.Passed += uptime.Passed
uptimeAgg.Failed += uptime.Failed
// Use highest latency for canary status
if latency.Rolling1H > highestLatency {
highestLatency = latency.Rolling1H
}
// Transition
q := query.CheckQueryParams{Check: checkID, StatusCount: 1}
if canary.Status.LastTransitionedTime != nil {
q.Start = canary.Status.LastTransitionedTime.Format(time.RFC3339)
}
latestCheckStatus, err := db.LatestCheckStatus(ctx, checkID)
if err != nil || latestCheckStatus == nil {
transitioned = true
} else if latestCheckStatus.Status != result.Pass {
transitioned = true
}
if transitioned {
transitionTime := time.Now()
if latestCheckStatus != nil {
transitionTime = latestCheckStatus.CreatedAt
}
checkStatus[checkID].LastTransitionedTime = &metav1.Time{Time: transitionTime}
lastTransitionedTime = &metav1.Time{Time: transitionTime}
}
// Update status message
if len(messages) == 1 {
msg = messages[0]
} else if len(messages) > 1 {
msg = fmt.Sprintf("%s, (%d more)", messages[0], len(messages)-1)
}
if len(errors) == 1 {
errorMsg = errors[0]
} else if len(errors) > 1 {
errorMsg = fmt.Sprintf("%s, (%d more)", errors[0], len(errors)-1)
}
if !result.Pass {
failEvents = append(failEvents, fmt.Sprintf("%s-%s: %s", result.Check.GetType(), result.Check.GetEndpoint(), result.Message))
pass = false
}
}
payload := CanaryStatusPayload{
Pass: pass,
CheckStatus: checkStatus,
FailEvents: failEvents,
LastTransitionedTime: lastTransitionedTime,
Message: msg,
ErrorMessage: errorMsg,
Uptime: uptimeAgg.String(),
Latency: utils.Age(time.Duration(highestLatency) * time.Millisecond),
NamespacedName: canary.GetNamespacedName(),
}
CanaryStatusChannel <- payload
}
type CanaryStatusPayload struct {
Pass bool
CheckStatus map[string]*v1.CheckStatus
FailEvents []string
LastTransitionedTime *metav1.Time
Message string
ErrorMessage string
Uptime string
Latency string
NamespacedName types.NamespacedName
}