-
Notifications
You must be signed in to change notification settings - Fork 99
/
reconcile_timer.go
128 lines (103 loc) · 3.14 KB
/
reconcile_timer.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
// Copyright 2024 The Carvel Authors.
// SPDX-License-Identifier: Apache-2.0
package pkgrepository
import (
"fmt"
"math"
"time"
"carvel.dev/kapp-controller/pkg/apis/kappctrl/v1alpha1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
)
type ReconcileTimer struct {
app v1alpha1.App
}
func NewReconcileTimer(app v1alpha1.App) ReconcileTimer {
return ReconcileTimer{*app.DeepCopy()}
}
func (rt ReconcileTimer) DurationUntilReady(err error) time.Duration {
if err != nil || rt.hasReconcileStatus(v1alpha1.ReconcileFailed) || rt.hasReconcileStatus(v1alpha1.DeleteFailed) {
return rt.failureSyncPeriod()
}
return rt.applyJitter(rt.syncPeriod())
}
func (rt ReconcileTimer) IsReadyAt(timeAt time.Time) bool {
// Did resource spec change?
if rt.app.Status.ObservedGeneration != rt.app.Generation {
return true
}
// TODO: Is this needed due to first case statement?
// If paused, then no reconciliation until unpaused
if rt.app.Spec.Paused {
return false
}
lastReconcileTime, err := rt.lastReconcileTime()
if err != nil {
return true
}
if rt.hasReconcileStatus(v1alpha1.ReconcileFailed) || rt.hasReconcileStatus(v1alpha1.DeleteFailed) {
if timeAt.UTC().Sub(lastReconcileTime) >= rt.failureSyncPeriod() {
return true
}
}
// Did we deploy too long ago?
if timeAt.UTC().Sub(lastReconcileTime) >= rt.syncPeriod() {
return true
}
return false
}
func (rt ReconcileTimer) syncPeriod() time.Duration {
const defaultSyncPeriod = 30 * time.Second
if sp := rt.app.Spec.SyncPeriod; sp != nil && sp.Duration > defaultSyncPeriod {
return sp.Duration
}
return defaultSyncPeriod
}
func (rt ReconcileTimer) failureSyncPeriod() time.Duration {
consecFailures := float64(rt.app.Status.ConsecutiveReconcileFailures)
// cap consec failures that we are willing to calculate for to avoid overflows
maxConsecFailures := float64(30)
if consecFailures > 0 && consecFailures < maxConsecFailures {
d := time.Duration(math.Exp2(consecFailures)) * time.Second
if d < rt.syncPeriod() {
return d
}
}
return rt.syncPeriod()
}
func (rt ReconcileTimer) hasReconcileStatus(c v1alpha1.ConditionType) bool {
for _, cond := range rt.app.Status.Conditions {
if cond.Type == c {
return true
}
}
return false
}
func (rt ReconcileTimer) applyJitter(t time.Duration) time.Duration {
const appJitter time.Duration = 5 * time.Second
return t - appJitter + wait.Jitter(appJitter, 1.0)
}
func (rt ReconcileTimer) lastReconcileTime() (time.Time, error) {
// Determine latest time from status and use that as the
// last reconcile time
lastReconcileTime := metav1.Time{}
times := []metav1.Time{}
if rt.app.Status.Fetch != nil {
times = append(times, rt.app.Status.Fetch.UpdatedAt)
}
if rt.app.Status.Template != nil {
times = append(times, rt.app.Status.Template.UpdatedAt)
}
if rt.app.Status.Deploy != nil {
times = append(times, rt.app.Status.Deploy.UpdatedAt)
}
for _, time := range times {
if lastReconcileTime.Before(&time) {
lastReconcileTime = time
}
}
if lastReconcileTime.IsZero() {
return time.Time{}, fmt.Errorf("could not determine time of last reconcile")
}
return lastReconcileTime.Time, nil
}