-
Notifications
You must be signed in to change notification settings - Fork 2
/
upgrade_status.go
128 lines (116 loc) · 3.87 KB
/
upgrade_status.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
package appliance
import (
"context"
"errors"
"fmt"
"regexp"
"strings"
"time"
"github.com/appgate/sdp-api-client-go/api/v20/openapi"
"github.com/appgate/sdpctl/pkg/tui"
"github.com/appgate/sdpctl/pkg/util"
"github.com/cenkalti/backoff/v4"
log "github.com/sirupsen/logrus"
)
const backoffDefaultTimeout = time.Minute * 30
var defaultExponentialBackOff = &backoff.ExponentialBackOff{
InitialInterval: 1 * time.Second,
Multiplier: 2,
RandomizationFactor: 0.7,
MaxInterval: 10 * time.Second,
MaxElapsedTime: backoffDefaultTimeout,
Stop: backoff.Stop,
Clock: backoff.SystemClock,
}
type WaitForUpgradeStatus interface {
// WaitForUpgradeStatus does exponential backoff retries on upgrade status until it reaches a desiredStatuses and reports it to current <- string
WaitForUpgradeStatus(ctx context.Context, appliance openapi.Appliance, desiredStatuses []string, undesiredStatuses []string, tracker *tui.Tracker) error
}
type UpgradeStatus struct {
Appliance *Appliance
}
type IsPrimaryUpgrade string
type CalledAs string
const (
PrimaryUpgrade IsPrimaryUpgrade = "IsPrimaryUpgrade"
Caller CalledAs = "calledAs"
)
func (u *UpgradeStatus) upgradeStatus(ctx context.Context, appliance openapi.Appliance, desiredStatuses []string, undesiredStatuses []string, tracker *tui.Tracker) backoff.Operation {
name := appliance.GetName()
logEntry := log.WithField("appliance", name)
logEntry.WithField("want", desiredStatuses).Info("Polling for upgrade status")
hasRebooted := false
offlineRegex := regexp.MustCompile(`No response Get`)
onlineRegex := regexp.MustCompile(`Bad Gateway`)
return func() error {
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
status, err := u.Appliance.UpgradeStatus(ctx, appliance.GetId())
if err != nil {
if tracker != nil {
msg := tracker.Current()
// Check if upgrading primary controller and apply logic for offline check
if primaryUpgrade, ok := ctx.Value(PrimaryUpgrade).(bool); ok && primaryUpgrade {
msg = "switching partition"
if offlineRegex.MatchString(err.Error()) {
hasRebooted = true
} else if onlineRegex.MatchString(err.Error()) {
if hasRebooted {
msg = "initializing"
} else {
msg = "installing"
}
} else {
msg = err.Error()
}
} else if !errors.Is(err, context.DeadlineExceeded) {
msg = err.Error()
}
tracker.Update(msg)
}
logEntry.WithError(err).Debug("No response, appliance offline")
return err
}
var s string
details := status.GetDetails()
if v, ok := status.GetStatusOk(); ok {
s = *v
logEntry.WithField("current", s).Debug("Received status")
if tracker != nil {
tracker.Update(s)
}
if util.InSlice(s, undesiredStatuses) {
if tracker != nil {
// send error details for tracker
tracker.Fail(s + " - " + details)
}
err := fmt.Errorf("Command failed on %s %s %s", name, s, details)
if calledAs, ok := ctx.Value(Caller).(string); ok {
err = fmt.Errorf("%s failed on %s %s %s", calledAs, name, s, details)
}
logEntry.WithError(err).WithFields(log.Fields{"status": s, "details": details}).Error("Unwanted status on appliance")
return backoff.Permanent(err)
}
if util.InSlice(s, desiredStatuses) {
logEntry.Info("Reached wanted status")
return nil
}
}
return fmt.Errorf(
"%s never reached %s, got %q %s",
name,
strings.Join(desiredStatuses, ", "),
s,
details,
)
}
}
func (u *UpgradeStatus) WaitForUpgradeStatus(ctx context.Context, appliance openapi.Appliance, desiredStatuses []string, undesiredStatuses []string, tracker *tui.Tracker) error {
b := backoff.WithContext(defaultExponentialBackOff, ctx)
select {
case <-ctx.Done():
return ctx.Err()
default:
return backoff.Retry(u.upgradeStatus(ctx, appliance, desiredStatuses, undesiredStatuses, tracker), b)
}
}