-
Notifications
You must be signed in to change notification settings - Fork 430
/
wait.go
109 lines (93 loc) · 3.4 KB
/
wait.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package framework
import (
"fmt"
"time"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/util/wait"
)
type timeoutError struct {
msg string
observedObjects []interface{}
}
func (e *timeoutError) Error() string {
return e.msg
}
func TimeoutError(msg string, observedObjects ...interface{}) *timeoutError {
return &timeoutError{
msg: msg,
observedObjects: observedObjects,
}
}
// maybeTimeoutError returns a TimeoutError if err is a timeout. Otherwise, wrap err.
// taskFormat and taskArgs should be the task being performed when the error occurred,
// e.g. "waiting for pod to be running".
func maybeTimeoutError(err error, taskFormat string, taskArgs ...interface{}) error {
if IsTimeout(err) {
return TimeoutError(fmt.Sprintf("timed out while "+taskFormat, taskArgs...))
} else if err != nil {
return fmt.Errorf("error while %s: %w", fmt.Sprintf(taskFormat, taskArgs...), err)
} else {
return nil
}
}
func IsTimeout(err error) bool {
if err == wait.ErrWaitTimeout {
return true
}
if _, ok := err.(*timeoutError); ok {
return true
}
return false
}
// handleWaitingAPIErrror handles an error from an API request in the context of a Wait function.
// If the error is retryable, sleep the recommended delay and ignore the error.
// If the erorr is terminal, return it.
func handleWaitingAPIError(err error, retryNotFound bool, taskFormat string, taskArgs ...interface{}) (bool, error) {
taskDescription := fmt.Sprintf(taskFormat, taskArgs...)
if retryNotFound && apierrors.IsNotFound(err) {
Logf("Ignoring NotFound error while " + taskDescription)
return false, nil
}
if retry, delay := shouldRetry(err); retry {
Logf("Retryable error while %s, retrying after %v: %v", taskDescription, delay, err)
if delay > 0 {
time.Sleep(delay)
}
return false, nil
}
Logf("Encountered non-retryable error while %s: %v", taskDescription, err)
return false, err
}
// Decide whether to retry an API request. Optionally include a delay to retry after.
func shouldRetry(err error) (retry bool, retryAfter time.Duration) {
// if the error sends the Retry-After header, we respect it as an explicit confirmation we should retry.
if delay, shouldRetry := apierrors.SuggestsClientDelay(err); shouldRetry {
return shouldRetry, time.Duration(delay) * time.Second
}
// these errors indicate a transient error that should be retried.
if apierrors.IsTimeout(err) || apierrors.IsTooManyRequests(err) {
return true, 0
}
return false, 0
}
// WaitUntil waits the condition to be met
func WaitUntil(cond func() (bool, error), condDesc string) {
if err := wait.PollImmediate(2*time.Second, timeout, cond); err != nil {
if IsTimeout(err) {
Failf("timed out while waiting for the condition to be met: %s", condDesc)
}
Fail(maybeTimeoutError(err, "waiting for the condition %q to be met", condDesc).Error())
}
}