-
-
Notifications
You must be signed in to change notification settings - Fork 957
/
retry.go
62 lines (49 loc) · 1.63 KB
/
retry.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
package util
import (
"context"
"fmt"
"time"
"github.com/gruntwork-io/terragrunt/pkg/log"
"github.com/sirupsen/logrus"
)
// DoWithRetry runs the specified action. If it returns a value, return that value. If it returns an error, sleep for
// sleepBetweenRetries and try again, up to a maximum of maxRetries retries. If maxRetries is exceeded, return a
// MaxRetriesExceeded error.
func DoWithRetry(ctx context.Context, actionDescription string, maxRetries int, sleepBetweenRetries time.Duration, logLevel logrus.Level, action func() error) error {
if ctx.Err() != nil {
return ctx.Err()
}
for i := 0; i <= maxRetries; i++ {
log.Logf(logLevel, actionDescription)
err := action()
if err == nil {
return nil
}
if _, isFatalErr := err.(FatalError); isFatalErr {
return err
}
log.Errorf("%s returned an error: %s. Retry %d of %d. Sleeping for %s and will try again.", actionDescription, err.Error(), i, maxRetries, sleepBetweenRetries)
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(sleepBetweenRetries):
// try again
}
}
return MaxRetriesExceeded{Description: actionDescription, MaxRetries: maxRetries}
}
// MaxRetriesExceeded is an error that occurs when the maximum amount of retries is exceeded.
type MaxRetriesExceeded struct {
Description string
MaxRetries int
}
func (err MaxRetriesExceeded) Error() string {
return fmt.Sprintf("'%s' unsuccessful after %d retries", err.Description, err.MaxRetries)
}
// FatalError is error interface for cases that should not be retried.
type FatalError struct {
Underlying error
}
func (err FatalError) Error() string {
return err.Underlying.Error()
}