forked from canonical/lxd
/
retry.go
58 lines (50 loc) · 1.19 KB
/
retry.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
package query
import (
"strings"
"time"
"github.com/CanonicalLtd/go-sqlite3"
"github.com/lxc/lxd/shared/logger"
"github.com/pkg/errors"
)
// Retry wraps a function that interacts with the database, and retries it in
// case a transient error is hit.
//
// This should by typically used to wrap transactions.
func Retry(f func() error) error {
// TODO: the retry loop should be configurable.
var err error
for i := 0; i < 20; i++ {
err = f()
if err != nil {
logger.Debugf("Database error %#v", err)
if IsRetriableError(err) {
logger.Debugf("Retry failed db interaction (%v)", err)
time.Sleep(250 * time.Millisecond)
continue
}
}
break
}
return err
}
// IsRetriableError returns true if the given error might be transient and the
// interaction can be safely retried.
func IsRetriableError(err error) bool {
err = errors.Cause(err)
if err == nil {
return false
}
if err == sqlite3.ErrLocked || err == sqlite3.ErrBusy {
return true
}
if strings.Contains(err.Error(), "database is locked") {
return true
}
if strings.Contains(err.Error(), "bad connection") {
return true
}
if strings.Contains(err.Error(), "leadership lost") {
return true
}
return false
}