forked from canonical/lxd
-
Notifications
You must be signed in to change notification settings - Fork 0
/
retry.go
84 lines (69 loc) · 1.76 KB
/
retry.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
package query
import (
"database/sql"
"strings"
"time"
"github.com/Rican7/retry/jitter"
"github.com/canonical/go-dqlite/driver"
"github.com/mattn/go-sqlite3"
"github.com/pkg/errors"
"github.com/lxc/lxd/shared/logger"
)
const maxRetries = 250
// Retry wraps a function that interacts with the database, and retries it in
// case a transient error is hit.
//
// This should by typically used to wrap transactions.
func Retry(f func() error) error {
// TODO: the retry loop should be configurable.
var err error
for i := 0; i < maxRetries; i++ {
err = f()
if err != nil {
// No point in re-trying or logging a no-row error.
if err == sql.ErrNoRows {
break
}
// Process actual errors.
logger.Debugf("Database error: %#v", err)
if IsRetriableError(err) {
if i == maxRetries {
logger.Warnf("Give up retrying database error: %v", err)
break
}
logger.Debugf("Retry failed db interaction (%v)", err)
time.Sleep(jitter.Deviation(nil, 0.8)(100 * time.Millisecond))
continue
}
}
break
}
return err
}
// IsRetriableError returns true if the given error might be transient and the
// interaction can be safely retried.
func IsRetriableError(err error) bool {
err = errors.Cause(err)
if err == nil {
return false
}
if err, ok := err.(driver.Error); ok && err.Code == driver.ErrBusy {
return true
}
if err == sqlite3.ErrLocked || err == sqlite3.ErrBusy {
return true
}
if strings.Contains(err.Error(), "database is locked") {
return true
}
if strings.Contains(err.Error(), "cannot start a transaction within a transaction") {
return true
}
if strings.Contains(err.Error(), "bad connection") {
return true
}
if strings.Contains(err.Error(), "checkpoint in progress") {
return true
}
return false
}