Ensures that the lease manager retries errors due to database

contention.

worker/lease/manager.go

@@ -20,6 +20,7 @@ import (
 	"gopkg.in/retry.v1"
 
 	"github.com/juju/juju/core/lease"
+	"github.com/juju/juju/database"
 )
 
 const (
@@ -648,6 +649,8 @@ func (manager *Manager) startRetry() *retry.Attempt {
 
 func isFatalRetryError(err error) bool {
 	switch {
+	case database.IsErrRetryable(err):
+		return false
 	case lease.IsTimeout(err):
 		return false
 	case lease.IsInvalid(err):
@@ -658,6 +661,8 @@ func isFatalRetryError(err error) bool {
 
 func isFatalClaimRetryError(act action, err error, count int) bool {
 	switch {
+	case database.IsErrRetryable(err):
+		return false
 	case lease.IsTimeout(err):
 		return false
 	case lease.IsInvalid(err):

worker/lease/manager_claim_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/juju/errors"
 	"github.com/juju/testing"
 	jc "github.com/juju/testing/checkers"
+	"github.com/mattn/go-sqlite3"
 	gc "gopkg.in/check.v1"
 
 	corelease "github.com/juju/juju/core/lease"
@@ -295,6 +296,50 @@ func (s *ClaimSuite) TestExtendLease_Failure_OtherHolder(c *gc.C) {
 	})
 }
 
+func (s *ClaimSuite) TestExtendLease_Failure_Retryable(c *gc.C) {
+	fix := &Fixture{
+		leases: map[corelease.Key]corelease.Info{
+			key("redis"): {
+				Holder: "redis/0",
+				Expiry: offset(time.Second),
+			},
+		},
+		expectCalls: []call{{
+			method: "ExtendLease",
+			args: []interface{}{
+				corelease.Key{
+					Namespace: "namespace",
+					ModelUUID: "modelUUID",
+					Lease:     "redis",
+				},
+				corelease.Request{Holder: "redis/0", Duration: time.Minute},
+			},
+			err: sqlite3.ErrLocked,
+			callback: func(leases map[corelease.Key]corelease.Info) {
+				leases[key("redis")] = corelease.Info{
+					Holder: "redis/1",
+					Expiry: offset(time.Second),
+				}
+			},
+		}},
+	}
+	fix.RunTest(c, func(manager *lease.Manager, clock *testclock.Clock) {
+		// When the Claim starts, it will first get a LeaseInvalid, it will then
+		// wait 50ms before trying again, since it is clear that our Leases map
+		// does not have the most up-to-date information. We then wake up again
+		// and see that our leases have expired and thus let things go.
+		var wg sync.WaitGroup
+		wg.Add(1)
+		go func() {
+			err := getClaimer(c, manager).Claim("redis", "redis/0", time.Minute)
+			c.Check(err, gc.Equals, corelease.ErrClaimDenied)
+			wg.Done()
+		}()
+		c.Check(clock.WaitAdvance(50*time.Millisecond, testing.LongWait, 2), jc.ErrorIsNil)
+		wg.Wait()
+	})
+}
+
 func (s *ClaimSuite) TestExtendLease_Failure_Error(c *gc.C) {
 	fix := &Fixture{
 		leases: map[corelease.Key]corelease.Info{