Permalink
Browse files

Make exponential backoff work exactly as per google specification - f…

…ixes #583
  • Loading branch information...
1 parent b7875fc commit 4803ce010e335442e4dc56a8f689fd791f3d357f @ncw committed Oct 17, 2016
Showing with 81 additions and 5 deletions.
  1. +1 −4 drive/drive.go
  2. +40 −1 pacer/pacer.go
  3. +40 −0 pacer/pacer_test.go
View
@@ -35,9 +35,6 @@ const (
timeFormatIn = time.RFC3339
timeFormatOut = "2006-01-02T15:04:05.000000000Z07:00"
minSleep = 10 * time.Millisecond
- maxSleep = 2000 * time.Millisecond
- decayConstant = 0 // bigger for slower decay, exponential
- attackConstant = 0 // bigger for slower attack, exponential
defaultExtensions = "docx,xlsx,pptx,svg"
)
@@ -295,7 +292,7 @@ func NewFs(name, path string) (fs.Fs, error) {
f := &Fs{
name: name,
root: root,
- pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant).SetAttackConstant(attackConstant),
+ pacer: pacer.New().SetMinSleep(minSleep).SetPacer(pacer.GoogleDrivePacer),
}
// Create a new authorized Drive client.
View
@@ -48,6 +48,16 @@ const (
//
// See https://developer.amazon.com/public/apis/experience/cloud-drive/content/restful-api-best-practices
AmazonCloudDrivePacer
+
+ // GoogleDrivePacer is a specialised pacer for Google Drive
+ //
+ // It implements a truncated exponential backoff strategy with
+ // randomization. Normally operations are paced at the
+ // interval set with SetMinSleep. On errors the sleep timer
+ // is set to (2 ^ n) + random_number_milliseconds seconds
+ //
+ // See https://developers.google.com/drive/v2/web/handle-errors#exponential-backoff
+ GoogleDrivePacer
)
// Paced is a function which is called by the Call and CallNoRetry
@@ -172,6 +182,8 @@ func (p *Pacer) SetPacer(t Type) *Pacer {
switch t {
case AmazonCloudDrivePacer:
p.calculatePace = p.acdPacer
+ case GoogleDrivePacer:
+ p.calculatePace = p.drivePacer
default:
p.calculatePace = p.defaultPacer
}
@@ -265,7 +277,34 @@ func (p *Pacer) acdPacer(retry bool) {
if p.sleepTime < p.minSleep {
p.sleepTime = p.minSleep
}
- fs.Debug("pacer", "Rate limited, sleeping for %v (%d consecutive low level retries)", p.sleepTime, consecutiveRetries)
+ fs.Debug("pacer", "Rate limited, sleeping for %v (%d consecutive low level retries)", p.sleepTime, p.consecutiveRetries)
+ }
+}
+
+// drivePacer implements a truncated exponential backoff strategy with
+// randomization for Google Drive
+//
+// See the description for GoogleDrivePacer
+//
+// This should calculate a new sleepTime. It takes a boolean as to
+// whether the operation should be retried or not.
+//
+// Call with p.mu held
+func (p *Pacer) drivePacer(retry bool) {
+ consecutiveRetries := p.consecutiveRetries
+ if consecutiveRetries == 0 {
+ if p.sleepTime != p.minSleep {
+ p.sleepTime = p.minSleep
+ fs.Debug("pacer", "Resetting sleep to minimum %v on success", p.sleepTime)
+ }
+ } else {
+ if consecutiveRetries > 5 {
+ consecutiveRetries = 5
+ }
+ // consecutiveRetries starts at 1 so go from 1,2,3,4,5,5 => 1,2,4,8,16,16
+ // maxSleep is 2**(consecutiveRetries-1) seconds + random milliseconds
+ p.sleepTime = time.Second<<uint(consecutiveRetries-1) + time.Duration(rand.Int63n(int64(time.Second)))
+ fs.Debug("pacer", "Rate limited, sleeping for %v (%d consecutive low level retries)", p.sleepTime, p.consecutiveRetries)
}
}
View
@@ -167,6 +167,10 @@ func TestSetPacer(t *testing.T) {
if fmt.Sprintf("%p", p.calculatePace) != fmt.Sprintf("%p", p.acdPacer) {
t.Errorf("calculatePace is not acdPacer")
}
+ p.SetPacer(GoogleDrivePacer)
+ if fmt.Sprintf("%p", p.calculatePace) != fmt.Sprintf("%p", p.drivePacer) {
+ t.Errorf("calculatePace is not drivePacer")
+ }
p.SetPacer(DefaultPacer)
if fmt.Sprintf("%p", p.calculatePace) != fmt.Sprintf("%p", p.defaultPacer) {
t.Errorf("calculatePace is not defaultPacer")
@@ -299,6 +303,42 @@ func TestAmazonCloudDrivePacer(t *testing.T) {
}
}
+func TestGoogleDrivePacer(t *testing.T) {
+ p := New().SetMinSleep(time.Millisecond).SetPacer(GoogleDrivePacer).SetMaxSleep(time.Second).SetDecayConstant(2)
+ // Do lots of times because of the random number!
+ for _, test := range []struct {
+ in time.Duration
+ consecutiveRetries int
+ retry bool
+ want time.Duration
+ }{
+ {time.Millisecond, 0, true, time.Millisecond},
+ {10 * time.Millisecond, 0, true, time.Millisecond},
+ {1 * time.Second, 1, true, 1*time.Second + 500*time.Millisecond},
+ {1 * time.Second, 2, true, 2*time.Second + 500*time.Millisecond},
+ {1 * time.Second, 3, true, 4*time.Second + 500*time.Millisecond},
+ {1 * time.Second, 4, true, 8*time.Second + 500*time.Millisecond},
+ {1 * time.Second, 5, true, 16*time.Second + 500*time.Millisecond},
+ {1 * time.Second, 6, true, 16*time.Second + 500*time.Millisecond},
+ {1 * time.Second, 7, true, 16*time.Second + 500*time.Millisecond},
+ } {
+ const n = 1000
+ var sum time.Duration
+ // measure average time over n cycles
+ for i := 0; i < n; i++ {
+ p.sleepTime = test.in
+ p.consecutiveRetries = test.consecutiveRetries
+ p.drivePacer(test.retry)
+ sum += p.sleepTime
+ }
+ got := sum / n
+ //t.Logf("%+v: got = %v", test, got)
+ if got < (test.want*9)/10 || got > (test.want*11)/10 {
+ t.Fatalf("%+v: bad sleep want %v+/-10%% got %v", test, test.want, got)
+ }
+ }
+}
+
func TestEndCall(t *testing.T) {
p := New().SetMaxConnections(5)
emptyTokens(p)

0 comments on commit 4803ce0

Please sign in to comment.