Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Exponential backoff when 503 is returned

Closes #7
  • Loading branch information...
commit eb72d395afbbb20f82d04fd5cce785927b56bf68 1 parent b9e064d
Evan Shaw authored March 05, 2012

Showing 1 changed file with 47 additions and 3 deletions. Show diff stats Hide diff stats

  1. 50  ironmq.go
50  ironmq.go
@@ -5,11 +5,36 @@ import (
5 5
 	"encoding/json"
6 6
 	"errors"
7 7
 	"fmt"
  8
+	"io/ioutil"
  9
+	"math/rand"
8 10
 	"net/http"
9 11
 	"net/http/httputil"
10 12
 	"path"
  13
+	"sync"
  14
+	"time"
11 15
 )
12 16
 
  17
+// Copied straight from Go's package rand since it's not exported.
  18
+type lockedSource struct {
  19
+	lk  sync.Mutex
  20
+	src rand.Source
  21
+}
  22
+
  23
+func (r *lockedSource) Int63() (n int64) {
  24
+	r.lk.Lock()
  25
+	n = r.src.Int63()
  26
+	r.lk.Unlock()
  27
+	return
  28
+}
  29
+
  30
+func (r *lockedSource) Seed(seed int64) {
  31
+	r.lk.Lock()
  32
+	r.src.Seed(seed)
  33
+	r.lk.Unlock()
  34
+}
  35
+
  36
+var localRand = rand.New(&lockedSource{src: rand.NewSource(time.Now().Unix())})
  37
+
13 38
 // A Client contains an Iron.io project ID and a token for authentication.
14 39
 type Client struct {
15 40
 	Debug     bool
@@ -45,9 +70,28 @@ func (c *Client) req(method, endpoint string, body []byte, data interface{}) err
45 70
 		req.Header.Set("Content-Type", "application/json")
46 71
 		req.ContentLength = int64(len(body))
47 72
 	}
48  
-	resp, err := http.DefaultClient.Do(req)
49  
-	if err != nil {
50  
-		return err
  73
+
  74
+	const maxRetries = 5
  75
+	tries := uint(0)
  76
+	var resp *http.Response
  77
+	for tries < maxRetries {
  78
+		resp, err = http.DefaultClient.Do(req)
  79
+		if err != nil {
  80
+			return err
  81
+		}
  82
+		// ELB sometimes returns this when load is increasing; we retry
  83
+		// with exponential backoff
  84
+		if resp.StatusCode == http.StatusServiceUnavailable {
  85
+			tries++
  86
+			// random delay between 0 and (4^tries*100) milliseconds
  87
+			pow := int64(1) << (2 * tries) * 100
  88
+			delayMs := time.Duration(localRand.Int63n(pow))
  89
+			fmt.Println("delay:", delayMs*time.Millisecond, tries)
  90
+			time.Sleep(delayMs * time.Millisecond)
  91
+			req.Body = ioutil.NopCloser(bytes.NewBuffer(body))
  92
+			continue
  93
+		}
  94
+		break
51 95
 	}
52 96
 
53 97
 	if c.Debug {

0 notes on commit eb72d39

Please sign in to comment.
Something went wrong with that request. Please try again.