Cycle through the list of available servers while retryingi.

-Always start with a random node.
- number of retries is equal to the number of available nodes
but cannot be less than 3 or more than 10

Change-Id: I5dffffe3f282ec0a1cda5440756f8d3d8b102bec
Reviewed-on: http://review.couchbase.org/48508
Tested-by: Manik Taneja <manik@couchbase.com>
Reviewed-by: Manik Taneja <manik@couchbase.com>

ddocs.go

@@ -72,25 +72,59 @@ func (b *Bucket) ddocURL(docname string) (string, error) {
 	return u.String(), nil
 }
 
+func (b *Bucket) ddocURLNext(nodeId int, docname string) (string, int, error) {
+	u, selected, err := b.randomNextURL(nodeId)
+	if err != nil {
+		return "", -1, err
+	}
+	u.Path = fmt.Sprintf("/%s/_design/%s", b.Name, docname)
+	return u.String(), selected, nil
+}
+
+const ABS_MAX_RETRIES = 10
+const ABS_MIN_RETRIES = 3
+
+func (b *Bucket) getMaxRetries() (int, error) {
+
+	maxRetries := len(b.Nodes())
+
+	if maxRetries == 0 {
+		return 0, fmt.Errorf("No available Couch rest URLs")
+	}
+
+	if maxRetries > ABS_MAX_RETRIES {
+		maxRetries = ABS_MAX_RETRIES
+	} else if maxRetries < ABS_MIN_RETRIES {
+		maxRetries = ABS_MIN_RETRIES
+	}
+
+	return maxRetries, nil
+}
+
 // PutDDoc installs a design document.
 func (b *Bucket) PutDDoc(docname string, value interface{}) error {
 
 	var Err error
 
-	nodes := b.Nodes()
-	if len(nodes) == 0 {
-		return fmt.Errorf("no couch rest URLs")
+	maxRetries, err := b.getMaxRetries()
+	if err != nil {
+		return err
 	}
-	maxRetries := len(nodes)
+
+	lastNode := START_NODE_ID
 
 	for retryCount := 0; retryCount < maxRetries; retryCount++ {
 
 		Err = nil
-		ddocU, err := b.ddocURL(docname)
+
+		ddocU, selectedNode, err := b.ddocURLNext(lastNode, docname)
 		if err != nil {
 			return err
 		}
 
+		lastNode = selectedNode
+
+		log.Printf(" Trying with selected node %d", selectedNode)
 		j, err := json.Marshal(value)
 		if err != nil {
 			return err
@@ -133,20 +167,23 @@ func (b *Bucket) GetDDoc(docname string, into interface{}) error {
 	var Err error
 	var res *http.Response
 
-	nodes := b.Nodes()
-	if len(nodes) == 0 {
-		return fmt.Errorf("no couch rest URLs")
+	maxRetries, err := b.getMaxRetries()
+	if err != nil {
+		return err
 	}
-	maxRetries := len(nodes)
 
+	lastNode := START_NODE_ID
 	for retryCount := 0; retryCount < maxRetries; retryCount++ {
 
 		Err = nil
-		ddocU, err := b.ddocURL(docname)
+		ddocU, selectedNode, err := b.ddocURLNext(lastNode, docname)
 		if err != nil {
 			return err
 		}
 
+		lastNode = selectedNode
+		log.Printf(" Trying with selected node %d", selectedNode)
+
 		req, err := http.NewRequest("GET", ddocU, nil)
 		if err != nil {
 			return err
@@ -186,20 +223,25 @@ func (b *Bucket) GetDDoc(docname string, into interface{}) error {
 func (b *Bucket) DeleteDDoc(docname string) error {
 
 	var Err error
-	nodes := b.Nodes()
-	if len(nodes) == 0 {
-		return fmt.Errorf("no couch rest URLs")
+
+	maxRetries, err := b.getMaxRetries()
+	if err != nil {
+		return err
 	}
-	maxRetries := len(nodes)
+
+	lastNode := START_NODE_ID
 
 	for retryCount := 0; retryCount < maxRetries; retryCount++ {
 
 		Err = nil
-		ddocU, err := b.ddocURL(docname)
+		ddocU, selectedNode, err := b.ddocURLNext(lastNode, docname)
 		if err != nil {
 			return err
 		}
 
+		lastNode = selectedNode
+		log.Printf(" Trying with selected node %d", selectedNode)
+
 		req, err := http.NewRequest("DELETE", ddocU, nil)
 		if err != nil {
 			return err

views.go

@@ -62,6 +62,39 @@ func (b *Bucket) randomBaseURL() (*url.URL, error) {
 	return u, err
 }
 
+const START_NODE_ID = -1
+
+func (b *Bucket) randomNextURL(lastNode int) (*url.URL, int, error) {
+	nodes := []Node{}
+	for _, n := range b.Nodes() {
+		if n.Status == "healthy" && n.CouchAPIBase != "" {
+			nodes = append(nodes, n)
+		}
+	}
+	if len(nodes) == 0 {
+		return nil, -1, errors.New("no available couch rest URLs")
+	}
+
+	var nodeNo int
+	if lastNode == START_NODE_ID || lastNode >= len(nodes) {
+		// randomly select a node if the value of lastNode is invalid
+		nodeNo = rand.Intn(len(nodes))
+	} else {
+		// wrap around the node list
+		nodeNo = (lastNode + 1) % len(nodes)
+	}
+
+	node := nodes[nodeNo]
+	u, err := ParseURL(node.CouchAPIBase)
+	if err != nil {
+		return nil, -1, fmt.Errorf("config error: Bucket %q node #%d CouchAPIBase=%q: %v",
+			b.Name, nodeNo, node.CouchAPIBase, err)
+	} else if b.pool != nil {
+		u.User = b.pool.client.BaseURL.User
+	}
+	return u, nodeNo, err
+}
+
 // DocID is the document ID type for the startkey_docid parameter in
 // views.
 type DocID string