NCBC-881: Ensure NRE are not thrown during retry

Motivation
----------
During rebalance/swap/failover scenarios an endpoint may not be available
for the client to execute an operation. This commit uses a backoff
strategy to wait and try again to acquire an endpoint during the retry
process.

Modifications
-------------
The CallbackFactory classes functors were updated so that they
use the back off strategy when when acquiring an endpoint (IServer) to
send the request.

Result
------
If the client cannot acquire an endpoint to send the query, a timeout
exception will be thrown.

Change-Id: Iad0a7fc5ce6dc826522ace574303b4027fc7bbed
Reviewed-on: http://review.couchbase.org/50611
Reviewed-by: Subhashni Balakrishnan <b.subhashni@gmail.com>
Tested-by: Jeffry Morris <jeffrymorris@gmail.com>

Src/Couchbase/Core/Buckets/CallbackFactory.cs

@@ -63,7 +63,14 @@ await op.ReadAsync(response, 0, response.Length)
 
                                     var keyMapper = c.GetKeyMapper();
                                     var mappedNode= keyMapper.MapKey(cloned.Key);
-                                    var server = mappedNode.LocatePrimary();
+
+                                    IServer server;
+                                    var attempts = 0;
+                                    while ((server = mappedNode.LocatePrimary()) == null)
+                                    {
+                                        if (attempts++ > 10) { throw new TimeoutException("Could not acquire a server."); }
+                                        Thread.Sleep((int)Math.Pow(2, attempts));
+                                    }
                                     server.SendAsync(o).ContinueOnAnyContext();
 
                                     return retryTcs.Task;
@@ -143,7 +150,14 @@ await op.ReadAsync(response, 0, response.Length)
 
                                     var keyMapper = c.GetKeyMapper();
                                     var mappedNode = keyMapper.MapKey(cloned.Key);
-                                    var server = mappedNode.LocatePrimary();
+
+                                    IServer server;
+                                    var attempts = 0;
+                                    while ((server = mappedNode.LocatePrimary()) == null)
+                                    {
+                                        if (attempts++ > 10) { throw new TimeoutException("Could not acquire a server."); }
+                                        Thread.Sleep((int)Math.Pow(2, attempts));
+                                    }
                                     server.SendAsync(o).ContinueOnAnyContext();
 
                                     return retryTcs.Task;
@@ -229,7 +243,13 @@ await op.ReadAsync(response, 0, response.Length)
                                     var vBucket = (IVBucket)keyMapper.MapKey(o.Key);
                                     o.VBucket = vBucket;
 
-                                    var server = vBucket.LocatePrimary();
+                                    IServer server;
+                                    var attempts = 0;
+                                    while ((server = vBucket.LocatePrimary()) == null)
+                                    {
+                                        if (attempts++ > 10) { throw new TimeoutException("Could not acquire a server."); }
+                                        Thread.Sleep((int)Math.Pow(2, attempts));
+                                    }
                                     server.SendAsync(o).ContinueOnAnyContext();
 
                                     return retryTcs.Task;
@@ -311,7 +331,13 @@ await op.ReadAsync(response, 0, response.Length)
                                     var vBucket = (IVBucket)keyMapper.MapKey(o.Key);
                                     o.VBucket = vBucket;
 
-                                    var server = vBucket.LocatePrimary();
+                                    IServer server;
+                                    var attempts = 0;
+                                    while ((server = vBucket.LocatePrimary()) == null)
+                                    {
+                                        if (attempts++ > 10) { throw new TimeoutException("Could not acquire a server."); }
+                                        Thread.Sleep((int)Math.Pow(2, attempts));
+                                    }
                                     server.SendAsync(o).ContinueOnAnyContext();
 
                                     return retryTcs.Task;