NCBC-1291: Reactivate search nodes after failure

Motivation
----------
Once a search node is marked as failed, it is never reactivated and
cannot be used by the client.

Modifications
-------------
When all search nodes are marked as failed, clear the failure on all
nodes before selecting a node.

Move failure threshold magic number to a constant.

Results
-------
If all search nodes get flagged as failed, then it will retry them all to
see if any are functional again.

This is not a complete fix, as a node will not be reactivated until all
search nodes have failed or there is a config change (i.e. rebalance).

Change-Id: I73974988c923a8999d691ec2d4e49cfbfdeafb85
Reviewed-on: http://review.couchbase.org/73277
Reviewed-by: Jeffry Morris <jeffrymorris@gmail.com>
Tested-by: Jeffry Morris <jeffrymorris@gmail.com>

Src/Couchbase.UnitTests/Configuration/ConfigContextBaseTests.cs

@@ -13,6 +13,8 @@ namespace Couchbase.UnitTests.Configuration
     [TestFixture]
     public class ConfigContextBaseTests
     {
+        #region GetQueryUri
+
         [Test]
         public void GetQueryUri_AllFailed_ResetsAll()
         {
@@ -77,6 +79,78 @@ public void GetQueryUri_PartiallyFailed_DoesntResetFailures()
             Assert.AreEqual(0, uri2.FailedCount);
         }
 
+        #endregion
+
+        #region GetSearchUri
+
+        [Test]
+        public void GetSearchUri_AllFailed_ResetsAll()
+        {
+            // Arrange
+
+            FailureCountingUri item;
+            while (ConfigContextBase.SearchUris.TryTake(out item))
+            {
+                // Clear the list
+            }
+
+            var uri1 = new FailureCountingUri("http://uri1/");
+            var uri2 = new FailureCountingUri("http://uri2/");
+            for (var i = 0; i < ConfigContextBase.SearchNodeFailureThreshold; i++)
+            {
+                uri1.IncrementFailed();
+                uri2.IncrementFailed();
+            }
+
+            ConfigContextBase.SearchUris.Add(uri1);
+            ConfigContextBase.SearchUris.Add(uri2);
+
+            // Act
+
+            var uri = ConfigContextBase.GetSearchUri();
+
+            // Assert
+
+            Assert.NotNull(uri);
+            Assert.AreEqual(0, uri1.FailedCount);
+            Assert.AreEqual(0, uri2.FailedCount);
+        }
+
+        [Test]
+        public void GetSearchUri_PartiallyFailed_DoesntResetFailures()
+        {
+            // Arrange
+
+            FailureCountingUri item;
+            while (ConfigContextBase.SearchUris.TryTake(out item))
+            {
+                // Clear the list
+            }
+
+            var uri1 = new FailureCountingUri("http://uri1/");
+            for (var i = 0; i < ConfigContextBase.SearchNodeFailureThreshold; i++)
+            {
+                uri1.IncrementFailed();
+            }
+
+            var uri2 = new FailureCountingUri("http://uri2/");
+
+            ConfigContextBase.SearchUris.Add(uri1);
+            ConfigContextBase.SearchUris.Add(uri2);
+
+            // Act
+
+            var uri = ConfigContextBase.GetSearchUri();
+
+            // Assert
+
+            Assert.AreEqual(uri2, uri);
+            Assert.AreEqual(2, uri1.FailedCount);
+            Assert.AreEqual(0, uri2.FailedCount);
+        }
+
+        #endregion
+
         [OneTimeTearDown]
         public void OneTimeTearDown()
         {
@@ -85,6 +159,11 @@ public void OneTimeTearDown()
             {
                 // Clear the list
             }
+
+            while (ConfigContextBase.SearchUris.TryTake(out item))
+            {
+                // Clear the list
+            }
         }
     }
 }

Src/Couchbase/Configuration/ConfigContextBase.cs

@@ -23,6 +23,8 @@ namespace Couchbase.Configuration
     /// </summary>
     internal abstract class ConfigContextBase : IConfigInfo
     {
+        public const int SearchNodeFailureThreshold = 2;
+
         protected static readonly ILog Log = LogManager.GetLogger<ConfigContextBase>();
         private static int _roundRobinPosition = 0;
         protected IKeyMapper KeyMapper;
@@ -105,7 +107,20 @@ private static FailureCountingUri RoundRobin(IReadOnlyList<FailureCountingUri> u
 
         public static FailureCountingUri GetSearchUri()
         {
-            return SearchUris.Where(x => x.IsHealthy(2)).GetRandom();
+            var searchUris = SearchUris.Where(x => x.IsHealthy(SearchNodeFailureThreshold)).ToList();
+            if (searchUris.Count == 0)
+            {
+                // All search URIs are unhealthy, so reset them all back to healthy and return the entire list
+                // It's better to at least try the nodes than assume they're all failing indefinitely
+
+                foreach (var searchUri in SearchUris)
+                {
+                    searchUri.ClearFailed();
+                    searchUris.Add(searchUri);
+                }
+            }
+
+            return searchUris.GetRandom();
         }
 
         protected ITypeTranscoder Transcoder { get; private set; }