JCBC-457: Force CCCP config fetching on node reconnect.

Motivation
----------
If a node needs to be reconnected, there is a strong indication that
the socket has been closed and this could be because of a topology
change.

Modification
------------
If a reconnect is scheduled, make sure it forces a config update.
Also, the method is added for memcache buckets to keep the behavior consistent.

Result
------
Quicker detection of topology changes, eventually getting quicker to
a valid config state.

Change-Id: I5244dfc6d6f19288977ef98745d47efe25773093
Reviewed-on: http://review.couchbase.org/36779
Reviewed-by: Matt Ingenthron <matt@couchbase.com>
Tested-by: Michael Nitschinger <michael.nitschinger@couchbase.com>

src/main/java/com/couchbase/client/CouchbaseConnection.java

@@ -24,10 +24,18 @@
 
 import com.couchbase.client.internal.AdaptiveThrottler;
 import com.couchbase.client.internal.ThrottleManager;
-import com.couchbase.client.vbucket.ConfigurationProvider;
 import com.couchbase.client.vbucket.Reconfigurable;
 import com.couchbase.client.vbucket.VBucketNodeLocator;
 import com.couchbase.client.vbucket.config.Bucket;
+import net.spy.memcached.ConnectionObserver;
+import net.spy.memcached.FailureMode;
+import net.spy.memcached.MemcachedConnection;
+import net.spy.memcached.MemcachedNode;
+import net.spy.memcached.OperationFactory;
+import net.spy.memcached.ops.KeyedOperation;
+import net.spy.memcached.ops.Operation;
+import net.spy.memcached.ops.ReplicaGetOperation;
+import net.spy.memcached.ops.VBucketAware;
 
 import java.io.IOException;
 import java.net.InetSocketAddress;
@@ -43,16 +51,6 @@
 import java.util.List;
 import java.util.Map;
 
-import net.spy.memcached.ConnectionObserver;
-import net.spy.memcached.FailureMode;
-import net.spy.memcached.MemcachedConnection;
-import net.spy.memcached.MemcachedNode;
-import net.spy.memcached.OperationFactory;
-import net.spy.memcached.ops.KeyedOperation;
-import net.spy.memcached.ops.Operation;
-import net.spy.memcached.ops.ReplicaGetOperation;
-import net.spy.memcached.ops.VBucketAware;
-
 /**
  * Maintains connections to each node in a cluster of Couchbase Nodes.
  *
@@ -326,10 +324,15 @@ protected void handleRetryInformation(byte[] retryMessage) {
   /**
    * Only queue for reconnect if the given node is still part of the cluster.
    *
+   * Since a node is queued to reconnect, it indicates a close socket and
+   * therefore an outdated configuration. With some providers, it is important
+   * to force a config reload which is also issued immediately.
+   *
    * @param node the node to check.
    */
   @Override
   protected void queueReconnect(final MemcachedNode node) {
+    cf.getConfigurationProvider().reloadConfig();
     if (isShutDown() || !locator.getAll().contains(node)) {
       getLogger().debug("Preventing reconnect for node " + node + " because it"
         + "is either not part of the cluster anymore or the connection is "

src/main/java/com/couchbase/client/CouchbaseMemcachedConnection.java

@@ -25,6 +25,12 @@
 import com.couchbase.client.vbucket.Reconfigurable;
 import com.couchbase.client.vbucket.VBucketNodeLocator;
 import com.couchbase.client.vbucket.config.Bucket;
+import net.spy.memcached.ConnectionObserver;
+import net.spy.memcached.FailureMode;
+import net.spy.memcached.MemcachedConnection;
+import net.spy.memcached.MemcachedNode;
+import net.spy.memcached.OperationFactory;
+import net.spy.memcached.ops.Operation;
 
 import java.io.IOException;
 import java.net.InetSocketAddress;
@@ -38,13 +44,6 @@
 import java.util.Iterator;
 import java.util.List;
 
-import net.spy.memcached.ConnectionObserver;
-import net.spy.memcached.FailureMode;
-import net.spy.memcached.MemcachedConnection;
-import net.spy.memcached.MemcachedNode;
-import net.spy.memcached.OperationFactory;
-import net.spy.memcached.ops.Operation;
-
 /**
  * Couchbase implementation of CouchbaseConnection.
  *
@@ -227,6 +226,28 @@ public void run() {
     getLogger().info("Shut down Couchbase client");
   }
 
+  /**
+   * Only queue for reconnect if the given node is still part of the cluster.
+   *
+   * Since a node is queued to reconnect, it indicates a close socket and
+   * therefore an outdated configuration. With some providers, it is important
+   * to force a config reload which is also issued immediately.
+   *
+   * @param node the node to check.
+   */
+  @Override
+  protected void queueReconnect(final MemcachedNode node) {
+    cf.getConfigurationProvider().reloadConfig();
+    if (isShutDown() || !locator.getAll().contains(node)) {
+      getLogger().debug("Preventing reconnect for node " + node + " because it"
+        + "is either not part of the cluster anymore or the connection is "
+        + "shutting down.");
+      return;
+    }
+
+    super.queueReconnect(node);
+  }
+
   private void logRunException(Exception e) {
     if (shutDown) {
       // There are a couple types of errors that occur during the