Added INFO level messages to better understand connection creation. E…

…xpet to remove most of this after debugging.

src/java/voldemort/store/socket/clientrequest/ClientRequestExecutorFactory.java
- print out time to establish connection (if it takes longer than 1 ms)

src/java/voldemort/utils/pool/KeyedResourcePool.java
- print out info about object creation (connection establishment). In
  particular, how many outstanding creations (connection
  establishments) are in flight and how many idle resources are in the
  pool after the newly created resource is checked in.

src/java/voldemort/store/socket/clientrequest/ClientRequestExecutorFactory.java

@@ -166,7 +166,16 @@ public ClientRequestExecutor create(SocketDestination dest) throws Exception {
                         logger.warn(e, e);
                 }
             }
-
+            duration = System.currentTimeMillis() - startTime;
+
+            if(duration > 0) {
+                if(logger.isInfoEnabled()) {
+                    logger.info("Created socket " + numCreated + " for " + dest.getHost() + ":"
+                                + dest.getPort() + " using protocol "
+                                + dest.getRequestFormatType().getCode() + " after " + duration
+                                + " ms.");
+                }
+            }
             if(logger.isDebugEnabled())
                 logger.debug("Created socket " + numCreated + " for " + dest.getHost() + ":"
                              + dest.getPort() + " using protocol "

src/java/voldemort/utils/pool/KeyedResourcePool.java

@@ -405,6 +405,7 @@ protected void checkNotClosed() {
 
         final private AtomicInteger size = new AtomicInteger(0);
         final private AtomicInteger blockingGets = new AtomicInteger(0);
+        final private AtomicInteger createsInFlight = new AtomicInteger(0);
         final private int maxPoolSize;
         final private BlockingQueue<V> queue;
 
@@ -435,13 +436,29 @@ public Pool(ResourcePoolConfig resourcePoolConfig) {
 
             if(this.size.incrementAndGet() <= this.maxPoolSize) {
                 try {
-                    V resource = objectFactory.create(key);
+                    V resource = null;
+                    int currentCreatesInFlight = 0;
+                    try {
+                        currentCreatesInFlight = createsInFlight.getAndIncrement();
+                        resource = objectFactory.create(key);
+                    } finally {
+                        createsInFlight.decrementAndGet();
+                    }
                     if(resource != null) {
                         if(!nonBlockingPut(resource)) {
                             this.size.decrementAndGet();
                             objectFactory.destroy(key, resource);
+                            logger.info("attemptGrow established new connection for key "
+                                        + key.toString() + " with " + currentCreatesInFlight
+                                        + " other connection establishments in flight."
+                                        + " And then promptly destroyed the new connection.");
                             return false;
                         }
+                        logger.info("attemptGrow established new connection for key "
+                                    + key.toString() + " with " + currentCreatesInFlight
+                                    + " other connection establishments in flight."
+                                    + " After checking in to KeyedResourcePool, there are "
+                                    + queue.size() + " destinations checked in.");
                     }
                 } catch(Exception e) {
                     // If nonBlockingPut throws an exception, then we could leak