Improved replica set failover tests. A few

improved exception messages.

lib/mongo/connection.rb

@@ -795,7 +795,7 @@ def receive_message_on_socket(length, socket)
       begin
         message = new_binary_string
         socket.read(length, message)
-        raise ConnectionFailure, "connection closed" unless message.length > 0
+        raise ConnectionFailure, "connection closed" unless message && message.length > 0
         if message.length < length
           chunk = new_binary_string
           while message.length < length

lib/mongo/repl_set_connection.rb

@@ -119,8 +119,10 @@ def connect
         BSON::BSON_CODER.update_max_bson_size(self)
       else
         if @secondary_pools.empty?
+          close # close any existing pools and sockets
           raise ConnectionFailure, "Failed to connect any given host:port"
         else
+          close # close any existing pools and sockets
           raise ConnectionFailure, "Failed to connect to primary node."
         end
       end
@@ -136,7 +138,7 @@ def connecting?
     #
     # @return [Boolean]
     def read_primary?
-      !@read_pool || @read_pool.length.zero?
+      !@read_pool
     end
     alias :primary? :read_primary?
 
@@ -194,9 +196,13 @@ def check_is_master(node)
 
         check_set_name(config, socket)
       rescue OperationFailure, SocketError, SystemCallError, IOError => ex
-        close unless connected?
+        # It's necessary to rescue here. The #connect method will keep trying
+        # until it has no more nodes to try and raise a ConnectionFailure if
+        # it can't connect to a primary.
       ensure
+        socket.close if socket
         @nodes_tried << node
+
         if config
           nodes = []
           nodes += config['hosts'] if config['hosts']
@@ -208,8 +214,6 @@ def check_is_master(node)
             @logger.warn("MONGODB #{config['msg']}")
           end
         end
-
-        socket.close if socket
       end
 
       config

lib/mongo/util/pool.rb

@@ -76,7 +76,7 @@ def checkout_new_socket
       socket = TCPSocket.new(@host, @port)
       socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
       rescue => ex
-        raise ConnectionFailure, "Failed to connect socket: #{ex}"
+        raise ConnectionFailure, "Failed to connect to host #{@host} and port #{@port}: #{ex}"
       end
 
       # If any saved authentications exist, we want to apply those

test/replica_sets/query_secondaries.rb

@@ -17,8 +17,10 @@ def teardown
   end
 
   def test_read_primary
-    assert !@conn.read_primary?
-    assert !@conn.primary?
+    rescue_connection_failure do
+      assert !@conn.read_primary?
+      assert !@conn.primary?
+    end
   end
 
   def test_con
@@ -59,6 +61,12 @@ def test_kill_primary
     # Should still be able to read immediately after killing master node
     RS.kill_primary
     assert_equal 2, @coll.find.to_a.length
+    rescue_connection_failure do
+      @coll.save({:a => 50}, :safe => {:w => 2, :wtimeout => 10000})
+    end
+    RS.restart_killed_nodes
+    @coll.save({:a => 50}, :safe => {:w => 2, :wtimeout => 10000})
+    assert_equal 4, @coll.find.to_a.length
   end
 
   def test_kill_secondary
@@ -71,6 +79,7 @@ def test_kill_secondary
     RS.kill(read_node)
 
     # Should fail immediately on next read
+    old_read_pool_port = @conn.read_pool.port
     assert_raise ConnectionFailure do
       @coll.find.to_a.length
     end
@@ -80,6 +89,8 @@ def test_kill_secondary
       length = @coll.find.to_a.length
       assert_equal 2, length
     end
+    new_read_pool_port = @conn.read_pool.port
+    assert old_read_pool != new_read_pool
   end
 
 end

test/replica_sets/rs_test_helper.rb

@@ -16,7 +16,7 @@ def rescue_connection_failure(max_retries=60)
     begin
       yield
     rescue Mongo::ConnectionFailure => ex
-      puts "Rescue attempt #{retries}"
+      puts "Rescue attempt #{retries}: from #{ex}"
       retries += 1
       raise ex if retries > max_retries
       sleep(1)