diff --git a/lib/mongodb/connection/repl_set.js b/lib/mongodb/connection/repl_set.js
index dafc1267c0..2a4bf1afae 100644
--- a/lib/mongodb/connection/repl_set.js
+++ b/lib/mongodb/connection/repl_set.js
@@ -475,13 +475,11 @@ var _connectHandler = function(self, candidateServers, instanceServer) {
       }
 
       // Make sure we have the right reference
-      if (self._state.addresses[instanceServer.host + ":" + instanceServer.port] != instanceServer) {
-        // Close the connection before deleting
-        if(self._state.addresses[instanceServer.host + ":" + instanceServer.port])
-          self._state.addresses[instanceServer.host + ":" + instanceServer.port].close();
-      }
+      var oldServer = self._state.addresses[userProvidedServerString]
+      if (oldServer && oldServer !== instanceServer) oldServer.close();
+      delete self._state.addresses[userProvidedServerString];
 
-      delete self._state.addresses[instanceServer.host + ":" + instanceServer.port];
+      if (self._state.addresses[me] && self._state.addresses[me] !== instanceServer) self._state.addresses[me].close();
       self._state.addresses[me] = instanceServer;
 
       // Let's add the server to our list of server types