diff --git a/hazelcast/src/main/java/com/hazelcast/nio/tcp/TcpIpConnectionManager.java b/hazelcast/src/main/java/com/hazelcast/nio/tcp/TcpIpConnectionManager.java
index 834e7311f96c..222b21498d84 100644
--- a/hazelcast/src/main/java/com/hazelcast/nio/tcp/TcpIpConnectionManager.java
+++ b/hazelcast/src/main/java/com/hazelcast/nio/tcp/TcpIpConnectionManager.java
@@ -403,33 +403,38 @@ public void destroyConnection(final Connection connection) {
         if (logger.isFinestEnabled()) {
             log(Level.FINEST, "Destroying " + connection);
         }
-        activeConnections.remove(connection);
+        if (activeConnections.remove(connection)) {
+            ioBalancer.connectionRemoved(connection);
+        }
         final Address endPoint = connection.getEndPoint();
         if (endPoint != null) {
             connectionsInProgress.remove(endPoint);
             connectionsMap.remove(endPoint, connection);
-            ioBalancer.connectionRemoved(connection);
-            if (live) {
-                ioService.getEventService().executeEventCallback(new StripedRunnable() {
-                    @Override
-                    public void run() {
-                        for (ConnectionListener listener : connectionListeners) {
-                            listener.connectionRemoved(connection);
-                        }
-                    }
-
-                    @Override
-                    public int getKey() {
-                        return endPoint.hashCode();
-                    }
-                });
-            }
+            fireConnectionRemovedEvent(connection, endPoint);
         }
         if (connection.isAlive()) {
             connection.close();
         }
     }
 
+    private void fireConnectionRemovedEvent(final Connection connection, final Address endPoint) {
+        if (live) {
+            ioService.getEventService().executeEventCallback(new StripedRunnable() {
+                @Override
+                public void run() {
+                    for (ConnectionListener listener : connectionListeners) {
+                        listener.connectionRemoved(connection);
+                    }
+                }
+
+                @Override
+                public int getKey() {
+                    return endPoint.hashCode();
+                }
+            });
+        }
+    }
+
     protected void initSocket(Socket socket) throws Exception {
         if (socketLingerSeconds > 0) {
             socket.setSoLinger(true, socketLingerSeconds);