diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
index c0f90d98bc6b4..89f65e2402264 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
@@ -1467,10 +1467,12 @@ Writable call(RPC.RpcKind rpcKind, Writable rpcRequest,
     final Connection connection = getConnection(remoteId, call, serviceClass,
         fallbackToSimpleAuth);
 
+    boolean success = false;
     try {
       checkAsyncCall();
       try {
         connection.sendRpcRequest(call);                 // send the rpc request
+        success = true;
       } catch (RejectedExecutionException e) {
         throw new IOException("connection has been closed", e);
       } catch (InterruptedException ie) {
@@ -1485,6 +1487,10 @@ Writable call(RPC.RpcKind rpcKind, Writable rpcRequest,
         releaseAsyncCall();
       }
       throw e;
+    } finally {
+      if (!success) {
+        connection.calls.remove(call.id);
+      }
     }
 
     if (isAsynchronousMode()) {