8293562: KeepAliveCache Blocks Threads while Closing Connections

Reviewed-by: sgehwolf
Backport-of: 03f25a9c6924430ec4063b801b2b6ca55b9067c9

Author: Pooja-DP

src/java.base/share/classes/sun/net/www/http/KeepAliveCache.java

@@ -113,38 +113,45 @@ public KeepAliveCache() {}
      * @param url  The URL contains info about the host and port
      * @param http The HttpClient to be cached
      */
-    public synchronized void put(final URL url, Object obj, HttpClient http) {
-        boolean startThread = (keepAliveTimer == null);
-        if (!startThread) {
-            if (!keepAliveTimer.isAlive()) {
-                startThread = true;
-            }
-        }
-        if (startThread) {
-            clear();
-            /* Unfortunately, we can't always believe the keep-alive timeout we got
-             * back from the server.  If I'm connected through a Netscape proxy
-             * to a server that sent me a keep-alive
-             * time of 15 sec, the proxy unilaterally terminates my connection
-             * The robustness to get around this is in HttpClient.parseHTTP()
-             */
-            final KeepAliveCache cache = this;
-            AccessController.doPrivileged(new PrivilegedAction<>() {
-                public Void run() {
-                    keepAliveTimer = InnocuousThread.newSystemThread("Keep-Alive-Timer", cache);
-                    keepAliveTimer.setDaemon(true);
-                    keepAliveTimer.setPriority(Thread.MAX_PRIORITY - 2);
-                    keepAliveTimer.start();
-                    return null;
+    public void put(final URL url, Object obj, HttpClient http) {
+        // this method may need to close an HttpClient, either because
+        // it is not cacheable, or because the cache is at its capacity.
+        // In the latter case, we close the least recently used client.
+        // The client to close is stored in oldClient, and is closed
+        // after cacheLock is released.
+        HttpClient oldClient = null;
+        synchronized (this) {
+            boolean startThread = (keepAliveTimer == null);
+            if (!startThread) {
+                if (!keepAliveTimer.isAlive()) {
+                    startThread = true;
                 }
-            });
-        }
+            }
+            if (startThread) {
+                clear();
+                /* Unfortunately, we can't always believe the keep-alive timeout we got
+                * back from the server.  If I'm connected through a Netscape proxy
+                * to a server that sent me a keep-alive
+                * time of 15 sec, the proxy unilaterally terminates my connection
+                * The robustness to get around this is in HttpClient.parseHTTP()
+                */
+                final KeepAliveCache cache = this;
+                AccessController.doPrivileged(new PrivilegedAction<>() {
+                    public Void run() {
+                        keepAliveTimer = InnocuousThread.newSystemThread("Keep-Alive-Timer", cache);
+                        keepAliveTimer.setDaemon(true);
+                        keepAliveTimer.setPriority(Thread.MAX_PRIORITY - 2);
+                        keepAliveTimer.start();
+                        return null;
+                    }
+                 });
+            }
 
-        KeepAliveKey key = new KeepAliveKey(url, obj);
-        ClientVector v = super.get(key);
+            KeepAliveKey key = new KeepAliveKey(url, obj);
+            ClientVector v = super.get(key);
 
-        if (v == null) {
-            int keepAliveTimeout = http.getKeepAliveTimeout();
+            if (v == null) {
+                int keepAliveTimeout = http.getKeepAliveTimeout();
                 if (keepAliveTimeout == 0) {
                     keepAliveTimeout = getUserKeepAlive(http.getUsingProxy());
                     if (keepAliveTimeout == -1) {
@@ -164,14 +171,19 @@ public Void run() {
                 // alive, which could be 0, if the user specified 0 for the property
                 assert keepAliveTimeout >= 0;
                 if (keepAliveTimeout == 0) {
-                    http.closeServer();
+                    oldClient = http;
                 } else {
                     v = new ClientVector(keepAliveTimeout * 1000);
                     v.put(http);
                     super.put(key, v);
                 }
-        } else {
-            v.put(http);
+            } else {
+                oldClient = v.put(http);
+            }
+        }
+        // close after releasing locks
+        if (oldClient != null) {
+            oldClient.closeServer();
         }
     }
 
@@ -221,6 +233,7 @@ public void run() {
             try {
                 Thread.sleep(LIFETIME);
             } catch (InterruptedException e) {}
+            List<HttpClient> closeList = null;
 
             // Remove all outdated HttpClients.
             synchronized (this) {
@@ -230,15 +243,18 @@ public void run() {
                 for (KeepAliveKey key : keySet()) {
                     ClientVector v = get(key);
                     synchronized (v) {
-                        KeepAliveEntry e = v.peek();
+                        KeepAliveEntry e = v.peekLast();
                         while (e != null) {
                             if ((currentTime - e.idleStartTime) > v.nap) {
-                                v.poll();
-                                e.hc.closeServer();
+                                v.pollLast();
+                                if (closeList == null) {
+                                    closeList = new ArrayList<>();
+                                }
+                                closeList.add(e.hc);
                             } else {
                                 break;
                             }
-                            e = v.peek();
+                            e = v.peekLast();
                         }
 
                         if (v.isEmpty()) {
@@ -251,6 +267,12 @@ public void run() {
                     removeVector(key);
                 }
             }
+            // close connections outside cacheLock
+            if (closeList != null) {
+                 for (HttpClient hc : closeList) {
+                     hc.closeServer();
+               }
+          }
         } while (!isEmpty());
     }
 
@@ -268,8 +290,8 @@ private void readObject(ObjectInputStream stream)
     }
 }
 
-/* FILO order for recycling HttpClients, should run in a thread
- * to time them out.  If > maxConns are in use, block.
+/* LIFO order for reusing HttpClients. Most recent entries at the front.
+ * If > maxConns are in use, discard oldest.
  */
 class ClientVector extends ArrayDeque<KeepAliveEntry> {
     private static final long serialVersionUID = -8680532108106489459L;
@@ -282,36 +304,37 @@ class ClientVector extends ArrayDeque<KeepAliveEntry> {
     }
 
     synchronized HttpClient get() {
-        if (isEmpty()) {
+        // check the most recent connection, use if still valid
+        KeepAliveEntry e = peekFirst();
+        if (e == null) {
             return null;
         }
 
-        // Loop until we find a connection that has not timed out
-        HttpClient hc = null;
         long currentTime = System.currentTimeMillis();
-        do {
-            KeepAliveEntry e = pop();
-            if ((currentTime - e.idleStartTime) > nap) {
-                e.hc.closeServer();
-            } else {
-                hc = e.hc;
-                if (KeepAliveCache.logger.isLoggable(PlatformLogger.Level.FINEST)) {
-                    String msg = "cached HttpClient was idle for "
+        if ((currentTime - e.idleStartTime) > nap) {
+            return null; // all connections stale - will be cleaned up later
+        } else {
+            pollFirst();
+            if (KeepAliveCache.logger.isLoggable(PlatformLogger.Level.FINEST)) {
+                String msg = "cached HttpClient was idle for "
                         + Long.toString(currentTime - e.idleStartTime);
-                    KeepAliveCache.logger.finest(msg);
-                }
+                KeepAliveCache.logger.finest(msg);
             }
-        } while ((hc == null) && (!isEmpty()));
-        return hc;
+            return e.hc;
+        }
     }
 
     /* return a still valid, unused HttpClient */
-    synchronized void put(HttpClient h) {
+    synchronized HttpClient put(HttpClient h) {
+        HttpClient staleClient = null;
+        assert KeepAliveCache.getMaxConnections() > 0;
         if (size() >= KeepAliveCache.getMaxConnections()) {
-            h.closeServer(); // otherwise the connection remains in limbo
-        } else {
-            push(new KeepAliveEntry(h, System.currentTimeMillis()));
+           // remove oldest connection
+           staleClient = removeLast().hc;
         }
+        addFirst(new KeepAliveEntry(h, System.currentTimeMillis()));
+        // close after releasing the locks
+        return staleClient;
     }
 
     /* remove an HttpClient */
@@ -339,10 +362,10 @@ private void readObject(ObjectInputStream stream)
 }
 
 class KeepAliveKey {
-    private String      protocol = null;
-    private String      host = null;
-    private int         port = 0;
-    private Object      obj = null; // additional key, such as socketfactory
+    private final String      protocol;
+    private final String      host;
+    private final int         port;
+    private final Object      obj; // additional key, such as socketfactory
 
     /**
      * Constructor
@@ -383,8 +406,8 @@ public int hashCode() {
 }
 
 class KeepAliveEntry {
-    HttpClient hc;
-    long idleStartTime;
+    final HttpClient hc;
+    final long idleStartTime;
 
     KeepAliveEntry(HttpClient hc, long idleStartTime) {
         this.hc = hc;

src/java.base/share/classes/sun/net/www/protocol/https/HttpsClient.java

@@ -428,6 +428,14 @@ protected Socket createSocket() throws IOException {
         }
     }
 
+    @Override
+    public void closeServer() {
+        try {
+            // SSLSocket.close may block up to timeout. Make sure it's short.
+            serverSocket.setSoTimeout(1);
+        } catch (Exception e) {}
+        super.closeServer();
+    }
 
     @Override
     public boolean needsTunneling() {