fixes networknt#1619 add maxQueueSize to egress-router and ingress-pr…

…oxy (networknt#1620)
younggwon1 · Feb 17, 2023 · 1e8ce2b · 1e8ce2b
1 parent 168816b
commit 1e8ce2b
Show file tree

Hide file tree

Showing 10 changed files with 47 additions and 1 deletion.
diff --git a/egress-router/src/main/java/com/networknt/router/RouterConfig.java b/egress-router/src/main/java/com/networknt/router/RouterConfig.java
@@ -26,7 +26,6 @@
 
 import java.io.IOException;
 import java.util.*;
-import java.util.regex.Pattern;
 
 /**
  * Config class for reverse router.
@@ -45,6 +44,7 @@ public class RouterConfig {
     private static final String CONNECTION_PER_THREAD = "connectionsPerThread";
     private static final String SOFT_MAX_CONNECTIONS_PER_THREAD = "softMaxConnectionsPerThread";
     private static final String MAX_CONNECTION_RETRIES = "maxConnectionRetries";
+    private static final String MAX_QUEUE_SIZE = "maxQueueSize";
     private static final String SERVICE_ID_QUERY_PARAMETER = "serviceIdQueryParameter";
     private static final String PRE_RESOLVE_FQDN_2_IP = "preResolveFQDN2IP";
 
@@ -57,6 +57,8 @@ public class RouterConfig {
     boolean rewriteHostHeader;
     boolean reuseXForwarded;
     int maxConnectionRetries;
+    int maxQueueSize;
+
 
     boolean preResolveFQDN2IP;
     List<String> hostWhitelist;
@@ -145,6 +147,10 @@ public void setConfigData() {
         if(object != null ) {
             maxConnectionRetries = (Integer)object;
         }
+        object = getMappedConfig().get(MAX_QUEUE_SIZE);
+        if(object != null ) {
+            maxQueueSize = (Integer)object;
+        }
         object = getMappedConfig().get(SERVICE_ID_QUERY_PARAMETER);
         if(object != null) {
             serviceIdQueryParameter = (Boolean)object;
@@ -185,6 +191,8 @@ public int getConnectionsPerThread() {
 
     public int getMaxConnectionRetries() { return maxConnectionRetries; }
 
+    public int getMaxQueueSize() { return maxQueueSize; }
+
     public List<String> getHostWhitelist() {
         return hostWhitelist;
     }

diff --git a/egress-router/src/main/java/com/networknt/router/RouterHandler.java b/egress-router/src/main/java/com/networknt/router/RouterHandler.java
@@ -57,6 +57,7 @@ public RouterHandler() {
         proxyHandler = ProxyHandler.builder()
                 .setProxyClient(client)
                 .setMaxConnectionRetries(config.maxConnectionRetries)
+                .setMaxQueueSize(config.maxQueueSize)
                 .setMaxRequestTime(config.maxRequestTime)
                 .setPathPrefixMaxRequestTime(config.pathPrefixMaxRequestTime)
                 .setReuseXForwarded(config.reuseXForwarded)
@@ -89,6 +90,7 @@ public void reload() {
         proxyHandler = ProxyHandler.builder()
                 .setProxyClient(client)
                 .setMaxConnectionRetries(config.maxConnectionRetries)
+                .setMaxQueueSize(config.maxQueueSize)
                 .setMaxRequestTime(config.maxRequestTime)
                 .setPathPrefixMaxRequestTime(config.pathPrefixMaxRequestTime)
                 .setReuseXForwarded(config.reuseXForwarded)

diff --git a/egress-router/src/main/resources/config/router.yml b/egress-router/src/main/resources/config/router.yml
@@ -28,6 +28,12 @@ pathPrefixMaxRequestTime: ${router.pathPrefixMaxRequestTime:}
 # Connections per thread.
 connectionsPerThread: ${router.connectionsPerThread:10}
 
+# The max queue size for the requests if there is no connection to the downstream API in the connection pool.
+# The default value is 0 that means there is queued requests. As we have maxConnectionRetries, there is no
+# need to use the request queue to increase the memory usage. It should only be used when you see 503 errors
+# in the log after maxConnectionRetries to accommodate slow backend API.
+maxQueueSize: ${router.maxQueueSize:0}
+
 # Soft max connections per thread.
 softMaxConnectionsPerThread: ${router.softMaxConnectionsPerThread:5}
 

diff --git a/egress-router/src/test/java/com/networknt/router/RouterConfigTest.java b/egress-router/src/test/java/com/networknt/router/RouterConfigTest.java
@@ -41,6 +41,7 @@ public void testConfig() {
         Assert.assertTrue(routerConfig.isRewriteHostHeader());
         Assert.assertEquals(routerConfig.getMaxRequestTime(), 1000);
         Assert.assertEquals(routerConfig.getMaxConnectionRetries(), 3);
+        Assert.assertEquals(routerConfig.getMaxQueueSize(), 0);
     }
 
     @Test

diff --git a/egress-router/src/test/resources/config/router.yml b/egress-router/src/test/resources/config/router.yml
@@ -19,6 +19,12 @@ pathPrefixMaxRequestTime: ${router.pathPrefixMaxRequestTime:}
 # Connections per thread.
 connectionsPerThread: 10
 
+# The max queue size for the requests if there is no connection to the downstream API in the connection pool.
+# The default value is 0 that means there is queued requests. As we have maxConnectionRetries, there is no
+# need to use the request queue to increase the memory usage. It should only be used when you see 503 errors
+# in the log after maxConnectionRetries to accommodate slow backend API.
+maxQueueSize: ${router.maxQueueSize:0}
+
 # Soft max connections per thread.
 softMaxConnectionsPerThread: 5
 

diff --git a/handler/src/main/java/com/networknt/handler/ProxyHandler.java b/handler/src/main/java/com/networknt/handler/ProxyHandler.java
@@ -80,6 +80,7 @@
 public class ProxyHandler implements HttpHandler {
 
     private static final int DEFAULT_MAX_RETRY_ATTEMPTS = Integer.getInteger("maxRetries", 1);
+    private static final int DEFAULT_MAX_QUEUE_SIZE = Integer.getInteger("maxQueueSize", 0);
 
     private static final Logger logger = LoggerFactory.getLogger(ProxyHandler.class);
 
@@ -103,6 +104,7 @@ public class ProxyHandler implements HttpHandler {
     private volatile boolean rewriteHostHeader;
     private volatile boolean reuseXForwarded;
     private volatile int maxConnectionRetries;
+    private volatile int maxQueueSize;
     private volatile List<UrlRewriteRule> urlRewriteRules;
     private volatile List<MethodRewriteRule> methodRewriteRules;
 
@@ -120,6 +122,7 @@ private ProxyHandler(Builder builder) {
         this.rewriteHostHeader = builder.rewriteHostHeader;
         this.reuseXForwarded = builder.reuseXForwarded;
         this.maxConnectionRetries = builder.maxConnectionRetries;
+        this.maxQueueSize = builder.maxQueueSize;
         this.urlRewriteRules = builder.urlRewriteRules;
         this.methodRewriteRules = builder.methodRewriteRules;
         this.queryParamRewriteRules = builder.queryParamRewriteRules;
@@ -1010,6 +1013,7 @@ public static class Builder {
         private boolean rewriteHostHeader;
         private boolean reuseXForwarded;
         private int maxConnectionRetries = DEFAULT_MAX_RETRY_ATTEMPTS;
+        private int maxQueueSize = DEFAULT_MAX_QUEUE_SIZE;
         private Predicate idempotentRequestPredicate = IdempotentPredicate.INSTANCE;
         private List<UrlRewriteRule> urlRewriteRules;
         private List<MethodRewriteRule> methodRewriteRules;
@@ -1095,6 +1099,11 @@ public Builder setMaxConnectionRetries(int maxConnectionRetries) {
             return this;
         }
 
+        public Builder setMaxQueueSize(int maxQueueSize) {
+            this.maxQueueSize = maxQueueSize;
+            return this;
+        }
+
         public Builder setIdempotentRequestPredicate(Predicate idempotentRequestPredicate) {
             if (idempotentRequestPredicate == null) {
                 throw UndertowMessages.MESSAGES.argumentCannotBeNull("idempotentRequestPredicate");

diff --git a/ingress-proxy/src/main/java/com/networknt/proxy/LightProxyHandler.java b/ingress-proxy/src/main/java/com/networknt/proxy/LightProxyHandler.java
@@ -90,6 +90,7 @@ public LightProxyHandler() {
         proxyHandler = ProxyHandler.builder()
                 .setProxyClient(loadBalancer)
                 .setMaxConnectionRetries(config.getMaxConnectionRetries())
+                .setMaxQueueSize(config.getMaxQueueSize())
                 .setMaxRequestTime(config.getMaxRequestTime())
                 .setReuseXForwarded(config.isReuseXForwarded())
                 .setRewriteHostHeader(config.isRewriteHostHeader())
@@ -172,6 +173,7 @@ public void reload() {
         proxyHandler = ProxyHandler.builder()
                 .setProxyClient(loadBalancer)
                 .setMaxConnectionRetries(config.getMaxConnectionRetries())
+                .setMaxQueueSize(config.getMaxQueueSize())
                 .setMaxRequestTime(config.getMaxRequestTime())
                 .setReuseXForwarded(config.isReuseXForwarded())
                 .setRewriteHostHeader(config.isRewriteHostHeader())

diff --git a/ingress-proxy/src/main/java/com/networknt/proxy/ProxyConfig.java b/ingress-proxy/src/main/java/com/networknt/proxy/ProxyConfig.java
@@ -35,6 +35,7 @@ public class ProxyConfig {
     private static final String REWRITE_HOST_HEADER = "rewriteHostHeader";
     private static final String REUSE_X_FORWARDED = "reuseXForwarded";
     private static final String MAX_CONNECTION_RETRIES = "maxConnectionRetries";
+    private static final String MAX_QUEUE_SIZE = "maxQueueSize";
     private static final String FORWARD_JWT_CLAIMS = "forwardJwtClaims";
 
     boolean enabled;
@@ -45,6 +46,7 @@ public class ProxyConfig {
     boolean rewriteHostHeader;
     boolean reuseXForwarded;
     int maxConnectionRetries;
+    int maxQueueSize;
     private boolean forwardJwtClaims;
 
     private Config config;
@@ -103,6 +105,8 @@ public int getMaxRequestTime() {
 
     public int getMaxConnectionRetries() { return maxConnectionRetries; }
 
+    public int getMaxQueueSize() { return maxQueueSize; }
+
     public boolean isForwardJwtClaims() {
         return forwardJwtClaims;
     }
@@ -128,5 +132,6 @@ private void setConfigData() {
         connectionsPerThread = (Integer)getMappedConfig().get(CONNECTIONS_PER_THREAD);
         maxRequestTime = (Integer)getMappedConfig().get(MAX_REQUEST_TIME);
         maxConnectionRetries = (Integer)getMappedConfig().get(MAX_CONNECTION_RETRIES);
+        maxQueueSize = (Integer)getMappedConfig().get(MAX_QUEUE_SIZE);
     }
 }
diff --git a/ingress-proxy/src/main/resources/config/proxy.yml b/ingress-proxy/src/main/resources/config/proxy.yml
@@ -24,5 +24,11 @@ reuseXForwarded: ${proxy.reuseXForwarded:false}
 # Max Connection Retries
 maxConnectionRetries: ${proxy.maxConnectionRetries:3}
 
+# The max queue size for the requests if there is no connection to the downstream API in the connection pool.
+# The default value is 0 that means there is queued requests. As we have maxConnectionRetries, there is no
+# need to use the request queue to increase the memory usage. It should only be used when you see 503 errors
+# in the log after maxConnectionRetries to accommodate slow backend API.
+maxQueueSize: ${proxy.maxQueueSize:0}
+
 # Decode the JWT token claims and forward to the backend api in the form of json string
 forwardJwtClaims: ${proxy.forwardJwtClaims:false}
diff --git a/ingress-proxy/src/test/java/com/networknt/proxy/ProxyConfigTest.java b/ingress-proxy/src/test/java/com/networknt/proxy/ProxyConfigTest.java
@@ -8,5 +8,6 @@ public class ProxyConfigTest {
     public void testLoadConfig() {
         ProxyConfig config = ProxyConfig.load();
         Assert.assertNotNull(config.getHosts());
+        Assert.assertEquals(config.getMaxQueueSize(), 0);
     }
 }