Skip to content
Permalink
Browse files

Made outgoing pools max total connections user configurable

For a finer control over the maximum simultaneously active outgoing
connections.
  • Loading branch information...
luccioman committed Jun 6, 2018
1 parent 387d646 commit b5dc1f376fbb3213a57a0d942c9fc8a70c9cc958
@@ -210,6 +210,12 @@ clientTimeout = 10000
# a limit on the number of concurrent connections
httpdMaxBusySessions = 200

# maximum number of simultaneously open outgoing HTTP connections in the general pool (net.yacy.cora.protocol.http.HTTPClient)
http.outgoing.pool.general.maxTotal = 200

# maximum number of simultaneously open outgoing HTTP connections in the remote Solr pool (net.yacy.cora.federate.solr.instance.RemoteInstance)
http.outgoing.pool.remoteSolr.maxTotal = 100

# default root path for the file server
# may be overridden by the htdocs parameter
# users shall be encouraged to use the htdocs path for individual content,
@@ -160,7 +160,7 @@ <h2>Performance Settings of Queues and Processes</h2>
</fieldset>
</form>

<form action="PerformanceQueues_p.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8" id="ThreadPoolSettings">
<form action="PerformanceQueues_p.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8" id="ThreadPoolSettings" class="col-sm-12 col-md-6">
<input type="hidden" name="transactionToken" value="#[transactionToken]#" />
<fieldset><legend>Thread Pool Settings:</legend>
<table border="0">
@@ -185,6 +185,50 @@ <h2>Performance Settings of Queues and Processes</h2>
</fieldset>
</form>

<form action="PerformanceQueues_p.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8" id="ConnectionPoolSettings" class="col-sm-12 col-md-6">
<input type="hidden" name="transactionToken" value="#[transactionToken]#" />
<fieldset><legend>Outgoing connections pools settings :</legend>
<table style="border-width:0px;">
<thead class="TableHeader">
<tr>
<th style="padding:0.4em;" rowspan="2">Connection Pool</th>
<th style="padding:0.4em;" rowspan="2" title="Total maximum number of simultaneously open connections in the pool">Total maximum</th>
<th style="padding:0.4em;" colspan="3" >Current statistics</th>
</tr>
<tr>
<th style="padding:0.4em;" title="Number of connections currently being used to execute requests.">Active</th>
<th style="padding:0.4em;" title="Number of reusable idle connections">Idle</th>
<th style="padding:0.4em;" title="Number of connection requests being blocked awaiting a free connection">Pending</th>
</tr>
</thead>
<tbody>
<tr class="TableCellDark">
<td>General</td>
<td><input name="http.outgoing.pool.general.maxTotal" type="number" value="#[http.outgoing.pool.general.maxTotal]#" min="1" max="2147483647"/></td>
<td>#[pool.general.leased]#</td>
<td>#[pool.general.available]#</td>
<td>#[pool.general.pending]#</td>
</tr>
<tr class="TableCellDark">
<td>Remote Solr servers</td>
<td><input name="http.outgoing.pool.remoteSolr.maxTotal" type="number" value="#[http.outgoing.pool.remoteSolr.maxTotal]#" min="1" max="2147483647"/></td>
<td>#[pool.remoteSolr.leased]#</td>
<td>#[pool.remoteSolr.available]#</td>
<td>#[pool.remoteSolr.pending]#</td>
</tr>
</tbody>
<tfoot>
<tr class="TableCellLight">
<td colspan="5">
<input type="submit" name="connectionPoolConfig" class="btn btn-sm btn-primary" value="Submit New Values" />
Changes take effect immediately
</td>
</tr>
</tfoot>
</table>
</fieldset>
</form>

#%env/templates/footer.template%#
</body>
</html>
@@ -28,8 +28,12 @@
import java.util.Iterator;
import java.util.Map;

import org.apache.http.pool.PoolStats;

import net.yacy.cora.federate.solr.instance.RemoteInstance;
import net.yacy.cora.protocol.ConnectionInfo;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.http.HTTPClient;
import net.yacy.data.TransactionManager;
import net.yacy.kelondro.data.word.WordReference;
import net.yacy.kelondro.rwi.IndexCell;
@@ -307,6 +311,25 @@ public static serverObjects respond(final RequestHeader header, final serverObje
sb.setConfig("httpdMaxBusySessions",maxBusy);

}

if ((post != null) && (post.containsKey("connectionPoolConfig"))) {

/* Configure the general outgoing HTTP connection pool */
int maxTotal = post.getInt(SwitchboardConstants.HTTP_OUTGOING_POOL_GENERAL_MAX_TOTAL,
SwitchboardConstants.HTTP_OUTGOING_POOL_GENERAL_MAX_TOTAL_DEFAULT);
if (maxTotal > 0) {
sb.setConfig(SwitchboardConstants.HTTP_OUTGOING_POOL_GENERAL_MAX_TOTAL, maxTotal);
HTTPClient.initPoolMaxConnections(HTTPClient.CONNECTION_MANAGER, maxTotal);
}

/* Configure the remote Solr outgoing HTTP connection pool */
maxTotal = post.getInt(SwitchboardConstants.HTTP_OUTGOING_POOL_REMOTE_SOLR_MAX_TOTAL,
SwitchboardConstants.HTTP_OUTGOING_POOL_REMOTE_SOLR_MAX_TOTAL_DEFAULT);
if (maxTotal > 0) {
sb.setConfig(SwitchboardConstants.HTTP_OUTGOING_POOL_REMOTE_SOLR_MAX_TOTAL, maxTotal);
RemoteInstance.initPoolMaxConnections(RemoteInstance.CONNECTION_MANAGER, maxTotal);
}
}

if ((post != null) && (post.containsKey("onlineCautionSubmit"))) {
sb.setConfig(SwitchboardConstants.PROXY_ONLINE_CAUTION_DELAY, Integer.toString(post.getInt("crawlPauseProxy", 30000)));
@@ -344,6 +367,24 @@ public static serverObjects respond(final RequestHeader header, final serverObje

prop.put("pool", "3");

/* Connection pools settings */
prop.put(SwitchboardConstants.HTTP_OUTGOING_POOL_GENERAL_MAX_TOTAL,
sb.getConfigInt(SwitchboardConstants.HTTP_OUTGOING_POOL_GENERAL_MAX_TOTAL,
SwitchboardConstants.HTTP_OUTGOING_POOL_GENERAL_MAX_TOTAL_DEFAULT));
prop.put(SwitchboardConstants.HTTP_OUTGOING_POOL_REMOTE_SOLR_MAX_TOTAL,
sb.getConfigInt(SwitchboardConstants.HTTP_OUTGOING_POOL_REMOTE_SOLR_MAX_TOTAL,
SwitchboardConstants.HTTP_OUTGOING_POOL_REMOTE_SOLR_MAX_TOTAL_DEFAULT));
/* Connection pools stats */
PoolStats stats = HTTPClient.CONNECTION_MANAGER.getTotalStats();
prop.put("pool.general.leased", stats.getLeased());
prop.put("pool.general.available", stats.getAvailable());
prop.put("pool.general.pending", stats.getPending());

stats = RemoteInstance.CONNECTION_MANAGER.getTotalStats();
prop.put("pool.remoteSolr.leased", stats.getLeased());
prop.put("pool.remoteSolr.available", stats.getAvailable());
prop.put("pool.remoteSolr.pending", stats.getPending());

/* Remote searches max loads settings */
prop.put("remoteSearchRWIMaxLoad", sb.getConfigFloat(SwitchboardConstants.REMOTESEARCH_MAXLOAD_RWI,
SwitchboardConstants.REMOTESEARCH_MAXLOAD_RWI_DEFAULT));
@@ -44,4 +44,20 @@
<numActive>#[numActive]#</numActive>
</Pool>#{/pool}#
</ThreadPools>
<ConnectionPools>
<ConnectionPool>
<Name>General</Name>
<maxTotal>#[http.outgoing.pool.general.maxTotal]#</maxTotal>
<active>#[pool.general.leased]#</active>
<idle>#[pool.general.available]#</idle>
<pending>#[pool.general.pending]#</pending>
</ConnectionPool>
<ConnectionPool>
<Name>Remote Solr servers</Name>
<maxTotal>#[http.outgoing.pool.remoteSolr.maxTotal]#</maxTotal>
<active>#[pool.remoteSolr.leased]#</active>
<idle>#[pool.remoteSolr.available]#</idle>
<pending>#[pool.remoteSolr.pending]#</pending>
</ConnectionPool>
</ConnectionPools>
</PerfmanceQueues>
@@ -65,6 +65,7 @@
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.util.CommonPattern;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.cora.util.Memory;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.search.schema.CollectionSchema;
import net.yacy.search.schema.WebgraphSchema;
@@ -75,8 +76,8 @@
@SuppressWarnings("deprecation")
public class RemoteInstance implements SolrInstance {

/** The connection manager used to handle the common HTTP connections pool. */
private static final org.apache.http.impl.conn.PoolingClientConnectionManager CONNECTION_MANAGER = buildConnectionManager();
/** The connection manager holding the HTTP connections pool shared between remote Solr clients. */
public static final org.apache.http.impl.conn.PoolingClientConnectionManager CONNECTION_MANAGER = buildConnectionManager();

/** A custom scheme registry allowing https connections to servers using self-signed certificate */
private static final SchemeRegistry SCHEME_REGISTRY = buildTrustSelfSignedSchemeRegistry();
@@ -205,19 +206,38 @@ public RemoteInstance(final String url, final Collection<String> coreNames, fina
if (this.defaultServer == null) throw new IOException("cannot connect to url " + url + " and connect core " + defaultCoreName);
}

/**
* Initialize the maximum connections for the given pool
*
* @param pool
* a pooling connection manager. Must not be null.
* @param maxConnections.
* The new maximum connections values. Must be greater than 0.
* @throws IllegalArgumentException
* when pool is null or when maxConnections is lower than 1
*/
public static void initPoolMaxConnections(final org.apache.http.impl.conn.PoolingClientConnectionManager pool, int maxConnections) {
if (pool == null) {
throw new IllegalArgumentException("pool parameter must not be null");
}
if (maxConnections <= 0) {
throw new IllegalArgumentException("maxConnections parameter must be greater than zero");
}
pool.setMaxTotal(maxConnections);

/* max connections per host */
pool.setDefaultMaxPerRoute((int) (2 * Memory.cores()));
}

/**
* @return a connection manager with a HTTP connection pool
*/
private static org.apache.http.impl.conn.PoolingClientConnectionManager buildConnectionManager() {
/* Important note : use of deprecated Apache classes is required because SolrJ still use them internally (see HttpClientUtil).
* Upgrade only when Solr implementation will become compatible */

org.apache.http.impl.conn.PoolingClientConnectionManager cm;

cm = new org.apache.http.impl.conn.PoolingClientConnectionManager(); // try also: ThreadSafeClientConnManager

cm.setMaxTotal(100);
cm.setDefaultMaxPerRoute(100);
final org.apache.http.impl.conn.PoolingClientConnectionManager cm = new org.apache.http.impl.conn.PoolingClientConnectionManager();
initPoolMaxConnections(cm, 100);
return cm;
}

@@ -118,9 +118,14 @@
public class HTTPClient {

private final static int default_timeout = 6000;
/** Maximum number of simultaneously open outgoing HTTP connections in the pool */
private final static int maxcon = 200;
private static IdleConnectionMonitorThread connectionMonitor = null;
private final static RequestConfig dfltReqConf = initRequestConfig();

/** The connection manager holding the configured connection pool for this client */
public static final PoolingHttpClientConnectionManager CONNECTION_MANAGER = initPoolingConnectionManager();

private final static HttpClientBuilder clientBuilder = initClientBuilder();
private final RequestConfig.Builder reqConfBuilder;
private Set<Entry<String, String>> headers = null;
@@ -171,7 +176,7 @@ private static RequestConfig initRequestConfig() {
private static HttpClientBuilder initClientBuilder() {
final HttpClientBuilder builder = HttpClientBuilder.create();

builder.setConnectionManager(initPoolingConnectionManager());
builder.setConnectionManager(CONNECTION_MANAGER);
builder.setDefaultRequestConfig(dfltReqConf);

// UserAgent
@@ -210,15 +215,8 @@ private static PoolingHttpClientConnectionManager initPoolingConnectionManager()
if (ip == null) throw new UnknownHostException(host0);
return new InetAddress[]{ip};
}});
// how much connections do we need? - default: 20
pooling.setMaxTotal(maxcon);
// for statistics same value should also be set here
ConnectionInfo.setMaxcount(maxcon);
// connections per host (2 default)
pooling.setDefaultMaxPerRoute((int) (2 * Memory.cores()));
// Increase max connections for localhost
final HttpHost localhost = new HttpHost(Domains.LOCALHOST);
pooling.setMaxPerRoute(new HttpRoute(localhost), maxcon);
initPoolMaxConnections(pooling, maxcon);

pooling.setValidateAfterInactivity(default_timeout); // on init set to default 5000ms
final SocketConfig socketConfig = SocketConfig.custom()
// Defines whether the socket can be bound even though a previous connection is still in a timeout state.
@@ -237,6 +235,35 @@ private static PoolingHttpClientConnectionManager initPoolingConnectionManager()

return pooling;
}

/**
* Initialize the maximum connections for the given pool
*
* @param pool
* a pooling connection manager. Must not be null.
* @param maxConnections.
* The new maximum connections values. Must be greater than 0.
* @throws IllegalArgumentException
* when pool is null or when maxConnections is lower than 1
*/
public static void initPoolMaxConnections(final PoolingHttpClientConnectionManager pool, int maxConnections) {
if (pool == null) {
throw new IllegalArgumentException("pool parameter must not be null");
}
if (maxConnections <= 0) {
throw new IllegalArgumentException("maxConnections parameter must be greater than zero");
}
pool.setMaxTotal(maxConnections);
// for statistics same value should also be set here
ConnectionInfo.setMaxcount(maxConnections);

// connections per host (2 default)
pool.setDefaultMaxPerRoute((int) (2 * Memory.cores()));

// Increase max connections for localhost
final HttpHost localhost = new HttpHost(Domains.LOCALHOST);
pool.setMaxPerRoute(new HttpRoute(localhost), maxConnections);
}

/**
* This method should be called just before shutdown
@@ -470,6 +470,9 @@ public void run() {
// set a high maximum cache size to current size; this is adopted later automatically
final int wordCacheMaxCount = (int) getConfigLong(SwitchboardConstants.WORDCACHE_MAX_COUNT, 20000);
setConfig(SwitchboardConstants.WORDCACHE_MAX_COUNT, Integer.toString(wordCacheMaxCount));

/* Init outgoing connections pools with user defined settings */
initOutgoingConnectionPools();

// load the network definition
try {
@@ -1270,6 +1273,29 @@ public boolean jobImpl() throws Exception {
this.log.config("Finished Switchboard Initialization");
}

/**
* Initialize outgoing connections pools with user defined settings
*/
private void initOutgoingConnectionPools() {
int generalPoolMaxTotal = getConfigInt(SwitchboardConstants.HTTP_OUTGOING_POOL_GENERAL_MAX_TOTAL,
SwitchboardConstants.HTTP_OUTGOING_POOL_GENERAL_MAX_TOTAL_DEFAULT);
if (generalPoolMaxTotal <= 0) {
/* Fix eventually wrong value from the config file */
generalPoolMaxTotal = SwitchboardConstants.HTTP_OUTGOING_POOL_GENERAL_MAX_TOTAL_DEFAULT;
setConfig(SwitchboardConstants.HTTP_OUTGOING_POOL_GENERAL_MAX_TOTAL, generalPoolMaxTotal);
}
HTTPClient.initPoolMaxConnections(HTTPClient.CONNECTION_MANAGER, generalPoolMaxTotal);

int remoteSolrPoolMaxTotal = getConfigInt(SwitchboardConstants.HTTP_OUTGOING_POOL_REMOTE_SOLR_MAX_TOTAL,
SwitchboardConstants.HTTP_OUTGOING_POOL_REMOTE_SOLR_MAX_TOTAL_DEFAULT);
if (remoteSolrPoolMaxTotal <= 0) {
/* Fix eventually wrong value from the config file */
remoteSolrPoolMaxTotal = SwitchboardConstants.HTTP_OUTGOING_POOL_REMOTE_SOLR_MAX_TOTAL_DEFAULT;
setConfig(SwitchboardConstants.HTTP_OUTGOING_POOL_REMOTE_SOLR_MAX_TOTAL, remoteSolrPoolMaxTotal);
}
RemoteInstance.initPoolMaxConnections(RemoteInstance.CONNECTION_MANAGER, remoteSolrPoolMaxTotal);
}

final String getSysinfo() {
return getConfig(SwitchboardConstants.NETWORK_NAME, "") + (isRobinsonMode() ? "-" : "/") + getConfig(SwitchboardConstants.NETWORK_DOMAIN, "global");
}
@@ -490,6 +490,19 @@

/** Default setting value controlling whether HTTP responses should be compressed */
public static final boolean SERVER_RESPONSE_COMPRESS_GZIP_DEFAULT = true;


/** Key of the setting controlling the maximum number of simultaneously open outgoing HTTP connections in the general pool (net.yacy.cora.protocol.http.HTTPClient) */
public static final String HTTP_OUTGOING_POOL_GENERAL_MAX_TOTAL = "http.outgoing.pool.general.maxTotal";

/** Default setting value controlling the maximum number of simultaneously open outgoing HTTP connections in the general pool */
public static final int HTTP_OUTGOING_POOL_GENERAL_MAX_TOTAL_DEFAULT = 200;

/** Key of the setting controlling the maximum number of simultaneously open outgoing HTTP connections in the remote Solr pool (net.yacy.cora.federate.solr.instance.RemoteInstance) */
public static final String HTTP_OUTGOING_POOL_REMOTE_SOLR_MAX_TOTAL = "http.outgoing.pool.remoteSolr.maxTotal";

/** Default setting value controlling the maximum number of simultaneously open outgoing HTTP connections in the remote Solr pool */
public static final int HTTP_OUTGOING_POOL_REMOTE_SOLR_MAX_TOTAL_DEFAULT = 100;


/*

0 comments on commit b5dc1f3

Please sign in to comment.
You can’t perform that action at this time.