Skip to content
Permalink
Browse files

Reuse expired connections evictors threads provided by apache and solr

  • Loading branch information...
luccioman committed Jun 6, 2018
1 parent b5dc1f3 commit cea81871616f7dd7ecdf897516d47a6074f9267e
@@ -26,6 +26,7 @@
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;

import javax.net.ssl.SSLContext;

@@ -53,13 +54,15 @@
import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
import org.apache.http.impl.auth.BasicScheme;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.SchemeRegistryFactory;
import org.apache.http.protocol.HTTP;
import org.apache.http.protocol.HttpContext;
import org.apache.http.ssl.SSLContextBuilder;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.update.UpdateShardHandler.IdleConnectionsEvictor;

import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.protocol.HeaderFramework;
@@ -76,12 +79,35 @@
@SuppressWarnings("deprecation")
public class RemoteInstance implements SolrInstance {

/** Default maximum time in seconds to keep alive an idle connection in the pool */
private static final int DEFAULT_POOLED_CONNECTION_TIME_TO_LIVE = 30;

/** Default sleep time in seconds between each run of the connection evictor */
private static final int DEFAULT_CONNECTION_EVICTOR_SLEEP_TIME = 5;

/** Default total maximum number of connections in the pool */
private static final int DEFAULT_POOL_MAX_TOTAL = 100;

/** The connection manager holding the HTTP connections pool shared between remote Solr clients. */
public static final org.apache.http.impl.conn.PoolingClientConnectionManager CONNECTION_MANAGER = buildConnectionManager();

/**
* Background daemon thread evicting expired idle connections from the pool.
* This may be eventually already done by the pool itself on connection request,
* but this background task helps when no request is made to the pool for a long
* time period.
*/
private static final IdleConnectionsEvictor EXPIRED_CONNECTIONS_EVICTOR = new IdleConnectionsEvictor(
CONNECTION_MANAGER, DEFAULT_CONNECTION_EVICTOR_SLEEP_TIME, TimeUnit.SECONDS,
DEFAULT_POOLED_CONNECTION_TIME_TO_LIVE, TimeUnit.SECONDS);

static {
EXPIRED_CONNECTIONS_EVICTOR.start();
}

/** A custom scheme registry allowing https connections to servers using self-signed certificate */
private static final SchemeRegistry SCHEME_REGISTRY = buildTrustSelfSignedSchemeRegistry();
private String solrurl;
private final HttpClient client;
private final String defaultCoreName;
@@ -236,8 +262,9 @@ public static void initPoolMaxConnections(final org.apache.http.impl.conn.Poolin
/* Important note : use of deprecated Apache classes is required because SolrJ still use them internally (see HttpClientUtil).
* Upgrade only when Solr implementation will become compatible */

final org.apache.http.impl.conn.PoolingClientConnectionManager cm = new org.apache.http.impl.conn.PoolingClientConnectionManager();
initPoolMaxConnections(cm, 100);
final org.apache.http.impl.conn.PoolingClientConnectionManager cm = new org.apache.http.impl.conn.PoolingClientConnectionManager(
SchemeRegistryFactory.createDefault(), DEFAULT_POOLED_CONNECTION_TIME_TO_LIVE, TimeUnit.SECONDS);
initPoolMaxConnections(cm, DEFAULT_POOL_MAX_TOTAL);
return cm;
}

@@ -461,7 +488,20 @@ public void close() {
* connections. Must be called at the end of the application.
*/
public static void closeConnectionManager() {
CONNECTION_MANAGER.shutdown();
try {
if (EXPIRED_CONNECTIONS_EVICTOR != null) {
// Shut down the evictor thread
EXPIRED_CONNECTIONS_EVICTOR.shutdown();
try {
EXPIRED_CONNECTIONS_EVICTOR.awaitTermination(1L, TimeUnit.SECONDS);
} catch (final InterruptedException ignored) {
}
}
} finally {
if (CONNECTION_MANAGER != null) {
CONNECTION_MANAGER.shutdown();
}
}
}

public static int queueSizeByMemory() {
@@ -76,7 +76,6 @@
import org.apache.http.config.SocketConfig;
import org.apache.http.conn.ConnectionKeepAliveStrategy;
import org.apache.http.conn.DnsResolver;
import org.apache.http.conn.HttpClientConnectionManager;
import org.apache.http.conn.routing.HttpRoute;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
@@ -90,6 +89,7 @@
import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.client.IdleConnectionEvictor;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.protocol.HTTP;
import org.apache.http.protocol.HttpContext;
@@ -120,12 +120,32 @@
private final static int default_timeout = 6000;
/** Maximum number of simultaneously open outgoing HTTP connections in the pool */
private final static int maxcon = 200;
private static IdleConnectionMonitorThread connectionMonitor = null;

/** Default sleep time in seconds between each run of the connection evictor */
private static final int DEFAULT_CONNECTION_EVICTOR_SLEEP_TIME = 5;

/** Default maximum time in seconds to keep alive an idle connection in the pool */
private static final int DEFAULT_POOLED_CONNECTION_TIME_TO_LIVE = 30;

private final static RequestConfig dfltReqConf = initRequestConfig();

/** The connection manager holding the configured connection pool for this client */
public static final PoolingHttpClientConnectionManager CONNECTION_MANAGER = initPoolingConnectionManager();

/**
* Background daemon thread evicting expired idle connections from the pool.
* This may be eventually already done by the pool itself on connection request,
* but this background task helps when no request is made to the pool for a long
* time period.
*/
private static final IdleConnectionEvictor EXPIRED_CONNECTIONS_EVICTOR = new IdleConnectionEvictor(
CONNECTION_MANAGER, DEFAULT_CONNECTION_EVICTOR_SLEEP_TIME, TimeUnit.SECONDS,
DEFAULT_POOLED_CONNECTION_TIME_TO_LIVE, TimeUnit.SECONDS);

static {
EXPIRED_CONNECTIONS_EVICTOR.start();
}

private final static HttpClientBuilder clientBuilder = initClientBuilder();
private final RequestConfig.Builder reqConfBuilder;
private Set<Entry<String, String>> headers = null;
@@ -208,13 +228,13 @@ private static PoolingHttpClientConnectionManager initPoolingConnectionManager()
.register("http", plainsf)
.register("https", getSSLSocketFactory())
.build();
final PoolingHttpClientConnectionManager pooling = new PoolingHttpClientConnectionManager(registry, new DnsResolver(){
final PoolingHttpClientConnectionManager pooling = new PoolingHttpClientConnectionManager(registry, null, null, new DnsResolver(){
@Override
public InetAddress[] resolve(final String host0)throws UnknownHostException {
final InetAddress ip = Domains.dnsResolve(host0);
if (ip == null) throw new UnknownHostException(host0);
return new InetAddress[]{ip};
}});
}}, DEFAULT_POOLED_CONNECTION_TIME_TO_LIVE, TimeUnit.SECONDS);
initPoolMaxConnections(pooling, maxcon);

pooling.setValidateAfterInactivity(default_timeout); // on init set to default 5000ms
@@ -228,11 +248,6 @@ private static PoolingHttpClientConnectionManager initPoolingConnectionManager()
.build();
pooling.setDefaultSocketConfig(socketConfig);

if (connectionMonitor == null) {
connectionMonitor = new IdleConnectionMonitorThread(pooling);
connectionMonitor.start();
}

return pooling;
}

@@ -265,19 +280,27 @@ public static void initPoolMaxConnections(final PoolingHttpClientConnectionManag
pool.setMaxPerRoute(new HttpRoute(localhost), maxConnections);
}

/**
* This method should be called just before shutdown
* to stop the ConnectionManager and idledConnectionEvictor
*
* @throws InterruptedException
*/
public static void closeConnectionManager() throws InterruptedException {
if (connectionMonitor != null) {
// Shut down the evictor thread
connectionMonitor.shutdown();
connectionMonitor.join();
}
}
/**
* This method should be called just before shutdown to stop the
* ConnectionManager and the idle connections evictor.
*
* @throws InterruptedException
* when the current thread is interrupted before the idle
* connections evictor thread termination.
*/
public static void closeConnectionManager() throws InterruptedException {
try {
if (EXPIRED_CONNECTIONS_EVICTOR != null) {
// Shut down the evictor thread
EXPIRED_CONNECTIONS_EVICTOR.shutdown();
EXPIRED_CONNECTIONS_EVICTOR.awaitTermination(1L, TimeUnit.SECONDS);
}
} finally {
if (CONNECTION_MANAGER != null) {
CONNECTION_MANAGER.shutdown();
}
}
}

/**
* This method sets the Header used for the request
@@ -1143,41 +1166,4 @@ public static void main(final String[] args) {
}
}

public static class IdleConnectionMonitorThread extends Thread {

private final HttpClientConnectionManager connMgr;
private volatile boolean shutdown;

public IdleConnectionMonitorThread(HttpClientConnectionManager connMgr) {
super("HTTPClient.IdleConnectionMonitorThread");
this.connMgr = connMgr;
}

@Override
public void run() {
try {
while (!shutdown) {
synchronized (this) {
wait(5000);
// Close expired connections
connMgr.closeExpiredConnections();
// Optionally, close connections
// that have been idle longer than 30 sec
connMgr.closeIdleConnections(30, TimeUnit.SECONDS);
}
}
connMgr.shutdown();
} catch (final InterruptedException ex) {
// terminate
}
}

public void shutdown() {
shutdown = true;
synchronized (this) {
notifyAll();
}
}
}

}

0 comments on commit cea8187

Please sign in to comment.
You can’t perform that action at this time.