Permalink
Browse files

Merge pull request #605 from dmdevito/fixingfirewallconnectioncut_3

added JMX counters for renewed idle/too long connections + added/modified tests
  • Loading branch information...
2 parents 1fe97f3 + 40932f7 commit b39268294ffd789227a0c4e89cec5608b2fe3990 @zznate zznate committed Mar 25, 2013
@@ -10,6 +10,8 @@
import me.prettyprint.cassandra.connection.client.HClient;
import me.prettyprint.cassandra.connection.factory.HClientFactory;
import me.prettyprint.cassandra.service.CassandraHost;
+import me.prettyprint.cassandra.service.CassandraClientMonitor;
+import me.prettyprint.cassandra.service.CassandraClientMonitor.Counter;
import me.prettyprint.hector.api.exceptions.HInactivePoolException;
import me.prettyprint.hector.api.exceptions.HPoolExhaustedException;
import me.prettyprint.hector.api.exceptions.HectorException;
@@ -36,9 +38,12 @@
private final HClientFactory clientFactory;
- public ConcurrentHClientPool(HClientFactory clientFactory, CassandraHost host) {
+ private final CassandraClientMonitor monitor;
+
+ public ConcurrentHClientPool(HClientFactory clientFactory, CassandraHost host, CassandraClientMonitor monitor) {
this.clientFactory = clientFactory;
this.cassandraHost = host;
+ this.monitor = monitor;
availableClientQueue = new ArrayBlockingQueue<HClient>(cassandraHost.getMaxActive(), true);
// This counter can be offset by as much as the number of threads.
@@ -79,6 +84,8 @@ public HClient borrowClient() throws HectorException {
System.currentTimeMillis() - cassandraClient.getLastSuccessTime());
cassandraClient.close();
cassandraClient = null;
+
+ monitor.incCounter(Counter.RENEWED_IDLE_CONNECTIONS);
}
}
if (cassandraClient != null) {
@@ -88,6 +95,8 @@ public HClient borrowClient() throws HectorException {
System.currentTimeMillis() - cassandraClient.getCreatedTime());
cassandraClient.close();
cassandraClient = null;
+
+ monitor.incCounter(Counter.RENEWED_TOO_LONG_CONNECTIONS);
}
}
if ( cassandraClient == null ) {
@@ -14,6 +14,7 @@
import me.prettyprint.cassandra.connection.factory.HClientFactory;
import me.prettyprint.cassandra.service.CassandraHost;
+import me.prettyprint.cassandra.service.CassandraClientMonitor;
import me.prettyprint.cassandra.utils.DaemonThreadPoolFactory;
import org.slf4j.Logger;
@@ -119,8 +120,8 @@ public int compare(HClientPool p1, HClientPool p2) {
}
@Override
- public HClientPool createConnection(HClientFactory clientFactory, CassandraHost host) {
- LatencyAwareHClientPool pool = new LatencyAwareHClientPool(clientFactory, host);
+ public HClientPool createConnection(HClientFactory clientFactory, CassandraHost host, CassandraClientMonitor monitor) {
+ LatencyAwareHClientPool pool = new LatencyAwareHClientPool(clientFactory, host, monitor);
add(pool);
return pool;
}
@@ -62,9 +62,10 @@ public HConnectionManager(String clusterName, CassandraHostConfigurator cassandr
if ( cassandraHostConfigurator.getRetryDownedHosts() ) {
cassandraHostRetryService = new CassandraHostRetryService(this, clientFactory, cassandraHostConfigurator, listenerHandler);
}
+ monitor = JmxMonitor.getInstance().getCassandraMonitor(this);
for ( CassandraHost host : cassandraHostConfigurator.buildCassandraHosts()) {
try {
- HClientPool hcp = loadBalancingPolicy.createConnection(clientFactory, host);
+ HClientPool hcp = loadBalancingPolicy.createConnection(clientFactory, host, monitor);
hostPools.put(host,hcp);
} catch (HectorTransportException hte) {
log.error("Could not start connection pool for host {}", host);
@@ -78,7 +79,6 @@ public HConnectionManager(String clusterName, CassandraHostConfigurator cassandr
if ( cassandraHostConfigurator.getUseHostTimeoutTracker() ) {
hostTimeoutTracker = new HostTimeoutTracker(this, cassandraHostConfigurator);
}
- monitor = JmxMonitor.getInstance().getCassandraMonitor(this);
exceptionsTranslator = new ExceptionsTranslatorImpl();
this.cassandraHostConfigurator = cassandraHostConfigurator;
hostPoolValues = hostPools.values();
@@ -109,7 +109,7 @@ public boolean addCassandraHost(CassandraHost cassandraHost) {
HClientPool pool = null;
try {
cassandraHostConfigurator.applyConfig(cassandraHost);
- pool = cassandraHostConfigurator.getLoadBalancingPolicy().createConnection(clientFactory, cassandraHost);
+ pool = cassandraHostConfigurator.getLoadBalancingPolicy().createConnection(clientFactory, cassandraHost, monitor);
hostPools.putIfAbsent(cassandraHost, pool);
log.info("Added host {} to pool", cassandraHost.getName());
listenerHandler.fireOnAddHost(cassandraHost, true, null, null);
@@ -6,6 +6,7 @@
import me.prettyprint.cassandra.connection.client.HClient;
import me.prettyprint.cassandra.connection.factory.HClientFactory;
import me.prettyprint.cassandra.service.CassandraHost;
+import me.prettyprint.cassandra.service.CassandraClientMonitor;
import me.prettyprint.hector.api.exceptions.HectorException;
/**
@@ -22,8 +23,8 @@
private static final double SENTINEL_COMPARE = 0.768;
private final LinkedBlockingDeque<Double> latencies;
- public LatencyAwareHClientPool(HClientFactory clientFactory, CassandraHost host) {
- super(clientFactory, host);
+ public LatencyAwareHClientPool(HClientFactory clientFactory, CassandraHost host, CassandraClientMonitor monitor) {
+ super(clientFactory, host, monitor);
latencies = new LinkedBlockingDeque<Double>(WINDOW_QUEUE_SIZE);
}
@@ -2,6 +2,7 @@
import me.prettyprint.cassandra.connection.factory.HClientFactory;
import me.prettyprint.cassandra.service.CassandraHost;
+import me.prettyprint.cassandra.service.CassandraClientMonitor;
import java.util.*;
@@ -55,7 +56,7 @@ public int compare(HClientPool o1, HClientPool o2) {
}
@Override
- public HClientPool createConnection(HClientFactory clientFactory, CassandraHost host) {
- return new ConcurrentHClientPool(clientFactory, host);
+ public HClientPool createConnection(HClientFactory clientFactory, CassandraHost host, CassandraClientMonitor monitor) {
+ return new ConcurrentHClientPool(clientFactory, host, monitor);
}
}
@@ -6,6 +6,7 @@
import me.prettyprint.cassandra.connection.factory.HClientFactory;
import me.prettyprint.cassandra.service.CassandraHost;
+import me.prettyprint.cassandra.service.CassandraClientMonitor;
/**
* Default interface for all load balancing policies.
@@ -27,7 +28,8 @@
*
* @param clientFactory an instance of {@link HClientFactory}
* @param host an instance of {@link CassandraHost} representing the host this pool will represent
+ * @param monitor the monitor exposing JMX methods
* @return a connection pool
*/
- HClientPool createConnection(HClientFactory clientFactory, CassandraHost host);
+ HClientPool createConnection(HClientFactory clientFactory, CassandraHost host, CassandraClientMonitor monitor);
}
@@ -5,6 +5,7 @@
import me.prettyprint.cassandra.connection.factory.HClientFactory;
import me.prettyprint.cassandra.service.CassandraHost;
+import me.prettyprint.cassandra.service.CassandraClientMonitor;
import com.google.common.collect.Iterables;
@@ -63,7 +64,7 @@ private int getAndIncrement(int size) {
}
@Override
- public HClientPool createConnection(HClientFactory clientFactory, CassandraHost host) {
- return new ConcurrentHClientPool(clientFactory, host);
+ public HClientPool createConnection(HClientFactory clientFactory, CassandraHost host, CassandraClientMonitor monitor) {
+ return new ConcurrentHClientPool(clientFactory, host, monitor);
}
}
@@ -40,6 +40,8 @@
RECOVERABLE_LB_CONNECT_ERRORS,
/** Connection time errors - unable to connect to host or something... */
CONNECT_ERROR,
+ RENEWED_IDLE_CONNECTIONS,
+ RENEWED_TOO_LONG_CONNECTIONS
}
public CassandraClientMonitor(HConnectionManager connectionManager) {
@@ -247,7 +249,14 @@ public boolean setCassandraHostRetryDelay(String retryDelay) {
return false;
}
}
-
-
+ @Override
+ public int getNumRenewedIdleConnections() {
+ return counters.get(Counter.RENEWED_IDLE_CONNECTIONS).intValue();
+ }
+
+ @Override
+ public int getNumRenewedTooLongConnections() {
+ return counters.get(Counter.RENEWED_TOO_LONG_CONNECTIONS).intValue();
+ }
}
@@ -152,4 +152,16 @@
Set<String> getSuspendedCassandraHosts();
boolean setCassandraHostRetryDelay(String retryDelay);
+
+
+ /**
+ * Total number of connections created due to previous idle connections.
+ */
+ int getNumRenewedIdleConnections();
+
+
+ /**
+ * Total number of connections created due to previous too long connections.
+ */
+ int getNumRenewedTooLongConnections();
}
@@ -7,6 +7,7 @@
import me.prettyprint.cassandra.connection.factory.HClientFactory;
import me.prettyprint.cassandra.connection.factory.HThriftClientFactoryImpl;
import me.prettyprint.cassandra.service.CassandraHost;
+import me.prettyprint.cassandra.service.CassandraClientMonitor;
import me.prettyprint.hector.api.exceptions.HInactivePoolException;
import org.junit.Before;
@@ -22,7 +23,7 @@ public void setupTest() {
setupClient();
cassandraHost = cassandraHostConfigurator.buildCassandraHosts()[0];
HClientFactory factory = new HThriftClientFactoryImpl();
- clientPool = new ConcurrentHClientPool(factory, cassandraHost);
+ clientPool = new ConcurrentHClientPool(factory, cassandraHost, new CassandraClientMonitor(connectionManager));
}
@Test
@@ -8,6 +8,7 @@
import me.prettyprint.cassandra.connection.factory.HClientFactory;
import me.prettyprint.cassandra.connection.factory.HThriftClientFactoryImpl;
import me.prettyprint.cassandra.service.CassandraHost;
+import me.prettyprint.cassandra.service.CassandraClientMonitor;
import me.prettyprint.cassandra.service.CassandraHostConfigurator;
import me.prettyprint.hector.api.exceptions.HInactivePoolException;
@@ -18,13 +19,15 @@
private CassandraHost cassandraHost;
private ConcurrentHClientPool clientPool;
+ private CassandraClientMonitor monitor;
@Before
public void setupTest() {
setupClient();
cassandraHost = cassandraHostConfigurator.buildCassandraHosts()[0];
HClientFactory factory = new HThriftClientFactoryImpl();
- clientPool = new ConcurrentHClientPool(factory, cassandraHost);
+ monitor = new CassandraClientMonitor(connectionManager);
+ clientPool = new ConcurrentHClientPool(factory, cassandraHost, monitor);
}
protected void configure(CassandraHostConfigurator configurator) {
@@ -39,13 +42,15 @@ public void testBorrowAndRenew() {
client1.updateLastSuccessTime();
clientPool.releaseClient(client1);
assertEquals(0, clientPool.getNumActive());
+ int count = monitor.getNumRenewedIdleConnections();
try {
Thread.sleep(4 * 1000);
} catch(InterruptedException ex) {
fail();
}
HClient client2 = clientPool.borrowClient();
assertEquals(1, clientPool.getNumActive());
+ assertEquals(count + 1, monitor.getNumRenewedIdleConnections());
assertNotSame(client1, client2);
}
}
@@ -5,6 +5,7 @@
import me.prettyprint.cassandra.connection.factory.HClientFactory;
import me.prettyprint.cassandra.connection.factory.HThriftClientFactoryImpl;
import me.prettyprint.cassandra.service.CassandraHost;
+import me.prettyprint.cassandra.service.CassandraClientMonitor;
import org.junit.Before;
import org.junit.Test;
@@ -20,7 +21,7 @@ public void setupTest() {
setupClient();
cassandraHost = cassandraHostConfigurator.buildCassandraHosts()[0];
HClientFactory factory = new HThriftClientFactoryImpl();
- clientPool = new LatencyAwareHClientPool(factory, cassandraHost);
+ clientPool = new LatencyAwareHClientPool(factory, cassandraHost, new CassandraClientMonitor(connectionManager));
}
@Test

0 comments on commit b392682

Please sign in to comment.