Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Merge remote-tracking branch 'origin/master' into SliceFilter

  • Loading branch information...
commit 66a10cc604d4357cea5c668d537c88fcfd64d640 2 parents 520c640 + 21ac921
Shane Perry authored
Showing with 325 additions and 8 deletions.
  1. +1 −1  README
  2. +4 −0 core/pom.xml
  3. +17 −1 core/src/main/java/me/prettyprint/cassandra/connection/ConcurrentHClientPool.java
  4. +2 −1  core/src/main/java/me/prettyprint/cassandra/connection/HConnectionManager.java
  5. +1 −1  core/src/main/java/me/prettyprint/cassandra/connection/HOpTimer.java
  6. +46 −0 core/src/main/java/me/prettyprint/cassandra/connection/MetricsOpTimer.java
  7. +1 −1  core/src/main/java/me/prettyprint/cassandra/connection/NullOpTimer.java
  8. +1 −1  core/src/main/java/me/prettyprint/cassandra/connection/SpeedForJOpTimer.java
  9. +18 −0 core/src/main/java/me/prettyprint/cassandra/connection/client/HClient.java
  10. +26 −0 core/src/main/java/me/prettyprint/cassandra/connection/client/HThriftClient.java
  11. +28 −0 core/src/main/java/me/prettyprint/cassandra/service/CassandraHost.java
  12. +35 −1 core/src/main/java/me/prettyprint/cassandra/service/CassandraHostConfigurator.java
  13. +70 −0 core/src/main/java/me/prettyprint/cassandra/service/template/ColumnFamilyResultIterator.java
  14. +2 −0  core/src/main/java/me/prettyprint/hector/api/beans/AbstractComposite.java
  15. +8 −0 core/src/main/java/me/prettyprint/hector/api/ddl/ColumnFamilyDefinition.java
  16. +59 −0 core/src/test/java/me/prettyprint/cassandra/connection/HConnectionManagerMetricsOpTimerTest.java
  17. +6 −1 pom.xml
2  README
View
@@ -12,7 +12,7 @@ Hector is the greatest warrior in the greek mythology, Troy's builder and brothe
http://en.wikipedia.org/wiki/Hector
http://en.wikipedia.org/wiki/Cassandra
-Hector is currently in use on a number of production systems some of which have node counts into the hundreds. Issues generally are fixed as quickly as possbile and releases done frequently.
+Hector is currently in use on a number of production systems some of which have node counts into the hundreds. Issues generally are fixed as quickly as possible and releases done frequently.
Some features provided by this client:
4 core/pom.xml
View
@@ -122,6 +122,10 @@
<groupId>com.ecyrd.speed4j</groupId>
<artifactId>speed4j</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.yammer.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ </dependency>
<!-- Note the optional tag. If you need to use fastinfoset serialization, you must include this dependency in your project! -->
<dependency>
<artifactId>FastInfoset</artifactId>
18 core/src/main/java/me/prettyprint/cassandra/connection/ConcurrentHClientPool.java
View
@@ -71,7 +71,23 @@ public HClient borrowClient() throws HectorException {
int currentActiveClients = activeClientsCount.incrementAndGet();
try {
-
+ if (cassandraClient != null) {
+ if (cassandraClient.getCassandraHost().getMaxLastSuccessTimeMillis() > 0
+ && cassandraClient.getLastSuccessTime() > 0
+ && System.currentTimeMillis() - cassandraClient.getLastSuccessTime() > cassandraClient.getCassandraHost().getMaxLastSuccessTimeMillis()) {
+ log.info("Closing connection to {} due to too long idle time of {} ms", cassandraClient.getCassandraHost().getHost(),
+ System.currentTimeMillis() - cassandraClient.getLastSuccessTime());
+ cassandraClient.close();
+ cassandraClient = null;
+ }
+ if (cassandraClient.getCassandraHost().getMaxConnectTimeMillis() > 0
+ && System.currentTimeMillis() - cassandraClient.getCreatedTime() > cassandraClient.getCassandraHost().getMaxConnectTimeMillis()) {
+ log.info("Closing connection to {} due to too long existence time of {} ms", cassandraClient.getCassandraHost().getHost(),
+ System.currentTimeMillis() - cassandraClient.getCreatedTime());
+ cassandraClient.close();
+ cassandraClient = null;
+ }
+ }
if ( cassandraClient == null ) {
if (currentActiveClients <= cassandraHost.getMaxActive()) {
3  core/src/main/java/me/prettyprint/cassandra/connection/HConnectionManager.java
View
@@ -235,7 +235,7 @@ public boolean unsuspendCassandraHost(CassandraHost cassandraHost) {
public void operateWithFailover(Operation<?> op) throws HectorException {
- final Object timerToken = timer.start();
+ final Object timerToken = timer.start(op.stopWatchTagName);
int retries = Math.min(op.failoverPolicy.numRetries, hostPools.size());
HClient client = null;
HClientPool pool = null;
@@ -257,6 +257,7 @@ public void operateWithFailover(Operation<?> op) throws HectorException {
op.executeAndSetResult(c, pool.getCassandraHost());
success = true;
+ client.updateLastSuccessTime();
timer.stop(timerToken, op.stopWatchTagName, true);
break;
2  core/src/main/java/me/prettyprint/cassandra/connection/HOpTimer.java
View
@@ -11,7 +11,7 @@
* @return - a token that will be returned to the timer when stop(...) in
* invoked
*/
- Object start();
+ Object start(String tagName);
/**
*
46 core/src/main/java/me/prettyprint/cassandra/connection/MetricsOpTimer.java
View
@@ -0,0 +1,46 @@
+package me.prettyprint.cassandra.connection;
+
+import java.util.concurrent.TimeUnit;
+
+import com.yammer.metrics.core.MetricName;
+import com.yammer.metrics.core.MetricsRegistry;
+import com.yammer.metrics.core.Timer;
+import com.yammer.metrics.core.TimerContext;
+
+public class MetricsOpTimer implements HOpTimer {
+
+ private static final String TIMER_TYPE = "hector";
+ private final MetricsRegistry metricsRegistry;
+ private final TimeUnit durationUnit;
+ private final TimeUnit rateUnit;
+ private final String clusterName;
+
+ public MetricsOpTimer(final MetricsRegistry metricsRegistry, final String clusterName,
+ final TimeUnit durationUnit, final TimeUnit rateUnit) {
+ this.metricsRegistry = metricsRegistry;
+ this.clusterName = clusterName;
+ this.durationUnit = durationUnit;
+ this.rateUnit = rateUnit;
+ }
+
+ public MetricsOpTimer(final String clusterName) {
+ this.metricsRegistry = new MetricsRegistry();
+ this.clusterName = clusterName;
+ this.durationUnit = TimeUnit.NANOSECONDS;
+ this.rateUnit = TimeUnit.SECONDS;
+ }
+
+ @Override
+ public Object start(final String tagName) {
+ final Timer timer = metricsRegistry.newTimer(new MetricName(clusterName, TIMER_TYPE, tagName),
+ durationUnit, rateUnit);
+ return timer.time();
+ }
+
+ @Override
+ public void stop(final Object token, final String tagName, final boolean success) {
+ final TimerContext timerContext = (TimerContext) token;
+ timerContext.stop();
+ }
+
+}
2  core/src/main/java/me/prettyprint/cassandra/connection/NullOpTimer.java
View
@@ -7,7 +7,7 @@
private static final long serialVersionUID = -4762728985083933452L;
@Override
- public Object start() {
+ public Object start(String tagName) {
return this;
}
2  core/src/main/java/me/prettyprint/cassandra/connection/SpeedForJOpTimer.java
View
@@ -16,7 +16,7 @@ public SpeedForJOpTimer(String clusterName) {
}
@Override
- public Object start() {
+ public Object start(String tagName) {
return stopWatchFactory.getStopWatch();
}
18 core/src/main/java/me/prettyprint/cassandra/connection/client/HClient.java
View
@@ -17,6 +17,12 @@
*
*/
public interface HClient {
+ /**
+ * Returns the time that this HClient was created.
+ *
+ * @return the time this client was created
+ */
+ long getCreatedTime();
/**
* Returns a new Cassandra.Client on each invocation using the underlying
@@ -111,4 +117,16 @@
*/
void clearAuthentication();
+ /**
+ * Retrieves the time of the last success in milliseconds.
+ *
+ * @return -1 if no successful operation has already happened, or the time
+ * of the last success in milliseconds.
+ */
+ long getLastSuccessTime();
+
+ /**
+ * Update the time of the last success with the current time.
+ */
+ void updateLastSuccessTime();
}
26 core/src/main/java/me/prettyprint/cassandra/connection/client/HThriftClient.java
View
@@ -36,6 +36,7 @@
* <p>
*/
public class HThriftClient implements HClient {
+ private long createdTime = System.currentTimeMillis();
private static Logger log = LoggerFactory.getLogger(HThriftClient.class);
@@ -53,6 +54,8 @@
protected TTransport transport;
protected Cassandra.Client cassandraClient;
private TSSLTransportParameters params;
+
+ private volatile long lastSuccessTime;
private final Map<String, String> credentials = new HashMap<String, String>();
@@ -287,4 +290,27 @@ public void setAuthenticated(Map<String, String> credentials) {
clearAuthentication();
this.credentials.putAll(credentials);
}
+
+ /**
+ * {@inheritDoc}
+ */
+ public long getCreatedTime() {
+ return createdTime;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public long getLastSuccessTime() {
+ return lastSuccessTime;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void updateLastSuccessTime() {
+ lastSuccessTime = System.currentTimeMillis();
+ }
}
28 core/src/main/java/me/prettyprint/cassandra/service/CassandraHost.java
View
@@ -40,6 +40,16 @@
public static final long DEFAULT_MAX_WAITTIME_WHEN_EXHAUSTED = -1;
public static final boolean DEFAULT_LIFO = true;
+ /**
+ * The default number of milliseconds (since creation time) we'll allow a connection
+ * to stay open. Default value is negative which means indefinitely.
+ */
+ public static final long DEFAULT_MAX_CONNECT_TIME = -1;
+ /**
+ * The default number of milliseconds (since last success) we'll allow a connection
+ * to stay open. Default value is negative which means indefinitely.
+ */
+ public static final long DEFAULT_MAX_LAST_SUCCESS_TIME = -1;
private final String host, ip, url;
private final int port;
@@ -54,6 +64,8 @@
private boolean useThriftFramedTransport = DEFAULT_USE_FRAMED_THRIFT_TRANSPORT;
private int maxFrameSize = DEFAULT_MAX_FRAME_SIZE;
private boolean useSocketKeepalive;
+ private long maxConnectTimeMillis = DEFAULT_MAX_CONNECT_TIME;
+ private long maxLastSuccessTimeMillis = DEFAULT_MAX_LAST_SUCCESS_TIME;
//TODO(ran): private FailoverPolicy failoverPolicy = DEFAULT_FAILOVER_POLICY;
public CassandraHost(String url) {
@@ -204,5 +216,21 @@ public void setUseSocketKeepalive(boolean useSocketKeepalive) {
this.useSocketKeepalive = useSocketKeepalive;
}
+ public long getMaxConnectTimeMillis() {
+ return this.maxConnectTimeMillis ;
+ }
+
+ public void setMaxConnectTimeMillis(long maxConnectTimeMillis) {
+ this.maxConnectTimeMillis = maxConnectTimeMillis;
+ }
+
+ public long getMaxLastSuccessTimeMillis() {
+ return this.maxLastSuccessTimeMillis;
+ }
+
+ public void setMaxLastSuccessTimeMillis(long maxLastSuccessTimeMillis) {
+ this.maxLastSuccessTimeMillis = maxLastSuccessTimeMillis;
+ }
+
}
36 core/src/main/java/me/prettyprint/cassandra/service/CassandraHostConfigurator.java
View
@@ -51,12 +51,18 @@
private boolean useSocketKeepalive = false;
private HOpTimer opTimer = new NullOpTimer();
private Class<? extends HClientFactory> clientFactoryClass = HThriftClientFactoryImpl.class;
-
+ private long maxConnectTimeMillis = CassandraHost.DEFAULT_MAX_CONNECT_TIME;
+ private long maxLastSuccessTimeMillis = CassandraHost.DEFAULT_MAX_LAST_SUCCESS_TIME;
public CassandraHostConfigurator() {
this.hosts = null;
}
+ /**
+ * Creates a new {@code CassandraHostConfigurator} from the specified hosts String, formatted as
+ * {@code host[:port][,host[:port]...]}.
+ * @param hosts The hosts to create {@link CassandraHost}s from.
+ */
public CassandraHostConfigurator(String hosts) {
this.hosts = hosts;
}
@@ -83,6 +89,8 @@ public void applyConfig(CassandraHost cassandraHost) {
cassandraHost.setUseThriftFramedTransport(useThriftFramedTransport);
cassandraHost.setMaxFrameSize(maxFrameSize);
cassandraHost.setUseSocketKeepalive(useSocketKeepalive);
+ cassandraHost.setMaxConnectTimeMillis(maxConnectTimeMillis);
+ cassandraHost.setMaxLastSuccessTimeMillis(maxLastSuccessTimeMillis);
// this is special as it can be passed in as a system property
if (cassandraThriftSocketTimeout > 0) {
@@ -90,6 +98,11 @@ public void applyConfig(CassandraHost cassandraHost) {
}
}
+ /**
+ * Specifies the hosts String, formatted as
+ * {@code host[:port][,host[:port]...]}.
+ * @param hosts The hosts to create {@link CassandraHost}s from.
+ */
public void setHosts(String hosts) {
this.hosts = hosts;
}
@@ -351,4 +364,25 @@ public void setClientFactoryClass(String cls) {
public Class<? extends HClientFactory> getClientFactoryClass() {
return clientFactoryClass;
}
+
+ /**
+ * The maximum time in milliseconds that we'll allow a connection to stay open to a host. A negative
+ * value indicates indefinitely (and is the default).
+ *
+ * @return the number of milliseconds
+ */
+ public long getMaxConnectTimeMillis() {
+ return maxConnectTimeMillis;
+ }
+
+ /**
+ * Set the maximum time in milliseconds that we'll allow a connection to stay open to a host. A negative
+ * value indicates indefinitely. This setting is useful if you you need to work around a firewall that
+ * forcefully closes connections after a fixed amount of time regardless of activity.
+ *
+ * @param maxConnectTimeMillis the maximum time to use a connection
+ */
+ public void setMaxConnectTimeMillis(long maxConnectTimeMillis) {
+ this.maxConnectTimeMillis = maxConnectTimeMillis;
+ }
}
70 core/src/main/java/me/prettyprint/cassandra/service/template/ColumnFamilyResultIterator.java
View
@@ -0,0 +1,70 @@
+/**
+ * This class will instill 'normal' iterator behavior to a ColumnFamilyResult.
+ * Simply instantiate this class while passing your ColumnFamilyResult as a
+ * constructor argument.
+ *
+ * Ex.
+ *
+ * ColumnFamilyResultIterator myResultsInterator =
+ * new ColumnFamilyResultIterator(someColumnFamilyResult);
+ *
+ * You can then use myResultsInterator with for loops or iterate with a while loop
+ * just as with any standard java iterator.
+ *
+ */
+package me.prettyprint.cassandra.service.template;
+
+import java.util.Iterator;
+
+import me.prettyprint.cassandra.service.template.ColumnFamilyResult;
+
+public class ColumnFamilyResultIterator implements Iterator<ColumnFamilyResult<?,?>> {
+ private ColumnFamilyResult<?, ?> res;
+ private boolean isStart = true;
+
+ public ColumnFamilyResultIterator(ColumnFamilyResult<?, ?> res) {
+ this.res = res;
+ }
+
+ public boolean hasNext()
+ {
+ boolean retval = false;
+ if (isStart)
+ {
+ retval = res.hasResults();
+ }
+ else
+ {
+ retval = res.hasNext();
+ }
+ return retval;
+ }
+
+ public ColumnFamilyResult<?, ?> getRes()
+ {
+ return res;
+ }
+
+ public void setRes(ColumnFamilyResult<?, ?> res)
+ {
+ this.res = res;
+ }
+
+ public ColumnFamilyResult<?, ?> next()
+ {
+ if (isStart)
+ {
+ isStart = false;
+ return res;
+ }
+ else
+ {
+ return (ColumnFamilyResult<?, ?>) res.next();
+ }
+ }
+
+ public void remove()
+ {
+ res.remove();
+ }
+}
2  core/src/main/java/me/prettyprint/hector/api/beans/AbstractComposite.java
View
@@ -81,6 +81,8 @@ public static ComponentEquality fromByte(byte equality) {
AsciiSerializer.get().getComparatorType().getTypeName())
.put(BigIntegerSerializer.class,
BigIntegerSerializer.get().getComparatorType().getTypeName())
+ .put(ByteBufferSerializer.class,
+ ByteBufferSerializer.get().getComparatorType().getTypeName())
.put(LongSerializer.class,
LongSerializer.get().getComparatorType().getTypeName())
.put(StringSerializer.class,
8 core/src/main/java/me/prettyprint/hector/api/ddl/ColumnFamilyDefinition.java
View
@@ -28,6 +28,14 @@
void setSubComparatorType(ComparatorType subComparitorType);
String getComparatorTypeAlias();
+
+ /**
+ * Sets the type alias for the comparator to be used for the row keys of the column family.
+ * For composite types, supply the alias in the following format:
+ * {@code (TypeName1, TypeName2, ...)}.
+ * @param alias An alias String defining the comparator to be used for the row keys.
+ * @see <a href="http://www.datastax.com/docs/1.1/ddl/column_family#about-data-types-comparators-and-validators">DataStax column family reference</a>
+ */
void setComparatorTypeAlias(String alias);
String getSubComparatorTypeAlias();
59 core/src/test/java/me/prettyprint/cassandra/connection/HConnectionManagerMetricsOpTimerTest.java
View
@@ -0,0 +1,59 @@
+package me.prettyprint.cassandra.connection;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+
+import java.util.Map.Entry;
+import java.util.SortedMap;
+import java.util.concurrent.TimeUnit;
+
+import me.prettyprint.cassandra.BaseEmbededServerSetupTest;
+import me.prettyprint.cassandra.service.Operation;
+import me.prettyprint.cassandra.service.OperationType;
+import me.prettyprint.hector.api.exceptions.HectorException;
+
+import org.apache.cassandra.thrift.Cassandra.Client;
+import org.junit.Test;
+
+import com.yammer.metrics.core.Metric;
+import com.yammer.metrics.core.MetricName;
+import com.yammer.metrics.core.MetricsRegistry;
+import com.yammer.metrics.core.Timer;
+
+public class HConnectionManagerMetricsOpTimerTest extends BaseEmbededServerSetupTest {
+
+ @Test
+ public void testWithOptimer() {
+ setupClient();
+ final MetricsRegistry registry = new MetricsRegistry();
+ final MetricsOpTimer opTimer = new MetricsOpTimer(registry, "TEST_CLUSTER",
+ TimeUnit.NANOSECONDS, TimeUnit.SECONDS);
+ connectionManager.setTimer(opTimer);
+ connectionManager.operateWithFailover(new NullOp());
+
+ final SortedMap<String, SortedMap<MetricName, Metric>> metrics = registry.groupedMetrics();
+ assertNotNull("Hector metric should exist in metrics registry", metrics);
+ assertFalse("Hector metrics should exist in metrics register", metrics.isEmpty());
+ final Entry<String, SortedMap<MetricName, Metric>> entry = metrics.entrySet().iterator().next();
+ assertEquals("Incorrect metrics key should be [cluster_name].hector", "TEST_CLUSTER.hector",
+ entry.getKey());
+ final Entry<MetricName, Metric> metric = entry.getValue().entrySet().iterator().next();
+ assertEquals("Incorrect metrics name should be META_READ", "META_READ", metric.getKey()
+ .getName());
+ assertEquals("Incorrect metrics type should be timer", Timer.class, metric.getValue()
+ .getClass());
+ }
+
+ class NullOp extends Operation<String> {
+
+ NullOp() {
+ super(OperationType.META_READ);
+ }
+
+ @Override
+ public String execute(final Client cassandra) throws HectorException {
+ return null;
+ }
+ }
+}
7 pom.xml
View
@@ -126,7 +126,7 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>r09</version>
+ <version>14.0</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
@@ -183,6 +183,11 @@
<version>0.9</version>
</dependency>
<dependency>
+ <groupId>com.yammer.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>2.2.0</version>
+ </dependency>
+ <dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>1.6.1</version>
Please sign in to comment.
Something went wrong with that request. Please try again.