Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Merge remote-tracking branch 'origin/master' into SliceFilter

  • Loading branch information...
commit 66a10cc604d4357cea5c668d537c88fcfd64d640 2 parents 520c640 + 21ac921
Shane Perry authored
Showing with 325 additions and 8 deletions.
  1. +1 −1  README
  2. +4 −0 core/pom.xml
  3. +17 −1 core/src/main/java/me/prettyprint/cassandra/connection/ConcurrentHClientPool.java
  4. +2 −1  core/src/main/java/me/prettyprint/cassandra/connection/HConnectionManager.java
  5. +1 −1  core/src/main/java/me/prettyprint/cassandra/connection/HOpTimer.java
  6. +46 −0 core/src/main/java/me/prettyprint/cassandra/connection/MetricsOpTimer.java
  7. +1 −1  core/src/main/java/me/prettyprint/cassandra/connection/NullOpTimer.java
  8. +1 −1  core/src/main/java/me/prettyprint/cassandra/connection/SpeedForJOpTimer.java
  9. +18 −0 core/src/main/java/me/prettyprint/cassandra/connection/client/HClient.java
  10. +26 −0 core/src/main/java/me/prettyprint/cassandra/connection/client/HThriftClient.java
  11. +28 −0 core/src/main/java/me/prettyprint/cassandra/service/CassandraHost.java
  12. +35 −1 core/src/main/java/me/prettyprint/cassandra/service/CassandraHostConfigurator.java
  13. +70 −0 core/src/main/java/me/prettyprint/cassandra/service/template/ColumnFamilyResultIterator.java
  14. +2 −0  core/src/main/java/me/prettyprint/hector/api/beans/AbstractComposite.java
  15. +8 −0 core/src/main/java/me/prettyprint/hector/api/ddl/ColumnFamilyDefinition.java
  16. +59 −0 core/src/test/java/me/prettyprint/cassandra/connection/HConnectionManagerMetricsOpTimerTest.java
  17. +6 −1 pom.xml
View
2  README
@@ -12,7 +12,7 @@ Hector is the greatest warrior in the greek mythology, Troy's builder and brothe
http://en.wikipedia.org/wiki/Hector
http://en.wikipedia.org/wiki/Cassandra
-Hector is currently in use on a number of production systems some of which have node counts into the hundreds. Issues generally are fixed as quickly as possbile and releases done frequently.
+Hector is currently in use on a number of production systems some of which have node counts into the hundreds. Issues generally are fixed as quickly as possible and releases done frequently.
Some features provided by this client:
View
4 core/pom.xml
@@ -122,6 +122,10 @@
<groupId>com.ecyrd.speed4j</groupId>
<artifactId>speed4j</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.yammer.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ </dependency>
<!-- Note the optional tag. If you need to use fastinfoset serialization, you must include this dependency in your project! -->
<dependency>
<artifactId>FastInfoset</artifactId>
View
18 core/src/main/java/me/prettyprint/cassandra/connection/ConcurrentHClientPool.java
@@ -71,7 +71,23 @@ public HClient borrowClient() throws HectorException {
int currentActiveClients = activeClientsCount.incrementAndGet();
try {
-
+ if (cassandraClient != null) {
+ if (cassandraClient.getCassandraHost().getMaxLastSuccessTimeMillis() > 0
+ && cassandraClient.getLastSuccessTime() > 0
+ && System.currentTimeMillis() - cassandraClient.getLastSuccessTime() > cassandraClient.getCassandraHost().getMaxLastSuccessTimeMillis()) {
+ log.info("Closing connection to {} due to too long idle time of {} ms", cassandraClient.getCassandraHost().getHost(),
+ System.currentTimeMillis() - cassandraClient.getLastSuccessTime());
+ cassandraClient.close();
+ cassandraClient = null;
+ }
+ if (cassandraClient.getCassandraHost().getMaxConnectTimeMillis() > 0
+ && System.currentTimeMillis() - cassandraClient.getCreatedTime() > cassandraClient.getCassandraHost().getMaxConnectTimeMillis()) {
+ log.info("Closing connection to {} due to too long existence time of {} ms", cassandraClient.getCassandraHost().getHost(),
+ System.currentTimeMillis() - cassandraClient.getCreatedTime());
+ cassandraClient.close();
+ cassandraClient = null;
+ }
+ }
if ( cassandraClient == null ) {
if (currentActiveClients <= cassandraHost.getMaxActive()) {
View
3  core/src/main/java/me/prettyprint/cassandra/connection/HConnectionManager.java
@@ -235,7 +235,7 @@ public boolean unsuspendCassandraHost(CassandraHost cassandraHost) {
public void operateWithFailover(Operation<?> op) throws HectorException {
- final Object timerToken = timer.start();
+ final Object timerToken = timer.start(op.stopWatchTagName);
int retries = Math.min(op.failoverPolicy.numRetries, hostPools.size());
HClient client = null;
HClientPool pool = null;
@@ -257,6 +257,7 @@ public void operateWithFailover(Operation<?> op) throws HectorException {
op.executeAndSetResult(c, pool.getCassandraHost());
success = true;
+ client.updateLastSuccessTime();
timer.stop(timerToken, op.stopWatchTagName, true);
break;
View
2  core/src/main/java/me/prettyprint/cassandra/connection/HOpTimer.java
@@ -11,7 +11,7 @@
* @return - a token that will be returned to the timer when stop(...) in
* invoked
*/
- Object start();
+ Object start(String tagName);
/**
*
View
46 core/src/main/java/me/prettyprint/cassandra/connection/MetricsOpTimer.java
@@ -0,0 +1,46 @@
+package me.prettyprint.cassandra.connection;
+
+import java.util.concurrent.TimeUnit;
+
+import com.yammer.metrics.core.MetricName;
+import com.yammer.metrics.core.MetricsRegistry;
+import com.yammer.metrics.core.Timer;
+import com.yammer.metrics.core.TimerContext;
+
+public class MetricsOpTimer implements HOpTimer {
+
+ private static final String TIMER_TYPE = "hector";
+ private final MetricsRegistry metricsRegistry;
+ private final TimeUnit durationUnit;
+ private final TimeUnit rateUnit;
+ private final String clusterName;
+
+ public MetricsOpTimer(final MetricsRegistry metricsRegistry, final String clusterName,
+ final TimeUnit durationUnit, final TimeUnit rateUnit) {
+ this.metricsRegistry = metricsRegistry;
+ this.clusterName = clusterName;
+ this.durationUnit = durationUnit;
+ this.rateUnit = rateUnit;
+ }
+
+ public MetricsOpTimer(final String clusterName) {
+ this.metricsRegistry = new MetricsRegistry();
+ this.clusterName = clusterName;
+ this.durationUnit = TimeUnit.NANOSECONDS;
+ this.rateUnit = TimeUnit.SECONDS;
+ }
+
+ @Override
+ public Object start(final String tagName) {
+ final Timer timer = metricsRegistry.newTimer(new MetricName(clusterName, TIMER_TYPE, tagName),
+ durationUnit, rateUnit);
+ return timer.time();
+ }
+
+ @Override
+ public void stop(final Object token, final String tagName, final boolean success) {
+ final TimerContext timerContext = (TimerContext) token;
+ timerContext.stop();
+ }
+
+}
View
2  core/src/main/java/me/prettyprint/cassandra/connection/NullOpTimer.java
@@ -7,7 +7,7 @@
private static final long serialVersionUID = -4762728985083933452L;
@Override
- public Object start() {
+ public Object start(String tagName) {
return this;
}
View
2  core/src/main/java/me/prettyprint/cassandra/connection/SpeedForJOpTimer.java
@@ -16,7 +16,7 @@ public SpeedForJOpTimer(String clusterName) {
}
@Override
- public Object start() {
+ public Object start(String tagName) {
return stopWatchFactory.getStopWatch();
}
View
18 core/src/main/java/me/prettyprint/cassandra/connection/client/HClient.java
@@ -17,6 +17,12 @@
*
*/
public interface HClient {
+ /**
+ * Returns the time that this HClient was created.
+ *
+ * @return the time this client was created
+ */
+ long getCreatedTime();
/**
* Returns a new Cassandra.Client on each invocation using the underlying
@@ -111,4 +117,16 @@
*/
void clearAuthentication();
+ /**
+ * Retrieves the time of the last success in milliseconds.
+ *
+ * @return -1 if no successful operation has already happened, or the time
+ * of the last success in milliseconds.
+ */
+ long getLastSuccessTime();
+
+ /**
+ * Update the time of the last success with the current time.
+ */
+ void updateLastSuccessTime();
}
View
26 core/src/main/java/me/prettyprint/cassandra/connection/client/HThriftClient.java
@@ -36,6 +36,7 @@
* <p>
*/
public class HThriftClient implements HClient {
+ private long createdTime = System.currentTimeMillis();
private static Logger log = LoggerFactory.getLogger(HThriftClient.class);
@@ -53,6 +54,8 @@
protected TTransport transport;
protected Cassandra.Client cassandraClient;
private TSSLTransportParameters params;
+
+ private volatile long lastSuccessTime;
private final Map<String, String> credentials = new HashMap<String, String>();
@@ -287,4 +290,27 @@ public void setAuthenticated(Map<String, String> credentials) {
clearAuthentication();
this.credentials.putAll(credentials);
}
+
+ /**
+ * {@inheritDoc}
+ */
+ public long getCreatedTime() {
+ return createdTime;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public long getLastSuccessTime() {
+ return lastSuccessTime;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void updateLastSuccessTime() {
+ lastSuccessTime = System.currentTimeMillis();
+ }
}
View
28 core/src/main/java/me/prettyprint/cassandra/service/CassandraHost.java
@@ -40,6 +40,16 @@
public static final long DEFAULT_MAX_WAITTIME_WHEN_EXHAUSTED = -1;
public static final boolean DEFAULT_LIFO = true;
+ /**
+ * The default number of milliseconds (since creation time) we'll allow a connection
+ * to stay open. Default value is negative which means indefinitely.
+ */
+ public static final long DEFAULT_MAX_CONNECT_TIME = -1;
+ /**
+ * The default number of milliseconds (since last success) we'll allow a connection
+ * to stay open. Default value is negative which means indefinitely.
+ */
+ public static final long DEFAULT_MAX_LAST_SUCCESS_TIME = -1;
private final String host, ip, url;
private final int port;
@@ -54,6 +64,8 @@
private boolean useThriftFramedTransport = DEFAULT_USE_FRAMED_THRIFT_TRANSPORT;
private int maxFrameSize = DEFAULT_MAX_FRAME_SIZE;
private boolean useSocketKeepalive;
+ private long maxConnectTimeMillis = DEFAULT_MAX_CONNECT_TIME;
+ private long maxLastSuccessTimeMillis = DEFAULT_MAX_LAST_SUCCESS_TIME;
//TODO(ran): private FailoverPolicy failoverPolicy = DEFAULT_FAILOVER_POLICY;
public CassandraHost(String url) {
@@ -204,5 +216,21 @@ public void setUseSocketKeepalive(boolean useSocketKeepalive) {
this.useSocketKeepalive = useSocketKeepalive;
}
+ public long getMaxConnectTimeMillis() {
+ return this.maxConnectTimeMillis ;
+ }
+
+ public void setMaxConnectTimeMillis(long maxConnectTimeMillis) {
+ this.maxConnectTimeMillis = maxConnectTimeMillis;
+ }
+
+ public long getMaxLastSuccessTimeMillis() {
+ return this.maxLastSuccessTimeMillis;
+ }
+
+ public void setMaxLastSuccessTimeMillis(long maxLastSuccessTimeMillis) {
+ this.maxLastSuccessTimeMillis = maxLastSuccessTimeMillis;
+ }
+
}
View
36 core/src/main/java/me/prettyprint/cassandra/service/CassandraHostConfigurator.java
@@ -51,12 +51,18 @@
private boolean useSocketKeepalive = false;
private HOpTimer opTimer = new NullOpTimer();
private Class<? extends HClientFactory> clientFactoryClass = HThriftClientFactoryImpl.class;
-
+ private long maxConnectTimeMillis = CassandraHost.DEFAULT_MAX_CONNECT_TIME;
+ private long maxLastSuccessTimeMillis = CassandraHost.DEFAULT_MAX_LAST_SUCCESS_TIME;
public CassandraHostConfigurator() {
this.hosts = null;
}
+ /**
+ * Creates a new {@code CassandraHostConfigurator} from the specified hosts String, formatted as
+ * {@code host[:port][,host[:port]...]}.
+ * @param hosts The hosts to create {@link CassandraHost}s from.
+ */
public CassandraHostConfigurator(String hosts) {
this.hosts = hosts;
}
@@ -83,6 +89,8 @@ public void applyConfig(CassandraHost cassandraHost) {
cassandraHost.setUseThriftFramedTransport(useThriftFramedTransport);
cassandraHost.setMaxFrameSize(maxFrameSize);
cassandraHost.setUseSocketKeepalive(useSocketKeepalive);
+ cassandraHost.setMaxConnectTimeMillis(maxConnectTimeMillis);
+ cassandraHost.setMaxLastSuccessTimeMillis(maxLastSuccessTimeMillis);
// this is special as it can be passed in as a system property
if (cassandraThriftSocketTimeout > 0) {
@@ -90,6 +98,11 @@ public void applyConfig(CassandraHost cassandraHost) {
}
}
+ /**
+ * Specifies the hosts String, formatted as
+ * {@code host[:port][,host[:port]...]}.
+ * @param hosts The hosts to create {@link CassandraHost}s from.
+ */
public void setHosts(String hosts) {
this.hosts = hosts;
}
@@ -351,4 +364,25 @@ public void setClientFactoryClass(String cls) {
public Class<? extends HClientFactory> getClientFactoryClass() {
return clientFactoryClass;
}
+
+ /**
+ * The maximum time in milliseconds that we'll allow a connection to stay open to a host. A negative
+ * value indicates indefinitely (and is the default).
+ *
+ * @return the number of milliseconds
+ */
+ public long getMaxConnectTimeMillis() {
+ return maxConnectTimeMillis;
+ }
+
+ /**
+ * Set the maximum time in milliseconds that we'll allow a connection to stay open to a host. A negative
+ * value indicates indefinitely. This setting is useful if you you need to work around a firewall that
+ * forcefully closes connections after a fixed amount of time regardless of activity.
+ *
+ * @param maxConnectTimeMillis the maximum time to use a connection
+ */
+ public void setMaxConnectTimeMillis(long maxConnectTimeMillis) {
+ this.maxConnectTimeMillis = maxConnectTimeMillis;
+ }
}
View
70 core/src/main/java/me/prettyprint/cassandra/service/template/ColumnFamilyResultIterator.java
@@ -0,0 +1,70 @@
+/**
+ * This class will instill 'normal' iterator behavior to a ColumnFamilyResult.
+ * Simply instantiate this class while passing your ColumnFamilyResult as a
+ * constructor argument.
+ *
+ * Ex.
+ *
+ * ColumnFamilyResultIterator myResultsInterator =
+ * new ColumnFamilyResultIterator(someColumnFamilyResult);
+ *
+ * You can then use myResultsInterator with for loops or iterate with a while loop
+ * just as with any standard java iterator.
+ *
+ */
+package me.prettyprint.cassandra.service.template;
+
+import java.util.Iterator;
+
+import me.prettyprint.cassandra.service.template.ColumnFamilyResult;
+
+public class ColumnFamilyResultIterator implements Iterator<ColumnFamilyResult<?,?>> {
+ private ColumnFamilyResult<?, ?> res;
+ private boolean isStart = true;
+
+ public ColumnFamilyResultIterator(ColumnFamilyResult<?, ?> res) {
+ this.res = res;
+ }
+
+ public boolean hasNext()
+ {
+ boolean retval = false;
+ if (isStart)
+ {
+ retval = res.hasResults();
+ }
+ else
+ {
+ retval = res.hasNext();
+ }
+ return retval;
+ }
+
+ public ColumnFamilyResult<?, ?> getRes()
+ {
+ return res;
+ }
+
+ public void setRes(ColumnFamilyResult<?, ?> res)
+ {
+ this.res = res;
+ }
+
+ public ColumnFamilyResult<?, ?> next()
+ {
+ if (isStart)
+ {
+ isStart = false;
+ return res;
+ }
+ else
+ {
+ return (ColumnFamilyResult<?, ?>) res.next();
+ }
+ }
+
+ public void remove()
+ {
+ res.remove();
+ }
+}
View
2  core/src/main/java/me/prettyprint/hector/api/beans/AbstractComposite.java
@@ -81,6 +81,8 @@ public static ComponentEquality fromByte(byte equality) {
AsciiSerializer.get().getComparatorType().getTypeName())
.put(BigIntegerSerializer.class,
BigIntegerSerializer.get().getComparatorType().getTypeName())
+ .put(ByteBufferSerializer.class,
+ ByteBufferSerializer.get().getComparatorType().getTypeName())
.put(LongSerializer.class,
LongSerializer.get().getComparatorType().getTypeName())
.put(StringSerializer.class,
View
8 core/src/main/java/me/prettyprint/hector/api/ddl/ColumnFamilyDefinition.java
@@ -28,6 +28,14 @@
void setSubComparatorType(ComparatorType subComparitorType);
String getComparatorTypeAlias();
+
+ /**
+ * Sets the type alias for the comparator to be used for the row keys of the column family.
+ * For composite types, supply the alias in the following format:
+ * {@code (TypeName1, TypeName2, ...)}.
+ * @param alias An alias String defining the comparator to be used for the row keys.
+ * @see <a href="http://www.datastax.com/docs/1.1/ddl/column_family#about-data-types-comparators-and-validators">DataStax column family reference</a>
+ */
void setComparatorTypeAlias(String alias);
String getSubComparatorTypeAlias();
View
59 core/src/test/java/me/prettyprint/cassandra/connection/HConnectionManagerMetricsOpTimerTest.java
@@ -0,0 +1,59 @@
+package me.prettyprint.cassandra.connection;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+
+import java.util.Map.Entry;
+import java.util.SortedMap;
+import java.util.concurrent.TimeUnit;
+
+import me.prettyprint.cassandra.BaseEmbededServerSetupTest;
+import me.prettyprint.cassandra.service.Operation;
+import me.prettyprint.cassandra.service.OperationType;
+import me.prettyprint.hector.api.exceptions.HectorException;
+
+import org.apache.cassandra.thrift.Cassandra.Client;
+import org.junit.Test;
+
+import com.yammer.metrics.core.Metric;
+import com.yammer.metrics.core.MetricName;
+import com.yammer.metrics.core.MetricsRegistry;
+import com.yammer.metrics.core.Timer;
+
+public class HConnectionManagerMetricsOpTimerTest extends BaseEmbededServerSetupTest {
+
+ @Test
+ public void testWithOptimer() {
+ setupClient();
+ final MetricsRegistry registry = new MetricsRegistry();
+ final MetricsOpTimer opTimer = new MetricsOpTimer(registry, "TEST_CLUSTER",
+ TimeUnit.NANOSECONDS, TimeUnit.SECONDS);
+ connectionManager.setTimer(opTimer);
+ connectionManager.operateWithFailover(new NullOp());
+
+ final SortedMap<String, SortedMap<MetricName, Metric>> metrics = registry.groupedMetrics();
+ assertNotNull("Hector metric should exist in metrics registry", metrics);
+ assertFalse("Hector metrics should exist in metrics register", metrics.isEmpty());
+ final Entry<String, SortedMap<MetricName, Metric>> entry = metrics.entrySet().iterator().next();
+ assertEquals("Incorrect metrics key should be [cluster_name].hector", "TEST_CLUSTER.hector",
+ entry.getKey());
+ final Entry<MetricName, Metric> metric = entry.getValue().entrySet().iterator().next();
+ assertEquals("Incorrect metrics name should be META_READ", "META_READ", metric.getKey()
+ .getName());
+ assertEquals("Incorrect metrics type should be timer", Timer.class, metric.getValue()
+ .getClass());
+ }
+
+ class NullOp extends Operation<String> {
+
+ NullOp() {
+ super(OperationType.META_READ);
+ }
+
+ @Override
+ public String execute(final Client cassandra) throws HectorException {
+ return null;
+ }
+ }
+}
View
7 pom.xml
@@ -126,7 +126,7 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>r09</version>
+ <version>14.0</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
@@ -183,6 +183,11 @@
<version>0.9</version>
</dependency>
<dependency>
+ <groupId>com.yammer.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>2.2.0</version>
+ </dependency>
+ <dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>1.6.1</version>
Please sign in to comment.
Something went wrong with that request. Please try again.