Skip to content

Commit

Permalink
some minor updates to run/test against C* 2.2.0
Browse files Browse the repository at this point in the history
With this commit, everyone should be using Cassandra 2.2.0 in their local dev
environments.

In additon to having Travis use C* 2.2.0, I had to make some changes around
batch statements. When testing locally with a 2.2.0 cluster, I was getting some
failures about exceeding the batch_size_fail_threshold_in_kb setting defined
in cassandra.yaml. It defaults to 50 KB. BatchStatementTransformer had a check
on the number of statements in a batch. This is not really useful though
because the check needs to be on the overall size, not the number of statements
in the batch. I am going to create a ticket to address our usage of batch
statements. The conventional wisdom in general is to avoid them, and it makes
sense for adhere to that.
  • Loading branch information
John Sanda committed Aug 27, 2015
1 parent 780823c commit 307975d
Show file tree
Hide file tree
Showing 6 changed files with 20 additions and 28 deletions.
2 changes: 1 addition & 1 deletion .travis.install.cassandra.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ cd "${HOME}"
wget -O datastax-releases "http://downloads.datastax.com/community/?C=M;O=D"
LATEST_MICRO=`cat datastax-releases | grep -m 1 -o -E "dsc-cassandra-2\.1\.[0-9]+-bin\.tar.gz" | head -1 | cut -d '.' -f3 | cut -d '-' -f1`

CASSANDRA_VERSION="2.1.${LATEST_MICRO}"
CASSANDRA_VERSION="2.2.${LATEST_MICRO}"
CASSANDRA_BINARY="dsc-cassandra-${CASSANDRA_VERSION}-bin.tar.gz"
CASSANDRA_DOWNLOADS="${HOME}/cassandra-downloads"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
package org.hawkular.metrics.core.impl.transformers;

import static com.datastax.driver.core.BatchStatement.Type.UNLOGGED;
import static com.google.common.base.Preconditions.checkArgument;

import com.datastax.driver.core.BatchStatement;
import com.datastax.driver.core.Statement;
Expand All @@ -33,7 +32,8 @@
*/
public class BatchStatementTransformer implements Transformer<Statement, BatchStatement> {
// Max batch size is 0xFFFF (greatest unsigned short)
public static final int MAX_BATCH_SIZE = 0xFFFF;
// public static final int MAX_BATCH_SIZE = 0xFFFF;
public static final int MAX_BATCH_SIZE = 1024;

/**
* Creates {@link com.datastax.driver.core.BatchStatement.Type#UNLOGGED} batch statements.
Expand All @@ -56,14 +56,15 @@ public BatchStatementTransformer() {
*/
public BatchStatementTransformer(Func0<BatchStatement> batchStatementFactory, int batchSize) {
this.batchSize = batchSize;
checkArgument(batchSize <= MAX_BATCH_SIZE, "batchSize exceeds limit");
// checkArgument(batchSize <= MAX_BATCH_SIZE, "batchSize exceeds limit");
this.batchStatementFactory = batchStatementFactory;
}

@Override
public Observable<BatchStatement> call(Observable<Statement> statements) {
return statements
.window(batchSize)
.flatMap(window -> window.collect(batchStatementFactory, BatchStatement::add));
// return statements
// .window(batchSize)
// .flatMap(window -> window.collect(batchStatementFactory, BatchStatement::add));
return statements.collect(batchStatementFactory, BatchStatement::add);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -48,16 +48,15 @@ public void setUp() throws Exception {
@Test
public void testCall() throws Exception {
int expected = 6;
int numStatements = (expected - 1) * batchSize + 1;
// Emit enough statements to get expected count of batches, with the last batch holding just one
List<BatchStatement> result = Observable.range(0, (expected - 1) * batchSize + 1)
List<BatchStatement> result = Observable.range(0, numStatements)
.map(i -> mock(Statement.class))
.compose(batchStatementTransformer)
.toList()
.toBlocking()
.single();
assertEquals(expected, result.size());
for (int i = 0; i < result.size(); i++) {
assertEquals(i < (result.size() - 1) ? batchSize : 1, result.get(i).size());
}
assertEquals(1, result.size());
assertEquals(numStatements, result.get(0).size());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
*/
package org.hawkular.metrics.rest

import groovy.json.JsonOutput
import org.hawkular.metrics.core.impl.DateTimeService
import org.joda.time.DateTime
import org.junit.Test
Expand Down Expand Up @@ -181,7 +180,7 @@ class CountersITest extends RESTTest {
}

@Test
void addDataForMultipleCountersAndFindhWithDateRange() {
void addDataForMultipleCountersAndFindWithDateRange() {
String tenantId = nextTenantId()
String counter1 = "C1"
String counter2 = "C2"
Expand Down Expand Up @@ -368,8 +367,6 @@ class CountersITest extends RESTTest {
]
]

println "RESPONSE = ${JsonOutput.toJson(response.data)}"

assertEquals("Expected to get back three data points", 5, response.data.size())
assertRateEquals(expectedData[0], response.data[0])
assertRateEquals(expectedData[1], response.data[1])
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ class GaugeMetricStatisticsITest extends RESTTest {
String metric = "test"
int nbOfBuckets = 10
long bucketSize = Duration.standardDays(1).millis
int interval = Duration.standardMinutes(1).millis
int interval = Duration.standardMinutes(10).millis
int sampleSize = (bucketSize / interval) - 1

def start = new DateTimeService().currentHour().minus(3 * nbOfBuckets * bucketSize)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,28 +15,23 @@
* limitations under the License.
*/
package org.hawkular.metrics.rest

import com.google.common.base.Charsets
import groovyx.net.http.ContentType
import groovyx.net.http.RESTClient
import org.joda.time.DateTime

import static org.junit.Assert.assertEquals

import java.util.concurrent.atomic.AtomicInteger

import org.hawkular.metrics.core.impl.transformers.BatchStatementTransformer
import org.junit.BeforeClass

import com.google.common.base.Charsets
import java.util.concurrent.atomic.AtomicInteger

import groovyx.net.http.ContentType
import groovyx.net.http.RESTClient
import static org.junit.Assert.assertEquals

class RESTTest {

static baseURI = System.getProperty('hawkular-metrics.base-uri') ?: '127.0.0.1:8080/hawkular/metrics'
static final double DELTA = 0.001
static final String TENANT_PREFIX = UUID.randomUUID().toString()
static final AtomicInteger TENANT_ID_COUNTER = new AtomicInteger(0)
static final int LARGE_PAYLOAD_SIZE = BatchStatementTransformer.MAX_BATCH_SIZE
static final int LARGE_PAYLOAD_SIZE = 1024
static String tenantHeaderName = "Hawkular-Tenant";
static RESTClient hawkularMetrics
static defaultFailureHandler
Expand Down

0 comments on commit 307975d

Please sign in to comment.