Skip to content

Commit

Permalink
use termsfilter for IN and ANY in COUNT and DELETE queries
Browse files Browse the repository at this point in the history
  • Loading branch information
@msbt
msbt committed Apr 23, 2015
1 parent 42b637a commit d5048e0
Show file tree
Hide file tree
Showing 6 changed files with 292 additions and 81 deletions.
27 changes: 19 additions & 8 deletions sql/src/main/java/io/crate/executor/transport/task/elasticsearch/ESQueryBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -635,13 +635,17 @@ public boolean convertArrayReference(Reference arrayReference, Literal literal,

@Override
public boolean convertArrayLiteral(Reference reference, Literal arrayLiteral, Context context) throws IOException {
String refName = reference.info().ident().columnIdent().fqn();
context.builder.startObject("terms").field(refName);
String columnName = reference.info().ident().columnIdent().fqn();
context.builder.startObject(Fields.FILTERED);
context.builder.startObject(Fields.QUERY)
.startObject(Fields.MATCH_ALL).endObject()
.endObject();
context.builder.startObject(Fields.FILTER).startObject("terms").field(columnName);
context.builder.startArray();
for (Object value: toIterable(arrayLiteral.value())){
context.builder.value(value);
}
context.builder.endArray().endObject();
context.builder.endArray().endObject().endObject().endObject();
return true;
}
}
Expand Down Expand Up @@ -681,16 +685,23 @@ public boolean convertArrayReference(Reference arrayReference, Literal literal,
public boolean convertArrayLiteral(Reference reference, Literal arrayLiteral, Context context) throws IOException {
// col != ANY ([1,2,3]) --> not(col=1 and col=2 and col=3)
String columnName = reference.info().ident().columnIdent().fqn();
context.builder.startObject("bool").startObject("must_not")
.startObject("bool").startArray("must");
context.builder.startObject(Fields.FILTERED)
.startObject(Fields.QUERY)
.startObject(Fields.MATCH_ALL).endObject()
.endObject()
.startObject(Fields.FILTER)
.startObject("bool").startObject("must_not")
.startObject("bool")
.startArray("must");
for (Object value: toIterable(arrayLiteral.value())) {
context.builder.startObject()
.startObject("term").field(columnName, value).endObject()
.endObject();
}
context.builder.endArray().endObject();

context.builder.endObject().endObject();
context.builder.endArray()
.endObject()
.endObject().endObject()
.endObject().endObject();
return true;
}
}
Expand Down
31 changes: 17 additions & 14 deletions sql/src/main/java/io/crate/lucene/LuceneQueryBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -269,30 +269,33 @@ public Object apply(@javax.annotation.Nullable Object input) {

static class AnyEqQuery extends AbstractAnyQuery {

@Override
protected Query applyArrayReference(Reference arrayReference, Literal literal, Context context) throws IOException {
QueryBuilderHelper builder = QueryBuilderHelper.forType(((CollectionType)arrayReference.valueType()).innerType());
return builder.eq(arrayReference.ident().columnIdent().fqn(), literal.value());
}

@Override
protected Query applyArrayLiteral(Reference reference, Literal arrayLiteral, Context context) throws IOException {
String columnName = reference.ident().columnIdent().fqn();

Object values = arrayLiteral.value();
private TermsFilter termsFilter(String columnName, Literal arrayLiteral) {
TermsFilter termsFilter;
Object values = arrayLiteral.value();
if (values instanceof Collection) {
termsFilter = new TermsFilter(
columnName,
getBytesRefs((Collection) arrayLiteral.value(),
getBytesRefs((Collection)values,
TermBuilder.forType(arrayLiteral.valueType())));
} else {
termsFilter = new TermsFilter(
columnName,
getBytesRefs((Object[]) arrayLiteral.value(),
getBytesRefs((Object[])values,
TermBuilder.forType(arrayLiteral.valueType())));
}
return new FilteredQuery(Queries.newMatchAllQuery(), termsFilter);
return termsFilter;
}

@Override
protected Query applyArrayReference(Reference arrayReference, Literal literal, Context context) throws IOException {
QueryBuilderHelper builder = QueryBuilderHelper.forType(((CollectionType)arrayReference.valueType()).innerType());
return builder.eq(arrayReference.ident().columnIdent().fqn(), literal.value());
}

@Override
protected Query applyArrayLiteral(Reference reference, Literal arrayLiteral, Context context) throws IOException {
String columnName = reference.ident().columnIdent().fqn();
return new FilteredQuery(Queries.newMatchAllQuery(), termsFilter(columnName, arrayLiteral));
}
}

Expand Down
21 changes: 11 additions & 10 deletions sql/src/test/java/io/crate/executor/transport/task/elasticsearch/ESQueryBuilderTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -514,7 +514,7 @@ public void testAnyEqReferenceAndArrayLiteral() throws Exception {

Function whereClause = new Function(anyEqImpl.info(),
Arrays.asList(doubleRef, doubleArrayLiteral));
xcontentAssert(whereClause, "{\"query\":{\"terms\":{\"d\":[-1.5,0.0,1.5]}}}");
xcontentAssert(whereClause, "{\"query\":{\"filtered\":{\"query\":{\"match_all\":{}},\"filter\":{\"terms\":{\"d\":[-1.5,0.0,1.5]}}}}}");
}

@Test
Expand All @@ -528,15 +528,16 @@ public void testAnyNeqReferenceAndArrayLiteral() throws Exception {

Function whereClause = new Function(anyNeqImpl.info(),
Arrays.asList(doubleRef, doubleArrayLiteral));
xcontentAssert(whereClause, "{\"query\":" +
"{\"bool\":{\"must_not\":" +
"{\"bool\":{\"must\":[" +
"{\"term\":{\"d\":-1.5}}," +
"{\"term\":{\"d\":0.0}}," +
"{\"term\":{\"d\":1.5}}" +
"]}}" +
"}}" +
"}");
xcontentAssert(whereClause, "{\"query\":{" +
"\"filtered\":{" +
"\"query\":{\"match_all\":{}}," +
"\"filter\":{" +
"\"bool\":{\"must_not\":{" +
"\"bool\":{\"must\":[" +
"{\"term\":{\"d\":-1.5}}," +
"{\"term\":{\"d\":0.0}}," +
"{\"term\":{\"d\":1.5}}]" +
"}}}}}}}");
}

@Test
Expand Down
2 changes: 1 addition & 1 deletion sql/src/test/java/io/crate/lucene/LuceneQueryBuilderTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,7 @@ public void testAnyGreaterAndSmaller() throws Exception {
@Test
public void testAnyOnArrayLiteral() throws Exception {
Reference ref = createReference("d", DataTypes.STRING);
Literal stringArrayLiteral = Literal.newLiteral(new Object[]{"a", "b", "c"}, new ArrayType(DataTypes.STRING));
Literal stringArrayLiteral = Literal.newLiteral(new Object[]{new BytesRef("a"), new BytesRef("b"), new BytesRef("c")}, new ArrayType(DataTypes.STRING));

// col != ANY (1,2,3)
Query neqQuery = convert(whereClause(AnyNeqOperator.NAME, ref, stringArrayLiteral));
Expand Down
144 changes: 96 additions & 48 deletions stresstest/src/test/java/io/crate/benchmark/AnyBenchmark.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,93 +27,141 @@
import com.carrotsearch.junitbenchmarks.annotation.BenchmarkHistoryChart;
import com.carrotsearch.junitbenchmarks.annotation.BenchmarkMethodChart;
import com.carrotsearch.junitbenchmarks.annotation.LabelType;
import com.carrotsearch.randomizedtesting.generators.RandomStrings;
import com.google.common.base.Joiner;
import com.google.common.collect.AbstractIterator;
import io.crate.action.sql.SQLRequest;
import io.crate.action.sql.SQLResponse;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.RuleChain;
import org.junit.rules.TestRule;

import java.util.Iterator;
import java.util.Random;
import java.io.IOException;
import java.util.Vector;
import java.util.concurrent.atomic.AtomicInteger;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.is;

@AxisRange(min = 0)
@BenchmarkHistoryChart(filePrefix="benchmark-any-history", labelWith = LabelType.CUSTOM_KEY)
@BenchmarkMethodChart(filePrefix = "benchmark-any")
public class AnyBenchmark extends BenchmarkBase {
public class AnyBenchmark extends BenchmarkBase {

static final int BENCHMARK_ROUNDS = 200;
static {
ClassLoader.getSystemClassLoader().setDefaultAssertionStatus(true);
}

private static Iterator<String> strings(final int num, final int length) {
return new AbstractIterator<String>() {
Random random = new Random(System.currentTimeMillis());
int i = num;

@Override
protected String computeNext() {
if (i-- <= 0) {
endOfData();
return null;
}
return RandomStrings.randomAsciiOfLength(random, length);
}
};
}

static final String SELECT_IN_2000_STATEMENT
= "select * from " + INDEX_NAME + " where \"countryCode\" in ('" +
Joiner.on("','").join(strings(2000, 2)) + "')";
static final int NUMBER_OF_DOCUMENTS = 100_000;
static AtomicInteger VALUE = new AtomicInteger(0);
static Vector<String> VALUES = new Vector<>();

static final String COUNT_IN_2000_STATEMENT = "select * from " + INDEX_NAME + " where \"countryCode\" in ('" +
Joiner.on("','").join(strings(2000, 2)) + "')";
static final String SELECT_ANY_PARAM = "select * from "+ INDEX_NAME + " where value = ANY (?)";

static final String SELECT_ANY_2K = "select * from "+ INDEX_NAME + " where \"countryCode\" = ANY (['" +
Joiner.on("','").join(strings(2000, 2)) + "'])";

private static final String SELECT_ANY_500 = "select * from "+ INDEX_NAME + " where \"countryCode\" = ANY (['" +
Joiner.on("','").join(strings(500, 2)) + "'])";
static final String SELECT_NOT_ANY_PARAM = "select * from "+ INDEX_NAME + " where value != ANY (?)";

@Rule
public TestRule benchmarkRun = RuleChain.outerRule(new BenchmarkRule()).around(super.ruleChain);

@Override
public boolean importData() {
public boolean generateData() {
return true;
}

public SQLResponse execute(String stmt) {
return super.execute(stmt, SQLRequest.EMPTY_ARGS, false);
@Override
protected int numberOfDocuments() {
return NUMBER_OF_DOCUMENTS;
}

@Override
protected void createTable() {
execute("create table " + INDEX_NAME + " (" +
" id int primary key, " +
" value string" +
") with (number_of_replicas=0)");
client().admin().cluster().prepareHealth(INDEX_NAME).setWaitForGreenStatus().execute().actionGet();
}

@Override
protected byte[] generateRowSource() throws IOException {
int value = VALUE.getAndIncrement();
String strValue = String.valueOf(value);
VALUES.add(strValue);

return XContentFactory.jsonBuilder()
.startObject()
.field("id", value)
.field("value", strValue)
.endObject()
.bytes().toBytes();
}

protected boolean generateNewRowForEveryDocument() {
return true;
}

@BenchmarkOptions(benchmarkRounds = BENCHMARK_ROUNDS, warmupRounds = 1)
@Test
public void testSelectWhereInWith2000Items() throws Exception {
// uses lucene query builder
execute(SELECT_IN_2000_STATEMENT);
public void testSelectAny5() throws Exception {
String statement = "select * from "+ INDEX_NAME + " where value = ANY (['" +
Joiner.on("','").join(VALUES.subList(0, 5)) + "'])";
SQLResponse response = execute(statement);
assertThat(response.rowCount(), is(5L));
}

@BenchmarkOptions(benchmarkRounds = BENCHMARK_ROUNDS, warmupRounds = 1)
@Test
public void testCountWhereInWith2000Items() throws Exception {
// uses ES query builder
execute(COUNT_IN_2000_STATEMENT);
public void testSelectAny500() throws Exception {
String statement = "select * from "+ INDEX_NAME + " where value = ANY (['" +
Joiner.on("','").join(VALUES.subList(0, 500)) + "'])";
SQLResponse response = execute(statement);
assertThat(response.rowCount(), is(500L));
}

@BenchmarkOptions(benchmarkRounds = BENCHMARK_ROUNDS, warmupRounds = 1)
@Test
public void testSelectAny2K() throws Exception {
execute(SELECT_ANY_2K);
String statement = "select * from "+ INDEX_NAME + " where value = ANY (['" +
Joiner.on("','").join(VALUES.subList(0, 2000)) + "'])";
SQLResponse response = execute(statement);
assertThat(response.rowCount(), is(2000L));
}

@BenchmarkOptions(benchmarkRounds = BENCHMARK_ROUNDS, warmupRounds = 1)
@Test
public void testSelectAny500() throws Exception {
execute(SELECT_ANY_500);
public void testSelectAny5Param() throws Exception {
SQLResponse response = execute(SELECT_ANY_PARAM, new Object[]{ VALUES.subList(0, 5).toArray() });
assertThat(response.rowCount(), is(5L));
}

@BenchmarkOptions(benchmarkRounds = BENCHMARK_ROUNDS, warmupRounds = 1)
@Test
public void testSelectAny500Param() throws Exception {
SQLResponse response = execute(SELECT_ANY_PARAM, new Object[]{ VALUES.subList(0, 500).toArray() });
assertThat(response.rowCount(), is(500L));
}

@BenchmarkOptions(benchmarkRounds = BENCHMARK_ROUNDS, warmupRounds = 1)
@Test
public void testSelectAny2KParam() throws Exception {
SQLResponse response = execute(SELECT_ANY_PARAM, new Object[]{ VALUES.subList(0, 2000).toArray() });
assertThat(response.rowCount(), is(2000L));
}

@BenchmarkOptions(benchmarkRounds = BENCHMARK_ROUNDS, warmupRounds = 1)
@Test
public void testSelectNotAny5Param() throws Exception {
SQLResponse response = execute(SELECT_NOT_ANY_PARAM, new Object[]{ VALUES.subList(0, 5).toArray() });
assertThat(response.rowCount(), is(10000L));
}

@BenchmarkOptions(benchmarkRounds = BENCHMARK_ROUNDS, warmupRounds = 1)
@Test
public void testSelectNotAny500Param() throws Exception {
SQLResponse response = execute(SELECT_NOT_ANY_PARAM, new Object[]{ VALUES.subList(0, 500).toArray() });
assertThat(response.rowCount(), is(10000L));
}

@BenchmarkOptions(benchmarkRounds = BENCHMARK_ROUNDS, warmupRounds = 1)
@Test
public void testSelectNotAny2KParam() throws Exception {
SQLResponse response = execute(SELECT_NOT_ANY_PARAM, new Object[]{ VALUES.subList(0, 2000).toArray() });
assertThat(response.rowCount(), is(10000L));
}
}
Loading

0 comments on commit d5048e0

Please sign in to comment.