Skip to content

Commit

Permalink
Support multiple ranges in ShardPredicate
Browse files Browse the repository at this point in the history
  • Loading branch information
@jessesleeping
jessesleeping committed May 25, 2018
1 parent dc1c925 commit 413e84b
Show file tree
Hide file tree
Showing 3 changed files with 90 additions and 39 deletions.
75 changes: 40 additions & 35 deletions presto-raptor/src/main/java/com/facebook/presto/raptor/metadata/ShardPredicate.java
View file
Expand Up @@ -39,7 +39,6 @@
import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR;
import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.MoreObjects.toStringHelper;
import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.collect.Iterables.getOnlyElement;
import static java.lang.String.format; import static java.lang.String.format;
import static java.util.Objects.requireNonNull; import static java.util.Objects.requireNonNull;


Expand All @@ -48,6 +47,7 @@ class ShardPredicate
private final String predicate; private final String predicate;
private final List<JDBCType> types; private final List<JDBCType> types;
private final List<Object> values; private final List<Object> values;
private static final int MAX_RANGE_COUNT = 100;


private ShardPredicate(String predicate, List<JDBCType> types, List<Object> values) private ShardPredicate(String predicate, List<JDBCType> types, List<Object> values)
{ {
Expand Down Expand Up @@ -108,50 +108,55 @@ public static ShardPredicate create(TupleDomain<RaptorColumnHandle> tupleDomain)
continue; continue;
} }


StringJoiner columnPredicate = new StringJoiner(" OR ", "(", ")").setEmptyValue("true");
Ranges ranges = domain.getValues().getRanges(); Ranges ranges = domain.getValues().getRanges();


// TODO: support multiple ranges // prevent generating complicated metadata queries
if (ranges.getRangeCount() != 1) { if (ranges.getRangeCount() > MAX_RANGE_COUNT) {
continue; continue;
} }
Range range = getOnlyElement(ranges.getOrderedRanges());


Object minValue = null; for (Range range : ranges.getOrderedRanges()) {
Object maxValue = null; Object minValue = null;
if (range.isSingleValue()) { Object maxValue = null;
minValue = range.getSingleValue(); if (range.isSingleValue()) {
maxValue = range.getSingleValue(); minValue = range.getSingleValue();
} maxValue = range.getSingleValue();
else {
if (!range.getLow().isLowerUnbounded()) {
minValue = range.getLow().getValue();
} }
if (!range.getHigh().isUpperUnbounded()) { else {
maxValue = range.getHigh().getValue(); if (!range.getLow().isLowerUnbounded()) {
minValue = range.getLow().getValue();
}
if (!range.getHigh().isUpperUnbounded()) {
maxValue = range.getHigh().getValue();
}
} }
}


String min; String min;
String max; String max;
if (handle.isBucketNumber()) { if (handle.isBucketNumber()) {
min = "bucket_number"; min = "bucket_number";
max = "bucket_number"; max = "bucket_number";
} }
else { else {
min = minColumn(handle.getColumnId()); min = minColumn(handle.getColumnId());
max = maxColumn(handle.getColumnId()); max = maxColumn(handle.getColumnId());
} }


if (minValue != null) { StringJoiner rangePredicate = new StringJoiner(" AND ", "(", ")").setEmptyValue("true");
predicate.add(format("(%s >= ? OR %s IS NULL)", max, max)); if (minValue != null) {
types.add(jdbcType); rangePredicate.add(format("(%s >= ? OR %s IS NULL)", max, max));
values.add(minValue); types.add(jdbcType);
} values.add(minValue);
if (maxValue != null) { }
predicate.add(format("(%s <= ? OR %s IS NULL)", min, min)); if (maxValue != null) {
types.add(jdbcType); rangePredicate.add(format("(%s <= ? OR %s IS NULL)", min, min));
values.add(maxValue); types.add(jdbcType);
values.add(maxValue);
}
columnPredicate.add(rangePredicate.toString());
} }
predicate.add(columnPredicate.toString());
} }
return new ShardPredicate(predicate.toString(), types.build(), values.build()); return new ShardPredicate(predicate.toString(), types.build(), values.build());
} }
Expand Down
22 changes: 20 additions & 2 deletions ...to-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestDatabaseShardManager.java
View file
Expand Up @@ -71,6 +71,7 @@
import static com.facebook.presto.spi.predicate.Range.greaterThan; import static com.facebook.presto.spi.predicate.Range.greaterThan;
import static com.facebook.presto.spi.predicate.Range.greaterThanOrEqual; import static com.facebook.presto.spi.predicate.Range.greaterThanOrEqual;
import static com.facebook.presto.spi.predicate.Range.lessThan; import static com.facebook.presto.spi.predicate.Range.lessThan;
import static com.facebook.presto.spi.predicate.Range.range;
import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BigintType.BIGINT;
import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN;
import static com.facebook.presto.spi.type.DateType.DATE; import static com.facebook.presto.spi.type.DateType.DATE;
Expand Down Expand Up @@ -570,10 +571,27 @@ public void testShardPruning()
shardAssertion(tableId).range(c6, lessThan(BOOLEAN, true)).expected(shards); shardAssertion(tableId).range(c6, lessThan(BOOLEAN, true)).expected(shards);
shardAssertion(tableId).range(c6, lessThan(BOOLEAN, false)).expected(shard1, shard3); shardAssertion(tableId).range(c6, lessThan(BOOLEAN, false)).expected(shard1, shard3);


// TODO: support multiple ranges // Test multiple ranges
shardAssertion(tableId) shardAssertion(tableId)
.domain(c1, createDomain(lessThan(BIGINT, 0L), greaterThan(BIGINT, 25L))) .domain(c1, createDomain(lessThan(BIGINT, 0L), greaterThan(BIGINT, 25L)))
.expected(shards); .expected();

shardAssertion(tableId)
.domain(c1, createDomain(range(BIGINT, 3L, true, 4L, true), range(BIGINT, 16L, true, 18L, true)))
.expected(shard2, shard3);

shardAssertion(tableId)
.domain(c5, createDomain(
range(createVarcharType(10), utf8Slice("gum"), true, utf8Slice("happy"), true),
range(createVarcharType(10), utf8Slice("pear"), true, utf8Slice("wall"), true)))
.expected(shard1, shard3);

shardAssertion(tableId)
.domain(c1, createDomain(range(BIGINT, 3L, true, 4L, true), range(BIGINT, 16L, true, 18L, true)))
.domain(c5, createDomain(
range(createVarcharType(10), utf8Slice("gum"), true, utf8Slice("happy"), true),
range(createVarcharType(10), utf8Slice("pear"), true, utf8Slice("wall"), true)))
.expected(shard3);
} }


@Test @Test
Expand Down
32 changes: 30 additions & 2 deletions presto-raptor/src/test/java/com/facebook/presto/raptor/metadata/TestShardPredicate.java
View file
Expand Up @@ -98,13 +98,41 @@ public void testRangeShardUuidPredicate()
} }


@Test @Test
public void testBucketNumber() public void testBucketNumberSingleRange()
{ {
TupleDomain<RaptorColumnHandle> tupleDomain = withColumnDomains(ImmutableMap.of( TupleDomain<RaptorColumnHandle> tupleDomain = withColumnDomains(ImmutableMap.of(
bucketNumberColumnHandle("test"), bucketNumberColumnHandle("test"),
create(SortedRangeSet.copyOf(INTEGER, ImmutableList.of(equal(INTEGER, 1L))), false))); create(SortedRangeSet.copyOf(INTEGER, ImmutableList.of(equal(INTEGER, 1L))), false)));


ShardPredicate shardPredicate = ShardPredicate.create(tupleDomain); ShardPredicate shardPredicate = ShardPredicate.create(tupleDomain);
assertEquals(shardPredicate.getPredicate(), "(bucket_number >= ? OR bucket_number IS NULL) AND (bucket_number <= ? OR bucket_number IS NULL)"); assertEquals(shardPredicate.getPredicate(), "(((bucket_number >= ? OR bucket_number IS NULL) AND (bucket_number <= ? OR bucket_number IS NULL)))");
}

@Test
public void testBucketNumberMultipleRanges()
{
TupleDomain<RaptorColumnHandle> tupleDomain = withColumnDomains(ImmutableMap.of(
bucketNumberColumnHandle("test"),
create(SortedRangeSet.copyOf(INTEGER, ImmutableList.of(equal(INTEGER, 1L), equal(INTEGER, 3L))), false)));

ShardPredicate shardPredicate = ShardPredicate.create(tupleDomain);
assertEquals(shardPredicate.getPredicate(),
"(((bucket_number >= ? OR bucket_number IS NULL) AND (bucket_number <= ? OR bucket_number IS NULL))" +
" OR ((bucket_number >= ? OR bucket_number IS NULL) AND (bucket_number <= ? OR bucket_number IS NULL)))");
}

@Test
public void testMultipleColumnsMultipleRanges()
{
TupleDomain<RaptorColumnHandle> tupleDomain = withColumnDomains(ImmutableMap.of(
bucketNumberColumnHandle("test"),
create(SortedRangeSet.copyOf(INTEGER, ImmutableList.of(equal(INTEGER, 1L), equal(INTEGER, 3L))), false),
new RaptorColumnHandle("test", "col", 1, INTEGER),
create(SortedRangeSet.copyOf(INTEGER, ImmutableList.of(equal(INTEGER, 1L), equal(INTEGER, 3L))), false)));
ShardPredicate shardPredicate = ShardPredicate.create(tupleDomain);
assertEquals(shardPredicate.getPredicate(), "(((c1_max >= ? OR c1_max IS NULL) AND (c1_min <= ? OR c1_min IS NULL)) " +
"OR ((c1_max >= ? OR c1_max IS NULL) AND (c1_min <= ? OR c1_min IS NULL))) " +
"AND (((bucket_number >= ? OR bucket_number IS NULL) AND (bucket_number <= ? OR bucket_number IS NULL)) " +
"OR ((bucket_number >= ? OR bucket_number IS NULL) AND (bucket_number <= ? OR bucket_number IS NULL)))");
} }
} }

0 comments on commit 413e84b

Please sign in to comment.