Skip to content

Commit

Permalink
feat: optimize scan filters (#3651)
Browse files Browse the repository at this point in the history 
When possible, bubble up the maxVersions filter to the start of the filter chain. Semantically we can reorder maxVersions anywhere in a filter chain as long as it doesnt skip over any cell filters (timerange, value, etc).
  • Loading branch information
@igorbernstein2
igorbernstein2 committed Jun 9, 2022
1 parent 49bfe24 commit 23af529
Show file tree
Hide file tree
Showing 4 changed files with 114 additions and 19 deletions.
36 changes: 25 additions & 11 deletions ...gtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/adapters/read/ScanAdapter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@
import com.google.common.collect.TreeRangeSet;
import com.google.protobuf.ByteString;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.NavigableSet;
import org.apache.hadoop.hbase.client.Scan;
Expand Down Expand Up @@ -115,18 +117,8 @@ public void throwIfUnsupportedScan(Scan scan) {
*/
public Filters.Filter buildFilter(Scan scan, ReadHooks hooks) {
ChainFilter chain = FILTERS.chain();
Optional<Filters.Filter> familyFilter = createColumnFamilyFilter(scan);
if (familyFilter.isPresent()) {
chain.filter(familyFilter.get());
}

if (scan.getTimeRange() != null && !scan.getTimeRange().isAllTime()) {
chain.filter(createTimeRangeFilter(scan.getTimeRange()));
}

if (scan.getMaxVersions() != Integer.MAX_VALUE) {
chain.filter(createColumnLimitFilter(scan.getMaxVersions()));
}
buildStartFilter(scan).forEach(chain::filter);

Optional<Filters.Filter> userFilter = createUserFilter(scan, hooks);
if (userFilter.isPresent()) {
Expand All @@ -140,6 +132,28 @@ public Filters.Filter buildFilter(Scan scan, ReadHooks hooks) {
return chain;
}

private List<Filters.Filter> buildStartFilter(Scan scan) {
List<Filters.Filter> filterList = new ArrayList<>();

Optional<Filters.Filter> familyFilter = createColumnFamilyFilter(scan);
if (familyFilter.isPresent()) {
filterList.add(familyFilter.get());
}

boolean hasTimeRange = false;
if (scan.getTimeRange() != null && !scan.getTimeRange().isAllTime()) {
filterList.add(createTimeRangeFilter(scan.getTimeRange()));
hasTimeRange = true;
}

// maxVersions should appear as early as possible, but it must appear after timeRange
if (scan.getMaxVersions() != Integer.MAX_VALUE) {
int i = hasTimeRange ? filterList.size() : 0;
filterList.add(i, createColumnLimitFilter(scan.getMaxVersions()));
}
return filterList;
}

/** {@inheritDoc} */
@Override
public void adapt(Scan scan, ReadHooks readHooks, Query query) {
Expand Down
2 changes: 1 addition & 1 deletion ...arent/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/TestBigtableTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,12 +160,12 @@ public void getRequestsAreFullyPopulated() throws Exception {
Assert.assertEquals(
FILTERS
.chain()
.filter(FILTERS.limit().cellsPerColumn(1))
.filter(
FILTERS
.chain()
.filter(FILTERS.family().regex("family"))
.filter(FILTERS.qualifier().regex("qualifier")))
.filter(FILTERS.limit().cellsPerColumn(1))
.toProto(),
filterCaptor.getValue().toProto());
}
Expand Down
15 changes: 8 additions & 7 deletions ...-hbase/src/test/java/com/google/cloud/bigtable/hbase/adapters/TestCheckAndMutateUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
package com.google.cloud.bigtable.hbase.adapters;

import static com.google.cloud.bigtable.data.v2.models.Filters.FILTERS;
import static com.google.common.truth.Truth.assertThat;

import com.google.bigtable.v2.CheckAndMutateRowRequest;
import com.google.bigtable.v2.Mutation;
Expand Down Expand Up @@ -75,8 +76,8 @@ private static void checkPredicate(CheckAndMutateRowRequest result) {
RowFilter expected =
FILTERS
.chain()
.filter(FAMILY_AND_QUAL_FILTER)
.filter(FILTERS.limit().cellsPerColumn(1))
.filter(FAMILY_AND_QUAL_FILTER)
.filter(
FILTERS
.value()
Expand Down Expand Up @@ -260,10 +261,10 @@ public void testIfNotExists() throws DoNotRetryIOException {
RowFilter expected =
FILTERS
.chain()
.filter(FAMILY_AND_QUAL_FILTER)
.filter(FILTERS.limit().cellsPerColumn(1))
.filter(FAMILY_AND_QUAL_FILTER)
.toProto();
Assert.assertEquals(expected, result.getPredicateFilter());
assertThat(result.getPredicateFilter()).isEqualTo(expected);
}

@Test
Expand All @@ -287,12 +288,12 @@ public void testNotEqualsNull() throws DoNotRetryIOException {
RowFilter expected =
FILTERS
.chain()
.filter(FAMILY_AND_QUAL_FILTER)
.filter(FILTERS.limit().cellsPerColumn(1))
.filter(FAMILY_AND_QUAL_FILTER)
.toProto();

checkPutMutation(result.getTrueMutations(0));
Assert.assertEquals(expected, result.getPredicateFilter());
assertThat(result.getPredicateFilter()).isEqualTo(expected);
}

@Test
Expand Down Expand Up @@ -320,11 +321,11 @@ public void testCompareOpsOtherThanNotEqualsNull() throws DoNotRetryIOException
RowFilter expected =
FILTERS
.chain()
.filter(FAMILY_AND_QUAL_FILTER)
.filter(FILTERS.limit().cellsPerColumn(1))
.filter(FAMILY_AND_QUAL_FILTER)
.toProto();

checkPutMutation(result.getFalseMutations(0));
Assert.assertEquals(expected, result.getPredicateFilter());
assertThat(result.getPredicateFilter()).isEqualTo(expected);
}
}
80 changes: 80 additions & 0 deletions ...le-hbase/src/test/java/com/google/cloud/bigtable/hbase/adapters/read/TestScanAdapter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterBase;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.PrefixFilter;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.Assert;
import org.junit.Test;
Expand Down Expand Up @@ -368,4 +369,83 @@ public void testColFamilyTimeRange() throws IOException {

Assert.assertEquals(expected.toProto(), query.toProto(requestContext).getFilter());
}

@Test
public void testMaxVersionsOptimizationDefault() {
Scan scan =
new Scan()
.addColumn("cf".getBytes(), "q".getBytes())
.setFilter(new PrefixFilter("blah".getBytes()));

scanAdapter.adapt(scan, throwingReadHooks, query);

Filters.Filter expected =
FILTERS
.chain()
// Optimization: limit is first
.filter(FILTERS.limit().cellsPerColumn(1))
// scan columns next
.filter(
FILTERS
.chain()
.filter(FILTERS.family().exactMatch("cf"))
.filter(FILTERS.qualifier().exactMatch("q")))
// user filter
.filter(FILTERS.key().regex("blah\\C*"));
Assert.assertEquals(expected.toProto(), query.toProto(requestContext).getFilter());
}

@Test
public void testMaxVersionsOptimization() {
Scan scan =
new Scan()
.addColumn("cf".getBytes(), "q".getBytes())
.setFilter(new PrefixFilter("blah".getBytes()))
.setMaxVersions(10);

scanAdapter.adapt(scan, throwingReadHooks, query);

Filters.Filter expected =
FILTERS
.chain()
// Optimization: limit is first
.filter(FILTERS.limit().cellsPerColumn(10))
// scan columns next
.filter(
FILTERS
.chain()
.filter(FILTERS.family().exactMatch("cf"))
.filter(FILTERS.qualifier().exactMatch("q")))
// user filter
.filter(FILTERS.key().regex("blah\\C*"));
Assert.assertEquals(expected.toProto(), query.toProto(requestContext).getFilter());
}

@Test
public void testMaxVersionsWithTimeRanges() throws IOException {
Scan scan =
new Scan()
.setTimeRange(0, 1_000)
.addColumn("cf".getBytes(), "q".getBytes())
.setFilter(new PrefixFilter("blah".getBytes()));

scanAdapter.adapt(scan, throwingReadHooks, query);

Filters.Filter expected =
FILTERS
.chain()
// scan columns first, since maxVersion must come after timeRange
.filter(
FILTERS
.chain()
.filter(FILTERS.family().exactMatch("cf"))
.filter(FILTERS.qualifier().exactMatch("q")))
// Timestamp range next
.filter(FILTERS.timestamp().range().of(0L, 1_000 * 1_000L))
// maxVersions after range
.filter(FILTERS.limit().cellsPerColumn(1))
// user filter
.filter(FILTERS.key().regex("blah\\C*"));
Assert.assertEquals(expected.toProto(), query.toProto(requestContext).getFilter());
}
}

0 comments on commit 23af529

Please sign in to comment.