Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
DRILL-3621: Fix incorrect result if HBase filter contains row_key "or…
…" filter or in list filter

Add unit test for row_key "or" filter and row_key in list filter.

Modify expected results for couple of existing unit tests, by specifying more strict regex pattern.

Add one row in Hbase test table, per review comment.
  • Loading branch information
jinfengni committed Aug 13, 2015
1 parent c0060f7 commit e3fbc6a
Show file tree
Hide file tree
Showing 5 changed files with 83 additions and 14 deletions.
Expand Up @@ -176,8 +176,10 @@ private HBaseScanSpec createHBaseScanSpec(FunctionCall call, CompareFunctionsPro
case "equal":
compareOp = CompareOp.EQUAL;
if (isRowKey) {
startRow = stopRow = fieldValue;
compareOp = null;
startRow = fieldValue;
/* stopRow should be just greater than 'value'*/
stopRow = Arrays.copyOf(fieldValue, fieldValue.length+1);
compareOp = CompareOp.EQUAL;
}
break;
case "not_equal":
Expand Down
Expand Up @@ -24,19 +24,19 @@ public class HBaseRecordReaderTest extends BaseHBaseTest {
@Test
public void testLocalDistributed() throws Exception {
String planName = "/hbase/hbase_scan_screen_physical.json";
runHBasePhysicalVerifyCount(planName, HBaseTestsSuite.TEST_TABLE_1, 6);
runHBasePhysicalVerifyCount(planName, HBaseTestsSuite.TEST_TABLE_1, 7);
}

@Test
public void testLocalDistributedColumnSelect() throws Exception {
String planName = "/hbase/hbase_scan_screen_physical_column_select.json";
runHBasePhysicalVerifyCount(planName, HBaseTestsSuite.TEST_TABLE_1, 2);
runHBasePhysicalVerifyCount(planName, HBaseTestsSuite.TEST_TABLE_1, 3);
}

@Test
public void testLocalDistributedFamilySelect() throws Exception {
String planName = "/hbase/hbase_scan_screen_physical_family_select.json";
runHBasePhysicalVerifyCount(planName, HBaseTestsSuite.TEST_TABLE_1, 3);
runHBasePhysicalVerifyCount(planName, HBaseTestsSuite.TEST_TABLE_1, 4);
}

}
Expand Up @@ -34,7 +34,7 @@ public void testFilterPushDownRowKeyEqual() throws Exception {

runHBaseSQLVerifyCount(sql, 1);

final String[] expectedPlan = {".*startRow=b4, stopRow=b4.*"};
final String[] expectedPlan = {".*startRow=b4, stopRow=b4\\\\x00, filter=null.*"};
final String[] excludedPlan ={};
final String sqlHBase = canonizeHBaseSQL(sql);
PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan);
Expand All @@ -52,7 +52,7 @@ public void testFilterPushDownRowKeyEqualWithItem() throws Exception {

runHBaseSQLVerifyCount(sql, 1);

final String[] expectedPlan = {".*startRow=b4, stopRow=b4.*"};
final String[] expectedPlan = {".*startRow=b4, stopRow=b4\\\\x00, filter=null.*"};
final String[] excludedPlan ={".*startRow=null, stopRow=null.*"};
final String sqlHBase = canonizeHBaseSQL(sql);
PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan);
Expand Down Expand Up @@ -122,9 +122,9 @@ public void testFilterPushDownRowKeyGreaterThan() throws Exception {
+ "WHERE\n"
+ " row_key > 'b4'";

runHBaseSQLVerifyCount(sql, 2);
runHBaseSQLVerifyCount(sql, 3);

final String[] expectedPlan = {".*startRow=b4.*stopRow=,.*"};
final String[] expectedPlan = {".*startRow=b4\\\\x00.*stopRow=,.*"};
final String[] excludedPlan ={};
final String sqlHBase = canonizeHBaseSQL(sql);
PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan);
Expand All @@ -142,7 +142,7 @@ public void testFilterPushDownRowKeyGreaterThanWithItem() throws Exception {

runHBaseSQLVerifyCount(sql, 2);

final String[] expectedPlan = {".*startRow=b4.*stopRow=,.*"};
final String[] expectedPlan = {".*startRow=b4\\\\x00.*stopRow=, filter=null.*"};
final String[] excludedPlan ={};
final String sqlHBase = canonizeHBaseSQL(sql);
PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan);
Expand Down Expand Up @@ -228,7 +228,7 @@ public void testFilterPushDownConvertExpression() throws Exception {
+ "WHERE\n"
+ " convert_from(row_key, 'UTF8') > 'b4'";

runHBaseSQLVerifyCount(sql, 2);
runHBaseSQLVerifyCount(sql, 3);

final String[] expectedPlan = {".*startRow=b4\\\\x00, stopRow=,.*"};
final String[] excludedPlan ={};
Expand Down Expand Up @@ -303,4 +303,62 @@ public void testFilterPushDownRowKeyLessThanOrEqualToWithItem() throws Exception
PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan);
}

@Test
public void testFilterPushDownOrRowKeyEqual() throws Exception {
setColumnWidths(new int[] {8, 38, 38});
final String sql = "SELECT\n"
+ " *\n"
+ "FROM\n"
+ " hbase.`[TABLE_NAME]` tableName\n"
+ "WHERE\n"
+ " row_key = 'b4' or row_key = 'a2'";

runHBaseSQLVerifyCount(sql, 2);

final String[] expectedPlan = {".*startRow=a2, stopRow=b4\\\\x00, filter=FilterList OR \\(2/2\\): \\[RowFilter \\(EQUAL, b4\\), RowFilter \\(EQUAL, a2\\).*"};
final String[] excludedPlan ={};
final String sqlHBase = canonizeHBaseSQL(sql);
PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan);

}

@Test
public void testFilterPushDownOrRowKeyInPred() throws Exception {
setColumnWidths(new int[] {8, 38, 38});
final String sql = "SELECT\n"
+ " *\n"
+ "FROM\n"
+ " hbase.`[TABLE_NAME]` tableName\n"
+ "WHERE\n"
+ " row_key in ('b4', 'a2')";

runHBaseSQLVerifyCount(sql, 2);

final String[] expectedPlan = {".*startRow=a2, stopRow=b4\\\\x00, filter=FilterList OR \\(2/2\\): \\[RowFilter \\(EQUAL, b4\\), RowFilter \\(EQUAL, a2\\).*"};
final String[] excludedPlan ={};
final String sqlHBase = canonizeHBaseSQL(sql);
PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan);

}

@Test
public void testFilterPushDownOrRowKeyEqualRangePred() throws Exception {
setColumnWidths(new int[] {8, 38, 38});
final String sql = "SELECT\n"
+ " *\n"
+ "FROM\n"
+ " hbase.`[TABLE_NAME]` tableName\n"
+ "WHERE\n"
+ " row_key = 'a2' or row_key between 'b5' and 'b6'";

runHBaseSQLVerifyCount(sql, 3);

final String[] expectedPlan = {".*startRow=a2, stopRow=b6\\\\x00, filter=FilterList OR \\(2/2\\): \\[RowFilter \\(EQUAL, a2\\), FilterList AND \\(2/2\\): \\[RowFilter \\(GREATER_OR_EQUAL, b5\\), RowFilter \\(LESS_OR_EQUAL, b6.*"};
final String[] excludedPlan ={};
final String sqlHBase = canonizeHBaseSQL(sql);
PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan);

}

}

Expand Up @@ -28,7 +28,7 @@ public void testRowKeyPushDown() throws Exception{
+ "row_key\n"
+ "FROM\n"
+ " hbase.`[TABLE_NAME]` tableName"
, 6);
, 7);
}

@Test
Expand All @@ -48,7 +48,7 @@ public void testRowKeyAndColumnPushDown() throws Exception{
+ "row_key, t.f.c1*31 as `t.f.c1*31`, t.f.c2 as `t.f.c2`, 5 as `5`, 'abc' as `'abc'`\n"
+ "FROM\n"
+ " hbase.`[TABLE_NAME]` t"
, 6);
, 7);
}

@Test
Expand All @@ -58,7 +58,7 @@ public void testColumnFamilyPushDown() throws Exception{
+ "row_key, f, f2\n"
+ "FROM\n"
+ " hbase.`[TABLE_NAME]` tableName"
, 6);
, 7);
}

}
Expand Up @@ -81,6 +81,15 @@ public static void generateHBaseDataset1(HBaseAdmin admin, String tableName, int
p.add("f".getBytes(), "c9".getBytes(), "6".getBytes());
table.put(p);

p = new Put(new byte[]{'b', '4', 0});
p.add("f".getBytes(), "c1".getBytes(), "1".getBytes());
p.add("f2".getBytes(), "c2".getBytes(), "2".getBytes());
p.add("f".getBytes(), "c3".getBytes(), "3".getBytes());
p.add("f2".getBytes(), "c4".getBytes(), "4".getBytes());
p.add("f".getBytes(), "c5".getBytes(), "5".getBytes());
p.add("f2".getBytes(), "c6".getBytes(), "6".getBytes());
table.put(p);

p = new Put("b4".getBytes());
p.add("f".getBytes(), "c1".getBytes(), "1".getBytes());
p.add("f2".getBytes(), "c2".getBytes(), "2".getBytes());
Expand Down

0 comments on commit e3fbc6a

Please sign in to comment.