From c4f59db8ca93732b6e5c896f1148cc47a31c9c4c Mon Sep 17 00:00:00 2001 From: Hsuan-Yi Chu Date: Sat, 26 Sep 2015 17:16:20 -0700 Subject: [PATCH] DRILL-3257: Disable FilterSetOpTransposeRule, DrillProjectSetOpTransposeRule and add test cases --- .../exec/planner/logical/DrillRuleSets.java | 6 +- .../java/org/apache/drill/TestUnionAll.java | 219 +++++++++++++----- 2 files changed, 168 insertions(+), 57 deletions(-) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java index 65d6e8930f3..16ec559e2b9 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java @@ -116,7 +116,8 @@ public static RuleSet getDrillUserConfigurableLogicalRules(OptimizerRulesContext Filter push-down related rules */ DrillPushFilterPastProjectRule.INSTANCE, - FilterSetOpTransposeRule.INSTANCE, + // Due to infinite loop in planning (DRILL-3257), temporarily disable this rule + //FilterSetOpTransposeRule.INSTANCE, FilterAggregateTransposeRule.INSTANCE, FilterMergeRule.INSTANCE, @@ -133,7 +134,8 @@ public static RuleSet getDrillUserConfigurableLogicalRules(OptimizerRulesContext DrillPushProjectPastFilterRule.INSTANCE, DrillPushProjectPastJoinRule.INSTANCE, DrillPushProjIntoScan.INSTANCE, - DrillProjectSetOpTransposeRule.INSTANCE, + // Due to infinite loop in planning (DRILL-3257), temporarily disable this rule + //DrillProjectSetOpTransposeRule.INSTANCE, ProjectWindowTransposeRule.INSTANCE, /* diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java b/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java index 3e5bfa2a937..32f7bcb1ada 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java @@ -534,14 +534,11 @@ public void testFilterPushDownOverUnionAll() throws Exception { + "order by n_regionkey"; // Validate the plan - final String[] expectedPlan = {"UnionAll.*\n" + - ".*SelectionVectorRemover.*\n" + - ".*Filter.*\n" + - ".*Scan.*\n" + - ".*SelectionVectorRemover.*\n" + - ".*Filter.*\n" + - ".*Scan"}; - final String[] excludedPlan = {"Filter.*\n.*UnionAll"}; + final String[] expectedPlan = {".*Filter.*\n" + + ".*UnionAll.*\n" + + ".*Scan.*columns=\\[`n_regionkey`\\].*\n" + + ".*Scan.*columns=\\[`r_regionkey`\\].*"}; + final String[] excludedPlan = {}; PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan); // Validate the result @@ -568,22 +565,19 @@ public void testInListPushDownOverUnionAll() throws Exception { "where n_nationkey in (1, 2)"; // Validate the plan - final String[] expectedPlan = {"UnionAll.*\n" + - ".*Project.*\n" + - ".*HashJoin.*\n" + - ".*SelectionVectorRemover.*\n" + - ".*Filter.*\n" + - ".*Project.*\n" + - ".*Scan.*\n" + - ".*Scan.*\n" + - ".*Project.*\n" + - ".*HashJoin.*\n" + - ".*SelectionVectorRemover.*\n" + - ".*Filter.*\n" + - ".*Project.*\n" + - ".*Scan.*\n" + - ".*Scan"}; - final String[] excludedPlan = {"Filter.*\n.*UnionAll"}; + final String[] expectedPlan = {"Filter.*\n" + + ".*UnionAll.*\n" + + ".*Project.*\n" + + ".*HashJoin.*\n" + + ".*Project.*\n" + + ".*Scan.*columns=\\[`n_regionkey`, `n_nationkey`\\].*\n" + + ".*Scan.*columns=\\[`r_regionkey`\\].*\n" + + ".*Project.*\n" + + ".*HashJoin.*\n" + + ".*Project.*\n" + + ".*Scan.*columns=\\[`n_regionkey`, `n_nationkey`\\].*\n" + + ".*Scan.*columns=\\[`r_regionkey`\\].*"}; + final String[] excludedPlan = {}; PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan); // Validate the result @@ -609,17 +603,14 @@ public void testFilterPushDownOverUnionAllCSV() throws Exception { "where ct < 100", root, root); // Validate the plan - final String[] expectedPlan = {"UnionAll.*\n" + - ".*SelectionVectorRemover.*\n" + - ".*Filter.*\n" + - ".*StreamAgg.*\n" + - ".*Project.*\n" + - ".*Scan.*\n" + - ".*SelectionVectorRemover.*\n" + - ".*Filter.*\n" + + final String[] expectedPlan = {"Filter.*\n" + + ".*UnionAll.*\n" + + ".*StreamAgg.*\n" + ".*Project.*\n" + - ".*Scan"}; - final String[] excludedPlan = {"Filter.*\n.*UnionAll"}; + ".*Scan.*columns=\\[`columns`\\[0\\]\\].*\n" + + ".*Project.*\n" + + ".*Scan.*columns=\\[`columns`\\[0\\]\\].*"}; + final String[] excludedPlan = {}; PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan); // Validate the result @@ -640,11 +631,13 @@ public void testProjectPushDownOverUnionAllWithProject() throws Exception { "union all select r_regionkey, r_name, r_comment from cp.`tpch/region.parquet`)"; // Validate the plan - final String[] expectedPlan = {"UnionAll.*\n" + - ".*Project.*\n" + - ".*Scan.*columns=\\[`n_nationkey`, `n_name`\\].*\n" + - ".*Project.*\n" + - ".*Scan.*columns=\\[`r_regionkey`, `r_name`\\]"}; + final String[] expectedPlan = {"Project\\(n_nationkey=\\[\\$0\\], n_name=\\[\\$1\\]\\).*\n" + + ".*UnionAll.*\n" + + ".*Project.*\n" + + ".*Scan.*columns=\\[`n_nationkey`, `n_name`, `n_comment`\\].*\n" + + ".*Project.*\n" + + ".*Scan.*columns=\\[`r_regionkey`, `r_name`, `r_comment`\\].*" + }; final String[] excludedPlan = {}; PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan); @@ -665,9 +658,12 @@ public void testProjectPushDownOverUnionAllWithoutProject() throws Exception { "union all select r_regionkey, r_name, r_comment from cp.`tpch/region.parquet`)"; // Validate the plan - final String[] expectedPlan = {"UnionAll.*\n" + - ".*Scan.*columns=\\[`n_nationkey`\\].*\n" + - ".*Scan.*columns=\\[`r_regionkey`\\].*"}; + final String[] expectedPlan = {"Project\\(n_nationkey=\\[\\$0\\]\\).*\n" + + ".*UnionAll.*\n" + + ".*Project.*\n" + + ".*Scan.*columns=\\[`n_nationkey`, `n_name`, `n_comment`\\].*\n" + + ".*Project.*\n" + + ".*Scan.*columns=\\[`r_regionkey`, `r_name`, `r_comment`\\].*"}; final String[] excludedPlan = {}; PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan); @@ -691,9 +687,9 @@ public void testProjectWithExpressionPushDownOverUnionAll() throws Exception { // Validate the plan final String[] expectedPlan = {"UnionAll.*\n" + ".*Project.*\n" + - ".*Scan.*columns=\\[`n_nationkey`\\].*\n" + + ".*Scan.*columns=\\[`n_nationkey`, `n_name`, `n_comment`\\].*\n" + ".*Project.*\n" + - ".*Scan.*columns=\\[`r_regionkey`\\].*"}; + ".*Scan.*columns=\\[`r_regionkey`, `r_name`, `r_comment`\\].*"}; final String[] excludedPlan = {}; PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan); @@ -719,9 +715,9 @@ public void testProjectDownOverUnionAllImplicitCasting() throws Exception { // Validate the plan final String[] expectedPlan = {"UnionAll.*\n." + ".*Project.*\n" + - ".*Scan.*columns=\\[`n_nationkey`\\].*\n" + + ".*Scan.*columns=\\[`n_nationkey`, `n_name`, `n_comment`\\].*\n" + ".*Project.*\n" + - ".*Scan.*columns=\\[`columns`\\[0\\]\\]"}; + ".*Scan.*columns=\\[`columns`\\[0\\], `columns`\\[1\\], `columns`\\[2\\]\\]"}; final String[] excludedPlan = {}; PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan); @@ -745,9 +741,9 @@ public void testProjectPushDownProjectColumnReorderingAndAlias() throws Exceptio // Validate the plan final String[] expectedPlan = {"UnionAll.*\n." + "*Project.*\n" + - ".*Scan.*columns=\\[`n_comment`, `n_nationkey`, `n_name`\\].*\n" + + ".*Scan.*columns=\\[`n_nationkey`, `n_name`, `n_comment`\\].*\n" + ".*Project.*\n" + - ".*Scan.*columns=\\[`r_comment`, `r_regionkey`, `r_name`\\]"}; + ".*Scan.*columns=\\[`r_regionkey`, `r_name`, `r_comment`\\]"}; final String[] excludedPlan = {}; PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan); @@ -770,13 +766,12 @@ public void testProjectFiltertPushDownOverUnionAll() throws Exception { "where n_nationkey > 0 and n_nationkey < 4"; // Validate the plan - final String[] expectedPlan = {"UnionAll.*\n" + - ".*SelectionVectorRemover.*\n" + - ".*Filter.*\n" + - ".*Scan.*columns=\\[`n_nationkey`\\].*\n" + - ".*SelectionVectorRemover.*\n" + - ".*Filter.*\n" + - ".*Scan.*columns=\\[`r_regionkey`\\]"}; + final String[] expectedPlan = {"Filter.*\n" + + ".*UnionAll.*\n" + + ".*Project.*\n" + + ".*Scan.*columns=\\[`n_nationkey`, `n_name`, `n_comment`\\].*\n" + + ".*Project.*\n" + + ".*Scan.*columns=\\[`r_regionkey`, `r_name`, `r_comment`\\]"}; final String[] excludedPlan = {}; PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan); @@ -790,4 +785,118 @@ public void testProjectFiltertPushDownOverUnionAll() throws Exception { .build() .run(); } + + @Test // DRILL-3257 (Simplified Query from TPC-DS query 74) + public void testUnionAllInWith() throws Exception { + final String query1 = "WITH year_total \n" + + " AS (SELECT c.r_regionkey customer_id,\n" + + " 1 year_total\n" + + " FROM cp.`tpch/region.parquet` c\n" + + " UNION ALL \n" + + " SELECT c.r_regionkey customer_id, \n" + + " 1 year_total\n" + + " FROM cp.`tpch/region.parquet` c) \n" + + "SELECT count(t_s_secyear.customer_id) as ct \n" + + "FROM year_total t_s_firstyear, \n" + + " year_total t_s_secyear, \n" + + " year_total t_w_firstyear, \n" + + " year_total t_w_secyear \n" + + "WHERE t_s_secyear.customer_id = t_s_firstyear.customer_id \n" + + " AND t_s_firstyear.customer_id = t_w_secyear.customer_id \n" + + " AND t_s_firstyear.customer_id = t_w_firstyear.customer_id \n" + + " AND CASE \n" + + " WHEN t_w_firstyear.year_total > 0 THEN t_w_secyear.year_total \n" + + " ELSE NULL \n" + + " END > -1"; + + final String query2 = "WITH year_total \n" + + " AS (SELECT c.r_regionkey customer_id,\n" + + " 1 year_total\n" + + " FROM cp.`tpch/region.parquet` c\n" + + " UNION ALL \n" + + " SELECT c.r_regionkey customer_id, \n" + + " 1 year_total\n" + + " FROM cp.`tpch/region.parquet` c) \n" + + "SELECT count(t_w_firstyear.customer_id) as ct \n" + + "FROM year_total t_w_firstyear, \n" + + " year_total t_w_secyear \n" + + "WHERE t_w_firstyear.year_total = t_w_secyear.year_total \n" + + " AND t_w_firstyear.year_total > 0 and t_w_secyear.year_total > 0"; + + final String query3 = "WITH year_total_1\n" + + " AS (SELECT c.r_regionkey customer_id,\n" + + " 1 year_total\n" + + " FROM cp.`tpch/region.parquet` c\n" + + " UNION ALL \n" + + " SELECT c.r_regionkey customer_id, \n" + + " 1 year_total\n" + + " FROM cp.`tpch/region.parquet` c) \n" + + " , year_total_2\n" + + " AS (SELECT c.n_nationkey customer_id,\n" + + " 1 year_total\n" + + " FROM cp.`tpch/nation.parquet` c\n" + + " UNION ALL \n" + + " SELECT c.n_nationkey customer_id, \n" + + " 1 year_total\n" + + " FROM cp.`tpch/nation.parquet` c) \n" + + " SELECT count(t_w_firstyear.customer_id) as ct\n" + + " FROM year_total_1 t_w_firstyear,\n" + + " year_total_2 t_w_secyear\n" + + " WHERE t_w_firstyear.year_total = t_w_secyear.year_total\n" + + " AND t_w_firstyear.year_total > 0 and t_w_secyear.year_total > 0"; + + final String query4 = "WITH year_total_1\n" + + " AS (SELECT c.r_regionkey customer_id,\n" + + " 1 year_total\n" + + " FROM cp.`tpch/region.parquet` c\n" + + " UNION ALL \n" + + " SELECT c.n_nationkey customer_id, \n" + + " 1 year_total\n" + + " FROM cp.`tpch/nation.parquet` c), \n" + + " year_total_2\n" + + " AS (SELECT c.r_regionkey customer_id,\n" + + " 1 year_total\n" + + " FROM cp.`tpch/region.parquet` c\n" + + " UNION ALL \n" + + " SELECT c.n_nationkey customer_id, \n" + + " 1 year_total\n" + + " FROM cp.`tpch/nation.parquet` c) \n" + + " SELECT count(t_w_firstyear.customer_id) as ct \n" + + " FROM year_total_1 t_w_firstyear,\n" + + " year_total_2 t_w_secyear\n" + + " WHERE t_w_firstyear.year_total = t_w_secyear.year_total\n" + + " AND t_w_firstyear.year_total > 0 and t_w_secyear.year_total > 0"; + + testBuilder() + .sqlQuery(query1) + .ordered() + .baselineColumns("ct") + .baselineValues((long) 80) + .build() + .run(); + + testBuilder() + .sqlQuery(query2) + .ordered() + .baselineColumns("ct") + .baselineValues((long) 100) + .build() + .run(); + + testBuilder() + .sqlQuery(query3) + .ordered() + .baselineColumns("ct") + .baselineValues((long) 500) + .build() + .run(); + + testBuilder() + .sqlQuery(query4) + .ordered() + .baselineColumns("ct") + .baselineValues((long) 900) + .build() + .run(); + } } \ No newline at end of file