From a5e22bdd48bcf7624615363d7994a137e652ae15 Mon Sep 17 00:00:00 2001 From: Hsuan-Yi Chu Date: Tue, 8 Sep 2015 10:42:51 -0700 Subject: [PATCH 1/7] DRILL-3580: Bump calcite version to 1.4.0-drill-test-r2 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8d4b318f26c..fc023afc153 100644 --- a/pom.xml +++ b/pom.xml @@ -1226,7 +1226,7 @@ org.apache.calcite calcite-core - 1.4.0-drill-r0 + 1.4.0-drill-test-r2 org.jgrapht From 12853282abc3d35ae0a7d2b6b630a3d9b9150761 Mon Sep 17 00:00:00 2001 From: Hsuan-Yi Chu Date: Tue, 8 Sep 2015 10:45:51 -0700 Subject: [PATCH 2/7] DRILL-3580: Add test case Fix is in CALCITE-841 --- .../drill/exec/TestWindowFunctions.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java index 29b702cba95..dbbc7773ba3 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java @@ -450,4 +450,32 @@ public void testConstantsInMultiplePartitions() throws Exception { .build() .run(); } + + @Test // DRILL-3580 + public void testExpressionInWindowFunction() throws Exception { + String root = FileUtils.getResourceAsFile("/store/text/data/t.json").toURI().toString(); + String query = String.format("select a1, b1, sum(b1) over (partition by a1) as c1, sum(a1 + b1) over (partition by a1) as c2\n" + + "from dfs_test.`%s`", root); + + // Validate the plan + final String[] expectedPlan = {"Window\\(window#0=\\[window\\(partition \\{0\\} order by \\[\\].*\\[SUM\\(\\$1\\), SUM\\(\\$2\\)\\]"}; + PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, new String[]{}); + + testBuilder() + .sqlQuery(query) + .unOrdered() + .baselineColumns("a1", "b1", "c1", "c2") + .baselineValues(0l, 1l, 8l, 8l) + .baselineValues(0l, 1l, 8l, 8l) + .baselineValues(0l, 2l, 8l, 8l) + .baselineValues(0l, 2l, 8l, 8l) + .baselineValues(0l, 2l, 8l, 8l) + .baselineValues(10l, 3l, 21l, 71l) + .baselineValues(10l, 3l, 21l, 71l) + .baselineValues(10l, 5l, 21l, 71l) + .baselineValues(10l, 5l, 21l, 71l) + .baselineValues(10l, 5l, 21l, 71l) + .build() + .run(); + } } \ No newline at end of file From 513805b193b786d548b4a876d9775fa848b55b8e Mon Sep 17 00:00:00 2001 From: Hsuan-Yi Chu Date: Thu, 20 Aug 2015 22:33:11 -0700 Subject: [PATCH 3/7] DRILL-3412: Add ProjectWindowTransposeRule to push Project past Window Fix is in CALCITE-844 --- .../exec/planner/logical/DrillRuleSets.java | 2 ++ .../drill/exec/TestWindowFunctions.java | 23 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java index 3ad617a46eb..ab70d93e1e5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java @@ -28,6 +28,7 @@ import org.apache.calcite.rel.rules.JoinPushExpressionsRule; import org.apache.calcite.rel.rules.JoinPushThroughJoinRule; import org.apache.calcite.rel.rules.ProjectRemoveRule; +import org.apache.calcite.rel.rules.ProjectWindowTransposeRule; import org.apache.calcite.rel.rules.ReduceExpressionsRule; import org.apache.calcite.rel.rules.SortRemoveRule; import org.apache.calcite.rel.rules.UnionToDistinctRule; @@ -133,6 +134,7 @@ public static RuleSet getDrillUserConfigurableLogicalRules(OptimizerRulesContext DrillPushProjectPastJoinRule.INSTANCE, DrillPushProjIntoScan.INSTANCE, DrillProjectSetOpTransposeRule.INSTANCE, + ProjectWindowTransposeRule.INSTANCE, /* Convert from Calcite Logical to Drill Logical Rules. diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java index dbbc7773ba3..0ce82f5a871 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java @@ -478,4 +478,27 @@ public void testExpressionInWindowFunction() throws Exception { .build() .run(); } + + @Test // see DRILL-3657 + public void testProjectPushPastWindow() throws Exception { + String query = "select sum(n_nationkey) over(partition by 1 order by 1) as col1, \n" + + "count(n_nationkey) over(partition by 1 order by 1) as col2 \n" + + "from cp.`tpch/nation.parquet` \n" + + "limit 5"; + + // Validate the plan + final String[] expectedPlan = {"Scan.*columns=\\[`n_nationkey`\\].*"}; + PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, new String[]{}); + testBuilder() + .sqlQuery(query) + .unOrdered() + .baselineColumns("col1", "col2") + .baselineValues(300l, 25l) + .baselineValues(300l, 25l) + .baselineValues(300l, 25l) + .baselineValues(300l, 25l) + .baselineValues(300l, 25l) + .build() + .run(); + } } \ No newline at end of file From 3648d91233e09203389cd94b5270dbbddc02553f Mon Sep 17 00:00:00 2001 From: Hsuan-Yi Chu Date: Fri, 21 Aug 2015 17:11:49 -0700 Subject: [PATCH 4/7] DRILL-3683: Add baseline and expected plan for TestWindowFunctions suite --- .../drill/exec/TestWindowFunctions.java | 300 +++++++++++++++++- 1 file changed, 285 insertions(+), 15 deletions(-) diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java index 0ce82f5a871..d26ddacb264 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java @@ -38,19 +38,91 @@ private static void throwAsUnsupportedException(UserException ex) throws Excepti @Test // DRILL-3196 public void testSinglePartition() throws Exception { - final String query = "explain plan for select sum(a2) over(partition by a2), count(*) over(partition by a2) \n" + + final String query = "select sum(n_nationKey) over(partition by n_nationKey) as col1, count(*) over(partition by n_nationKey) as col2 \n" + "from cp.`tpch/nation.parquet`"; - test(query); + // Validate the plan + final String[] expectedPlan = {"Window.*partition \\{0\\} order by \\[\\].*\\[SUM\\(\\$0\\), COUNT\\(\\)", + "Scan.*columns=\\[`n_nationKey`\\].*"}; + final String[] excludedPatterns = {"Scan.*columns=\\[`\\*`\\].*"}; + PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPatterns); + + testBuilder() + .sqlQuery(query) + .unOrdered() + .baselineColumns("col1", "col2") + .baselineValues(0l, 1l) + .baselineValues(1l, 1l) + .baselineValues(2l, 1l) + .baselineValues(3l, 1l) + .baselineValues(4l, 1l) + .baselineValues(5l, 1l) + .baselineValues(6l, 1l) + .baselineValues(7l, 1l) + .baselineValues(8l, 1l) + .baselineValues(9l, 1l) + .baselineValues(10l, 1l) + .baselineValues(11l, 1l) + .baselineValues(12l, 1l) + .baselineValues(13l, 1l) + .baselineValues(14l, 1l) + .baselineValues(15l, 1l) + .baselineValues(16l, 1l) + .baselineValues(17l, 1l) + .baselineValues(18l, 1l) + .baselineValues(19l, 1l) + .baselineValues(20l, 1l) + .baselineValues(21l, 1l) + .baselineValues(22l, 1l) + .baselineValues(23l, 1l) + .baselineValues(24l, 1l) + .build() + .run(); } @Test // DRILL-3196 public void testSinglePartitionDefinedInWindowList() throws Exception { - final String query = "explain plan for select sum(a2) over w \n" + + final String query = "select sum(n_nationKey) over w as col \n" + "from cp.`tpch/nation.parquet` \n" + - "window w as (partition by a2 order by a2)"; + "window w as (partition by n_nationKey order by n_nationKey)"; + + // Validate the plan + final String[] expectedPlan = {"Window.*partition \\{0\\} order by \\[0\\].*SUM\\(\\$0\\)", + "Scan.*columns=\\[`n_nationKey`\\].*"}; + final String[] excludedPatterns = {"Scan.*columns=\\[`\\*`\\].*"}; + PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPatterns); - test(query); + testBuilder() + .sqlQuery(query) + .unOrdered() + .baselineColumns("col") + .baselineValues(0l) + .baselineValues(1l) + .baselineValues(2l) + .baselineValues(3l) + .baselineValues(4l) + .baselineValues(5l) + .baselineValues(6l) + .baselineValues(7l) + .baselineValues(8l) + .baselineValues(9l) + .baselineValues(10l) + .baselineValues(11l) + .baselineValues(12l) + .baselineValues(13l) + .baselineValues(14l) + .baselineValues(15l) + .baselineValues(16l) + .baselineValues(17l) + .baselineValues(18l) + .baselineValues(19l) + .baselineValues(20l) + .baselineValues(21l) + .baselineValues(22l) + .baselineValues(23l) + .baselineValues(24l) + .build() + .run(); } @Test(expected = UnsupportedFunctionException.class) // DRILL-3182 @@ -165,39 +237,171 @@ public void testWindowGroupByOnView() throws Exception { @Test // DRILL-3188 public void testWindowFrameEquivalentToDefault() throws Exception { - final String query1 = "explain plan for select sum(n_nationKey) over(partition by n_nationKey order by n_nationKey) \n" + + final String query1 = "select sum(n_nationKey) over(partition by n_nationKey order by n_nationKey) as col\n" + "from cp.`tpch/nation.parquet` t \n" + "order by n_nationKey"; - final String query2 = "explain plan for select sum(n_nationKey) over(partition by n_nationKey order by n_nationKey \n" + - "range between unbounded preceding and current row) \n" + + final String query2 = "select sum(n_nationKey) over(partition by n_nationKey order by n_nationKey \n" + + "range between unbounded preceding and current row) as col \n" + "from cp.`tpch/nation.parquet` t \n" + "order by n_nationKey"; - final String query3 = "explain plan for select sum(n_nationKey) over(partition by n_nationKey \n" + - "rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)" + + final String query3 = "select sum(n_nationKey) over(partition by n_nationKey \n" + + "rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) as col \n" + "from cp.`tpch/nation.parquet` t \n" + "order by n_nationKey"; - test(query1); - test(query2); - test(query3); + // Validate the plan + final String[] expectedPlan1 = {"Window.*partition \\{0\\} order by \\[0\\].*SUM\\(\\$0\\)", + "Scan.*columns=\\[`n_nationKey`\\].*"}; + final String[] excludedPatterns1 = {"Scan.*columns=\\[`\\*`\\].*"}; + PlanTestBase.testPlanMatchingPatterns(query1, expectedPlan1, excludedPatterns1); + + testBuilder() + .sqlQuery(query1) + .unOrdered() + .baselineColumns("col") + .baselineValues(0l) + .baselineValues(1l) + .baselineValues(2l) + .baselineValues(3l) + .baselineValues(4l) + .baselineValues(5l) + .baselineValues(6l) + .baselineValues(7l) + .baselineValues(8l) + .baselineValues(9l) + .baselineValues(10l) + .baselineValues(11l) + .baselineValues(12l) + .baselineValues(13l) + .baselineValues(14l) + .baselineValues(15l) + .baselineValues(16l) + .baselineValues(17l) + .baselineValues(18l) + .baselineValues(19l) + .baselineValues(20l) + .baselineValues(21l) + .baselineValues(22l) + .baselineValues(23l) + .baselineValues(24l) + .build() + .run(); + + final String[] expectedPlan2 = {"Window.*partition \\{0\\} order by \\[0\\].*SUM\\(\\$0\\)", + "Scan.*columns=\\[`n_nationKey`\\].*"}; + final String[] excludedPatterns2 = {"Scan.*columns=\\[`\\*`\\].*"}; + PlanTestBase.testPlanMatchingPatterns(query2, expectedPlan2, excludedPatterns2); + + testBuilder() + .sqlQuery(query2) + .unOrdered() + .baselineColumns("col") + .baselineValues(0l) + .baselineValues(1l) + .baselineValues(2l) + .baselineValues(3l) + .baselineValues(4l) + .baselineValues(5l) + .baselineValues(6l) + .baselineValues(7l) + .baselineValues(8l) + .baselineValues(9l) + .baselineValues(10l) + .baselineValues(11l) + .baselineValues(12l) + .baselineValues(13l) + .baselineValues(14l) + .baselineValues(15l) + .baselineValues(16l) + .baselineValues(17l) + .baselineValues(18l) + .baselineValues(19l) + .baselineValues(20l) + .baselineValues(21l) + .baselineValues(22l) + .baselineValues(23l) + .baselineValues(24l) + .build() + .run(); + + final String[] expectedPlan3 = {"Window.*partition \\{0\\}.*SUM\\(\\$0\\)", + "Scan.*columns=\\[`n_nationKey`\\].*"}; + final String[] excludedPatterns3 = {"Scan.*columns=\\[`\\*`\\].*"}; + PlanTestBase.testPlanMatchingPatterns(query3, expectedPlan3, excludedPatterns3); + + testBuilder() + .sqlQuery(query3) + .unOrdered() + .baselineColumns("col") + .baselineValues(0l) + .baselineValues(1l) + .baselineValues(2l) + .baselineValues(3l) + .baselineValues(4l) + .baselineValues(5l) + .baselineValues(6l) + .baselineValues(7l) + .baselineValues(8l) + .baselineValues(9l) + .baselineValues(10l) + .baselineValues(11l) + .baselineValues(12l) + .baselineValues(13l) + .baselineValues(14l) + .baselineValues(15l) + .baselineValues(16l) + .baselineValues(17l) + .baselineValues(18l) + .baselineValues(19l) + .baselineValues(20l) + .baselineValues(21l) + .baselineValues(22l) + .baselineValues(23l) + .baselineValues(24l) + .build() + .run(); } @Test // DRILL-3204 public void testWindowWithJoin() throws Exception { - final String query = "select sum(t1.r_regionKey) over(partition by t1.r_regionKey) \n" + + final String query = "select sum(t1.r_regionKey) over(partition by t1.r_regionKey) as col \n" + "from cp.`tpch/region.parquet` t1, cp.`tpch/nation.parquet` t2 \n" + "where t1.r_regionKey = t2.n_nationKey \n" + "group by t1.r_regionKey"; - test(query); + // Validate the plan + final String[] expectedPlan = {"Window.*partition \\{0\\}.*SUM\\(\\$0\\)", + "Scan.*columns=\\[`n_nationKey`\\].*", + "Scan.*columns=\\[`n_nationKey`\\].*"}; + final String[] excludedPatterns = {"Scan.*columns=\\[`\\*`\\].*"}; + PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPatterns); + + testBuilder() + .sqlQuery(query) + .unOrdered() + .baselineColumns("col") + .baselineValues(0l) + .baselineValues(1l) + .baselineValues(2l) + .baselineValues(3l) + .baselineValues(4l) + .build() + .run(); } @Test // DRILL-3298 public void testCountEmptyPartitionByWithExchange() throws Exception { String query = String.format("select count(*) over (order by o_orderpriority) as cnt from dfs.`%s/multilevel/parquet` where o_custkey < 100", TEST_RES_PATH); try { + // Validate the plan + final String[] expectedPlan = {"Window.*partition \\{\\} order by \\[0\\].*COUNT\\(\\)", + "Scan.*columns=\\[`o_custkey`, `o_orderpriority`\\]"}; + final String[] excludedPatterns = {"Scan.*columns=\\[`\\*`\\]"}; + test("alter session set `planner.slice_target` = 1"); + PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPatterns); + testBuilder() .sqlQuery(query) .ordered() @@ -223,6 +427,12 @@ public void testAvgVarianceWindowFunctions() throws Exception { "from cp.`tpch/nation.parquet` " + "where n_nationkey = 1"; + // Validate the plan + final String[] expectedPlan1 = {"Window.*partition \\{0\\} order by \\[\\].*SUM\\(\\$0\\), COUNT\\(\\$0\\)", + "Scan.*columns=\\[`n_nationkey`\\]"}; + final String[] excludedPatterns1 = {"Scan.*columns=\\[`\\*`\\]"}; + PlanTestBase.testPlanMatchingPatterns(avgQuery, expectedPlan1, excludedPatterns1); + testBuilder() .sqlQuery(avgQuery) .unOrdered() @@ -234,6 +444,12 @@ public void testAvgVarianceWindowFunctions() throws Exception { "from cp.`tpch/nation.parquet` " + "where n_nationkey = 1"; + // Validate the plan + final String[] expectedPlan2 = {"Window.*partition \\{0\\} order by \\[\\].*SUM\\(\\$1\\), SUM\\(\\$0\\), COUNT\\(\\$0\\)", + "Scan.*columns=\\[`n_nationkey`\\]"}; + final String[] excludedPatterns2 = {"Scan.*columns=\\[`\\*`\\]"}; + PlanTestBase.testPlanMatchingPatterns(varianceQuery, expectedPlan2, excludedPatterns2); + testBuilder() .sqlQuery(varianceQuery) .unOrdered() @@ -247,6 +463,12 @@ public void testWindowFunctionWithKnownType() throws Exception { final String query = "select sum(cast(col_int as int)) over (partition by col_varchar) as col1 " + "from cp.`jsoninput/large_int.json` limit 1"; + // Validate the plan + final String[] expectedPlan1 = {"Window.*partition \\{0\\} order by \\[\\].*SUM\\(\\$1\\)", + "Scan.*columns=\\[`col_varchar`, `col_int`\\]"}; + final String[] excludedPatterns1 = {"Scan.*columns=\\[`\\*`\\]"}; + PlanTestBase.testPlanMatchingPatterns(query, expectedPlan1, excludedPatterns1); + testBuilder() .sqlQuery(query) .unOrdered() @@ -257,6 +479,12 @@ public void testWindowFunctionWithKnownType() throws Exception { final String avgQuery = "select avg(cast(col_int as int)) over (partition by col_varchar) as col1 " + "from cp.`jsoninput/large_int.json` limit 1"; + // Validate the plan + final String[] expectedPlan2 = {"Window.*partition \\{0\\} order by \\[\\].*SUM\\(\\$1\\), COUNT\\(\\$1\\)", + "Scan.*columns=\\[`col_varchar`, `col_int`\\]"}; + final String[] excludedPatterns2 = {"Scan.*columns=\\[`\\*`\\]"}; + PlanTestBase.testPlanMatchingPatterns(avgQuery, expectedPlan2, excludedPatterns2); + testBuilder() .sqlQuery(avgQuery) .unOrdered() @@ -272,6 +500,12 @@ public void testCompoundIdentifierInWindowDefinition() throws Exception { "FROM dfs_test.`%s` \n" + "WINDOW w AS (PARTITION BY columns[1] ORDER BY columns[0] DESC)", root); + // Validate the plan + final String[] expectedPlan = {"Window.*partition \\{1\\} order by \\[0 DESC\\].*COUNT\\(\\)", + "Scan.*columns=\\[`columns`\\[0\\], `columns`\\[1\\]\\]"}; + final String[] excludedPatterns = {"Scan.*columns=\\[`\\*`\\]"}; + PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPatterns); + // Validate the result testBuilder() .sqlQuery(query) @@ -296,6 +530,12 @@ public void testRankWithGroupBy() throws Exception { final String query = "select dense_rank() over (order by l_suppkey) as rank1 " + " from cp.`tpch/lineitem.parquet` group by l_partkey, l_suppkey order by 1 desc limit 1"; + // Validate the plan + final String[] expectedPlan = {"Window.*partition \\{\\} order by \\[1\\].*DENSE_RANK\\(\\)", + "Scan.*columns=\\[`l_partkey`, `l_suppkey`\\]"}; + final String[] excludedPatterns = {"Scan.*columns=\\[`\\*`\\]"}; + PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPatterns); + testBuilder() .sqlQuery(query) .unOrdered() @@ -307,6 +547,13 @@ public void testRankWithGroupBy() throws Exception { @Test // DRILL-3404 public void testWindowSumAggIsNotNull() throws Exception { String query = String.format("select count(*) cnt from (select sum ( c1 ) over ( partition by c2 order by c1 asc nulls first ) w_sum from dfs.`%s/window/table_with_nulls.parquet` ) sub_query where w_sum is not null", TEST_RES_PATH); + + // Validate the plan + final String[] expectedPlan = {"Window.*partition \\{1\\} order by \\[0 ASC-nulls-first\\].*SUM\\(\\$0\\)", + "Scan.*columns=\\[`c1`, `c2`\\]"}; + final String[] excludedPatterns = {"Scan.*columns=\\[`\\*`\\]"}; + PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPatterns); + testBuilder() .sqlQuery(query) .ordered() @@ -322,6 +569,12 @@ public void testWindowConstants() throws Exception { "where position_id = 2 \n" + "window w as(partition by position_id order by employee_id)"; + // Validate the plan + final String[] expectedPlan = {"Window.*partition \\{0\\} order by \\[1\\].*RANK\\(\\), SUM\\(\\$2\\), SUM\\(\\$1\\), SUM\\(\\$3\\)", + "Scan.*columns=\\[`position_id`, `employee_id`\\]"}; + final String[] excludedPatterns = {"Scan.*columns=\\[`\\*`\\]"}; + PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPatterns); + testBuilder() .sqlQuery(query) .ordered() @@ -344,6 +597,13 @@ public void testMultiplePartitions1() throws Exception { "count(*) over(partition by a1 order by c1) as count2 \n" + "from dfs_test.`%s`", root); + // Validate the plan + final String[] expectedPlan = {"Window.*partition \\{2\\} order by \\[1\\].*COUNT\\(\\)", + "Window.*partition \\{0\\} order by \\[1\\].*COUNT\\(\\), SUM\\(\\$2\\)", + "Scan.*columns=\\[`b1`, `c1`, `a1`\\]"}; + final String[] excludedPatterns = {"Scan.*columns=\\[`\\*`\\]"}; + PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPatterns); + testBuilder() .sqlQuery(query) .unOrdered() @@ -370,6 +630,13 @@ public void testMultiplePartitions2() throws Exception { "sum(a1) over(partition by b1 order by c1) as sum1 \n" + "from dfs_test.`%s`", root); + // Validate the plan + final String[] expectedPlan = {"Window.*partition \\{2\\} order by \\[1\\].*COUNT\\(\\)", + "Window.*partition \\{0\\} order by \\[1\\].*COUNT\\(\\), SUM\\(\\$2\\)", + "Scan.*columns=\\[`b1`, `c1`, `a1`\\]"}; + final String[] excludedPatterns = {"Scan.*columns=\\[`\\*`\\]"}; + PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPatterns); + testBuilder() .sqlQuery(query) .unOrdered() @@ -400,6 +667,7 @@ public void testWithAndWithoutPartitions() throws Exception { final String[] expectedPlan = {"Window\\(window#0=\\[window\\(partition \\{\\}.*\n" + ".*UnionExchange"}; PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, new String[]{}); + testBuilder() .sqlQuery(query) .unOrdered() @@ -433,6 +701,7 @@ public void testConstantsInMultiplePartitions() throws Exception { ".*Window.*SUM\\(\\$2\\).*" }; PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, new String[]{}); + testBuilder() .sqlQuery(query) .unOrdered() @@ -489,6 +758,7 @@ public void testProjectPushPastWindow() throws Exception { // Validate the plan final String[] expectedPlan = {"Scan.*columns=\\[`n_nationkey`\\].*"}; PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, new String[]{}); + testBuilder() .sqlQuery(query) .unOrdered() From 2ce5af0debf690ed17f29ba5c0f9c3e91efbe299 Mon Sep 17 00:00:00 2001 From: Hsuan-Yi Chu Date: Fri, 4 Sep 2015 11:35:45 -0700 Subject: [PATCH 5/7] DRILL-2190, DRILL-2313, DRILL-2318: Add test cases Fixes are in CALCITE-634, CALCITE-613, CALCITE-662 --- .../org/apache/drill/TestExampleQueries.java | 69 +++++++++++++++++++ .../apache/drill/exec/sql/TestWithClause.java | 22 ++++++ 2 files changed, 91 insertions(+) diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java index d15cff2543e..58c7862468b 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java @@ -1121,4 +1121,73 @@ public void testNegativeExtractOperator() throws Exception { .build() .run(); } + + @Test // see DRILL-2313 + public void testDistinctOverAggFunctionWithGroupBy() throws Exception { + String query1 = "select distinct count(distinct n_nationkey) as col from cp.`tpch/nation.parquet` group by n_regionkey order by 1"; + String query2 = "select distinct count(distinct n_nationkey) as col from cp.`tpch/nation.parquet` group by n_regionkey order by count(distinct n_nationkey)"; + String query3 = "select distinct sum(n_nationkey) as col from cp.`tpch/nation.parquet` group by n_regionkey order by 1"; + String query4 = "select distinct sum(n_nationkey) as col from cp.`tpch/nation.parquet` group by n_regionkey order by col"; + + testBuilder() + .sqlQuery(query1) + .unOrdered() + .baselineColumns("col") + .baselineValues((long) 5) + .build() + .run(); + + testBuilder() + .sqlQuery(query2) + .unOrdered() + .baselineColumns("col") + .baselineValues((long) 5) + .build() + .run(); + + testBuilder() + .sqlQuery(query3) + .ordered() + .baselineColumns("col") + .baselineValues((long) 47) + .baselineValues((long) 50) + .baselineValues((long) 58) + .baselineValues((long) 68) + .baselineValues((long) 77) + .build() + .run(); + + testBuilder() + .sqlQuery(query4) + .ordered() + .baselineColumns("col") + .baselineValues((long) 47) + .baselineValues((long) 50) + .baselineValues((long) 58) + .baselineValues((long) 68) + .baselineValues((long) 77) + .build() + .run(); + } + + @Test // DRILL-2190 + public void testDateImplicitCasting() throws Exception { + String query = "SELECT birth_date \n" + + "FROM cp.`employee.json` \n" + + "WHERE birth_date BETWEEN '1920-01-01' AND cast('1931-01-01' AS DATE) \n" + + "order by birth_date"; + + testBuilder() + .sqlQuery(query) + .unOrdered() + .baselineColumns("birth_date") + .baselineValues("1920-04-17") + .baselineValues("1921-12-04") + .baselineValues("1922-08-10") + .baselineValues("1926-10-27") + .baselineValues("1928-03-20") + .baselineValues("1930-01-08") + .build() + .run(); + } } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestWithClause.java b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestWithClause.java index 3a3204587d3..ff04f8c7b7f 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestWithClause.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestWithClause.java @@ -39,4 +39,26 @@ public void withClauseWithAliases() throws Exception { "select x, y from alpha"); } + @Test // DRILL-2318 + public void withClauseOrderBy() throws Exception { + String query = "WITH x \n" + + "AS (SELECT n_nationkey a1 \n" + + "FROM cp.`tpch/nation.parquet`) \n" + + "SELECT x.a1 \n" + + "FROM x \n" + + "ORDER BY x.a1 \n" + + "limit 5"; + + testBuilder() + .sqlQuery(query) + .ordered() + .baselineColumns("a1") + .baselineValues(0) + .baselineValues(1) + .baselineValues(2) + .baselineValues(3) + .baselineValues(4) + .build() + .run(); + } } From 759a2853c0713162bcbe85d5e55ca2b86453786c Mon Sep 17 00:00:00 2001 From: Hsuan-Yi Chu Date: Mon, 7 Sep 2015 16:47:18 -0700 Subject: [PATCH 6/7] DRILL-3755: In DrillSqlWorker, give UserException.validationError if ValidationException is thrown from Calcite --- .../exec/planner/sql/DrillSqlWorker.java | 10 ++++---- .../java/org/apache/drill/BaseTestQuery.java | 10 ++++++++ .../drill/exec/TestWindowFunctions.java | 23 ++++++++++--------- 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java index bc79cffcb61..64a2e1cbdaf 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java @@ -178,15 +178,15 @@ public PhysicalPlan getPlan(String sql, Pointer textPlan) throws Foreman return handler.getPlan(sqlNode); } catch(ValidationException e) { String errorMessage = e.getCause() != null ? e.getCause().getMessage() : e.getMessage(); - throw UserException.parseError(e) - .message(errorMessage) - .build(logger); + throw UserException.validationError(e) + .message(errorMessage) + .build(logger); } catch (AccessControlException e) { throw UserException.permissionError(e) - .build(logger); + .build(logger); } catch(SqlUnsupportedException e) { throw UserException.unsupportedError(e) - .build(logger); + .build(logger); } catch (IOException | RelConversionException e) { throw new QueryInputException("Failure handling SQL.", e); } diff --git a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java index 13819492fe8..2916a678094 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java +++ b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java @@ -379,6 +379,16 @@ protected static void parseErrorHelper(final String testSqlQuery) throws Excepti errorMsgTestHelper(testSqlQuery, UserBitShared.DrillPBError.ErrorType.PARSE.name()); } + /** + * Utility method which tests given query produces a {@link UserException} + * with {@link org.apache.drill.exec.proto.UserBitShared.DrillPBError.ErrorType} + * being DrillPBError.ErrorType.VALIDATION the given message. + * @param testSqlQuery Test query + */ + protected static void validationErrorHelper(final String testSqlQuery) throws Exception { + errorMsgTestHelper(testSqlQuery, UserBitShared.DrillPBError.ErrorType.VALIDATION.name()); + } + public static String getFile(String resource) throws IOException{ final URL url = Resources.getResource(resource); if (url == null) { diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java index d26ddacb264..07dc288cb56 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java @@ -215,7 +215,7 @@ public void testWindowGroupBy() throws Exception { "from cp.`tpch/nation.parquet` \n" + "group by n_name"; - parseErrorHelper(query); + validationErrorHelper(query); } @Test // DRILL-3346 @@ -229,7 +229,7 @@ public void testWindowGroupByOnView() throws Exception { test("use dfs_test.tmp"); test(createView); - parseErrorHelper(query); + validationErrorHelper(query); } finally { test("drop view testWindowGroupByOnView"); } @@ -403,15 +403,16 @@ public void testCountEmptyPartitionByWithExchange() throws Exception { PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPatterns); testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("cnt") - .optionSettingQueriesForTestQuery("alter session set `planner.slice_target` = 1") - .baselineValues(1l) - .baselineValues(4l) - .baselineValues(4l) - .baselineValues(4l) - .build().run(); + .sqlQuery(query) + .ordered() + .baselineColumns("cnt") + .optionSettingQueriesForTestQuery("alter session set `planner.slice_target` = 1") + .baselineValues(1l) + .baselineValues(4l) + .baselineValues(4l) + .baselineValues(4l) + .build() + .run(); } finally { test("alter session set `planner.slice_target` = " + ExecConstants.SLICE_TARGET_DEFAULT); } From 9e96545c65c6c2c1b8619207a7006ae2391651ac Mon Sep 17 00:00:00 2001 From: Hsuan-Yi Chu Date: Wed, 9 Sep 2015 16:21:54 -0700 Subject: [PATCH 7/7] DRILL-3280, DRILL-3360, DRILL-3601, DRILL-3649: Add test cases Fix is in CALCITE-820 --- .../drill/exec/TestWindowFunctions.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java index 07dc288cb56..245ae4a1b36 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/TestWindowFunctions.java @@ -209,6 +209,27 @@ public void testWindowWithAllowDisallow() throws Exception { } } + @Test // DRILL-3360 + public void testWindowInWindow() throws Exception { + String query = "select rank() over(order by row_number() over(order by n_nationkey)) \n" + + "from cp.`tpch/nation.parquet`"; + + validationErrorHelper(query); + } + + @Test // DRILL-3280, DRILL-3601, DRILL-3649 + public void testMissingOver() throws Exception { + String query1 = "select lead(n_nationkey) from cp.`tpch/nation.parquet`"; + String query2 = "select rank(), cume_dist() over w \n" + + "from cp.`tpch/nation.parquet` \n" + + "window w as (partition by n_name order by n_nationkey)"; + String query3 = "select NTILE(1) from cp.`tpch/nation.parquet`"; + + validationErrorHelper(query1); + validationErrorHelper(query2); + validationErrorHelper(query3); + } + @Test // DRILL-3344 public void testWindowGroupBy() throws Exception { String query = "explain plan for SELECT max(n_nationkey) OVER (), n_name as col2 \n" +