From 428160fd824309f83127ad68efabd0595d614abd Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Sun, 10 Jan 2016 17:12:48 -0800 Subject: [PATCH 1/5] The Limit Clause can be applied inside the set operation --- .../org/apache/spark/sql/catalyst/parser/SparkSqlParser.g | 7 ++++--- .../org/apache/spark/sql/catalyst/CatalystQlSuite.scala | 7 +++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/antlr3/org/apache/spark/sql/catalyst/parser/SparkSqlParser.g b/sql/catalyst/src/main/antlr3/org/apache/spark/sql/catalyst/parser/SparkSqlParser.g index 2c13d3056f468..3cd31e11ebdd7 100644 --- a/sql/catalyst/src/main/antlr3/org/apache/spark/sql/catalyst/parser/SparkSqlParser.g +++ b/sql/catalyst/src/main/antlr3/org/apache/spark/sql/catalyst/parser/SparkSqlParser.g @@ -651,7 +651,7 @@ import java.util.HashMap; return false; } private CommonTree throwSetOpException() throws RecognitionException { - throw new FailedPredicateException(input, "orderByClause clusterByClause distributeByClause sortByClause limitClause can only be applied to the whole union.", ""); + throw new FailedPredicateException(input, "orderByClause clusterByClause distributeByClause sortByClause can only be applied to the whole union.", ""); } private CommonTree throwColumnNameException() throws RecognitionException { throw new FailedPredicateException(input, Arrays.toString(excludedCharForColumnName) + " can not be used in column name in create table statement.", ""); @@ -2250,7 +2250,7 @@ selectStatement[boolean topLevel] (set=setOpSelectStatement[$selectStatement.tree, topLevel])? -> {set == null}? {$selectStatement.tree} - -> {o==null && c==null && d==null && sort==null && l==null}? + -> {o==null && c==null && d==null && sort==null}? {$set.tree} -> {throwSetOpException()} ; @@ -2322,8 +2322,9 @@ simpleSelectStatement groupByClause? havingClause? ((window_clause) => window_clause)? + ((limitClause) => limitClause)? -> ^(TOK_QUERY fromClause? ^(TOK_INSERT ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE)) - selectClause whereClause? groupByClause? havingClause? window_clause?)) + selectClause whereClause? groupByClause? havingClause? window_clause? limitClause?)) ; selectStatementWithCTE diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala index d7204c3488313..82bf075ef89fd 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala @@ -49,4 +49,11 @@ class CatalystQlSuite extends PlanTest { parser.createPlan("select sum(product + 1) over (partition by (product + (1)) order by 2) " + "from windowData") } + + test("limit clause: a support in set operation") { + parser.createPlan("select key from (select * from t1) x limit 1") + parser.createPlan("select key from (select * from t1 limit 2) x limit 1") + parser.createPlan("select key from ((select * from testData limit 1) " + + "union all (select * from testData limit 1)) x limit 1") + } } From 6244975948c016f5adc7dedef825d472f01c8846 Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Sun, 10 Jan 2016 22:05:58 -0800 Subject: [PATCH 2/5] improved the test case. --- .../apache/spark/sql/catalyst/CatalystQlSuite.scala | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala index 82bf075ef89fd..6578bca130f49 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql.catalyst import org.apache.spark.sql.catalyst.plans.PlanTest +import org.apache.spark.sql.catalyst.plans.logical.Limit class CatalystQlSuite extends PlanTest { val parser = new CatalystQl() @@ -51,9 +52,14 @@ class CatalystQlSuite extends PlanTest { } test("limit clause: a support in set operation") { - parser.createPlan("select key from (select * from t1) x limit 1") - parser.createPlan("select key from (select * from t1 limit 2) x limit 1") - parser.createPlan("select key from ((select * from testData limit 1) " + + val plan1 = parser.createPlan("select key from (select * from t1) x limit 1") + assert(plan1.collect{ case w: Limit => w }.size === 1) + + val plan2 = parser.createPlan("select key from (select * from t1 limit 2) x limit 1") + assert(plan2.collect{ case w: Limit => w }.size === 2) + + val plan3 = parser.createPlan("select key from ((select * from testData limit 1) " + "union all (select * from testData limit 1)) x limit 1") + assert(plan3.collect{ case w: Limit => w }.size === 3) } } From 94386aa0fb392c51aa0862bc208cff63614b3c62 Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Mon, 11 Jan 2016 09:51:45 -0800 Subject: [PATCH 3/5] updated the test case. --- .../spark/sql/catalyst/CatalystQlSuite.scala | 34 ++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala index 6578bca130f49..ad76e1f2969f1 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala @@ -17,8 +17,10 @@ package org.apache.spark.sql.catalyst +import org.apache.spark.sql.catalyst.analysis.{UnresolvedStar, UnresolvedAlias, UnresolvedAttribute, UnresolvedRelation} +import org.apache.spark.sql.catalyst.expressions.Literal import org.apache.spark.sql.catalyst.plans.PlanTest -import org.apache.spark.sql.catalyst.plans.logical.Limit +import org.apache.spark.sql.catalyst.plans.logical.{Union, Subquery, Project, Limit} class CatalystQlSuite extends PlanTest { val parser = new CatalystQl() @@ -53,13 +55,37 @@ class CatalystQlSuite extends PlanTest { test("limit clause: a support in set operation") { val plan1 = parser.createPlan("select key from (select * from t1) x limit 1") - assert(plan1.collect{ case w: Limit => w }.size === 1) + val correctPlan1 = + Limit (Literal(1), + Project(Seq(UnresolvedAlias(UnresolvedAttribute("key"))), + Subquery("x", + Project(Seq(UnresolvedAlias(UnresolvedStar(None))), + UnresolvedRelation(TableIdentifier("t1")))))) + comparePlans(plan1, correctPlan1) val plan2 = parser.createPlan("select key from (select * from t1 limit 2) x limit 1") - assert(plan2.collect{ case w: Limit => w }.size === 2) + val correctPlan2 = + Limit (Literal(1), + Project(Seq(UnresolvedAlias(UnresolvedAttribute("key"))), + Subquery("x", + Limit (Literal(2), + Project(Seq(UnresolvedAlias(UnresolvedStar(None))), + UnresolvedRelation(TableIdentifier("t1"))))))) + comparePlans(plan2, correctPlan2) val plan3 = parser.createPlan("select key from ((select * from testData limit 1) " + "union all (select * from testData limit 1)) x limit 1") - assert(plan3.collect{ case w: Limit => w }.size === 3) + val correctPlan3 = + Limit (Literal(1), + Project(Seq(UnresolvedAlias(UnresolvedAttribute("key"))), + Subquery("x", + Union( + Limit (Literal(1), + Project(Seq(UnresolvedAlias(UnresolvedStar(None))), + UnresolvedRelation(TableIdentifier("testData")))), + Limit (Literal(1), + Project(Seq(UnresolvedAlias(UnresolvedStar(None))), + UnresolvedRelation(TableIdentifier("testData")))))))) + comparePlans(plan3, correctPlan3) } } From 7d55c10316c075a0a1c297ab28087aa0ed3556c5 Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Mon, 11 Jan 2016 09:56:30 -0800 Subject: [PATCH 4/5] style fix. --- .../scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala index ad76e1f2969f1..309ac61696c72 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.catalyst -import org.apache.spark.sql.catalyst.analysis.{UnresolvedStar, UnresolvedAlias, UnresolvedAttribute, UnresolvedRelation} +import org.apache.spark.sql.catalyst.analysis._ import org.apache.spark.sql.catalyst.expressions.Literal import org.apache.spark.sql.catalyst.plans.PlanTest import org.apache.spark.sql.catalyst.plans.logical.{Union, Subquery, Project, Limit} From b9ba021cd276a3c53dbc83f7af3046dab5f09706 Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Mon, 11 Jan 2016 10:00:02 -0800 Subject: [PATCH 5/5] style fix. --- .../scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala index 309ac61696c72..84dc9ac596493 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala @@ -20,7 +20,7 @@ package org.apache.spark.sql.catalyst import org.apache.spark.sql.catalyst.analysis._ import org.apache.spark.sql.catalyst.expressions.Literal import org.apache.spark.sql.catalyst.plans.PlanTest -import org.apache.spark.sql.catalyst.plans.logical.{Union, Subquery, Project, Limit} +import org.apache.spark.sql.catalyst.plans.logical.{Limit, Project, Subquery, Union} class CatalystQlSuite extends PlanTest { val parser = new CatalystQl()