From 93bd51eba6a07468743e087ec137606d3ba588e6 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Mon, 15 Feb 2016 06:04:22 +0000 Subject: [PATCH 1/3] Support nested UNION in parser. --- .../spark/sql/catalyst/parser/SparkSqlParser.g | 13 +++++++++++++ .../apache/spark/sql/catalyst/CatalystQlSuite.scala | 9 +++++++++ 2 files changed, 22 insertions(+) diff --git a/sql/catalyst/src/main/antlr3/org/apache/spark/sql/catalyst/parser/SparkSqlParser.g b/sql/catalyst/src/main/antlr3/org/apache/spark/sql/catalyst/parser/SparkSqlParser.g index e1908a8e03b30..c3e1eb9107180 100644 --- a/sql/catalyst/src/main/antlr3/org/apache/spark/sql/catalyst/parser/SparkSqlParser.g +++ b/sql/catalyst/src/main/antlr3/org/apache/spark/sql/catalyst/parser/SparkSqlParser.g @@ -2320,6 +2320,19 @@ regularBody[boolean topLevel] ) | selectStatement[topLevel] + | + (LPAREN selectStatement[true]) => nestedSetOpSelectStatement[topLevel] + ; + +nestedSetOpSelectStatement[boolean topLevel] + : + ( + LPAREN s=selectStatement[topLevel] RPAREN -> {$s.tree} + ) + (set=setOpSelectStatement[$nestedSetOpSelectStatement.tree, topLevel]) + -> {set == null}? + {$nestedSetOpSelectStatement.tree} + -> {$set.tree} ; selectStatement[boolean topLevel] diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala index 8d7d6b5bf52ea..b94f417bced66 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala @@ -201,4 +201,13 @@ class CatalystQlSuite extends PlanTest { parser.parsePlan("select sum(product + 1) over (partition by (product + (1)) order by 2) " + "from windowData") } + + test("nesting UNION") { + parser.parsePlan("SELECT `u_1`.`id` FROM (((SELECT `t0`.`id` FROM `default`.`t0`) " + + "UNION ALL (SELECT `t0`.`id` FROM `default`.`t0`)) UNION ALL " + + "(SELECT `t0`.`id` FROM `default`.`t0`)) AS u_1") + parser.parsePlan("SELECT `u_1`.`id` FROM ((SELECT `t0`.`id` FROM `default`.`t0`) " + + "UNION ALL (SELECT `t0`.`id` FROM `default`.`t0`) UNION ALL " + + "(SELECT `t0`.`id` FROM `default`.`t0`)) AS u_1") + } } From f462997728021f1f2e1091ceb38cbc7caddd5e12 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Mon, 15 Feb 2016 13:38:43 +0000 Subject: [PATCH 2/3] Test actual parsed plan. --- .../spark/sql/catalyst/CatalystQlSuite.scala | 38 ++++++++++++++++--- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala index b94f417bced66..aadcfd136c367 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala @@ -203,11 +203,37 @@ class CatalystQlSuite extends PlanTest { } test("nesting UNION") { - parser.parsePlan("SELECT `u_1`.`id` FROM (((SELECT `t0`.`id` FROM `default`.`t0`) " + - "UNION ALL (SELECT `t0`.`id` FROM `default`.`t0`)) UNION ALL " + - "(SELECT `t0`.`id` FROM `default`.`t0`)) AS u_1") - parser.parsePlan("SELECT `u_1`.`id` FROM ((SELECT `t0`.`id` FROM `default`.`t0`) " + - "UNION ALL (SELECT `t0`.`id` FROM `default`.`t0`) UNION ALL " + - "(SELECT `t0`.`id` FROM `default`.`t0`)) AS u_1") + val parsed = parser.parsePlan( + """ + |SELECT `u_1`.`id` FROM (((SELECT `t0`.`id` FROM `default`.`t0`) + |UNION ALL (SELECT `t0`.`id` FROM `default`.`t0`)) UNION ALL + |(SELECT `t0`.`id` FROM `default`.`t0`)) AS u_1 + """.stripMargin) + + val expected = Project( + UnresolvedAlias(UnresolvedAttribute("u_1.id"), None) :: Nil, + Subquery("u_1", + Union( + Union( + Project( + UnresolvedAlias(UnresolvedAttribute("t0.id"), None) :: Nil, + UnresolvedRelation(TableIdentifier("t0", Some("default")), None)), + Project( + UnresolvedAlias(UnresolvedAttribute("t0.id"), None) :: Nil, + UnresolvedRelation(TableIdentifier("t0", Some("default")), None))), + Project( + UnresolvedAlias(UnresolvedAttribute("t0.id"), None) :: Nil, + UnresolvedRelation(TableIdentifier("t0", Some("default")), None))))) + + comparePlans(parsed, expected) + + val parsed2 = parser.parsePlan( + """ + |SELECT `u_1`.`id` FROM ((SELECT `t0`.`id` FROM `default`.`t0`) + |UNION ALL (SELECT `t0`.`id` FROM `default`.`t0`) UNION ALL + |(SELECT `t0`.`id` FROM `default`.`t0`)) AS u_1 + """.stripMargin) + + comparePlans(parsed2, expected) } } From 190b326ab4c23da0ed7ebf8f7254dc8a60ea1e84 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Mon, 15 Feb 2016 15:15:08 +0000 Subject: [PATCH 3/3] Support recursively nested one. --- .../sql/catalyst/parser/SparkSqlParser.g | 11 +++++-- .../spark/sql/catalyst/CatalystQlSuite.scala | 33 +++++++++++++++++-- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/antlr3/org/apache/spark/sql/catalyst/parser/SparkSqlParser.g b/sql/catalyst/src/main/antlr3/org/apache/spark/sql/catalyst/parser/SparkSqlParser.g index c3e1eb9107180..9aeea69cd2d9b 100644 --- a/sql/catalyst/src/main/antlr3/org/apache/spark/sql/catalyst/parser/SparkSqlParser.g +++ b/sql/catalyst/src/main/antlr3/org/apache/spark/sql/catalyst/parser/SparkSqlParser.g @@ -2321,13 +2321,13 @@ regularBody[boolean topLevel] | selectStatement[topLevel] | - (LPAREN selectStatement[true]) => nestedSetOpSelectStatement[topLevel] + (LPAREN selectStatement0[true]) => nestedSetOpSelectStatement[topLevel] ; nestedSetOpSelectStatement[boolean topLevel] : ( - LPAREN s=selectStatement[topLevel] RPAREN -> {$s.tree} + LPAREN s=selectStatement0[topLevel] RPAREN -> {$s.tree} ) (set=setOpSelectStatement[$nestedSetOpSelectStatement.tree, topLevel]) -> {set == null}? @@ -2335,6 +2335,13 @@ nestedSetOpSelectStatement[boolean topLevel] -> {$set.tree} ; +selectStatement0[boolean topLevel] + : + (selectStatement[true]) => selectStatement[topLevel] + | + (nestedSetOpSelectStatement[true]) => nestedSetOpSelectStatement[topLevel] + ; + selectStatement[boolean topLevel] : ( diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala index aadcfd136c367..7ee0046c95b06 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala @@ -227,13 +227,42 @@ class CatalystQlSuite extends PlanTest { comparePlans(parsed, expected) - val parsed2 = parser.parsePlan( + val parsedSame = parser.parsePlan( """ |SELECT `u_1`.`id` FROM ((SELECT `t0`.`id` FROM `default`.`t0`) |UNION ALL (SELECT `t0`.`id` FROM `default`.`t0`) UNION ALL |(SELECT `t0`.`id` FROM `default`.`t0`)) AS u_1 """.stripMargin) - comparePlans(parsed2, expected) + comparePlans(parsedSame, expected) + + val parsed2 = parser.parsePlan( + """ + |SELECT `u_1`.`id` FROM ((((SELECT `t0`.`id` FROM `default`.`t0`) + |UNION ALL (SELECT `t0`.`id` FROM `default`.`t0`)) UNION ALL + |(SELECT `t0`.`id` FROM `default`.`t0`)) + |UNION ALL (SELECT `t0`.`id` FROM `default`.`t0`)) AS u_1 + """.stripMargin) + + val expected2 = Project( + UnresolvedAlias(UnresolvedAttribute("u_1.id"), None) :: Nil, + Subquery("u_1", + Union( + Union( + Union( + Project( + UnresolvedAlias(UnresolvedAttribute("t0.id"), None) :: Nil, + UnresolvedRelation(TableIdentifier("t0", Some("default")), None)), + Project( + UnresolvedAlias(UnresolvedAttribute("t0.id"), None) :: Nil, + UnresolvedRelation(TableIdentifier("t0", Some("default")), None))), + Project( + UnresolvedAlias(UnresolvedAttribute("t0.id"), None) :: Nil, + UnresolvedRelation(TableIdentifier("t0", Some("default")), None))), + Project( + UnresolvedAlias(UnresolvedAttribute("t0.id"), None) :: Nil, + UnresolvedRelation(TableIdentifier("t0", Some("default")), None))))) + + comparePlans(parsed2, expected2) } }