From 709684f1036e1ab8595f94c2d3c5314c29a20063 Mon Sep 17 00:00:00 2001 From: ravipesala Date: Mon, 6 Oct 2014 21:12:02 +0530 Subject: [PATCH 1/3] Changed parser to support case when function. --- .../apache/spark/sql/catalyst/SqlParser.scala | 23 +++++++++++++++++++ .../org/apache/spark/sql/SQLQuerySuite.scala | 15 ++++++++++-- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala index 854b5b461bdc8..f1cbac10b7707 100755 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala @@ -128,6 +128,11 @@ class SqlParser extends StandardTokenParsers with PackratParsers { protected val UNION = Keyword("UNION") protected val UPPER = Keyword("UPPER") protected val WHERE = Keyword("WHERE") + protected val CASE = Keyword("CASE") + protected val WHEN = Keyword("WHEN") + protected val THEN = Keyword("THEN") + protected val ELSE = Keyword("ELSE") + protected val END = Keyword("END") // Use reflection to find the reserved words defined in this class. protected val reservedWords = @@ -333,6 +338,24 @@ class SqlParser extends StandardTokenParsers with PackratParsers { IF ~> "(" ~> expression ~ "," ~ expression ~ "," ~ expression <~ ")" ^^ { case c ~ "," ~ t ~ "," ~ f => If(c,t,f) } | + CASE ~> opt(expression) ~ (WHEN ~ expression ~ THEN ~ expression).* ~ + opt(ELSE ~> expression) <~ END ^^ { + case c ~ l ~ el => + var caseWhenExpr = l.map{x => + x match { + case w ~ we ~ t ~ te => + c match { + case Some(e) => Seq(EqualTo(e, we), te) + case None => Seq(we, te) + } + } + }.toSeq.reduce(_ ++ _) + caseWhenExpr = el match { + case Some(e) => caseWhenExpr ++ Seq(e) + case None => caseWhenExpr + } + CaseWhen(caseWhenExpr) + } | (SUBSTR | SUBSTRING) ~> "(" ~> expression ~ "," ~ expression <~ ")" ^^ { case s ~ "," ~ p => Substring(s,p,Literal(Integer.MAX_VALUE)) } | diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index 6fb6cb8db0c8f..0d32276d37e2c 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -680,9 +680,20 @@ class SQLQuerySuite extends QueryTest with BeforeAndAfterAll { sql("SELECT CAST(TRUE AS STRING), CAST(FALSE AS STRING) FROM testData LIMIT 1"), ("true", "false") :: Nil) } - + test("SPARK-3371 Renaming a function expression with group by gives error") { registerFunction("len", (s: String) => s.length) checkAnswer( - sql("SELECT len(value) as temp FROM testData WHERE key = 1 group by len(value)"), 1)} + sql("SELECT len(value) as temp FROM testData WHERE key = 1 group by len(value)"), 1) + } + + test("SPARK-3813 CASE a WHEN b THEN c [WHEN d THEN e]* [ELSE f] END") { + checkAnswer( + sql("SELECT CASE key WHEN 1 THEN 1 ELSE 0 END FROM testData WHERE key = 1 group by key"), 1) + } + + test("SPARK-3813 CASE WHEN a THEN b [WHEN c THEN d]* [ELSE e] END") { + checkAnswer( + sql("SELECT CASE WHEN key=1 THEN 1 ELSE 2 END FROM testData WHERE key = 1 group by key"), 1) + } } From 713ea84a4b751ef061d0f991620dc7ce0d7b3afe Mon Sep 17 00:00:00 2001 From: ravipesala Date: Mon, 6 Oct 2014 23:00:30 +0530 Subject: [PATCH 2/3] Updated as per admin comments --- .../apache/spark/sql/catalyst/SqlParser.scala | 23 ++++++------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala index f1cbac10b7707..f84aeb8ed9be5 100755 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala @@ -338,23 +338,14 @@ class SqlParser extends StandardTokenParsers with PackratParsers { IF ~> "(" ~> expression ~ "," ~ expression ~ "," ~ expression <~ ")" ^^ { case c ~ "," ~ t ~ "," ~ f => If(c,t,f) } | - CASE ~> opt(expression) ~ (WHEN ~ expression ~ THEN ~ expression).* ~ - opt(ELSE ~> expression) <~ END ^^ { - case c ~ l ~ el => - var caseWhenExpr = l.map{x => - x match { - case w ~ we ~ t ~ te => - c match { - case Some(e) => Seq(EqualTo(e, we), te) - case None => Seq(we, te) - } - } - }.toSeq.reduce(_ ++ _) - caseWhenExpr = el match { - case Some(e) => caseWhenExpr ++ Seq(e) - case None => caseWhenExpr + CASE ~> expression.? ~ (WHEN ~> expression ~ (THEN ~> expression)).* ~ + (ELSE ~> expression).? <~ END ^^ { + case casePart ~ altPart ~ elsePart => + val altExprs = altPart.flatMap{ + case we ~ te => + Seq(casePart.fold(we)(EqualTo(_, we)),te) } - CaseWhen(caseWhenExpr) + CaseWhen(altExprs ++ elsePart.toList) } | (SUBSTR | SUBSTRING) ~> "(" ~> expression ~ "," ~ expression <~ ")" ^^ { case s ~ "," ~ p => Substring(s,p,Literal(Integer.MAX_VALUE)) From 70c75a78053f08e33ebff5c778d857043d2c634d Mon Sep 17 00:00:00 2001 From: ravipesala Date: Tue, 7 Oct 2014 07:15:21 +0530 Subject: [PATCH 3/3] Fixed styles --- .../org/apache/spark/sql/catalyst/SqlParser.scala | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala index f84aeb8ed9be5..4662f585cfe15 100755 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala @@ -77,10 +77,13 @@ class SqlParser extends StandardTokenParsers with PackratParsers { protected val BETWEEN = Keyword("BETWEEN") protected val BY = Keyword("BY") protected val CACHE = Keyword("CACHE") + protected val CASE = Keyword("CASE") protected val CAST = Keyword("CAST") protected val COUNT = Keyword("COUNT") protected val DESC = Keyword("DESC") protected val DISTINCT = Keyword("DISTINCT") + protected val ELSE = Keyword("ELSE") + protected val END = Keyword("END") protected val EXCEPT = Keyword("EXCEPT") protected val FALSE = Keyword("FALSE") protected val FIRST = Keyword("FIRST") @@ -122,17 +125,14 @@ class SqlParser extends StandardTokenParsers with PackratParsers { protected val SUBSTRING = Keyword("SUBSTRING") protected val SUM = Keyword("SUM") protected val TABLE = Keyword("TABLE") + protected val THEN = Keyword("THEN") protected val TIMESTAMP = Keyword("TIMESTAMP") protected val TRUE = Keyword("TRUE") protected val UNCACHE = Keyword("UNCACHE") protected val UNION = Keyword("UNION") protected val UPPER = Keyword("UPPER") - protected val WHERE = Keyword("WHERE") - protected val CASE = Keyword("CASE") protected val WHEN = Keyword("WHEN") - protected val THEN = Keyword("THEN") - protected val ELSE = Keyword("ELSE") - protected val END = Keyword("END") + protected val WHERE = Keyword("WHERE") // Use reflection to find the reserved words defined in this class. protected val reservedWords = @@ -341,9 +341,9 @@ class SqlParser extends StandardTokenParsers with PackratParsers { CASE ~> expression.? ~ (WHEN ~> expression ~ (THEN ~> expression)).* ~ (ELSE ~> expression).? <~ END ^^ { case casePart ~ altPart ~ elsePart => - val altExprs = altPart.flatMap{ + val altExprs = altPart.flatMap { case we ~ te => - Seq(casePart.fold(we)(EqualTo(_, we)),te) + Seq(casePart.fold(we)(EqualTo(_, we)), te) } CaseWhen(altExprs ++ elsePart.toList) } |