From 7cbb530539641bb75639144a859609833aee515f Mon Sep 17 00:00:00 2001 From: Michael Armbrust Date: Sun, 14 Jun 2015 04:56:19 +0000 Subject: [PATCH 1/3] [SPARK-8358][SQL] Wait for child resolution when resolving generatorsa --- .../org/apache/spark/sql/catalyst/analysis/Analyzer.scala | 6 ++++-- .../test/scala/org/apache/spark/sql/DataFrameSuite.scala | 8 ++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index cbd8def4f1d3c..9993f21923740 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -563,7 +563,9 @@ class Analyzer( private object AliasedGenerator { def unapply(e: Expression): Option[(Generator, Seq[String])] = e match { case Alias(g: Generator, name) - if g.elementTypes.size > 1 && java.util.regex.Pattern.matches("_c[0-9]+", name) => { + if g.resolved && + g.elementTypes.size > 1 && + java.util.regex.Pattern.matches("_c[0-9]+", name) => { // Assume the default name given by parser is "_c[0-9]+", // TODO in long term, move the naming logic from Parser to Analyzer. // In projection, Parser gave default name for TGF as does for normal UDF, @@ -572,7 +574,7 @@ class Analyzer( // Let's simply ignore the default given name for this case. Some((g, Nil)) } - case Alias(g: Generator, name) if g.elementTypes.size > 1 => + case Alias(g: Generator, name) if g.resolved && g.elementTypes.size > 1 => // If not given the default names, and the TGF with multiple output columns failAnalysis( s"""Expect multiple names given for ${g.getClass.getName}, diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala index 84835c0db765d..007d7cd755547 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala @@ -133,6 +133,14 @@ class DataFrameSuite extends QueryTest { Row("a", 3) :: Row("b", 2) :: Row("c", 1) :: Nil ) } + + test("explode alias and star") { + val df = Seq((Array("a"), 1)).toDF("a", "b") + + checkAnswer( + df.select(explode($"a").as("a"), $"*"), + Row("a", Seq("a"), 1) :: Nil) + } test("selectExpr") { checkAnswer( From 806a3738a46af2bf47713045f54f37dc3243fb37 Mon Sep 17 00:00:00 2001 From: Michael Armbrust Date: Sun, 14 Jun 2015 10:53:55 +0000 Subject: [PATCH 2/3] fix style --- .../org/apache/spark/sql/catalyst/analysis/Analyzer.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 9993f21923740..4b7fef7126989 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -563,8 +563,8 @@ class Analyzer( private object AliasedGenerator { def unapply(e: Expression): Option[(Generator, Seq[String])] = e match { case Alias(g: Generator, name) - if g.resolved && - g.elementTypes.size > 1 && + if g.resolved && + g.elementTypes.size > 1 && java.util.regex.Pattern.matches("_c[0-9]+", name) => { // Assume the default name given by parser is "_c[0-9]+", // TODO in long term, move the naming logic from Parser to Analyzer. From fbd206586ab817f0b8152ad9f39674311934d664 Mon Sep 17 00:00:00 2001 From: Michael Armbrust Date: Sun, 14 Jun 2015 11:36:27 +0000 Subject: [PATCH 3/3] more style --- .../src/test/scala/org/apache/spark/sql/DataFrameSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala index 007d7cd755547..fa98e23e3d147 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala @@ -133,10 +133,10 @@ class DataFrameSuite extends QueryTest { Row("a", 3) :: Row("b", 2) :: Row("c", 1) :: Nil ) } - + test("explode alias and star") { val df = Seq((Array("a"), 1)).toDF("a", "b") - + checkAnswer( df.select(explode($"a").as("a"), $"*"), Row("a", Seq("a"), 1) :: Nil)