Fix NullType

apache · Apr 25, 2020 · 67f34a1 · 67f34a1
1 parent 7357427
commit 67f34a1
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 3 deletions.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
@@ -426,7 +426,10 @@ case class In(value: Expression, list: Seq[Expression]) extends Predicate {
  * Optimized version of In clause, when all filter values of In clause are
  * static.
  */
-case class InSet(child: Expression, hset: Set[Any]) extends UnaryExpression with Predicate {
+case class InSet(
+    child: Expression,
+    hset: Set[Any],
+    hsetElemType: Option[DataType] = None) extends UnaryExpression with Predicate {
 
   require(hset != null, "hset could not be null")
 
@@ -520,8 +523,9 @@ case class InSet(child: Expression, hset: Set[Any]) extends UnaryExpression with
 
   override def sql: String = {
     val valueSQL = child.sql
+    val elemType = hsetElemType.getOrElse(child.dataType)
     val listSQL = hset.toSeq
-      .map(elem => Literal(convertToScala(elem, child.dataType)).sql)
+      .map(elem => Literal(convertToScala(elem, elemType)).sql)
       .mkString(", ")
     s"($valueSQL IN ($listSQL))"
   }

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
@@ -830,7 +830,7 @@ class Column(val expr: Expression) extends Logging {
   def isInCollection(values: scala.collection.Iterable[_]): Column = withExpr {
     val exprValues = values.toSeq.map(lit(_).expr)
     if (exprValues.size > SQLConf.get.optimizerInSetConversionThreshold) {
-      InSet(expr, exprValues.map(_.eval()).toSet)
+      InSet(expr, exprValues.map(_.eval()).toSet, exprValues.headOption.map(_.dataType))
     } else {
       In(expr, exprValues)
     }