From 69be01dd36af7badd84725ecd87f8f7a0cdbbf42 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Wed, 2 Dec 2015 00:12:55 +0800 Subject: [PATCH 1/2] Check bitmasks to set nullable property. --- .../apache/spark/sql/catalyst/analysis/Analyzer.scala | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index b8f212fca7509..50d482fcffc8d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -223,10 +223,13 @@ class Analyzer( case other => Alias(other, other.toString)() } - // TODO: We need to use bitmasks to determine which grouping expressions need to be - // set as nullable. For example, if we have GROUPING SETS ((a,b), a), we do not need - // to change the nullability of a. - val attributeMap = groupByAliases.map(a => (a -> a.toAttribute.withNullability(true))).toMap + val attributeMap = groupByAliases.zipWithIndex.map { case (a, idx) => + if (x.bitmasks.exists(bitmask => (bitmask & 1 << idx) == 0)) { + (a -> a.toAttribute.withNullability(true)) + } else { + (a -> a.toAttribute) + } + }.toMap val aggregations: Seq[NamedExpression] = x.aggregations.map { // If an expression is an aggregate (contains a AggregateExpression) then we dont change From 723d24ab6a89a09b90d9de65a0e72f0c07bfef4b Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Wed, 2 Dec 2015 07:57:06 +0800 Subject: [PATCH 2/2] For comment. --- .../org/apache/spark/sql/catalyst/analysis/Analyzer.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 50d482fcffc8d..3545afee5aa5f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -223,8 +223,10 @@ class Analyzer( case other => Alias(other, other.toString)() } + val nonNullBitmask = x.bitmasks.reduce(_ & _) + val attributeMap = groupByAliases.zipWithIndex.map { case (a, idx) => - if (x.bitmasks.exists(bitmask => (bitmask & 1 << idx) == 0)) { + if ((nonNullBitmask & 1 << idx) == 0) { (a -> a.toAttribute.withNullability(true)) } else { (a -> a.toAttribute)