From a7c750ec236973bec29380c39f9b7e1627979d04 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Sat, 9 Jul 2016 22:41:01 +0900 Subject: [PATCH 1/2] Support partition batch pruning with `<=>` (EqualNullSafe) predicate in InMemoryTableScanExec --- .../spark/sql/execution/columnar/InMemoryTableScanExec.scala | 5 +++++ .../sql/execution/columnar/PartitionBatchPruningSuite.scala | 2 ++ 2 files changed, 7 insertions(+) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala index 67a410f539b60..2380fd46a9177 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala @@ -65,6 +65,11 @@ private[sql] case class InMemoryTableScanExec( case EqualTo(l: Literal, a: AttributeReference) => statsFor(a).lowerBound <= l && l <= statsFor(a).upperBound + case EqualNullSafe(a: AttributeReference, l: Literal) if !l.nullable => + statsFor(a).lowerBound <= l && l <= statsFor(a).upperBound + case EqualNullSafe(l: Literal, a: AttributeReference) if !l.nullable => + statsFor(a).lowerBound <= l && l <= statsFor(a).upperBound + case LessThan(a: AttributeReference, l: Literal) => statsFor(a).lowerBound < l case LessThan(l: Literal, a: AttributeReference) => l < statsFor(a).upperBound diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/PartitionBatchPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/PartitionBatchPruningSuite.scala index b99cd67a6344c..9d862cfdecb21 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/PartitionBatchPruningSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/PartitionBatchPruningSuite.scala @@ -85,6 +85,8 @@ class PartitionBatchPruningSuite // Comparisons checkBatchPruning("SELECT key FROM pruningData WHERE key = 1", 1, 1)(Seq(1)) checkBatchPruning("SELECT key FROM pruningData WHERE 1 = key", 1, 1)(Seq(1)) + checkBatchPruning("SELECT key FROM pruningData WHERE key <=> 1", 1, 1)(Seq(1)) + checkBatchPruning("SELECT key FROM pruningData WHERE 1 <=> key", 1, 1)(Seq(1)) checkBatchPruning("SELECT key FROM pruningData WHERE key < 12", 1, 2)(1 to 11) checkBatchPruning("SELECT key FROM pruningData WHERE key <= 11", 1, 2)(1 to 11) checkBatchPruning("SELECT key FROM pruningData WHERE key > 88", 1, 2)(89 to 100) From 3da26601cc4833db96251c7f29f7ba362a4968a0 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Sun, 10 Jul 2016 01:27:33 +0900 Subject: [PATCH 2/2] Fix this identical with equalto --- .../spark/sql/execution/columnar/InMemoryTableScanExec.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala index 2380fd46a9177..d72e75ff5fd59 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala @@ -65,9 +65,9 @@ private[sql] case class InMemoryTableScanExec( case EqualTo(l: Literal, a: AttributeReference) => statsFor(a).lowerBound <= l && l <= statsFor(a).upperBound - case EqualNullSafe(a: AttributeReference, l: Literal) if !l.nullable => + case EqualNullSafe(a: AttributeReference, l: Literal) => statsFor(a).lowerBound <= l && l <= statsFor(a).upperBound - case EqualNullSafe(l: Literal, a: AttributeReference) if !l.nullable => + case EqualNullSafe(l: Literal, a: AttributeReference) => statsFor(a).lowerBound <= l && l <= statsFor(a).upperBound case LessThan(a: AttributeReference, l: Literal) => statsFor(a).lowerBound < l