Hive partition pruning support Contains, StartsWith and EndsWith pred…

…icate
apache · Nov 16, 2020 · 9c40971 · 9c40971
1 parent 0046222
commit 9c40971
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 0 deletions.
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
@@ -753,6 +753,15 @@ private[client] class Shim_v0_13 extends Shim_v0_12 {
           ExtractableLiteral(value), ExtractAttribute(SupportedAttribute(name))) =>
         Some(s"$value ${op.symbol} $name")
 
+      case Contains(ExtractAttribute(SupportedAttribute(name)), ExtractableLiteral(value)) =>
+        Some(s"$name like " + (("\".*" + value.drop(1)).dropRight(1) + ".*\""))
+
+      case StartsWith(ExtractAttribute(SupportedAttribute(name)), ExtractableLiteral(value)) =>
+        Some(s"$name like " + (value.dropRight(1) + ".*\""))
+
+      case EndsWith(ExtractAttribute(SupportedAttribute(name)), ExtractableLiteral(value)) =>
+        Some(s"$name like " + ("\".*" + value.drop(1)))
+
       case And(expr1, expr2) if useAdvanced =>
         val converted = convert(expr1) ++ convert(expr2)
         if (converted.isEmpty) {

diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HivePartitionFilteringSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HivePartitionFilteringSuite.scala
@@ -272,6 +272,24 @@ class HivePartitionFilteringSuite(version: String)
       day1 :: day2 :: Nil)
   }
 
+  test("getPartitionsByFilter: chunk contains bb") {
+    testMetastorePartitionFiltering(
+      attr("chunk").contains("bb"),
+      (20170101 to 20170103, 0 to 4, Seq("bb")) :: Nil)
+  }
+
+  test("getPartitionsByFilter: chunk startsWith b") {
+    testMetastorePartitionFiltering(
+      attr("chunk").startsWith("b"),
+      (20170101 to 20170103, 0 to 4, Seq("ba", "bb")) :: Nil)
+  }
+
+  test("getPartitionsByFilter: chunk endsWith b") {
+    testMetastorePartitionFiltering(
+      attr("chunk").endsWith("b"),
+      (20170101 to 20170103, 0 to 4, Seq("ab", "bb")) :: Nil)
+  }
+
   private def testMetastorePartitionFiltering(
       filterExpr: Expression,
       expectedDs: Seq[Int],