From 426dd2e8c9aba6bff15cc5d4c1755730bc378fca Mon Sep 17 00:00:00 2001 From: ouyangxiaochen Date: Fri, 24 Aug 2018 21:15:49 +0800 Subject: [PATCH] ExternalCatalogUtils.prunePartitionsByFilter throw an AnalysisException when partition name contains upper letter --- .../sql/catalyst/catalog/ExternalCatalogUtils.scala | 9 ++++++--- .../sql/catalyst/catalog/ExternalCatalogSuite.scala | 5 +++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogUtils.scala index 50f32e81d997d..74c519233ddbb 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogUtils.scala @@ -132,7 +132,8 @@ object ExternalCatalogUtils { catalogTable: CatalogTable, inputPartitions: Seq[CatalogTablePartition], predicates: Seq[Expression], - defaultTimeZoneId: String): Seq[CatalogTablePartition] = { + defaultTimeZoneId: String, + caseSensitive: Boolean = false): Seq[CatalogTablePartition] = { if (predicates.isEmpty) { inputPartitions } else { @@ -140,7 +141,8 @@ object ExternalCatalogUtils { val partitionColumnNames = catalogTable.partitionColumnNames.toSet val nonPartitionPruningPredicates = predicates.filterNot { - _.references.map(_.name).toSet.subsetOf(partitionColumnNames) + _.references.map(r => if (caseSensitive) r.name else r.name.toLowerCase(Locale.ROOT)) + .toSet.subsetOf(partitionColumnNames) } if (nonPartitionPruningPredicates.nonEmpty) { throw new AnalysisException("Expected only partition pruning predicates: " + @@ -150,7 +152,8 @@ object ExternalCatalogUtils { val boundPredicate = InterpretedPredicate.create(predicates.reduce(And).transform { case att: AttributeReference => - val index = partitionSchema.indexWhere(_.name == att.name) + val attributeName = if (caseSensitive) att.name else att.name.toLowerCase(Locale.ROOT) + val index = partitionSchema.indexWhere(_.name == attributeName) BoundReference(index, partitionSchema(index).dataType, nullable = true) }) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala index b376108399c1c..1160238aa8663 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala @@ -494,6 +494,11 @@ abstract class ExternalCatalogSuite extends SparkFunSuite with BeforeAndAfterEac throw new AnalysisException(ex.getMessage) } } + + val m = intercept[AnalysisException] { + checkAnswer(tbl2, Seq('A.int === 1 && 'b.string === "2"), Set(part1)) + }.getMessage + assert(m.contains("Expected only partition pruning predicates")) } test("drop partitions") {