diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala index 3741a9cb32fd4..aa502c63a9a3e 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala @@ -255,9 +255,18 @@ private[sql] object DataSourceStrategy extends Strategy with Logging { predicates: Seq[Expression], partitionSpec: PartitionSpec): Seq[Partition] = { val PartitionSpec(partitionColumns, partitions) = partitionSpec - val partitionColumnNames = partitionColumns.map(_.name).toSet + val isCaseSensitive = SQLContext.getActive().get.conf.caseSensitiveAnalysis + val partitionColumnNames = if (isCaseSensitive) { + partitionColumns.map(_.name).toSet + } else { + partitionColumns.map(_.name.toLowerCase).toSet + } val partitionPruningPredicates = predicates.filter { - _.references.map(_.name).toSet.subsetOf(partitionColumnNames) + if (isCaseSensitive) { + _.references.map(_.name).toSet.subsetOf(partitionColumnNames) + } else { + _.references.map(_.name.toLowerCase).toSet.subsetOf(partitionColumnNames) + } } if (partitionPruningPredicates.nonEmpty) { @@ -268,7 +277,11 @@ private[sql] object DataSourceStrategy extends Strategy with Logging { val boundPredicate = InterpretedPredicate.create(predicate.transform { case a: AttributeReference => - val index = partitionColumns.indexWhere(a.name == _.name) + val index = if (isCaseSensitive) { + partitionColumns.indexWhere(a.name == _.name) + } else { + partitionColumns.indexWhere(c => a.name.equalsIgnoreCase(c.name)) + } BoundReference(index, partitionColumns(index).dataType, nullable = true) }) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala index 06436a1f3b5ce..542e4c2d77957 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala @@ -1013,7 +1013,8 @@ class DataFrameSuite extends QueryTest with SharedSQLContext { test("SPARK-11301: fix case sensitivity for filter on partitioned columns") { withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") { withTempPath { path => - Seq(2012 -> "a").toDF("year", "val").write.partitionBy("year").parquet(path.getAbsolutePath) + Seq(2012 -> "a", 1999 -> "b").toDF("year", "val").write.partitionBy("year") + .parquet(path.getAbsolutePath) val df = sqlContext.read.parquet(path.getAbsolutePath) checkAnswer(df.filter($"yEAr" > 2000).select($"val"), Row("a")) }