Thu Oct 20 14:39:28 PDT 2016

apache · Oct 20, 2016 · 3b6398b · 3b6398b
1 parent 11f3654
commit 3b6398b
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 5 deletions.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileStatusCache.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileStatusCache.scala
@@ -67,7 +67,7 @@ object FileStatusCache {
    */
   def getOrInitializeShared(clientId: ClientId, session: SparkSession): FileStatusCache = {
     synchronized {
-      if (session.sqlContext.conf.filesourcePartitionPruning &&
+      if (session.sqlContext.conf.filesourcePartitionManagement &&
           session.sqlContext.conf.filesourcePartitionFileCacheSize > 0) {
         if (sharedCache == null) {
           sharedCache = new SharedInMemoryCache(

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/TableFileCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/TableFileCatalog.scala
@@ -36,9 +36,9 @@ import org.apache.spark.sql.types.StructType
  */
 class TableFileCatalog(
     sparkSession: SparkSession,
-    db: String,
-    table: String,
-    partitionSchema: Option[StructType],
+    val db: String,
+    val table: String,
+    override val partitionSchema: StructType,
     override val sizeInBytes: Long,
     fileStatusCache: FileStatusCache = NoopCache) extends FileCatalog {
 

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
@@ -230,7 +230,7 @@ private[hive] class HiveMetastoreCatalog(sparkSession: SparkSession) extends Log
         val sizeInBytes = metastoreRelation.statistics.sizeInBytes.toLong
         val fileCatalog = {
           val catalog = new TableFileCatalog(
-            sparkSession, db, table, Some(partitionSchema), sizeInBytes,
+            sparkSession, db, table, partitionSchema, sizeInBytes,
             FileStatusCache.getOrInitializeShared(new Object(), sparkSession))
           if (lazyPruningEnabled) {
             catalog