From 32389107c17ccd84a62912230daec61c8b058f99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=AE=B8=E9=B9=8F?= Date: Fri, 1 May 2015 17:02:15 +0800 Subject: [PATCH 1/2] remove temporary directories after the file is inserted into hive table --- .../apache/spark/sql/hive/execution/InsertIntoHiveTable.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala index 89995a91b1a92..7a87f7ffbd019 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala @@ -251,6 +251,10 @@ case class InsertIntoHiveTable( } } + //remove temporary directories + val fs = outputPath.getFileSystem(jobConf) + fs.delete(outputPath,true) + // Invalidate the cache. sqlContext.cacheManager.invalidateCache(table) From d28c5566065079c76bb76ca19cdcb3b17c19091b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=AE=B8=E9=B9=8F?= Date: Sat, 2 May 2015 09:28:18 +0800 Subject: [PATCH 2/2] remove the parent directory instead of the file itself --- .../apache/spark/sql/hive/execution/InsertIntoHiveTable.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala index 7a87f7ffbd019..d4ad47227e02f 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala @@ -253,7 +253,8 @@ case class InsertIntoHiveTable( //remove temporary directories val fs = outputPath.getFileSystem(jobConf) - fs.delete(outputPath,true) + if ( outputPath.getParent.isRoot == false ) + fs.delete(outputPath.getParent,true) // Invalidate the cache. sqlContext.cacheManager.invalidateCache(table)