From 6c0997c5c286d80503bbe1423902318571663365 Mon Sep 17 00:00:00 2001 From: Kousuke Saruta Date: Mon, 11 Aug 2014 20:56:50 +0900 Subject: [PATCH 1/3] Modified the timing of shutdown hook execution. It should be executed before shutdown hook of o.a.h.f.FileSystem --- .../spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala index 4d0c506c5a397..fa67a3222c484 100755 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala @@ -26,6 +26,8 @@ import jline.{ConsoleReader, History} import org.apache.commons.lang.StringUtils import org.apache.commons.logging.LogFactory import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.FileSystem +import org.apache.hadoop.util.ShutdownHookManager import org.apache.hadoop.hive.cli.{CliDriver, CliSessionState, OptionsProcessor} import org.apache.hadoop.hive.common.LogUtils.LogInitializationException import org.apache.hadoop.hive.common.{HiveInterruptCallback, HiveInterruptUtils, LogUtils} @@ -116,13 +118,14 @@ private[hive] object SparkSQLCLIDriver { SessionState.start(sessionState) // Clean up after we exit - Runtime.getRuntime.addShutdownHook( + // This should be executed before shutdown hook of FileSystem to avoid race condition of FileSystem operation + ShutdownHookManager.get.addShutdownHook( new Thread() { override def run() { SparkSQLEnv.stop() } } - ) + , FileSystem.SHUTDOWN_HOOK_PRIORITY - 1) // "-h" option has been passed, so connect to Hive thrift server. if (sessionState.getHost != null) { From 8bd833cd6355d8813f821b8e48283ce89aadb222 Mon Sep 17 00:00:00 2001 From: Kousuke Saruta Date: Mon, 11 Aug 2014 21:43:20 +0900 Subject: [PATCH 2/3] Modified style --- .../spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala index fa67a3222c484..9a7a4ea46a06d 100755 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala @@ -118,7 +118,10 @@ private[hive] object SparkSQLCLIDriver { SessionState.start(sessionState) // Clean up after we exit - // This should be executed before shutdown hook of FileSystem to avoid race condition of FileSystem operation + /** + This should be executed before shutdown hook of + * FileSystem to avoid race condition of FileSystem operation + */ ShutdownHookManager.get.addShutdownHook( new Thread() { override def run() { From 4a2d2fe4f01b559abfaba1efc890416bd43e88d6 Mon Sep 17 00:00:00 2001 From: Kousuke Saruta Date: Tue, 12 Aug 2014 03:21:03 +0900 Subject: [PATCH 3/3] Modified comment style --- .../apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala index 9a7a4ea46a06d..4ed0f58ebc531 100755 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala @@ -119,7 +119,7 @@ private[hive] object SparkSQLCLIDriver { // Clean up after we exit /** - This should be executed before shutdown hook of + * This should be executed before shutdown hook of * FileSystem to avoid race condition of FileSystem operation */ ShutdownHookManager.get.addShutdownHook(