From 92c1452da5f994a96f1bf5cf90df75492e742746 Mon Sep 17 00:00:00 2001 From: windpiger Date: Mon, 20 Feb 2017 13:04:17 +0800 Subject: [PATCH 1/7] [SPARK-19664][SQL]put hive.metastore.warehouse.dir in hadoopconf to overwrite its original value --- .../main/scala/org/apache/spark/sql/internal/SharedState.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala index 8de95fe64e663..51cf3d3a6a9e2 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala @@ -61,7 +61,7 @@ private[sql] class SharedState(val sparkContext: SparkContext) extends Logging { // When neither spark.sql.warehouse.dir nor hive.metastore.warehouse.dir is set, // we will set hive.metastore.warehouse.dir to the default value of spark.sql.warehouse.dir. val sparkWarehouseDir = sparkContext.conf.get(WAREHOUSE_PATH) - sparkContext.conf.set("hive.metastore.warehouse.dir", sparkWarehouseDir) + sparkContext.hadoopConfiguration.set("hive.metastore.warehouse.dir", sparkWarehouseDir) sparkWarehouseDir } From ac0a1c61d6794de4d049b4dd50593da0aa4f9cfe Mon Sep 17 00:00:00 2001 From: windpiger Date: Mon, 20 Feb 2017 14:56:35 +0800 Subject: [PATCH 2/7] add some loginfo, and fix a bug --- .../org/apache/spark/sql/internal/SharedState.scala | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala index 51cf3d3a6a9e2..e1a5b0785aa1a 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala @@ -42,6 +42,7 @@ private[sql] class SharedState(val sparkContext: SparkContext) extends Logging { val warehousePath = { val configFile = Utils.getContextOrSparkClassLoader.getResource("hive-site.xml") if (configFile != null) { + logInfo(s"load config from hive-site.xml $configFile") sparkContext.hadoopConfiguration.addResource(configFile) } @@ -61,7 +62,13 @@ private[sql] class SharedState(val sparkContext: SparkContext) extends Logging { // When neither spark.sql.warehouse.dir nor hive.metastore.warehouse.dir is set, // we will set hive.metastore.warehouse.dir to the default value of spark.sql.warehouse.dir. val sparkWarehouseDir = sparkContext.conf.get(WAREHOUSE_PATH) - sparkContext.hadoopConfiguration.set("hive.metastore.warehouse.dir", sparkWarehouseDir) + logInfo(s"${WAREHOUSE_PATH.key} is set, Setting " + + s"hive.metastore.warehouse.dir ($hiveWarehouseDir) to the value of " + + s"${WAREHOUSE_PATH.key} ('$sparkWarehouseDir').") + if (hiveWarehouseDir != null) { + sparkContext.hadoopConfiguration.set("hive.metastore.warehouse.dir", sparkWarehouseDir) + } + sparkContext.conf.set("hive.metastore.warehouse.dir", sparkWarehouseDir) sparkWarehouseDir } From 91b9fd2cec7ef8ef9ab4bcf2c5468ef19139f647 Mon Sep 17 00:00:00 2001 From: windpiger Date: Tue, 21 Feb 2017 11:54:13 +0800 Subject: [PATCH 3/7] remove a log info --- .../main/scala/org/apache/spark/sql/internal/SharedState.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala index e1a5b0785aa1a..ccb6e20a17f9a 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala @@ -62,8 +62,7 @@ private[sql] class SharedState(val sparkContext: SparkContext) extends Logging { // When neither spark.sql.warehouse.dir nor hive.metastore.warehouse.dir is set, // we will set hive.metastore.warehouse.dir to the default value of spark.sql.warehouse.dir. val sparkWarehouseDir = sparkContext.conf.get(WAREHOUSE_PATH) - logInfo(s"${WAREHOUSE_PATH.key} is set, Setting " + - s"hive.metastore.warehouse.dir ($hiveWarehouseDir) to the value of " + + logInfo(s"Setting hive.metastore.warehouse.dir ($hiveWarehouseDir) to the value of " + s"${WAREHOUSE_PATH.key} ('$sparkWarehouseDir').") if (hiveWarehouseDir != null) { sparkContext.hadoopConfiguration.set("hive.metastore.warehouse.dir", sparkWarehouseDir) From 7429cd807a2c615803bf4062392ddf9329bb03ee Mon Sep 17 00:00:00 2001 From: windpiger Date: Thu, 23 Feb 2017 18:03:43 +0800 Subject: [PATCH 4/7] always set in hadoopconf --- .../scala/org/apache/spark/sql/internal/SharedState.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala index ccb6e20a17f9a..9a1280789f1ea 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala @@ -64,9 +64,7 @@ private[sql] class SharedState(val sparkContext: SparkContext) extends Logging { val sparkWarehouseDir = sparkContext.conf.get(WAREHOUSE_PATH) logInfo(s"Setting hive.metastore.warehouse.dir ($hiveWarehouseDir) to the value of " + s"${WAREHOUSE_PATH.key} ('$sparkWarehouseDir').") - if (hiveWarehouseDir != null) { - sparkContext.hadoopConfiguration.set("hive.metastore.warehouse.dir", sparkWarehouseDir) - } + sparkContext.hadoopConfiguration.set("hive.metastore.warehouse.dir", sparkWarehouseDir) sparkContext.conf.set("hive.metastore.warehouse.dir", sparkWarehouseDir) sparkWarehouseDir } From 26ba21257aac29e2def5a9db32dd62f052fec2de Mon Sep 17 00:00:00 2001 From: windpiger Date: Thu, 23 Feb 2017 18:07:24 +0800 Subject: [PATCH 5/7] modify a comment --- .../main/scala/org/apache/spark/sql/internal/SharedState.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala index 9a1280789f1ea..ae2f5bef6612d 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala @@ -42,7 +42,7 @@ private[sql] class SharedState(val sparkContext: SparkContext) extends Logging { val warehousePath = { val configFile = Utils.getContextOrSparkClassLoader.getResource("hive-site.xml") if (configFile != null) { - logInfo(s"load config from hive-site.xml $configFile") + logInfo(s"loading hive config file $configFile") sparkContext.hadoopConfiguration.addResource(configFile) } From 1d18636c0b27a25e504eb1ff273354cf6a87e99d Mon Sep 17 00:00:00 2001 From: windpiger Date: Thu, 23 Feb 2017 18:08:52 +0800 Subject: [PATCH 6/7] modify a comment --- .../main/scala/org/apache/spark/sql/internal/SharedState.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala index ae2f5bef6612d..4f1c94a80a81d 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala @@ -42,7 +42,7 @@ private[sql] class SharedState(val sparkContext: SparkContext) extends Logging { val warehousePath = { val configFile = Utils.getContextOrSparkClassLoader.getResource("hive-site.xml") if (configFile != null) { - logInfo(s"loading hive config file $configFile") + logInfo(s"loading hive config file: $configFile") sparkContext.hadoopConfiguration.addResource(configFile) } From 86deb6233faa3b64c999786741a0b0cf3cbbe457 Mon Sep 17 00:00:00 2001 From: windpiger Date: Fri, 24 Feb 2017 10:33:59 +0800 Subject: [PATCH 7/7] remove hive.metastore.warehouse.dir in sparkconf --- .../scala/org/apache/spark/sql/internal/SharedState.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala index 4f1c94a80a81d..5827b28c6d50f 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala @@ -46,6 +46,8 @@ private[sql] class SharedState(val sparkContext: SparkContext) extends Logging { sparkContext.hadoopConfiguration.addResource(configFile) } + // hive.metastore.warehouse.dir only stay in hadoopConf + sparkContext.conf.remove("hive.metastore.warehouse.dir") // Set the Hive metastore warehouse path to the one we use val hiveWarehouseDir = sparkContext.hadoopConfiguration.get("hive.metastore.warehouse.dir") if (hiveWarehouseDir != null && !sparkContext.conf.contains(WAREHOUSE_PATH.key)) { @@ -62,13 +64,11 @@ private[sql] class SharedState(val sparkContext: SparkContext) extends Logging { // When neither spark.sql.warehouse.dir nor hive.metastore.warehouse.dir is set, // we will set hive.metastore.warehouse.dir to the default value of spark.sql.warehouse.dir. val sparkWarehouseDir = sparkContext.conf.get(WAREHOUSE_PATH) - logInfo(s"Setting hive.metastore.warehouse.dir ($hiveWarehouseDir) to the value of " + + logInfo(s"Setting hive.metastore.warehouse.dir ('$hiveWarehouseDir') to the value of " + s"${WAREHOUSE_PATH.key} ('$sparkWarehouseDir').") sparkContext.hadoopConfiguration.set("hive.metastore.warehouse.dir", sparkWarehouseDir) - sparkContext.conf.set("hive.metastore.warehouse.dir", sparkWarehouseDir) sparkWarehouseDir } - } logInfo(s"Warehouse path is '$warehousePath'.")