From 8553125f6f6649e7e5760e05a043f60fc4e2da23 Mon Sep 17 00:00:00 2001 From: Venkata Ramana Gollamudi Date: Fri, 10 Apr 2015 22:50:29 +0530 Subject: [PATCH 1/3] Fix Hive setConf issue --- .../src/main/scala/org/apache/spark/sql/SQLConf.scala | 11 +++++++++-- .../main/scala/org/apache/spark/sql/SQLContext.scala | 2 +- .../scala/org/apache/spark/sql/hive/HiveContext.scala | 2 +- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala index ee641bdfeb2d7..5ed3fe31c7f6a 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala @@ -199,8 +199,15 @@ private[sql] class SQLConf extends Serializable { /** ********************** SQLConf functionality methods ************ */ /** Set Spark SQL configuration properties. */ - def setConf(props: Properties): Unit = settings.synchronized { - props.foreach { case (k, v) => settings.put(k, v) } + def setConf(props: Properties, overwrite: Boolean = true): Unit = settings.synchronized { + if (overwrite) { + props.foreach { case (k, v) => settings.put(k, v) } + } else { + props + .filter(p => !settings.containsKey(p._1)) + .foreach { case (k, v) => settings.put(k, v) } + } + } /** Set the given Spark SQL configuration property. */ diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala index c25ef58e6f62a..e761009f7db79 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala @@ -73,7 +73,7 @@ class SQLContext(@transient val sparkContext: SparkContext) * * @group config */ - def setConf(props: Properties): Unit = conf.setConf(props) + def setConf(props: Properties, overwrite: Boolean = true): Unit = conf.setConf(props, overwrite) /** * Set the given Spark SQL configuration property. diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala index 7c6a7df2bd01e..e13f0e98e0f65 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala @@ -264,7 +264,7 @@ class HiveContext(sc: SparkContext) extends SQLContext(sc) { } protected[hive] lazy val hiveconf: HiveConf = { - setConf(sessionState.getConf.getAllProperties) + setConf(sessionState.getConf.getAllProperties, false) sessionState.getConf } From bb2fb758030f0b29dd0d76c9284781d73ea96151 Mon Sep 17 00:00:00 2001 From: Venkata Ramana Gollamudi Date: Fri, 10 Apr 2015 22:51:07 +0530 Subject: [PATCH 2/3] Added Test --- .../spark/sql/hive/HiveContextInitSuite.scala | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveContextInitSuite.scala diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveContextInitSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveContextInitSuite.scala new file mode 100644 index 0000000000000..7ee7a74865204 --- /dev/null +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveContextInitSuite.scala @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hive + +import org.apache.spark.{SparkConf, SparkContext} +import org.apache.spark.sql.hive.test.TestHiveContext +import org.scalatest.{BeforeAndAfterAll, FunSuite} + +class HiveContextInitSuite extends FunSuite with BeforeAndAfterAll { + test("SPARK-6675 Hive setConf") { + val sparkConf = new SparkConf() + val hc = + new TestHiveContext(new SparkContext("local", s"TestSQLContext", sparkConf)) + hc.setConf("hive.metastore.warehouse.dir", "/home/spark/hive/warehouse_test") + hc.setConf("spark.sql.shuffle.partitions", "10") + assert(hc.getAllConfs.get("hive.metastore.warehouse.dir") + .toList.contains("/home/spark/hive/warehouse_test")) + assert(hc.getAllConfs.get("spark.sql.shuffle.partitions").toList.contains("10")) + } +} From 6c2ae871109dde59d720efc33e7cd6717b30f5a6 Mon Sep 17 00:00:00 2001 From: Venkata Ramana Gollamudi Date: Mon, 13 Apr 2015 18:58:29 +0530 Subject: [PATCH 3/3] corrected testcase --- .../spark/sql/hive/HiveContextInitSuite.scala | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveContextInitSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveContextInitSuite.scala index 7ee7a74865204..4f06d673e97b6 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveContextInitSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveContextInitSuite.scala @@ -17,19 +17,16 @@ package org.apache.spark.sql.hive -import org.apache.spark.{SparkConf, SparkContext} -import org.apache.spark.sql.hive.test.TestHiveContext -import org.scalatest.{BeforeAndAfterAll, FunSuite} +import org.apache.spark.sql.hive.test.TestHive +import org.scalatest.FunSuite -class HiveContextInitSuite extends FunSuite with BeforeAndAfterAll { +class HiveContextInitSuite extends FunSuite { test("SPARK-6675 Hive setConf") { - val sparkConf = new SparkConf() - val hc = - new TestHiveContext(new SparkContext("local", s"TestSQLContext", sparkConf)) - hc.setConf("hive.metastore.warehouse.dir", "/home/spark/hive/warehouse_test") - hc.setConf("spark.sql.shuffle.partitions", "10") - assert(hc.getAllConfs.get("hive.metastore.warehouse.dir") + val hc = new HiveContext(TestHive.sparkContext) + hc.setConf("hive.metastore.warehouse.dir", "/home/spark/hive/warehouse_test") + hc.setConf("spark.sql.shuffle.partitions", "10") + assert(hc.getAllConfs.get("hive.metastore.warehouse.dir") .toList.contains("/home/spark/hive/warehouse_test")) - assert(hc.getAllConfs.get("spark.sql.shuffle.partitions").toList.contains("10")) + assert(hc.getAllConfs.get("spark.sql.shuffle.partitions").toList.contains("10")) } }