diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index 0b155a77d8d42..f4973d5f1bc09 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -361,10 +361,15 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli override protected def initialValue(): Properties = new Properties() } + // Keys of local properties that should not be inherited by children threads + private val nonInheritedLocalProperties: HashSet[String] = new HashSet[String] + /** - * Keys of local properties that should not be inherited by children threads. + * Mark a local property such that its values are never inherited across the thread hierarchy. */ - private[spark] val nonInheritedLocalProperties: HashSet[String] = new HashSet[String] + private[spark] def markLocalPropertyNonInherited(key: String): Unit = { + nonInheritedLocalProperties += key + } /* ------------------------------------------------------------------------------------- * | Initialization. This code initializes the context in a manner that is exception-safe. | diff --git a/core/src/test/scala/org/apache/spark/ThreadingSuite.scala b/core/src/test/scala/org/apache/spark/ThreadingSuite.scala index 17b1e30dd8212..5c1ca00e7cf3e 100644 --- a/core/src/test/scala/org/apache/spark/ThreadingSuite.scala +++ b/core/src/test/scala/org/apache/spark/ThreadingSuite.scala @@ -214,7 +214,7 @@ class ThreadingSuite extends SparkFunSuite with LocalSparkContext with Logging { test("inheritance exclusions (SPARK-10548)") { sc = new SparkContext("local", "test") - sc.nonInheritedLocalProperties.add("do-not-inherit-me") + sc.markLocalPropertyNonInherited("do-not-inherit-me") sc.setLocalProperty("do-inherit-me", "parent") sc.setLocalProperty("do-not-inherit-me", "parent") var throwable: Option[Throwable] = None diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala index f13f3c88ac6b4..5d0236653040e 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala @@ -81,7 +81,7 @@ class SQLContext(@transient val sparkContext: SparkContext) // Ensure query execution IDs are not inherited across the thread hierarchy, which is // the default behavior for SparkContext local properties. Otherwise, we may confuse // the listener as to which query is being executed. (SPARK-10548) - sparkContext.nonInheritedLocalProperties.add(SQLExecution.EXECUTION_ID_KEY) + sparkContext.markLocalPropertyNonInherited(SQLExecution.EXECUTION_ID_KEY) /** * Set Spark SQL configuration properties.