Skip to content

Commit

Permalink
[SPARK-20204][SQL][FOLLOWUP] SQLConf should react to change in defaul…
Browse files Browse the repository at this point in the history
…t timezone settings

## What changes were proposed in this pull request?
Make sure SESSION_LOCAL_TIMEZONE reflects the change in JVM's default timezone setting. Currently several timezone related tests fail as the change to default timezone is not picked up by SQLConf.

## How was this patch tested?
Added an unit test in ConfigEntrySuite

Author: Dilip Biswal <dbiswal@us.ibm.com>

Closes #17537 from dilipbiswal/timezone_debug.
  • Loading branch information
dilipbiswal authored and cloud-fan committed Apr 6, 2017
1 parent 9543fc0 commit 9d68c67
Show file tree
Hide file tree
Showing 4 changed files with 35 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,14 @@ private[spark] class TypedConfigBuilder[T](
}
}

/** Creates a [[ConfigEntry]] with a function to determine the default value */
def createWithDefaultFunction(defaultFunc: () => T): ConfigEntry[T] = {
val entry = new ConfigEntryWithDefaultFunction[T](parent.key, defaultFunc, converter,
stringConverter, parent._doc, parent._public)
parent._onCreate.foreach(_ (entry))
entry
}

/**
* Creates a [[ConfigEntry]] that has a default value. The default value is provided as a
* [[String]] and must be a valid value for the entry.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,24 @@ private class ConfigEntryWithDefault[T] (
def readFrom(reader: ConfigReader): T = {
reader.get(key).map(valueConverter).getOrElse(_defaultValue)
}
}

private class ConfigEntryWithDefaultFunction[T] (
key: String,
_defaultFunction: () => T,
valueConverter: String => T,
stringConverter: T => String,
doc: String,
isPublic: Boolean)
extends ConfigEntry(key, valueConverter, stringConverter, doc, isPublic) {

override def defaultValue: Option[T] = Some(_defaultFunction())

override def defaultValueString: String = stringConverter(_defaultFunction())

def readFrom(reader: ConfigReader): T = {
reader.get(key).map(valueConverter).getOrElse(_defaultFunction())
}
}

private class ConfigEntryWithDefaultString[T] (
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -251,4 +251,13 @@ class ConfigEntrySuite extends SparkFunSuite {
.createWithDefault(null)
testEntryRef(nullConf, ref(nullConf))
}

test("conf entry : default function") {
var data = 0
val conf = new SparkConf()
val iConf = ConfigBuilder(testKey("intval")).intConf.createWithDefaultFunction(() => data)
assert(conf.get(iConf) === 0)
data = 2
assert(conf.get(iConf) === 2)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -752,7 +752,7 @@ object SQLConf {
buildConf("spark.sql.session.timeZone")
.doc("""The ID of session local timezone, e.g. "GMT", "America/Los_Angeles", etc.""")
.stringConf
.createWithDefault(TimeZone.getDefault().getID())
.createWithDefaultFunction(() => TimeZone.getDefault.getID)

val WINDOW_EXEC_BUFFER_SPILL_THRESHOLD =
buildConf("spark.sql.windowExec.buffer.spill.threshold")
Expand Down

0 comments on commit 9d68c67

Please sign in to comment.