Skip to content

Commit

Permalink
[KYUUBI-169]Let spark prefer heap buffer by default while using netty…
Browse files Browse the repository at this point in the history
… rpc #172

fix #169 set spark.shuffle.io.preferDirectBufs = false by default (#172)
  • Loading branch information
yaooqinn committed Mar 21, 2019
1 parent ba9b424 commit 7790bbd
Showing 1 changed file with 5 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,10 @@ object KyuubiSparkUtil extends Logging {
val MULTIPLE_CONTEXTS: String = SPARK_PREFIX + DRIVER_PREFIX + "allowMultipleContexts"
val MULTIPLE_CONTEXTS_DEFAULT = "true"

// Shuffle
val PREFER_DIRECTBUF: String = SPARK_PREFIX + "shuffle." + "io.preferDirectBufs"
val PREFER_DIRECTBUF_DEFAULT = "false"

// Spark SQL
val CATALOG_IMPL: String = SPARK_PREFIX + SQL_PREFIX + "catalogImplementation"
val CATALOG_IMPL_DEFAULT = "hive"
Expand Down Expand Up @@ -295,6 +299,7 @@ object KyuubiSparkUtil extends Logging {
// Set missing Kyuubi configs to SparkConf
KyuubiConf.getAllDefaults.foreach(kv => conf.setIfMissing(kv._1, kv._2))

conf.setIfMissing(PREFER_DIRECTBUF, PREFER_DIRECTBUF_DEFAULT)
conf.setIfMissing(SPARK_LOCAL_DIR, conf.get(KyuubiConf.BACKEND_SESSION_LOCAL_DIR.key))
conf.setIfMissing(GC_INTERVAL, GC_INTERVAL_DEFAULT)
if (UserGroupInformation.isSecurityEnabled) {
Expand Down

0 comments on commit 7790bbd

Please sign in to comment.