From 51812a75caaed811153223e01a82407fc71102f6 Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Tue, 12 May 2015 17:03:15 -0700 Subject: [PATCH] Change shuffle manager sort name to tungsten-sort --- core/src/main/scala/org/apache/spark/SparkEnv.scala | 2 +- .../apache/spark/shuffle/unsafe/UnsafeShuffleManager.scala | 4 ++-- .../org/apache/spark/shuffle/unsafe/UnsafeShuffleSuite.scala | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/SparkEnv.scala b/core/src/main/scala/org/apache/spark/SparkEnv.scala index 8c40bc93863b2..a5d831c7e68ad 100644 --- a/core/src/main/scala/org/apache/spark/SparkEnv.scala +++ b/core/src/main/scala/org/apache/spark/SparkEnv.scala @@ -314,7 +314,7 @@ object SparkEnv extends Logging { val shortShuffleMgrNames = Map( "hash" -> "org.apache.spark.shuffle.hash.HashShuffleManager", "sort" -> "org.apache.spark.shuffle.sort.SortShuffleManager", - "unsafe" -> "org.apache.spark.shuffle.unsafe.UnsafeShuffleManager") + "tungsten-sort" -> "org.apache.spark.shuffle.unsafe.UnsafeShuffleManager") val shuffleMgrName = conf.get("spark.shuffle.manager", "sort") val shuffleMgrClass = shortShuffleMgrNames.getOrElse(shuffleMgrName.toLowerCase, shuffleMgrName) val shuffleManager = instantiateClass[ShuffleManager](shuffleMgrClass) diff --git a/core/src/main/scala/org/apache/spark/shuffle/unsafe/UnsafeShuffleManager.scala b/core/src/main/scala/org/apache/spark/shuffle/unsafe/UnsafeShuffleManager.scala index 2da7691f5af8f..ce684fbe59d79 100644 --- a/core/src/main/scala/org/apache/spark/shuffle/unsafe/UnsafeShuffleManager.scala +++ b/core/src/main/scala/org/apache/spark/shuffle/unsafe/UnsafeShuffleManager.scala @@ -117,8 +117,8 @@ private[spark] class UnsafeShuffleManager(conf: SparkConf) extends ShuffleManage if (!conf.getBoolean("spark.shuffle.spill", true)) { logWarning( - "spark.shuffle.spill was set to false, but this is ignored by UnsafeShuffleManager; " + - "its optimized shuffles will continue to spill to disk when necessary.") + "spark.shuffle.spill was set to false, but this is ignored by the tungsten-sort shuffle " + + "manager; its optimized shuffles will continue to spill to disk when necessary.") } diff --git a/core/src/test/scala/org/apache/spark/shuffle/unsafe/UnsafeShuffleSuite.scala b/core/src/test/scala/org/apache/spark/shuffle/unsafe/UnsafeShuffleSuite.scala index f7eefa2a3f40c..e68261a730d3a 100644 --- a/core/src/test/scala/org/apache/spark/shuffle/unsafe/UnsafeShuffleSuite.scala +++ b/core/src/test/scala/org/apache/spark/shuffle/unsafe/UnsafeShuffleSuite.scala @@ -25,7 +25,7 @@ class UnsafeShuffleSuite extends ShuffleSuite with BeforeAndAfterAll { // This test suite should run all tests in ShuffleSuite with unsafe-based shuffle. override def beforeAll() { - conf.set("spark.shuffle.manager", "unsafe") + conf.set("spark.shuffle.manager", "tungsten-sort") // UnsafeShuffleManager requires at least 128 MB of memory per task in order to be able to sort // shuffle records. conf.set("spark.shuffle.memoryFraction", "0.5")