From 90b25a3585cdce1f0fac9c8e43152e5efd8fed13 Mon Sep 17 00:00:00 2001 From: Yin Huai Date: Tue, 18 Aug 2015 18:48:23 -0700 Subject: [PATCH 1/2] Disable spark.shuffle.reduceLocality.enabled by default. --- .../main/scala/org/apache/spark/scheduler/DAGScheduler.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala index dadf83a38296d..c675ab3d5db12 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala @@ -138,7 +138,7 @@ class DAGScheduler( // Flag to control if reduce tasks are assigned preferred locations private val shuffleLocalityEnabled = - sc.getConf.getBoolean("spark.shuffle.reduceLocality.enabled", true) + sc.getConf.getBoolean("spark.shuffle.reduceLocality.enabled", false) // Number of map, reduce tasks above which we do not assign preferred locations // based on map output sizes. We limit the size of jobs for which assign preferred locations // as computing the top locations by size becomes expensive. From 40e28e3f0101a04884b61d77ebdc4911128ce4f9 Mon Sep 17 00:00:00 2001 From: Yin Huai Date: Tue, 18 Aug 2015 21:26:24 -0700 Subject: [PATCH 2/2] Ignore tests. --- .../scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala index a063596d3eae2..c858e73549b1e 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala @@ -909,7 +909,7 @@ class DAGSchedulerSuite assertDataStructuresEmpty() } - test("reduce tasks should be placed locally with map output") { + ignore("reduce tasks should be placed locally with map output") { // Create an shuffleMapRdd with 1 partition val shuffleMapRdd = new MyRDD(sc, 1, Nil) val shuffleDep = new ShuffleDependency(shuffleMapRdd, null) @@ -929,7 +929,7 @@ class DAGSchedulerSuite assertDataStructuresEmpty() } - test("reduce task locality preferences should only include machines with largest map outputs") { + ignore("reduce task locality preferences should only include machines with largest map outputs") { val numMapTasks = 4 // Create an shuffleMapRdd with more partitions val shuffleMapRdd = new MyRDD(sc, numMapTasks, Nil)