fix narrow dependency

apache · Apr 7, 2015 · e3b8eab · e3b8eab
1 parent 2a1857a
commit e3b8eab
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/python/pyspark/rdd.py b/python/pyspark/rdd.py
@@ -1802,7 +1802,7 @@ def combine(iterator):
             merger.mergeValues(iterator)
             return merger.iteritems()
 
-        locally_combined = self.mapPartitions(combine)
+        locally_combined = self.mapPartitions(combine, preservesPartitioning=True)
         shuffled = locally_combined.partitionBy(numPartitions)
 
         def groupByKey(it):

diff --git a/python/pyspark/tests.py b/python/pyspark/tests.py
@@ -795,9 +795,9 @@ def test_narrow_dependency_in_join(self):
         self.assertEqual(rdd.getNumPartitions() + 2, parted.union(rdd).getNumPartitions())
         self.assertEqual(rdd.getNumPartitions() + 2, rdd.union(parted).getNumPartitions())
 
-        self.sc.setJobGroup("test1", "test", True)
         tracker = self.sc.statusTracker()
 
+        self.sc.setJobGroup("test1", "test", True)
         d = sorted(parted.join(parted).collect())
         self.assertEqual(10, len(d))
         self.assertEqual((0, (0, 0)), d[0])