changed for SCCallSiteSync

YanTangZhai · Sep 21, 2014 · 5cdb6fa · 5cdb6fa
1 parent e685853
commit 5cdb6fa
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 7 deletions.
diff --git a/core/pom.xml b/core/pom.xml
@@ -21,11 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent</artifactId>
-<<<<<<< HEAD
     <version>1.2.0-SNAPSHOT</version>
-=======
-    <version>1.1.0-SNAPSHOT</version>
->>>>>>> modified the code base on comment in https://github.com/tdas/spark/pull/10
     <relativePath>../pom.xml</relativePath>
   </parent>
 

diff --git a/python/pyspark/streaming/dstream.py b/python/pyspark/streaming/dstream.py
@@ -22,11 +22,10 @@
 from pyspark.serializers import NoOpSerializer,\
     BatchedSerializer, CloudPickleSerializer, pack_long,\
     CompressedSerializer
-from pyspark.rdd import _JavaStackTrace
 from pyspark.storagelevel import StorageLevel
 from pyspark.resultiterable import ResultIterable
 from pyspark.streaming.util import rddToFileName, RDDFunction
-
+from pyspark.traceback_utils import SCCallSiteSync
 
 from py4j.java_collections import ListConverter, MapConverter
 
@@ -187,7 +186,7 @@ def add_shuffle_key(split, iterator):
                 yield outputSerializer.dumps(items)
         keyed = PipelinedDStream(self, add_shuffle_key)
         keyed._bypass_serializer = True
-        with _JavaStackTrace(self.ctx) as st:
+        with SCCallSiteSync(self.context) as css:
             partitioner = self.ctx._jvm.PythonPartitioner(numPartitions,
                                                           id(partitionFunc))
             jdstream = self.ctx._jvm.PythonPairwiseDStream(keyed._jdstream.dstream(),