added count operation but this implementation need double check

YanTangZhai · Sep 20, 2014 · 98c2a00 · 98c2a00
1 parent 58591d2
commit 98c2a00
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/python/pyspark/streaming/dstream.py b/python/pyspark/streaming/dstream.py
@@ -139,6 +139,12 @@ def combineByKey(self, createCombiner, mergeValue, mergeCombiners,
         def combineLocally(iterator):
             combiners = {}
             for x in iterator:
+
+                #TODO for count operation make sure count implementation
+                # This is different from what pyspark does
+                if isinstance(x, int):
+                    x = ("", x)
+
                 (k, v) = x
                 if k not in combiners:
                     combiners[k] = createCombiner(v)