-
Notifications
You must be signed in to change notification settings - Fork 44
/
test_streaming_queue.py
55 lines (42 loc) · 1.57 KB
/
test_streaming_queue.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
from __future__ import print_function
import tornado.testing
import pysparkling
class TestCount(tornado.testing.AsyncTestCase):
def test_count(self):
sc = pysparkling.Context()
ssc = pysparkling.streaming.StreamingContext(sc, 0.1)
result = []
(
ssc.queueStream([range(20), ['a', 'b'], ['c']])
.count()
.foreachRDD(lambda rdd: result.append(rdd.collect()[0]))
)
ssc.start()
ssc.awaitTermination(timeout=0.3)
self.assertEqual(sum(result), 23)
def test_groupByKey(self):
sc = pysparkling.Context()
ssc = pysparkling.streaming.StreamingContext(sc, 0.1)
result = []
(
ssc.queueStream([[('a', 5), ('b', 8), ('a', 2)],
[('a', 2), ('b', 3)]])
.groupByKey().mapPartitions(sorted).mapValues(sorted)
.foreachRDD(lambda rdd: result.append(rdd.collect()))
)
ssc.start()
ssc.awaitTermination(timeout=0.25)
self.assertEqual(
result, [[('a', [2, 5]), ('b', [8])], [('a', [2]), ('b', [3])]])
def test_mapValues(self):
sc = pysparkling.Context()
ssc = pysparkling.streaming.StreamingContext(sc, 0.1)
result = []
(
ssc.queueStream([[('a', [5, 8, 2]), ('b', [6, 3, 8])]])
.mapValues(sorted)
.foreachRDD(lambda rdd: result.append(rdd.collect()))
)
ssc.start()
ssc.awaitTermination(timeout=0.15)
self.assertEqual(result, [[('a', [2, 5, 8]), ('b', [3, 6, 8])]])