Parsing workers

cemsbr · May 14, 2015 · 798b0ff · 798b0ff
1 parent 397c446
commit 798b0ff
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 0 deletions.
diff --git a/sparklogstats/logparser.py b/sparklogstats/logparser.py
@@ -5,6 +5,7 @@ class LogParser:
     def __init__(self, filename):
         self.start = -1
         self.end = -1
+        self.workers = set()
 
         self._parse_file(filename)
 
@@ -20,6 +21,8 @@ def _parse_json(self, json):
             self.start = json["Timestamp"]
         elif event == "SparkListenerApplicationEnd":
             self.end = json["Timestamp"]
+        elif event == "SparkListenerExecutorAdded":
+            self.workers.add(json["Executor Info"]["Host"])
 
     @property
     def duration(self):

diff --git a/tests/test_app.py b/tests/test_app.py
@@ -9,6 +9,10 @@ def test_total_time(self):
         rounded_secs = round(log.duration / 1000)
         self.assertEqual(rounded_secs, 33)
 
+    def test_worker_amount(self):
+        log = LogParser('app-20150427122457-0000')
+        self.assertEqual(len(log.workers), 2)
+
 
 if __name__ == '__main__':
     unittest.main()