From 798b0ffb0a32ba51997d165647090bc4248896c6 Mon Sep 17 00:00:00 2001 From: Carlos Eduardo Moreira dos Santos Date: Thu, 14 May 2015 09:58:43 +0900 Subject: [PATCH] Parsing workers --- sparklogstats/logparser.py | 3 +++ tests/test_app.py | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/sparklogstats/logparser.py b/sparklogstats/logparser.py index 8660575..3af6c98 100644 --- a/sparklogstats/logparser.py +++ b/sparklogstats/logparser.py @@ -5,6 +5,7 @@ class LogParser: def __init__(self, filename): self.start = -1 self.end = -1 + self.workers = set() self._parse_file(filename) @@ -20,6 +21,8 @@ def _parse_json(self, json): self.start = json["Timestamp"] elif event == "SparkListenerApplicationEnd": self.end = json["Timestamp"] + elif event == "SparkListenerExecutorAdded": + self.workers.add(json["Executor Info"]["Host"]) @property def duration(self): diff --git a/tests/test_app.py b/tests/test_app.py index d3b8db1..d7fef29 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -9,6 +9,10 @@ def test_total_time(self): rounded_secs = round(log.duration / 1000) self.assertEqual(rounded_secs, 33) + def test_worker_amount(self): + log = LogParser('app-20150427122457-0000') + self.assertEqual(len(log.workers), 2) + if __name__ == '__main__': unittest.main()