From dfbed38cb1e89725fc45e746d3662f45b1f3370d Mon Sep 17 00:00:00 2001
From: John Penner <jpenner@gmail.com>
Date: Thu, 3 Aug 2017 19:40:41 -0400
Subject: [PATCH] Add error catching for Activity polling

---
 garcon/activity.py     | 17 ++++++++++++++++-
 tests/test_activity.py | 24 +++++++++++++++++++++++-
 2 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/garcon/activity.py b/garcon/activity.py
index f023e57..10c26da 100755
--- a/garcon/activity.py
+++ b/garcon/activity.py
@@ -253,7 +253,22 @@ def run(self):
         previous activity is consumed (context).
         """
 
-        activity_task = self.poll()
+        try:
+            activity_task = self.poll()
+        except Exception as error:
+            # Catch exceptions raised during poll() to avoid an Activity thread
+            # dying & worker daemon unable to process the affected Activity.
+            # AWS api limits on SWF calls are a common source of such
+            # exceptions (see https://github.com/xethorn/garcon/pull/75)
+
+            # on_exception() can be overriden by the flow to send an alert
+            # when an exception occurs.
+            if self.on_exception:
+                self.on_exception(self, error)
+
+            self.logger.error(error, exc_info=True)
+            return True
+
         packed_context = activity_task.get('input')
         context = dict()
 
diff --git a/tests/test_activity.py b/tests/test_activity.py
index 22effbe..7213801 100755
--- a/tests/test_activity.py
+++ b/tests/test_activity.py
@@ -75,7 +75,7 @@ def test_run_activity(monkeypatch, poll):
 
 
 def test_run_capture_exception(monkeypatch, poll):
-    """Run an activity with an exception raised.
+    """Run an activity with an exception raised during activity execution.
     """
 
     current_activity = activity_run(monkeypatch, poll=poll)
@@ -93,6 +93,28 @@ def test_run_capture_exception(monkeypatch, poll):
     assert not current_activity.complete.called
 
 
+def test_run_capture_poll_exception(monkeypatch, poll):
+    """Run an activity with an exception raised during poll.
+    """
+
+    current_activity = activity_run(monkeypatch, poll=poll)
+    current_activity.on_exception = MagicMock()
+    current_activity.execute_activity = MagicMock()
+    exception = Exception('poll exception')
+    current_activity.poll.side_effect = exception
+    current_activity.run()
+
+    assert current_activity.poll.called
+    assert current_activity.on_exception.called
+    assert not current_activity.execute_activity.called
+    assert not current_activity.complete.called
+
+    current_activity.on_exception = None
+    current_activity.logger.error = MagicMock()
+    current_activity.run()
+    current_activity.logger.error.assert_called_with(exception, exc_info=True)
+
+
 def test_run_activity_without_id(monkeypatch):
     """Run an activity without an activity id.
     """