Merge pull request #75 from someboredkiddo/fix_activity_poll_exception

Add error catching for Activity polling

garcon/activity.py

@@ -253,7 +253,22 @@ def run(self):
         previous activity is consumed (context).
         """
 
-        activity_task = self.poll()
+        try:
+            activity_task = self.poll()
+        except Exception as error:
+            # Catch exceptions raised during poll() to avoid an Activity thread
+            # dying & worker daemon unable to process the affected Activity.
+            # AWS api limits on SWF calls are a common source of such
+            # exceptions (see https://github.com/xethorn/garcon/pull/75)
+
+            # on_exception() can be overriden by the flow to send an alert
+            # when an exception occurs.
+            if self.on_exception:
+                self.on_exception(self, error)
+
+            self.logger.error(error, exc_info=True)
+            return True
+
         packed_context = activity_task.get('input')
         context = dict()
 

tests/test_activity.py

@@ -75,7 +75,7 @@ def test_run_activity(monkeypatch, poll):
 
 
 def test_run_capture_exception(monkeypatch, poll):
-    """Run an activity with an exception raised.
+    """Run an activity with an exception raised during activity execution.
     """
 
     current_activity = activity_run(monkeypatch, poll=poll)
@@ -93,6 +93,28 @@ def test_run_capture_exception(monkeypatch, poll):
     assert not current_activity.complete.called
 
 
+def test_run_capture_poll_exception(monkeypatch, poll):
+    """Run an activity with an exception raised during poll.
+    """
+
+    current_activity = activity_run(monkeypatch, poll=poll)
+    current_activity.on_exception = MagicMock()
+    current_activity.execute_activity = MagicMock()
+    exception = Exception('poll exception')
+    current_activity.poll.side_effect = exception
+    current_activity.run()
+
+    assert current_activity.poll.called
+    assert current_activity.on_exception.called
+    assert not current_activity.execute_activity.called
+    assert not current_activity.complete.called
+
+    current_activity.on_exception = None
+    current_activity.logger.error = MagicMock()
+    current_activity.run()
+    current_activity.logger.error.assert_called_with(exception, exc_info=True)
+
+
 def test_run_activity_without_id(monkeypatch):
     """Run an activity without an activity id.
     """