Retry failed backport due to redis ConnectionError up to 5 times (#229)

Closes python/miss-islington#228

miss_islington/backport_pr.py

@@ -1,9 +1,11 @@
-import gidgethub.routing
-
+import asyncio
+import os
 import random
 
-from . import tasks
-from . import util
+import gidgethub.routing
+import redis
+
+from . import tasks, util
 
 EASTER_EGG = "I'm not a witch! I'm not a witch!"
 
@@ -46,7 +48,10 @@ async def backport_pr(event, gh, *args, **kwargs):
                 thanks_to = f"Thanks @{created_by} for the PR 🌮🎉."
             else:
                 thanks_to = f"Thanks @{created_by} for the PR, and @{merged_by} for merging it 🌮🎉."
-            message = f"{thanks_to}. I'm working now to backport this PR to: {', '.join(branches)}." f"\n🐍🍒⛏🤖 {easter_egg}"
+            message = (
+                f"{thanks_to}. I'm working now to backport this PR to: {', '.join(branches)}."
+                f"\n🐍🍒⛏🤖 {easter_egg}"
+            )
 
             await util.leave_comment(gh, issue_number, message)
 
@@ -55,10 +60,38 @@ async def backport_pr(event, gh, *args, **kwargs):
             )
 
             for branch in sorted_branches:
-                tasks.backport_task.delay(
-                    commit_hash,
-                    branch,
-                    issue_number=issue_number,
-                    created_by=created_by,
-                    merged_by=merged_by,
+                await kickoff_backport_task(
+                    gh, commit_hash, branch, issue_number, created_by, merged_by
                 )
+
+
+async def kickoff_backport_task(
+    gh, commit_hash, branch, issue_number, created_by, merged_by, retry_num=0
+):
+    try:
+        tasks.backport_task.delay(
+            commit_hash,
+            branch,
+            issue_number=issue_number,
+            created_by=created_by,
+            merged_by=merged_by,
+        )
+    except redis.exceptions.ConnectionError as ce:
+        retry_num = retry_num + 1
+        if retry_num < 5:
+            err_message = f"I'm having trouble backporting to `{branch}`. Reason: '`{ce}`'. Will retry in 1 minute. Retry # {retry_num}"
+            await util.leave_comment(gh, issue_number, err_message)
+            await asyncio.sleep(int(os.environ.get("RETRY_SLEEP_TIME", "60")))
+            await kickoff_backport_task(
+                gh,
+                commit_hash,
+                branch,
+                issue_number,
+                created_by,
+                merged_by,
+                retry_num=retry_num,
+            )
+        else:
+            err_message = f"I'm still having trouble backporting after {retry_num} attempts. Please backport manually."
+            await util.leave_comment(gh, issue_number, err_message)
+            await util.assign_pr_to_core_dev(gh, issue_number, merged_by)

tests/test_backport_pr.py

@@ -1,19 +1,23 @@
 import os
-
-
 from unittest import mock
+
+import redis
 from gidgethub import sansio
 
 os.environ["REDIS_URL"] = "someurl"
+os.environ["RETRY_SLEEP_TIME"] = "1"
+
 from miss_islington import backport_pr
 
 
 class FakeGH:
-    def __init__(self, *, getitem=None, post=None):
+    def __init__(self, *, getitem=None, post=None, patch=None):
         self._getitem_return = getitem
         self.getitem_url = None
         self.getiter_url = None
         self._post_return = post
+        self._patch_return = patch
+        self.patch_url = self.patch_data = None
 
     async def getitem(self, url, url_vars={}):
         self.getitem_url = sansio.format_url(url, url_vars)
@@ -24,6 +28,11 @@ async def post(self, url, *, data):
         self.post_data = data
         return self._post_return
 
+    async def patch(self, url, *, data):
+        self.patch_url = url
+        self.patch_data = data
+        return self._patch_return
+
 
 async def test_unmerged_pr_is_ignored():
     data = {"action": "closed", "pull_request": {"merged": False}}
@@ -206,3 +215,36 @@ async def test_easter_egg():
         assert "Thanks @gvanrossum for the PR" in gh.post_data["body"]
         assert "I'm not a witch" in gh.post_data["body"]
         assert gh.post_url == "/repos/python/cpython/issues/1/comments"
+
+
+async def test_backport_pr_redis_connection_error():
+    data = {
+        "action": "closed",
+        "pull_request": {
+            "merged": True,
+            "number": 1,
+            "merged_by": {"login": "Mariatta"},
+            "user": {"login": "gvanrossum"},
+            "merge_commit_sha": "f2393593c99dd2d3ab8bfab6fcc5ddee540518a9",
+        },
+        "repository": {
+            "issues_url": "https://api.github.com/repos/python/cpython/issues/1"
+        },
+    }
+    event = sansio.Event(data, event="pull_request", delivery_id="1")
+
+    getitem = {
+        "https://api.github.com/repos/python/cpython/issues/1": {
+            "labels_url": "https://api.github.com/repos/python/cpython/issues/1/labels{/name}"
+        },
+        "https://api.github.com/repos/python/cpython/issues/1/labels": [
+            {"name": "CLA signed"},
+            {"name": "needs backport to 3.7"},
+        ],
+    }
+
+    gh = FakeGH(getitem=getitem)
+    with mock.patch("miss_islington.tasks.backport_task.delay") as backport_delay_mock:
+        backport_delay_mock.side_effect = redis.exceptions.ConnectionError
+        await backport_pr.router.dispatch(event, gh)
+        assert "trouble backporting after 5 attempts" in gh.post_data["body"]