Retry HTTP requests after HTTP errors

Retry HTTP requests if we get a HTTP 500 from the server when collecting
 data.  It is a temporary condition that has been encountered with
 hg.mozilla.org and can crash the program on long runs.

Author: mars-f

committelemetry/classifier.py

@@ -13,6 +13,7 @@
 from mozautomation.commitparser import parse_bugs
 
 from committelemetry import config
+from committelemetry.http import requests_retry_session
 
 log = logging.getLogger(__name__)
 
@@ -58,7 +59,7 @@ def fetch_attachments(bug_id):
     """Fetch the given bug's attachment list from Bugzilla."""
     # Example: https://bugzilla.mozilla.org/rest/bug/1447193/attachment?exclude_fields=data
     url = f'{config.BMO_API_URL}/bug/{bug_id}/attachment?exclude_fields=data'
-    response = requests.get(url)
+    response = requests_retry_session().get(url)
 
     # TODO Workaround for bug 1462349.  Can be removed when API calls return the correct value.
     if 'error' in response.json():
@@ -73,7 +74,7 @@ def fetch_bug_history(bug_id):
     """Fetch the given bug's history from Bugzilla."""
     # Example: https://bugzilla.mozilla.org/rest/bug/1447193/history
     url = f'{config.BMO_API_URL}/bug/{bug_id}/history'
-    response = requests.get(url)
+    response = requests_retry_session().get(url)
 
     # TODO Workaround for bug 1462349.  Can be removed when API calls return the correct value.
     if 'error' in response.json():

committelemetry/http.py

@@ -0,0 +1,44 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Functions related to basic network communication tasks.
+"""
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3 import Retry
+
+
+def requests_retry_session(
+    retries=3,
+    backoff_factor=0.3,
+    status_forcelist=(500, 502, 504),
+    session=None
+):
+    """Return a python-requests Session that retries on HTTP failure.
+
+    For usage see
+    https://www.peterbe.com/plog/best-practice-with-retries-with-requests.
+
+    Args:
+        retries: optional int, number of retries to attempt.
+        backoff_factor: See https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#module-urllib3.util.retry.
+        status_forcelist: optional list of HTTP status codes that will trigger
+            a retry.
+        session: optional pre-built requests.Session object.
+
+    Returns:
+        A requests.Session object we can use to call .get(), post() etc.
+    """
+    session = session or requests.Session()
+    retry = Retry(
+        total=retries,
+        read=retries,
+        connect=retries,
+        backoff_factor=backoff_factor,
+        status_forcelist=status_forcelist,
+    )
+    adapter = HTTPAdapter(max_retries=retry)
+    session.mount('http://', adapter)
+    session.mount('https://', adapter)
+    return session

committelemetry/pulse.py

@@ -10,10 +10,10 @@
 from contextlib import closing
 from functools import partial
 
-import requests
 from kombu import Connection, Exchange, Queue
 
 from committelemetry import config
+from committelemetry.http import requests_retry_session
 from committelemetry.telemetry import payload_for_changeset, send_ping
 
 log = logging.getLogger(__name__)
@@ -41,7 +41,7 @@ def changesets_for_pushid(pushid, push_json_url):
         A list of changeset ID strings (40 char hex strings).
     """
     log.info(f'processing pushid {pushid}')
-    response = requests.get(push_json_url)
+    response = requests_retry_session().get(push_json_url)
     response.raise_for_status()
 
     # See https://mozilla-version-control-tools.readthedocs.io/en/latest/hgmo/pushlog.html#version-2

committelemetry/pushlog.py

@@ -8,8 +8,7 @@
 """
 import logging
 
-import requests
-
+from committelemetry.http import requests_retry_session
 from committelemetry.telemetry import payload_for_changeset, send_ping
 
 log = logging.getLogger(__name__)
@@ -30,7 +29,9 @@ def pushes_for_range(repo_url, starting_push_id, ending_push_id):
     """
     # See https://mozilla-version-control-tools.readthedocs.io/en/latest/hgmo/pushlog.html#version-2
     params = dict(startID=starting_push_id, endID=ending_push_id, version=2)
-    response = requests.get(f'{repo_url}/json-pushes/', params=params)
+    response = requests_retry_session().get(
+        f'{repo_url}/json-pushes/', params=params
+    )
     response.raise_for_status()
     pushlog = response.json()
     return pushlog['pushes']
@@ -49,7 +50,9 @@ def send_pings_by_pushid(repo_url, starting_push_id, ending_push_id, no_send):
     if no_send:
         log.info('transmission of ping data has been disabled')
 
-    for pushid, pushdata in pushes_for_range(repo_url, starting_push_id, ending_push_id).items():
+    for pushid, pushdata in pushes_for_range(
+        repo_url, starting_push_id, ending_push_id
+    ).items():
         log.info(f'processing pushid {pushid}')
 
         # See https://mozilla-version-control-tools.readthedocs.io/en/latest/hgmo/pushlog.html#version-2

committelemetry/telemetry.py

@@ -11,6 +11,7 @@
 
 from committelemetry import config
 from committelemetry.classifier import determine_review_system
+from committelemetry.http import requests_retry_session
 
 log = logging.getLogger(__name__)
 
@@ -33,7 +34,8 @@ def payload_for_changeset(changesetid, repo_url):
         mercurial repository.
     """
     # Example URL: https://hg.mozilla.org/mozilla-central/json-rev/deafa2891c61
-    response = requests.get(f'{repo_url}/json-rev/{changesetid}')
+    response = requests_retry_session(
+    ).get(f'{repo_url}/json-rev/{changesetid}')
 
     if response.status_code == 404:
         raise NoSuchChangeset(