Merge pull request #293 from openedx/alangsto/add_error_handling_for_…

…rate_limiting

fix: add rate-limiting retry for enterprise requests

CHANGELOG.rst

@@ -15,6 +15,10 @@ Unreleased
 ----------
 
 =========================
+[4.2.1] - 2022-03-15
+---------------------
+  * Added error handling for rate limit exceptions
+
 [4.2.0] - 2022-03-15
 ---------------------
   * Removed currently broken admin url inclusion from enterprise-data.

enterprise_data/__init__.py

@@ -2,6 +2,6 @@
 Enterprise data api application. This Django app exposes API endpoints used by enterprises.
 """
 
-__version__ = "4.2.0"
+__version__ = "4.2.1"
 
 default_app_config = "enterprise_data.apps.EnterpriseDataAppConfig"  # pylint: disable=invalid-name

enterprise_reporting/clients/__init__.py

@@ -6,8 +6,10 @@
 from datetime import datetime
 from functools import wraps
 from urllib.parse import parse_qs, urlparse
+import time
 
 from edx_rest_api_client.client import EdxRestApiClient
+from edx_rest_api_client.exceptions import HttpClientError
 
 
 class EdxOAuth2APIClient:
@@ -129,7 +131,16 @@ def traverse_pagination(response, endpoint):
     next_page = response.get('next')
     while next_page:
         querystring = parse_qs(urlparse(next_page).query, True)
-        response = endpoint.get(**querystring)
+        try:
+            response = endpoint.get(**querystring)
+        except HttpClientError as exc:
+            # If we get HTTP429, we should pause for a period of time before re-requesting one more time.
+            # This status code is expected because of rate limiting on the endpoint
+            if exc.response.status_code == 429:
+                time.sleep(5)
+                response = endpoint.get(**querystring)
+            else:
+                raise exc
         results += response.get('results', [])
         next_page = response.get('next')