Merge pull request #479 from geopy/add-rate-limited-exc

Add `GeocoderRateLimited` error, raise it for 429 instead of `GeocoderQuotaExceeded`

docs/index.rst

@@ -394,6 +394,9 @@ Exceptions
 .. autoclass:: geopy.exc.GeocoderQuotaExceeded
     :show-inheritance:
 
+.. autoclass:: geopy.exc.GeocoderRateLimited
+    :show-inheritance:
+
 .. autoclass:: geopy.exc.GeocoderAuthenticationFailure
     :show-inheritance:
 

geopy/adapters.py

@@ -16,7 +16,9 @@
 import abc
 import asyncio
 import contextlib
+import email
 import json
+import time
 import warnings
 from socket import timeout as SocketTimeout
 from ssl import SSLError
@@ -68,18 +70,63 @@ class AdapterHTTPError(IOError):
 
     """
 
-    def __init__(self, message, *, status_code, text):
+    def __init__(self, message, *, status_code, headers, text):
         """
 
         :param str message: Standard exception message.
-        :param int status_code: HTTP status code
-        :param str text: HTTP body text
+        :param int status_code: HTTP status code.
+        :param dict headers: HTTP response readers. A mapping object
+            with lowercased or case-insensitive keys.
+        :param str text: HTTP body text.
+
+        .. versionchanged:: 2.2
+           Added ``headers``.
         """
         self.status_code = status_code
+        self.headers = headers
         self.text = text
         super().__init__(message)
 
 
+def get_retry_after(headers):
+    """Return Retry-After header value in seconds.
+
+    .. versionadded:: 2.2
+    """
+    # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
+    # https://github.com/urllib3/urllib3/blob/1.26.4/src/urllib3/util/retry.py#L376
+
+    try:
+        retry_after = headers['retry-after']
+    except KeyError:
+        return None
+
+    if not retry_after:  # None, ''
+        return None
+
+    retry_after = retry_after.strip()
+
+    # RFC7231 section-7.1.3:
+    # Retry-After = HTTP-date / delay-seconds
+
+    try:
+        # Retry-After: 120
+        seconds = int(retry_after)
+    except ValueError:
+        # Retry-After: Fri, 31 Dec 1999 23:59:59 GMT
+        retry_date_tuple = email.utils.parsedate_tz(retry_after)
+        if retry_date_tuple is None:
+            logger.warning('Invalid Retry-After header: %s', retry_after)
+            return None
+        retry_date = email.utils.mktime_tz(retry_date_tuple)
+        seconds = retry_date - time.time()
+
+    if seconds < 0:
+        seconds = 0
+
+    return seconds
+
+
 class BaseAdapter(abc.ABC):
     """Base class for an Adapter.
 
@@ -253,8 +300,17 @@ def get_text(self, url, *, timeout, headers):
             message = str(error.args[0]) if len(error.args) else str(error)
             if isinstance(error, HTTPError):
                 code = error.getcode()
+                response_headers = {
+                    name.lower(): value
+                    for name, value in error.headers.items()
+                }
                 body = self._read_http_error_body(error)
-                raise AdapterHTTPError(message, status_code=code, text=body)
+                raise AdapterHTTPError(
+                    message,
+                    status_code=code,
+                    headers=response_headers,
+                    text=body,
+                )
             elif isinstance(error, URLError):
                 if "timed out" in message:
                     raise GeocoderTimedOut("Service timed out")
@@ -270,9 +326,15 @@ def get_text(self, url, *, timeout, headers):
             text = self._decode_page(page)
             status_code = page.getcode()
             if status_code >= 400:
+                response_headers = {
+                    name.lower(): value
+                    for name, value in page.headers.items()
+                }
                 raise AdapterHTTPError(
                     "Non-successful status code %s" % status_code,
-                    status_code=status_code, text=text
+                    status_code=status_code,
+                    headers=response_headers,
+                    text=text,
                 )
 
         return text
@@ -405,6 +467,7 @@ def _request(self, url, *, timeout, headers):
                 raise AdapterHTTPError(
                     "Non-successful status code %s" % resp.status_code,
                     status_code=resp.status_code,
+                    headers=resp.headers,
                     text=resp.text,
                 )
 
@@ -490,6 +553,7 @@ async def _raise_for_status(self, resp):
             raise AdapterHTTPError(
                 "Non-successful status code %s" % resp.status,
                 status_code=resp.status,
+                headers=resp.headers,
                 text=await resp.text(),
             )
 

geopy/exc.py

@@ -42,6 +42,24 @@ class GeocoderQuotaExceeded(GeocoderServiceError):
     """
 
 
+class GeocoderRateLimited(GeocoderQuotaExceeded):
+    """
+    The remote geocoding service has rate-limited the request.
+    Retrying later might help.
+
+    Exception of this type has a ``retry_after`` attribute,
+    which contains amount of time (in seconds) the service
+    has asked to wait. Might be ``None`` if there were no such
+    data in response.
+
+    .. versionadded:: 2.2
+    """
+
+    def __init__(self, message, *, retry_after=None):
+        super().__init__(message)
+        self.retry_after = retry_after
+
+
 class GeocoderAuthenticationFailure(GeocoderServiceError):
     """
     The remote geocoding service rejected the API key or account

geopy/geocoders/base.py

@@ -10,13 +10,15 @@
     BaseSyncAdapter,
     RequestsAdapter,
     URLLibAdapter,
+    get_retry_after,
 )
 from geopy.exc import (
     ConfigurationError,
     GeocoderAuthenticationFailure,
     GeocoderInsufficientPrivileges,
     GeocoderQueryError,
     GeocoderQuotaExceeded,
+    GeocoderRateLimited,
     GeocoderServiceError,
     GeocoderTimedOut,
 )
@@ -196,10 +198,11 @@ class options:
     402: GeocoderQuotaExceeded,
     403: GeocoderInsufficientPrivileges,
     407: GeocoderAuthenticationFailure,
+    408: GeocoderTimedOut,
     412: GeocoderQueryError,
     413: GeocoderQueryError,
     414: GeocoderQueryError,
-    429: GeocoderQuotaExceeded,
+    429: GeocoderRateLimited,
     502: GeocoderServiceError,
     503: GeocoderTimedOut,
     504: GeocoderTimedOut
@@ -389,7 +392,12 @@ def _adapter_error_handler(self, error):
                 )
             self._geocoder_exception_handler(error)
             exc_cls = ERROR_CODE_MAP.get(error.status_code, GeocoderServiceError)
-            raise exc_cls(str(error)) from error
+            if issubclass(exc_cls, GeocoderRateLimited):
+                raise exc_cls(
+                    str(error), retry_after=get_retry_after(error.headers)
+                ) from error
+            else:
+                raise exc_cls(str(error)) from error
         else:
             self._geocoder_exception_handler(error)
 

geopy/geocoders/bing.py

@@ -5,7 +5,7 @@
 from geopy.exc import (
     GeocoderAuthenticationFailure,
     GeocoderInsufficientPrivileges,
-    GeocoderQuotaExceeded,
+    GeocoderRateLimited,
     GeocoderServiceError,
     GeocoderUnavailable,
 )
@@ -222,7 +222,7 @@ def _parse_json(self, doc, exactly_one=True):
             elif status_code == 403:
                 raise GeocoderInsufficientPrivileges(err)
             elif status_code == 429:
-                raise GeocoderQuotaExceeded(err)
+                raise GeocoderRateLimited(err)
             elif status_code == 503:
                 raise GeocoderUnavailable(err)
             else:

geopy/geocoders/googlev3.py

@@ -410,13 +410,11 @@ def parse_place(place):
             return [parse_place(place) for place in places]
 
     def _check_status(self, status):
-        """
-        Validates error statuses.
-        """
+        # https://developers.google.com/maps/documentation/geocoding/overview#StatusCodes
         if status == 'ZERO_RESULTS':
             # When there are no results, just return.
             return
-        if status == 'OVER_QUERY_LIMIT':
+        if status in ('OVER_QUERY_LIMIT', 'OVER_DAILY_LIMIT'):
             raise GeocoderQuotaExceeded(
                 'The given key has gone over the requests limit in the 24'
                 ' hour period or has submitted too many requests in too'

geopy/geocoders/here.py

@@ -7,7 +7,7 @@
     ConfigurationError,
     GeocoderAuthenticationFailure,
     GeocoderInsufficientPrivileges,
-    GeocoderQuotaExceeded,
+    GeocoderRateLimited,
     GeocoderServiceError,
     GeocoderUnavailable,
 )
@@ -333,7 +333,7 @@ def _parse_json(self, doc, exactly_one=True):
             elif status_code == 403:
                 raise GeocoderInsufficientPrivileges(err)
             elif status_code == 429:
-                raise GeocoderQuotaExceeded(err)
+                raise GeocoderRateLimited(err)
             elif status_code == 503:
                 raise GeocoderUnavailable(err)
             else:

geopy/geocoders/opencage.py

@@ -1,8 +1,8 @@
 from functools import partial
 from urllib.parse import urlencode
 
-from geopy.exc import GeocoderQueryError, GeocoderQuotaExceeded
-from geopy.geocoders.base import DEFAULT_SENTINEL, Geocoder
+from geopy.exc import GeocoderServiceError
+from geopy.geocoders.base import DEFAULT_SENTINEL, ERROR_CODE_MAP, Geocoder
 from geopy.location import Location
 from geopy.util import logger
 
@@ -216,24 +216,10 @@ def parse_place(place):
             return [parse_place(place) for place in places]
 
     def _check_status(self, status):
-        """
-        Validates error statuses.
-        """
         status_code = status['code']
-        if status_code == 429:
-            # Rate limit exceeded
-            raise GeocoderQuotaExceeded(
-                'The given key has gone over the requests limit in the 24'
-                ' hour period or has submitted too many requests in too'
-                ' short a period of time.'
-            )
+        message = status['message']
         if status_code == 200:
-            # When there are no results, just return.
             return
-
-        if status_code == 403:
-            raise GeocoderQueryError(
-                'Your request was denied.'
-            )
-        else:
-            raise GeocoderQueryError('Unknown error.')
+        # https://opencagedata.com/api#codes
+        exc_cls = ERROR_CODE_MAP.get(status_code, GeocoderServiceError)
+        raise exc_cls(message)

pytest.ini

@@ -1,5 +1,9 @@
 [pytest]
-python_files = test/test_*.py test/extra/*.py test/geocoders/*.py
+python_files =
+    test/test_*.py
+    test/adapters/*.py
+    test/extra/*.py
+    test/geocoders/*.py
 
 ; Bodies of HTTP errors are logged with INFO level
 log_level = INFO

test/test_adapters.py → test/adapters/each_adapter.py

@@ -23,7 +23,7 @@
 from geopy.geocoders.base import Geocoder
 from test.proxy_server import HttpServerThread, ProxyServerThread
 
-CERT_SELFSIGNED_CA = os.path.join(os.path.dirname(__file__), "selfsigned_ca.pem")
+CERT_SELFSIGNED_CA = os.path.join(os.path.dirname(__file__), "..", "selfsigned_ca.pem")
 
 # Are system proxies set? System proxies are set in:
 # - Environment variables (HTTP_PROXY/HTTPS_PROXY) on Unix;
@@ -336,6 +336,11 @@ async def test_adapter_exception_for_non_200_response(remote_website_http_404, t
         assert isinstance(excinfo.value.__cause__, AdapterHTTPError)
         assert isinstance(excinfo.value.__cause__, IOError)
 
+        adapter_http_error = excinfo.value.__cause__
+        assert adapter_http_error.status_code == 404
+        assert adapter_http_error.headers['x-test-header'] == 'hello'
+        assert adapter_http_error.text == 'Not found'
+
 
 async def test_system_proxies_are_respected_by_default(
     inject_proxy_to_system_env,

test/adapters/retry_after.py

@@ -0,0 +1,25 @@
+import datetime
+import time
+from unittest.mock import patch
+
+import pytest
+
+from geopy.adapters import get_retry_after
+
+
+@pytest.mark.parametrize(
+    "headers, expected_retry_after",
+    [
+        ({}, None),
+        ({"retry-after": "42"}, 42),
+        ({"retry-after": "Wed, 21 Oct 2015 07:28:44 GMT"}, 43),
+        ({"retry-after": "Wed, 21 Oct 2015 06:28:44 GMT"}, 0),
+        ({"retry-after": "Wed"}, None),
+    ],
+)
+def test_get_retry_after(headers, expected_retry_after):
+    current_time = datetime.datetime(
+        2015, 10, 21, 7, 28, 1, tzinfo=datetime.timezone.utc
+    ).timestamp()
+    with patch.object(time, "time", return_value=current_time):
+        assert expected_retry_after == get_retry_after(headers)