Merge pull request #131 from gogasca/RateLimitException

Add new type of Exception to handle 429 Errors

gcsfs/core.py

@@ -30,7 +30,7 @@
 import warnings
 
 from requests.exceptions import RequestException
-from .utils import HtmlError, is_retriable, read_block
+from .utils import HtmlError, RateLimitException, is_retriable, read_block
 
 PY2 = sys.version_info.major == 2
 
@@ -156,6 +156,8 @@ def validate_response(r, path):
             raise FileNotFoundError(path)
         elif r.status_code == 403:
             raise IOError("Forbidden: %s\n%s" % (path, msg))
+        elif r.status_code == 429:
+            raise RateLimitException(error)
         elif "invalid" in m:
             raise ValueError("Bad Request: %s\n%s" % (path, msg))
         elif error:

gcsfs/tests/test_utils.py

@@ -1,7 +1,8 @@
 import io
 import os
 import requests
-from gcsfs.utils import read_block, seek_delimiter, HtmlError, is_retriable
+from gcsfs.utils import read_block, seek_delimiter, HtmlError, \
+    RateLimitException, is_retriable
 from gcsfs.tests.utils import tmpfile
 
 
@@ -72,6 +73,14 @@ def retriable_exception():
     e = HtmlError({'message': '', 'code': '500'})
     assert is_retriable(e)
     e = HtmlError({'message': '', 'code': 400})
-    assert notis_retriable(e)
+    assert not is_retriable(e)
     e = HtmlError()
     assert not is_retriable(e)
+    e = RateLimitException()
+    assert not is_retriable(e)
+    e = RateLimitException({'message': '', 'code': 501})
+    assert is_retriable(e)
+    e = RateLimitException({'message': '', 'code': '501'})
+    assert is_retriable(e)
+    e = RateLimitException({'message': '', 'code': 400})
+    assert not is_retriable(e)

gcsfs/utils.py

@@ -88,6 +88,15 @@ def read_block(f, offset, length, delimiter=None):
     return bytes
 
 
+class RateLimitException(Exception):
+    """Holds the message and code from cloud errors."""
+    def __init__(self, error_response=None):
+        self.message = error_response.get('message', '')
+        self.code = error_response.get('code', None)
+        # Call the base class constructor with the parameters it needs
+        super(RateLimitException, self).__init__(self.message)
+
+
 class HtmlError(Exception):
     """Holds the message and code from cloud errors."""
     def __init__(self, error_response=None):
@@ -119,4 +128,7 @@ def is_retriable(exception):
     errs += [str(e) for e in errs]
     if isinstance(exception, HtmlError):
         return exception.code in errs
+    # https://cloud.google.com/storage/docs/key-terms#immutability
+    if isinstance(exception, RateLimitException):
+        return exception.code in errs
     return isinstance(exception, RETRIABLE_EXCEPTIONS)