fix: Automatically delay and refetch if 429

blockapi/test/v2/api/nft/test_simple_hash.py

@@ -5,6 +5,7 @@
 from dateutil.tz import tzutc
 
 from blockapi.test.v2.api.conftest import read_file
+from blockapi.test.v2.api.fake_sleep_provider import FakeSleepProvider
 from blockapi.v2.api.nft import SimpleHashBitcoinApi, SimpleHashSolanaApi
 from blockapi.v2.coins import COIN_BTC
 from blockapi.v2.models import AssetType, Blockchain, NftOfferDirection
@@ -251,13 +252,18 @@ def test_parse_listings(requests_mock, api, listings_response):
 
 
 @pytest.fixture
-def api():
-    return SimpleHashBitcoinApi('fake_key')
+def fake_sleep_provider():
+    return FakeSleepProvider()
 
 
 @pytest.fixture
-def solana_api():
-    return SimpleHashSolanaApi('fake_key')
+def api(fake_sleep_provider):
+    return SimpleHashBitcoinApi('fake_key', fake_sleep_provider)
+
+
+@pytest.fixture
+def solana_api(fake_sleep_provider):
+    return SimpleHashSolanaApi('fake_key', fake_sleep_provider)
 
 
 @pytest.fixture

blockapi/v2/api/nft/simple_hash.py

@@ -1,7 +1,7 @@
 import logging
 from typing import Iterable, Optional
 
-from blockapi.v2.base import BlockchainApi, INftParser, INftProvider
+from blockapi.v2.base import BlockchainApi, INftParser, INftProvider, ISleepProvider
 from blockapi.v2.coins import COIN_BTC, COIN_ETH, COIN_SOL
 from blockapi.v2.models import (
     ApiOptions,
@@ -54,8 +54,8 @@ class SimpleHashApi(BlockchainApi, INftProvider, INftParser):
         '&include_nft_details={include_nft_details}',
     }
 
-    def __init__(self, blockchain, simplehash_blockchain, api_key):
-        super().__init__()
+    def __init__(self, blockchain, simplehash_blockchain, api_key, sleep_provider):
+        super().__init__(sleep_provider=sleep_provider)
 
         self._api_key = api_key
         self.headers = {'accept': 'application/json', 'X-API-KEY': api_key}
@@ -402,8 +402,8 @@ class SimpleHashBitcoinApi(SimpleHashApi):
     default_blockchain = Blockchain.BITCOIN
     default_collection = 'inscriptions'
 
-    def __init__(self, api_key: str):
-        super().__init__(self.default_blockchain, 'bitcoin', api_key)
+    def __init__(self, api_key: str, sleep_provider: ISleepProvider):
+        super().__init__(self.default_blockchain, 'bitcoin', api_key, sleep_provider)
 
     def fetch_collection(self, collection: str) -> FetchResult:
         if collection == self.default_collection:
@@ -426,8 +426,8 @@ class SimpleHashSolanaApi(SimpleHashApi):
     coin = COIN_SOL
     default_blockchain = Blockchain.SOLANA
 
-    def __init__(self, api_key: str):
-        super().__init__(self.default_blockchain, 'solana', api_key)
+    def __init__(self, api_key: str, sleep_provider: ISleepProvider):
+        super().__init__(self.default_blockchain, 'solana', api_key, sleep_provider)
 
     def fetch_nfts(self, address: str, cursor: Optional[str] = None) -> FetchResult:
         token_cursor = None
@@ -516,9 +516,11 @@ class SimpleHashEthereumApi(SimpleHashApi):
     simplehash_blockchains_map = {n: b for b, n in supported_blockchains_map.items()}
     supported_blockchains = list(supported_blockchains_map.keys())
 
-    def __init__(self, blockchain: Blockchain, api_key: str):
+    def __init__(
+        self, blockchain: Blockchain, api_key: str, sleep_provider: ISleepProvider
+    ):
         chain = self.supported_blockchains_map.get(blockchain)
         if not chain:
             raise Exception(f'Blockchain not supported {blockchain}')
 
-        super().__init__(blockchain, chain, api_key)
+        super().__init__(blockchain, chain, api_key, sleep_provider)

blockapi/v2/base.py

@@ -1,3 +1,4 @@
+import json
 import logging
 import time
 from abc import ABC
@@ -22,6 +23,16 @@
 logger = logging.getLogger(__name__)
 
 
+class ISleepProvider(ABC):
+    def sleep(self, url: str, seconds: float) -> None:
+        raise NotImplementedError
+
+
+class SleepProvider(ISleepProvider):
+    def sleep(self, url: str, seconds: float):
+        time.sleep(seconds)
+
+
 class CustomizableBlockchainApi(ABC):
     """
     Class for handling blockchain API services with customizable base URL,
@@ -37,11 +48,18 @@ class CustomizableBlockchainApi(ABC):
     supported_requests: Dict[str, str] = {}
 
     json_parse_args = dict()
+    max_rate_limit_retries = 5
 
-    def __init__(self, base_url: Optional[str] = None, api_key: Optional[str] = None):
+    def __init__(
+        self,
+        base_url: Optional[str] = None,
+        api_key: Optional[str] = None,
+        sleep_provider: Optional[ISleepProvider] = None,
+    ):
         self.api_key = api_key
         self._session = Session()
         self.base_url = base_url or self.api_options.base_url
+        self.sleep_provider = sleep_provider
 
         if not self.base_url:
             raise NotImplementedError(
@@ -73,17 +91,34 @@ def get_data(
         **req_args,
     ) -> FetchResult:
         try:
-            response = self._get_response(request_method, headers, params, req_args)
-            time = self._get_response_time(response.headers)
-            if response.status_code == 200:
-                return FetchResult(
-                    status_code=response.status_code,
-                    headers=self._get_headers_dict(response.headers),
-                    data=response.json(**self.json_parse_args),
-                    extra=extra,
-                    time=time,
-                )
-
+            retries = self.max_rate_limit_retries
+            while True:
+                response = self._get_response(request_method, headers, params, req_args)
+                time = self._get_response_time(response.headers)
+                if response.status_code == 200:
+                    return FetchResult(
+                        status_code=response.status_code,
+                        headers=self._get_headers_dict(response.headers),
+                        data=response.json(**self.json_parse_args),
+                        extra=extra,
+                        time=time,
+                    )
+
+                if response.status_code == 429 and self.sleep_provider and retries > 0:
+                    retries -= 1
+                    delay = response.headers.get('retry-after', '60')
+                    try:
+                        seconds = int(delay)
+                    except ValueError:
+                        seconds = 60
+
+                    logger.warning(
+                        f'Too Many Requests: Will retry after {seconds}s sleep. Remaining attempts {retries}.'
+                    )
+                    self.sleep_provider.sleep(self.base_url, seconds=seconds)
+                    continue
+
+                break
             return FetchResult(
                 status_code=response.status_code,
                 headers=self._get_headers_dict(response.headers),
@@ -179,8 +214,10 @@ class BlockchainApi(CustomizableBlockchainApi, ABC):
     General class for handling blockchain API services.
     """
 
-    def __init__(self, api_key: Optional[str] = None):
-        super().__init__(base_url=None, api_key=api_key)
+    def __init__(
+        self, api_key: Optional[str] = None, sleep_provider: ISleepProvider = None
+    ):
+        super().__init__(base_url=None, api_key=api_key, sleep_provider=sleep_provider)
 
 
 class IBalance(ABC):
@@ -205,16 +242,6 @@ def get_portfolio(self, address: str) -> List[Pool]:
         raise NotImplementedError
 
 
-class ISleepProvider(ABC):
-    def sleep(self, url: str, seconds: float) -> None:
-        raise NotImplementedError
-
-
-class SleepProvider(ISleepProvider):
-    def sleep(self, url: str, seconds: float):
-        time.sleep(seconds)
-
-
 class INftProvider(ABC):
     def fetch_nfts(self, address: str) -> FetchResult:
         raise NotImplementedError