From 81db9b34b0a08873026c0fa34b6ab92389a8fdb2 Mon Sep 17 00:00:00 2001 From: hanjano Date: Thu, 29 Feb 2024 06:50:08 +0100 Subject: [PATCH 1/3] fix: Automatically delay and refetch if 429 --- blockapi/v2/api/nft/simple_hash.py | 20 +++++---- blockapi/v2/base.py | 72 ++++++++++++++++++++---------- 2 files changed, 59 insertions(+), 33 deletions(-) diff --git a/blockapi/v2/api/nft/simple_hash.py b/blockapi/v2/api/nft/simple_hash.py index 704008ee..70de7140 100644 --- a/blockapi/v2/api/nft/simple_hash.py +++ b/blockapi/v2/api/nft/simple_hash.py @@ -1,7 +1,7 @@ import logging from typing import Iterable, Optional -from blockapi.v2.base import BlockchainApi, INftParser, INftProvider +from blockapi.v2.base import BlockchainApi, INftParser, INftProvider, ISleepProvider from blockapi.v2.coins import COIN_BTC, COIN_ETH, COIN_SOL from blockapi.v2.models import ( ApiOptions, @@ -54,8 +54,8 @@ class SimpleHashApi(BlockchainApi, INftProvider, INftParser): '&include_nft_details={include_nft_details}', } - def __init__(self, blockchain, simplehash_blockchain, api_key): - super().__init__() + def __init__(self, blockchain, simplehash_blockchain, api_key, sleep_provider): + super().__init__(sleep_provider=sleep_provider) self._api_key = api_key self.headers = {'accept': 'application/json', 'X-API-KEY': api_key} @@ -402,8 +402,8 @@ class SimpleHashBitcoinApi(SimpleHashApi): default_blockchain = Blockchain.BITCOIN default_collection = 'inscriptions' - def __init__(self, api_key: str): - super().__init__(self.default_blockchain, 'bitcoin', api_key) + def __init__(self, api_key: str, sleep_provider: ISleepProvider): + super().__init__(self.default_blockchain, 'bitcoin', api_key, sleep_provider) def fetch_collection(self, collection: str) -> FetchResult: if collection == self.default_collection: @@ -426,8 +426,8 @@ class SimpleHashSolanaApi(SimpleHashApi): coin = COIN_SOL default_blockchain = Blockchain.SOLANA - def __init__(self, api_key: str): - super().__init__(self.default_blockchain, 'solana', api_key) + def __init__(self, api_key: str, sleep_provider: ISleepProvider): + super().__init__(self.default_blockchain, 'solana', api_key, sleep_provider) def fetch_nfts(self, address: str, cursor: Optional[str] = None) -> FetchResult: token_cursor = None @@ -516,9 +516,11 @@ class SimpleHashEthereumApi(SimpleHashApi): simplehash_blockchains_map = {n: b for b, n in supported_blockchains_map.items()} supported_blockchains = list(supported_blockchains_map.keys()) - def __init__(self, blockchain: Blockchain, api_key: str): + def __init__( + self, blockchain: Blockchain, api_key: str, sleep_provider: ISleepProvider + ): chain = self.supported_blockchains_map.get(blockchain) if not chain: raise Exception(f'Blockchain not supported {blockchain}') - super().__init__(blockchain, chain, api_key) + super().__init__(blockchain, chain, api_key, sleep_provider) diff --git a/blockapi/v2/base.py b/blockapi/v2/base.py index 54d35f82..6c70dc98 100644 --- a/blockapi/v2/base.py +++ b/blockapi/v2/base.py @@ -1,3 +1,4 @@ +import json import logging import time from abc import ABC @@ -22,6 +23,16 @@ logger = logging.getLogger(__name__) +class ISleepProvider(ABC): + def sleep(self, url: str, seconds: float) -> None: + raise NotImplementedError + + +class SleepProvider(ISleepProvider): + def sleep(self, url: str, seconds: float): + time.sleep(seconds) + + class CustomizableBlockchainApi(ABC): """ Class for handling blockchain API services with customizable base URL, @@ -38,10 +49,16 @@ class CustomizableBlockchainApi(ABC): json_parse_args = dict() - def __init__(self, base_url: Optional[str] = None, api_key: Optional[str] = None): + def __init__( + self, + base_url: Optional[str] = None, + api_key: Optional[str] = None, + sleep_provider: Optional[ISleepProvider] = None, + ): self.api_key = api_key self._session = Session() self.base_url = base_url or self.api_options.base_url + self.sleep_provider = sleep_provider if not self.base_url: raise NotImplementedError( @@ -73,17 +90,32 @@ def get_data( **req_args, ) -> FetchResult: try: - response = self._get_response(request_method, headers, params, req_args) - time = self._get_response_time(response.headers) - if response.status_code == 200: - return FetchResult( - status_code=response.status_code, - headers=self._get_headers_dict(response.headers), - data=response.json(**self.json_parse_args), - extra=extra, - time=time, - ) - + while True: + response = self._get_response(request_method, headers, params, req_args) + time = self._get_response_time(response.headers) + if response.status_code == 200: + return FetchResult( + status_code=response.status_code, + headers=self._get_headers_dict(response.headers), + data=response.json(**self.json_parse_args), + extra=extra, + time=time, + ) + + if response.status_code == 429 and self.sleep_provider: + delay = response.headers.get('retry-after', '60') + try: + seconds = int(delay) + except ValueError: + seconds = 60 + + logger.warning( + f'Too Many Requests: Will retry after {seconds}s sleep' + ) + self.sleep_provider.sleep(self.base_url, seconds=seconds) + continue + + break return FetchResult( status_code=response.status_code, headers=self._get_headers_dict(response.headers), @@ -179,8 +211,10 @@ class BlockchainApi(CustomizableBlockchainApi, ABC): General class for handling blockchain API services. """ - def __init__(self, api_key: Optional[str] = None): - super().__init__(base_url=None, api_key=api_key) + def __init__( + self, api_key: Optional[str] = None, sleep_provider: ISleepProvider = None + ): + super().__init__(base_url=None, api_key=api_key, sleep_provider=sleep_provider) class IBalance(ABC): @@ -205,16 +239,6 @@ def get_portfolio(self, address: str) -> List[Pool]: raise NotImplementedError -class ISleepProvider(ABC): - def sleep(self, url: str, seconds: float) -> None: - raise NotImplementedError - - -class SleepProvider(ISleepProvider): - def sleep(self, url: str, seconds: float): - time.sleep(seconds) - - class INftProvider(ABC): def fetch_nfts(self, address: str) -> FetchResult: raise NotImplementedError From 9540209a06a8da716e4138a2fc8e680d42ca7af7 Mon Sep 17 00:00:00 2001 From: hanjano Date: Thu, 29 Feb 2024 09:40:52 +0100 Subject: [PATCH 2/3] fix tests --- blockapi/test/v2/api/nft/test_simple_hash.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/blockapi/test/v2/api/nft/test_simple_hash.py b/blockapi/test/v2/api/nft/test_simple_hash.py index b927e5ae..4678d20f 100644 --- a/blockapi/test/v2/api/nft/test_simple_hash.py +++ b/blockapi/test/v2/api/nft/test_simple_hash.py @@ -5,6 +5,7 @@ from dateutil.tz import tzutc from blockapi.test.v2.api.conftest import read_file +from blockapi.test.v2.api.fake_sleep_provider import FakeSleepProvider from blockapi.v2.api.nft import SimpleHashBitcoinApi, SimpleHashSolanaApi from blockapi.v2.coins import COIN_BTC from blockapi.v2.models import AssetType, Blockchain, NftOfferDirection @@ -251,13 +252,18 @@ def test_parse_listings(requests_mock, api, listings_response): @pytest.fixture -def api(): - return SimpleHashBitcoinApi('fake_key') +def fake_sleep_provider(): + return FakeSleepProvider() @pytest.fixture -def solana_api(): - return SimpleHashSolanaApi('fake_key') +def api(fake_sleep_provider): + return SimpleHashBitcoinApi('fake_key', fake_sleep_provider) + + +@pytest.fixture +def solana_api(fake_sleep_provider): + return SimpleHashSolanaApi('fake_key', fake_sleep_provider) @pytest.fixture From dbf6296370497a9ad4004fc076b6b11b20f56047 Mon Sep 17 00:00:00 2001 From: hanjano Date: Thu, 29 Feb 2024 11:48:17 +0100 Subject: [PATCH 3/3] fix: limit number of retries --- blockapi/v2/base.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/blockapi/v2/base.py b/blockapi/v2/base.py index 6c70dc98..bf6adfdf 100644 --- a/blockapi/v2/base.py +++ b/blockapi/v2/base.py @@ -48,6 +48,7 @@ class CustomizableBlockchainApi(ABC): supported_requests: Dict[str, str] = {} json_parse_args = dict() + max_rate_limit_retries = 5 def __init__( self, @@ -90,6 +91,7 @@ def get_data( **req_args, ) -> FetchResult: try: + retries = self.max_rate_limit_retries while True: response = self._get_response(request_method, headers, params, req_args) time = self._get_response_time(response.headers) @@ -102,7 +104,8 @@ def get_data( time=time, ) - if response.status_code == 429 and self.sleep_provider: + if response.status_code == 429 and self.sleep_provider and retries > 0: + retries -= 1 delay = response.headers.get('retry-after', '60') try: seconds = int(delay) @@ -110,7 +113,7 @@ def get_data( seconds = 60 logger.warning( - f'Too Many Requests: Will retry after {seconds}s sleep' + f'Too Many Requests: Will retry after {seconds}s sleep. Remaining attempts {retries}.' ) self.sleep_provider.sleep(self.base_url, seconds=seconds) continue