diff --git a/rotkehlchen/constants/misc.py b/rotkehlchen/constants/misc.py index 90847d03cb..f4e895b23d 100644 --- a/rotkehlchen/constants/misc.py +++ b/rotkehlchen/constants/misc.py @@ -24,7 +24,7 @@ # API URLS KRAKEN_BASE_URL = 'https://api.kraken.com' KRAKEN_API_VERSION = '0' -BINANCE_BASE_URL = 'https://api.binance.com/api/' +BINANCE_BASE_URL = 'https://api.binance.com/' # KRAKEN_BASE_URL = 'http://localhost:5001/kraken' # KRAKEN_API_VERSION = 'mock' # BINANCE_BASE_URL = 'http://localhost:5001/binance/api/' diff --git a/rotkehlchen/exchanges/binance.py b/rotkehlchen/exchanges/binance.py index a6776f96e4..d6929966de 100644 --- a/rotkehlchen/exchanges/binance.py +++ b/rotkehlchen/exchanges/binance.py @@ -2,15 +2,22 @@ import hmac import logging import time -from typing import Any, Dict, List, NamedTuple, Optional, Tuple, Union +from typing import Any, Dict, List, NamedTuple, Optional, Tuple, Union, cast from urllib.parse import urlencode import gevent +from typing_extensions import Literal from rotkehlchen.assets.converters import asset_from_binance from rotkehlchen.constants import BINANCE_BASE_URL +from rotkehlchen.constants.misc import ZERO from rotkehlchen.errors import DeserializationError, RemoteError, UnknownAsset, UnsupportedAsset -from rotkehlchen.exchanges.data_structures import Trade, TradeType, trade_pair_from_assets +from rotkehlchen.exchanges.data_structures import ( + AssetMovement, + Trade, + TradeType, + trade_pair_from_assets, +) from rotkehlchen.exchanges.exchange import ExchangeInterface from rotkehlchen.fval import FVal from rotkehlchen.inquirer import Inquirer @@ -21,7 +28,7 @@ deserialize_price, deserialize_timestamp_from_binance, ) -from rotkehlchen.typing import ApiKey, ApiSecret, FilePath, Timestamp +from rotkehlchen.typing import ApiKey, ApiSecret, Exchange, Fee, FilePath, Timestamp from rotkehlchen.user_messages import MessagesAggregator from rotkehlchen.utils.misc import cache_response_timewise from rotkehlchen.utils.serialization import rlk_jsonloads @@ -39,6 +46,11 @@ 'exchangeInfo' ) +WAPI_ENDPOINTS = ( + 'depositHistory.html', + 'withdrawHistory.html', +) + class BinancePair(NamedTuple): """A binance pair. Contains the symbol in the Binance mode e.g. "ETHBTC" and @@ -136,10 +148,10 @@ def create_binance_symbols_to_pair(exchange_data: Dict[str, Any]) -> Dict[str, B class Binance(ExchangeInterface): """Binance exchange api docs: - https://github.com/binance-exchange/binance-official-api-docs/blob/master/rest-api.md + https://github.com/binance-exchange/binance-official-api-docs/ An unofficial python binance package: - https://github.com/sammchardy/python-binance + https://github.com/binance-exchange/python-binance/ """ def __init__( self, @@ -203,7 +215,7 @@ def api_query(self, method: str, options: Optional[Dict] = None) -> Union[List, # Protect this region with a lock since binance will reject # non-increasing nonces. So if two greenlets come in here at # the same time one of them will fail - if method in V3_ENDPOINTS: + if method in V3_ENDPOINTS or method in WAPI_ENDPOINTS: api_version = 3 # Recommended recvWindows is 5000 but we get timeouts with it options['recvWindow'] = 10000 @@ -219,7 +231,8 @@ def api_query(self, method: str, options: Optional[Dict] = None) -> Union[List, else: raise ValueError('Unexpected binance api method {}'.format(method)) - request_url = self.uri + 'v' + str(api_version) + '/' + method + '?' + apistr = 'wapi/' if method in WAPI_ENDPOINTS else 'api/' + request_url = f'{self.uri}{apistr}v{str(api_version)}/{method}?' request_url += urlencode(options) log.debug('Binance API request', request_url=request_url) @@ -414,3 +427,99 @@ def query_trade_history( trades.append(trade) return trades + + def _deserialize_asset_movement(self, raw_data: Dict[str, Any]) -> Optional[AssetMovement]: + """Processes a single deposit/withdrawal from binance and deserializes it + + Can log error/warning and return None if something went wrong at deserialization + """ + try: + if 'insertTime' in raw_data: + category = 'deposit' + time_key = 'insertTime' + else: + category = 'withdrawal' + time_key = 'applyTime' + + timestamp = deserialize_timestamp_from_binance(raw_data[time_key]) + asset = asset_from_binance(raw_data['asset']) + amount = deserialize_asset_amount(raw_data['amount']) + return AssetMovement( + exchange=Exchange.BINANCE, + category=cast(Literal['deposit', 'withdrawal'], category), + timestamp=timestamp, + asset=asset, + amount=amount, + # Binance does not include withdrawal fees neither in the API nor in their UI + fee=Fee(ZERO), + ) + + except UnknownAsset as e: + self.msg_aggregator.add_warning( + f'Found binance deposit/withdrawal with unknown asset ' + f'{e.asset_name}. Ignoring it.', + ) + except UnsupportedAsset as e: + self.msg_aggregator.add_warning( + f'Found binance deposit/withdrawal with unsupported asset ' + f'{e.asset_name}. Ignoring it.', + ) + except (DeserializationError, KeyError) as e: + msg = str(e) + if isinstance(e, KeyError): + msg = f'Missing key entry for {msg}.' + self.msg_aggregator.add_error( + 'Error processing a binance deposit/withdrawal. Check logs ' + 'for details. Ignoring it.', + ) + log.error( + 'Error processing a binance deposit_withdrawal', + asset_movement=raw_data, + error=msg, + ) + + return None + + def query_deposits_withdrawals( + self, + start_ts: Timestamp, + end_ts: Timestamp, + end_at_least_ts: Timestamp, + ) -> List[AssetMovement]: + cache = self.check_trades_cache_list( + start_ts, + end_at_least_ts, + special_name='deposits_withdrawals', + ) + if cache is not None: + raw_data = cache + else: + # This does not check for any limits. Can there be any limits like with trades + # in the deposit/withdrawal binance api? Can't see anything in the docs: + # https://github.com/binance-exchange/binance-official-api-docs/blob/master/wapi-api.md#deposit-history-user_data + result = self.api_query_dict( + 'depositHistory.html', + options={'timestamp': 0}, + ) + raw_data = result['depositList'] + result = self.api_query_dict( + 'withdrawHistory.html', + options={'timestamp': 0}, + ) + raw_data.extend(result['withdrawList']) + log.debug('binance deposit/withdrawal history result', results_num=len(raw_data)) + self.update_trades_cache( + raw_data, + start_ts, + end_ts, + special_name='deposits_withdrawals', + ) + + + movements = [] + for raw_movement in raw_data: + movement = self._deserialize_asset_movement(raw_movement) + if movement: + movements.append(movement) + + return movements diff --git a/rotkehlchen/history.py b/rotkehlchen/history.py index 8f8bdfb81b..ad4263a71e 100644 --- a/rotkehlchen/history.py +++ b/rotkehlchen/history.py @@ -506,6 +506,14 @@ def create_history(self, start_ts, end_ts, end_at_least_ts): end_at_least_ts=end_at_least_ts, ) history.extend(binance_history) + + binance_asset_movements = self.binance.query_deposits_withdrawals( + start_ts=start_ts, + end_ts=end_ts, + end_at_least_ts=end_at_least_ts, + ) + asset_movements.extend(binance_asset_movements) + except RemoteError as e: empty_or_error += '\n' + str(e) diff --git a/rotkehlchen/tests/test_binance.py b/rotkehlchen/tests/test_binance.py index be04f4990c..fe0de0f918 100644 --- a/rotkehlchen/tests/test_binance.py +++ b/rotkehlchen/tests/test_binance.py @@ -12,13 +12,15 @@ ) from rotkehlchen.assets.resolver import AssetResolver from rotkehlchen.constants.assets import A_BTC, A_ETH +from rotkehlchen.constants.misc import ZERO from rotkehlchen.errors import RemoteError, UnsupportedAsset from rotkehlchen.exchanges.binance import Binance, trade_from_binance -from rotkehlchen.exchanges.data_structures import Trade, TradeType +from rotkehlchen.exchanges.data_structures import Exchange, Trade, TradeType from rotkehlchen.fval import FVal -from rotkehlchen.tests.utils.constants import A_BNB, A_RDN, A_USDT +from rotkehlchen.tests.utils.constants import A_BNB, A_RDN, A_USDT, A_XMR from rotkehlchen.tests.utils.exchanges import BINANCE_BALANCES_RESPONSE from rotkehlchen.tests.utils.factories import make_random_b64bytes +from rotkehlchen.tests.utils.history import TEST_END_TS from rotkehlchen.tests.utils.mock import MockResponse from rotkehlchen.user_messages import MessagesAggregator @@ -389,3 +391,108 @@ def query_binance_and_test( expected_errors_num=0, warning_str_test='Found binance trade with unknown asset DSDSDS', ) + + +BINANCE_DEPOSITS_HISTORY_RESPONSE = """{ + "depositList": [ + { + "insertTime": 1508198532000, + "amount": 0.04670582, + "asset": "ETH", + "address": "0x6915f16f8791d0a1cc2bf47c13a6b2a92000504b", + "txId": "0xdf33b22bdb2b28b1f75ccd201a4a4m6e7g83jy5fc5d5a9d1340961598cfcb0a1", + "status": 1 + }, + { + "insertTime": 1508398632000, + "amount": 1000, + "asset": "XMR", + "address": "463tWEBn5XZJSxLU34r6g7h8jtxuNcDbjLSjkn3XAXHCbLrTTErJrBWYgHJQyrCwkNgYvV38", + "addressTag": "342341222", + "txId": "b3c6219639c8ae3f9cf010cdc24fw7f7yt8j1e063f9b4bd1a05cb44c4b6e2509", + "status": 1 + } + ], + "success": true +}""" + +BINANCE_WITHDRAWALS_HISTORY_RESPONSE = """{ + "withdrawList": [ + { + "id":"7213fea8e94b4a5593d507237e5a555b", + "amount": 1, + "address": "0x6915f16f8791d0a1cc2bf47c13a6b2a92000504b", + "asset": "ETH", + "txId": "0xdf33b22bdb2b28b1f75ccd201a4a4m6e7g83jy5fc5d5a9d1340961598cfcb0a1", + "applyTime": 1518192542000, + "status": 4 + }, + { + "id":"7213fea8e94b4a5534ggsd237e5a555b", + "amount": 850.1, + "address": "463tWEBn5XZJSxLU34r6g7h8jtxuNcDbjLSjkn3XAXHCbLrTTErJrBWYgHJQyrCwkNgYvyVz8", + "addressTag": "342341222", + "txId": "b3c6219639c8ae3f9cf010cdc24fw7f7yt8j1e063f9b4bd1a05cb44c4b6e2509", + "asset": "XMR", + "applyTime": 1529198732000, + "status": 4 + } + ], + "success": true +}""" + + +def test_binance_query_deposits_withdrawals(function_scope_binance): + """Test the happy case of binance deposit withdrawal query""" + binance = function_scope_binance + binance.cache_ttl_secs = 0 + + def mock_get_deposit_withdrawal(url): # pylint: disable=unused-argument + if 'deposit' in url: + response_str = BINANCE_DEPOSITS_HISTORY_RESPONSE + else: + response_str = BINANCE_WITHDRAWALS_HISTORY_RESPONSE + + return MockResponse(200, response_str) + + with patch.object(binance.session, 'get', side_effect=mock_get_deposit_withdrawal): + movements = binance.query_deposits_withdrawals(0, TEST_END_TS, TEST_END_TS) + + errors = binance.msg_aggregator.consume_errors() + warnings = binance.msg_aggregator.consume_warnings() + assert len(errors) == 0 + assert len(warnings) == 0 + + assert len(movements) == 4 + + assert movements[0].exchange == Exchange.BINANCE + assert movements[0].category == 'deposit' + assert movements[0].timestamp == 1508198532 + assert isinstance(movements[0].asset, Asset) + assert movements[0].asset == A_ETH + assert movements[0].amount == FVal('0.04670582') + assert movements[0].fee == ZERO + + assert movements[1].exchange == Exchange.BINANCE + assert movements[1].category == 'deposit' + assert movements[1].timestamp == 1508398632 + assert isinstance(movements[1].asset, Asset) + assert movements[1].asset == A_XMR + assert movements[1].amount == FVal('1000') + assert movements[1].fee == ZERO + + assert movements[2].exchange == Exchange.BINANCE + assert movements[2].category == 'withdrawal' + assert movements[2].timestamp == 1518192542 + assert isinstance(movements[2].asset, Asset) + assert movements[2].asset == A_ETH + assert movements[2].amount == FVal('1') + assert movements[2].fee == ZERO + + assert movements[3].exchange == Exchange.BINANCE + assert movements[3].category == 'withdrawal' + assert movements[3].timestamp == 1529198732 + assert isinstance(movements[3].asset, Asset) + assert movements[3].asset == A_XMR + assert movements[3].amount == FVal('850.1') + assert movements[3].fee == ZERO