Support deposit/withdrawal queries for Binance

Fix #458

rotkehlchen/constants/misc.py

@@ -24,7 +24,7 @@
 # API URLS
 KRAKEN_BASE_URL = 'https://api.kraken.com'
 KRAKEN_API_VERSION = '0'
-BINANCE_BASE_URL = 'https://api.binance.com/api/'
+BINANCE_BASE_URL = 'https://api.binance.com/'
 # KRAKEN_BASE_URL = 'http://localhost:5001/kraken'
 # KRAKEN_API_VERSION = 'mock'
 # BINANCE_BASE_URL = 'http://localhost:5001/binance/api/'

rotkehlchen/exchanges/binance.py

@@ -2,15 +2,22 @@
 import hmac
 import logging
 import time
-from typing import Any, Dict, List, NamedTuple, Optional, Tuple, Union
+from typing import Any, Dict, List, NamedTuple, Optional, Tuple, Union, cast
 from urllib.parse import urlencode
 
 import gevent
+from typing_extensions import Literal
 
 from rotkehlchen.assets.converters import asset_from_binance
 from rotkehlchen.constants import BINANCE_BASE_URL
+from rotkehlchen.constants.misc import ZERO
 from rotkehlchen.errors import DeserializationError, RemoteError, UnknownAsset, UnsupportedAsset
-from rotkehlchen.exchanges.data_structures import Trade, TradeType, trade_pair_from_assets
+from rotkehlchen.exchanges.data_structures import (
+    AssetMovement,
+    Trade,
+    TradeType,
+    trade_pair_from_assets,
+)
 from rotkehlchen.exchanges.exchange import ExchangeInterface
 from rotkehlchen.fval import FVal
 from rotkehlchen.inquirer import Inquirer
@@ -21,7 +28,7 @@
     deserialize_price,
     deserialize_timestamp_from_binance,
 )
-from rotkehlchen.typing import ApiKey, ApiSecret, FilePath, Timestamp
+from rotkehlchen.typing import ApiKey, ApiSecret, Exchange, Fee, FilePath, Timestamp
 from rotkehlchen.user_messages import MessagesAggregator
 from rotkehlchen.utils.misc import cache_response_timewise
 from rotkehlchen.utils.serialization import rlk_jsonloads
@@ -39,6 +46,11 @@
     'exchangeInfo'
 )
 
+WAPI_ENDPOINTS = (
+    'depositHistory.html',
+    'withdrawHistory.html',
+)
+
 
 class BinancePair(NamedTuple):
     """A binance pair. Contains the symbol in the Binance mode e.g. "ETHBTC" and
@@ -136,10 +148,10 @@ def create_binance_symbols_to_pair(exchange_data: Dict[str, Any]) -> Dict[str, B
 
 class Binance(ExchangeInterface):
     """Binance exchange api docs:
-    https://github.com/binance-exchange/binance-official-api-docs/blob/master/rest-api.md
+    https://github.com/binance-exchange/binance-official-api-docs/
 
     An unofficial python binance package:
-    https://github.com/sammchardy/python-binance
+    https://github.com/binance-exchange/python-binance/
     """
     def __init__(
             self,
@@ -203,7 +215,7 @@ def api_query(self, method: str, options: Optional[Dict] = None) -> Union[List,
                 # Protect this region with a lock since binance will reject
                 # non-increasing nonces. So if two greenlets come in here at
                 # the same time one of them will fail
-                if method in V3_ENDPOINTS:
+                if method in V3_ENDPOINTS or method in WAPI_ENDPOINTS:
                     api_version = 3
                     # Recommended recvWindows is 5000 but we get timeouts with it
                     options['recvWindow'] = 10000
@@ -219,7 +231,8 @@ def api_query(self, method: str, options: Optional[Dict] = None) -> Union[List,
                 else:
                     raise ValueError('Unexpected binance api method {}'.format(method))
 
-                request_url = self.uri + 'v' + str(api_version) + '/' + method + '?'
+                apistr = 'wapi/' if method in WAPI_ENDPOINTS else 'api/'
+                request_url = f'{self.uri}{apistr}v{str(api_version)}/{method}?'
                 request_url += urlencode(options)
 
                 log.debug('Binance API request', request_url=request_url)
@@ -414,3 +427,99 @@ def query_trade_history(
             trades.append(trade)
 
         return trades
+
+    def _deserialize_asset_movement(self, raw_data: Dict[str, Any]) -> Optional[AssetMovement]:
+        """Processes a single deposit/withdrawal from binance and deserializes it
+
+        Can log error/warning and return None if something went wrong at deserialization
+        """
+        try:
+            if 'insertTime' in raw_data:
+                category = 'deposit'
+                time_key = 'insertTime'
+            else:
+                category = 'withdrawal'
+                time_key = 'applyTime'
+
+            timestamp = deserialize_timestamp_from_binance(raw_data[time_key])
+            asset = asset_from_binance(raw_data['asset'])
+            amount = deserialize_asset_amount(raw_data['amount'])
+            return AssetMovement(
+                exchange=Exchange.BINANCE,
+                category=cast(Literal['deposit', 'withdrawal'], category),
+                timestamp=timestamp,
+                asset=asset,
+                amount=amount,
+                # Binance does not include withdrawal fees neither in the API nor in their UI
+                fee=Fee(ZERO),
+            )
+
+        except UnknownAsset as e:
+            self.msg_aggregator.add_warning(
+                f'Found binance deposit/withdrawal with unknown asset '
+                f'{e.asset_name}. Ignoring it.',
+            )
+        except UnsupportedAsset as e:
+            self.msg_aggregator.add_warning(
+                f'Found binance deposit/withdrawal with unsupported asset '
+                f'{e.asset_name}. Ignoring it.',
+            )
+        except (DeserializationError, KeyError) as e:
+            msg = str(e)
+            if isinstance(e, KeyError):
+                msg = f'Missing key entry for {msg}.'
+            self.msg_aggregator.add_error(
+                'Error processing a binance deposit/withdrawal. Check logs '
+                'for details. Ignoring it.',
+            )
+            log.error(
+                'Error processing a binance deposit_withdrawal',
+                asset_movement=raw_data,
+                error=msg,
+            )
+
+        return None
+
+    def query_deposits_withdrawals(
+            self,
+            start_ts: Timestamp,
+            end_ts: Timestamp,
+            end_at_least_ts: Timestamp,
+    ) -> List[AssetMovement]:
+        cache = self.check_trades_cache_list(
+            start_ts,
+            end_at_least_ts,
+            special_name='deposits_withdrawals',
+        )
+        if cache is not None:
+            raw_data = cache
+        else:
+            # This does not check for any limits. Can there be any limits like with trades
+            # in the deposit/withdrawal binance api? Can't see anything in the docs:
+            # https://github.com/binance-exchange/binance-official-api-docs/blob/master/wapi-api.md#deposit-history-user_data
+            result = self.api_query_dict(
+                'depositHistory.html',
+                options={'timestamp': 0},
+            )
+            raw_data = result['depositList']
+            result = self.api_query_dict(
+                'withdrawHistory.html',
+                options={'timestamp': 0},
+            )
+            raw_data.extend(result['withdrawList'])
+            log.debug('binance deposit/withdrawal history result', results_num=len(raw_data))
+            self.update_trades_cache(
+                raw_data,
+                start_ts,
+                end_ts,
+                special_name='deposits_withdrawals',
+            )
+
+
+        movements = []
+        for raw_movement in raw_data:
+            movement = self._deserialize_asset_movement(raw_movement)
+            if movement:
+                movements.append(movement)
+
+        return movements

rotkehlchen/history.py

@@ -506,6 +506,14 @@ def create_history(self, start_ts, end_ts, end_at_least_ts):
                     end_at_least_ts=end_at_least_ts,
                 )
                 history.extend(binance_history)
+
+                binance_asset_movements = self.binance.query_deposits_withdrawals(
+                    start_ts=start_ts,
+                    end_ts=end_ts,
+                    end_at_least_ts=end_at_least_ts,
+                )
+                asset_movements.extend(binance_asset_movements)
+
             except RemoteError as e:
                 empty_or_error += '\n' + str(e)
 

rotkehlchen/tests/test_binance.py

@@ -12,13 +12,15 @@
 )
 from rotkehlchen.assets.resolver import AssetResolver
 from rotkehlchen.constants.assets import A_BTC, A_ETH
+from rotkehlchen.constants.misc import ZERO
 from rotkehlchen.errors import RemoteError, UnsupportedAsset
 from rotkehlchen.exchanges.binance import Binance, trade_from_binance
-from rotkehlchen.exchanges.data_structures import Trade, TradeType
+from rotkehlchen.exchanges.data_structures import Exchange, Trade, TradeType
 from rotkehlchen.fval import FVal
-from rotkehlchen.tests.utils.constants import A_BNB, A_RDN, A_USDT
+from rotkehlchen.tests.utils.constants import A_BNB, A_RDN, A_USDT, A_XMR
 from rotkehlchen.tests.utils.exchanges import BINANCE_BALANCES_RESPONSE
 from rotkehlchen.tests.utils.factories import make_random_b64bytes
+from rotkehlchen.tests.utils.history import TEST_END_TS
 from rotkehlchen.tests.utils.mock import MockResponse
 from rotkehlchen.user_messages import MessagesAggregator
 
@@ -389,3 +391,108 @@ def query_binance_and_test(
         expected_errors_num=0,
         warning_str_test='Found binance trade with unknown asset DSDSDS',
     )
+
+
+BINANCE_DEPOSITS_HISTORY_RESPONSE = """{
+    "depositList": [
+        {
+            "insertTime": 1508198532000,
+            "amount": 0.04670582,
+            "asset": "ETH",
+            "address": "0x6915f16f8791d0a1cc2bf47c13a6b2a92000504b",
+            "txId": "0xdf33b22bdb2b28b1f75ccd201a4a4m6e7g83jy5fc5d5a9d1340961598cfcb0a1",
+            "status": 1
+        },
+        {
+            "insertTime": 1508398632000,
+            "amount": 1000,
+            "asset": "XMR",
+            "address": "463tWEBn5XZJSxLU34r6g7h8jtxuNcDbjLSjkn3XAXHCbLrTTErJrBWYgHJQyrCwkNgYvV38",
+            "addressTag": "342341222",
+            "txId": "b3c6219639c8ae3f9cf010cdc24fw7f7yt8j1e063f9b4bd1a05cb44c4b6e2509",
+            "status": 1
+        }
+    ],
+    "success": true
+}"""
+
+BINANCE_WITHDRAWALS_HISTORY_RESPONSE = """{
+    "withdrawList": [
+        {
+            "id":"7213fea8e94b4a5593d507237e5a555b",
+            "amount": 1,
+            "address": "0x6915f16f8791d0a1cc2bf47c13a6b2a92000504b",
+            "asset": "ETH",
+            "txId": "0xdf33b22bdb2b28b1f75ccd201a4a4m6e7g83jy5fc5d5a9d1340961598cfcb0a1",
+            "applyTime": 1518192542000,
+            "status": 4
+        },
+        {
+            "id":"7213fea8e94b4a5534ggsd237e5a555b",
+            "amount": 850.1,
+            "address": "463tWEBn5XZJSxLU34r6g7h8jtxuNcDbjLSjkn3XAXHCbLrTTErJrBWYgHJQyrCwkNgYvyVz8",
+            "addressTag": "342341222",
+            "txId": "b3c6219639c8ae3f9cf010cdc24fw7f7yt8j1e063f9b4bd1a05cb44c4b6e2509",
+            "asset": "XMR",
+            "applyTime": 1529198732000,
+            "status": 4
+        }
+    ],
+    "success": true
+}"""
+
+
+def test_binance_query_deposits_withdrawals(function_scope_binance):
+    """Test the happy case of binance deposit withdrawal query"""
+    binance = function_scope_binance
+    binance.cache_ttl_secs = 0
+
+    def mock_get_deposit_withdrawal(url):  # pylint: disable=unused-argument
+        if 'deposit' in url:
+            response_str = BINANCE_DEPOSITS_HISTORY_RESPONSE
+        else:
+            response_str = BINANCE_WITHDRAWALS_HISTORY_RESPONSE
+
+        return MockResponse(200, response_str)
+
+    with patch.object(binance.session, 'get', side_effect=mock_get_deposit_withdrawal):
+        movements = binance.query_deposits_withdrawals(0, TEST_END_TS, TEST_END_TS)
+
+    errors = binance.msg_aggregator.consume_errors()
+    warnings = binance.msg_aggregator.consume_warnings()
+    assert len(errors) == 0
+    assert len(warnings) == 0
+
+    assert len(movements) == 4
+
+    assert movements[0].exchange == Exchange.BINANCE
+    assert movements[0].category == 'deposit'
+    assert movements[0].timestamp == 1508198532
+    assert isinstance(movements[0].asset, Asset)
+    assert movements[0].asset == A_ETH
+    assert movements[0].amount == FVal('0.04670582')
+    assert movements[0].fee == ZERO
+
+    assert movements[1].exchange == Exchange.BINANCE
+    assert movements[1].category == 'deposit'
+    assert movements[1].timestamp == 1508398632
+    assert isinstance(movements[1].asset, Asset)
+    assert movements[1].asset == A_XMR
+    assert movements[1].amount == FVal('1000')
+    assert movements[1].fee == ZERO
+
+    assert movements[2].exchange == Exchange.BINANCE
+    assert movements[2].category == 'withdrawal'
+    assert movements[2].timestamp == 1518192542
+    assert isinstance(movements[2].asset, Asset)
+    assert movements[2].asset == A_ETH
+    assert movements[2].amount == FVal('1')
+    assert movements[2].fee == ZERO
+
+    assert movements[3].exchange == Exchange.BINANCE
+    assert movements[3].category == 'withdrawal'
+    assert movements[3].timestamp == 1529198732
+    assert isinstance(movements[3].asset, Asset)
+    assert movements[3].asset == A_XMR
+    assert movements[3].amount == FVal('850.1')
+    assert movements[3].fee == ZERO