Use blockstream API for native segwit address balance queries

docs/api.rst

@@ -4242,7 +4242,7 @@ Adding BTC xpubs
    :statuscode 400: Provided JSON or data is in some way malformed. The accounts to add contained invalid addresses or were an empty list.
    :statuscode 409: User is not logged in. Some error occured when re-querying the balances after addition. Provided tags do not exist. Check message for details.
    :statuscode 500: Internal Rotki error
-   :statuscode 502: Error occured with some external service query such as blockcypher. Check message for details.
+   :statuscode 502: Error occured with some external service query such as blockstream. Check message for details.
 
 Deleting BTC xpubs
 ========================
@@ -4323,7 +4323,7 @@ Deleting BTC xpubs
    :statuscode 400: Provided JSON or data is in some way malformed. The accounts to add contained invalid addresses or were an empty list.
    :statuscode 409: User is not logged in. Some error occured when re-querying the balances after addition. Check message for details.
    :statuscode 500: Internal Rotki error
-   :statuscode 502: Error occured with some external service query such as blockcypher. Check message for details.
+   :statuscode 502: Error occured with some external service query such as blockstream. Check message for details.
 
 Editing blockchain account data
 =================================

docs/changelog.rst

@@ -4,6 +4,7 @@ Changelog
 
 * :bug:`1635` Application will now continue running when changing log level on Windows.
 * :feature:`1642` Force pull/push buttons for premium sync are now accessible in the floppy disk icon on the toolbar.
+* :bug:`1639` Native segwit xpubs will now properly query and display the balances of their derived addresses. Rotki switched to using blockstream's API instead of blockcypher for native segwit addresses.
 * :bug:`1638` Balances displayed in dashboard cards should now be properly sorted by value in descending order.
 * :bug:`-` If the DB has not been uploaded in this run of Rotki, the last upload time indicator now shows the last time data was uploaded and not "Never".
 * :bug:`1641` Rotki only accepts derivation paths in the form of m/X/Y/Z... where ``X``, ``Y`` and ``Z`` are integers. Anything else is not processable and invalid. We now check that the given path is valid and reject the addition if not. Also the DB is upgraded and any xpubs with such invalid derivation path are automatically deleted.

rotkehlchen/chain/bitcoin/__init__.py

@@ -5,65 +5,32 @@
 from rotkehlchen.errors import RemoteError, UnableToDecryptRemoteData
 from rotkehlchen.fval import FVal
 from rotkehlchen.typing import BTCAddress
-from rotkehlchen.utils.misc import request_get, request_get_dict, satoshis_to_btc
-
-
-def _prepare_blockcypher_accounts(accounts: List[BTCAddress]) -> List[BTCAddress]:
-    """bech32 accounts have to be given lowercase to the blockcypher query.
-
-    No idea why.
-    """
-    new_accounts: List[BTCAddress] = []
-    for x in accounts:
-        lowered = x.lower()
-        if lowered[0:3] == 'bc1':
-            new_accounts.append(BTCAddress(lowered))
-        else:
-            new_accounts.append(x)
-
-    return new_accounts
+from rotkehlchen.utils.misc import request_get_dict, satoshis_to_btc
 
 
 def _have_bc1_accounts(accounts: List[BTCAddress]) -> bool:
     return any(account.lower()[0:3] == 'bc1' for account in accounts)
 
 
 def get_bitcoin_addresses_balances(accounts: List[BTCAddress]) -> Dict[BTCAddress, FVal]:
-    """Queries blockchain.info or blockcypher for the balances of accounts
+    """Queries blockchain.info or blockstream for the balances of accounts
 
     May raise:
-    - RemotError if there is a problem querying blockchain.info or blockcypher
+    - RemotError if there is a problem querying blockchain.info or blockstream
     """
     source = 'blockchain.info'
-    balances = {}
+    balances: Dict[BTCAddress, FVal] = {}
     try:
         if _have_bc1_accounts(accounts):
-            # if 1 account is bech32 we have to query blockcypher. blockchaininfo won't work
-            source = 'blockcypher.com'
-            new_accounts = _prepare_blockcypher_accounts(accounts)
-
-            # blockcypher's batching takes up as many api queries as the batch,
-            # and the api rate limit is 3 requests per second. So we should make
-            # sure each batch is of max size 3
-            # https://www.blockcypher.com/dev/bitcoin/#batching
-            batches = [new_accounts[x: x + 3] for x in range(0, len(new_accounts), 3)]
-            total_idx = 0
-            for batch in batches:
-                params = ';'.join(batch)
-                url = f'https://api.blockcypher.com/v1/btc/main/addrs/{params}/balance'
-                response_data = request_get(url=url, handle_429=True, backoff_in_seconds=4)
-
-                if isinstance(response_data, dict):
-                    # If only one account was requested put it in a list so the
-                    # rest of the code works
-                    response_data = [response_data]
-
-                for idx, entry in enumerate(response_data):
-                    # we don't use the returned address as it may be lowercased
-                    balances[accounts[total_idx + idx]] = satoshis_to_btc(
-                        FVal(entry['final_balance']),
-                    )
-                total_idx += len(batch)
+            # if 1 account is bech32 we have to query blockstream. blockchaininfo won't work
+            source = 'blockstream'
+            balances = {}
+            for account in accounts:
+                url = f'https://blockstream.info/api/address/{account}'
+                response_data = request_get_dict(url=url, handle_429=True, backoff_in_seconds=4)
+                stats = response_data['chain_stats']
+                balance = int(stats['funded_txo_sum']) - int(stats['spent_txo_sum'])
+                balances[account] = satoshis_to_btc(balance)
         else:
             params = '|'.join(accounts)
             btc_resp = request_get_dict(
@@ -80,49 +47,36 @@ def get_bitcoin_addresses_balances(accounts: List[BTCAddress]) -> Dict[BTCAddres
             UnableToDecryptRemoteData,
             requests.exceptions.Timeout,
     ) as e:
-        raise RemoteError(f'bitcoin external API request for balances failed due to {str(e)}')
+        raise RemoteError(f'bitcoin external API request for balances failed due to {str(e)}') from e  # noqa: E501
     except KeyError as e:
         raise RemoteError(
             f'Malformed response when querying bitcoin blockchain via {source}.'
             f'Did not find key {e}',
-        )
+        ) from e
 
     return balances
 
 
-def _check_blockcypher_for_transactions(
+def _check_blockstream_for_transactions(
         accounts: List[BTCAddress],
 ) -> Dict[BTCAddress, Tuple[bool, FVal]]:
+    """May raise connection errors or KeyError"""
     have_transactions = {}
-    new_accounts = _prepare_blockcypher_accounts(accounts)
-    # blockcypher's batching takes up as many api queries as the batch,
-    # and the api rate limit is 3 requests per second. So we should make
-    # sure each batch is of max size 3
-    # https://www.blockcypher.com/dev/bitcoin/#batching
-    batches = [new_accounts[x: x + 3] for x in range(0, len(new_accounts), 3)]
-    total_idx = 0
-    for batch in batches:
-        params = ';'.join(batch)
-        url = f'https://api.blockcypher.com/v1/btc/main/addrs/{params}/balance'
-        response_data = request_get(url=url, handle_429=True, backoff_in_seconds=4)
-
-        if isinstance(response_data, dict):
-            # If only one account was requested put it in a list so the
-            # rest of the code works
-            response_data = [response_data]
-
-        for idx, entry in enumerate(response_data):
-            balance = satoshis_to_btc(FVal(entry['final_balance']))
-            # we don't use the returned address as it may be lowercased
-            have_transactions[accounts[total_idx + idx]] = (entry['final_n_tx'] != 0, balance)
-        total_idx += len(batch)
+    for account in accounts:
+        url = f'https://blockstream.info/api/address/{account}'
+        response_data = request_get_dict(url=url, handle_429=True, backoff_in_seconds=4)
+        stats = response_data['chain_stats']
+        balance = satoshis_to_btc(int(stats['funded_txo_sum']) - int(stats['spent_txo_sum']))
+        have_txs = stats['tx_count'] != 0
+        have_transactions[account] = (have_txs, balance)
 
     return have_transactions
 
 
 def _check_blockchaininfo_for_transactions(
         accounts: List[BTCAddress],
 ) -> Dict[BTCAddress, Tuple[bool, FVal]]:
+    """May raise connection errors or KeyError"""
     have_transactions = {}
     params = '|'.join(accounts)
     btc_resp = request_get_dict(
@@ -149,14 +103,21 @@ def have_bitcoin_transactions(accounts: List[BTCAddress]) -> Dict[BTCAddress, Tu
     """
     try:
         if _have_bc1_accounts(accounts):
-            have_transactions = _check_blockcypher_for_transactions(accounts)
+            source = 'blockstream'
+            have_transactions = _check_blockstream_for_transactions(accounts)
         else:
+            source = 'blockchain.info'
             have_transactions = _check_blockchaininfo_for_transactions(accounts)
     except (
             requests.exceptions.ConnectionError,
             UnableToDecryptRemoteData,
             requests.exceptions.Timeout,
     ) as e:
-        raise RemoteError(f'bitcoin external API request for transactions failed due to {str(e)}')
+        raise RemoteError(f'bitcoin external API request for transactions failed due to {str(e)}') from e  # noqa: E501
+    except KeyError as e:
+        raise RemoteError(
+            f'Malformed response when querying bitcoin blockchain via {source}.'
+            f'Did not find key {str(e)}',
+        ) from e
 
     return have_transactions

rotkehlchen/chain/bitcoin/xpub.py

@@ -67,7 +67,7 @@ def _derive_addresses_loop(
         root: HDKey,
 ) -> List[XpubDerivedAddressData]:
     """May raise:
-    - RemoteError: if blockcypher/blockchain.info can't be reached
+    - RemoteError: if blockstream/blockchain.info can't be reached
     """
     step_index = start_index
     addresses: List[XpubDerivedAddressData] = []
@@ -120,7 +120,7 @@ def derive_addresses_from_xpub_data(
     This is to make it easier to later derive and check more addresses
 
     May raise:
-    - RemoteError: if blockcypher/blockchain.info and others can't be reached
+    - RemoteError: if blockstream/blockchain.info and others can't be reached
     """
     if xpub_data.derivation_path is not None:
         account_xpub = xpub_data.xpub.derive_path(xpub_data.derivation_path)
@@ -158,7 +158,7 @@ def _derive_xpub_addresses(self, xpub_data: XpubData, new_xpub: bool) -> None:
         have not had any transactions to the tracked bitcoin addresses
 
         May raise:
-        - RemoteError: if blockcypher/blockchain.info and others can't be reached
+        - RemoteError: if blockstream/blockchain.info and others can't be reached
         """
         last_receiving_idx, last_change_idx = self.db.get_last_xpub_derived_indices(xpub_data)
         derived_addresses_data = derive_addresses_from_xpub_data(
@@ -220,7 +220,7 @@ def add_bitcoin_xpub(self, xpub_data: XpubData) -> 'BlockchainBalancesUpdate':
         May raise:
         - InputError: If the xpub already exists in the DB
         - TagConstraintError if any of the given account data contain unknown tags.
-        - RemoteError if an external service such as blockcypher is queried and
+        - RemoteError if an external service such as blockstream is queried and
           there is a problem with its query.
         """
         # First try to add the xpub, and if it already exists raise

rotkehlchen/chain/manager.py

@@ -409,10 +409,10 @@ def query_balances(
         return self.get_balances_update()
 
     def query_btc_balances(self) -> None:
-        """Queries blockchain.info/blockcypher for the balance of all BTC accounts
+        """Queries blockchain.info/blockstream for the balance of all BTC accounts
 
         May raise:
-        - RemotError if there is a problem querying blockchain.info or cryptocompare
+        - RemotError if there is a problem querying any remote
         """
         if len(self.accounts.btc) == 0:
             return

rotkehlchen/externalapis/etherscan.py

@@ -457,4 +457,10 @@ def get_blocknumber_by_time(self, ts: Timestamp) -> int:
             action='getblocknobytime',
             options=options,
         )
+        if not isinstance(result, int):
+            # At this point the blocknumber string returned by etherscan should be an int
+            raise RemoteError(
+                f'Got unexpected etherscan response: {result} to getblocknobytime call',
+            )
+
         return result

rotkehlchen/tests/utils/blockchain.py

@@ -547,21 +547,13 @@ def mock_requests_get(url, *args, **kwargs):
                 if idx < len(addresses) - 1:
                     response += ','
             response += ']}'
-        elif 'blockcypher.com' in url:
-            addresses = url.split('addrs/')[1].split('/balance')[0].split(';')
-            response = '['
-            for idx, address in enumerate(addresses):
-                balance = btc_map.get(address, '0')
-                entry = f'{{"address": "{address}", "final_balance": {balance}}}'
-                if len(addresses) == 1:
-                    response = entry
-                else:
-                    response += entry
-                if idx < len(addresses) - 1:
-                    response += ','
-
-            if len(addresses) > 1:
-                response += ']'
+        elif 'blockstream.info' in url:
+            split_result = url.rsplit('/', 1)
+            if len(split_result) != 2:
+                raise AssertionError(f'Could not find bitcoin address at url {url}')
+            address = split_result[1]
+            balance = btc_map.get(address, '0')
+            response = f"""{{"address":"{address}","chain_stats":{{"funded_txo_count":1,"funded_txo_sum":{balance},"spent_txo_count":0,"spent_txo_sum":0,"tx_count":1}},"mempool_stats":{{"funded_txo_count":0,"funded_txo_sum":0,"spent_txo_count":0,"spent_txo_sum":0,"tx_count":0}}}}"""  # noqa: E501
         else:
             return original_requests_get(url, *args, **kwargs)
 

rotkehlchen/utils/misc.py

@@ -90,7 +90,7 @@ def timestamp_to_iso8601(ts: Timestamp, utc_as_z: bool = False) -> str:
     return res if utc_as_z is False else res.replace('+00:00', 'Z')
 
 
-def satoshis_to_btc(satoshis: FVal) -> FVal:
+def satoshis_to_btc(satoshis: Union[int, FVal]) -> FVal:
     return satoshis * FVal('0.00000001')
 
 
@@ -228,8 +228,8 @@ def request_get(
 
     try:
         result = rlk_jsonloads(response.text)
-    except json.decoder.JSONDecodeError:
-        raise UnableToDecryptRemoteData(f'{url} returned malformed json')
+    except json.decoder.JSONDecodeError as e:
+        raise UnableToDecryptRemoteData(f'{url} returned malformed json. Error: {str(e)}') from e
 
     return result