Add all DeFi balances querying

docs/api.rst

@@ -2637,6 +2637,154 @@ Getting blockchain account data
    :statuscode 409: User is not logged in.
    :statuscode 500: Internal Rotki error
 
+Getting all DeFi balances
+=========================
+
+.. http:get:: /api/(version)/blockchains/ETH/defi
+
+   Doing a GET on the DeFi balances endpoint will return a mapping of all accounts to their respective balances in DeFi protocols.
+
+   .. note::
+      This endpoint can also be queried asynchronously by using ``"async_query": true``
+
+   .. note::
+      This endpoint also accepts parameters as query arguments.
+
+   **Example Request**:
+
+   .. http:example:: curl wget httpie python-requests
+
+      GET /api/1/blockchains/ETH/defi HTTP/1.1
+      Host: localhost:5042
+
+   :reqjson bool async_query: Boolean denoting whether this is an asynchronous query or not
+
+   **Example Response**:
+
+   .. sourcecode:: http
+
+      HTTP/1.1 200 OK
+      Content-Type: application/json
+
+      {
+          "result": {
+              "0xA0B6B7fEa3a3ce3b9e6512c0c5A157a385e81056": [{
+                  "protocol": "Curve",
+                  "balance_type": "Asset",
+                  "base_balance": {
+                      "token_address": "0xdF5e0e81Dff6FAF3A7e52BA697820c5e32D806A8",
+                      "token_name": "Y Pool",
+                      "token_symbol": "yDAI+yUSDC+yUSDT+yTUSD",
+                      "balance": {
+                          "amount": "1000",
+                          "usd_value": "1009.12"
+                      }
+                  },
+                  "underlying_balances": [{
+                      "token_address": "0x6B175474E89094C44Da98b954EedeAC495271d0F",
+                      "token_name": "Dai Stablecoin",
+                      "token_symbol": "DAI",
+                      "balance": {
+                          "amount": "200",
+                          "usd_value": "201.12"
+                      }
+                  }, {
+                      "token_address": "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48",
+                      "token_name": "USD//C",
+                      "token_symbol": "USDC",
+                      "balance": {
+                          "amount": "300",
+                          "usd_value": "302.14"
+                      }
+                  }, {
+                      "token_address": "0xdAC17F958D2ee523a2206206994597C13D831ec7",
+                      "token_name": "Tether USD",
+                      "token_symbol": "USDT",
+                      "balance": {
+                          "amount": "280",
+                          "usd_value": "281.98"
+                      }
+                  }, {
+                      "token_address": "0x0000000000085d4780B73119b644AE5ecd22b376",
+                      "token_name": "TrueUSD",
+                      "token_symbol": "TUSD",
+                      "balance": {
+                          "amount": "220",
+                          "usd_value": "221.201"
+                      }
+                  }]
+              }, {
+                  "protocol": "Compound",
+                  "balance_type": "Asset",
+                  "base_balance": {
+                      "token_address": "0x6C8c6b02E7b2BE14d4fA6022Dfd6d75921D90E4E",
+                      "token_name": "Compound Basic Attention Token",
+                      "token_symbol": "cBAT",
+                      "balance": {
+                          "amount": "8000",
+                          "usd_value": "36.22"
+                      }
+                  },
+                  "underlying_balances": [{
+                      "token_address": "0x0D8775F648430679A709E98d2b0Cb6250d2887EF",
+                      "token_name": "Basic Attention Token",
+                      "token_symbol": "BAT",
+                      "balance": {
+                          "amount": "150",
+                          "usd_value": "36.21"
+                      }
+                  }]
+              }, {
+                  "protocol": "Compound",
+                  "balance_type": "Asset",
+                  "base_balance": {
+                      "token_address": "0xc00e94Cb662C3520282E6f5717214004A7f26888",
+                      "token_name": "Compound",
+                      "token_symbol": "COMP",
+                      "balance": {
+                          "amount": "0.01",
+                          "usd_value": "1.9"
+                      }
+                  },
+                  "underlying_balances": []
+              }],
+              "0x78b0AD50E768D2376C6BA7de33F426ecE4e03e0B": [{
+                  "protocol": "Aave",
+                  "balance_type": "Asset",
+                  "base_balance": {
+                      "token_address": "0xfC1E690f61EFd961294b3e1Ce3313fBD8aa4f85d",
+                      "token_name": "Aave Interest bearing DAI",
+                      "token_symbol": "aDAI",
+                      "balance": {
+                          "amount": "2000",
+                          "usd_value": "2001.95"
+                      }
+                  },
+                  "underlying_balances": [{
+                      "token_address": "0x6B175474E89094C44Da98b954EedeAC495271d0F",
+                      "token_name": "Dai Stablecoin",
+                      "token_symbol": "DAI",
+                      "balance": {
+                          "amount": "2000",
+                          "usd_value": "2001.95"
+                      }
+                  }]
+              }],
+          },
+          "message": ""
+      }
+
+   :resjson object result: A mapping from account to list of DeFi balances.
+   :resjsonarr string protocol: The name of the protocol. Since these names come from Zerion check `here <https://github.com/zeriontech/defi-sdk#supported-protocols>`__ for supported names.
+   :resjsonarr string balance_type: One of ``"Asset"`` or ``"Debt"`` denoting that one if deposited asset in DeFi and the other a debt or liability.
+   :resjsonarr string base_balance: A single DefiBalance entry. It's comprised of a token address, name, symbol and a balance. This is the actually deposited asset in the protocol. Can also be a synthetic in case of synthetic protocols or lending pools.
+   :resjsonarr string underlying_balances: A list of underlying DefiBalances supporting the base balance. Can also be an empty list. The format of each balance is thesame as that of base_balance. For lending this is going to be the normal token. For example for aDAI this is DAI. For cBAT this is BAT etc. For pools this list contains all tokens that are contained in the pool.
+
+   :statuscode 200: Balances succesfully queried.
+   :statuscode 409: User is not logged in or if using own chain the chain is not synced.
+   :statuscode 500: Internal Rotki error.
+   :statuscode 502: An external service used in the query such as etherscan could not be reached or returned unexpected response.
+
 
 Getting current ethereum MakerDAO DSR balance
 =================================================

rotkehlchen/api/rest.py

@@ -1375,6 +1375,35 @@ def import_data(
         self.rotkehlchen.data_importer.import_cointracking_csv(filepath)
         return api_response(OK_RESULT, status_code=HTTPStatus.OK)
 
+    def _get_defi_balances(self) -> Dict[str, Any]:
+        """
+        This returns the typical async response dict but with the
+        extra status code argument for errors
+        """
+        try:
+            balances = self.rotkehlchen.chain_manager.query_defi_balances()
+        except EthSyncError as e:
+            return {'result': None, 'message': str(e), 'status_code': HTTPStatus.CONFLICT}
+        except RemoteError as e:
+            return {'result': None, 'message': str(e), 'status_code': HTTPStatus.BAD_GATEWAY}
+
+        return {'result': process_result(balances), 'message': ''}
+
+    @require_loggedin_user()
+    def get_defi_balances(self, async_query: bool) -> Response:
+        if async_query:
+            return self._query_async(command='_get_defi_balances')
+
+        response = self._get_defi_balances()
+        result = response['result']
+        msg = response['message']
+        if result is None:
+            return api_response(wrap_in_fail_result(msg), status_code=response['status_code'])
+
+        # success
+        result_dict = _wrap_in_result(result, msg)
+        return api_response(result_dict, status_code=HTTPStatus.OK)
+
     def _eth_module_query(self, module: str, method: str, **kwargs: Any) -> Dict[str, Any]:
         result = None
         msg = ''

rotkehlchen/api/server.py

@@ -24,6 +24,7 @@
     BlockchainBalancesResource,
     BlockchainsAccountsResource,
     DataImportResource,
+    DefiBalancesResource,
     EthereumTokensResource,
     ExchangeBalancesResource,
     ExchangesResource,
@@ -108,6 +109,7 @@
     ('/history/', HistoryProcessingResource),
     ('/history/export/', HistoryExportingResource),
     ('/blockchains/ETH/tokens', EthereumTokensResource),
+    ('/blockchains/ETH/defi', DefiBalancesResource),
     ('/blockchains/ETH/modules/makerdao/dsrbalance', MakerDAODSRBalanceResource),
     ('/blockchains/ETH/modules/makerdao/dsrhistory', MakerDAODSRHistoryResource),
     ('/blockchains/ETH/modules/makerdao/vaults', MakerDAOVaultsResource),

rotkehlchen/api/v1/resources.py

@@ -778,6 +778,15 @@ def put(self, source: Literal['cointracking.info'], filepath: Path) -> None:
         return self.rest_api.import_data(source=source, filepath=filepath)
 
 
+class DefiBalancesResource(BaseResource):
+
+    get_schema = AsyncQueryArgumentSchema()
+
+    @use_kwargs(get_schema, location='json_and_query')  # type: ignore
+    def get(self, async_query: bool) -> Response:
+        return self.rest_api.get_defi_balances(async_query)
+
+
 class MakerDAODSRBalanceResource(BaseResource):
 
     get_schema = AsyncQueryArgumentSchema()

rotkehlchen/chain/ethereum/zerion.py

@@ -28,6 +28,9 @@ class DefiProtocol(NamedTuple):
     icon_link: str
     version: int
 
+    def serialize(self) -> str:
+        return self.name
+
 
 class DefiBalance(NamedTuple):
     token_address: ChecksumEthAddress
@@ -93,6 +96,13 @@ def all_balances_for_account(self, account: ChecksumEthAddress) -> List[DefiProt
                         defi_balance = self._get_single_balance(balance)
                         underlying_balances.append(defi_balance)
 
+                    if base_balance.balance.usd_value == ZERO:
+                        # This can happen. We can't find a price for some assets
+                        # such as combined pool assets. But we can instead use
+                        # the sum of the usd_value of the underlying_balances
+                        usd_sum = sum(x.balance.usd_value for x in underlying_balances)
+                        base_balance.balance.usd_value = usd_sum  # type: ignore
+
                 protocol_balances.append(DefiProtocolBalances(
                     protocol=protocol,
                     balance_type=balance_type,
@@ -108,18 +118,18 @@ def _get_single_balance(self, entry: Tuple[Tuple[str, str, str, int], int]) -> D
         decimals = metadata[3]
         normalized_value = token_normalized_value(balance_value, decimals)
         token_symbol = metadata[2]
+
         try:
             asset = Asset(token_symbol)
             usd_price = Inquirer().find_usd_price(asset)
         except (UnknownAsset, UnsupportedAsset):
-            if '+' not in token_symbol:  # ignore the curve fi "pool" combined base asset
+            if '+' not in token_symbol:  # ignore the curvefinance pool combined base asset
                 self.msg_aggregator.add_error(
                     f'Unsupported asset {token_symbol} encountered during DeFi protocol queries',
                 )
             usd_price = Price(ZERO)
 
         usd_value = normalized_value * usd_price
-
         defi_balance = DefiBalance(
             token_address=to_checksum_address(metadata[0]),
             token_name=metadata[1],

rotkehlchen/serialization/serialize.py

@@ -19,6 +19,7 @@
     VaultEvent,
     VaultEventType,
 )
+from rotkehlchen.chain.ethereum.zerion import DefiBalance, DefiProtocol, DefiProtocolBalances
 from rotkehlchen.db.settings import DBSettings
 from rotkehlchen.db.utils import AssetBalance, LocationData, SingleAssetBalance
 from rotkehlchen.exchanges.data_structures import Trade
@@ -63,6 +64,8 @@ def _process_entry(entry: Any) -> Union[str, List[Any], Dict[str, Any], Any]:
             'amount': entry.amount,
             'usd_value': entry.usd_value,
         }
+    elif isinstance(entry, DefiProtocol):
+        return entry.serialize()
     elif isinstance(entry, (
             Trade,
             MakerDAOVault,
@@ -84,6 +87,8 @@ def _process_entry(entry: Any) -> Union[str, List[Any], Dict[str, Any], Any]:
             AaveBalances,
             AaveEvent,
             AaveHistory,
+            DefiBalance,
+            DefiProtocolBalances,
     )):
         return process_result(entry._asdict())
     elif isinstance(entry, tuple):

rotkehlchen/tests/api/test_defi.py

@@ -0,0 +1,28 @@
+import pytest
+import requests
+
+from rotkehlchen.tests.utils.api import api_url_for, assert_proper_response_with_result
+
+
+@pytest.mark.parametrize('ethereum_accounts', [['0x2B888954421b424C5D3D9Ce9bB67c9bD47537d12']])
+def test_query_defi_balances(rotkehlchen_api_server, ethereum_accounts):  # pylint: disable=unused-argument  # noqa: E501
+    """Check querying the defi balances endpoint works. Uses real data.
+
+    TODO: Here we should use a test account for which we will know what balances
+    it has and we never modify
+    """
+    response = requests.get(api_url_for(
+        rotkehlchen_api_server,
+        "defibalancesresource",
+    ))
+    result = assert_proper_response_with_result(response)
+
+    assert len(result) == 1
+    # Since we can't really be sure of latest balance of a non-test account just check
+    # for correctness of result if there is any balance
+    if len(result['0x2B888954421b424C5D3D9Ce9bB67c9bD47537d12']) != 0:
+        first_entry = result['0x2B888954421b424C5D3D9Ce9bB67c9bD47537d12'][0]
+        assert first_entry['protocol'] is not None
+        assert first_entry['balance_type'] in ('Asset', 'Debt')
+        assert first_entry['base_balance'] is not None
+        assert first_entry['underlying_balances'] is not None