Add total earned from liquidations in aave history endpoint

docs/api.rst

@@ -3341,13 +3341,14 @@ Getting Aave historical data
                       "tx_hash": "0x618fc9542890a2f58ab20a3c12d173b3638af11fda813e61788e242b4fc9a755",
 		      "log_index": 1
                   }],
-                  "total_earned": {
+                  "total_earned_interest": {
 		      "BAT": {
 		          "amount": "0.9482",
 			  "usd_value": "0.2312"
 		      }
 		  },
-		  "total_lost": {}
+		  "total_lost": {},
+		  "total_earned_liquidations": {},
               }
           },
           "message": ""
@@ -3370,8 +3371,9 @@ Getting Aave historical data
    :resjsonarr string collateral_balance: This attribute appears only in ``"liquidation"`` events. It shows the value (amount and usd_value mapping) of the collateral asset that the user loses due to liquidation. The rate is the asset/USD rate at the events's timestamp.
    :resjsonarr string principal_asset: This attribute appears only in ``"liquidation"`` events. It shows the principal debt asset that is repaid due to the liquidation due to liquidation.
    :resjsonarr string principal_balance: This attribute appears only in ``"liquidation"`` events. It shows the value (amount and usd_value mapping) of the principal asset whose debt is repaid due to liquidation. The rate is the asset/USD rate at the events's timestamp.
-   :resjson object total_earned: A mapping of asset identifier to total earned (amount + usd_value mapping) for each asset. The total earned is essentially the sum of all interest payments plus the difference between ``balanceOf`` and ``principalBalanceOf`` for each asset.
+   :resjson object total_earned_interest: A mapping of asset identifier to total earned (amount + usd_value mapping) for each asset's interest earnings. The total earned is essentially the sum of all interest payments plus the difference between ``balanceOf`` and ``principalBalanceOf`` for each asset.
    :resjson object total_lost: A mapping of asset identifier to total lost (amount + usd_value mapping) for each asset. The total losst for each asset is essentially the accrued interest from borrowing and the collateral lost from liquidations.
+   :resjson object total_earned_liquidations: A mapping of asset identifier to total earned (amount + usd_value mapping) for each repaid assets during liquidations.
 
    :statuscode 200: Aave history succesfully queried.
    :statuscode 409: No user is currently logged in or currently logged in user does not have a premium subscription. Or aave module is not activated.

rotkehlchen/chain/ethereum/aave/__init__.py

@@ -166,6 +166,10 @@ def get_history(
 
             if self.use_graph:
                 if self.graph_inquirer is None:  # could not initialize graph
+                    log.error(
+                        "Tried to query Aave's history via the subgraph "
+                        "without an initialized graph_inquirer",
+                    )
                     return {}
 
                 aave_balances = self.get_balances(given_defi_balances)

rotkehlchen/chain/ethereum/aave/blockchain.py

@@ -205,8 +205,9 @@ def get_history_for_address(
         total_address_events.sort(key=lambda event: event.timestamp)
         return AaveHistory(
             events=total_address_events,
-            total_earned=total_earned_map,
+            total_earned_interest=total_earned_map,
             total_lost={},
+            total_earned_liquidations={},
         )
 
     def get_events_for_atoken_and_address(

rotkehlchen/chain/ethereum/aave/common.py

@@ -120,8 +120,9 @@ class AaveHistory(NamedTuple):
     """All events and total interest accrued for all Atoken of an address
     """
     events: List[AaveEvent]
-    total_earned: Dict[Asset, Balance]
+    total_earned_interest: Dict[Asset, Balance]
     total_lost: Dict[Asset, Balance]
+    total_earned_liquidations: Dict[Asset, Balance]
 
 
 class AaveInquirer():

rotkehlchen/chain/ethereum/aave/graph.py

@@ -154,13 +154,23 @@ class AaveUserReserve(NamedTuple):
     symbol: str
 
 
+class AaveEventProcessingResult(NamedTuple):
+    interest_events: List[AaveSimpleEvent]
+    total_earned_interest: Dict[Asset, Balance]
+    total_lost: Dict[Asset, Balance]
+    total_earned_liquidations: Dict[Asset, Balance]
+
+
 def _calculate_loss(
         borrow_actions: List[AaveEvent],
         balances: AaveBalances,
-) -> Dict[Asset, Balance]:
+) -> Tuple[Dict[Asset, Balance], Dict[Asset, Balance]]:
+    """Returns a tuple of mapping of losses due to liquidation/borrowing and
+    earnings due to keeping the principal repaid by the liquidation"""
     borrow_actions.sort(key=lambda event: event.timestamp)
     historical_borrow_balances: Dict[Asset, FVal] = defaultdict(FVal)
     total_lost: Dict[Asset, Balance] = defaultdict(Balance)
+    total_earned: Dict[Asset, Balance] = defaultdict(Balance)
 
     for b_action in borrow_actions:
         if b_action.event_type == 'borrow':
@@ -170,8 +180,9 @@ def _calculate_loss(
         elif b_action.event_type == 'liquidation':
             # At liquidation you lose the collateral asset
             total_lost[b_action.collateral_asset] += b_action.collateral_balance  # type: ignore  # noqa: E501
-            # And your principal asset is repaid
+            # And your principal asset is repaid and you gain it
             historical_borrow_balances[b_action.principal_asset] += b_action.principal_balance.amount  # type: ignore # noqa: E501
+            total_earned[b_action.principal_asset] += b_action.principal_balance  # type: ignore
 
     for b_asset, amount in historical_borrow_balances.items():
         borrow_balance = balances.borrowing.get(b_asset.identifier, None)
@@ -185,7 +196,7 @@ def _calculate_loss(
             usd_value=amount * usd_price,
         )
 
-    return total_lost
+    return total_lost, total_earned
 
 
 def _parse_common_event_data(
@@ -481,7 +492,7 @@ def _process_events(
             liquidations: List[AaveLiquidationEvent],
             db_events: List[AaveEvent],
             balances: AaveBalances,
-    ) -> Tuple[List[AaveSimpleEvent], Dict[Asset, Balance], Dict[Asset, Balance]]:
+    ) -> AaveEventProcessingResult:
         """Calculates the interest events and the total earned from all the given events.
         Also calculates total loss from borrowing and liquidations.
 
@@ -513,12 +524,16 @@ def _process_events(
             from_ts=from_ts,
             to_ts=to_ts,
         )
-        total_lost = _calculate_loss(
+        total_lost, total_earned_liquidations = _calculate_loss(
             borrow_actions=borrow_actions + borrows + repays + liquidations,  # type: ignore
             balances=balances,
         )
-
-        return interest_events, total_earned, total_lost
+        return AaveEventProcessingResult(
+            interest_events=interest_events,
+            total_earned_interest=total_earned,
+            total_lost=total_lost,
+            total_earned_liquidations=total_earned_liquidations,
+        )
 
     def _get_user_data(
             self,
@@ -562,7 +577,7 @@ def _get_user_data(
                 to_ts,
             )
 
-        interest_events, total_earned, total_lost = self._process_events(
+        result = self._process_events(
             user_address=address,
             user_result=user_result,
             from_ts=from_ts,
@@ -577,7 +592,7 @@ def _get_user_data(
         )
 
         # Add all new events to the DB
-        new_events: List[AaveEvent] = deposits + withdrawals + interest_events + borrows + repays + liquidation_calls  # type: ignore  # noqa: E501
+        new_events: List[AaveEvent] = deposits + withdrawals + result.interest_events + borrows + repays + liquidation_calls  # type: ignore  # noqa: E501
         self.database.add_aave_events(address, new_events)
         # After all events have been queried then also update the query range.
         # Even if no events are found for an address we need to remember the range
@@ -593,8 +608,9 @@ def _get_user_data(
         all_events.sort(key=lambda event: sort_map[event.event_type] + event.timestamp)
         return AaveHistory(
             events=all_events,
-            total_earned=total_earned,
-            total_lost=total_lost,
+            total_earned_interest=result.total_earned_interest,
+            total_lost=result.total_lost,
+            total_earned_liquidations=result.total_earned_liquidations,
         )
 
     def _parse_deposits(

rotkehlchen/history/trades.py

@@ -291,7 +291,7 @@ def fail_history_cb(error_msg: str) -> None:
                         ))
                         total_amount_per_token[event.asset] += event.value.amount
 
-                for token, balance in aave_history.total_earned.items():
+                for token, balance in aave_history.total_earned_interest.items():
                     # Αdd an extra event per token per address for the remaining not paid amount
                     if token in total_amount_per_token:
                         defi_events.append(DefiEvent(
@@ -301,15 +301,24 @@ def fail_history_cb(error_msg: str) -> None:
                             amount=balance.amount - total_amount_per_token[token],
                         ))
 
+                # Add all losses from aave borrowing/liquidations
                 for asset, balance in aave_history.total_lost.items():
-                    # Add all losses from aave borrowing/liquidations
                     defi_events.append(DefiEvent(
                         timestamp=now,
                         event_type=DefiEventType.AAVE_LOSS,
                         asset=asset,
                         amount=balance.amount,
                     ))
 
+                # Add earned assets from aave liquidations
+                for asset, balance in aave_history.total_earned_liquidations.items():
+                    defi_events.append(DefiEvent(
+                        timestamp=now,
+                        event_type=DefiEventType.AAVE_LOAN_INTEREST,
+                        asset=asset,
+                        amount=balance.amount,
+                    ))
+
         history.sort(key=lambda trade: action_get_timestamp(trade))
         return (
             empty_or_error,

rotkehlchen/premium/sync.py

@@ -151,8 +151,12 @@ def maybe_upload_data_to_server(self, force_upload: bool = False) -> bool:
         if diff < 3600 and not force_upload:
             return False
 
+        try:
+            metadata = self.premium.query_last_data_metadata()
+        except RemoteError as e:
+            log.debug('upload to server -- fetching metadata error', error=str(e))
+            return False
         b64_encoded_data, our_hash = self.data.compress_and_encrypt_db(self.password)
-        metadata = self.premium.query_last_data_metadata()
 
         log.debug(
             'CAN_PUSH',

rotkehlchen/tests/api/test_aave.py

@@ -136,15 +136,17 @@ def _query_simple_aave_history_test(
             result = assert_proper_response_with_result(response)
 
     assert len(result) == 1
-    assert len(result[AAVE_TEST_ACC_2]) == 3
+    assert len(result[AAVE_TEST_ACC_2]) == 4
     events = result[AAVE_TEST_ACC_2]['events']
-    total_earned = result[AAVE_TEST_ACC_2]['total_earned']
+    total_earned_interest = result[AAVE_TEST_ACC_2]['total_earned_interest']
     total_lost = result[AAVE_TEST_ACC_2]['total_lost']
+    total_earned_liquidations = result[AAVE_TEST_ACC_2]['total_earned_liquidations']
     assert len(total_lost) == 0
-    assert len(total_earned) == 1
-    assert len(total_earned['aDAI']) == 2
-    assert FVal(total_earned['aDAI']['amount']) >= FVal('24.207179802347627414')
-    assert FVal(total_earned['aDAI']['usd_value']) >= FVal('24.580592532348742989192')
+    assert len(total_earned_liquidations) == 0
+    assert len(total_earned_interest) == 1
+    assert len(total_earned_interest['aDAI']) == 2
+    assert FVal(total_earned_interest['aDAI']['amount']) >= FVal('24.207179802347627414')
+    assert FVal(total_earned_interest['aDAI']['usd_value']) >= FVal('24.580592532348742989192')
 
     expected_events = process_result_list(expected_aave_deposit_test_events)
     if use_graph:
@@ -198,15 +200,21 @@ def _query_borrowing_aave_history_test(setup: BalancesTestSetup, server: APIServ
         result = assert_proper_response_with_result(response)
 
     assert len(result) == 1
-    assert len(result[AAVE_TEST_ACC_3]) == 3
+    assert len(result[AAVE_TEST_ACC_3]) == 4
     events = result[AAVE_TEST_ACC_3]['events']
-    total_earned = result[AAVE_TEST_ACC_3]['total_earned']
+    total_earned_interest = result[AAVE_TEST_ACC_3]['total_earned_interest']
     total_lost = result[AAVE_TEST_ACC_3]['total_lost']
+    total_earned_liquidations = result[AAVE_TEST_ACC_3]['total_earned_liquidations']
 
-    assert len(total_earned) == 1
-    assert len(total_earned['aWBTC']) == 2
-    assert FVal(total_earned['aWBTC']['amount']) >= FVal('0.00000833')
-    assert FVal(total_earned['aWBTC']['usd_value']) >= ZERO
+    assert len(total_earned_interest) == 1
+    assert len(total_earned_interest['aWBTC']) == 2
+    assert FVal(total_earned_interest['aWBTC']['amount']) >= FVal('0.00000833')
+    assert FVal(total_earned_interest['aWBTC']['usd_value']) >= ZERO
+
+    assert len(total_earned_liquidations) == 1
+    assert len(total_earned_liquidations['ETH']) == 2
+    assert FVal(total_earned_liquidations['ETH']['amount']) >= FVal('9.251070299427409111')
+    assert FVal(total_earned_liquidations['ETH']['usd_value']) >= ZERO
 
     assert len(total_lost) == 3
     eth_lost = total_lost['ETH']
@@ -264,12 +272,14 @@ def _test_for_duplicates_and_negatives(setup: BalancesTestSetup, server: APIServ
 
     assert len(result) == 1
     result = result[AAVE_TEST_ACC_1]
-    assert len(result) == 3
+    assert len(result) == 4
 
-    for _, entry in result['total_earned'].items():
+    for _, entry in result['total_earned_interest'].items():
         assert FVal(entry['amount']) > ZERO
     for _, entry in result['total_lost'].items():
         assert FVal(entry['amount']) > ZERO
+    for _, entry in result['total_earned_liquidations'].items():
+        assert FVal(entry['amount']) > ZERO
 
     events = result['events']
     events_set = set()