Merge branch 'feat/objectify-ccxt' into feature/catch-exchange-errors

freqtrade/exchange/__init__.py

@@ -3,6 +3,7 @@
 import logging
 from random import randint
 from typing import List, Dict, Any, Optional
+from datetime import datetime
 
 import ccxt
 import arrow
@@ -134,7 +135,8 @@ def buy(pair: str, rate: float, amount: float) -> Dict:
             'side': 'buy',
             'remaining': 0.0,
             'datetime': arrow.utcnow().isoformat(),
-            'status': 'closed'
+            'status': 'closed',
+            'fee': None
         }
         return {'id': order_id}
 
@@ -324,6 +326,25 @@ def get_order(order_id: str, pair: str) -> Dict:
 
 
 @retrier
+def get_trades_for_order(order_id: str, pair: str, since: datetime) -> List:
+    if _CONF['dry_run']:
+        return []
+    if not exchange_has('fetchMyTrades'):
+        return []
+    try:
+        my_trades = _API.fetch_my_trades(pair, since.timestamp())
+        matched_trades = [trade for trade in my_trades if trade['order'] == order_id]
+
+        return matched_trades
+
+    except ccxt.NetworkError as e:
+        raise TemporaryError(
+            'Could not get trades due to networking error. Message: {}'.format(e)
+        )
+    except ccxt.BaseError as e:
+        raise OperationalException(e)
+
+
 def get_pair_detail_url(pair: str) -> str:
     try:
         url_base = _API.urls.get('www')
@@ -371,3 +392,13 @@ def get_fee(symbol='ETH/BTC', type='', side='', amount=1,
                 e.__class__.__name__, e))
     except ccxt.BaseError as e:
         raise OperationalException(e)
+
+
+def get_amount_lots(pair: str, amount: float) -> float:
+    """
+    get buyable amount rounding, ..
+    """
+    # validate that markets are loaded before trying to get fee
+    if not _API.markets:
+        _API.load_markets()
+    return _API.amount_to_lots(pair, amount)

freqtrade/freqtradebot.py

@@ -287,7 +287,7 @@ def create_trade(self) -> bool:
         if not whitelist:
             raise DependencyException('No currency pairs in whitelist')
 
-        # Pick pair based on StochRSI buy signals
+        # Pick pair based on buy signals
         for _pair in whitelist:
             (buy, sell) = self.analyze.get_signal(_pair, interval)
             if buy and not sell:
@@ -323,11 +323,13 @@ def create_trade(self) -> bool:
             )
         )
         # Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL
+        fee = exchange.get_fee(symbol=pair, taker_or_maker='maker')
         trade = Trade(
             pair=pair,
             stake_amount=stake_amount,
             amount=amount,
-            fee=exchange.get_fee(taker_or_maker='maker'),
+            fee_open=fee,
+            fee_close=fee,
             open_rate=buy_limit,
             open_date=datetime.utcnow(),
             exchange=exchange.get_id(),
@@ -363,7 +365,19 @@ def process_maybe_execute_sell(self, trade: Trade) -> bool:
             if trade.open_order_id:
                 # Update trade with order values
                 logger.info('Found open order for %s', trade)
-                trade.update(exchange.get_order(trade.open_order_id, trade.pair))
+                order = exchange.get_order(trade.open_order_id, trade.pair)
+                # Try update amount (binance-fix)
+                try:
+                    new_amount = self.get_real_amount(trade, order)
+                    if order['amount'] != new_amount:
+                        order['amount'] = new_amount
+                        # Fee was applied, so set to 0
+                        trade.fee_open = 0
+
+                except OperationalException as exception:
+                    logger.warning("could not update trade amount: %s", exception)
+
+                trade.update(order)
 
             if trade.is_open and trade.open_order_id is None:
                 # Check if we can sell our current pair
@@ -372,6 +386,48 @@ def process_maybe_execute_sell(self, trade: Trade) -> bool:
             logger.warning('Unable to sell trade: %s', exception)
         return False
 
+    def get_real_amount(self, trade: Trade, order: Dict) -> float:
+        """
+        Get real amount for the trade
+        Necessary for exchanges which charge fees in base currency (e.g. binance)
+        """
+        order_amount = order['amount']
+        # Only run for closed orders
+        if trade.fee_open == 0 or order['status'] == 'open':
+            return order_amount
+
+        # use fee from order-dict if possible
+        if 'fee' in order and order['fee']:
+            if trade.pair.startswith(order['fee']['currency']):
+                new_amount = order_amount - order['fee']['cost']
+                logger.info("Applying fee on amount for %s (from %s to %s) from Order",
+                            trade, order['amount'], new_amount)
+                return new_amount
+
+        # Fallback to Trades
+        trades = exchange.get_trades_for_order(trade.open_order_id, trade.pair, trade.open_date)
+
+        if len(trades) == 0:
+            logger.info("Applying fee on amount for %s failed: myTrade-Dict empty found", trade)
+            return order_amount
+        amount = 0
+        fee_abs = 0
+        for exectrade in trades:
+            amount += exectrade['amount']
+            if "fee" in exectrade:
+                # only applies if fee is in quote currency!
+                if trade.pair.startswith(exectrade['fee']['currency']):
+                    fee_abs += exectrade['fee']['cost']
+
+        if amount != order_amount:
+            logger.warning("amount {} does not match amount {}".format(amount, trade.amount))
+            raise OperationalException("Half bought? Amounts don't match")
+        real_amount = amount - fee_abs
+        if fee_abs != 0:
+            logger.info("Applying fee on amount for {} (from {} to {}) from Trades".format(
+                        trade, order['amount'], real_amount))
+        return real_amount
+
     def handle_trade(self, trade: Trade) -> bool:
         """
         Sells the current pair if the threshold is reached and updates the trade record.

freqtrade/optimize/backtesting.py

@@ -113,12 +113,14 @@ def _get_sell_trade_entry(
 
         stake_amount = args['stake_amount']
         max_open_trades = args.get('max_open_trades', 0)
+        fee = exchange.get_fee()
         trade = Trade(
             open_rate=buy_row.close,
             open_date=buy_row.date,
             stake_amount=stake_amount,
             amount=stake_amount / buy_row.open,
-            fee=exchange.get_fee()
+            fee_open=fee,
+            fee_close=fee
         )
 
         # calculate win/lose forwards from buy point

freqtrade/persistence.py

@@ -85,7 +85,8 @@ class Trade(_DECL_BASE):
     exchange = Column(String, nullable=False)
     pair = Column(String, nullable=False)
     is_open = Column(Boolean, nullable=False, default=True)
-    fee = Column(Float, nullable=False, default=0.0)
+    fee_open = Column(Float, nullable=False, default=0.0)
+    fee_close = Column(Float, nullable=False, default=0.0)
     open_rate = Column(Float)
     close_rate = Column(Float)
     close_profit = Column(Float)
@@ -156,7 +157,7 @@ def calc_open_trade_price(
         getcontext().prec = 8
 
         buy_trade = (Decimal(self.amount) * Decimal(self.open_rate))
-        fees = buy_trade * Decimal(fee or self.fee)
+        fees = buy_trade * Decimal(fee or self.fee_open)
         return float(buy_trade + fees)
 
     def calc_close_trade_price(
@@ -177,7 +178,7 @@ def calc_close_trade_price(
             return 0.0
 
         sell_trade = (Decimal(self.amount) * Decimal(rate or self.close_rate))
-        fees = sell_trade * Decimal(fee or self.fee)
+        fees = sell_trade * Decimal(fee or self.fee_close)
         return float(sell_trade - fees)
 
     def calc_profit(
@@ -195,7 +196,7 @@ def calc_profit(
         open_trade_price = self.calc_open_trade_price()
         close_trade_price = self.calc_close_trade_price(
             rate=(rate or self.close_rate),
-            fee=(fee or self.fee)
+            fee=(fee or self.fee_close)
         )
         return float("{0:.8f}".format(close_trade_price - open_trade_price))
 
@@ -215,7 +216,7 @@ def calc_profit_percent(
         open_trade_price = self.calc_open_trade_price()
         close_trade_price = self.calc_close_trade_price(
             rate=(rate or self.close_rate),
-            fee=(fee or self.fee)
+            fee=(fee or self.fee_close)
         )
 
         return float("{0:.8f}".format((close_trade_price / open_trade_price) - 1))

freqtrade/tests/conftest.py

@@ -208,7 +208,7 @@ def markets_empty():
     return MagicMock(return_value=[])
 
 
-@pytest.fixture
+@pytest.fixture(scope='function')
 def limit_buy_order():
     return {
         'id': 'mocked_limit_buy',
@@ -499,3 +499,90 @@ def result():
 # that inserts a trade of some type and open-status
 # return the open-order-id
 # See tests in rpc/main that could use this
+
+
+@pytest.fixture(scope="function")
+def trades_for_order():
+    return [{'info': {'id': 34567,
+                      'orderId': 123456,
+                      'price': '0.24544100',
+                      'qty': '8.00000000',
+                      'commission': '0.00800000',
+                      'commissionAsset': 'LTC',
+                      'time': 1521663363189,
+                      'isBuyer': True,
+                      'isMaker': False,
+                      'isBestMatch': True},
+             'timestamp': 1521663363189,
+             'datetime': '2018-03-21T20:16:03.189Z',
+             'symbol': 'LTC/ETH',
+             'id': '34567',
+             'order': '123456',
+             'type': None,
+             'side': 'buy',
+             'price': 0.245441,
+             'cost': 1.963528,
+             'amount': 8.0,
+             'fee': {'cost': 0.008, 'currency': 'LTC'}}]
+
+
+@pytest.fixture(scope="function")
+def trades_for_order2():
+    return [{'info': {'id': 34567,
+                      'orderId': 123456,
+                      'price': '0.24544100',
+                      'qty': '8.00000000',
+                      'commission': '0.00800000',
+                      'commissionAsset': 'LTC',
+                      'time': 1521663363189,
+                      'isBuyer': True,
+                      'isMaker': False,
+                      'isBestMatch': True},
+             'timestamp': 1521663363189,
+             'datetime': '2018-03-21T20:16:03.189Z',
+             'symbol': 'LTC/ETH',
+             'id': '34567',
+             'order': '123456',
+             'type': None,
+             'side': 'buy',
+             'price': 0.245441,
+             'cost': 1.963528,
+             'amount': 4.0,
+             'fee': {'cost': 0.004, 'currency': 'LTC'}},
+            {'info': {'id': 34567,
+                      'orderId': 123456,
+                      'price': '0.24544100',
+                      'qty': '8.00000000',
+                      'commission': '0.00800000',
+                      'commissionAsset': 'LTC',
+                      'time': 1521663363189,
+                      'isBuyer': True,
+                      'isMaker': False,
+                      'isBestMatch': True},
+             'timestamp': 1521663363189,
+             'datetime': '2018-03-21T20:16:03.189Z',
+             'symbol': 'LTC/ETH',
+             'id': '34567',
+             'order': '123456',
+             'type': None,
+             'side': 'buy',
+             'price': 0.245441,
+             'cost': 1.963528,
+             'amount': 4.0,
+             'fee': {'cost': 0.004, 'currency': 'LTC'}}]
+
+
+@pytest.fixture
+def buy_order_fee():
+    return {
+        'id': 'mocked_limit_buy_old',
+        'type': 'limit',
+        'side': 'buy',
+        'pair': 'mocked',
+        'datetime': str(arrow.utcnow().shift(minutes=-601).datetime),
+        'price': 0.245441,
+        'amount': 8.0,
+        'remaining': 90.99181073,
+        'status': 'closed',
+        'fee': None
+    }

freqtrade/tests/exchange/test_exchange.py

@@ -4,14 +4,15 @@
 from copy import deepcopy
 from random import randint
 from unittest.mock import MagicMock, PropertyMock
-import ccxt
 
+import ccxt
 import pytest
 
-from freqtrade import OperationalException, DependencyException, TemporaryError
-from freqtrade.exchange import init, validate_pairs, buy, sell, get_balance, get_balances, \
-    get_ticker, get_ticker_history, cancel_order, get_name, get_fee, get_id, get_pair_detail_url
 import freqtrade.exchange as exchange
+from freqtrade import OperationalException, DependencyException, TemporaryError
+from freqtrade.exchange import (init, validate_pairs, buy, sell, get_balance, get_balances,
+                                get_ticker, get_ticker_history, cancel_order, get_name, get_fee,
+                                get_id, get_pair_detail_url, get_amount_lots)
 from freqtrade.tests.conftest import log_has
 
 API_INIT = False
@@ -507,3 +508,10 @@ def test_get_fee(default_conf, mocker):
     })
     mocker.patch('freqtrade.exchange._API', api_mock)
     assert get_fee() == 0.025
+
+
+def test_get_amount_lots(default_conf, mocker):
+    api_mock = MagicMock()
+    api_mock.amount_to_lots = MagicMock(return_value=1.0)
+    mocker.patch('freqtrade.exchange._API', api_mock)
+    assert get_amount_lots('LTC/BTC', 1.54) == 1