Merge branch 'freqtrade:develop' into develop

docs/backtesting.md

@@ -320,6 +320,9 @@ A backtesting result will look like that:
 | Avg. Duration Loser         | 6:55:00             |
 | Rejected Entry signals      | 3089                |
 | Entry/Exit Timeouts         | 0 / 0               |
+| Canceled Trade Entries      | 34                  |
+| Canceled Entry Orders       | 123                 |
+| Replaced Entry Orders       | 89                  |
 |                             |                     |
 | Min balance                 | 0.00945123 BTC      |
 | Max balance                 | 0.01846651 BTC      |
@@ -416,6 +419,9 @@ It contains some useful key metrics about performance of your strategy on backte
 | Avg. Duration Loser         | 6:55:00             |
 | Rejected Entry signals      | 3089                |
 | Entry/Exit Timeouts         | 0 / 0               |
+| Canceled Trade Entries      | 34                  |
+| Canceled Entry Orders       | 123                 |
+| Replaced Entry Orders       | 89                  |
 |                             |                     |
 | Min balance                 | 0.00945123 BTC      |
 | Max balance                 | 0.01846651 BTC      |
@@ -447,6 +453,9 @@ It contains some useful key metrics about performance of your strategy on backte
 - `Avg. Duration Winners` / `Avg. Duration Loser`: Average durations for winning and losing trades.
 - `Rejected Entry signals`: Trade entry signals that could not be acted upon due to `max_open_trades` being reached.
 - `Entry/Exit Timeouts`: Entry/exit orders which did not fill (only applicable if custom pricing is used).
+- `Canceled Trade Entries`: Number of trades that have been canceled by user request via `adjust_entry_price`.
+- `Canceled Entry Orders`: Number of entry orders that have been canceled by user request via `adjust_entry_price`.
+- `Replaced Entry Orders`: Number of entry orders that have been replaced by user request via `adjust_entry_price`.
 - `Min balance` / `Max balance`: Lowest and Highest Wallet balance during the backtest period.
 - `Max % of account underwater`: Maximum percentage your account has decreased from the top since the simulation started.
 Calculated as the maximum of `(Max Balance - Current Balance) / (Max Balance)`.

freqtrade/data/history/hdf5datahandler.py

@@ -40,7 +40,7 @@ def ohlcv_get_available_data(
         return [
             (
                 cls.rebuild_pair_from_filename(match[1]),
-                match[2],
+                cls.rebuild_timeframe_from_filename(match[2]),
                 CandleType.from_string(match[3])
             ) for match in _tmp if match and len(match.groups()) > 1]
 
@@ -109,7 +109,11 @@ def _ohlcv_load(self, pair: str, timeframe: str,
         )
 
         if not filename.exists():
-            return pd.DataFrame(columns=self._columns)
+            # Fallback mode for 1M files
+            filename = self._pair_data_filename(
+                self._datadir, pair, timeframe, candle_type=candle_type, no_timeframe_modify=True)
+            if not filename.exists():
+                return pd.DataFrame(columns=self._columns)
         where = []
         if timerange:
             if timerange.starttype == 'date':

freqtrade/data/history/idatahandler.py

@@ -26,7 +26,7 @@
 
 class IDataHandler(ABC):
 
-    _OHLCV_REGEX = r'^([a-zA-Z_-]+)\-(\d+\S)\-?([a-zA-Z_]*)?(?=\.)'
+    _OHLCV_REGEX = r'^([a-zA-Z_-]+)\-(\d+[a-zA-Z]{1,2})\-?([a-zA-Z_]*)?(?=\.)'
 
     def __init__(self, datadir: Path) -> None:
         self._datadir = datadir
@@ -193,10 +193,14 @@ def _pair_data_filename(
         datadir: Path,
         pair: str,
         timeframe: str,
-        candle_type: CandleType
+        candle_type: CandleType,
+        no_timeframe_modify: bool = False
     ) -> Path:
         pair_s = misc.pair_to_filename(pair)
         candle = ""
+        if not no_timeframe_modify:
+            timeframe = cls.timeframe_to_file(timeframe)
+
         if candle_type != CandleType.SPOT:
             datadir = datadir.joinpath('futures')
             candle = f"-{candle_type}"
@@ -210,6 +214,18 @@ def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path:
         filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}')
         return filename
 
+    @staticmethod
+    def timeframe_to_file(timeframe: str):
+        return timeframe.replace('M', 'Mo')
+
+    @staticmethod
+    def rebuild_timeframe_from_filename(timeframe: str) -> str:
+        """
+        converts timeframe from disk to file
+        Replaces mo with M (to avoid problems on case-insensitive filesystems)
+        """
+        return re.sub('1mo', '1M', timeframe, flags=re.IGNORECASE)
+
     @staticmethod
     def rebuild_pair_from_filename(pair: str) -> str:
         """

freqtrade/data/history/jsondatahandler.py

@@ -41,7 +41,7 @@ def ohlcv_get_available_data(
         return [
             (
                 cls.rebuild_pair_from_filename(match[1]),
-                match[2],
+                cls.rebuild_timeframe_from_filename(match[2]),
                 CandleType.from_string(match[3])
             ) for match in _tmp if match and len(match.groups()) > 1]
 
@@ -103,9 +103,14 @@ def _ohlcv_load(self, pair: str, timeframe: str,
         :param candle_type: Any of the enum CandleType (must match trading mode!)
         :return: DataFrame with ohlcv data, or empty DataFrame
         """
-        filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type=candle_type)
+        filename = self._pair_data_filename(
+            self._datadir, pair, timeframe, candle_type=candle_type)
         if not filename.exists():
-            return DataFrame(columns=self._columns)
+            # Fallback mode for 1M files
+            filename = self._pair_data_filename(
+                self._datadir, pair, timeframe, candle_type=candle_type, no_timeframe_modify=True)
+            if not filename.exists():
+                return DataFrame(columns=self._columns)
         try:
             pairdata = read_json(filename, orient='values')
             pairdata.columns = self._columns

freqtrade/exchange/exchange.py

@@ -16,8 +16,7 @@
 import ccxt
 import ccxt.async_support as ccxt_async
 from cachetools import TTLCache
-from ccxt.base.decimal_to_precision import (ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE,
-                                            decimal_to_precision)
+from ccxt import ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE, Precise, decimal_to_precision
 from pandas import DataFrame
 
 from freqtrade.constants import (DEFAULT_AMOUNT_RESERVE_PERCENT, NON_OPEN_EXCHANGE_STATES, BuySell,
@@ -704,10 +703,11 @@ def price_to_precision(self, pair: str, price: float) -> float:
             #                                    counting_mode=self.precisionMode,
             #                                    ))
             if self.precisionMode == TICK_SIZE:
-                precision = self.markets[pair]['precision']['price']
-                missing = price % precision
-                if missing != 0:
-                    price = round(price - missing + precision, 10)
+                precision = Precise(str(self.markets[pair]['precision']['price']))
+                price_str = Precise(str(price))
+                missing = price_str % precision
+                if not missing == Precise("0"):
+                    price = round(float(str(price_str - missing + precision)), 14)
             else:
                 symbol_prec = self.markets[pair]['precision']['price']
                 big_price = price * pow(10, symbol_prec)
@@ -1457,6 +1457,23 @@ def fetch_l2_order_book(self, pair: str, limit: int = 100) -> dict:
         except ccxt.BaseError as e:
             raise OperationalException(e) from e
 
+    def _get_price_side(self, side: str, is_short: bool, conf_strategy: Dict) -> str:
+        price_side = conf_strategy['price_side']
+
+        if price_side in ('same', 'other'):
+            price_map = {
+                ('entry', 'long', 'same'): 'bid',
+                ('entry', 'long', 'other'): 'ask',
+                ('entry', 'short', 'same'): 'ask',
+                ('entry', 'short', 'other'): 'bid',
+                ('exit', 'long', 'same'): 'ask',
+                ('exit', 'long', 'other'): 'bid',
+                ('exit', 'short', 'same'): 'bid',
+                ('exit', 'short', 'other'): 'ask',
+            }
+            price_side = price_map[(side, 'short' if is_short else 'long', price_side)]
+        return price_side
+
     def get_rate(self, pair: str, refresh: bool,
                  side: EntryExit, is_short: bool) -> float:
         """
@@ -1483,20 +1500,7 @@ def get_rate(self, pair: str, refresh: bool,
 
         conf_strategy = self._config.get(strat_name, {})
 
-        price_side = conf_strategy['price_side']
-
-        if price_side in ('same', 'other'):
-            price_map = {
-                ('entry', 'long', 'same'): 'bid',
-                ('entry', 'long', 'other'): 'ask',
-                ('entry', 'short', 'same'): 'ask',
-                ('entry', 'short', 'other'): 'bid',
-                ('exit', 'long', 'same'): 'ask',
-                ('exit', 'long', 'other'): 'bid',
-                ('exit', 'short', 'same'): 'bid',
-                ('exit', 'short', 'other'): 'ask',
-            }
-            price_side = price_map[(side, 'short' if is_short else 'long', price_side)]
+        price_side = self._get_price_side(side, is_short, conf_strategy)
 
         price_side_word = price_side.capitalize()
 

freqtrade/optimize/backtesting.py

@@ -297,6 +297,9 @@ def prepare_backtest(self, enable_protections):
         self.rejected_trades = 0
         self.timedout_entry_orders = 0
         self.timedout_exit_orders = 0
+        self.canceled_trade_entries = 0
+        self.canceled_entry_orders = 0
+        self.replaced_entry_orders = 0
         self.dataprovider.clear_cache()
         if enable_protections:
             self._load_protections(self.strategy)
@@ -539,11 +542,11 @@ def _get_exit_trade_entry_for_candle(self, trade: LocalTrade,
 
             trade_dur = int((trade.close_date_utc - trade.open_date_utc).total_seconds() // 60)
             try:
-                closerate = self._get_close_rate(row, trade, exit_, trade_dur)
+                close_rate = self._get_close_rate(row, trade, exit_, trade_dur)
             except ValueError:
                 return None
-            # call the custom exit price,with default value as previous closerate
-            current_profit = trade.calc_profit_ratio(closerate)
+            # call the custom exit price,with default value as previous close_rate
+            current_profit = trade.calc_profit_ratio(close_rate)
             order_type = self.strategy.order_types['exit']
             if exit_.exit_type in (ExitType.EXIT_SIGNAL, ExitType.CUSTOM_EXIT):
                 # Checks and adds an exit tag, after checking that the length of the
@@ -557,24 +560,24 @@ def _get_exit_trade_entry_for_candle(self, trade: LocalTrade,
                     exit_reason = row[EXIT_TAG_IDX]
                 # Custom exit pricing only for exit-signals
                 if order_type == 'limit':
-                    closerate = strategy_safe_wrapper(self.strategy.custom_exit_price,
-                                                      default_retval=closerate)(
+                    close_rate = strategy_safe_wrapper(self.strategy.custom_exit_price,
+                                                       default_retval=close_rate)(
                         pair=trade.pair, trade=trade,
                         current_time=exit_candle_time,
-                        proposed_rate=closerate, current_profit=current_profit,
+                        proposed_rate=close_rate, current_profit=current_profit,
                         exit_tag=exit_reason)
                     # We can't place orders lower than current low.
                     # freqtrade does not support this in live, and the order would fill immediately
                     if trade.is_short:
-                        closerate = min(closerate, row[HIGH_IDX])
+                        close_rate = min(close_rate, row[HIGH_IDX])
                     else:
-                        closerate = max(closerate, row[LOW_IDX])
+                        close_rate = max(close_rate, row[LOW_IDX])
             # Confirm trade exit:
             time_in_force = self.strategy.order_time_in_force['exit']
 
             if not strategy_safe_wrapper(self.strategy.confirm_trade_exit, default_retval=True)(
                     pair=trade.pair, trade=trade, order_type='limit', amount=trade.amount,
-                    rate=closerate,
+                    rate=close_rate,
                     time_in_force=time_in_force,
                     sell_reason=exit_reason,  # deprecated
                     exit_reason=exit_reason,
@@ -597,12 +600,12 @@ def _get_exit_trade_entry_for_candle(self, trade: LocalTrade,
                 side=trade.exit_side,
                 order_type=order_type,
                 status="open",
-                price=closerate,
-                average=closerate,
+                price=close_rate,
+                average=close_rate,
                 amount=trade.amount,
                 filled=0,
                 remaining=trade.amount,
-                cost=trade.amount * closerate,
+                cost=trade.amount * close_rate,
             )
             trade.orders.append(order)
             return trade
@@ -884,6 +887,7 @@ def manage_open_orders(self, trade: LocalTrade, current_time, row: Tuple) -> boo
                 return True
             elif self.check_order_replace(trade, order, current_time, row):
                 # delete trade due to user request
+                self.canceled_trade_entries += 1
                 return True
         # default maintain trade
         return False
@@ -933,13 +937,15 @@ def check_order_replace(self, trade: LocalTrade, order: Order, current_time,
                 return False
             else:
                 del trade.orders[trade.orders.index(order)]
+                self.canceled_entry_orders += 1
 
             # place new order if result was not None
             if requested_rate:
                 self._enter_trade(pair=trade.pair, row=row, trade=trade,
                                   requested_rate=requested_rate,
                                   requested_stake=(order.remaining * order.price),
                                   direction='short' if trade.is_short else 'long')
+                self.replaced_entry_orders += 1
             else:
                 # assumption: there can't be multiple open entry orders at any given time
                 return (trade.nr_of_successful_entries == 0)
@@ -1087,6 +1093,9 @@ def backtest(self, processed: Dict,
             'rejected_signals': self.rejected_trades,
             'timedout_entry_orders': self.timedout_entry_orders,
             'timedout_exit_orders': self.timedout_exit_orders,
+            'canceled_trade_entries': self.canceled_trade_entries,
+            'canceled_entry_orders': self.canceled_entry_orders,
+            'replaced_entry_orders': self.replaced_entry_orders,
             'final_balance': self.wallets.get_total(self.strategy.config['stake_currency']),
         }
 

freqtrade/optimize/optimize_reports.py

@@ -468,6 +468,9 @@ def generate_strategy_stats(pairlist: List[str],
         'rejected_signals': content['rejected_signals'],
         'timedout_entry_orders': content['timedout_entry_orders'],
         'timedout_exit_orders': content['timedout_exit_orders'],
+        'canceled_trade_entries': content['canceled_trade_entries'],
+        'canceled_entry_orders': content['canceled_entry_orders'],
+        'replaced_entry_orders': content['replaced_entry_orders'],
         'max_open_trades': max_open_trades,
         'max_open_trades_setting': (config['max_open_trades']
                                     if config['max_open_trades'] != float('inf') else -1),
@@ -753,6 +756,12 @@ def text_table_add_metrics(strat_results: Dict) -> str:
             ('Drawdown End', strat_results['drawdown_end']),
         ])
 
+        entry_adjustment_metrics = [
+            ('Canceled Trade Entries', strat_results.get('canceled_trade_entries', 'N/A')),
+            ('Canceled Entry Orders', strat_results.get('canceled_entry_orders', 'N/A')),
+            ('Replaced Entry Orders', strat_results.get('replaced_entry_orders', 'N/A')),
+        ] if strat_results.get('canceled_entry_orders', 0) > 0 else []
+
         # Newly added fields should be ignored if they are missing in strat_results. hyperopt-show
         # command stores these results and newer version of freqtrade must be able to handle old
         # results with missing new fields.
@@ -801,6 +810,7 @@ def text_table_add_metrics(strat_results: Dict) -> str:
             ('Entry/Exit Timeouts',
              f"{strat_results.get('timedout_entry_orders', 'N/A')} / "
              f"{strat_results.get('timedout_exit_orders', 'N/A')}"),
+            *entry_adjustment_metrics,
             ('', ''),  # Empty line to improve readability
 
             ('Min balance', round_coin_value(strat_results['csum_min'],

freqtrade/plugins/pairlist/SpreadFilter.py

@@ -50,7 +50,7 @@ def _validate_pair(self, pair: str, ticker: Dict[str, Any]) -> bool:
         :param ticker: ticker dict as returned from ccxt.fetch_tickers()
         :return: True if the pair can stay, false if it should be removed
         """
-        if 'bid' in ticker and 'ask' in ticker and ticker['ask']:
+        if 'bid' in ticker and 'ask' in ticker and ticker['ask'] and ticker['bid']:
             spread = 1 - ticker['bid'] / ticker['ask']
             if spread > self._max_spread_ratio:
                 self.log_once(f"Removed {pair} from whitelist, because spread "

requirements-plot.txt

@@ -1,4 +1,4 @@
 # Include all requirements to run the bot.
 -r requirements.txt
 
-plotly==5.7.0
+plotly==5.8.0

setup.py

@@ -42,7 +42,7 @@
     ],
     install_requires=[
         # from requirements.txt
-        'ccxt>=1.79.69',
+        'ccxt>=1.80.67',
         'SQLAlchemy',
         'python-telegram-bot>=13.4',
         'arrow>=0.17.0',

tests/data/test_history.py

@@ -158,29 +158,30 @@ def test_testdata_path(testdatadir) -> None:
     assert str(Path('tests') / 'testdata') in str(testdatadir)
 
 
-@pytest.mark.parametrize("pair,expected_result,candle_type", [
-    ("ETH/BTC", 'freqtrade/hello/world/ETH_BTC-5m.json', ""),
-    ("Fabric Token/ETH", 'freqtrade/hello/world/Fabric_Token_ETH-5m.json', ""),
-    ("ETHH20", 'freqtrade/hello/world/ETHH20-5m.json', ""),
-    (".XBTBON2H", 'freqtrade/hello/world/_XBTBON2H-5m.json', ""),
-    ("ETHUSD.d", 'freqtrade/hello/world/ETHUSD_d-5m.json', ""),
-    ("ACC_OLD/BTC", 'freqtrade/hello/world/ACC_OLD_BTC-5m.json', ""),
-    ("ETH/BTC", 'freqtrade/hello/world/futures/ETH_BTC-5m-mark.json', "mark"),
-    ("ACC_OLD/BTC", 'freqtrade/hello/world/futures/ACC_OLD_BTC-5m-index.json', "index"),
+@pytest.mark.parametrize("pair,timeframe,expected_result,candle_type", [
+    ("ETH/BTC", "5m", "freqtrade/hello/world/ETH_BTC-5m.json", ""),
+    ("ETH/USDT", "1M", "freqtrade/hello/world/ETH_USDT-1Mo.json", ""),
+    ("Fabric Token/ETH", "5m", "freqtrade/hello/world/Fabric_Token_ETH-5m.json", ""),
+    ("ETHH20", "5m", "freqtrade/hello/world/ETHH20-5m.json", ""),
+    (".XBTBON2H", "5m", "freqtrade/hello/world/_XBTBON2H-5m.json", ""),
+    ("ETHUSD.d", "5m", "freqtrade/hello/world/ETHUSD_d-5m.json", ""),
+    ("ACC_OLD/BTC", "5m", "freqtrade/hello/world/ACC_OLD_BTC-5m.json", ""),
+    ("ETH/BTC", "5m", "freqtrade/hello/world/futures/ETH_BTC-5m-mark.json", "mark"),
+    ("ACC_OLD/BTC", "5m", "freqtrade/hello/world/futures/ACC_OLD_BTC-5m-index.json", "index"),
 ])
-def test_json_pair_data_filename(pair, expected_result, candle_type):
+def test_json_pair_data_filename(pair, timeframe, expected_result, candle_type):
     fn = JsonDataHandler._pair_data_filename(
         Path('freqtrade/hello/world'),
         pair,
-        '5m',
+        timeframe,
         CandleType.from_string(candle_type)
     )
     assert isinstance(fn, Path)
     assert fn == Path(expected_result)
     fn = JsonGzDataHandler._pair_data_filename(
         Path('freqtrade/hello/world'),
         pair,
-        '5m',
+        timeframe,
         candle_type=CandleType.from_string(candle_type)
     )
     assert isinstance(fn, Path)