From 1b6a60ecb21bdbdf6f4c32781e4c5cbfa370c995 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Wed, 15 Nov 2017 20:06:37 +0200 Subject: [PATCH 1/8] refactor backtesting to avoid recalculating indicators in hyperopt --- freqtrade/tests/test_backtesting.py | 22 +++++++++++++++------- freqtrade/tests/test_hyperopt.py | 7 ++++--- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/freqtrade/tests/test_backtesting.py b/freqtrade/tests/test_backtesting.py index 6a8ecacb0f4..8e518425ab0 100644 --- a/freqtrade/tests/test_backtesting.py +++ b/freqtrade/tests/test_backtesting.py @@ -1,4 +1,5 @@ # pragma pylint: disable=missing-docstring +from typing import Dict import logging import os @@ -7,7 +8,8 @@ from pandas import DataFrame from freqtrade import exchange -from freqtrade.analyze import analyze_ticker +from freqtrade.analyze import parse_ticker_dataframe, populate_indicators, \ + populate_buy_trend, populate_sell_trend from freqtrade.exchange import Bittrex from freqtrade.main import min_roi_reached from freqtrade.persistence import Trade @@ -25,15 +27,21 @@ def print_pair_results(pair, results): print(format_results(results[results.currency == pair])) -def backtest(backtest_conf, backdata, mocker): +def preprocess(backdata) -> Dict[str, DataFrame]: + processed = {} + for pair, pair_data in backdata.items(): + processed[pair] = populate_indicators(parse_ticker_dataframe(pair_data)) + return processed + + +def backtest(backtest_conf, processed, mocker): trades = [] exchange._API = Bittrex({'key': '', 'secret': ''}) - mocked_history = mocker.patch('freqtrade.analyze.get_ticker_history') mocker.patch.dict('freqtrade.main._CONF', backtest_conf) mocker.patch('arrow.utcnow', return_value=arrow.get('2017-08-20T14:50:00')) - for pair, pair_data in backdata.items(): - mocked_history.return_value = pair_data - ticker = analyze_ticker(pair)[['close', 'date', 'buy', 'sell']].copy() + for pair, pair_data in processed.items(): + ticker = populate_sell_trend(populate_buy_trend(pair_data))[['close', 'date', 'buy', 'sell']].copy() + # for each buy point for row in ticker[ticker.buy == 1].itertuples(index=True): trade = Trade( @@ -56,7 +64,7 @@ def backtest(backtest_conf, backdata, mocker): @pytest.mark.skipif(not os.environ.get('BACKTEST', False), reason="BACKTEST not set") def test_backtest(backtest_conf, backdata, mocker, report=True): - results = backtest(backtest_conf, backdata, mocker) + results = backtest(backtest_conf, preprocess(backdata), mocker) print('====================== BACKTESTING REPORT ================================') for pair in backdata: diff --git a/freqtrade/tests/test_hyperopt.py b/freqtrade/tests/test_hyperopt.py index bece2edae71..2ca7d868a70 100644 --- a/freqtrade/tests/test_hyperopt.py +++ b/freqtrade/tests/test_hyperopt.py @@ -9,7 +9,7 @@ from hyperopt import fmin, tpe, hp, Trials, STATUS_OK from pandas import DataFrame -from freqtrade.tests.test_backtesting import backtest, format_results +from freqtrade.tests.test_backtesting import backtest, format_results, preprocess from freqtrade.vendor.qtpylib.indicators import crossed_above logging.disable(logging.DEBUG) # disable debug logs that slow backtesting a lot @@ -67,12 +67,13 @@ def populate_buy_trend(dataframe: DataFrame) -> DataFrame: @pytest.mark.skipif(not os.environ.get('BACKTEST', False), reason="BACKTEST not set") def test_hyperopt(backtest_conf, backdata, mocker): - mocked_buy_trend = mocker.patch('freqtrade.analyze.populate_buy_trend') + mocked_buy_trend = mocker.patch('freqtrade.tests.test_backtesting.populate_buy_trend') + processed = preprocess(backdata) def optimizer(params): mocked_buy_trend.side_effect = buy_strategy_generator(params) - results = backtest(backtest_conf, backdata, mocker) + results = backtest(backtest_conf, processed, mocker) result = format_results(results) From 174122a09b544dca6967294d2b9201c3d2a81c7e Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Wed, 15 Nov 2017 20:07:57 +0200 Subject: [PATCH 2/8] remove unnecessary calculation --- freqtrade/tests/test_hyperopt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/tests/test_hyperopt.py b/freqtrade/tests/test_hyperopt.py index 2ca7d868a70..74344b8f74b 100644 --- a/freqtrade/tests/test_hyperopt.py +++ b/freqtrade/tests/test_hyperopt.py @@ -59,7 +59,6 @@ def populate_buy_trend(dataframe: DataFrame) -> DataFrame: dataframe.loc[ reduce(lambda x, y: x & y, conditions), 'buy'] = 1 - dataframe.loc[dataframe['buy'] == 1, 'buy_price'] = dataframe['close'] return dataframe return populate_buy_trend From 5d1f87404139c4e8ac62c7b7a7f972080fa451cc Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Thu, 16 Nov 2017 20:16:54 +0200 Subject: [PATCH 3/8] switch ix to loc, ix is apparently deprecated --- freqtrade/analyze.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index b1279071e5b..7008c69bfee 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -66,14 +66,14 @@ def populate_buy_trend(dataframe: DataFrame) -> DataFrame: :param dataframe: DataFrame :return: DataFrame with buy column """ - dataframe.ix[ + dataframe.loc[ (dataframe['close'] < dataframe['sma']) & (dataframe['tema'] <= dataframe['blower']) & (dataframe['mfi'] < 25) & (dataframe['fastd'] < 25) & (dataframe['adx'] > 30), 'buy'] = 1 - dataframe.ix[dataframe['buy'] == 1, 'buy_price'] = dataframe['close'] + dataframe.loc[dataframe['buy'] == 1, 'buy_price'] = dataframe['close'] return dataframe @@ -83,10 +83,10 @@ def populate_sell_trend(dataframe: DataFrame) -> DataFrame: :param dataframe: DataFrame :return: DataFrame with buy column """ - dataframe.ix[ + dataframe.loc[ (crossed_above(dataframe['rsi'], 70)), 'sell'] = 1 - dataframe.ix[dataframe['sell'] == 1, 'sell_price'] = dataframe['close'] + dataframe.loc[dataframe['sell'] == 1, 'sell_price'] = dataframe['close'] return dataframe From 27a6b29c80b093689d433895b5599c67a0b1abee Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Thu, 16 Nov 2017 20:17:23 +0200 Subject: [PATCH 4/8] move time diff calculation out of a loop --- freqtrade/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index 316e0c960f8..8508bb878f1 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -163,9 +163,9 @@ def min_roi_reached(trade: Trade, current_rate: float, current_time: datetime) - logger.debug('Stop loss hit.') return True + # Check if time matches and current rate is above threshold + time_diff = (current_time - trade.open_date).total_seconds() / 60 for duration, threshold in sorted(_CONF['minimal_roi'].items()): - # Check if time matches and current rate is above threshold - time_diff = (current_time - trade.open_date).total_seconds() / 60 if time_diff > float(duration) and current_profit > threshold: return True From 16d412323c119684294e93a3ad6d6d2b17ec6288 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Thu, 16 Nov 2017 20:21:59 +0200 Subject: [PATCH 5/8] add a little snippet to allow running line_profiler with hyperopt --- freqtrade/tests/test_hyperopt.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/freqtrade/tests/test_hyperopt.py b/freqtrade/tests/test_hyperopt.py index 74344b8f74b..8f754cb179b 100644 --- a/freqtrade/tests/test_hyperopt.py +++ b/freqtrade/tests/test_hyperopt.py @@ -146,3 +146,8 @@ def optimizer(params): print('Best parameters {}'.format(best)) newlist = sorted(trials.results, key=itemgetter('loss')) print('Result: {}'.format(newlist[0]['result'])) + + +if __name__ == '__main__': + # for profiling with cProfile and line_profiler + pytest.main([__file__, '-s']) From 2a56031cdc5ac98476457a5bfc36ed58d399b755 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 17 Nov 2017 08:02:06 +0200 Subject: [PATCH 6/8] remove unnecessary line --- freqtrade/tests/test_backtesting.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/tests/test_backtesting.py b/freqtrade/tests/test_backtesting.py index 8e518425ab0..b03bdcd0eca 100644 --- a/freqtrade/tests/test_backtesting.py +++ b/freqtrade/tests/test_backtesting.py @@ -58,8 +58,7 @@ def backtest(backtest_conf, processed, mocker): trades.append((pair, current_profit, row2.Index - row.Index)) break labels = ['currency', 'profit', 'duration'] - results = DataFrame.from_records(trades, columns=labels) - return results + return DataFrame.from_records(trades, columns=labels) @pytest.mark.skipif(not os.environ.get('BACKTEST', False), reason="BACKTEST not set") From 632d00e01d49c364d8b6beb109615a89e86f14d2 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 17 Nov 2017 12:08:23 +0200 Subject: [PATCH 7/8] move price point calculations out from populate functions --- freqtrade/analyze.py | 5 +++-- freqtrade/tests/test_analyze.py | 8 ++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 7008c69bfee..347d127adb8 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -73,7 +73,6 @@ def populate_buy_trend(dataframe: DataFrame) -> DataFrame: (dataframe['fastd'] < 25) & (dataframe['adx'] > 30), 'buy'] = 1 - dataframe.loc[dataframe['buy'] == 1, 'buy_price'] = dataframe['close'] return dataframe @@ -86,7 +85,6 @@ def populate_sell_trend(dataframe: DataFrame) -> DataFrame: dataframe.loc[ (crossed_above(dataframe['rsi'], 70)), 'sell'] = 1 - dataframe.loc[dataframe['sell'] == 1, 'sell_price'] = dataframe['close'] return dataframe @@ -106,6 +104,9 @@ def analyze_ticker(pair: str) -> DataFrame: dataframe = populate_indicators(dataframe) dataframe = populate_buy_trend(dataframe) dataframe = populate_sell_trend(dataframe) + # TODO: buy_price and sell_price are only used by the plotter, should probably be moved there + dataframe.loc[dataframe['buy'] == 1, 'buy_price'] = dataframe['close'] + dataframe.loc[dataframe['sell'] == 1, 'sell_price'] = dataframe['close'] return dataframe diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index 7b8c0749061..8b789c08966 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -5,7 +5,7 @@ from pandas import DataFrame from freqtrade.analyze import parse_ticker_dataframe, populate_buy_trend, populate_indicators, \ - get_signal, SignalType + get_signal, SignalType, populate_sell_trend @pytest.fixture @@ -26,7 +26,11 @@ def test_dataframe_correct_length(result): def test_populates_buy_trend(result): dataframe = populate_buy_trend(populate_indicators(result)) assert 'buy' in dataframe.columns - assert 'buy_price' in dataframe.columns + + +def test_populates_buy_trend(result): + dataframe = populate_sell_trend(populate_indicators(result)) + assert 'sell' in dataframe.columns def test_returns_latest_buy_signal(mocker): From d89db50465e237d9575d8d218bab055584d9b804 Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Fri, 17 Nov 2017 12:27:33 +0200 Subject: [PATCH 8/8] avoid copy operation due to memory consumption --- freqtrade/tests/test_backtesting.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/test_backtesting.py b/freqtrade/tests/test_backtesting.py index b03bdcd0eca..f13d4241857 100644 --- a/freqtrade/tests/test_backtesting.py +++ b/freqtrade/tests/test_backtesting.py @@ -40,8 +40,9 @@ def backtest(backtest_conf, processed, mocker): mocker.patch.dict('freqtrade.main._CONF', backtest_conf) mocker.patch('arrow.utcnow', return_value=arrow.get('2017-08-20T14:50:00')) for pair, pair_data in processed.items(): - ticker = populate_sell_trend(populate_buy_trend(pair_data))[['close', 'date', 'buy', 'sell']].copy() - + pair_data['buy'] = 0 + pair_data['sell'] = 0 + ticker = populate_sell_trend(populate_buy_trend(pair_data)) # for each buy point for row in ticker[ticker.buy == 1].itertuples(index=True): trade = Trade(