Use handler for trades

freqtrade/data/converter.py

@@ -155,12 +155,12 @@ def order_book_to_dataframe(bids: list, asks: list) -> DataFrame:
     return frame
 
 
-def trades_to_ohlcv(trades: list, timeframe: str) -> list:
+def trades_to_ohlcv(trades: list, timeframe: str) -> DataFrame:
     """
     Converts trades list to ohlcv list
     :param trades: List of trades, as returned by ccxt.fetch_trades.
     :param timeframe: Ticker timeframe to resample data to
-    :return: ohlcv timeframe as list (as returned by ccxt.fetch_ohlcv)
+    :return: ohlcv Dataframe.
     """
     from freqtrade.exchange import timeframe_to_minutes
     ticker_minutes = timeframe_to_minutes(timeframe)
@@ -170,8 +170,7 @@ def trades_to_ohlcv(trades: list, timeframe: str) -> list:
 
     df_new = df['price'].resample(f'{ticker_minutes}min').ohlc()
     df_new['volume'] = df['amount'].resample(f'{ticker_minutes}min').sum()
-    df_new['date'] = df_new.index.astype("int64") // 10 ** 6
+    df_new['date'] = df_new.index
     # Drop 0 volume rows
     df_new = df_new.dropna()
-    columns = ["date", "open", "high", "low", "close", "volume"]
-    return list(zip(*[df_new[x].values.tolist() for x in columns]))
+    return df_new[['date', 'open', 'high', 'low', 'close', 'volume']]

freqtrade/data/history.py

@@ -18,7 +18,7 @@
 from freqtrade import OperationalException, misc
 from freqtrade.configuration import TimeRange
 from freqtrade.data.converter import trades_to_ohlcv
-from freqtrade.data.datahandlers import get_datahandler, get_datahandlerclass
+from freqtrade.data.datahandlers import get_datahandler
 from freqtrade.data.datahandlers.idatahandler import IDataHandler
 from freqtrade.exchange import Exchange, timeframe_to_minutes
 
@@ -90,15 +90,6 @@ def load_trades_file(datadir: Path, pair: str,
     return tradesdata
 
 
-def store_trades_file(datadir: Path, pair: str,
-                      data: list, is_zip: bool = True):
-    """
-    Stores tickerdata to file
-    """
-    filename = pair_trades_filename(datadir, pair)
-    misc.file_dump_json(filename, data, is_zip=is_zip)
-
-
 def load_pair_history(pair: str,
                       timeframe: str,
                       datadir: Path, *,
@@ -339,10 +330,11 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
     return pairs_not_available
 
 
-def _download_trades_history(datadir: Path,
-                             exchange: Exchange,
-                             pair: str,
-                             timerange: Optional[TimeRange] = None) -> bool:
+def _download_trades_history(exchange: Exchange,
+                             pair: str, *,
+                             timerange: Optional[TimeRange] = None,
+                             data_handler: IDataHandler
+                             ) -> bool:
     """
     Download trade history from the exchange.
     Appends to previously downloaded trades data.
@@ -351,7 +343,7 @@ def _download_trades_history(datadir: Path,
 
         since = timerange.startts * 1000 if timerange and timerange.starttype == 'date' else None
 
-        trades = load_trades_file(datadir, pair)
+        trades = data_handler.trades_load(pair)
 
         from_id = trades[-1]['id'] if trades else None
 
@@ -366,7 +358,7 @@ def _download_trades_history(datadir: Path,
                                                   from_id=from_id,
                                                   )
         trades.extend(new_trades[1])
-        store_trades_file(datadir, pair, trades)
+        data_handler.trades_store(pair, data=trades)
 
         logger.debug("New Start: %s", trades[0]['datetime'])
         logger.debug("New End: %s", trades[-1]['datetime'])
@@ -382,13 +374,15 @@ def _download_trades_history(datadir: Path,
 
 
 def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path,
-                                 timerange: TimeRange, erase=False) -> List[str]:
+                                 timerange: TimeRange, erase=False,
+                                 data_format: str = 'jsongz') -> List[str]:
     """
     Refresh stored trades data for backtesting and hyperopt operations.
     Used by freqtrade download-data subcommand.
     :return: List of pairs that are not available.
     """
     pairs_not_available = []
+    data_handler = get_datahandler(datadir, data_format=data_format)
     for pair in pairs:
         if pair not in exchange.markets:
             pairs_not_available.append(pair)
@@ -404,7 +398,8 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir:
         logger.info(f'Downloading trades for pair {pair}.')
         _download_trades_history(datadir=datadir, exchange=exchange,
                                  pair=pair,
-                                 timerange=timerange)
+                                 timerange=timerange,
+                                 data_handler=data_handler)
     return pairs_not_available
 
 
@@ -413,16 +408,19 @@ def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str],
     """
     Convert stored trades data to ohlcv data
     """
+    data_handler_trades = get_datahandler(datadir, data_format='jsongz')
+    data_handler_ohlcv = get_datahandler(datadir, data_format='json')
+
     for pair in pairs:
-        trades = load_trades_file(datadir, pair)
+        trades = data_handler_trades.trades_load(pair)
         for timeframe in timeframes:
             ohlcv_file = pair_data_filename(datadir, pair, timeframe)
             if erase and ohlcv_file.exists():
                 logger.info(f'Deleting existing data for pair {pair}, interval {timeframe}.')
                 ohlcv_file.unlink()
             ohlcv = trades_to_ohlcv(trades, timeframe)
             # Store ohlcv
-            store_tickerdata_file(datadir, pair, timeframe, data=ohlcv)
+            data_handler_ohlcv.ohlcv_store(pair, timeframe, data=ohlcv)
 
 
 def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]:

tests/data/test_history.py

@@ -10,6 +10,7 @@
 from pandas import DataFrame
 
 from freqtrade.configuration import TimeRange
+from freqtrade.data.datahandlers import get_datahandler
 from freqtrade.data.history import (_download_pair_history,
                                     _download_trades_history,
                                     _load_cached_data_for_updating,
@@ -597,12 +598,12 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad
                  ght_mock)
     exchange = get_patched_exchange(mocker, default_conf)
     file1 = testdatadir / 'ETH_BTC-trades.json.gz'
-
+    data_handler = get_datahandler(testdatadir, data_format='jsongz')
     _backup_file(file1)
 
     assert not file1.is_file()
 
-    assert _download_trades_history(datadir=testdatadir, exchange=exchange,
+    assert _download_trades_history(data_handler=data_handler, exchange=exchange,
                                     pair='ETH/BTC')
     assert log_has("New Amount of trades: 5", caplog)
     assert file1.is_file()
@@ -613,7 +614,7 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad
     mocker.patch('freqtrade.exchange.Exchange.get_historic_trades',
                  MagicMock(side_effect=ValueError))
 
-    assert not _download_trades_history(datadir=testdatadir, exchange=exchange,
+    assert not _download_trades_history(data_handler=data_handler, exchange=exchange,
                                         pair='ETH/BTC')
     assert log_has_re('Failed to download historic trades for pair: "ETH/BTC".*', caplog)