data_collector_update_for_web_interface_call

octobot_backtesting/api/data_file.py

@@ -13,12 +13,28 @@
 #
 #  You should have received a copy of the GNU Lesser General Public
 #  License along with this library.
+from os import path
 from octobot_backtesting.constants import BACKTESTING_FILE_PATH
-from octobot_backtesting.data import data_file_manager as data_manager
+from octobot_backtesting.data import data_file_manager as data_manager, DataBaseNotExists
+from octobot_backtesting.data.database import DataBase
 
 
-def get_file_description(file_name, data_path=BACKTESTING_FILE_PATH):
-    return data_manager.get_file_description(data_path, file_name)
+async def get_file_description(file_name, data_path=BACKTESTING_FILE_PATH):
+    database = None
+    try:
+        database = DataBase(path.join(data_path, file_name))
+        await database.initialize()
+        description = await data_manager.get_file_description(database)
+    except DataBaseNotExists as e:
+        print(e)
+        description = None
+    except TypeError as e:
+        print(e)
+        description = None
+    finally:
+        if database is not None:
+            await database.stop()
+    return description
 
 
 def get_all_available_data_files(data_path=BACKTESTING_FILE_PATH):

octobot_backtesting/api/exchange_data_collector.py

@@ -13,15 +13,15 @@
 #
 #  You should have received a copy of the GNU Lesser General Public
 #  License along with this library.
-from octobot_backtesting.collectors.exchanges.exchange_collector import ExchangeDataCollector
+from octobot_backtesting.collectors.exchanges.abstract_exchange_history_collector import \
+    AbstractExchangeHistoryCollector
 from octobot_commons.tentacles_management.advanced_manager import get_single_deepest_child_class
 
 
-async def collect_exchange_historical_data(config, exchange_name, symbols, time_frames=None):
-    collector_class = get_single_deepest_child_class(ExchangeDataCollector)
-    collector_instance = collector_class(config, exchange_name, symbols, time_frames)
-    if time_frames is None:
-        collector_instance.use_all_available_timeframes()
-    collector_instance.initialize()
+async def collect_exchange_historical_data(exchange_name, symbols, time_frames=None):
+    collector_class = get_single_deepest_child_class(AbstractExchangeHistoryCollector)
+    collector_instance = collector_class({}, exchange_name, symbols, time_frames,
+                                         use_all_available_timeframes=time_frames is None)
+    await collector_instance.initialize()
     await collector_instance.start()
-    return collector_instance.file_path
+    return collector_instance.file_name

octobot_backtesting/collectors/data_collector.pxd

@@ -34,3 +34,5 @@ cdef class DataCollector:
     cpdef void create_database(self)
     cpdef void create_aiohttp_session(self)
     cpdef void set_file_path(self)
+
+    cdef void _ensure_file_path(self)

octobot_backtesting/collectors/data_collector.py

@@ -16,33 +16,36 @@
 import asyncio
 import json
 import time
-from os.path import join
+from os.path import join, isdir
+from os import makedirs
 
 from aiohttp import ClientSession, ClientPayloadError
 
+from octobot_backtesting.enums import DataFormats
 from octobot_commons.logging.logging_util import get_logger
 
-from octobot_backtesting.constants import BACKTESTING_DATA_FILE_EXT, BACKTESTING_FILE_PATH, \
-    BACKTESTING_DATA_FILE_SEPARATOR
+from octobot_backtesting.constants import BACKTESTING_FILE_PATH, BACKTESTING_DATA_FILE_SEPARATOR
 from octobot_backtesting.data.database import DataBase
+from octobot_backtesting.data.data_file_manager import get_file_ending
 from octobot_backtesting.importers.data_importer import DataImporter
 
 
 class DataCollector:
     IMPORTER = DataImporter
 
-    def __init__(self, config, path=BACKTESTING_FILE_PATH):
+    def __init__(self, config, path=BACKTESTING_FILE_PATH, data_format=DataFormats.REGULAR_COLLECTOR_DATA):
         self.config = config
         self.path = path
         self.logger = get_logger(self.__class__.__name__)
 
         self.should_stop = False
         self.file_name = f"{self.__class__.__name__}{BACKTESTING_DATA_FILE_SEPARATOR}" \
-                         f"{time.time()}{BACKTESTING_DATA_FILE_EXT}"
+                         f"{time.time()}{get_file_ending(data_format)}"
 
         self.database = None
         self.aiohttp_session = None
         self.file_path = None
+        self._ensure_file_path()
         self.set_file_path()
 
     async def initialize(self) -> None:
@@ -54,6 +57,10 @@ async def stop(self) -> None:
     async def start(self) -> None:
         raise NotImplementedError("Start is not implemented")
 
+    def _ensure_file_path(self):
+        if not isdir(self.path):
+            makedirs(self.path)
+
     def set_file_path(self) -> None:
         self.file_path = join(self.path, self.file_name) if self.path else self.file_name
 

octobot_backtesting/collectors/exchanges/abstract_exchange_history_collector.pxd

@@ -0,0 +1,20 @@
+#  Drakkar-Software OctoBot
+#  Copyright (c) Drakkar-Software, All rights reserved.
+#
+#  This library is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 3.0 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library.
+from octobot_backtesting.collectors.exchanges.exchange_collector cimport ExchangeDataCollector
+
+
+cdef class AbstractExchangeHistoryCollector(ExchangeDataCollector):
+    pass

octobot_backtesting/collectors/exchanges/abstract_exchange_history_collector.py

@@ -0,0 +1,20 @@
+#  Drakkar-Software OctoBot
+#  Copyright (c) Drakkar-Software, All rights reserved.
+#
+#  This library is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 3.0 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library.
+from octobot_backtesting.collectors.exchanges.exchange_collector import ExchangeDataCollector
+
+
+class AbstractExchangeHistoryCollector(ExchangeDataCollector):
+    pass

octobot_backtesting/collectors/exchanges/abstract_exchange_live_collector.pxd

@@ -0,0 +1,20 @@
+#  Drakkar-Software OctoBot
+#  Copyright (c) Drakkar-Software, All rights reserved.
+#
+#  This library is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 3.0 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library.
+from octobot_backtesting.collectors.exchanges.exchange_collector cimport ExchangeDataCollector
+
+
+cdef class AbstractExchangeLiveCollector(ExchangeDataCollector):
+    pass

octobot_backtesting/collectors/exchanges/abstract_exchange_live_collector.py

@@ -0,0 +1,20 @@
+#  Drakkar-Software OctoBot
+#  Copyright (c) Drakkar-Software, All rights reserved.
+#
+#  This library is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 3.0 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library.
+from octobot_backtesting.collectors.exchanges.exchange_collector import ExchangeDataCollector
+
+
+class AbstractExchangeLiveCollector(ExchangeDataCollector):
+    pass

octobot_backtesting/collectors/exchanges/exchange_collector.pxd

@@ -23,3 +23,8 @@ cdef class ExchangeDataCollector(DataCollector):
 
     cdef public list symbols
     cdef public list time_frames
+
+    cdef bint use_all_available_timeframes
+
+    cdef void _load_timeframes_if_necessary(self)
+    cdef void _load_all_available_timeframes(self)

octobot_backtesting/collectors/exchanges/exchange_collector.py

@@ -22,7 +22,7 @@
 from octobot_commons.constants import CONFIG_TIME_FRAME, CONFIG_CRYPTO_CURRENCIES, CONFIG_CRYPTO_PAIRS
 
 from octobot_backtesting.collectors.data_collector import DataCollector
-from octobot_backtesting.enums import ExchangeDataTables, DataTables
+from octobot_backtesting.enums import ExchangeDataTables, DataTables, DataFormats
 from octobot_backtesting.importers.exchanges.exchange_importer import ExchangeDataImporter
 
 try:
@@ -35,13 +35,19 @@ class ExchangeDataCollector(DataCollector):
     VERSION = "1.0"
     IMPORTER = ExchangeDataImporter
 
-    def __init__(self, config, exchange_name, symbols, time_frames):
-        super().__init__(config)
+    def __init__(self, config, exchange_name, symbols, time_frames, use_all_available_timeframes=False,
+                 data_format=DataFormats.REGULAR_COLLECTOR_DATA):
+        super().__init__(config, data_format=data_format)
         self.exchange_name = exchange_name
         self.symbols = symbols if symbols else []
         self.time_frames = time_frames if time_frames else []
+        self.use_all_available_timeframes = use_all_available_timeframes
         self.set_file_path()
 
+    @abstractmethod
+    def _load_all_available_timeframes(self):
+        raise NotImplementedError("_load_all_available_timeframes is not implemented")
+
     async def initialize(self):
         self.create_database()
         await self.database.initialize()
@@ -51,18 +57,19 @@ async def initialize(self):
         self.config[CONFIG_EXCHANGES] = {self.exchange_name: {}}
         self.config[CONFIG_CRYPTO_CURRENCIES] = {"Symbols": {CONFIG_CRYPTO_PAIRS: self.symbols}}
 
-        # create description
+    def _load_timeframes_if_necessary(self):
+        if self.use_all_available_timeframes:
+            self._load_all_available_timeframes()
+        self.config[CONFIG_TIME_FRAME] = self.time_frames
+
+    async def _create_description(self):
         await self.database.insert(DataTables.DESCRIPTION,
                                    timestamp=time.time(),
                                    version=self.VERSION,
                                    exchange=self.exchange_name,
                                    symbols=json.dumps(self.symbols),
                                    time_frames=json.dumps([tf.value for tf in self.time_frames]))
 
-    @abstractmethod
-    def use_all_available_timeframes(self):
-        raise NotImplementedError("use_all_available_timeframes is not implemented")
-
     async def save_ticker(self, timestamp, exchange, symbol, ticker, multiple=False):
         if not multiple:
             await self.database.insert(ExchangeDataTables.TICKER, timestamp,

octobot_backtesting/data/data_file_manager.pxd

@@ -15,18 +15,9 @@
 #  You should have received a copy of the GNU Lesser General Public
 #  License along with this library.
 
-cpdef object interpret_file_name(str file_name)
-cpdef str build_file_name(str exchange, str symbol, str ending=*)
-cpdef void write_data_file(str file_name, dict content)
-cpdef dict read_data_file(str file_name)
 cpdef object get_data_type(str file_name)
 cpdef str get_file_ending(object data_type)
-cpdef dict get_time_frames(str file_path, dict content)
-cpdef dict get_ohlcv_per_timeframe(str file_path, dict content)
-cpdef int get_candles_count(str file_path, list tf_content)
-cpdef int get_number_of_candles(str file_path)
 cpdef float get_date(str time_info)
-cpdef dict get_file_description(str data_collector_path, str file_name)
 cpdef bint is_valid_ending(str ending)
 cpdef list get_all_available_data_files(str data_collector_path)
 cpdef object delete_data_file(str data_collector_path, str file_name)