Merge pull request #885 from ricequant/develop

Develop

messages.pot

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PROJECT VERSION\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2024-04-24 13:56+0800\n"
+"POT-Creation-Date: 2024-05-15 14:39+0800\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -73,8 +73,8 @@ msgstr ""
 #: rqalpha/apis/api_base.py:173
 #: rqalpha/mod/rqalpha_mod_sys_accounts/api/api_future.py:66
 #: rqalpha/mod/rqalpha_mod_sys_accounts/api/api_stock.py:103
-#: rqalpha/mod/rqalpha_mod_sys_accounts/api/api_stock.py:140
-#: rqalpha/mod/rqalpha_mod_sys_accounts/api/api_stock.py:343
+#: rqalpha/mod/rqalpha_mod_sys_accounts/api/api_stock.py:141
+#: rqalpha/mod/rqalpha_mod_sys_accounts/api/api_stock.py:344
 msgid "Order Creation Failed: [{order_book_id}] No market data"
 msgstr ""
 
@@ -247,41 +247,16 @@ msgstr ""
 msgid "deprecated parameter[bar_dict] in before_trading function."
 msgstr ""
 
-#: rqalpha/data/bundle.py:460
-msgid ""
-"RQAlpha already supports backtesting using futures historical margins and"
-" rates, please upgrade RQDatac to version 2.11.12 and above to use it"
-msgstr ""
-
-#: rqalpha/data/bundle.py:470 rqalpha/data/bundle.py:531
-msgid ""
-"Futures historical trading parameters data is being updated, please "
-"wait......"
-msgstr ""
-
-#: rqalpha/data/bundle.py:522 rqalpha/data/bundle.py:706
+#: rqalpha/data/bundle.py:522
 msgid ""
 "File {} update failed, if it is using, please update later, or you can "
 "delete then update again"
 msgstr ""
 
-#: rqalpha/data/base_data_source/data_source.py:140
-msgid ""
-"RQDatac is not installed, "
-"\"config.base.futures_time_series_trading_parameters\" will be disabled."
-msgstr ""
-
-#: rqalpha/data/base_data_source/data_source.py:145
-msgid ""
-"RQDatac does not have permission to obtain futures histrical trading "
-"parameters, \"config.base.futures_time_series_trading_parameters\" will "
-"be disabled."
-msgstr ""
-
 #: rqalpha/data/base_data_source/storages.py:81
 msgid ""
-"Your bundle data is too old, please use 'rqalpha update-bundle' or "
-"'rqalpha download-bundle' to update it to lastest before using"
+"The bundle data you are using is too old, please update it to lastest "
+"before using"
 msgstr ""
 
 #: rqalpha/data/base_data_source/storages.py:97
@@ -337,8 +312,8 @@ msgid ""
 msgstr ""
 
 #: rqalpha/mod/rqalpha_mod_sys_accounts/api/api_future.py:50
-#: rqalpha/mod/rqalpha_mod_sys_accounts/api/api_stock.py:112
-#: rqalpha/mod/rqalpha_mod_sys_accounts/api/api_stock.py:157
+#: rqalpha/mod/rqalpha_mod_sys_accounts/api/api_stock.py:113
+#: rqalpha/mod/rqalpha_mod_sys_accounts/api/api_stock.py:158
 msgid "Order Creation Failed: 0 order quantity, order_book_id={order_book_id}"
 msgstr ""
 
@@ -371,39 +346,39 @@ msgid ""
 "again!"
 msgstr ""
 
-#: rqalpha/mod/rqalpha_mod_sys_accounts/api/api_stock.py:119
+#: rqalpha/mod/rqalpha_mod_sys_accounts/api/api_stock.py:120
 msgid "insufficient cash, use all remaining cash({}) to create order"
 msgstr ""
 
-#: rqalpha/mod/rqalpha_mod_sys_accounts/api/api_stock.py:332
+#: rqalpha/mod/rqalpha_mod_sys_accounts/api/api_stock.py:333
 msgid ""
 "function order_target_portfolio: invalid keys of target_portfolio, "
 "expected order_book_ids or Instrument objects, got {} (type: {})"
 msgstr ""
 
-#: rqalpha/mod/rqalpha_mod_sys_accounts/api/api_stock.py:337
+#: rqalpha/mod/rqalpha_mod_sys_accounts/api/api_stock.py:338
 msgid ""
 "function order_target_portfolio: invalid instrument type, excepted "
 "CS/ETF/LOF/INDX, got {}"
 msgstr ""
 
-#: rqalpha/mod/rqalpha_mod_sys_accounts/api/api_stock.py:351
+#: rqalpha/mod/rqalpha_mod_sys_accounts/api/api_stock.py:352
 msgid ""
 "function order_target_portfolio: invalid values of target_portfolio, "
 "excepted float between 0 and 1, got {} (key: {})"
 msgstr ""
 
-#: rqalpha/mod/rqalpha_mod_sys_accounts/api/api_stock.py:360
+#: rqalpha/mod/rqalpha_mod_sys_accounts/api/api_stock.py:361
 msgid "total percent should be lower than 1, current: {}"
 msgstr ""
 
-#: rqalpha/mod/rqalpha_mod_sys_accounts/api/api_stock.py:380
+#: rqalpha/mod/rqalpha_mod_sys_accounts/api/api_stock.py:381
 msgid ""
 "Adjust position of {id_or_ins} Failed: Invalid close/open price "
 "{close_price}/{open_price}"
 msgstr ""
 
-#: rqalpha/mod/rqalpha_mod_sys_accounts/api/api_stock.py:693
+#: rqalpha/mod/rqalpha_mod_sys_accounts/api/api_stock.py:694
 msgid "in get_dividend, start_date {} is later than the previous test day {}"
 msgstr ""
 
@@ -455,31 +430,31 @@ msgstr ""
 msgid "[sys_analyser] Generate report from strategy output file"
 msgstr ""
 
-#: rqalpha/mod/rqalpha_mod_sys_analyser/mod.py:113
+#: rqalpha/mod/rqalpha_mod_sys_analyser/mod.py:111
 msgid ""
 "config 'base.benchmark' is deprecated, use 'mod.sys_analyser.benchmark' "
 "instead"
 msgstr ""
 
-#: rqalpha/mod/rqalpha_mod_sys_analyser/mod.py:151
+#: rqalpha/mod/rqalpha_mod_sys_analyser/mod.py:149
 msgid "benchmark {} not exists, please entry correct order_book_id"
 msgstr ""
 
-#: rqalpha/mod/rqalpha_mod_sys_analyser/mod.py:163
+#: rqalpha/mod/rqalpha_mod_sys_analyser/mod.py:161
 msgid "benchmark {} missing data between backtest start date {} and end date {}"
 msgstr ""
 
-#: rqalpha/mod/rqalpha_mod_sys_analyser/mod.py:175
+#: rqalpha/mod/rqalpha_mod_sys_analyser/mod.py:173
 msgid ""
 "benchmark {} available data start date {} >= backtest start date {} or "
 "end date {} <= backtest end date {}"
 msgstr ""
 
-#: rqalpha/mod/rqalpha_mod_sys_analyser/mod.py:243
+#: rqalpha/mod/rqalpha_mod_sys_analyser/mod.py:241
 msgid "invalid init benchmark {}, should be in format 'order_book_id:weight'"
 msgstr ""
 
-#: rqalpha/mod/rqalpha_mod_sys_analyser/mod.py:248
+#: rqalpha/mod/rqalpha_mod_sys_analyser/mod.py:246
 msgid "invalid weight for instrument {order_book_id}: {weight}"
 msgstr ""
 

requirements.txt

@@ -10,7 +10,8 @@ simplejson >=3.10.0
 dill ==0.2.5
 PyYAML >=3.12
 tabulate
-rqrisk >=1.0.8
+rqrisk >=1.0.9
 h5py
 matplotlib >=1.5.1 ; python_version >= '3.6'
 matplotlib >=1.5.1,<=3.0.3 ; python_version == '3.5'
+filelock

rqalpha/config.yml

@@ -41,6 +41,11 @@ base:
   forced_liquidation: true
   # 是否开启期货历史交易参数进行回测，默认为 False
   futures_time_series_trading_parameters: false
+  # 是否开启在回测过程中自动下载所需的 bundle 数据
+  # 当前支持数据：1. 盘前集合竞价成交量；2. 期货历史交易参数
+  auto_update_bundle: false
+  # 自动下载的 bundle 文件支持单独设置存储路径，若不设置则使用 data_bundle_path 路径
+  auto_update_bundle_path: ~
 
 
 extra:

rqalpha/data/base_data_source/data_source.py

@@ -24,23 +24,21 @@
 import pandas as pd
 import six
 from rqalpha.utils.i18n import gettext as _
-from rqalpha.const import INSTRUMENT_TYPE, TRADING_CALENDAR_TYPE, DEFAULT_ACCOUNT_TYPE
+from rqalpha.const import INSTRUMENT_TYPE, TRADING_CALENDAR_TYPE
 from rqalpha.interface import AbstractDataSource
 from rqalpha.model.instrument import Instrument
 from rqalpha.utils.datetime_func import (convert_date_to_int, convert_int_to_date, convert_int_to_datetime)
-from rqalpha.utils.exception import RQInvalidArgument, RQDatacVersionTooLow
+from rqalpha.utils.exception import RQInvalidArgument
 from rqalpha.utils.functools import lru_cache
 from rqalpha.utils.typing import DateLike
 from rqalpha.environment import Environment
-from rqalpha.data.bundle import update_futures_trading_parameters
-from rqalpha.utils.logger import user_system_log
 from rqalpha.data.base_data_source.adjust import FIELDS_REQUIRE_ADJUSTMENT, adjust_bars
 from rqalpha.data.base_data_source.storage_interface import (AbstractCalendarStore, AbstractDateSet,
                                 AbstractDayBarStore, AbstractDividendStore,
                                 AbstractInstrumentStore)
 from rqalpha.data.base_data_source.storages import (DateSet, DayBarStore, DividendStore,
                        ExchangeTradingCalendarStore, FutureDayBarStore,
-                       FutureInfoStore, FuturesTradingParametersStore,InstrumentStore,
+                       FutureInfoStore,InstrumentStore,
                        ShareTransformationStore, SimpleFactorStore,
                        YieldCurveStore, FuturesTradingParameters)
 
@@ -72,8 +70,7 @@ class BaseDataSource(AbstractDataSource):
         INSTRUMENT_TYPE.PUBLIC_FUND,
     )
 
-    def __init__(self, path, custom_future_info, futures_time_series_trading_parameters=False, end_date=None):       
-        # type: (str, dict, bool, date) -> None
+    def __init__(self, path: str, custom_future_info: dict, *args, **kwargs) -> None:
         if not os.path.exists(path):
             raise RuntimeError('bundle path {} not exist'.format(os.path.abspath(path)))
 
@@ -89,22 +86,22 @@ def _p(name):
             INSTRUMENT_TYPE.LOF: funds_day_bar_store
         }  # type: Dict[INSTRUMENT_TYPE, AbstractDayBarStore]
 
-        self._futures_trading_parameters_store = None
         self._future_info_store = FutureInfoStore(_p("future_info.json"), custom_future_info)
 
         self._instruments_stores = {}  # type: Dict[INSTRUMENT_TYPE, AbstractInstrumentStore]
         self._ins_id_or_sym_type_map = {}  # type: Dict[str, INSTRUMENT_TYPE]
         instruments = []
 
+        env = Environment.get_instance()
         with open(_p('instruments.pk'), 'rb') as f:
             for i in pickle.load(f):
                 if i["type"] == "Future" and Instrument.is_future_continuous_contract(i["order_book_id"]):
                     i["listed_date"] = datetime(1990, 1, 1)
                 instruments.append(Instrument(
                     i, 
                     lambda i: self._future_info_store.get_tick_size(i),
-                    lambda i, dt: self.get_futures_trading_parameters(i, dt).long_margin_ratio,
-                    lambda i, dt: self.get_futures_trading_parameters(i, dt).short_margin_ratio
+                    # lambda i, dt: env.data_proxy.get_futures_trading_parameters(i, dt).long_margin_ratio,
+                    # lambda i, dt: env.data_proxy.get_futures_trading_parameters(i, dt).short_margin_ratio
                     ))
         for ins_type in self.DEFAULT_INS_TYPES:
             self.register_instruments_store(InstrumentStore(instruments, ins_type))
@@ -133,20 +130,6 @@ def _p(name):
         self._suspend_days = [DateSet(_p('suspended_days.h5'))]  # type: List[AbstractDateSet]
         self._st_stock_days = DateSet(_p('st_stock_days.h5'))
 
-        if futures_time_series_trading_parameters:
-            try:
-                import rqdatac
-            except ImportError:
-                user_system_log.warn(_("RQDatac is not installed, \"config.base.futures_time_series_trading_parameters\" will be disabled."))
-            else:
-                try:
-                    update_futures_trading_parameters(path, end_date)
-                except (rqdatac.share.errors.PermissionDenied, RQDatacVersionTooLow):
-                    user_system_log.warn(_("RQDatac does not have permission to obtain futures histrical trading parameters, \"config.base.futures_time_series_trading_parameters\" will be disabled."))
-                else:
-                    file = os.path.join(path, "futures_trading_parameters.h5")
-                    self._futures_trading_parameters_store = FuturesTradingParametersStore(file, custom_future_info)
-
     def register_day_bar_store(self, instrument_type, store):
         #  type: (INSTRUMENT_TYPE, AbstractDayBarStore) -> None
         self._day_bars[instrument_type] = store
@@ -382,15 +365,8 @@ def get_yield_curve(self, start_date, end_date, tenor=None):
         return self._yield_curve.get_yield_curve(start_date, end_date, tenor=tenor)
 
     @lru_cache(1024)
-    def get_futures_trading_parameters(self, instrument, dt):
-        # type: (Instrument, datetime.date) -> FuturesTradingParameters
-        if self._futures_trading_parameters_store:
-            trading_parameters = self._futures_trading_parameters_store.get_futures_trading_parameters(instrument, dt)
-            if trading_parameters is None:
-                return self._future_info_store.get_future_info(instrument.order_book_id, instrument.underlying_symbol)
-            return trading_parameters
-        else:
-            return self._future_info_store.get_future_info(instrument.order_book_id, instrument.underlying_symbol)
+    def get_futures_trading_parameters(self, instrument: Instrument, dt: datetime.date) -> FuturesTradingParameters:
+        return self._future_info_store.get_future_info(instrument.order_book_id, instrument.underlying_symbol)
 
     def get_merge_ticks(self, order_book_id_list, trading_date, last_dt=None):
         raise NotImplementedError

rqalpha/data/base_data_source/storages.py

@@ -78,7 +78,7 @@ def __init__(self, f, custom_future_info):
             }
         self._custom_data = custom_future_info
         if "margin_rate" not in self._default_data[next(iter(self._default_data))]:
-            raise RuntimeError(_("Your bundle data is too old, please use 'rqalpha update-bundle' or 'rqalpha download-bundle' to update it to lastest before using"))
+            raise RuntimeError(_("The bundle data you are using is too old, please update it to lastest before using"))
 
     @classmethod
     def _process_future_info_item(cls, item):
@@ -247,72 +247,6 @@ class FutureDayBarStore(DayBarStore):
     DEFAULT_DTYPE = np.dtype(DayBarStore.DEFAULT_DTYPE.descr + [("open_interest", '<f8')])
 
 
-class FuturesTradingParametersStore(object):
-    COMMISSION_TYPE_MAP = {
-        0: COMMISSION_TYPE.BY_MONEY,
-        1: COMMISSION_TYPE.BY_VOLUME
-    }
-
-    # 历史期货交易参数的数据在2010年4月之后才有
-    FUTURES_TRADING_PARAMETERS_START_DATE = 20100401
-
-    def __init__(self, path, custom_future_info):
-        self._path = path
-        self._custom_data = custom_future_info
-
-    def get_futures_trading_parameters(self, instrument, dt):
-        # type: (Instrument, datetime.date) -> FuturesTradingParameters or None
-        dt = convert_date_to_date_int(dt)
-        if dt < self.FUTURES_TRADING_PARAMETERS_START_DATE:
-            return None
-        order_book_id = instrument.order_book_id
-        underlying_symbol = instrument.underlying_symbol
-        data = self.get_futures_trading_parameters_all_time(order_book_id)
-        if data is None: 
-            return None
-        else:
-            arr = data[data['datetime'] == dt]
-            if len(arr) == 0:
-                if dt >= convert_date_to_date_int(instrument.listed_date) and dt <= convert_date_to_date_int(instrument.de_listed_date):
-                    user_system_log.info("Historical futures trading parameters are abnormal, the lastst parameters will be used for calculations.\nPlease contract RiceQuant to repair: 0755-26569969")
-                return None
-            custom_info = self._custom_data.get(order_book_id) or self._custom_data.get(underlying_symbol)
-            if custom_info:
-                arr[0] = self.set_custom_info(arr[0], custom_info)        
-            futures_trading_parameters = self._to_namedtuple(arr[0])
-        return futures_trading_parameters
-
-    @lru_cache(1024)
-    def get_futures_trading_parameters_all_time(self, order_book_id):
-        # type: (str) -> numpy.ndarray or None
-        with h5_file(self._path) as h5:
-            try:
-                data = h5[order_book_id][:]
-            except KeyError:
-                return None
-        return data
-
-    def set_custom_info(self, arr, custom_info):
-        for field in custom_info:
-            if field == "commission_type":
-                if custom_info[field] == COMMISSION_TYPE.BY_MONEY:
-                    value = 0
-                elif custom_info[field] == COMMISSION_TYPE.BY_VOLUME:
-                    value = 1
-            else:
-                value = custom_info[field]
-            arr[field] = value
-        return arr
-
-    def _to_namedtuple(self, arr):
-        # type: (numpy.void) -> FuturesTradingParameters
-        dic = dict(zip(arr.dtype.names, arr))
-        del dic['datetime']
-        dic["commission_type"] = self.COMMISSION_TYPE_MAP[dic['commission_type']]
-        futures_trading_parameters = FuturesTradingParameters(**dic)
-        return futures_trading_parameters
-
-
 class DividendStore(AbstractDividendStore):
     def __init__(self, path):
         self._path = path