Merge pull request #229 from fasiondog/feature/tmpstock

add temp stock support setKRecordList

hikyuu_cpp/hikyuu/Stock.cpp

@@ -290,15 +290,21 @@ void Stock::loadKDataToBuffer(KQuery::KType inkType) {
 
     releaseKDataBuffer(kType);
 
-    const auto& param = StockManager::instance().getPreloadParameter();
-    string preload_type = fmt::format("{}_max", kType);
-    to_lower(preload_type);
-    int max_num = param.tryGet<int>(preload_type, 4096);
-    HKU_ERROR_IF_RETURN(max_num < 0, void(), "Invalid preload {} param: {}", preload_type, max_num);
-
+    int start = 0;
     auto driver = m_kdataDriver->getConnect();
     size_t total = driver->getCount(m_data->m_market, m_data->m_code, kType);
-    int start = total <= max_num ? 0 : total - max_num;
+
+    // CSV 直接全部加载至内存，其他类型依据配置的预加载参数进行加载
+    if (driver->name() != "TMPCSV") {
+        const auto& param = StockManager::instance().getPreloadParameter();
+        string preload_type = fmt::format("{}_max", kType);
+        to_lower(preload_type);
+        int max_num = param.tryGet<int>(preload_type, 4096);
+        HKU_ERROR_IF_RETURN(max_num < 0, void(), "Invalid preload {} param: {}", preload_type,
+                            max_num);
+        start = total <= max_num ? 0 : total - max_num;
+    }
+
     {
         std::unique_lock<std::shared_mutex> lock(*(m_data->pMutex[kType]));
         // 需要对是否已缓存进行二次判定，防止加锁之前已被缓存
@@ -749,6 +755,54 @@ void Stock::realtimeUpdate(KRecord record, KQuery::KType inktype) {
     }
 }
 
+void Stock::setKRecordList(const KRecordList& ks, const KQuery::KType& ktype) {
+    HKU_IF_RETURN(ks.empty(), void());
+    string nktype(ktype);
+    to_upper(nktype);
+
+    // 写锁
+    std::unique_lock<std::shared_mutex> lock(*(m_data->pMutex[ktype]));
+    HKU_CHECK(m_data->pKData.find(nktype) != m_data->pKData.end(), "Invalid ktype: {}", ktype);
+
+    if (!m_data->pKData[nktype]) {
+        m_data->pKData[nktype] = new KRecordList();
+    }
+
+    (*(m_data->pKData[nktype])) = ks;
+
+    Parameter param;
+    param.set<string>("type", "DoNothin");
+    m_kdataDriver = DataDriverFactory::getKDataDriverPool(param);
+
+    m_data->m_valid = true;
+    m_data->m_startDate = ks.front().datetime;
+    m_data->m_lastDate = ks.back().datetime;
+}
+
+void Stock::setKRecordList(KRecordList&& ks, const KQuery::KType& ktype) {
+    HKU_IF_RETURN(ks.empty(), void());
+    string nktype(ktype);
+    to_upper(nktype);
+
+    // 写锁
+    std::unique_lock<std::shared_mutex> lock(*(m_data->pMutex[ktype]));
+    HKU_CHECK(m_data->pKData.find(nktype) != m_data->pKData.end(), "Invalid ktype: {}", ktype);
+
+    if (!m_data->pKData[nktype]) {
+        m_data->pKData[nktype] = new KRecordList();
+    }
+
+    (*m_data->pKData[nktype]) = std::move(ks);
+
+    Parameter param;
+    param.set<string>("type", "DoNothin");
+    m_kdataDriver = DataDriverFactory::getKDataDriverPool(param);
+
+    m_data->m_valid = true;
+    m_data->m_startDate = ks.front().datetime;
+    m_data->m_lastDate = ks.back().datetime;
+}
+
 const vector<HistoryFinanceInfo>& Stock::getHistoryFinance() const {
     std::lock_guard<std::mutex> lock(m_data->m_history_finance_mutex);
     if (!m_data->m_history_finance_ready) {

hikyuu_cpp/hikyuu/Stock.h

@@ -216,6 +216,13 @@ class HKU_API Stock {
     /** （临时函数）只用于更新缓存中的K线数据 **/
     void realtimeUpdate(KRecord, KQuery::KType ktype = KQuery::DAY);
 
+    /**
+     * 部分临时创建的 Stock, 直接设置KRecordList
+     * @note 谨慎调用，通常供外部数据源直接设定数据
+     */
+    void setKRecordList(const KRecordList& ks, const KQuery::KType& ktype = KQuery::DAY);
+    void setKRecordList(KRecordList&& ks, const KQuery::KType& ktype = KQuery::DAY);
+
     /** 仅用于python的__str__ */
     string toString() const;
 

hikyuu_cpp/hikyuu/StockManager.cpp

@@ -429,23 +429,13 @@ Stock StockManager::addTempCsvStock(const string& code, const string& day_filena
     p->setDayFileName(day_filename);
     p->setMinFileName(min_filename);
     result.setKDataDriver(driver_pool);
-    const auto& preload_param = getPreloadParameter();
-    if (preload_param.tryGet<bool>("day", true)) {
-        result.loadKDataToBuffer(KQuery::DAY);
-    }
-    if (preload_param.tryGet<bool>("min", false)) {
-        result.loadKDataToBuffer(KQuery::MIN);
-    }
+    result.loadKDataToBuffer(KQuery::DAY);
+    result.loadKDataToBuffer(KQuery::MIN);
     return addStock(result) ? result : Null<Stock>();
 }
 
 void StockManager::removeTempCsvStock(const string& code) {
-    string query_str = "TMP" + code;
-    to_upper(query_str);
-    auto iter = m_stockDict.find(query_str);
-    if (iter != m_stockDict.end()) {
-        m_stockDict.erase(iter);
-    }
+    removeStock(fmt::format("TMP{}", code));
 }
 
 bool StockManager::addStock(const Stock& stock) {
@@ -458,6 +448,16 @@ bool StockManager::addStock(const Stock& stock) {
     return true;
 }
 
+void StockManager::removeStock(const string& market_code) {
+    string n_market_code(market_code);
+    to_upper(n_market_code);
+    std::lock_guard<std::mutex> lock(*m_stockDict_mutex);
+    auto iter = m_stockDict.find(n_market_code);
+    if (iter != m_stockDict.end()) {
+        m_stockDict.erase(iter);
+    }
+}
+
 void StockManager::loadAllStocks() {
     HKU_INFO("Loading stock information...");
     vector<StockInfo> stockInfos;

hikyuu_cpp/hikyuu/StockManager.h

@@ -174,6 +174,12 @@ class HKU_API StockManager {
      */
     bool addStock(const Stock& stock);
 
+    /**
+     * 从 StockManager 中移除相应的 Stock，一般用于将临时增加的 Stock 从 sm 中移除
+     * @param market_code
+     */
+    void removeStock(const string& market_code);
+
     /**
      * 从CSV文件（K线数据）增加临时的Stock，可用于只有CSV格式的K线数据时，进行临时测试
      * @details 增加的临时Stock，其market为“TMP”

hikyuu_cpp/hikyuu/data_driver/DataDriverFactory.cpp

@@ -8,6 +8,7 @@
 #include "../GlobalInitializer.h"
 #include <boost/algorithm/string.hpp>
 #include "block_info/qianlong/QLBlockInfoDriver.h"
+#include "kdata/DoNothingKDataDriver.h"
 #include "kdata/cvs/KDataTempCsvDriver.h"
 #include "DataDriverFactory.h"
 #include "KDataDriver.h"
@@ -61,6 +62,7 @@ void DataDriverFactory::init() {
     m_kdataPrototypeDrivers = new map<string, KDataDriverPtr>();
     m_kdataDriverPools = new map<string, KDataDriverConnectPoolPtr>();
 
+    DataDriverFactory::regKDataDriver(make_shared<DoNothingKDataDriver>());
     DataDriverFactory::regKDataDriver(make_shared<KDataTempCsvDriver>());
 
 #if HKU_ENABLE_TDX_KDATA

hikyuu_cpp/hikyuu/data_driver/base_info/mysql/MySQLBaseInfoDriver.cpp

@@ -403,7 +403,7 @@ vector<HistoryFinanceInfo> MySQLBaseInfoDriver::getHistoryFinance(const string &
         to_upper(market_code);
         vector<HistoryFinanceTable> finances;
         con->batchLoad(finances, ((Field("market_code") == market_code) &
-                                  (Field("report_date") >= start.ymd())) +
+                                  (Field("report_date") >= new_start.ymd())) +
                                    ASC("report_date"));
         size_t total = finances.size();
         result.resize(total);

hikyuu_cpp/hikyuu/data_driver/base_info/table/HistoryFinanceTable.h

@@ -12,7 +12,7 @@
 namespace hku {
 
 struct HistoryFinanceTable {
-    TABLE_BIND4(HistoryFinanceTable, HistoryFinance, file_date, market_code, report_date, values)
+    TABLE_BIND4(HistoryFinanceTable, HistoryFinance, file_date, report_date, market_code, values)
     uint64_t file_date;
     uint64_t report_date;
     std::string market_code;

hikyuu_cpp/hikyuu/data_driver/kdata/DoNothingKDataDriver.h

@@ -0,0 +1,37 @@
+/*
+ *  Copyright (c) 2024 hikyuu.org
+ *
+ *  Created on: 2024-04-16
+ *      Author: fasiondog
+ */
+
+#include "../KDataDriver.h"
+
+namespace hku {
+
+/**
+ * 一个特殊的 KDatadriver，不实际读取数据，用于增加外部临时Stock时使用
+ */
+class DoNothingKDataDriver : public KDataDriver {
+public:
+    DoNothingKDataDriver() : KDataDriver("DoNothing") {}
+    virtual ~DoNothingKDataDriver() = default;
+
+    virtual KDataDriverPtr _clone() override {
+        return std::make_shared<DoNothingKDataDriver>();
+    }
+
+    virtual bool _init() override {
+        return true;
+    }
+
+    virtual bool isIndexFirst() override {
+        return true;
+    }
+
+    virtual bool canParallelLoad() override {
+        return true;
+    }
+};
+
+}  // namespace hku

hikyuu_cpp/hikyuu/data_driver/kdata/cvs/KDataTempCsvDriver.cpp

@@ -153,7 +153,7 @@ KRecordList KDataTempCsvDriver::_getKRecordListByIndex(const string& market, con
     }
 
     std::ifstream infile(filename.c_str());
-    HKU_ERROR_IF_RETURN(!infile, result, "Can't open this file: {}", filename);
+    HKU_ERROR_IF_RETURN(!infile, result, "Can't open this file: {}, ktype: {}", filename, kType);
     string line;
     if (!std::getline(infile, line)) {
         infile.close();

hikyuu_pywrap/_Stock.cpp

@@ -190,6 +190,24 @@ void export_Stock(py::module& m) {
 
         :param Query.KType ktype: K线类型)")
 
+      .def(
+        "set_krecord_list",
+        [](Stock& self, const py::object& obj) {
+            if (py::isinstance<KRecordList>(obj)) {
+                const auto& ks = obj.cast<const KRecordList&>();
+                self.setKRecordList(ks);
+            } else if (py::isinstance<py::sequence>(obj)) {
+                auto seq = obj.cast<py::sequence>();
+                auto ks = python_list_to_vector<KRecord>(seq);
+                self.setKRecordList(ks);
+            } else {
+                HKU_THROW("Unusable input data type");
+            }
+        },
+        R"(set_krecord_list(self, krecord_list)
+
+        "谨慎调用！！！直接设置当前内存 KRecordList, 仅供需临时增加的外部 Stock 设置 K 线数据)")
+
       .def(py::self == py::self)
       .def(py::self != py::self)
 

hikyuu_pywrap/_StockManager.cpp

@@ -192,6 +192,15 @@ void export_StockManager(py::module& m) {
                return ret;
            })
 
+      .def("add_stock", &StockManager::addStock, R"(add_stock(self, stock)
+      
+    谨慎调用！！！仅供增加某些临时的外部 Stock)
+    @return True | False)")
+
+      .def("remove_stock", &StockManager::removeStock, R"(remove_stock(self, market_code)
+    
+    从 sm 中移除 market_code 代表的证券，谨慎使用！！！通常用于移除临时增加的外布 Stock)")
+
       .def("__len__", &StockManager::size, "返回证券数量")
       .def("__getitem__", &StockManager::getStock, "同 get_stock")
       .def(