Merge pull request #228 from fasiondog/feature/finance

历史财务信息入库，实现 FINANCE 指标（财务数据指标化）

hikyuu/data/common.py

@@ -27,6 +27,7 @@
 import akshare as ak
 import pandas as pd
 import datetime
+from hikyuu.core import cpp_bytes_to_vector_float_blob
 from hikyuu.util import *
 
 
@@ -191,3 +192,53 @@ def get_china_bond10_rate(start_date="19901219"):
     bond_zh_us_rate_df = ak.bond_zh_us_rate(start_date)
     df = bond_zh_us_rate_df[['中国国债收益率10年', '日期']].dropna()
     return [(v[1].strftime('%Y%m%d'), int(v[0]*10000)) for v in df.values]
+
+
+def modifiy_code(code):
+    if code.startswith(('0', '3')):
+        return 'SZ' + code
+    if code.startswith(('4', '8')):
+        return 'BJ' + code
+    if code.startswith('6'):
+        return 'SH' + code
+    else:
+        hku_warn("Unknow code: {}", code)
+        return None
+
+
+def historyfinancialreader(filepath):
+    """
+    读取解析通达信目录的历史财务数据（来源: onefish, 公众号：一鱼策略）
+    :param filepath: 字符串类型。传入文件路径
+    :return: DataFrame格式。返回解析出的财务文件内容
+    """
+    import struct
+
+    cw_file = open(filepath, 'rb')
+    header_pack_format = '<1hI1H3L'
+    header_size = struct.calcsize(header_pack_format)
+    stock_item_size = struct.calcsize("<6s1c1L")
+    data_header = cw_file.read(header_size)
+    stock_header = struct.unpack(header_pack_format, data_header)
+    max_count = stock_header[2]
+    file_date = stock_header[1]
+    print(file_date)
+    report_size = stock_header[4]
+    report_fields_count = int(report_size / 4)
+    report_pack_format = '<{}f'.format(report_fields_count)
+    results = []
+    for stock_idx in range(0, max_count):
+        cw_file.seek(header_size + stock_idx * struct.calcsize("<6s1c1L"))
+        si = cw_file.read(stock_item_size)
+        stock_item = struct.unpack("<6s1c1L", si)
+        code = stock_item[0].decode("utf-8")
+        foa = stock_item[2]
+        cw_file.seek(foa)
+        info_data = cw_file.read(struct.calcsize(report_pack_format))
+        cw_info = list(struct.unpack(report_pack_format, info_data))
+        report_date = int(cw_info[313])  # 财务公告日期
+        report_date = 19000000 + report_date if report_date > 800000 else 20000000 + report_date
+        # results.append((modifiy_code(code), report_date, cw_info))
+        results.append((file_date, modifiy_code(code), report_date, cpp_bytes_to_vector_float_blob(info_data)))
+    cw_file.close()
+    return results

hikyuu/data/pytdx_finance_to_mysql.py

@@ -22,10 +22,11 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-from hikyuu.data.common import MARKETID, STOCKTYPE
+from hikyuu.data.common import MARKETID, STOCKTYPE, historyfinancialreader
 from hikyuu.data.common_mysql import get_marketid
 from hikyuu.util import *
 
+
 @hku_catch(trace=True)
 def pytdx_import_finance_to_mysql(db_connect, pytdx_connect, market):
     """导入公司财务信息"""
@@ -41,7 +42,7 @@ def pytdx_import_finance_to_mysql(db_connect, pytdx_connect, market):
     records = []
     for stk in all_list:
         x = pytdx_connect.get_finance_info(1 if stk[1] == MARKETID.SH else 0, stk[2])
-        #print(stk[2])
+        # print(stk[2])
         if x is not None and x['code'] == stk[2]:
             cur.execute(
                 "select updated_date from `hku_base`.`stkfinance` where stockid={} and updated_date={}".format(
@@ -51,9 +52,9 @@ def pytdx_import_finance_to_mysql(db_connect, pytdx_connect, market):
             a = cur.fetchall()
             a = [x[0] for x in a]
             if a:
-                #print(a)
+                # print(a)
                 continue
-            #else:
+            # else:
             #    print(market, stk[2])
             records.append(
                 (
@@ -114,4 +115,15 @@ def pytdx_import_finance_to_mysql(db_connect, pytdx_connect, market):
         db_connect.commit()
 
     cur.close()
-    return len(records)
+    return len(records)
+
+
+def history_finance_import_mysql(connect, filename):
+    file_date = filename[-12:-4]
+    ret = historyfinancialreader(filename)
+    cur = connect.cursor()
+    cur.execute(f"delete from `hku_base`.`HistoryFinance` where file_date={file_date}")
+    cur.executemany(
+        "insert into `hku_base`.`HistoryFinance` (`file_date`, `market_code`, `report_date`, `values`) values (%s, %s, %s, %s)", ret)
+    connect.commit()
+    cur.close()

hikyuu/data/pytdx_finance_to_sqlite.py

@@ -22,7 +22,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-from hikyuu.data.common import MARKETID, STOCKTYPE
+from hikyuu.data.common import MARKETID, STOCKTYPE, historyfinancialreader
 from hikyuu.data.common_sqlite3 import get_marketid, create_database
 
 
@@ -39,7 +39,7 @@ def pytdx_import_finance_to_sqlite(db_connect, pytdx_connect, market):
     records = []
     for stk in all_list:
         x = pytdx_connect.get_finance_info(1 if stk[1] == MARKETID.SH else 0, stk[2])
-        #print(stk[2])
+        # print(stk[2])
         if x is not None and x['code'] == stk[2]:
             cur.execute(
                 "select updated_date from stkfinance where stockid={} and updated_date={}".format(
@@ -49,9 +49,9 @@ def pytdx_import_finance_to_sqlite(db_connect, pytdx_connect, market):
             a = cur.fetchall()
             a = [x[0] for x in a]
             if a:
-                #print(a)
+                # print(a)
                 continue
-            #else:
+            # else:
             #    print(market, stk[2])
             records.append(
                 (
@@ -115,6 +115,17 @@ def pytdx_import_finance_to_sqlite(db_connect, pytdx_connect, market):
     return len(records)
 
 
+def history_finance_import_sqlite(connect, filename):
+    file_date = filename[-12:-4]
+    ret = historyfinancialreader(filename)
+    cur = connect.cursor()
+    cur.execute(f"delete from `HistoryFinance` where file_date={file_date}")
+    cur.executemany(
+        "insert into `HistoryFinance` (`file_date`, `market_code`, `report_date`, `values`) values (?,?,?,?)", ret)
+    connect.commit()
+    cur.close()
+
+
 if __name__ == '__main__':
     import os
     import time
@@ -123,7 +134,7 @@ def pytdx_import_finance_to_sqlite(db_connect, pytdx_connect, market):
     starttime = time.time()
 
     dest_dir = "d:\\stock"
-    tdx_server = '120.76.152.87'  #'119.147.212.81'
+    tdx_server = '120.76.152.87'  # '119.147.212.81'
     tdx_port = 7709
 
     connect = sqlite3.connect(dest_dir + "\\stock.db")
@@ -148,4 +159,4 @@ def pytdx_import_finance_to_sqlite(db_connect, pytdx_connect, market):
     endtime = time.time()
     print("\nTotal time:")
     print("%.2fs" % (endtime - starttime))
-    print("%.2fm" % ((endtime - starttime) / 60))
+    print("%.2fm" % ((endtime - starttime) / 60))