Merge pull request #379 from zhupr/fix_usinex_collector

Fix us_index collector
microsoft · Mar 29, 2021 · 45f7867 · 45f7867
2 parents 253378a + 4b66304
commit 45f7867
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 3 deletions.
diff --git a/scripts/data_collector/index.py b/scripts/data_collector/index.py
@@ -114,6 +114,8 @@ def save_new_companies(self):
             $ python collector.py save_new_companies --index_name CSI300 --qlib_dir ~/.qlib/qlib_data/cn_data
         """
         df = self.get_new_companies()
+        if df is None or df.empty:
+            raise ValueError(f"get new companies error: {self.index_name}")
         df = df.drop_duplicates([self.SYMBOL_FIELD_NAME])
         df.loc[:, self.INSTRUMENTS_COLUMNS].to_csv(
             self.instruments_dir.joinpath(f"{self.index_name.lower()}_only_new.txt"), sep="\t", index=False, header=None
@@ -184,7 +186,10 @@ def parse_instruments(self):
         logger.info(f"start parse {self.index_name.lower()} companies.....")
         instruments_columns = [self.SYMBOL_FIELD_NAME, self.START_DATE_FIELD, self.END_DATE_FIELD]
         changers_df = self.get_changes()
-        new_df = self.get_new_companies().copy()
+        new_df = self.get_new_companies()
+        if new_df is None or new_df.empty:
+            raise ValueError(f"get new companies error: {self.index_name}")
+        new_df = new_df.copy()
         logger.info("parse history companies by changes......")
         for _row in tqdm(changers_df.sort_values(self.DATE_FIELD_NAME, ascending=False).itertuples(index=False)):
             if _row.type == self.ADD:

diff --git a/scripts/data_collector/us_index/collector.py b/scripts/data_collector/us_index/collector.py
@@ -35,7 +35,7 @@
 class WIKIIndex(IndexBase):
     # NOTE: The US stock code contains "PRN", and the directory cannot be created on Windows system, use the "_" prefix
     # https://superuser.com/questions/613313/why-cant-we-make-con-prn-null-folder-in-windows
-    INST_PREFIX = "_"
+    INST_PREFIX = ""
 
     def __init__(self, index_name: str, qlib_dir: [str, Path] = None, request_retry: int = 5, retry_sleep: int = 3):
         super(WIKIIndex, self).__init__(
@@ -123,7 +123,7 @@ class NASDAQ100Index(WIKIIndex):
     MAX_WORKERS = 16
 
     def filter_df(self, df: pd.DataFrame) -> pd.DataFrame:
-        if not (set(df.columns) - {"Company", "Ticker"}):
+        if len(df) >= 100 and "Ticker" in df.columns:
             return df.loc[:, ["Ticker"]].copy()
 
     @property