From fd8bbb20cd74d8d5578cac82b03474576d41d356 Mon Sep 17 00:00:00 2001
From: Boreas514 <zio_liu@qq.com>
Date: Wed, 23 Sep 2020 19:04:54 +0800
Subject: [PATCH 1/6] add most method

---
 .gitignore                                 |   3 +
 python/fastquant/data/forex/__init__.py    |   0
 python/fastquant/data/forex/forextester.py | 132 +++++++++++++++++++++
 python/fastquant/data/forex/forey.py       |  44 +++++++
 4 files changed, 179 insertions(+)
 create mode 100644 python/fastquant/data/forex/__init__.py
 create mode 100644 python/fastquant/data/forex/forextester.py
 create mode 100644 python/fastquant/data/forex/forey.py

diff --git a/.gitignore b/.gitignore
index 94d18c2a..721d52d0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -135,3 +135,6 @@ fastquant_0.0.0.9000.tar.gz
 
 # VSCode Workspace files
 *.code-workspace
+
+# PyCharm Workspace files
+*.idea/
\ No newline at end of file
diff --git a/python/fastquant/data/forex/__init__.py b/python/fastquant/data/forex/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/python/fastquant/data/forex/forextester.py b/python/fastquant/data/forex/forextester.py
new file mode 100644
index 00000000..455fce55
--- /dev/null
+++ b/python/fastquant/data/forex/forextester.py
@@ -0,0 +1,132 @@
+import os
+from pathlib import Path
+import zipfile
+
+import requests
+import pandas as pd
+from pandas import DataFrame
+
+from fastquant.config import DATA_PATH
+
+def get_forextester_data(symbol, start_date, end_date,  time_frame):
+    allowed_forex_symbol_list = [
+        'AUDJPY',
+        'AUDUSD',
+        'CHFJPY',
+        'EURCAD',
+        'EURCHF',
+        'EURGBP',
+        'EURJPY',
+        'EURUSD',
+        'GBPCHF',
+        'GBPJPY',
+        'GBPUSD',
+        'NZDJPY',
+        'NZDUSD',
+        'USDCAD',
+        'USDJPY',
+        'USDCHF',
+        'XAGUSD',
+        'XAUUSD'
+    ]
+    if symbol not in allowed_forex_symbol_list:
+        raise ValueError('your symbol is not supported by forextester')
+
+    print('start downloading forex data zip file...')
+    res = requests.get(f'http://www.forextester.com/templates/data/files/{symbol}.zip', stream=True)
+    with open(Path(DATA_PATH, f'{symbol}.zip'), 'wb') as forex_zip_file:
+        for chunk in res.iter_content(chunk_size=1024):
+            forex_zip_file.write(chunk)
+
+    zip_file = zipfile.ZipFile(Path(DATA_PATH, f'{symbol}.zip'))
+    zip_file.extractall(Path(DATA_PATH))
+
+    tmp_dict = {
+        'dt':[],
+        'minute':[],
+        'open':[],
+        'high':[],
+        'low':[],
+        'close':[]
+    }
+    with open(Path(DATA_PATH, f'{symbol}.txt')) as forex_txt_file:
+        for line in forex_txt_file:
+            symbol, data, minute, open_price, high, low, close, vol = line.strip().split(',')
+            if data == '<DTYYYYMMDD>':
+                continue
+            tmp_dict['dt'].append(data+minute)
+            tmp_dict['open'].append(open_price)
+            tmp_dict['high'].append(high)
+            tmp_dict['low'].append(low)
+            tmp_dict['close'].append(close)
+
+    forex_dataframe = DataFrame(tmp_dict, columns=list(tmp_dict.keys()))
+    forex_dataframe = forex_dataframe.reset_index()
+    forex_dataframe["dt"] = pd.to_datetime(forex_dataframe.dt)
+    return forex_dataframe.set_index('dt')
+
+
+
+def shape_data_from_1min_to_other(df, time_frame):
+    if time_frame not in ["M1", "M5", "M15", "H1", "D1", "W1"]:
+        raise ValueError('time_frame must in this list:["M1", "M5", "M15", "H1", "D1", "W1"]')
+
+    tmp_dict = {
+        'dt': [],
+        'minute': [],
+        'open': [],
+        'high': [],
+        'low': [],
+        'close': []
+    }
+
+    if time_frame == 'M1':
+        return df
+    elif time_frame == 'M5':
+        pass
+    elif time_frame == 'M15':
+        tmp_datetime = None
+        tmp_hour = None
+        tmp_minute = None
+        tmp_open = None
+        tmp_high = None
+        tmp_low = None
+
+        for index, row in df.iterrows():
+            if tmp_open is None:
+                tmp_open = row['open']
+                tmp_high = row['high']
+                tmp_low = row['low']
+                tmp_datetime = row['dt']
+                tmp_hour = row['dt'].hour
+                tmp_minute = row['dt'].minute
+            else:
+                if row['high']>tmp_high:
+                    tmp_high = row['high']
+                if row['low']<tmp_low:
+                    tmp_low = row['low']
+
+                if tmp_minute<15:
+                    if df[index+1:index+2]['dt'].values[0].minute >=15 and df[index+1:index+2]['dt'].values[0].hour == tmp_hour:
+                        tmp_dict['dt'] = tmp_datetime
+                        tmp_dict['open'] = tmp_open
+                        tmp_dict['high'] = tmp_high
+                        tmp_dict['low'] = tmp_low
+                        tmp_dict['close'] = row['close']
+                        tmp_datetime = None
+                        tmp_hour = None
+                        tmp_minute = None
+                        tmp_open = None
+                        tmp_high = None
+                        tmp_low = None
+                elif 15<tmp_minute<30:
+                    pass
+                elif 30<tmp_minute<45:
+                    pass
+                elif tmp_minute>45:
+                    pass
+                else:
+                    pass
+
+    else:
+        pass
\ No newline at end of file
diff --git a/python/fastquant/data/forex/forey.py b/python/fastquant/data/forex/forey.py
new file mode 100644
index 00000000..2f2fc86b
--- /dev/null
+++ b/python/fastquant/data/forex/forey.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import numpy as np
+
+# Import from config
+from fastquant.config import DATA_FORMAT_COLS
+
+# Import package
+from fastquant.data.forex.forextester import get_forextester_data
+
+
+def get_forex_data(symbol, start_date=None, end_date=None, source="forextester", time_frame='day'):
+    """Returns pricing data for a specified forex pair.
+
+    Parameters
+    ----------
+    symbol : str
+        Symbol of the forex in the forextester.
+        https://forextester.com/data/datasources
+    start_date : str
+        Starting date (YYYY-MM-DD) of the period that you want to get data on
+        in most cases we need more wide time period to test strategy, so keep this to None
+    end_date : str
+        Ending date (YYYY-MM-DD) of the period you want to get data on
+        in most cases we need more wide time period to test strategy, so keep this to None
+    source : str
+         Source of forex history data
+    time_frame : str
+         time frame you want, support 1 minute,15 minutes,1 hour,1 day,1 week
+         this parameter must one of them:["M1", "M5", "M15", "H1", "D1", "W1"]
+
+    Returns
+    -------
+    pandas.DataFrame
+        Forex data (in the specified `format`) for the specified company and date range
+    """
+
+    if source == "forextester":
+        df = get_forextester_data(symbol, start_date, end_date, time_frame)
+    else:
+        raise Exception("Source must be either 'phisix' or 'yahoo'")
+
+    return df

From 491e86b66e698405cee57113d61a15ca07dc688e Mon Sep 17 00:00:00 2001
From: Boreas514 <zio_liu@qq.com>
Date: Thu, 1 Oct 2020 14:27:45 +0800
Subject: [PATCH 2/6] get forex data method done

---
 .gitignore                                    |   9 +-
 .../data/forex/{forey.py => forex.py}         |  17 +-
 python/fastquant/data/forex/forextester.py    | 359 +++++++++++++++---
 python/fastquant/data/forex/tests.py          |  44 +++
 4 files changed, 372 insertions(+), 57 deletions(-)
 rename python/fastquant/data/forex/{forey.py => forex.py} (65%)
 create mode 100644 python/fastquant/data/forex/tests.py

diff --git a/.gitignore b/.gitignore
index 721d52d0..854da391 100644
--- a/.gitignore
+++ b/.gitignore
@@ -137,4 +137,11 @@ fastquant_0.0.0.9000.tar.gz
 *.code-workspace
 
 # PyCharm Workspace files
-*.idea/
\ No newline at end of file
+*.idea/
+
+# tmp pickle test files
+*.pickle
+
+# tmp forex data
+/python/fastquant/data/*.txt
+/python/fastquant/data/*.zip
diff --git a/python/fastquant/data/forex/forey.py b/python/fastquant/data/forex/forex.py
similarity index 65%
rename from python/fastquant/data/forex/forey.py
rename to python/fastquant/data/forex/forex.py
index 2f2fc86b..e27afcd0 100644
--- a/python/fastquant/data/forex/forey.py
+++ b/python/fastquant/data/forex/forex.py
@@ -7,10 +7,10 @@
 from fastquant.config import DATA_FORMAT_COLS
 
 # Import package
-from fastquant.data.forex.forextester import get_forextester_data
+from fastquant.data.forex.forextester import get_forextester_data, get_local_data
 
 
-def get_forex_data(symbol, start_date=None, end_date=None, source="forextester", time_frame='day'):
+def get_forex_data(symbol, start_date=None, end_date=None, source="forextester", time_frame='D1', read_from_local=False):
     """Returns pricing data for a specified forex pair.
 
     Parameters
@@ -28,17 +28,22 @@ def get_forex_data(symbol, start_date=None, end_date=None, source="forextester",
          Source of forex history data
     time_frame : str
          time frame you want, support 1 minute,15 minutes,1 hour,1 day,1 week
-         this parameter must one of them:["M1", "M5", "M15", "H1", "D1", "W1"]
+         this parameter must one of them:["M1", "M15", "H1", "D1", "W1"]
+    read_from_local : bull
+        if this parameter set False, method get data from online
+        if set it to True, method get data from local pickle file, faster than set it to False
 
     Returns
     -------
     pandas.DataFrame
-        Forex data (in the specified `format`) for the specified company and date range
     """
 
     if source == "forextester":
-        df = get_forextester_data(symbol, start_date, end_date, time_frame)
+        if read_from_local is False:
+            df = get_forextester_data(symbol, start_date, end_date, time_frame)
+        else:
+            df = get_local_data(symbol, start_date, end_date, time_frame)
     else:
-        raise Exception("Source must be either 'phisix' or 'yahoo'")
+        raise Exception("Source must be forextester")
 
     return df
diff --git a/python/fastquant/data/forex/forextester.py b/python/fastquant/data/forex/forextester.py
index 455fce55..98c84ba4 100644
--- a/python/fastquant/data/forex/forextester.py
+++ b/python/fastquant/data/forex/forextester.py
@@ -1,14 +1,25 @@
 import os
 from pathlib import Path
 import zipfile
+import logging
+import pickle
 
 import requests
 import pandas as pd
 from pandas import DataFrame
 
 from fastquant.config import DATA_PATH
+from fastquant.data.stocks.pse import datestring_to_datetime
 
-def get_forextester_data(symbol, start_date, end_date,  time_frame):
+handler_format = logging.Formatter('%(asctime)s-%(name)s-%(levelname)s-%(message)s',datefmt='%Y-%m-%d %H:%M:%S')
+logger = logging.getLogger('forex_logger')
+logger_handler = logging.StreamHandler()
+logger_handler.setFormatter(handler_format)
+logger.setLevel(logging.INFO)
+logger.addHandler(logger_handler)
+
+
+def get_forextester_data(symbol, start_date=None, end_date=None,  time_frame='D1'):
     allowed_forex_symbol_list = [
         'AUDJPY',
         'AUDUSD',
@@ -32,48 +43,92 @@ def get_forextester_data(symbol, start_date, end_date,  time_frame):
     if symbol not in allowed_forex_symbol_list:
         raise ValueError('your symbol is not supported by forextester')
 
-    print('start downloading forex data zip file...')
+    if start_date is not None:
+        start_date = datestring_to_datetime(start_date)
+        end_date = datestring_to_datetime(end_date)
+
+    logger.info(f'start downloading forex {symbol} data zip file...')
+    file_size = 0
     res = requests.get(f'http://www.forextester.com/templates/data/files/{symbol}.zip', stream=True)
     with open(Path(DATA_PATH, f'{symbol}.zip'), 'wb') as forex_zip_file:
-        for chunk in res.iter_content(chunk_size=1024):
+        for chunk in res.iter_content(chunk_size=4096):
+            file_size+=4096
+            if file_size<1048576:
+                logger.info(f'download {round(file_size/1024)}KB')
+            else:
+                logger.info(f'download {round(file_size/1048576)}MB')
             forex_zip_file.write(chunk)
-
+    logger.info('download success')
     zip_file = zipfile.ZipFile(Path(DATA_PATH, f'{symbol}.zip'))
     zip_file.extractall(Path(DATA_PATH))
+    logger.info('unzip success')
 
-    tmp_dict = {
-        'dt':[],
-        'minute':[],
-        'open':[],
-        'high':[],
-        'low':[],
-        'close':[]
+    forex_dataframe = pd.read_csv(
+        Path(DATA_PATH, f'{symbol}.txt'),
+        dtype={'<DTYYYYMMDD>': str, '<TIME>': str, '<OPEN>': float, '<HIGH>': float, '<LOW>': float, '<CLOSE>': float}
+    )
+    logger.info('load txt success')
+    del(forex_dataframe['<TICKER>'])
+    del(forex_dataframe['<VOL>'])
+    rename_list = [
+        'date',
+        'time',
+        'open',
+        'high',
+        'low',
+        'close',
+    ]
+    rename_dict = {
+        '<DTYYYYMMDD>': 'date',
+        '<TIME>': 'time',
+        '<OPEN>': 'open',
+        '<HIGH>': 'high',
+        '<LOW>': 'low',
+        '<CLOSE>': 'close',
     }
-    with open(Path(DATA_PATH, f'{symbol}.txt')) as forex_txt_file:
-        for line in forex_txt_file:
-            symbol, data, minute, open_price, high, low, close, vol = line.strip().split(',')
-            if data == '<DTYYYYMMDD>':
-                continue
-            tmp_dict['dt'].append(data+minute)
-            tmp_dict['open'].append(open_price)
-            tmp_dict['high'].append(high)
-            tmp_dict['low'].append(low)
-            tmp_dict['close'].append(close)
+    forex_dataframe = forex_dataframe.rename(columns=rename_dict)[rename_list].drop_duplicates()
+    logger.info('change column name success')
+    forex_dataframe['dt'] = forex_dataframe['date'] + forex_dataframe['time']
+    logger.info('sum date and time success')
+    del(forex_dataframe['date'])
+    del(forex_dataframe['time'])
+    logger.info('start to transfer str datetime to pd.timestamp, this process requires 6 minutes on INTEL-9900k')
+    forex_dataframe['dt'] = pd.to_datetime(forex_dataframe.dt)
+    forex_dataframe = shape_data_from_1min_to_other(forex_dataframe, time_frame)
+    logger.info('start write backup file to disk')
+    with open(f'{symbol}_{time_frame}.pickle', 'wb') as df_file:
+        pickle.dump(forex_dataframe, df_file, True)
+    logger.info('all is done')
+    if start_date is None:
+        return forex_dataframe
+    else:
+        forex_dataframe = forex_dataframe[(forex_dataframe['dt'] >= start_date) & (forex_dataframe['dt'] <= end_date)]
+        return forex_dataframe
 
-    forex_dataframe = DataFrame(tmp_dict, columns=list(tmp_dict.keys()))
-    forex_dataframe = forex_dataframe.reset_index()
-    forex_dataframe["dt"] = pd.to_datetime(forex_dataframe.dt)
-    return forex_dataframe.set_index('dt')
 
+def get_local_data(symbol, start_date=None, end_date=None,  time_frame='D1'):
+    if start_date is not None:
+        start_date = datestring_to_datetime(start_date)
+        end_date = datestring_to_datetime(end_date)
+    logger.info('start load pickle to memory')
+    try:
+        with open(f'{symbol}_{time_frame}.pickle', 'rb') as df_file:
+            df = pickle.load(df_file)
+    except Exception as e:
+        raise FileNotFoundError('backup file not found, try get_forextester_data method first')
+    if start_date is None:
+        return df
+    else:
+        df = df[(df['dt'] >= start_date) & (df['dt'] <= end_date)]
+        return df
 
 
 def shape_data_from_1min_to_other(df, time_frame):
-    if time_frame not in ["M1", "M5", "M15", "H1", "D1", "W1"]:
-        raise ValueError('time_frame must in this list:["M1", "M5", "M15", "H1", "D1", "W1"]')
+    if time_frame not in ["M1", "M15", "H1", "D1", "W1"]:
+        raise ValueError('time_frame must in this list:["M1", "M15", "H1", "D1", "W1"]')
 
     tmp_dict = {
         'dt': [],
-        'minute': [],
         'open': [],
         'high': [],
         'low': [],
@@ -82,8 +137,6 @@ def shape_data_from_1min_to_other(df, time_frame):
 
     if time_frame == 'M1':
         return df
-    elif time_frame == 'M5':
-        pass
     elif time_frame == 'M15':
         tmp_datetime = None
         tmp_hour = None
@@ -91,6 +144,7 @@ def shape_data_from_1min_to_other(df, time_frame):
         tmp_open = None
         tmp_high = None
         tmp_low = None
+        tmp_close = None
 
         for index, row in df.iterrows():
             if tmp_open is None:
@@ -105,28 +159,233 @@ def shape_data_from_1min_to_other(df, time_frame):
                     tmp_high = row['high']
                 if row['low']<tmp_low:
                     tmp_low = row['low']
+                tmp_close = row['close']
 
                 if tmp_minute<15:
-                    if df[index+1:index+2]['dt'].values[0].minute >=15 and df[index+1:index+2]['dt'].values[0].hour == tmp_hour:
-                        tmp_dict['dt'] = tmp_datetime
-                        tmp_dict['open'] = tmp_open
-                        tmp_dict['high'] = tmp_high
-                        tmp_dict['low'] = tmp_low
-                        tmp_dict['close'] = row['close']
-                        tmp_datetime = None
-                        tmp_hour = None
-                        tmp_minute = None
-                        tmp_open = None
-                        tmp_high = None
-                        tmp_low = None
-                elif 15<tmp_minute<30:
-                    pass
-                elif 30<tmp_minute<45:
-                    pass
-                elif tmp_minute>45:
-                    pass
+                    # continuous situation
+                    if row['dt'].minute >=15 and row['dt'].hour == tmp_hour:
+                        tmp_dict['dt'].append(tmp_datetime)
+                        tmp_dict['open'].append(tmp_open)
+                        tmp_dict['high'].append(tmp_high)
+                        tmp_dict['low'].append(tmp_low)
+                        tmp_dict['close'].append(tmp_close)
+                        tmp_open = row['open']
+                        tmp_high = row['high']
+                        tmp_low = row['low']
+                        tmp_datetime = row['dt']
+                        tmp_hour = row['dt'].hour
+                        tmp_minute = row['dt'].minute
+                        continue
+                    # discontinuous situation
+                    elif row['dt'].hour != tmp_hour:
+                        tmp_dict['dt'].append(tmp_datetime)
+                        tmp_dict['open'].append(tmp_open)
+                        tmp_dict['high'].append(tmp_high)
+                        tmp_dict['low'].append(tmp_low)
+                        tmp_dict['close'].append(tmp_close)
+                        tmp_open = row['open']
+                        tmp_high = row['high']
+                        tmp_low = row['low']
+                        tmp_datetime = row['dt']
+                        tmp_hour = row['dt'].hour
+                        tmp_minute = row['dt'].minute
+                        continue
+                    elif row['dt'].minute <15:
+                        continue
+                    else:
+                        raise ValueError(f'M1 to M15 unkown error, start datetime:{tmp_hour} {tmp_minute} end datetime:{row["dt"].hour} {row["dt"].minute}')
+                elif 15<=tmp_minute<30:
+                    if row['dt'].minute >=30 and row['dt'].hour == tmp_hour:
+                        tmp_dict['dt'].append(tmp_datetime)
+                        tmp_dict['open'].append(tmp_open)
+                        tmp_dict['high'].append(tmp_high)
+                        tmp_dict['low'].append(tmp_low)
+                        tmp_dict['close'].append(tmp_close)
+                        tmp_open = row['open']
+                        tmp_high = row['high']
+                        tmp_low = row['low']
+                        tmp_datetime = row['dt']
+                        tmp_hour = row['dt'].hour
+                        tmp_minute = row['dt'].minute
+                        continue
+                    elif row['dt'].hour != tmp_hour:
+                        tmp_dict['dt'].append(tmp_datetime)
+                        tmp_dict['open'].append(tmp_open)
+                        tmp_dict['high'].append(tmp_high)
+                        tmp_dict['low'].append(tmp_low)
+                        tmp_dict['close'].append(tmp_close)
+                        tmp_open = row['open']
+                        tmp_high = row['high']
+                        tmp_low = row['low']
+                        tmp_datetime = row['dt']
+                        tmp_hour = row['dt'].hour
+                        tmp_minute = row['dt'].minute
+                        continue
+                    elif 15<row['dt'].minute <30:
+                        continue
+                    else:
+                        raise ValueError(
+                            f'M1 to M15 unkown error, start datetime:{tmp_hour} {tmp_minute} end datetime:{row["dt"].hour} {row["dt"].minute}')
+                elif 30<=tmp_minute<45:
+                    if row['dt'].minute >=45 and row['dt'].hour == tmp_hour:
+                        tmp_dict['dt'].append(tmp_datetime)
+                        tmp_dict['open'].append(tmp_open)
+                        tmp_dict['high'].append(tmp_high)
+                        tmp_dict['low'].append(tmp_low)
+                        tmp_dict['close'].append(tmp_close)
+                        tmp_open = row['open']
+                        tmp_high = row['high']
+                        tmp_low = row['low']
+                        tmp_datetime = row['dt']
+                        tmp_hour = row['dt'].hour
+                        tmp_minute = row['dt'].minute
+                    elif row['dt'].hour != tmp_hour:
+                        tmp_dict['dt'].append(tmp_datetime)
+                        tmp_dict['open'].append(tmp_open)
+                        tmp_dict['high'].append(tmp_high)
+                        tmp_dict['low'].append(tmp_low)
+                        tmp_dict['close'].append(tmp_close)
+                        tmp_open = row['open']
+                        tmp_high = row['high']
+                        tmp_low = row['low']
+                        tmp_datetime = row['dt']
+                        tmp_hour = row['dt'].hour
+                        tmp_minute = row['dt'].minute
+                    elif 30<row['dt'].minute<45:
+                        continue
+                    else:
+                        raise ValueError(f'M1 to M15 unkown error, start datetime:{tmp_hour} {tmp_minute} end datetime:{row["dt"].hour} {row["dt"].minute}')
+                elif tmp_minute>=45:
+                    if row['dt'].hour != tmp_hour:
+                        tmp_dict['dt'].append(tmp_datetime)
+                        tmp_dict['open'].append(tmp_open)
+                        tmp_dict['high'].append(tmp_high)
+                        tmp_dict['low'].append(tmp_low)
+                        tmp_dict['close'].append(tmp_close)
+                        tmp_open = row['open']
+                        tmp_high = row['high']
+                        tmp_low = row['low']
+                        tmp_datetime = row['dt']
+                        tmp_hour = row['dt'].hour
+                        tmp_minute = row['dt'].minute
+                    elif 45<row['dt'].minute<=59:
+                        continue
+                    else:
+                        raise ValueError(f'M1 to M15 unkown error, start datetime:{tmp_hour} {tmp_minute} end datetime:{row["dt"].hour} {row["dt"].minute}')
+                else:
+                    raise ValueError(f'M1 to M15 unkown error, start datetime:{tmp_hour} {tmp_minute} end datetime:{row["dt"].hour} {row["dt"].minute}')
+        df = DataFrame(tmp_dict, columns=list(tmp_dict.keys()))
+        return df
+    elif time_frame == 'H1':
+        tmp_datetime = None
+        tmp_hour = None
+        tmp_open = None
+        tmp_high = None
+        tmp_low = None
+        tmp_close = None
+
+        for index, row in df.iterrows():
+            if tmp_open is None:
+                tmp_open = row['open']
+                tmp_high = row['high']
+                tmp_low = row['low']
+                tmp_datetime = row['dt']
+                tmp_hour = row['dt'].hour
+            else:
+                if row['high'] > tmp_high:
+                    tmp_high = row['high']
+                if row['low'] < tmp_low:
+                    tmp_low = row['low']
+                tmp_close = row['close']
+
+                if row['dt'].hour != tmp_hour:
+                    tmp_dict['dt'].append(tmp_datetime)
+                    tmp_dict['open'].append(tmp_open)
+                    tmp_dict['high'].append(tmp_high)
+                    tmp_dict['low'].append(tmp_low)
+                    tmp_dict['close'].append(tmp_close)
+                    tmp_open = row['open']
+                    tmp_high = row['high']
+                    tmp_low = row['low']
+                    tmp_datetime = row['dt']
+                    tmp_hour = row['dt'].hour
+                else:
+                    continue
+        df = DataFrame(tmp_dict, columns=list(tmp_dict.keys()))
+        return df
+    elif time_frame == 'D1':
+        tmp_datetime = None
+        tmp_day = None
+        tmp_open = None
+        tmp_high = None
+        tmp_low = None
+        tmp_close = None
+
+        for index, row in df.iterrows():
+            if tmp_day is None:
+                tmp_open = row['open']
+                tmp_high = row['high']
+                tmp_low = row['low']
+                tmp_datetime = row['dt']
+                tmp_day = row['dt'].day
+            else:
+                if row['high'] > tmp_high:
+                    tmp_high = row['high']
+                if row['low'] < tmp_low:
+                    tmp_low = row['low']
+                tmp_close = row['close']
+
+                if row['dt'].day != tmp_day:
+                    tmp_dict['dt'].append(tmp_datetime)
+                    tmp_dict['open'].append(tmp_open)
+                    tmp_dict['high'].append(tmp_high)
+                    tmp_dict['low'].append(tmp_low)
+                    tmp_dict['close'].append(tmp_close)
+                    tmp_open = row['open']
+                    tmp_high = row['high']
+                    tmp_low = row['low']
+                    tmp_datetime = row['dt']
+                    tmp_day = row['dt'].day
                 else:
-                    pass
+                    continue
+        df = DataFrame(tmp_dict, columns=list(tmp_dict.keys()))
+        return df
+    elif time_frame == 'W1':
+        tmp_datetime = None
+        tmp_week = None
+        tmp_open = None
+        tmp_high = None
+        tmp_low = None
+        tmp_close = None
 
+        for index, row in df.iterrows():
+            if tmp_week is None:
+                tmp_open = row['open']
+                tmp_high = row['high']
+                tmp_low = row['low']
+                tmp_datetime = row['dt']
+                tmp_week = row['dt'].week
+            else:
+                if row['high'] > tmp_high:
+                    tmp_high = row['high']
+                if row['low'] < tmp_low:
+                    tmp_low = row['low']
+                tmp_close = row['close']
+
+                if row['dt'].week != tmp_week:
+                    tmp_dict['dt'].append(tmp_datetime)
+                    tmp_dict['open'].append(tmp_open)
+                    tmp_dict['high'].append(tmp_high)
+                    tmp_dict['low'].append(tmp_low)
+                    tmp_dict['close'].append(tmp_close)
+                    tmp_open = row['open']
+                    tmp_high = row['high']
+                    tmp_low = row['low']
+                    tmp_datetime = row['dt']
+                    tmp_week = row['dt'].week
+                else:
+                    continue
+        df = DataFrame(tmp_dict, columns=list(tmp_dict.keys()))
+        return df
     else:
-        pass
\ No newline at end of file
+        pass
diff --git a/python/fastquant/data/forex/tests.py b/python/fastquant/data/forex/tests.py
new file mode 100644
index 00000000..453ceb27
--- /dev/null
+++ b/python/fastquant/data/forex/tests.py
@@ -0,0 +1,44 @@
+import pickle
+import time
+
+from fastquant.data.forex.forextester import *
+
+
+def get_forex_data_to_pickle(symbol, size):
+    res_dict = get_forextester_data(symbol)[0:size]
+    with open(f'{symbol}_pickle.pickle', 'wb') as file:
+        pickle.dump(res_dict, file, True)
+
+
+def read_forex_data_from_pickle(symbol):
+    with open(f'{symbol}_pickle.pickle', 'rb') as file:
+        df = pickle.load(file)
+    return df
+
+
+def test_shape_data_from_1min_to_other(df):
+    return shape_data_from_1min_to_other(df, time_frame='D1')
+
+
+def gen_candle_chart(df, start_time=None, end_time=None):
+    import mplfinance as mpf
+    reformatted_data = dict()
+    reformatted_data['Date'] = []
+    reformatted_data['Open'] = []
+    reformatted_data['High'] = []
+    reformatted_data['Low'] = []
+    reformatted_data['Close'] = []
+
+    for index, row in df.iterrows():
+        reformatted_data['Date'].append(row['dt'])
+        reformatted_data['Open'].append(row['open'])
+        reformatted_data['High'].append(row['high'])
+        reformatted_data['Low'].append(row['low'])
+        reformatted_data['Close'].append(row['close'])
+
+    pdata = pd.DataFrame.from_dict(reformatted_data)
+    pdata.set_index('Date', inplace=True)
+    if start_time:
+        pdata = pdata.loc[start_time:end_time, :]
+    mpf.plot(pdata, type='candle')
+

From e496a816b053aac1c2c958d92ff4854035d184c8 Mon Sep 17 00:00:00 2001
From: Boreas514 <zio_liu@qq.com>
Date: Tue, 6 Oct 2020 13:40:08 +0800
Subject: [PATCH 3/6] fix bugs, use mulitprocess, write readme

---
 README.md                                  |  37 ++++-
 python/fastquant/data/forex/forextester.py | 162 ++++++++++++++++-----
 python/fastquant/data/forex/tests.py       | 145 ++++++++++++++----
 3 files changed, 277 insertions(+), 67 deletions(-)

diff --git a/README.md b/README.md
index 8aa66999..2904c823 100644
--- a/README.md
+++ b/README.md
@@ -8,11 +8,13 @@
 **fastquant** allows you to easily backtest investment strategies with as few as 3 lines of python code. Its goal is to promote data driven investments by making quantitative analysis in finance accessible to everyone.
 
 ## Features
-1. Easily access historical stock data
+1. Easily access historical stock and forex data
 2. Backtest and optimize trading strategies with only 3 lines of code
 
 <sup>`*` - Both Yahoo Finance and Philippine stock data data are accessible straight from fastquant<sup>
 
+<sup>`*` - Support one minute foreign exchange data from Forextester.com and conversion to other time scales<sup>
+
 Check out our blog posts in the fastquant [website](https://enzoampil.github.io/fastquant-blog/) and this intro [article](https://towardsdatascience.com/backtest-your-trading-strategy-with-only-3-lines-of-python-3859b4a4ab44?source=friends_link&sk=ec647b6bb43fe322013248fd1d473015) on Medium!
 
 ## Installation
@@ -110,6 +112,39 @@ get_crypto_data("BTCUSDT", "2019-01-01", "2019-03-01")
 
 *R does NOT have support for backtesting yet*
 
+## Get forex data
+
+All symbols from [forextester.com](https://forextester.com/) are accessible via `get_forex_data`.This mehtod fetch one minute data from source and shape it to other time frame, M1, M15, H1, D1, W1 are supported time frame now.
+
+### Python
+First, call `get_forex_data` with the following parameters:
+```
+from fastquant import get_forex_data
+df = get_forex_data('EURUSD', time_frame='D1', read_from_local=False)
+print(df)
+                      dt    open    high     low   close
+    2001-01-02 23:01:00  0.9507  0.9509  0.9505  0.9507
+    2001-01-03 00:00:00  0.9506  0.9569  0.9262  0.9283
+    2001-01-04 00:00:00  0.9283  0.9536  0.9283  0.9532
+    2001-01-05 00:00:00  0.9532  0.9591  0.9464  0.9583
+    2001-01-07 23:02:00  0.9583  0.9585  0.9576  0.9584
+                  ...     ...     ...     ...     ...
+    2020-08-25 00:00:00  1.1792  1.1841  1.1784  1.1833
+    2020-08-26 00:00:00  1.1833  1.1839  1.1772  1.1839
+    2020-08-27 00:00:00  1.1839  1.1895  1.1767  1.1819
+    2020-08-28 00:00:00  1.1819  1.1918  1.1810  1.1906
+    2020-08-30 22:00:00  1.1906  1.1918  1.1901  1.1916
+[6127 rows x 5 columns]
+```
+this process download online data, transfer it, save cache files in disk, first call will need 25 minutes or more and 7GB free RAM to generate cache files.
+
+After then, set `read_from_local` to True and you get a second response if don't need update local data to newest.
+
+### R
+
+```
+pass
+```
 ## Backtest trading strategies
 
 *Note: Support for backtesting in R is pending*
diff --git a/python/fastquant/data/forex/forextester.py b/python/fastquant/data/forex/forextester.py
index 98c84ba4..24b32030 100644
--- a/python/fastquant/data/forex/forextester.py
+++ b/python/fastquant/data/forex/forextester.py
@@ -3,6 +3,7 @@
 import zipfile
 import logging
 import pickle
+import multiprocessing
 
 import requests
 import pandas as pd
@@ -52,24 +53,20 @@ def get_forextester_data(symbol, start_date=None, end_date=None,  time_frame='D1
     res = requests.get(f'http://www.forextester.com/templates/data/files/{symbol}.zip', stream=True)
     with open(Path(DATA_PATH, f'{symbol}.zip'), 'wb') as forex_zip_file:
         for chunk in res.iter_content(chunk_size=4096):
-            file_size+=4096
-            if file_size<1048576:
-                logger.info(f'download {round(file_size/1024)}KB')
-            else:
-                logger.info(f'download {round(file_size/1048576)}MB')
+            file_size += 4096
             forex_zip_file.write(chunk)
-    logger.info('download success')
+    logger.info(f'download {symbol} success')
     zip_file = zipfile.ZipFile(Path(DATA_PATH, f'{symbol}.zip'))
     zip_file.extractall(Path(DATA_PATH))
     logger.info('unzip success')
 
-    forex_dataframe = pd.read_csv(
+    forex_df = pd.read_csv(
         Path(DATA_PATH, f'{symbol}.txt'),
         dtype={'<DTYYYYMMDD>': str, '<TIME>': str, '<OPEN>': float, '<HIGH>': float, '<LOW>': float, '<CLOSE>': float}
     )
     logger.info('load txt success')
-    del(forex_dataframe['<TICKER>'])
-    del(forex_dataframe['<VOL>'])
+    del(forex_df['<TICKER>'])
+    del(forex_df['<VOL>'])
     rename_list = [
         'date',
         'time',
@@ -86,47 +83,58 @@ def get_forextester_data(symbol, start_date=None, end_date=None,  time_frame='D1
         '<LOW>': 'low',
         '<CLOSE>': 'close',
     }
-    forex_dataframe = forex_dataframe.rename(columns=rename_dict)[rename_list].drop_duplicates()
+    forex_df = forex_df.rename(columns=rename_dict)[rename_list].drop_duplicates()
     logger.info('change column name success')
-    forex_dataframe['dt'] = forex_dataframe['date'] + forex_dataframe['time']
+    forex_df['dt'] = forex_df['date'] + forex_df['time']
     logger.info('sum date and time success')
-    del(forex_dataframe['date'])
-    del(forex_dataframe['time'])
-    logger.info('start to transfer str datetime to pd.timestamp, this process requires 6 minutes on INTEL-9900k')
-    forex_dataframe['dt'] = pd.to_datetime(forex_dataframe.dt)
-    forex_dataframe = shape_data_from_1min_to_other(forex_dataframe, time_frame)
-    logger.info('start write backup file to disk')
-    with open(f'{symbol}_{time_frame}.pickle', 'wb') as df_file:
-        pickle.dump(forex_dataframe, df_file, True)
+    del(forex_df['date'])
+    del(forex_df['time'])
+    logger.info('start transfer str datetime to pd.timestamp, this step needs 6 minutes on INTEL-9900k')
+    forex_df['dt'] = pd.to_datetime(forex_df.dt)
+    logger.info('transfer is done')
+    # make other timeframe pickle
+    logger.info('start shape M1 data to other time frame, this step needs 15 minutes and 7GB RAM free')
+    multi_jobs = [
+        multiprocessing.Process(target=shape_data_from_1min_to_other, args=((symbol, forex_df, _)))
+        for _ in {"M1", "M15", "H1", "D1", "W1"} - {time_frame}
+    ]
+    for job in multi_jobs:
+        job.start()
+    while multi_jobs:
+        multi_jobs.pop().join()
+    logger.info('shape data to other time frame is done')
+
+    # make goal timeframe pickle
+    logger.info(f'start shape {symbol} to {time_frame}')
+    forex_df = shape_data_from_1min_to_other(symbol, forex_df, time_frame)
     logger.info('all is done')
-    if start_date is None:
-        return forex_dataframe
-    else:
-        forex_dataframe = forex_dataframe[(forex_dataframe['dt'] >= start_date) & (forex_dataframe['dt'] <= end_date)]
-        return forex_dataframe
+
+    if start_date is not None:
+        forex_df = forex_df[(forex_df['dt'] >= start_date) & (forex_df['dt'] <= end_date)]
+    return forex_df.set_index('dt')
 
 
 def get_local_data(symbol, start_date=None, end_date=None,  time_frame='D1'):
     if start_date is not None:
         start_date = datestring_to_datetime(start_date)
         end_date = datestring_to_datetime(end_date)
-    logger.info('start load pickle to memory')
+    logger.info(f'start load {symbol} pickle to memory')
     try:
-        with open(f'{symbol}_{time_frame}.pickle', 'rb') as df_file:
+        with open(Path(DATA_PATH, f'{symbol}_{time_frame}.pickle'), 'rb') as df_file:
             df = pickle.load(df_file)
     except Exception as e:
         raise FileNotFoundError('backup file not found, try get_forextester_data method first')
     if start_date is None:
-        return df
+        return df.set_index('dt')
     else:
         df = df[(df['dt'] >= start_date) & (df['dt'] <= end_date)]
-        return df
+        return df.set_index('dt')
 
 
-def shape_data_from_1min_to_other(df, time_frame):
+def shape_data_from_1min_to_other(symbol, df, time_frame, has_pickle=True):
     if time_frame not in ["M1", "M15", "H1", "D1", "W1"]:
         raise ValueError('time_frame must in this list:["M1", "M15", "H1", "D1", "W1"]')
-
+    logger.info(f'start shape {symbol} to {time_frame}')
     tmp_dict = {
         'dt': [],
         'open': [],
@@ -136,9 +144,10 @@ def shape_data_from_1min_to_other(df, time_frame):
     }
 
     if time_frame == 'M1':
-        return df
+        pass
     elif time_frame == 'M15':
         tmp_datetime = None
+        tmp_day = None
         tmp_hour = None
         tmp_minute = None
         tmp_open = None
@@ -154,6 +163,7 @@ def shape_data_from_1min_to_other(df, time_frame):
                 tmp_datetime = row['dt']
                 tmp_hour = row['dt'].hour
                 tmp_minute = row['dt'].minute
+                tmp_day = row['dt'].day
             else:
                 if row['high']>tmp_high:
                     tmp_high = row['high']
@@ -175,6 +185,7 @@ def shape_data_from_1min_to_other(df, time_frame):
                         tmp_datetime = row['dt']
                         tmp_hour = row['dt'].hour
                         tmp_minute = row['dt'].minute
+                        tmp_day = row['dt'].day
                         continue
                     # discontinuous situation
                     elif row['dt'].hour != tmp_hour:
@@ -189,9 +200,23 @@ def shape_data_from_1min_to_other(df, time_frame):
                         tmp_datetime = row['dt']
                         tmp_hour = row['dt'].hour
                         tmp_minute = row['dt'].minute
+                        tmp_day = row['dt'].day
                         continue
                     elif row['dt'].minute <15:
                         continue
+                    elif row['dt'].day != tmp_day:
+                        tmp_dict['dt'].append(tmp_datetime)
+                        tmp_dict['open'].append(tmp_open)
+                        tmp_dict['high'].append(tmp_high)
+                        tmp_dict['low'].append(tmp_low)
+                        tmp_dict['close'].append(tmp_close)
+                        tmp_open = row['open']
+                        tmp_high = row['high']
+                        tmp_low = row['low']
+                        tmp_datetime = row['dt']
+                        tmp_hour = row['dt'].hour
+                        tmp_minute = row['dt'].minute
+                        tmp_day = row['dt'].day
                     else:
                         raise ValueError(f'M1 to M15 unkown error, start datetime:{tmp_hour} {tmp_minute} end datetime:{row["dt"].hour} {row["dt"].minute}')
                 elif 15<=tmp_minute<30:
@@ -207,6 +232,7 @@ def shape_data_from_1min_to_other(df, time_frame):
                         tmp_datetime = row['dt']
                         tmp_hour = row['dt'].hour
                         tmp_minute = row['dt'].minute
+                        tmp_day = row['dt'].day
                         continue
                     elif row['dt'].hour != tmp_hour:
                         tmp_dict['dt'].append(tmp_datetime)
@@ -220,9 +246,23 @@ def shape_data_from_1min_to_other(df, time_frame):
                         tmp_datetime = row['dt']
                         tmp_hour = row['dt'].hour
                         tmp_minute = row['dt'].minute
+                        tmp_day = row['dt'].day
                         continue
                     elif 15<row['dt'].minute <30:
                         continue
+                    elif row['dt'].day != tmp_day:
+                        tmp_dict['dt'].append(tmp_datetime)
+                        tmp_dict['open'].append(tmp_open)
+                        tmp_dict['high'].append(tmp_high)
+                        tmp_dict['low'].append(tmp_low)
+                        tmp_dict['close'].append(tmp_close)
+                        tmp_open = row['open']
+                        tmp_high = row['high']
+                        tmp_low = row['low']
+                        tmp_datetime = row['dt']
+                        tmp_hour = row['dt'].hour
+                        tmp_minute = row['dt'].minute
+                        tmp_day = row['dt'].day
                     else:
                         raise ValueError(
                             f'M1 to M15 unkown error, start datetime:{tmp_hour} {tmp_minute} end datetime:{row["dt"].hour} {row["dt"].minute}')
@@ -239,6 +279,7 @@ def shape_data_from_1min_to_other(df, time_frame):
                         tmp_datetime = row['dt']
                         tmp_hour = row['dt'].hour
                         tmp_minute = row['dt'].minute
+                        tmp_day = row['dt'].day
                     elif row['dt'].hour != tmp_hour:
                         tmp_dict['dt'].append(tmp_datetime)
                         tmp_dict['open'].append(tmp_open)
@@ -251,8 +292,22 @@ def shape_data_from_1min_to_other(df, time_frame):
                         tmp_datetime = row['dt']
                         tmp_hour = row['dt'].hour
                         tmp_minute = row['dt'].minute
+                        tmp_day = row['dt'].day
                     elif 30<row['dt'].minute<45:
                         continue
+                    elif row['dt'].day != tmp_day:
+                        tmp_dict['dt'].append(tmp_datetime)
+                        tmp_dict['open'].append(tmp_open)
+                        tmp_dict['high'].append(tmp_high)
+                        tmp_dict['low'].append(tmp_low)
+                        tmp_dict['close'].append(tmp_close)
+                        tmp_open = row['open']
+                        tmp_high = row['high']
+                        tmp_low = row['low']
+                        tmp_datetime = row['dt']
+                        tmp_hour = row['dt'].hour
+                        tmp_minute = row['dt'].minute
+                        tmp_day = row['dt'].day
                     else:
                         raise ValueError(f'M1 to M15 unkown error, start datetime:{tmp_hour} {tmp_minute} end datetime:{row["dt"].hour} {row["dt"].minute}')
                 elif tmp_minute>=45:
@@ -268,14 +323,27 @@ def shape_data_from_1min_to_other(df, time_frame):
                         tmp_datetime = row['dt']
                         tmp_hour = row['dt'].hour
                         tmp_minute = row['dt'].minute
+                        tmp_day = row['dt'].day
                     elif 45<row['dt'].minute<=59:
                         continue
+                    elif row['dt'].day != tmp_day:
+                        tmp_dict['dt'].append(tmp_datetime)
+                        tmp_dict['open'].append(tmp_open)
+                        tmp_dict['high'].append(tmp_high)
+                        tmp_dict['low'].append(tmp_low)
+                        tmp_dict['close'].append(tmp_close)
+                        tmp_open = row['open']
+                        tmp_high = row['high']
+                        tmp_low = row['low']
+                        tmp_datetime = row['dt']
+                        tmp_hour = row['dt'].hour
+                        tmp_minute = row['dt'].minute
+                        tmp_day = row['dt'].day
                     else:
                         raise ValueError(f'M1 to M15 unkown error, start datetime:{tmp_hour} {tmp_minute} end datetime:{row["dt"].hour} {row["dt"].minute}')
                 else:
                     raise ValueError(f'M1 to M15 unkown error, start datetime:{tmp_hour} {tmp_minute} end datetime:{row["dt"].hour} {row["dt"].minute}')
         df = DataFrame(tmp_dict, columns=list(tmp_dict.keys()))
-        return df
     elif time_frame == 'H1':
         tmp_datetime = None
         tmp_hour = None
@@ -312,7 +380,6 @@ def shape_data_from_1min_to_other(df, time_frame):
                 else:
                     continue
         df = DataFrame(tmp_dict, columns=list(tmp_dict.keys()))
-        return df
     elif time_frame == 'D1':
         tmp_datetime = None
         tmp_day = None
@@ -349,7 +416,6 @@ def shape_data_from_1min_to_other(df, time_frame):
                 else:
                     continue
         df = DataFrame(tmp_dict, columns=list(tmp_dict.keys()))
-        return df
     elif time_frame == 'W1':
         tmp_datetime = None
         tmp_week = None
@@ -386,6 +452,32 @@ def shape_data_from_1min_to_other(df, time_frame):
                 else:
                     continue
         df = DataFrame(tmp_dict, columns=list(tmp_dict.keys()))
-        return df
     else:
         pass
+    if has_pickle:
+        with open(Path(DATA_PATH, f'{symbol}_{time_frame}.pickle'), 'wb') as df_file:
+            pickle.dump(df, df_file, True)
+    return df
+
+
+def gen_candle_chart(df, start_time=None, end_time=None):
+    import mplfinance as mpf
+    reformatted_data = dict()
+    reformatted_data['Date'] = []
+    reformatted_data['Open'] = []
+    reformatted_data['High'] = []
+    reformatted_data['Low'] = []
+    reformatted_data['Close'] = []
+
+    for index, row in df.iterrows():
+        reformatted_data['Date'].append(row['dt'])
+        reformatted_data['Open'].append(row['open'])
+        reformatted_data['High'].append(row['high'])
+        reformatted_data['Low'].append(row['low'])
+        reformatted_data['Close'].append(row['close'])
+
+    pdata = pd.DataFrame.from_dict(reformatted_data)
+    pdata.set_index('Date', inplace=True)
+    if start_time:
+        pdata = pdata.loc[start_time:end_time, :]
+    mpf.plot(pdata, type='candle')
diff --git a/python/fastquant/data/forex/tests.py b/python/fastquant/data/forex/tests.py
index 453ceb27..022df599 100644
--- a/python/fastquant/data/forex/tests.py
+++ b/python/fastquant/data/forex/tests.py
@@ -1,44 +1,127 @@
-import pickle
-import time
+import unittest
+import multiprocessing
 
 from fastquant.data.forex.forextester import *
+from fastquant.data.forex.forex import get_forex_data
 
+class TestForexMethod(unittest.TestCase):
+    allowed_symbol_list = [
+        'AUDJPY',
+        'AUDUSD',
+        'CHFJPY',
+        'EURCAD',
+        'EURCHF',
+        'EURGBP',
+        'EURJPY',
+        'EURUSD',
+        'GBPCHF',
+        'GBPJPY',
+        'GBPUSD',
+        'NZDJPY',
+        'NZDUSD',
+        'USDCAD',
+        'USDJPY',
+        'USDCHF',
+        'XAGUSD',
+        'XAUUSD'
+    ]
 
-def get_forex_data_to_pickle(symbol, size):
-    res_dict = get_forextester_data(symbol)[0:size]
-    with open(f'{symbol}_pickle.pickle', 'wb') as file:
-        pickle.dump(res_dict, file, True)
+    def test_download_and_unzip(self):
+        for symbol in self.allowed_symbol_list:
+            logger.info(f'start downloading forex {symbol} data zip file...')
+            file_size = 0
+            res = requests.get(f'http://www.forextester.com/templates/data/files/{symbol}.zip', stream=True)
+            with open(Path(DATA_PATH, f'{symbol}.zip'), 'wb') as forex_zip_file:
+                for chunk in res.iter_content(chunk_size=4096):
+                    file_size += 4096
+                    if file_size < 1048576:
+                        logger.info(f'download {round(file_size / 1024)}KB')
+                    else:
+                        logger.info(f'download {round(file_size / 1048576)}MB')
+                    forex_zip_file.write(chunk)
+            logger.info('download success')
+            zip_file = zipfile.ZipFile(Path(DATA_PATH, f'{symbol}.zip'))
+            zip_file.extractall(Path(DATA_PATH))
+            logger.info('unzip success')
 
+    def test_shape_and_make_tmp(self):
+        '''
+        transfer txt to pd.timestamp 6min
+        M1 to H1 11min
+        M1 to M15 15min
+        M1 to W1 11min
+        M1 to D1 13min
 
-def read_forex_data_from_pickle(symbol):
-    with open(f'{symbol}_pickle.pickle', 'rb') as file:
-        df = pickle.load(file)
-    return df
+        :return:
+        '''
+        for symbol in self.allowed_symbol_list:
+            time_frame = 'D1'
+            forex_dataframe = pd.read_csv(
+                Path(DATA_PATH, f'{symbol}.txt'),
+                dtype={'<DTYYYYMMDD>': str, '<TIME>': str, '<OPEN>': float, '<HIGH>': float, '<LOW>': float,
+                       '<CLOSE>': float}
+            )
+            logger.info('load txt success')
+            del (forex_dataframe['<TICKER>'])
+            del (forex_dataframe['<VOL>'])
+            rename_list = [
+                'date',
+                'time',
+                'open',
+                'high',
+                'low',
+                'close',
+            ]
+            rename_dict = {
+                '<DTYYYYMMDD>': 'date',
+                '<TIME>': 'time',
+                '<OPEN>': 'open',
+                '<HIGH>': 'high',
+                '<LOW>': 'low',
+                '<CLOSE>': 'close',
+            }
+            forex_dataframe = forex_dataframe.rename(columns=rename_dict)[rename_list].drop_duplicates()
+            logger.info('change column name success')
+            forex_dataframe['dt'] = forex_dataframe['date'] + forex_dataframe['time']
+            logger.info('sum date and time success')
+            del (forex_dataframe['date'])
+            del (forex_dataframe['time'])
+            logger.info('start transfer str datetime to pd.timestamp, this step requires 6 minutes on INTEL-9900k')
+            forex_dataframe['dt'] = pd.to_datetime(forex_dataframe.dt)
+            logger.info('transfer is done')
 
+            # make other timeframe pickle
+            logger.info('start shape M1 data to other time frame, this step need 7GB RAM free')
+            multi_jobs = [
+                multiprocessing.Process(target=shape_data_from_1min_to_other, args=((symbol, forex_dataframe, _)))
+                for _ in {"M1", "M15", "H1", "D1", "W1"} - {time_frame}
+            ]
+            for job in multi_jobs:
+                job.start()
+            while multi_jobs:
+                multi_jobs.pop().join()
+            logger.info('shape other is done')
 
-def test_shape_data_from_1min_to_other(df):
-    return shape_data_from_1min_to_other(df, time_frame='D1')
+            # make goal timeframe pickle
+            logger.info(f'start shape {symbol} to {time_frame}')
+            forex_dataframe = shape_data_from_1min_to_other(symbol, forex_dataframe, time_frame)
 
+            logger.info('all is done')
 
-def gen_candle_chart(df, start_time=None, end_time=None):
-    import mplfinance as mpf
-    reformatted_data = dict()
-    reformatted_data['Date'] = []
-    reformatted_data['Open'] = []
-    reformatted_data['High'] = []
-    reformatted_data['Low'] = []
-    reformatted_data['Close'] = []
+    def test_combining_test(self):
+        from fastquant import backtest
+        df = get_forex_data('EURUSD')
+        backtest('smac', df, fast_period=20, slow_period=40)
 
-    for index, row in df.iterrows():
-        reformatted_data['Date'].append(row['dt'])
-        reformatted_data['Open'].append(row['open'])
-        reformatted_data['High'].append(row['high'])
-        reformatted_data['Low'].append(row['low'])
-        reformatted_data['Close'].append(row['close'])
+        df = get_forex_data('EURUSD', start_date='2018-01-01', end_date='2019-01-01')
+        backtest('smac', df, fast_period=20, slow_period=40)
 
-    pdata = pd.DataFrame.from_dict(reformatted_data)
-    pdata.set_index('Date', inplace=True)
-    if start_time:
-        pdata = pdata.loc[start_time:end_time, :]
-    mpf.plot(pdata, type='candle')
+        df = get_forex_data('EURUSD', read_from_local=True)
+        backtest('smac', df, fast_period=20, slow_period=40)
 
+        df = get_forex_data('EURUSD', start_date='2018-01-01', end_date='2019-01-01', read_from_local=True)
+        backtest('smac', df, fast_period=20, slow_period=40)
+
+
+if __name__ == '__main__':
+    unittest.main()

From 4073e301de3da052c9d5372472b290d9fd78b565 Mon Sep 17 00:00:00 2001
From: Boreas514 <zio_liu@qq.com>
Date: Tue, 13 Oct 2020 20:24:45 +0800
Subject: [PATCH 4/6] repair work

---
 python/fastquant/data/__init__.py             |  3 +++
 python/fastquant/data/forex/forextester.py    |  1 +
 .../forex/tests.py => tests/tset_forex.py}    | 24 ++++++++++---------
 3 files changed, 17 insertions(+), 11 deletions(-)
 rename python/{fastquant/data/forex/tests.py => tests/tset_forex.py} (95%)

diff --git a/python/fastquant/data/__init__.py b/python/fastquant/data/__init__.py
index 12cb1ab6..eca5535a 100644
--- a/python/fastquant/data/__init__.py
+++ b/python/fastquant/data/__init__.py
@@ -25,3 +25,6 @@
 # Twitter
 from fastquant.data.web.twitter import tweepy_api
 from fastquant.data.web.twitter import get_twitter_sentiment
+
+# Forex
+from fastquant.data.forex.forex import get_forex_data
\ No newline at end of file
diff --git a/python/fastquant/data/forex/forextester.py b/python/fastquant/data/forex/forextester.py
index 24b32030..3187c1b8 100644
--- a/python/fastquant/data/forex/forextester.py
+++ b/python/fastquant/data/forex/forextester.py
@@ -461,6 +461,7 @@ def shape_data_from_1min_to_other(symbol, df, time_frame, has_pickle=True):
 
 
 def gen_candle_chart(df, start_time=None, end_time=None):
+    df = df.reset_index()
     import mplfinance as mpf
     reformatted_data = dict()
     reformatted_data['Date'] = []
diff --git a/python/fastquant/data/forex/tests.py b/python/tests/tset_forex.py
similarity index 95%
rename from python/fastquant/data/forex/tests.py
rename to python/tests/tset_forex.py
index 022df599..2a8ef0ea 100644
--- a/python/fastquant/data/forex/tests.py
+++ b/python/tests/tset_forex.py
@@ -43,7 +43,7 @@ def test_download_and_unzip(self):
             zip_file = zipfile.ZipFile(Path(DATA_PATH, f'{symbol}.zip'))
             zip_file.extractall(Path(DATA_PATH))
             logger.info('unzip success')
-
+    
     def test_shape_and_make_tmp(self):
         '''
         transfer txt to pd.timestamp 6min
@@ -51,7 +51,7 @@ def test_shape_and_make_tmp(self):
         M1 to M15 15min
         M1 to W1 11min
         M1 to D1 13min
-
+    
         :return:
         '''
         for symbol in self.allowed_symbol_list:
@@ -89,7 +89,7 @@ def test_shape_and_make_tmp(self):
             logger.info('start transfer str datetime to pd.timestamp, this step requires 6 minutes on INTEL-9900k')
             forex_dataframe['dt'] = pd.to_datetime(forex_dataframe.dt)
             logger.info('transfer is done')
-
+    
             # make other timeframe pickle
             logger.info('start shape M1 data to other time frame, this step need 7GB RAM free')
             multi_jobs = [
@@ -101,27 +101,29 @@ def test_shape_and_make_tmp(self):
             while multi_jobs:
                 multi_jobs.pop().join()
             logger.info('shape other is done')
-
+    
             # make goal timeframe pickle
             logger.info(f'start shape {symbol} to {time_frame}')
             forex_dataframe = shape_data_from_1min_to_other(symbol, forex_dataframe, time_frame)
-
+    
             logger.info('all is done')
-
+    
     def test_combining_test(self):
         from fastquant import backtest
         df = get_forex_data('EURUSD')
         backtest('smac', df, fast_period=20, slow_period=40)
-
+    
         df = get_forex_data('EURUSD', start_date='2018-01-01', end_date='2019-01-01')
         backtest('smac', df, fast_period=20, slow_period=40)
-
+    
         df = get_forex_data('EURUSD', read_from_local=True)
         backtest('smac', df, fast_period=20, slow_period=40)
-
+    
         df = get_forex_data('EURUSD', start_date='2018-01-01', end_date='2019-01-01', read_from_local=True)
         backtest('smac', df, fast_period=20, slow_period=40)
-
-
+        
+    # def test_aaa(self):
+    #     df = get_forex_data('GBPUSD', start_date='2019-08-25', end_date='2019-09-10', time_frame='D1',read_from_local=True)
+    #     gen_candle_chart(df)
 if __name__ == '__main__':
     unittest.main()

From 3cf0a9a5bd48cb7db2812028c1bbe8487e2bf249 Mon Sep 17 00:00:00 2001
From: Jerome de Leon <jpdeleon.bsap@gmail.com>
Date: Sat, 17 Oct 2020 18:44:03 +0900
Subject: [PATCH 5/6] Rename tset_forex.py to test_forex.py

---
 python/tests/{tset_forex.py => test_forex.py} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename python/tests/{tset_forex.py => test_forex.py} (100%)

diff --git a/python/tests/tset_forex.py b/python/tests/test_forex.py
similarity index 100%
rename from python/tests/tset_forex.py
rename to python/tests/test_forex.py

From 61731c56c7a61ec31d9906f38be9878bd400c134 Mon Sep 17 00:00:00 2001
From: Boreas514 <zio_liu@qq.com>
Date: Thu, 22 Oct 2020 21:41:05 +0800
Subject: [PATCH 6/6] split function, cancel multi - process

---
 README.md                                  |  10 +-
 python/fastquant/data/forex/forex.py       |  12 +-
 python/fastquant/data/forex/forextester.py | 119 +++++++++----------
 python/tests/test_forex.py                 |   0
 python/tests/tset_forex.py                 | 129 ---------------------
 5 files changed, 68 insertions(+), 202 deletions(-)
 create mode 100644 python/tests/test_forex.py
 delete mode 100644 python/tests/tset_forex.py

diff --git a/README.md b/README.md
index 2904c823..2da01ae9 100644
--- a/README.md
+++ b/README.md
@@ -13,7 +13,7 @@
 
 <sup>`*` - Both Yahoo Finance and Philippine stock data data are accessible straight from fastquant<sup>
 
-<sup>`*` - Support one minute foreign exchange data from Forextester.com and conversion to other time scales<sup>
+<sup>`*` - Support one minute foreign exchange data from Forextester.com and conversion to other time period<sup>
 
 Check out our blog posts in the fastquant [website](https://enzoampil.github.io/fastquant-blog/) and this intro [article](https://towardsdatascience.com/backtest-your-trading-strategy-with-only-3-lines-of-python-3859b4a4ab44?source=friends_link&sk=ec647b6bb43fe322013248fd1d473015) on Medium!
 
@@ -114,13 +114,13 @@ get_crypto_data("BTCUSDT", "2019-01-01", "2019-03-01")
 
 ## Get forex data
 
-All symbols from [forextester.com](https://forextester.com/) are accessible via `get_forex_data`.This mehtod fetch one minute data from source and shape it to other time frame, M1, M15, H1, D1, W1 are supported time frame now.
+All symbols from [forextester.com](https://forextester.com/) are accessible via `get_forex_data`.This method fetch one minute data from source and convert it to other time period, M1, M15, H1, D1, W1 are supported time period now.
 
 ### Python
 First, call `get_forex_data` with the following parameters:
 ```
 from fastquant import get_forex_data
-df = get_forex_data('EURUSD', time_frame='D1', read_from_local=False)
+df = get_forex_data('EURUSD', period='D1', read_from_local=False)
 print(df)
                       dt    open    high     low   close
     2001-01-02 23:01:00  0.9507  0.9509  0.9505  0.9507
@@ -136,9 +136,9 @@ print(df)
     2020-08-30 22:00:00  1.1906  1.1918  1.1901  1.1916
 [6127 rows x 5 columns]
 ```
-this process download online data, transfer it, save cache files in disk, first call will need 25 minutes or more and 7GB free RAM to generate cache files.
+this process download online data, transfer it, save cache files in disk, first call will need 20 minutes or more and 2.3GB free RAM to generate cache files.
 
-After then, set `read_from_local` to True and you get a second response if don't need update local data to newest.
+After then, set `read_from_local=Ture` and you will get a second response if don't need update local data to newest.
 
 ### R
 
diff --git a/python/fastquant/data/forex/forex.py b/python/fastquant/data/forex/forex.py
index e27afcd0..7e445a29 100644
--- a/python/fastquant/data/forex/forex.py
+++ b/python/fastquant/data/forex/forex.py
@@ -10,7 +10,7 @@
 from fastquant.data.forex.forextester import get_forextester_data, get_local_data
 
 
-def get_forex_data(symbol, start_date=None, end_date=None, source="forextester", time_frame='D1', read_from_local=False):
+def get_forex_data(symbol, start_date=None, end_date=None, source="forextester", period='D1', read_from_local=False):
     """Returns pricing data for a specified forex pair.
 
     Parameters
@@ -26,10 +26,10 @@ def get_forex_data(symbol, start_date=None, end_date=None, source="forextester",
         in most cases we need more wide time period to test strategy, so keep this to None
     source : str
          Source of forex history data
-    time_frame : str
-         time frame you want, support 1 minute,15 minutes,1 hour,1 day,1 week
+    period : str
+         time period you want, support 1 minute,15 minutes,1 hour,1 day,1 week
          this parameter must one of them:["M1", "M15", "H1", "D1", "W1"]
-    read_from_local : bull
+    read_from_local : bool
         if this parameter set False, method get data from online
         if set it to True, method get data from local pickle file, faster than set it to False
 
@@ -40,9 +40,9 @@ def get_forex_data(symbol, start_date=None, end_date=None, source="forextester",
 
     if source == "forextester":
         if read_from_local is False:
-            df = get_forextester_data(symbol, start_date, end_date, time_frame)
+            df = get_forextester_data(symbol, start_date, end_date, period)
         else:
-            df = get_local_data(symbol, start_date, end_date, time_frame)
+            df = get_local_data(symbol, start_date, end_date, period)
     else:
         raise Exception("Source must be forextester")
 
diff --git a/python/fastquant/data/forex/forextester.py b/python/fastquant/data/forex/forextester.py
index 3187c1b8..789e8237 100644
--- a/python/fastquant/data/forex/forextester.py
+++ b/python/fastquant/data/forex/forextester.py
@@ -1,9 +1,7 @@
-import os
 from pathlib import Path
 import zipfile
 import logging
 import pickle
-import multiprocessing
 
 import requests
 import pandas as pd
@@ -19,9 +17,7 @@
 logger.setLevel(logging.INFO)
 logger.addHandler(logger_handler)
 
-
-def get_forextester_data(symbol, start_date=None, end_date=None,  time_frame='D1'):
-    allowed_forex_symbol_list = [
+ALLOWED_SYMBOL_LIST = [
         'AUDJPY',
         'AUDUSD',
         'CHFJPY',
@@ -41,13 +37,9 @@ def get_forextester_data(symbol, start_date=None, end_date=None,  time_frame='D1
         'XAGUSD',
         'XAUUSD'
     ]
-    if symbol not in allowed_forex_symbol_list:
-        raise ValueError('your symbol is not supported by forextester')
 
-    if start_date is not None:
-        start_date = datestring_to_datetime(start_date)
-        end_date = datestring_to_datetime(end_date)
 
+def download_and_unzip_data_from_forextester(symbol):
     logger.info(f'start downloading forex {symbol} data zip file...')
     file_size = 0
     res = requests.get(f'http://www.forextester.com/templates/data/files/{symbol}.zip', stream=True)
@@ -60,13 +52,14 @@ def get_forextester_data(symbol, start_date=None, end_date=None,  time_frame='D1
     zip_file.extractall(Path(DATA_PATH))
     logger.info('unzip success')
 
+def load_data_from_text(symbol):
     forex_df = pd.read_csv(
         Path(DATA_PATH, f'{symbol}.txt'),
         dtype={'<DTYYYYMMDD>': str, '<TIME>': str, '<OPEN>': float, '<HIGH>': float, '<LOW>': float, '<CLOSE>': float}
     )
     logger.info('load txt success')
-    del(forex_df['<TICKER>'])
-    del(forex_df['<VOL>'])
+    del (forex_df['<TICKER>'])
+    del (forex_df['<VOL>'])
     rename_list = [
         'date',
         'time',
@@ -87,54 +80,18 @@ def get_forextester_data(symbol, start_date=None, end_date=None,  time_frame='D1
     logger.info('change column name success')
     forex_df['dt'] = forex_df['date'] + forex_df['time']
     logger.info('sum date and time success')
-    del(forex_df['date'])
-    del(forex_df['time'])
-    logger.info('start transfer str datetime to pd.timestamp, this step needs 6 minutes on INTEL-9900k')
+    del (forex_df['date'])
+    del (forex_df['time'])
+    logger.info('start normalize the datetime of 1-minute chart, this step needs 6 minutes on INTEL-9900k')
     forex_df['dt'] = pd.to_datetime(forex_df.dt)
     logger.info('transfer is done')
-    # make other timeframe pickle
-    logger.info('start shape M1 data to other time frame, this step needs 15 minutes and 7GB RAM free')
-    multi_jobs = [
-        multiprocessing.Process(target=shape_data_from_1min_to_other, args=((symbol, forex_df, _)))
-        for _ in {"M1", "M15", "H1", "D1", "W1"} - {time_frame}
-    ]
-    for job in multi_jobs:
-        job.start()
-    while multi_jobs:
-        multi_jobs.pop().join()
-    logger.info('shape data to other time frame is done')
-
-    # make goal timeframe pickle
-    logger.info(f'start shape {symbol} to {time_frame}')
-    forex_df = shape_data_from_1min_to_other(symbol, forex_df, time_frame)
-    logger.info('all is done')
-
-    if start_date is not None:
-        forex_df = forex_df[(forex_df['dt'] >= start_date) & (forex_df['dt'] <= end_date)]
-    return forex_df.set_index('dt')
-
-
-def get_local_data(symbol, start_date=None, end_date=None,  time_frame='D1'):
-    if start_date is not None:
-        start_date = datestring_to_datetime(start_date)
-        end_date = datestring_to_datetime(end_date)
-    logger.info(f'start load {symbol} pickle to memory')
-    try:
-        with open(Path(DATA_PATH, f'{symbol}_{time_frame}.pickle'), 'rb') as df_file:
-            df = pickle.load(df_file)
-    except Exception as e:
-        raise FileNotFoundError('backup file not found, try get_forextester_data method first')
-    if start_date is None:
-        return df.set_index('dt')
-    else:
-        df = df[(df['dt'] >= start_date) & (df['dt'] <= end_date)]
-        return df.set_index('dt')
+    return forex_df
 
 
-def shape_data_from_1min_to_other(symbol, df, time_frame, has_pickle=True):
-    if time_frame not in ["M1", "M15", "H1", "D1", "W1"]:
+def convert_data_from_1min_to_other(symbol, df, period, has_pickle=True):
+    if period not in ["M1", "M15", "H1", "D1", "W1"]:
         raise ValueError('time_frame must in this list:["M1", "M15", "H1", "D1", "W1"]')
-    logger.info(f'start shape {symbol} to {time_frame}')
+    logger.info(f'start shape {symbol} to {period}, this step needs 13 minutes on INTEL-9900K')
     tmp_dict = {
         'dt': [],
         'open': [],
@@ -143,9 +100,9 @@ def shape_data_from_1min_to_other(symbol, df, time_frame, has_pickle=True):
         'close': []
     }
 
-    if time_frame == 'M1':
+    if period == 'M1':
         pass
-    elif time_frame == 'M15':
+    elif period == 'M15':
         tmp_datetime = None
         tmp_day = None
         tmp_hour = None
@@ -344,7 +301,7 @@ def shape_data_from_1min_to_other(symbol, df, time_frame, has_pickle=True):
                 else:
                     raise ValueError(f'M1 to M15 unkown error, start datetime:{tmp_hour} {tmp_minute} end datetime:{row["dt"].hour} {row["dt"].minute}')
         df = DataFrame(tmp_dict, columns=list(tmp_dict.keys()))
-    elif time_frame == 'H1':
+    elif period == 'H1':
         tmp_datetime = None
         tmp_hour = None
         tmp_open = None
@@ -380,7 +337,7 @@ def shape_data_from_1min_to_other(symbol, df, time_frame, has_pickle=True):
                 else:
                     continue
         df = DataFrame(tmp_dict, columns=list(tmp_dict.keys()))
-    elif time_frame == 'D1':
+    elif period == 'D1':
         tmp_datetime = None
         tmp_day = None
         tmp_open = None
@@ -416,7 +373,7 @@ def shape_data_from_1min_to_other(symbol, df, time_frame, has_pickle=True):
                 else:
                     continue
         df = DataFrame(tmp_dict, columns=list(tmp_dict.keys()))
-    elif time_frame == 'W1':
+    elif period == 'W1':
         tmp_datetime = None
         tmp_week = None
         tmp_open = None
@@ -455,12 +412,50 @@ def shape_data_from_1min_to_other(symbol, df, time_frame, has_pickle=True):
     else:
         pass
     if has_pickle:
-        with open(Path(DATA_PATH, f'{symbol}_{time_frame}.pickle'), 'wb') as df_file:
+        with open(Path(DATA_PATH, f'{symbol}_{period}.pickle'), 'wb') as df_file:
             pickle.dump(df, df_file, True)
     return df
 
 
-def gen_candle_chart(df, start_time=None, end_time=None):
+def get_forextester_data(symbol, start_date=None, end_date=None,  period='D1'):
+
+    if symbol not in ALLOWED_SYMBOL_LIST:
+        raise ValueError('your symbol is not supported by forextester')
+
+    if start_date is not None:
+        start_date = datestring_to_datetime(start_date)
+        end_date = datestring_to_datetime(end_date)
+
+    download_and_unzip_data_from_forextester(symbol)
+    forex_df = load_data_from_text(symbol)
+
+    # make goal timeframe pickle
+    logger.info(f'start shape {symbol} to {period}')
+    forex_df = convert_data_from_1min_to_other(symbol, forex_df, period)
+    logger.info('all is done')
+    if start_date is not None:
+        forex_df = forex_df[(forex_df['dt'] >= start_date) & (forex_df['dt'] <= end_date)]
+    return forex_df.set_index('dt')
+
+
+def get_local_data(symbol, start_date=None, end_date=None,  period='D1'):
+    if start_date is not None:
+        start_date = datestring_to_datetime(start_date)
+        end_date = datestring_to_datetime(end_date)
+    logger.info(f'start load {symbol} pickle to memory')
+    try:
+        with open(Path(DATA_PATH, f'{symbol}_{period}.pickle'), 'rb') as df_file:
+            df = pickle.load(df_file)
+    except Exception as e:
+        raise FileNotFoundError('backup file not found, try get_forex_data method setting read_from_local=False')
+    if start_date is None:
+        return df.set_index('dt')
+    else:
+        df = df[(df['dt'] >= start_date) & (df['dt'] <= end_date)]
+        return df.set_index('dt')
+
+
+def _gen_candle_chart_for_test(df, start_time=None, end_time=None):
     df = df.reset_index()
     import mplfinance as mpf
     reformatted_data = dict()
diff --git a/python/tests/test_forex.py b/python/tests/test_forex.py
new file mode 100644
index 00000000..e69de29b
diff --git a/python/tests/tset_forex.py b/python/tests/tset_forex.py
deleted file mode 100644
index 2a8ef0ea..00000000
--- a/python/tests/tset_forex.py
+++ /dev/null
@@ -1,129 +0,0 @@
-import unittest
-import multiprocessing
-
-from fastquant.data.forex.forextester import *
-from fastquant.data.forex.forex import get_forex_data
-
-class TestForexMethod(unittest.TestCase):
-    allowed_symbol_list = [
-        'AUDJPY',
-        'AUDUSD',
-        'CHFJPY',
-        'EURCAD',
-        'EURCHF',
-        'EURGBP',
-        'EURJPY',
-        'EURUSD',
-        'GBPCHF',
-        'GBPJPY',
-        'GBPUSD',
-        'NZDJPY',
-        'NZDUSD',
-        'USDCAD',
-        'USDJPY',
-        'USDCHF',
-        'XAGUSD',
-        'XAUUSD'
-    ]
-
-    def test_download_and_unzip(self):
-        for symbol in self.allowed_symbol_list:
-            logger.info(f'start downloading forex {symbol} data zip file...')
-            file_size = 0
-            res = requests.get(f'http://www.forextester.com/templates/data/files/{symbol}.zip', stream=True)
-            with open(Path(DATA_PATH, f'{symbol}.zip'), 'wb') as forex_zip_file:
-                for chunk in res.iter_content(chunk_size=4096):
-                    file_size += 4096
-                    if file_size < 1048576:
-                        logger.info(f'download {round(file_size / 1024)}KB')
-                    else:
-                        logger.info(f'download {round(file_size / 1048576)}MB')
-                    forex_zip_file.write(chunk)
-            logger.info('download success')
-            zip_file = zipfile.ZipFile(Path(DATA_PATH, f'{symbol}.zip'))
-            zip_file.extractall(Path(DATA_PATH))
-            logger.info('unzip success')
-    
-    def test_shape_and_make_tmp(self):
-        '''
-        transfer txt to pd.timestamp 6min
-        M1 to H1 11min
-        M1 to M15 15min
-        M1 to W1 11min
-        M1 to D1 13min
-    
-        :return:
-        '''
-        for symbol in self.allowed_symbol_list:
-            time_frame = 'D1'
-            forex_dataframe = pd.read_csv(
-                Path(DATA_PATH, f'{symbol}.txt'),
-                dtype={'<DTYYYYMMDD>': str, '<TIME>': str, '<OPEN>': float, '<HIGH>': float, '<LOW>': float,
-                       '<CLOSE>': float}
-            )
-            logger.info('load txt success')
-            del (forex_dataframe['<TICKER>'])
-            del (forex_dataframe['<VOL>'])
-            rename_list = [
-                'date',
-                'time',
-                'open',
-                'high',
-                'low',
-                'close',
-            ]
-            rename_dict = {
-                '<DTYYYYMMDD>': 'date',
-                '<TIME>': 'time',
-                '<OPEN>': 'open',
-                '<HIGH>': 'high',
-                '<LOW>': 'low',
-                '<CLOSE>': 'close',
-            }
-            forex_dataframe = forex_dataframe.rename(columns=rename_dict)[rename_list].drop_duplicates()
-            logger.info('change column name success')
-            forex_dataframe['dt'] = forex_dataframe['date'] + forex_dataframe['time']
-            logger.info('sum date and time success')
-            del (forex_dataframe['date'])
-            del (forex_dataframe['time'])
-            logger.info('start transfer str datetime to pd.timestamp, this step requires 6 minutes on INTEL-9900k')
-            forex_dataframe['dt'] = pd.to_datetime(forex_dataframe.dt)
-            logger.info('transfer is done')
-    
-            # make other timeframe pickle
-            logger.info('start shape M1 data to other time frame, this step need 7GB RAM free')
-            multi_jobs = [
-                multiprocessing.Process(target=shape_data_from_1min_to_other, args=((symbol, forex_dataframe, _)))
-                for _ in {"M1", "M15", "H1", "D1", "W1"} - {time_frame}
-            ]
-            for job in multi_jobs:
-                job.start()
-            while multi_jobs:
-                multi_jobs.pop().join()
-            logger.info('shape other is done')
-    
-            # make goal timeframe pickle
-            logger.info(f'start shape {symbol} to {time_frame}')
-            forex_dataframe = shape_data_from_1min_to_other(symbol, forex_dataframe, time_frame)
-    
-            logger.info('all is done')
-    
-    def test_combining_test(self):
-        from fastquant import backtest
-        df = get_forex_data('EURUSD')
-        backtest('smac', df, fast_period=20, slow_period=40)
-    
-        df = get_forex_data('EURUSD', start_date='2018-01-01', end_date='2019-01-01')
-        backtest('smac', df, fast_period=20, slow_period=40)
-    
-        df = get_forex_data('EURUSD', read_from_local=True)
-        backtest('smac', df, fast_period=20, slow_period=40)
-    
-        df = get_forex_data('EURUSD', start_date='2018-01-01', end_date='2019-01-01', read_from_local=True)
-        backtest('smac', df, fast_period=20, slow_period=40)
-        
-    # def test_aaa(self):
-    #     df = get_forex_data('GBPUSD', start_date='2019-08-25', end_date='2019-09-10', time_frame='D1',read_from_local=True)
-    #     gen_candle_chart(df)
-if __name__ == '__main__':
-    unittest.main()