Merge 34e117b into 6e4a2f0

ivopetiz · Feb 2, 2019 · 1f7b8dc · 1f7b8dc
2 parents 6e4a2f0 + 34e117b
commit 1f7b8dc
Show file tree

Hide file tree

Showing 4 changed files with 63 additions and 20 deletions.
diff --git a/.coverage b/.coverage
@@ -1 +1 @@
-!coverage.py: This is a private format, don't read it directly!{"lines":{"/home/petiz/Dropbox/Code/python/crypto-algotrading/cryptoalgotrading/var.py":[8,10,13,14,19,20,24,26,27,28,30,33,35,38,40,42,43,46,48,49,50,51,52,54,55,58,60,62,63,65,67],"/home/petiz/Dropbox/Code/python/crypto-algotrading/cryptoalgotrading/cryptoalgotrading.py":[512,513,514,515,516,5,518,519,520,521,522,523,525,526,528,18,19,20,533,22,23,24,25,26,27,28,29,30,31,39,44,47,50,563,52,53,54,55,569,570,572,573,575,577,578,579,68,70,585,74,587,76,590,591,81,84,85,612,90,91,92,94,613,100,101,102,103,104,105,106,619,530,622,624,625,627,628,630,617,635,636,618,638,639,640,641,642,643,644,133,646,647,649,651,652,653,142,621,656,145,658,147,148,150,663,664,153,154,160,517,686,688,691,187,188,189,191,194,195,197,198,200,201,631,204,206,207,210,214,215,217,218,219,220,221,222,223,225,226,228,230,231,232,235,237,244,245,247,252,253,254,255,256,257,258,129,131,135,51,137,565,614,141,684,655,144,65,152,67,433,436,437,438,439,440,441,442,443,444,445,446,75,78,203,476,478,479,484,487,498,499,500,503,505,508,511],"/home/petiz/Dropbox/Code/python/crypto-algotrading/cryptoalgotrading/aux.py":[512,5,8,9,10,87,12,13,14,15,16,17,18,19,20,21,534,23,25,539,28,29,30,31,544,36,549,550,551,40,41,42,43,556,45,46,48,51,55,568,569,570,572,575,67,580,582,583,585,74,75,76,77,590,591,592,593,594,596,526,599,527,604,605,606,607,528,610,611,612,614,529,108,530,120,532,122,123,124,126,130,131,132,133,535,541,201,202,203,204,234,553,248,249,250,251,252,253,254,255,256,257,258,56,338,339,340,341,57,523,372,609,63,524,384,386,387,388,389,392,396,398,399,401,579,404,73,446,448,449,451,452,454,455,456,461,462,463,468,78,474,477,489,490,492,495,84,506,507,509],"/home/petiz/Dropbox/Code/python/crypto-algotrading/cryptoalgotrading/exit.py":[5,8,12,13,14,15,16,18],"/home/petiz/Dropbox/Code/python/crypto-algotrading/cryptoalgotrading/funcs.py":[],"/home/petiz/Dropbox/Code/python/crypto-algotrading/cryptoalgotrading/extra.py":[],"/home/petiz/Dropbox/Code/python/crypto-algotrading/cryptoalgotrading/lib_bittrex.py":[257,130,9,11,12,13,14,15,144,18,20,21,22,24,26,283,28,31,34,35,39,180,195,71,268,162,80,212,89,223,99,234,107,237,115,246],"/home/petiz/Dropbox/Code/python/crypto-algotrading/cryptoalgotrading/entry.py":[5,8,12,13,14,15,16,18],"/home/petiz/Dropbox/Code/python/crypto-algotrading/cryptoalgotrading/riskmanagement.py":[32,3,5,6,8,73,10,27,37],"/home/petiz/Dropbox/Code/python/crypto-algotrading/cryptoalgotrading/finance.py":[5,8,12,13,14,15,17],"/home/petiz/Dropbox/Code/python/crypto-algotrading/cryptoalgotrading/__init__.py":[1]}}
+!coverage.py: This is a private format, don't read it directly!{"lines":{"/home/petiz/Dropbox/Code/python/crypto-algotrading/cryptoalgotrading/var.py":[8,10,13,14,19,20,24,26,27,28,30,33,35,38,40,42,43,46,48,49,50,51,52,54,55,58,60,62,63,65,67],"/home/petiz/Dropbox/Code/python/crypto-algotrading/cryptoalgotrading/cryptoalgotrading.py":[512,513,514,515,516,5,518,519,520,521,522,523,524,614,526,527,529,19,20,21,534,23,24,25,26,27,28,29,30,517,32,40,615,45,48,51,564,53,54,55,56,570,571,573,574,576,578,579,580,69,71,586,75,588,77,591,592,82,85,86,91,92,93,613,101,102,103,104,105,106,107,620,622,623,625,626,531,628,629,631,632,636,618,639,640,641,642,619,644,645,134,647,648,650,652,653,654,143,656,657,146,659,148,149,151,664,665,154,155,161,685,687,689,692,31,188,189,190,192,195,196,198,199,201,202,204,205,207,208,211,215,216,218,219,220,221,222,223,224,226,227,229,231,232,233,236,238,637,245,246,248,253,254,255,256,257,258,259,130,643,132,95,136,52,138,566,142,145,66,153,68,434,437,438,439,440,441,442,443,444,445,446,447,76,79,477,479,480,485,488,499,500,501,504,506,509],"/home/petiz/Dropbox/Code/python/crypto-algotrading/cryptoalgotrading/aux.py":[5,8,9,10,87,12,13,14,15,16,17,18,19,20,21,23,25,538,28,29,30,31,36,40,41,42,555,557,46,559,48,561,51,565,566,55,56,57,570,572,575,67,580,581,582,584,73,74,587,76,77,78,84,526,599,600,601,603,606,610,611,613,614,616,108,621,622,623,624,625,627,630,120,122,635,636,637,638,640,641,130,131,132,133,537,540,543,201,202,203,204,123,124,234,126,248,249,250,251,252,554,254,255,256,257,258,43,520,642,45,643,558,645,560,521,563,338,339,340,341,342,523,63,392,393,394,410,413,415,416,417,423,427,429,430,432,435,75,477,479,480,482,483,485,486,487,492,493,494,253,499,505,508],"/home/petiz/Dropbox/Code/python/crypto-algotrading/cryptoalgotrading/exit.py":[5,8,12,13,14,15,16,18],"/home/petiz/Dropbox/Code/python/crypto-algotrading/cryptoalgotrading/funcs.py":[],"/home/petiz/Dropbox/Code/python/crypto-algotrading/cryptoalgotrading/extra.py":[],"/home/petiz/Dropbox/Code/python/crypto-algotrading/cryptoalgotrading/lib_bittrex.py":[257,130,9,11,12,13,14,15,144,18,20,21,22,24,26,283,28,31,34,35,39,180,195,71,268,162,80,212,89,223,99,234,107,237,115,246],"/home/petiz/Dropbox/Code/python/crypto-algotrading/cryptoalgotrading/entry.py":[5,8,12,13,14,15,16,18],"/home/petiz/Dropbox/Code/python/crypto-algotrading/cryptoalgotrading/riskmanagement.py":[32,3,5,6,8,73,10,27,37],"/home/petiz/Dropbox/Code/python/crypto-algotrading/cryptoalgotrading/finance.py":[5,8,12,13,14,15,17],"/home/petiz/Dropbox/Code/python/crypto-algotrading/cryptoalgotrading/__init__.py":[1]}}
diff --git a/changelog.md b/changelog.md
@@ -1,5 +1,14 @@
 # changelog
 
+## 2019-01-29
+
+*   Added HDF5 Filetype support.
+
+## 2019-01-03
+
+*   Added Codecy badge.
+*   Implemented new unit tests to more aux funcs. 
+
 ## 2018-11-24
 
 *   Improved performance on csv reading method.

diff --git a/cryptoalgotrading/aux.py b/cryptoalgotrading/aux.py
@@ -11,7 +11,7 @@
 
 from os import system, listdir, path
 from numpy import isnan
-from pandas import read_csv, DataFrame
+from pandas import read_csv, read_hdf, DataFrame
 from time import time, localtime
 from datetime import datetime, timedelta
 from finance import bollinger_bands
@@ -192,6 +192,8 @@ def get_historical_data(market,
 
     res = db_client.query(command)
 
+    db_client.close()
+
     # returning Pandas DataFrame.
     #return pd.DataFrame(list(res.get_points(measurement=exchange)))
     return detect_init(DataFrame(list(res.get_points(measurement=var.exchange))))
@@ -338,7 +340,8 @@ def plot_data(data,
 def get_histdata_to_file(markets=[],
                          interval=var.default_interval,
                          base_market='BTC',
-                         exchange='bittrex'):
+                         exchange='bittrex',
+                         filetype='csv'):
     '''
     Gets data from DB to file.
     Prevents excess of DB accesses.
@@ -348,45 +351,74 @@ def get_histdata_to_file(markets=[],
     - market: list of str with markets.
     - interval: str with time between measures.
         Empty for default_interval.
+    - base_market: str with base market.
+        Default is BTC.
+    - exchange: str with crypto exchange.
+        Default is Bittrex.
+    - filetype: str with filetype to save.
+        Default is csv 
 
     Returns:
     - 'True'
     '''
 
-    if type(markets) is str: markets = [markets]
+    if isinstance(markets,str): markets = [markets]
 
     if not markets: markets = get_markets_list(base_market)
 
     for market in markets:
         verified_market = check_market_name(market, exchange=exchange)
-        print verified_market
-        get_historical_data(verified_market,
-                            interval=interval, 
-                            exchange=exchange).to_csv(
-            var.data_dir + '/hist-' + interval +
-            '/' + verified_market + '.csv')
+        log(verified_market, 2)
 
+        data_ = get_historical_data(verified_market,
+                            interval=interval, 
+                            exchange=exchange)
+
+        filename_ = var.data_dir + '/hist-' + \
+                    interval + '/' + \
+                    verified_market + '.'
+
+        if filetype is 'csv':
+            data_.to_csv(filename_ + filetype)
+        elif filetype is 'hdf':
+            data_.to_hdf(filename_ + filetype, 'data', 
+                         mode='w', format='t',
+                         complevel=9, complib='bzip2')
+        #TEST
+        del data_
+
     return True
 
 
-def get_data_from_file(market, interval=var.default_interval, exchange='bittrex'):
+def get_data_from_file(market,
+                       interval=var.default_interval,
+                       exchange='bittrex',
+                       filetype='csv'):
     '''
     Gets data from file.
 
     Args:
     - market: str with market.
     - interval: str with time between measures.
         Empty for default_interval.
+    - exchange: str with crypto exchange.
+        bittrex as default exchange.
+    - filetype: str with filetype.
+        default type is csv.
 
     Returns:
     - pd.DataFrame
     '''
     verified_market = check_market_name(market, exchange=exchange)
 
-    return read_csv(var.data_dir + '/hist-' + interval 
-                    + '/' + verified_market + '.csv',
-                    sep=',', engine='c',            # Optimized for a faster
-                    index_col=0, low_memory=True)   # csv reading.
+    filename_ = var.data_dir + '/hist-' + interval \
+                + '/' + verified_market + '.' + filetype
+
+    if filetype is 'csv':
+        return read_csv(filename_,  sep=',', engine='c') # Optimized.
+    elif filetype is 'h5':
+        return read_hdf(filename_,'data')
+    else: return 0
 
 
 def check_market_name(market, exchange='bittrex'):

diff --git a/cryptoalgotrading/cryptoalgotrading.py b/cryptoalgotrading/cryptoalgotrading.py
@@ -13,6 +13,7 @@
 #   improve multiprocessing display information.
 #   break big files into smaller ones.
 #   backtest based on pandas.Dataframe.
+# > add HDF5 support.
 
 
 import var
@@ -430,6 +431,7 @@ def realtime(entry_funcs,
         if refresh_interval - (time()-start_time) > 0:
             sleep(refresh_interval - (time()-start_time))
 
+
 @timeit
 def backtest(markets,
              entry_funcs,
@@ -479,7 +481,7 @@ def backtest(markets,
             db_client = connect_db()
         except Exception as e:
             log(str(e),0,log_level)
-            raise e
+            sys.exit(1)
     else:
         db_client = 0
 
@@ -495,7 +497,7 @@ def backtest(markets,
             log("Without files to analyse.",0,log_level)
 
     # Prevents errors from markets and funcs as str.
-    if isinstance(markets,str): markets=[markets]
+    if not isinstance(markets,list): markets=[markets]
     if not isinstance(entry_funcs,list): entry_funcs=[entry_funcs]
     if not isinstance(exit_funcs,list): exit_funcs=[exit_funcs]
 
@@ -657,6 +659,7 @@ def backtest_market(entry_funcs,
 
                 full_log += '[P&L] > ' + str(total) + '%.' + '\n'
 
+    del data_init
 
     # Use plot_data for just a few markets. If you try to run plot_data for several markets, 
     # computer can start run really slow.
@@ -677,15 +680,14 @@ def backtest_market(entry_funcs,
     except Exception as e:
         log("[ERROR] Ploting data: " + str(e), 0, log_level)
 
-
+    del data
     #if len(exit_points_x):
     #    log(market + ' > ' + str(total), log_level)
 
     log('[' + market + '][TOTAL]> ' + str(total)  + '%.', 0, log_level)
 
     log(full_log, 1, log_level)
 
-    if isnan(total):
-        return 0
+    if isnan(total): return 0
 
     return total