Merge branch 'feature/ledger-manager' into dev

config/babao.conf

@@ -3,20 +3,17 @@
 # hardcoded values in config.py will be used for omited variables
 
 [babao]
-# time interval for data resampling (minutes)
-# 1, 5, 15, 30, 60, 240, 1440, 10080, 21600
-TIME_INTERVAL = 5
+# the quote asset you want to use, should be one of:
+# CAD EUR GBP JPY USD
+QUOTE = EUR
+
+# the crypto(s) asset(s) you want to use, should be a space-separated list of:
+# ETC ETH LTC REP XBT XLM XMR XRP ZEC
+CRYPTOS = ETH LTC XBT
 
-# currency pair to trade
-# (full list at https://api.kraken.com/0/public/AssetPairs)
-ASSET_PAIR = XXBTZEUR
+# time interval for data resampling (minutes)
+TIME_INTERVAL = 60
 
 # number of points to graph at once
 # (increase to show more points, decrease to improve performances)
-MAX_GRAPH_POINTS = 420
-
-# where to write logs
-#LOG_DIR = /tmp
-
-# where to write data dumps
-#DATA_DIR = /tmp
+MAX_GRAPH_POINTS = 168

src/babao/babao.py

@@ -19,11 +19,32 @@
 # from ipdb import set_trace; set_trace()
 # import babao; args = babao.babao._init(["-vv", "d"]); args.func(args)
 
+from multiprocessing import Process, Lock
+from prwlock import RWLock
+
 import babao.utils.log as log
 import babao.utils.file as fu
 import babao.utils.lock as lock
+# import babao.utils.signal as sig
 import babao.config as conf
 import babao.parser as pars
+import babao.inputs.ledger.ledgerManager as lm
+import babao.strategy.modelManager as modelManager
+
+
+def _launchGraph():
+    """Start the graph process"""
+
+    # we import here, so matplotlib can stay an optional dependency
+    import babao.graph as graph
+
+    p = Process(
+        target=graph.initGraph,
+        args=(log.LOCK, fu.LOCK),
+        name="babao-graph",
+        daemon=True  # so we don't have to terminate it
+    )
+    p.start()
 
 
 def _kthxbye():
@@ -42,7 +63,22 @@ def _init(args=None):
 
     if not lock.tryLock(conf.LOCK_FILE) and not args.fuckit:
         log.error("Lock found (" + conf.LOCK_FILE + "), abort.")
+    if args.func.__name__ not in ["train", "backtest"]:
+        log.setLock(Lock())
+    if args.graph:
+        fu.setLock(RWLock())
     fu.initStore(conf.DB_FILE)
+    lm.initLedger(
+        simulate=args.func.__name__ != "wetRun",
+        log_to_file=args.func.__name__ != "train",
+    )
+    try:
+        modelManager.loadModels()
+    except FileNotFoundError:
+        log.warning("No model found.")
+    if args.graph:
+        _launchGraph()
+    # sig.catchSignal()
 
     return args
 

src/babao/commands.py

@@ -3,8 +3,6 @@
 import time
 import os
 from multiprocessing.dummy import Pool as ThreadPool
-from multiprocessing import Process, Lock
-from prwlock import RWLock
 import numpy as np
 import pandas as pd
 
@@ -14,7 +12,7 @@
 import babao.utils.date as du
 import babao.utils.file as fu
 import babao.config as conf
-import babao.strategy.transaction as tx
+import babao.inputs.ledger.ledgerManager as lm
 import babao.strategy.strategy as strat
 import babao.strategy.modelManager as modelManager
 
@@ -32,61 +30,27 @@
 from babao.inputs.trades.krakenTradesInput import KrakenTradesXXBTZJPYInput
 from babao.inputs.trades.krakenTradesInput import KrakenTradesXXBTZUSDInput
 
-K = None
+K = [
+    KrakenTradesXXBTZEURInput(),
+    KrakenTradesXETCZEURInput(),
+    KrakenTradesXETHZEURInput(),
+    KrakenTradesXLTCZEURInput(),
+    KrakenTradesXREPZEURInput(),
+    KrakenTradesXXLMZEURInput(),
+    KrakenTradesXXMRZEURInput(),
+    KrakenTradesXXRPZEURInput(),
+    KrakenTradesXZECZEURInput(),
+    KrakenTradesXXBTZCADInput(),
+    KrakenTradesXXBTZGBPInput(),
+    KrakenTradesXXBTZJPYInput(),
+    KrakenTradesXXBTZUSDInput(),
+]
 
 TRAIN_SET_LEN = 850  # TODO: config-var?
 TEST_SET_LEN = 850  # TODO: config-var?
 NUMBER_OF_TRAIN_SETS = 36  # TODO: config-var?
 
 
-def _launchGraph():
-    """Start the graph process"""
-
-    # we import here, so matplotlib can stay an optional dependency
-    import babao.graph as graph
-
-    p = Process(
-        target=graph.initGraph,
-        args=(log.LOCK, fu.LOCK),
-        name="babao-graph",
-        daemon=True  # so we don't have to terminate it
-    )
-    p.start()
-
-
-def _initCmd(graph=False, simulate=True):
-    """
-    Generic command init function
-
-    Init: signal handlers, api key, graph
-    """
-
-    log.setLock(Lock())
-    if graph:
-        fu.setLock(RWLock())
-    global K
-    K = [
-        KrakenTradesXXBTZEURInput(),
-        KrakenTradesXETCZEURInput(),
-        KrakenTradesXETHZEURInput(),
-        KrakenTradesXLTCZEURInput(),
-        KrakenTradesXREPZEURInput(),
-        KrakenTradesXXLMZEURInput(),
-        KrakenTradesXXMRZEURInput(),
-        KrakenTradesXXRPZEURInput(),
-        KrakenTradesXZECZEURInput(),
-        KrakenTradesXXBTZCADInput(),
-        KrakenTradesXXBTZGBPInput(),
-        KrakenTradesXXBTZJPYInput(),
-        KrakenTradesXXBTZUSDInput(),
-    ]
-    tx.initLedger(simulate)
-    modelManager.loadModels()
-    if graph:
-        _launchGraph()
-    sig.catchSignal()
-
-
 def _getData():
     """Return the whole dataset splitted in two parts: (train, test)"""
 
@@ -96,14 +60,12 @@ def _getData():
 
 def wetRun(args):
     """Dummy"""
-    _initCmd(args.graph, simulate=False)
     print("Sorry, this is not implemented yet :/")
 
 
 def dryRun(args):
     """Real-time bot simulation"""
 
-    _initCmd(args.graph)
     pool = ThreadPool(
         initializer=lambda x, y: [log.setLock(x), fu.setLock(y)],
         initargs=(log.LOCK, fu.LOCK)
@@ -131,11 +93,6 @@ def dryRun(args):
 def fetch(args):
     """Fetch raw trade data since the beginning of times"""
 
-    try:
-        _initCmd(args.graph)
-    except FileNotFoundError:
-        log.warning("No model found.")
-
     for f in [conf.DB_FILE]:
         if os.path.isfile(f):
             # os.remove(f)  # TODO: warn user / create backup?
@@ -155,8 +112,6 @@ def backtest(args):
     It will call the trained strategies on each test data point
     """
 
-    _initCmd(args.graph)
-
     big_fat_data = _getData()[1]
     modelManager.prepareModels(big_fat_data)
     big_fat_data_index = big_fat_data.index.values
@@ -176,7 +131,7 @@ def backtest(args):
             return
 
     price = big_fat_data_prices[-1]
-    score = tx.L["crypto"].balance * price + tx.L["quote"].balance
+    score = lm.getGlobalBalanceInQuote()
     hodl = price / big_fat_data_prices[0] * 100
     log.info(
         "Backtesting done! Score: " + str(round(float(score)))
@@ -196,8 +151,6 @@ def backtest(args):
 def train(args):
     """Train the various (awesome) algorithms"""
 
-    # _initCmd(args.graph)
-
     train_data, test_data = _getData()
 
     splits_size = int(len(train_data) / TRAIN_SET_LEN)

src/babao/config.py

@@ -3,75 +3,66 @@
 import os
 import configparser as cp
 
+from babao.utils.enum import QuoteEnum, CryptoEnum
+
 # globad vars
 CONFIG_DIR = os.path.join(os.path.expanduser("~"), ".babao.d")
 CONFIG_FILE = os.path.join(CONFIG_DIR, "babao.conf")
 API_KEY_FILE = os.path.join(CONFIG_DIR, "kraken.key")
 LOCK_FILE = os.path.join(CONFIG_DIR, "babao.lock")
+LOG_DIR = os.path.join(CONFIG_DIR, "log")
+DATA_DIR = os.path.join(CONFIG_DIR, "data")
+DB_FILE = os.path.join(DATA_DIR, "babao-database.hdf")
 
-MODEL_MACD_FILE = None
-MODEL_EXTREMA_FILE = None
-MODEL_TENDENCY_FILE = None
-MODEL_QLEARN_FILE = None
+# TODO: don't
+MODEL_MACD_FILE = os.path.join(DATA_DIR, "macd.pkl")
+MODEL_EXTREMA_FILE = os.path.join(DATA_DIR, "extrema.pkl")
+MODEL_TENDENCY_FILE = os.path.join(DATA_DIR, "tendency.h5")
+MODEL_QLEARN_FILE = os.path.join(DATA_DIR, "qlearn.h5")
 
 # config vars
-LOG_DIR = None  # useless and annoying... just let the user symlink if needed
-DATA_DIR = None  # idem
-ASSET_PAIR = None  # TODO: QUOTE, CRYPTOS
+QUOTE = None
+CRYPTOS = None
 TIME_INTERVAL = None
 MAX_GRAPH_POINTS = None
-DB_FILE = None
 
 
 def readConfigFile(unused_cmd_name="unamed"):
     """Read config file and initialize file/dir paths"""
 
     # TODO: find a better way to handle config
-    global LOG_DIR
-    global DATA_DIR
-    global ASSET_PAIR
+    global QUOTE
+    global CRYPTOS
     global TIME_INTERVAL
     global MAX_GRAPH_POINTS
-    global DB_FILE
-    global MODEL_MACD_FILE
-    global MODEL_EXTREMA_FILE
-    global MODEL_TENDENCY_FILE
-    global MODEL_QLEARN_FILE
 
     config = cp.RawConfigParser()
     config.read(CONFIG_FILE)
 
-    LOG_DIR = config.get(
-        "babao",
-        "LOG_DIR",
-        fallback="/tmp"
-    )
-    DATA_DIR = config.get(
+    QUOTE = config.get(
         "babao",
-        "DATA_DIR",
-        fallback=os.path.join(CONFIG_DIR, "data")
+        "QUOTE",
+        fallback="EUR"
     )
-    ASSET_PAIR = config.get(
+    QUOTE = QuoteEnum[QUOTE]
+
+    CRYPTOS = config.get(
         "babao",
-        "ASSET_PAIR",
-        fallback="XXBTZEUR"
+        "CRYPTOS",
+        fallback="ETH LTC XBT"
     )
+    CRYPTOS = [CryptoEnum[c] for c in CRYPTOS.split()]
+
     TIME_INTERVAL = config.getint(
         "babao",
         "TIME_INTERVAL",
         fallback=5
     )
+
     MAX_GRAPH_POINTS = config.getint(
         "babao",
         "MAX_GRAPH_POINTS",
         fallback=420
     )
-    # TODO: check if these vars are valid
 
-    pre = os.path.join(DATA_DIR, ASSET_PAIR)
-    DB_FILE = os.path.join(DATA_DIR, "babao-database.hdf")
-
-    MODEL_MACD_FILE = pre + "-macd.pkl"
-    MODEL_EXTREMA_FILE = pre + "-extrema.pkl"
-    MODEL_TENDENCY_FILE = pre + "-tendency.h5"
-    MODEL_QLEARN_FILE = pre + "-qlearn.h5"
+    # TODO: check if these vars are valid

src/babao/graph.py

@@ -14,8 +14,9 @@
 import babao.config as conf
 import babao.utils.indicators as indic
 import babao.strategy.models.macd as macd  # TODO: this is weird
-import babao.strategy.transaction as tx
+import babao.inputs.ledger.ledgerManager as lm
 import babao.inputs.inputHelper as ih
+from babao.utils.enum import CryptoEnum
 from babao.inputs.trades.krakenTradesInput import KrakenTradesXXBTZEURInput
 
 K = None
@@ -47,7 +48,10 @@ def _getData():
     since = K.last_row.name - du.secToNano(
         (MAX_LOOK_BACK + conf.MAX_GRAPH_POINTS) * 60 * 60
     )
-    DATA = ih.readInputs([K, tx.L["quote"], tx.L["crypto"]], since)
+    DATA = ih.readInputs(
+        [K, lm.LEDGERS[conf.QUOTE], lm.LEDGERS[CryptoEnum.XBT]],
+        since
+    )
     DATA = indic.get(DATA, INDICATORS_COLUMNS)
     DATA["macd_line"], DATA["signal_line"], DATA["macd"] = indic.MACD(
         DATA["KrakenTradesXXBTZEURInput-vwap"],
@@ -176,10 +180,10 @@ def _initGraph():
     axes["macd"].set_ylim(bottom=-y_max, top=y_max)
     axes["bal"].set_ylim(bottom=0, top=200)
 
-    axes["KrakenTradesXXBTZEURInput-vwap"].set_ylabel(conf.ASSET_PAIR[4:])
-    axes["KrakenTradesXXBTZEURInput-volume"].set_ylabel(conf.ASSET_PAIR[:4])
+    axes["KrakenTradesXXBTZEURInput-vwap"].set_ylabel("XBT")
+    axes["KrakenTradesXXBTZEURInput-volume"].set_ylabel("EUR")
     axes["macd"].set_ylabel("%")
-    axes["bal"].set_ylabel(conf.ASSET_PAIR[4:])
+    axes["bal"].set_ylabel("XBT")
 
     for key in axes:
         axes[key].grid(True)

src/babao/inputs/krakenBase.py

@@ -23,7 +23,7 @@ def _initAPI():
     API = krakenex.API()
     try:
         API.load_key(conf.API_KEY_FILE)
-    except Exception as e:  # TODO
+    except FileNotFoundError as e:
         log.warning(
             "Couldn't load kraken api key file '"
             + conf.API_KEY_FILE + "': " + repr(e)

src/babao/inputs/ledger/ledgerInputBase.py

@@ -29,6 +29,12 @@ class ABCLedgerInput(ABCInput):
         "balance"
     ]
 
+    @property
+    @abstractmethod
+    def asset(self):
+        """TODO"""
+        pass
+
     def __init__(self):
         super().__init__()
         self.verbose = True