Merge branch 'feature/cache' into dev

TODO.md

@@ -3,12 +3,8 @@
 * grep -ri todo
 * add other market api (-> gdax?)
 * babao is now too slow for dry-run on an old raspberry :/ solve that!
-    * (might be linked to the recent kraken '0 fee' policy...)
-* optimize real-time graph: since kraken is (more) laggy, it's really slow
 * handle log files
-* ideally the test/train data should look like the real-time data (new slice every few seconds)
 * write more tests!
 * pip package
 * refactor graph: one page per crypto, one page for all at once, one menu page
 * add type hints
-* cache db data (~1 week in dry-run mode, 1 read needed, then fetch to cache)

src/babao/babao.py

@@ -19,12 +19,14 @@
 # from ipdb import set_trace; set_trace()
 # import babao; args = babao.babao._init(["-vv", "d"]); args.func(args)
 
+import os
 from multiprocessing import Process, Lock
 
 from prwlock import RWLock
 
 import babao.arg as arg
 import babao.config as conf
+import babao.inputs.inputBase as ib
 import babao.inputs.ledger.ledgerManager as lm
 import babao.utils.date as du
 import babao.utils.file as fu
@@ -40,13 +42,16 @@ def _launchGraph():
     # we import here, so matplotlib can stay an optional dependency
     import babao.graph as graph
 
-    p = Process(
-        target=graph.initGraph,
-        args=(log.LOCK, fu.LOCK),
-        name="babao-graph",
-        daemon=True  # so we don't have to terminate it
-    )
-    p.start()
+    if os.environ.get("DEBUG_GRAPH"):
+        graph.initGraph(log.LOCK, fu.LOCK)  # no fork
+    else:
+        p = Process(
+            target=graph.initGraph,
+            args=(log.LOCK, fu.LOCK),
+            name="babao-graph",
+            daemon=True  # so we don't have to terminate it
+        )
+        p.start()
 
 
 def _kthxbye():
@@ -62,26 +67,33 @@ def _init(args=None):
     args = arg.parseArgv(args)
     log.initLogLevel(args.verbose, args.quiet)
     conf.readConfigFile(args.func.__name__)
+    real_time = conf.CURRENT_COMMAND not in ["train", "backtest"]
+
     if not lock.tryLock(conf.LOCK_FILE) and not args.fuckit:
         log.error("Lock found (" + conf.LOCK_FILE + "), abort.")
-
-    if args.func.__name__ in ["train", "backtest"]:
-        du.setTime(du.EPOCH)
-    else:
+    if real_time:
         log.setLock(Lock())
     if args.graph:
         fu.setLock(RWLock())
     fu.initStore(conf.DB_FILE)
 
-    if args.func.__name__ in ["train", "backtest"]:
-        du.setTime(du.EPOCH)
+    if conf.CURRENT_COMMAND == "train":
+        du.setTime(
+            du.EPOCH + du.secToNano(ib.REAL_TIME_LOOKBACK_DAYS * 24 * 3600)
+        )
+    elif conf.CURRENT_COMMAND == "backtest":
+        du.setTime(
+            ib.SPLIT_DATE + du.secToNano(ib.REAL_TIME_LOOKBACK_DAYS * 24 * 3600)
+        )
+
     lm.initLedgers(
-        simulate=args.func.__name__ != "wetRun",
-        log_to_file=args.func.__name__ not in ["train", "backtest"]
+        simulate=conf.CURRENT_COMMAND != "wetRun",
+        log_to_file=real_time
     )
     RootModel()
 
-    if args.graph and args.func.__name__ != "train":
+    if args.graph and conf.CURRENT_COMMAND != "train":
+        fu.closeStore()
         _launchGraph()
     if args.func.__name__ != "train":
         sig.catchSignal()

src/babao/commands.py

@@ -2,6 +2,7 @@
 
 import time
 
+import babao.config as conf
 import babao.inputs.ledger.ledgerManager as lm
 import babao.inputs.inputManager as im
 import babao.inputs.inputBase as ib
@@ -22,7 +23,7 @@ def dryRun(unused_args):
     while not sig.EXIT:
         if im.fetchInputs():
             mm.predictModelsMaybeTrade(
-                since=du.nowMinus(weeks=1)  # TODO: do not hardcode the lookback
+                since=du.nowMinus(days=ib.REAL_TIME_LOOKBACK_DAYS)
             )
 
 
@@ -47,19 +48,16 @@ def backtest(args):
     It will call the trained strategies on each test data point
     """
     now = du.getTime(force=True)
-    epoch_to_now = now - du.EPOCH
-    t = du.EPOCH + epoch_to_now / 2
+    t = du.getTime()
     log.info(
         "Test data: from", du.toStr(t),
         "to", du.toStr(now)
     )
-
     while t < now and not lm.gameOver() and not sig.EXIT:
-        t += du.secToNano(4 * 60 * 60)
+        t += du.secToNano(conf.TIME_INTERVAL * 60)  # TODO
         im.timeTravel(t)
         mm.predictModelsMaybeTrade(
-            since=du.nowMinus(weeks=1)
-            # TODO:  do not hardcode the lookback
+            since=du.nowMinus(days=ib.REAL_TIME_LOOKBACK_DAYS)
         )
 
     score = lm.getGlobalBalanceInQuote()
@@ -79,13 +77,11 @@ def backtest(args):
 
 def train(args):
     """Train the various (awesome) algorithms"""
-    epoch_to_now = du.getTime(force=True) - du.EPOCH
-    till = du.EPOCH + epoch_to_now / 2
-    im.timeTravel(till)
+    im.timeTravel(ib.SPLIT_DATE)
 
     log.debug(
         "Train data: from", du.toStr(du.EPOCH),
-        "to", du.toStr(till)
+        "to", du.toStr(ib.SPLIT_DATE)
     )
     mm.trainModels(since=du.EPOCH)
 
@@ -96,10 +92,10 @@ def train(args):
         log.debug("Plot models on test data")
         im.timeTravel(du.getTime(force=True))  # back to the future
         log.debug(
-            "Test data: from", du.toStr(till),
+            "Test data: from", du.toStr(du.toStr(ib.SPLIT_DATE)),
             "to", du.toStr(du.getTime())
         )
-        mm.plotModels(since=till)
+        mm.plotModels(since=ib.SPLIT_DATE)
 
     log.debug("Job done!")
 

src/babao/config.py

@@ -16,20 +16,24 @@
 
 # config vars
 QUOTE = None
-CRYPTOS = None
+CRYPTOS = None  # TODO: infere from models
 TIME_INTERVAL = None
 MAX_GRAPH_POINTS = None
 
+CURRENT_COMMAND = None
 
-def readConfigFile(unused_cmd_name="unamed"):
+
+def readConfigFile(cmd_name="dry-run"):
     """Read config file and initialize file/dir paths"""
 
     # TODO: find a better way to handle config
     global QUOTE
     global CRYPTOS
     global TIME_INTERVAL
     global MAX_GRAPH_POINTS
+    global CURRENT_COMMAND
 
+    CURRENT_COMMAND = cmd_name
     config = cp.RawConfigParser()
     config.read(CONFIG_FILE)
 

src/babao/graph.py

@@ -2,7 +2,7 @@
 
 import os
 import sys
-import traceback
+# import traceback
 
 import matplotlib.animation as animation
 import matplotlib.pyplot as plt
@@ -16,10 +16,8 @@
 import babao.utils.indicators as indic
 import babao.utils.log as log
 import babao.utils.signal as sig
-from babao.inputs.trades.krakenTradesInput import KrakenTradesXXBTZEURInput
 from babao.utils.enum import CryptoEnum
 
-K = None
 DATA = None
 INDICATORS_COLUMNS = [
     "sma_KrakenTradesXXBTZEURInput-vwap_9",
@@ -45,13 +43,16 @@ def _getData():
         # TODO: catch missing frame errors
         return False
 
-    since = K.current_row.name - du.secToNano(
+    inputs = [
+        lm.TRADES[CryptoEnum.XBT],
+        lm.LEDGERS[conf.QUOTE],
+        lm.LEDGERS[CryptoEnum.XBT]
+    ]
+    im.refreshInputs(inputs)
+    since = lm.TRADES[CryptoEnum.XBT].current_row.name - du.secToNano(
         (MAX_LOOK_BACK + conf.MAX_GRAPH_POINTS) * conf.TIME_INTERVAL * 60
     )
-    DATA = im.readInputs(
-        [K, lm.LEDGERS[conf.QUOTE], lm.LEDGERS[CryptoEnum.XBT]],
-        since
-    )
+    DATA = im.readInputs(inputs, since)
     DATA = indic.get(DATA, INDICATORS_COLUMNS)
     DATA["macd_line"], DATA["signal_line"], DATA["macd"] = indic.macd(
         DATA["KrakenTradesXXBTZEURInput-vwap"],
@@ -227,7 +228,7 @@ def _initGraph():
         _updateGraph,
         fargs=(lines,),
         # blit=True,  # bug?
-        interval=10000
+        interval=5000
     )
     plt.show()  # this is blocking!
 
@@ -237,14 +238,13 @@ def initGraph(log_lock, file_lock):
 
     log.setLock(log_lock)
     fu.setLock(file_lock)
-    global K
-    K = KrakenTradesXXBTZEURInput()
-    fu.closeStore()
     sig.catchSignal()
     try:
         _getData()
         _initGraph()
-    except Exception:  # pylint: disable=broad-except
-        traceback.print_exc()
-        log.error("Something's bjorked in your graph :/")
+    except Exception as e:
+        log.warning("Something's bjorked in your graph :/")
+        log.info("Try to run babao again with the env variable DEBUG_GRAPH set")
+        # traceback.print_exc()
+        raise e
     sys.exit(0)  # we exit explicitly in the subprocess, to avoid double clean