Merge branch 'feature/multi-inputs' into dev

Makefile

@@ -1,7 +1,8 @@
 # yep
 
 NAME = babao
-
+AUTHOR = JeanMax
+VERSION = 0.1
 
 ROOT_DIR = $(HOME)/.$(NAME).d
 SRC_DIR = src
@@ -15,14 +16,12 @@ KRAKEN_KEY_FILE = $(CONFIG_DIR)/kraken.key
 TMP_FILES = build dist temp $(shell find . -name __pycache__) \
             $(NAME).egg-info $(SRC_DIR)/$(NAME).egg-info $(DOC_DIR)
 
+EUID = $(shell id -u)
+
 RM = rm -rfv
 MKDIR = mkdir -pv
 CP = cp -nv
-PY = python -u
-
-EUID = $(shell id -u)
 
-DEBUGER = ipython --no-confirm-exit --no-banner -i --pdb
 TESTER = pytest --fulltrace
 ifndef TRAVIS
 TESTER += $(shell if [ "$(TERM)" != dumb ]; then echo "--pdb"; fi)
@@ -33,14 +32,12 @@ PIP_INSTALL = pip install $(shell if [ "$(EUID)" != 0 ] && [ "$(READTHEDOCS)$(TR
 PIP_UNINSTALL = pip uninstall -y
 
 ifdef DEBUG
-EXEC = $(DEBUGER) -m $(NAME) --
+EXEC = ipython --no-confirm-exit --no-banner -i --pdb -m $(NAME) --
 else
-EXEC = $(PY) -m $(NAME)
+EXEC = python -u -m $(NAME)
 endif
 
 
-.PHONY: conf install install_test install_graph clean fclean uninstall reinstall flake lint test check commit coverage doc html man
-
 $(NAME): install
 	printf '#!/bin/bash\n\n$(EXEC) "$$@"\n' > $(NAME)
 	chmod 755 $(NAME)
@@ -75,6 +72,7 @@ clean:
 fclean: clean
 	$(RM) $(NAME)
 
+# TODO
 uninstall: fclean
 	$(PIP_UNINSTALL) $(NAME)
 # $(RM) $(ROOT_DIR)
@@ -86,7 +84,7 @@ flake:
 	$(FLAKE)
 
 lint:
-	find $(SRC_DIR) -name \*.py | grep -vE '\.#|flycheck_' | xargs $(LINTER)
+	find $(SRC_DIR) -name \*.py | grep -vE '\.#|flycheck_|eggs' | xargs $(LINTER)
 
 test:
 	$(TESTER)
@@ -98,7 +96,7 @@ coverage:
 check: flake lint test
 
 $(DOC_BUILD_DIR):
-	sphinx-apidoc --ext-coverage -H $(NAME) -A JeanMax -V 0.1 -F -o $(DOC_DIR) $(SRC_DIR)/$(NAME)
+	sphinx-apidoc --ext-coverage -H $(NAME) -A $(AUTHOR) -V $(VERSION) -F -o $(DOC_DIR) $(SRC_DIR)/$(NAME)
 
 html: $(DOC_BUILD_DIR)
 	sphinx-build -M html $(DOC_DIR) $(DOC_BUILD_DIR)
@@ -113,3 +111,9 @@ commit: reinstall check fclean
 	git add -A .
 	git diff --cached --minimal
 	git commit
+
+
+.PHONY: conf install install_test install_graph \
+		clean fclean uninstall reinstall \
+		flake lint test check commit coverage \
+		doc html man

TODO.md

@@ -10,7 +10,7 @@
 * handle log files
 * ideally the test/train data should look like the real-time data (new slice every few seconds)
 * write more tests!
-* mkdir/multiproc.lock for lock
+* multiproc.lock for lock
 
 
 ## Work-around'd:

setup.cfg

@@ -9,7 +9,7 @@ optimize = 2
 
 [flake8]
 max-line-length = 80
-exclude = tests, build, dist, .git, __pycache__
+exclude = tests, build, dist, .git, __pycache__, .eggs
 show_source = True
 ignore = E722, W503
 

setup.py

@@ -56,7 +56,7 @@
         install_requires=[
             # machine learning
             'keras>=2.2.0',
-            'tensorflow', # >=1.8.0',
+            'tensorflow',  # >=1.8.0',
             # TODO: readthedocs prefers tensorflow-1.9.0rc1
             'scipy>=1.1.0',
             'scikit-learn>=0.19.1',
@@ -67,6 +67,9 @@
             'pandas>=0.23.1',
             'tables>=3.4.4',
 
+            # reader-writer lock
+            'prwlock>=0.4.0',
+
             # parsing
             'configparser>=3.5.0',
             'argparse>=1.4.0',

src/babao/babao.py

@@ -29,7 +29,6 @@ def _kthxbye():
     """KTHXBYE"""
 
     lock.tryUnlock(conf.LOCK_FILE)
-    # TODO: the graph process probably unlock that first...
 
 
 def _init(args=None):

src/babao/commands.py

@@ -2,93 +2,55 @@
 
 import time
 import os
-import signal
+from multiprocessing import Process
+from multiprocessing.dummy import Pool as ThreadPool
 import numpy as np
 import pandas as pd
 
+
+import babao.utils.signal as sig
 import babao.utils.log as log
 import babao.utils.date as du
-import babao.utils.file as fu
 import babao.config as conf
-import babao.strategy.strategy as strat
 import babao.strategy.transaction as tx
+import babao.strategy.strategy as strat
 import babao.strategy.modelManager as modelManager
 
-from babao.inputs.kraken.krakenTradesInput import KrakenTradesXXBTZEURInput
+from babao.inputs.inputBase import ABCInput
+from babao.inputs.trades.krakenTradesInput import KrakenTradesXXBTZEURInput
+from babao.inputs.trades.krakenTradesInput import KrakenTradesXETCZEURInput
+from babao.inputs.trades.krakenTradesInput import KrakenTradesXETHZEURInput
+from babao.inputs.trades.krakenTradesInput import KrakenTradesXLTCZEURInput
+from babao.inputs.trades.krakenTradesInput import KrakenTradesXREPZEURInput
+from babao.inputs.trades.krakenTradesInput import KrakenTradesXXLMZEURInput
+from babao.inputs.trades.krakenTradesInput import KrakenTradesXXMRZEURInput
+from babao.inputs.trades.krakenTradesInput import KrakenTradesXXRPZEURInput
+from babao.inputs.trades.krakenTradesInput import KrakenTradesXZECZEURInput
+from babao.inputs.trades.krakenTradesInput import KrakenTradesXXBTZCADInput
+from babao.inputs.trades.krakenTradesInput import KrakenTradesXXBTZGBPInput
+from babao.inputs.trades.krakenTradesInput import KrakenTradesXXBTZJPYInput
+from babao.inputs.trades.krakenTradesInput import KrakenTradesXXBTZUSDInput
 
 K = None
 
-EXIT = 0
-TICK = None
-LOCK = None
-
 TRAIN_SET_LEN = 850  # TODO: config-var?
 TEST_SET_LEN = 850  # TODO: config-var?
 NUMBER_OF_TRAIN_SETS = 36  # TODO: config-var?
 
 
-def _signalHandler(signal_code, unused_frame):
-    """Catch signal INT/TERM, so we won't exit while playing with data files"""
-
-    global EXIT
-    EXIT = 128 + signal_code
-
-
-def _delay(block=True):
-    """
-    Sleep the min amount of time required to still be friend with the api
-
-    Also handle a lock to avoid having the graph read data while we write
-    Return False if a signal have been caught (you need to exit).
-    """
-
-    if EXIT:
-        return False
-
-    if LOCK:
-        try:
-            LOCK.release()
-        except ValueError:
-            pass
-
-    global TICK
-    if TICK is not None:
-        delta = time.time() - TICK
-        log.debug("Loop took " + str(round(delta, 3)) + "s")
-    else:
-        delta = 0
-
-    delta = 3 - delta  # TODO: define API_DELAY
-    if delta > 0:
-        time.sleep(delta)
-    TICK = time.time()
-
-    if LOCK:
-        if delta < 0:
-            time.sleep(1)  # TODO: workaround: graph need time
-        LOCK.acquire(block=block)
-
-    return not bool(EXIT)
-
-
 def _launchGraph():
     """Start the graph process"""
 
     # we import here, so matplotlib can stay an optional dependency
-    from multiprocessing import Process, Lock
     import babao.graph as graph
 
-    global LOCK
-    LOCK = Lock()
     p = Process(
         target=graph.initGraph,
-        args=(LOCK,),
+        args=(log.LOCK, ABCInput.rw_lock),
         name="babao-graph",
         daemon=True  # so we don't have to terminate it
     )
     p.start()
-    time.sleep(0.5)  # TODO: define LIL_DELAY_JUST_IN_CASE
-    LOCK.acquire()
 
 
 def _initCmd(graph=False, simulate=True):
@@ -98,46 +60,65 @@ def _initCmd(graph=False, simulate=True):
     Init: signal handlers, api key, graph
     """
 
-    signal.signal(signal.SIGINT, _signalHandler)
-    signal.signal(signal.SIGTERM, _signalHandler)
-
     global K
-    K = KrakenTradesXXBTZEURInput()
-
-    ledger.initBalance()
-    if simulate and fu.getLastRows(conf.DB_FILE, conf.LEDGER_FRAME, 1).empty:
-        ledger.logQuoteDeposit(100)
-
+    K = [
+        KrakenTradesXXBTZEURInput(),
+        KrakenTradesXETCZEURInput(),
+        KrakenTradesXETHZEURInput(),
+        KrakenTradesXLTCZEURInput(),
+        KrakenTradesXREPZEURInput(),
+        KrakenTradesXXLMZEURInput(),
+        KrakenTradesXXMRZEURInput(),
+        KrakenTradesXXRPZEURInput(),
+        KrakenTradesXZECZEURInput(),
+        KrakenTradesXXBTZCADInput(),
+        KrakenTradesXXBTZGBPInput(),
+        KrakenTradesXXBTZJPYInput(),
+        KrakenTradesXXBTZUSDInput(),
+    ]
+    tx.initLedger(simulate)
+    modelManager.loadModels()
     if graph:
         _launchGraph()
-
-    tx.initLastTransaction()
-    modelManager.loadModels()
+    sig.catchSignal()
 
 
 def _getData():
     """Return the whole dataset splitted in two parts: (train, test)"""
 
-    full_data = K.read()
+    full_data = K[0].resample(K[0].read())
     return full_data[:-TEST_SET_LEN], full_data[-TEST_SET_LEN:]
 
 
+def _initLocks(log_lock, rw_lock):
+    """TODO: same in graph"""
+    log.setLock(log_lock)
+    ABCInput.rw_lock = rw_lock
+
+
+def _poolFetcher(pool_input):
+    """TODO"""
+    return pool_input.write(pool_input.fetch())
+
+
 def wetRun(args):
     """Dummy"""
-    print(repr(args))
+    _initCmd(args.graph, simulate=False)
     print("Sorry, this is not implemented yet :/")
 
 
 def dryRun(args):
     """Real-time bot simulation"""
 
     _initCmd(args.graph)
-    ledger.setVerbose(True)
-
-    while True:
-        K.write()
+    pool = ThreadPool(
+        initializer=_initLocks,
+        initargs=(log.LOCK, ABCInput.rw_lock)
+    )
+    while not sig.EXIT:
+        pool.map(_poolFetcher, K)
         # TODO:  do not hardcode the lookback
-        fresh_data = K.read(since=du.nowMinus(weeks=1))
+        fresh_data = K[0].resample(K[0].read(since=du.nowMinus(weeks=1)))
         if not fresh_data.empty:
             modelManager.prepareModels(fresh_data)
             timestamp = fresh_data.index[-1]
@@ -147,9 +128,8 @@ def dryRun(args):
                 price=price,
                 timestamp=timestamp
             )
-
-        if not _delay():
-            return
+    pool.close()
+    pool.join()
 
 
 def fetch(args):
@@ -165,13 +145,11 @@ def fetch(args):
             # os.remove(f)  # TODO: warn user / create backup?
             log.warning("Database file already exists (" + f + ").")
 
-    while True:
-        K.write()
+    while not sig.EXIT:
+        K[0].write(K[0].fetch())
         log.debug(
-            "Fetched data till " + pd.to_datetime(K.last_row.name, unit="ns")
+            "Fetched data till " + pd.to_datetime(K[0].last_row.name, unit="ns")
         )
-        if not _delay():
-            return
 
 
 def backtest(args):
@@ -182,7 +160,6 @@ def backtest(args):
     """
 
     _initCmd(args.graph)
-    ledger.setVerbose(True)
 
     big_fat_data = _getData()[1]
     modelManager.prepareModels(big_fat_data)
@@ -199,11 +176,11 @@ def backtest(args):
             price=big_fat_data_prices[i],
             timestamp=big_fat_data_index[i]
         )
-        if EXIT:
+        if sig.EXIT:
             return
 
     price = big_fat_data_prices[-1]
-    score = ledger.BALANCE["crypto"] * price + ledger.BALANCE["quote"]
+    score = tx.L["crypto"].balance * price + tx.L["quote"].balance
     hodl = price / big_fat_data_prices[0] * 100
     log.info(
         "Backtesting done! Score: " + str(round(float(score)))
@@ -216,8 +193,8 @@ def backtest(args):
 
     if args.graph:
         # TODO: exit if graph is closed
-        while _delay(block=False):
-            pass
+        while not sig.EXIT:
+            time.sleep(0.1)
 
 
 def train(args):

src/babao/config.py

@@ -23,7 +23,7 @@
 DB_FILE = None
 
 
-def readConfigFile(cmd_name="unamed"):
+def readConfigFile(unused_cmd_name="unamed"):
     """Read config file and initialize file/dir paths"""
 
     # TODO: find a better way to handle config