Merge branch 'feature/models-tree' into dev

Makefile

@@ -91,7 +91,7 @@ lint:
 
 pyre:
     # TODO: debug cache, opt stubs import
-	$(PYRE) check 2>/dev/null | grep -v 'Undefined import \[21\]' || true
+	$(PYRE) check 2>/dev/null | grep -vE 'Undefined (import|attribute)' || true
 
 mypy:
 	$(MYPY)
@@ -123,7 +123,7 @@ commit: reinstall check fclean
 	git commit
 
 todo:
-	grep -rin todo . | grep -vE '^(Binary file|\./Makefile|\./TODO.md|\./\.travis\.yml.* make todo)'
+	grep -rin todo . | grep -vE '^(Binary file|\./\.git|\./Makefile|\./TODO.md|\./\.travis\.yml.* make todo)'
 	grep -iHn todo ./Makefile | head -n -$(shell grep -A1000 'todo:' Makefile | grep -ic todo)
 	cat TODO.md
 

setup.cfg

@@ -14,12 +14,12 @@ exclude = tests, build, dist, .git, __pycache__, .eggs
 format = pylint
 show_source = True
 max-line-length = 80
-max-complexity = 10
+max-complexity = 8
 
 [pylint]
 output-format = colorized
 max-line-length = 80
-disable = global-statement,  # invalid-name
+disable = global-statement, unused-import, useless-import-alias # invalid-name
 notes = FIXME, DEBUG
 score = no
 function-naming-style = camelCase

setup.py

@@ -55,17 +55,17 @@
 
         install_requires=[
             # machine learning
-            'keras>=2.2.0',
-            'tensorflow',  # >=1.8.0',
-            # TODO: readthedocs prefers tensorflow-1.9.0rc1
+            'keras>=2.1.2',
+            # 'tensorflow',  # >=1.8.0', readthedocs prefers tensorflow-1.9.0rc1
+            'theano>=1.0.2',
             'scipy>=1.1.0',
             'scikit-learn>=0.19.1',
             'joblib>=0.11',  # just for saving scikit models...
 
             # data handling
             'numpy>=1.14.5',
             'pandas>=0.23.1',
-            'tables>=3.4.4',
+            'tables>=3.4.2',
 
             # reader-writer lock
             'prwlock>=0.4.0',

src/babao/babao.py

@@ -83,7 +83,8 @@ def _init(args=None):
 
     if args.graph and args.func.__name__ != "train":
         _launchGraph()
-    sig.catchSignal()
+    if args.func.__name__ != "train":
+        sig.catchSignal()
 
     return args
 

src/babao/commands.py

@@ -1,9 +1,7 @@
 """Commands launched by parseArgv"""
 
-import os
 import time
 
-import babao.config as conf
 import babao.inputs.ledger.ledgerManager as lm
 import babao.inputs.inputManager as im
 import babao.inputs.inputBase as ib

src/babao/config.py

@@ -14,12 +14,6 @@
 DATA_DIR = os.path.join(CONFIG_DIR, "data")
 DB_FILE = os.path.join(DATA_DIR, "babao-database.hdf")  # TODO: move
 
-# TODO: don't
-MODEL_MACD_FILE = os.path.join(DATA_DIR, "macd.pkl")
-MODEL_EXTREMA_FILE = os.path.join(DATA_DIR, "extrema.pkl")
-MODEL_TENDENCY_FILE = os.path.join(DATA_DIR, "tendency.h5")
-MODEL_QLEARN_FILE = os.path.join(DATA_DIR, "qlearn.h5")
-
 # config vars
 QUOTE = None
 CRYPTOS = None

src/babao/graph.py

@@ -11,7 +11,6 @@
 import babao.config as conf
 import babao.inputs.inputManager as im
 import babao.inputs.ledger.ledgerManager as lm
-import babao.models.tree.macdModel as macd  # TODO: this is weird
 import babao.utils.date as du
 import babao.utils.file as fu
 import babao.utils.indicators as indic
@@ -56,7 +55,8 @@ def _getData():
     DATA = indic.get(DATA, INDICATORS_COLUMNS)
     DATA["macd_line"], DATA["signal_line"], DATA["macd"] = indic.macd(
         DATA["KrakenTradesXXBTZEURInput-vwap"],
-        macd.MODEL["a"], macd.MODEL["b"], macd.MODEL["c"], True
+        46, 75, 22,
+        True
     )
     DATA = DATA.dropna()
     DATA["bal"] = DATA["FakeLedgerEURInput-balance"] \
@@ -80,10 +80,8 @@ def _updateGraph(unused_counter, lines):
     return lines.values()
 
 
-def _initGraph():
-    """Wrapped to display errors (this is running in a separate process)"""
-
-    fig = plt.figure()
+def _createAxes():
+    """TODO"""
     axes = {}
     axes["KrakenTradesXXBTZEURInput-vwap"] = plt.subplot2grid(
         (8, 1), (0, 0), rowspan=5
@@ -97,7 +95,11 @@ def _initGraph():
     axes["bal"] = plt.subplot2grid(
         (8, 1), (7, 0), sharex=axes["KrakenTradesXXBTZEURInput-vwap"]
     )
+    return axes
+
 
+def _createLines(axes):
+    """TODO"""
     lines = {}
     for key in axes:  # TODO: this is *really* ugly
         lines[key], = axes[key].plot(
@@ -146,6 +148,15 @@ def _initGraph():
                 color="g",
                 alpha=0.5
             )
+    return lines
+
+
+def _initGraph():
+    """Wrapped to display errors (this is running in a separate process)"""
+
+    fig = plt.figure()
+    axes = _createAxes()
+    lines = _createLines(axes)
 
     # the assignation is needed to avoid garbage collection...
     unused_cursor = MultiCursor(  # NOQA: F841

src/babao/inputs/inputManager.py

@@ -5,7 +5,7 @@
 
 from multiprocessing.dummy import Pool as ThreadPool
 from functools import partial, reduce
-from typing import List, Union
+from typing import List, Optional
 
 import babao.utils.date as du
 import babao.utils.file as fu
@@ -30,7 +30,7 @@ def fetchInputs():
     return reduce(lambda acc, inp: acc & inp.up_to_date, ib.INPUTS, True)
 
 
-def readInputs(input_list: Union[List[ib.ABCInput], None] = None, since=None):
+def readInputs(input_list: Optional[List[ib.ABCInput]] = None, since=None):
     """TODO"""
 
     def _renamer(prefix, s):

src/babao/inputs/ledger/fakeLedgerInput.py

@@ -4,6 +4,7 @@
 """
 
 import sys
+from abc import abstractmethod
 
 import pandas as pd
 
@@ -16,6 +17,12 @@
 class ABCFakeLedgerInput(ABCLedgerInput):
     """TODO"""
 
+    @property
+    @abstractmethod
+    def asset(self):
+        """TODO: warning trap"""
+        pass
+
     def __init__(self, log_to_file=True):
         ABCLedgerInput.__init__(self)
         self.log_to_file = log_to_file
@@ -64,7 +71,7 @@ def logTransaction(
         ).fillna(0)
 
         for c in df.columns:
-            if c == "type" or c == "product":
+            if c in ["type", "product"]:
                 df[c] = df[c].astype(int)
             else:
                 df[c] = df[c].astype(float)

src/babao/inputs/ledger/krakenLedgerInput.py

@@ -44,7 +44,7 @@ def fetch(self):
         if res["count"] == 0:
             self.up_to_date = True
             return None
-        elif res["count"] > 50 and self.current_row is None:
+        if res["count"] > 50 and self.current_row is None:
             # kraken api is *STOOPID*: if we don't have the exact date of the
             # first transaction, we can't fetch the ledger data starting from
             # the begining... so we'll need a couple extra requests, sorry!

src/babao/inputs/ledger/ledgerManager.py

@@ -4,17 +4,22 @@
 """
 
 import re
+from typing import Optional, Dict, Union, TYPE_CHECKING  # noqa: F401
 
 import babao.config as conf
 import babao.inputs.trades.krakenTradesInput as tra
 import babao.utils.date as du
 import babao.utils.log as log
-from babao.utils.enum import ActionEnum
+from babao.utils.enum import ActionEnum, CryptoEnum, QuoteEnum  # noqa: F401
 
 MIN_BAL = 50  # maximum drawdown  # TODO: this should be a percent of... hmm
 
-LEDGERS = None
-TRADES = None  # TODO: all prices are going to be desync in simulation
+if TYPE_CHECKING:
+    from babao.inputs.ledger.ledgerInputBase import ABCLedgerInput  # noqa: F401
+    AssetEnum = Union["CryptoEnum", "QuoteEnum"]
+
+LEDGERS = None  # type: Optional[Dict[AssetEnum, ABCLedgerInput]]
+TRADES = None  # type: Optional[Dict[CryptoEnum, ABCLedgerInput]]
 
 
 def initLedgers(simulate=True, log_to_file=True):
@@ -114,7 +119,7 @@ def _canSell(crypto_enum):
     # if LAST_TX["type"] == "s":
     #     return False
     if getBalanceInQuote(crypto_enum) < MIN_BAL:
-        # TODO: this can be quite high actually
+        # this can be quite high actually
         # support.kraken.com/ \
         # hc/en-us/articles/205893708-What-is-the-minimum-order-size-
         if LEDGERS[conf.QUOTE].verbose:
@@ -162,7 +167,7 @@ def buyOrSell(action_enum, crypto_enum, volume=None):
         if volume is None:
             volume = LEDGERS[conf.QUOTE].balance
         return buy(crypto_enum, volume)
-    elif action_enum == ActionEnum.SELL:
+    if action_enum == ActionEnum.SELL:
         if volume is None:
             volume = LEDGERS[crypto_enum].balance
         return sell(crypto_enum, volume)

src/babao/inputs/trades/krakenTradesInputBase.py

@@ -68,8 +68,8 @@ def fetch(self):
             fresh_data.index = fresh_data["time"]
 
         del fresh_data["misc"]
-        del fresh_data["market-limit"]  # TODO: this could be useful
-        del fresh_data["buy-sell"]  # TODO: this could be useful
+        del fresh_data["market-limit"]  # this could be useful
+        del fresh_data["buy-sell"]  # idem
         del fresh_data["time"]
 
         self.up_to_date = len(fresh_data) != 1000