Merge branch 'feature/models-tree' into dev

.gitignore

@@ -14,6 +14,12 @@ docs/
 # coveralls
 .coverage
 
+# pyre
+.pyre/
+
+# mypy
+.mypy_cache/
+
 # python gifts
 *.egg-info/
 *.egg
@@ -22,3 +28,6 @@ docs/
 __pycache__/
 *.so
 *~
+
+# don't tell emacs please
+.idea

.travis.yml

@@ -53,8 +53,11 @@ script:
   - babao
   - make flake
   - make lint
+  - make pyre
+  - make mypy
   - make test
   - make coverage
+  - make todo
 
 after_script:
   - make develop

Makefile

@@ -5,16 +5,17 @@ AUTHOR = JeanMax
 VERSION = 0.1
 
 ROOT_DIR = $(HOME)/.$(NAME).d
-SRC_DIR = src
+SRC_DIR = src/$(NAME)
 DOC_DIR = docs
 DOC_BUILD_DIR = docs/_build
+TEST_DIR = test
 
 CONFIG_DIR = config
 CONFIG_FILE = $(CONFIG_DIR)/$(NAME).conf
 KRAKEN_KEY_FILE = $(CONFIG_DIR)/kraken.key
 
 TMP_FILES = build dist temp $(shell find . -name __pycache__) \
-            $(NAME).egg-info $(SRC_DIR)/$(NAME).egg-info $(DOC_DIR)
+            $(NAME).egg-info $(SRC_DIR).egg-info $(DOC_DIR) .pyre
 
 EUID = $(shell id -u)
 
@@ -27,6 +28,8 @@ ifndef TRAVIS
 TESTER += $(shell if [ "$(TERM)" != dumb ]; then echo "--pdb"; fi)
 endif
 FLAKE = flake8
+PYRE = pyre --source-directory $(SRC_DIR) $(shell if [ "$(TERM)" = dumb ]; then echo "--noninteractive"; fi) # server looks broken :/ $(shell python -c 'import sys; import os; print(" ".join(["--search-path " + i for i in sys.path if os.path.isdir(i)]))')
+MYPY = mypy --ignore-missing-imports $(SRC_DIR)
 LINTER = pylint --rcfile=setup.cfg $(shell if [ "$(TERM)" = dumb ]; then echo "-fparseable"; fi)
 PIP_INSTALL = pip install $(shell if [ "$(EUID)" != 0 ] && [ "$(READTHEDOCS)$(TRAVIS)$(PYENV_VERSION)" = "" ]; then echo "--user"; fi)
 PIP_UNINSTALL = pip uninstall -y
@@ -72,7 +75,7 @@ clean:
 fclean: clean
 	$(RM) $(NAME)
 
-# TODO
+# TODO: clean all installed files
 uninstall: fclean
 	$(PIP_UNINSTALL) $(NAME)
 # $(RM) $(ROOT_DIR)
@@ -86,17 +89,24 @@ flake:
 lint:
 	find $(SRC_DIR) -name \*.py | grep -vE '\.#|flycheck_|eggs' | xargs $(LINTER)
 
+pyre:
+    # TODO: debug cache, opt stubs import
+	$(PYRE) check 2>/dev/null | grep -v 'Undefined import \[21\]' || true
+
+mypy:
+	$(MYPY)
+
 test:
 	$(TESTER)
 
 coverage:
 	coverage run --source=$(NAME) setup.py test
 	coveralls
 
-check: flake lint test
+check: lint flake pyre mypy test
 
 $(DOC_BUILD_DIR):
-	sphinx-apidoc --ext-coverage -H $(NAME) -A $(AUTHOR) -V $(VERSION) -F -o $(DOC_DIR) $(SRC_DIR)/$(NAME)
+	sphinx-apidoc --ext-coverage -H $(NAME) -A $(AUTHOR) -V $(VERSION) -F -o $(DOC_DIR) $(SRC_DIR)
 
 html: $(DOC_BUILD_DIR)
 	sphinx-build -M html $(DOC_DIR) $(DOC_BUILD_DIR)
@@ -112,8 +122,12 @@ commit: reinstall check fclean
 	git diff --cached --minimal
 	git commit
 
+todo:
+	grep -rin todo . | grep -vE '^(Binary file|\./Makefile|\./TODO.md|\./\.travis\.yml.* make todo)'
+	grep -iHn todo ./Makefile | head -n -$(shell grep -A1000 'todo:' Makefile | grep -ic todo)
+	cat TODO.md
 
 .PHONY: conf install install_test install_graph \
 		clean fclean uninstall reinstall \
-		flake lint test check commit coverage \
-		doc html man
+		flake pyre mypy lint test check commit coverage \
+		doc html man todo

TODO.md

@@ -1,18 +1,14 @@
 # TODO:
 
 * grep -ri todo
-* switch to another market api (-> gdax?)
-* refactor models as objects?
-    * there is some redundant code between models, handle that while you're at it
+* add other market api (-> gdax?)
 * babao is now too slow for dry-run on an old raspberry :/ solve that!
     * (might be linked to the recent kraken '0 fee' policy...)
 * optimize real-time graph: since kraken is (more) laggy, it's really slow
 * handle log files
 * ideally the test/train data should look like the real-time data (new slice every few seconds)
 * write more tests!
-* multiproc.lock for lock
-
-
-## Work-around'd:
-
-* there is a concurrent access issue with the hdf database (core/graph)
+* pip package
+* refactor graph: one page per crypto, one page for all at once, one menu page
+* add type hints
+* cache db data (~1 week in dry-run mode, 1 read needed, then fetch to cache)

config/babao.conf

@@ -8,8 +8,10 @@
 QUOTE = EUR
 
 # the crypto(s) asset(s) you want to use, should be a space-separated list of:
-# ETC ETH LTC REP XBT XLM XMR XRP ZEC
-CRYPTOS = ETH LTC XBT
+# BCH DASH EOS GNO ETC ETH LTC REP XBT XLM XMR XRP ZEC
+# note: for BCH, DASH, EOS, or GNO crypto trading,
+#       only EUR and USD are available as quote (kraken)
+CRYPTOS = DASH LTC ETH XBT XMR XRP
 
 # time interval for data resampling (minutes)
 TIME_INTERVAL = 60

setup.cfg

@@ -8,14 +8,22 @@ description-file = README.md
 optimize = 2
 
 [flake8]
-max-line-length = 80
+# select = C,E,F,W,B,B9
+ignore = E722, W503
 exclude = tests, build, dist, .git, __pycache__, .eggs
+format = pylint
 show_source = True
-ignore = E722, W503
+max-line-length = 80
+max-complexity = 10
 
 [pylint]
 output-format = colorized
 max-line-length = 80
-disable=invalid-name,global-statement,invalid-unary-operand-type
-notes=FIXME,DEBUG #,TODO
-score=no
+disable = global-statement,  # invalid-name
+notes = FIXME, DEBUG
+score = no
+function-naming-style = camelCase
+method-naming-style = camelCase
+module-rgx = ^[_a-z][_a-zA-Z]*$  # TODO
+variable-rgx = ^[_a-z]*$
+argument-rgx = ^[_a-z]*$

setup.py

@@ -85,8 +85,11 @@
                 'pytest',
                 'pylint',
                 'flake8',
+                'flake8-bugbear',
                 'coveralls',
-                'sphinx'
+                'sphinx',
+                'pyre-check',
+                'mypy'
             ],
         },
         setup_requires=['pytest-runner'],

src/babao/babao.py

@@ -20,16 +20,18 @@
 # import babao; args = babao.babao._init(["-vv", "d"]); args.func(args)
 
 from multiprocessing import Process, Lock
+
 from prwlock import RWLock
 
-import babao.utils.log as log
-import babao.utils.file as fu
-import babao.utils.lock as lock
-# import babao.utils.signal as sig
+import babao.arg as arg
 import babao.config as conf
-import babao.parser as pars
 import babao.inputs.ledger.ledgerManager as lm
-import babao.strategy.modelManager as modelManager
+import babao.utils.date as du
+import babao.utils.file as fu
+import babao.utils.lock as lock
+import babao.utils.log as log
+import babao.utils.signal as sig
+from babao.models.rootModel import RootModel
 
 
 def _launchGraph():
@@ -57,28 +59,31 @@ def _kthxbye():
 def _init(args=None):
     """Initialize config and parse argv"""
 
-    args = pars.parseArgv(args)
+    args = arg.parseArgv(args)
     log.initLogLevel(args.verbose, args.quiet)
     conf.readConfigFile(args.func.__name__)
-
     if not lock.tryLock(conf.LOCK_FILE) and not args.fuckit:
         log.error("Lock found (" + conf.LOCK_FILE + "), abort.")
-    if args.func.__name__ not in ["train", "backtest"]:
+
+    if args.func.__name__ in ["train", "backtest"]:
+        du.setTime(du.EPOCH)
+    else:
         log.setLock(Lock())
     if args.graph:
         fu.setLock(RWLock())
     fu.initStore(conf.DB_FILE)
-    lm.initLedger(
+
+    if args.func.__name__ in ["train", "backtest"]:
+        du.setTime(du.EPOCH)
+    lm.initLedgers(
         simulate=args.func.__name__ != "wetRun",
-        log_to_file=args.func.__name__ != "train",
+        log_to_file=args.func.__name__ not in ["train", "backtest"]
     )
-    try:
-        modelManager.loadModels()
-    except FileNotFoundError:
-        log.warning("No model found.")
-    if args.graph:
+    RootModel()
+
+    if args.graph and args.func.__name__ != "train":
         _launchGraph()
-    # sig.catchSignal()
+    sig.catchSignal()
 
     return args