feat(*): new datasource api with transparent backtest to live transit…

…ion, explicit market scheme structure and management, various improvements related to data and apis

Dockerfile

@@ -9,15 +9,13 @@ FROM hivetech/pyscience
 MAINTAINER Xavier Bruhiere <xavier.bruhiere@gmail.com>
 
 # Local settings
-RUN apt-get install -y wget git-core libssl-dev
-#RUN apt-get install -y language-pack-fr wget git-core
+RUN apt-get update && \
+  apt-get install -y language-pack-fr wget git-core libssl-dev
 #ENV LANGUAGE fr_FR.UTF-8
 #ENV LANG fr_FR.UTF-8
 #ENV LC_ALL fr_FR.UTF-8
 #RUN locale-gen fr_FR.UTF-8 && dpkg-reconfigure locales
 
-#RUN pip install --use-mirrors intuition
-#RUN pip install --use-mirrors insights
 RUN git clone https://github.com/hackliff/intuition.git -b develop --depth 1 && \
   cd intuition && python setup.py install
 

Makefile

@@ -25,6 +25,11 @@ package:
 	python setup.py sdist upload
 
 tests: warn_missing_linters
+	@hr '-'
+	# TODO Recursively analyze all files and fail on conditions
+	@echo -e '\tChecking complexity ...'
+	@hr '-'
+	radon cc -ana intuition/core/engine.py
 	@hr '-'
 	@echo -e '\tChecking requirements ...'
 	@hr '-'

config/live.json

@@ -0,0 +1,25 @@
+
+{
+  "universe": "nasdaq,5",
+  "end": "17h30",
+  "modules": {
+    "manager": "insights.managers.fair.Fair",
+    "algorithm": "insights.algorithms.dummy.Random.BuyAndHold",
+    "data": "insights.sources.hybridforex.ForexRates"
+  },
+  "algorithm": {
+    "start_day": -1,
+    "rate": -1,
+    "hipchat": false,
+    "notify": false,
+    "interactive": false,
+    "save": false,
+  },
+  "manager": {
+    "cash": 10000,
+    "perc_sell": 1.0,
+    "max_weight": 0.3,
+    "sell_scale": 100,
+    "buy_scale": 150
+  }
+}

data/market.yml

@@ -0,0 +1,171 @@
+---
+forex:
+  timezone: null
+  schedule: null
+  benchmark: null
+  pairs:
+    - eur/usd
+    - usd/jpy
+    - gbp/usd
+    - eur/gbp
+    - usd/chf
+    - eur/jpy
+    - eur/chf
+    - usd/cad
+    - aud/usd
+    - gbp/jpy
+    - aud/cad
+    - aud/chf
+    - aud/jpy
+    - aud/nzd
+    - cad/chf
+    - chf/jpy
+    - eur/aud
+    - eur/cad
+    - eur/nok
+    - eur/nzd
+    - gbp/cad
+    - gbp/chf
+    - nzd/jpy
+    - nzd/usd
+    - usd/nok
+    - usd/sek
+stocks:
+    paris:
+      # Paris not supported yet
+      timezone: Europe/London
+      schedule: '8h,16h30'
+      code: epa
+      benchmark: fchi
+      cac40:
+        #fp:
+          #name: Total
+          #sector: oil
+        vk:
+          name: Vallourec
+          sector: engineering
+        ml:
+          name: Michelin
+          sector: automobile
+        #sol:
+          #name: Solvay
+          #sector: chemistry
+        or:
+          name: L'Oréal
+          sector: home,hygiene
+        #edf:
+          #name: EDF - Electricite De France
+          #sector: energy,public utility
+        viv:
+          name: Vivendi
+          sector: medias
+        #ul:
+          #name: Unibail-Rodamco
+          #sector: real estate
+        #mt:
+          #name: ArcelorMittal
+          #sector: or,precious materials
+        ri:
+          name: Pernod-Ricard
+          sector: food,drink
+        #bn:
+          #name: Danone
+          #sector: food,drink
+        ead:
+          name: Airbus Group
+          sector: transport
+        #ef:
+          #name: Essilor International
+          #sector: services
+        saf:
+          name: Safran
+          sector: engineering
+        #gsz:
+          #name: GDF Suez
+          #sector: gaz,electricity,public,services
+        #lr1:
+          #name: Legrand
+          #sector: electronic
+        san:
+          name: Sanofi
+          sector: pharmacy
+        su:
+          name: Schneider Electric
+          sector: engineering
+        sgo:
+          name: Saint-Gobain
+          sector: engineering
+        #gt1:
+          #name: Gemalto
+          #sector: electronic
+        ai:
+          name: Air Liquide
+          sector: chemistry
+        #vie:
+          #name: Veolia Environment
+          #sector: gaz,electricity,services
+        #mv:
+          #name: LVMH - Moet Hennessy Louis Vuitton
+          #sector: sell
+        pub:
+          name: Publicis Groupe
+          sector: medias
+        #dg:
+          #name: Vinci
+          #sector: engineering
+        #pp:
+          #name: Kering
+          #sector: construction,materials,real estate
+        tec:
+          name: Technip
+          sector: engineering
+        rno:
+          name: Renault
+          sector: automobile
+        #cap:
+          #name: Cap Gemini
+          #sector: information technology
+        #alu:
+          #name: Alcatel-Lucent
+          #sector: telecommunications
+        alo:
+          name: Alstom
+          sector: engineering
+        cs:
+          name: AXA
+          sector: finance
+        #fte:
+          #name: Orange
+          #sector: telecommunications
+        #ca:
+          #name: Carrefour
+          #sector: sell
+        #bnp:
+          #name: BNP Paribas
+          #sector: finance
+        #gle:
+          #name: Societe Generale
+          #sector: finance
+        #aca:
+          #name: Credit agricole
+          #sector: finance
+        lg:
+          name: Lafarge
+          sector: construction,materials,real estate
+        en:
+          name: Bouygues
+          sector: engineering
+        #ac:
+          #name: Accor
+          #sector: services
+      sbf120: []
+      srd: []
+    nasdaq:
+      code: NASDAQ
+      timezone: US/Eastern
+      benchmark: ^GSPC
+      nasdaq100: []
+      others: []
+    london:
+      ftse100: []
+      others: []

dev-requirements.txt

@@ -4,3 +4,4 @@ flake8==2.1.0
 coveralls==0.4.1
 ipdb==0.8
 piprot==0.5.0
+radon==0.5

intuition/analysis.py

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+# vim:fenc=utf-8
+
+'''
+  Intuition analysis module
+  -------------------------
+
+  Provides high level building blocks to extract insights from various datasets
+
+  :copyright (c) 2014 Xavier Bruhiere
+  :license: Apache 2.0, see LICENSE for more details.
+'''
+
+import os
+import shutil
+import rpy2.robjects as robjects
+import dna.logging
+
+log = dna.logging.logger(__name__)
+
+
+class Stocks(object):
+    ''' Produce R report of stocks opportunities '''
+    knitr_report = '~/.intuition/assets/report.rnw'
+
+    def __init__(self, report_template=None):
+        log.info('loading R context')
+        self.r = robjects.r
+        self.report_template = report_template \
+            or os.path.expanduser(self.knitr_report)
+        self.r('require("knitr")')
+
+    def clean(self, everything=False):
+        log.debug('cleaning garbage')
+        for extension in ['aux', 'log', 'out', 'tex']:
+            os.remove('report.{}'.format(extension))
+        if everything:
+            os.remove('report.{}'.format('pdf'))
+
+        shutil.rmtree('figure')
+
+    def process(self):
+        log.info('processing report')
+        self.r('knit2pdf("{}")'.format(self.report_template))

intuition/api/algorithm.py

@@ -17,8 +17,12 @@
 import abc
 import datetime as dt
 from zipline.algorithm import TradingAlgorithm
-from zipline.sources import DataFrameSource
+import zipline.finance.commission as commission
 from intuition.errors import AlgorithmEventFailed
+import insights.plugins.database as database
+import insights.plugins.mobile as mobile
+import insights.plugins.hipchat as hipchat
+import insights.plugins.messaging as msg
 
 
 class TradingFactory(TradingAlgorithm):
@@ -40,7 +44,6 @@ class TradingFactory(TradingAlgorithm):
     auto = False
 
     def __init__(self, *args, **kwargs):
-        self.data_generator = DataFrameSource
         self.realworld = kwargs['properties'].get('realworld')
         TradingAlgorithm.__init__(self, *args, **kwargs)
 
@@ -49,6 +52,24 @@ def _is_interactive(self):
         return not (
             self.realworld and (dt.date.today() > self.datetime.date()))
 
+    def use_default_middlewares(self, properties):
+        if properties.get('interactive'):
+            self.use(msg.RedisProtocol(self.identity).check)
+        device = properties.get('mobile')
+        if device:
+            self.use(mobile.AndroidPush(device).notify)
+        if properties.get('save'):
+            self.use(database.RethinkdbBackend(
+                table=self.identity, db='portfolios', reset=True)
+                .save_portfolio)
+        hipchat_room = properties.get('hipchat')
+        if hipchat_room:
+            self.use(hipchat.Bot(
+                hipchat_room, name=self.identity).notify)
+
+        self.set_commission(commission.PerTrade(
+            cost=properties.get('commission', 2.5)))
+
     def use(self, func, when='whenever'):
         ''' Append a middleware to the algorithm '''
         #NOTE A middleware Object ?
@@ -59,25 +80,14 @@ def use(self, func, when='whenever'):
             'args': func.func_code.co_varnames,
             'when': when})
 
-    def trade(self, source, sim_params=None):
-        if isinstance(source, dict):
-            source = self.data_generator(source)
-
-        return self.run(source, sim_params)
-
-    #TODO How can I use several sources ?
-    def set_data_generator(self, generator_class):
-        ''' Register a data source to the algorithm '''
-        self.data_generator = generator_class
-
     #NOTE I'm not superfan of initialize + warm
     def warm(self, data):
         ''' Called at the first handle_data frame '''
         pass
 
     @abc.abstractmethod
     def event(self, data):
-        ''' Users should overwrite this method '''
+        ''' User should overwrite this method '''
         pass
 
     def handle_data(self, data):
@@ -121,7 +131,7 @@ def process_orders(self, orderbook):
         ''' Default and costant orders processor. Overwrite it for more
         sophisticated strategies '''
         for stock, alloc in orderbook.iteritems():
-            self.logger.debug('{}: Ordered {} {} stocks'.format(
+            self.logger.info('{}: Ordered {} {} stocks'.format(
                 self.datetime, stock, alloc))
             if isinstance(alloc, int):
                 self.order(stock, alloc)