diff --git a/.gitignore b/.gitignore index 3bb96ee..772d262 100755 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,4 @@ tmp/* .envrc dist +bkp.README.md diff --git a/Dockerfile b/Dockerfile index a4e3d71..6651f01 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,28 +1,34 @@ # hivetech/intuition image # A raring box with Intuition (https://github.com/hackliff/intuition installed # and ready to use +# docker run \ +# -e DB_HOST=172.17.0.4 \ +# -e LANG=fr_FR.UTF-8 \ +# -e LOG=info \ +# -e QUANDL_API_KEY=$QUANDL_API_KEY \ +# hivetech/intuition --context insights.contexts.mongodb.MongodbContext://192.168.0.19:27017/intuition/contexts/bt-yahoo --id chuck --bot --showlog # VERSION 0.1.0 # Administration -# hivetech/pyscience is an ubuntu 13.10 image with most popular python packages +# hivetech/pyscience is an ubuntu 12.04 image with most popular python packages FROM hivetech/pyscience MAINTAINER Xavier Bruhiere # Local settings -RUN apt-get update && \ - apt-get install -y language-pack-fr wget git-core libssl-dev +#RUN apt-get update && \ + #apt-get install -y language-pack-fr wget git-core libssl-dev #ENV LANGUAGE fr_FR.UTF-8 #ENV LANG fr_FR.UTF-8 #ENV LC_ALL fr_FR.UTF-8 #RUN locale-gen fr_FR.UTF-8 && dpkg-reconfigure locales -RUN git clone https://github.com/hackliff/intuition.git -b develop --depth 1 && \ - cd intuition && python setup.py install +RUN git clone https://github.com/intuition-io/intuition.git -b develop --depth 1 && \ + cd /intuition && python setup.py install # Install modules -RUN git clone https://github.com/hackliff/insights.git -b develop --depth 1 && \ +RUN git clone https://github.com/intuition-io/insights.git -b develop --depth 1 && \ + apt-get update && apt-get install -y libreadline-dev && \ cd insights && python setup.py install - # Install R libraries RUN wget -qO- http://bit.ly/L39jeY | R --no-save diff --git a/Makefile b/Makefile index 399d48f..bdfe964 100755 --- a/Makefile +++ b/Makefile @@ -25,23 +25,15 @@ package: python setup.py sdist upload tests: warn_missing_linters - @hr '-' # TODO Recursively analyze all files and fail on conditions - @echo -e '\tChecking complexity ...' - @hr '-' + @echo -e '\tChecking complexity (experimental) ...' radon cc -ana intuition/core/engine.py - @hr '-' @echo -e '\tChecking requirements ...' - @hr '-' #TODO Fail if outdated piprot --outdated requirements.txt dev-requirements.txt - @hr '-' @echo -e '\tChecking syntax ...' - @hr '-' flake8 tests intuition - @hr '-' @echo -e '\tRunning tests ...' - @hr '-' nosetests -w tests --with-yanc --with-coverage --cover-package=intuition watch: warn_missing_linters diff --git a/README.md b/README.md index bebf883..02b3f88 100755 --- a/README.md +++ b/README.md @@ -1,10 +1,11 @@ Intuition ========= -> Automated quantitative trading kit, for hackers +> Quantitative trading kit, for hackers -![Dashboard](https://raw.github.com/hivetech/hivetech.github.io/master/images/QuantDashboard.png) + +![Dashboard](http://intuition.io/img/flat-showoff.png) > The intuitive mind is a sacred gift and the rational mind is a faithful > servant. We have created a society that honors the servant and has forgotten @@ -35,16 +36,16 @@ portfolio optimization, genetic optimization, sentiment analysis from twitter, . Features -------- -* Highly configurable trading environment, powered by [zipline](https://github.com/quantopian/zipline) +* Highly configurable trading environment, powered by [zipline][9] * From instant kickstart to full control * Made to let you tweak algorithms, portfolio manager, data sources, contexts and plugins -* Already includes many +* [Plugin friendly][8]. Enjoy mail or mobile notifications. +* Already includes [many][2] * Experimental live trading on different markets (Nyse, Nasdaq, CAC40 and Forex for now) * Experimental R integration in your algorithms * Results analyser -* Mail and Android notifications (for now with the help of freely available [NotifyMyAndroid](http://www.notifymyandroid.com/) or [PushBullet](https://www.pushbullet.com)) * Financial library, with common used trading functions, data fetchers, ... used for example to solve Coursera econometrics assignments -* Easy to use data management, powered by [rethinkdb](rethinkdb.com) +* Easy to use data management, powered by [rethinkdb][6] * [Docker](docker.io) support for development workflow and deployment * Kind of a CI showcase as I am testing [travis](https://travis-ci.org), [wercker](wercker.com), [shippable](shippable.com), [drone.io](shippable.com), [coveralls](coveralls.io) and [landscape](landscape.io) @@ -65,197 +66,40 @@ Status [Development Board][1] [Chat about the project][3] -**Attention** Project is in an *early alpha*, and under heavy development. - The new version 0.3.0 revises a lot of code : - -* Algoithms, managers and data sources have their [own repository][2] -* More powerful API to build custom versions of them -* The context module now handles configuration -* [Shiny](http://www.rstudio.com/shiny/) interface, [Dashboard](http://fdietz.github.io/team_dashboard/) and clustering will have their intuition-plugins repository (soon) -* ZeroMQ messaging is for now removed but might be back for inter-algo communication -* So is MySQL, that has been removed and will be re-implemented as a [data plugin](https://github.com/hackliff/insights/tree/master/insights/plugins) -* But currently it has been replaced by [Rethinkdb](rethinkdb.com) -* Installation is much simpler and a docker image is available for development and deployment -* More intuitive configuration splitted between the context mentioned, command line argument and environment variables -* And a lot (I mean A LOT) of house keeping and code desgin stuff - - -Installation ------------- - -You are just a few steps away from algoritmic trading. Choose one of the -following installation method - -* The usual way - -```console -$ pip install intuition -$ # Optionnaly, install offcial algorithms, managers, ... -$ pip install insights -``` - -* One-liner for the full installation (i.e. with packages and official - [modules](https://github.com/hackliff/insights)) - -```console -$ wget -qO- http://bit.ly/1anxGhf | sudo FULL_INTUITION=true bash -$ # ... Go grab a coffee -``` - -* From source - -```console -$ git clone https://github.com/hackliff/intuition.git -$ cd intuition && sudo make -``` - -* Sexy, early-adopter style - -```console -$ docker pull hivetech/intuition -``` Getting started --------------- -Intuition wires 4 primitives to build up the system : A data source generates -events, processed by the algorithm, that can optionnaly use a portfolio manager -to compute assets allocation. They are configured through a Context, while -third party services use environment variables (take a look in -config/local.env). - -The following example trades in real time forex, with a simple buy and hold -algorithm and a portfolio manager that allocates same amount for each asset. -Their configuration below is stored in a json file. The `--bot` flag allows -the portfolio to process orders on its own. - -```console -$ intuition --context file::liveForex.json --id chuck --showlog --bot -``` - -```json -{ - "id": "liveForex", - "end": "22h", - "universe": "forex,5", - "algorithm": { - "notify": "", - "save": false - }, - "manager": { - "cash": 10000, - "buy_scale": 150, - "max_weight": 0.3 - }, - "modules": { - "algorithm": "insights.algorithms.buyandhold.BuyAndHold", - "data": "insights.sources.live.forex.ForexLiveSource", - "manager": "insightsmanagers.fair.Fair" - } -} -``` - -Note that in the current implementation, Nasdaq, Nyse, Cac 40 and Forex markets -are available. +Learn more and find the (in progress) documentation at http://doc.intuition.io. -Alternatively you can use docker. Here we also fire up a [rethinkdb](rethinkdb.com) -database to store portfolios while trading, and -[mongodb](http://www.mongodb.org/) to store configurations. +You can follow the development on this [trello board][1] and chat about the +project on [Gitter][3]. -```console -$ docker run -d -name mongodb -p 27017:27017 -p 28017:28017 waitingkuo/mongodb - -$ docker run -d -name rethinkdb crosbymichael/rethinkdb --bind all - -$ docker run \ - -e PUSHBULLET_API_KEY=$PUSHBULLET_API_KEY \ - -e QUANDL_API_KEY=$QUANDL_API_KEY \ - -e MAILGUN_API_KEY=$MAILGUN_API_KEY \ - -e TRUEFX_API=$TRUEFX_API \ - -e DB_HOST=$DB_HOST \ - -e DB_PORT=$DB_PORT \ - -e DB_NAME=$DB_NAME \ - -e LOG=debug \ - -e LANGUAGE="fr_FR.UTF-8" \ - -e LANG="fr_FR.UTF-8" \ - -e LC_ALL="fr_FR.UTF-8" \ - hivetech/intuition --context mongodb::${host_ip}:27017/{{ config_id }} --showlog -``` - -For Hackers ------------ - -You can easily work out and plug your own strategies : - -* [Algorithm API](https://github.com/hackliff/insights/blob/master/insights/algorithms/readme.md) -* [Portfolio API](https://github.com/hackliff/insights/blob/master/insights/managers/readme.md) -* [Data API](https://github.com/hackliff/insights/blob/master/insights/sources/readme.md) -* [Contexts](https://github.com/hackliff/insights/blob/master/insights/contexts/readme.md) -* [Middlewares](https://github.com/hackliff/insights/blob/master/insights/contexts/readme.md) - -Either clone the [insights repository][2] -and hack it or start from scratch. Just make sure the modules paths you give in -the configuration are in the python path. - - -The [provided](https://github.com/hackliff/intuition/blob/master/app/intuition) -``intuition`` command does already a lot of things but why not improve it or -write your own. Here is a minimal implementation, assuming you installed -[insights][2]. - -```python -from datetime import datetime -from intuition.core.engine import Simulation - -data = {'universe': 'nasdaq,10', - 'index': pd.date_range(datetime.now(), datetime(2014, 1, 7))} - -modules = { - 'algorithm': 'algorithms.movingaverage.DualMovingAverage', - 'manager': 'managers.gmv.GlobalMinimumVariance', - 'data': 'sources.live.Equities.EquitiesLiveSource'}}) - -engine = Simulation() - -# Use the configuration to prepare the trading environment -engine.configure_environment(data['index'][-1], 'nasdaq') -engine.build('chuck_norris', modules) -analyzes = engine.run(data, auto=True) - -# Explore the analyzes object -print analyzes.overall_metrics('one_month') -print analyzes.results.tail() -``` +A webapp built on top of Intuition is also in preparation, get your early +ticket at http://intuition.io ! Contributing ------------ -> Fork, implement, add tests, pull request, get my everlasting thanks and a -> respectable place here [=)](https://github.com/jondot/groundcontrol) +Contributors are happily welcome, [here is the place to start][10]. License ------- Copyright 2014 Xavier Bruhiere -Intuition is available under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.html). +Intuition is available under the [Apache License, Version 2.0][5]. --------------------------------------------------------------- -Credits -------- - -* [Zipline](http://github.com/quantopian/zipline) -* [Quantopian](http://www.quantopian.com/) -* [Pandas](http://github.com/pydata/pandas) -* [R-bloggers](http://www.r-bloggers.com/) -* [QSTK](https://github.com/tucker777/QSTK) -* [Coursera](http://www.coursera.org/) -* [Udacity](http://www.udacity.com/) -* [Babypips](http://www.babypips.com/) -* [GLMF](http://www.unixgarden.com/) - [1]: https://trello.com/b/WvJDlynt/intuition -[2]: https://github.com/hackliff/insights -[3]: https://gitter.im/hackliff/intuition +[2]: https://github.com/intuition-io/insights +[3]: https://gitter.im/intuition-io +[4]: https://github.com/jondot/groundcontrol +[5]: http://www.apache.org/licenses/LICENSE-2.0.html +[6]: rethinkdb.com +[7]: http://www.mongodb.org +[8]: https://github.com/intuition-io/insights/tree/develop/insights/plugins +[9]: https://github.com/quantopian/zipline +[10]: http://doc.intuition.io/articles/contributors.html diff --git a/config/backtest.yaml b/config/backtest.yaml deleted file mode 100755 index 430a4a5..0000000 --- a/config/backtest.yaml +++ /dev/null @@ -1,26 +0,0 @@ -universe: nasdaq,5 -start: 2011-05-10 -end: 2013-11-10 -modules: - manager: insights.managers.constant.Constant - algorithm: insights.algorithms.dummy.Random - data: insights.sources.backtest.database.RethinkdbPrices - -data: - database: quotes - -algorithm: - start_day: -1 - rate: -1 - notify: - interactive: false - save: false - gradient_iterations: 5 - long_window: 30 - short_window: 20 - threshold: 0.5 - -manager: - buy_amount: 100 - sell_amount: 100 - cash: 2027 diff --git a/config/backtest.yml b/config/backtest.yml new file mode 100755 index 0000000..c0fb1e0 --- /dev/null +++ b/config/backtest.yml @@ -0,0 +1,12 @@ +universe: stocks:paris:cac40,5 +start: 2013-02-10 +end: 2013-11-10 +modules: + manager: insights.managers.linear.Constant + algorithm: insights.algorithms.dummy.Random + backtest: insights.sources.backtest.yahoo.YahooPrices + +manager: + buy_amount: 150 + sell_amount: 100 + cash: 20027 diff --git a/dev-requirements.txt b/dev-requirements.txt index 777c335..38f7c96 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,7 @@ -nose==1.3.0 +nose==1.3.1 yanc==0.2.4 flake8==2.1.0 coveralls==0.4.1 ipdb==0.8 piprot==0.5.0 -radon==0.5 +radon==0.5.1 diff --git a/intuition/api/context.py b/intuition/api/context.py index 289c060..354924c 100644 --- a/intuition/api/context.py +++ b/intuition/api/context.py @@ -37,7 +37,7 @@ def parse_storage(storage): return { 'uri': uri, 'path': path, - 'param': params + 'params': params } diff --git a/intuition/constants.py b/intuition/constants.py index e7c90c6..d8234da 100644 --- a/intuition/constants.py +++ b/intuition/constants.py @@ -30,7 +30,9 @@ # TODO It will be at least one Optional('backtest'): basestring, Optional('live'): basestring, - Optional('manager'): Or(basestring, None)}}) + Optional('manager'): Or(basestring, None) + } +}) PANDAS_FREQ = { diff --git a/intuition/core/configuration.py b/intuition/core/configuration.py index abac16b..038b77b 100755 --- a/intuition/core/configuration.py +++ b/intuition/core/configuration.py @@ -87,8 +87,8 @@ def _validate(self, config): log.info('validating configuration', config=config) try: assert intuition.constants.CONFIG_SCHEMA.validate(config) - except: - raise InvalidConfiguration(config=config, module=__name__) + except Exception as error: + raise InvalidConfiguration(config=config, reason=error) def logfile(session_id): diff --git a/intuition/data/remote.py b/intuition/data/remote.py index 46a06ba..240eb0e 100755 --- a/intuition/data/remote.py +++ b/intuition/data/remote.py @@ -150,7 +150,6 @@ def snapshot_yahoo_pandas(symbols): #NOTE Can use symbol with market: 'goog:nasdaq', any difference ? @use_google_symbol -# Index symbol ex: PX1 def snapshot_google_light(symbols): payload = {'client': 'ig', 'q': ','.join(symbols)} response = requests.get(finance_urls['snapshot_google_light'], @@ -234,11 +233,6 @@ def lookup_symbol(company): u'symbol': u'AAPL', u'type': u'Equity'} ''' - infos = {} request = requests.get(finance_urls['info_lookup'].format(company)) - if request.ok: - infos = json.loads(request.text[39:-1])["ResultSet"]["Result"][0] - infos["market"] = infos.pop("exchDisp") - infos["type"] = infos.pop("typeDisp") - - return infos + return json.loads(request.text[39:-1])["ResultSet"]["Result"] \ + if request.ok else {'error': request.reason} diff --git a/intuition/data/utils.py b/intuition/data/utils.py index e07f457..2ba8e1f 100755 --- a/intuition/data/utils.py +++ b/intuition/data/utils.py @@ -9,13 +9,7 @@ :license: Apache 2.0, see LICENSE for more details. ''' - -import os -import random -import yaml import pandas as pd -import intuition.data.data as data -from intuition.errors import ExchangeIsClosed def apply_mapping(raw_row, mapping): diff --git a/intuition/errors.py b/intuition/errors.py index 60dbcb5..1e17e93 100755 --- a/intuition/errors.py +++ b/intuition/errors.py @@ -14,7 +14,7 @@ class InvalidConfiguration(dna.errors.FactoryError): - msg = "invalid configuration: {config} ({module})" + msg = "invalid configuration: {reason} ({config})" class PortfolioOptimizationFailed(dna.errors.FactoryError): diff --git a/intuition/utils.py b/intuition/utils.py index 8f5cda9..ddf178e 100755 --- a/intuition/utils.py +++ b/intuition/utils.py @@ -51,8 +51,10 @@ def _detect_timezone(): Experimental and temporary (since there is a world module) get timezone as set by the system ''' - locale_code = locale.getdefaultlocale()[0] - return str(pytz.country_timezones[locale_code[-2:]][0]) + default_timezone = 'America/New_York' + locale_code = locale.getdefaultlocale() + return default_timezone if not locale_code else \ + str(pytz.country_timezones[locale_code[0][-2:]][0]) def build_date_index(start='', end='', freq='D'): diff --git a/requirements.txt b/requirements.txt index d135470..84fb0ad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,16 @@ beautifulsoup4==4.3.2 ystockquote -blist==1.3.4 +blist==1.3.6 schematics==0.9-4 schema==0.2.0 Cython==0.20.1 -numpy==1.8.0 +numpy==1.8.1 python-dateutil==2.2 -pytz==2013.9 -Quandl==1.8.1 +pytz==2014.2 +Quandl==1.9.1 requests==2.2.1 -six==1.5.2 -dna +six==1.6.1 +PyYAML==3.11 +dna==0.0.2 -e git+https://github.com/pydata/pandas.git@master#egg=pandas-0.13.0-27-gf7aeaeb -e git+https://github.com/quantopian/zipline.git@master#egg=zipline-0.5.11.dev diff --git a/setup.py b/setup.py index f141373..f67530f 100644 --- a/setup.py +++ b/setup.py @@ -33,18 +33,19 @@ def get_requirements(): requires = [ 'beautifulsoup4>=4.3.2', - 'blist>=1.3.4', + 'blist>=1.3.6', 'Cython>=0.20.1', 'ystockquote', - 'numpy>=1.8.0', + 'numpy>=1.8.1', 'schematics>=0.9-4', 'schema==0.2.0', 'python-dateutil>=2.2', - 'pytz>=2013.9', - 'Quandl>=1.8.1', - 'dna', + 'pytz>=2014.2', + 'PyYAML>=3.11', + 'Quandl>=1.9.1', + 'dna>=0.0.2', 'requests>=2.2.1', - 'six>=1.5.2', + 'six>=1.6.1', 'zipline>=0.5.11.dev', 'pandas>=0.13.0.dev'] @@ -86,8 +87,7 @@ def long_description(): 'Topic :: Scientific/Engineering :: Information Analysis', 'Topic :: System :: Distributed Computing', ], - data_files=[(os.path.expanduser('~/.intuition/data'), glob('./data/*')), - (os.path.expanduser('~/.intuition/logs'), ['logs.doc.md'])], + data_files=[(os.path.expanduser('~/.intuition/data'), glob('./data/*'))], dependency_links=[ 'http://github.com/pydata/pandas/tarball/master#egg=pandas-0.13.0.dev', 'http://github.com/quantopian/zipline/tarball/master#egg=zipline-0.5.11.dev'] diff --git a/tests/test_setup.py b/tests/test_setup.py index ece2a0c..c6dd564 100644 --- a/tests/test_setup.py +++ b/tests/test_setup.py @@ -14,16 +14,10 @@ class ConfigurationTestCase(unittest.TestCase): # FIXME Needs insights installed for these tests good_driver = '{}://{}'.format( 'insights.contexts.file.FileContext', - 'intuition.io/../config/backtest.yaml') - good_driver = '{}://{}'.format( - 'insights.contexts.file.FileContext', - 'intuition.io/../config/backtest.yaml') - good_driver = '{}://{}'.format( - 'insights.contexts.file.FileContext', - 'intuition.io/../config/backtest.yaml') - bad_driver = 'no.file.FileContext://intuition.io/../config/backtest.yaml' - bad_config = 'insights.contexts.file.FileContext://intuition.io/fake.yaml' - bad_formatted_config = 'insights.contexts.file.FileContext::/fake.yaml' + 'intuition.io/../config/backtest.yml') + bad_driver = 'no.file.FileContext://intuition.io/../config/backtest.yml' + bad_config = 'insights.contexts.file.FileContext://intuition.io/fake.yml' + bad_formatted_config = 'insights.contexts.file.FileContext::/fake.yml' def test_logfile(self): logfile = configuration.logfile('fake_id') @@ -52,11 +46,9 @@ def test_validate_bad_config(self): def test_validate_good_config(self): good_config = { 'universe': 'nasdaq,4', - 'exchange': 'nasdaq', 'index': pd.date_range('2014/2/3', periods=30), 'modules': { - 'algorithm': 'dualma', - 'data': 'database' + 'algorithm': 'dualma' } } ctx = configuration.Context(self.bad_driver) @@ -70,7 +62,7 @@ def test_loaded_configuration(self): with configuration.Context(self.good_driver) as context: for field in ['manager', 'algorithm', 'data']: self.assertIn(field, context['strategy']) - for field in ['index', 'live', 'exchange']: + for field in ['index', 'live']: self.assertIn(field, context['config']) def test_absent_driver_context_load(self):