From 6d2a69d9f7261c8cd3bb12b139b63cb7c6e91adc Mon Sep 17 00:00:00 2001 From: hackliff Date: Fri, 28 Mar 2014 10:43:04 +0100 Subject: [PATCH] feat(*): new datasource api with transparent backtest to live transition, explicit market scheme structure and management, various improvements related to data and apis --- Dockerfile | 6 +- Makefile | 5 + config/backtest.json | 36 ---- config/live.json | 25 +++ data/market.yml | 171 ++++++++++++++++++ dev-requirements.txt | 1 + intuition/analysis.py | 44 +++++ intuition/api/algorithm.py | 40 ++-- intuition/api/context.py | 41 +---- intuition/api/{data_source.py => datafeed.py} | 136 +++++--------- intuition/cli.py | 58 +++--- intuition/constants.py | 7 +- intuition/core/analyzes.py | 13 +- intuition/core/configuration.py | 42 ++++- intuition/core/engine.py | 37 ++-- intuition/data/data.py | 25 +-- intuition/data/forex.py | 113 ++++++------ intuition/data/loader.py | 10 +- intuition/data/quandl.py | 117 +++++++----- intuition/data/remote.py | 13 +- intuition/data/universe.py | 119 ++++++++++++ intuition/data/utils.py | 78 +------- logs.doc.md | 8 - requirements.txt | 1 + setup.py | 6 +- tests/test_configuration.py | 55 ------ tests/test_setup.py | 79 ++++++++ tests/test_utils.py | 35 +++- wercker.yml | 3 +- 29 files changed, 792 insertions(+), 532 deletions(-) delete mode 100755 config/backtest.json create mode 100755 config/live.json create mode 100644 data/market.yml create mode 100644 intuition/analysis.py rename intuition/api/{data_source.py => datafeed.py} (51%) create mode 100755 intuition/data/universe.py delete mode 100644 logs.doc.md delete mode 100644 tests/test_configuration.py create mode 100644 tests/test_setup.py diff --git a/Dockerfile b/Dockerfile index 2c637d2..a4e3d71 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,15 +9,13 @@ FROM hivetech/pyscience MAINTAINER Xavier Bruhiere # Local settings -RUN apt-get install -y wget git-core libssl-dev -#RUN apt-get install -y language-pack-fr wget git-core +RUN apt-get update && \ + apt-get install -y language-pack-fr wget git-core libssl-dev #ENV LANGUAGE fr_FR.UTF-8 #ENV LANG fr_FR.UTF-8 #ENV LC_ALL fr_FR.UTF-8 #RUN locale-gen fr_FR.UTF-8 && dpkg-reconfigure locales -#RUN pip install --use-mirrors intuition -#RUN pip install --use-mirrors insights RUN git clone https://github.com/hackliff/intuition.git -b develop --depth 1 && \ cd intuition && python setup.py install diff --git a/Makefile b/Makefile index ca7a1a4..399d48f 100755 --- a/Makefile +++ b/Makefile @@ -25,6 +25,11 @@ package: python setup.py sdist upload tests: warn_missing_linters + @hr '-' + # TODO Recursively analyze all files and fail on conditions + @echo -e '\tChecking complexity ...' + @hr '-' + radon cc -ana intuition/core/engine.py @hr '-' @echo -e '\tChecking requirements ...' @hr '-' diff --git a/config/backtest.json b/config/backtest.json deleted file mode 100755 index 083d98b..0000000 --- a/config/backtest.json +++ /dev/null @@ -1,36 +0,0 @@ - -{ - "universe": "nasdaq,5", - "start": "2011-05-10", - "end": "2013-11-10", - "modules": { - "manager": "insights.managers.fair.Fair", - "algorithm": "insights.algorithms.dummy.Random.BuyAndHold", - "data": "insights.sources.backtest.yahoo.YahooPriceSource" - }, - "algorithm": { - "debug": true, - "save": false, - "rebalance_period": 15, - "refresh_period": 1, - "window_length": 60, - "stddev_window": 9, - "vwap_window": 5, - "long_window" : 25, - "short_window" : 5, - "threshold" : 0, - "ignored": ["volume", "pouet"], - "gradient_iterations": 5, - "signal_frontier": 0.5, - "decision_frontier": 0.5 - }, - "manager": { - "cash": 10000, - "load_backup": 0, - "loopback": 60, - "perc_sell": 1.0, - "max_weight": 0.3, - "sell_scale": 100, - "buy_scale": 150 - } -} diff --git a/config/live.json b/config/live.json new file mode 100755 index 0000000..fb41daa --- /dev/null +++ b/config/live.json @@ -0,0 +1,25 @@ + +{ + "universe": "nasdaq,5", + "end": "17h30", + "modules": { + "manager": "insights.managers.fair.Fair", + "algorithm": "insights.algorithms.dummy.Random.BuyAndHold", + "data": "insights.sources.hybridforex.ForexRates" + }, + "algorithm": { + "start_day": -1, + "rate": -1, + "hipchat": false, + "notify": false, + "interactive": false, + "save": false, + }, + "manager": { + "cash": 10000, + "perc_sell": 1.0, + "max_weight": 0.3, + "sell_scale": 100, + "buy_scale": 150 + } +} diff --git a/data/market.yml b/data/market.yml new file mode 100644 index 0000000..5e2650e --- /dev/null +++ b/data/market.yml @@ -0,0 +1,171 @@ +--- +forex: + timezone: null + schedule: null + benchmark: null + pairs: + - eur/usd + - usd/jpy + - gbp/usd + - eur/gbp + - usd/chf + - eur/jpy + - eur/chf + - usd/cad + - aud/usd + - gbp/jpy + - aud/cad + - aud/chf + - aud/jpy + - aud/nzd + - cad/chf + - chf/jpy + - eur/aud + - eur/cad + - eur/nok + - eur/nzd + - gbp/cad + - gbp/chf + - nzd/jpy + - nzd/usd + - usd/nok + - usd/sek +stocks: + paris: + # Paris not supported yet + timezone: Europe/London + schedule: '8h,16h30' + code: epa + benchmark: fchi + cac40: + #fp: + #name: Total + #sector: oil + vk: + name: Vallourec + sector: engineering + ml: + name: Michelin + sector: automobile + #sol: + #name: Solvay + #sector: chemistry + or: + name: L'Oréal + sector: home,hygiene + #edf: + #name: EDF - Electricite De France + #sector: energy,public utility + viv: + name: Vivendi + sector: medias + #ul: + #name: Unibail-Rodamco + #sector: real estate + #mt: + #name: ArcelorMittal + #sector: or,precious materials + ri: + name: Pernod-Ricard + sector: food,drink + #bn: + #name: Danone + #sector: food,drink + ead: + name: Airbus Group + sector: transport + #ef: + #name: Essilor International + #sector: services + saf: + name: Safran + sector: engineering + #gsz: + #name: GDF Suez + #sector: gaz,electricity,public,services + #lr1: + #name: Legrand + #sector: electronic + san: + name: Sanofi + sector: pharmacy + su: + name: Schneider Electric + sector: engineering + sgo: + name: Saint-Gobain + sector: engineering + #gt1: + #name: Gemalto + #sector: electronic + ai: + name: Air Liquide + sector: chemistry + #vie: + #name: Veolia Environment + #sector: gaz,electricity,services + #mv: + #name: LVMH - Moet Hennessy Louis Vuitton + #sector: sell + pub: + name: Publicis Groupe + sector: medias + #dg: + #name: Vinci + #sector: engineering + #pp: + #name: Kering + #sector: construction,materials,real estate + tec: + name: Technip + sector: engineering + rno: + name: Renault + sector: automobile + #cap: + #name: Cap Gemini + #sector: information technology + #alu: + #name: Alcatel-Lucent + #sector: telecommunications + alo: + name: Alstom + sector: engineering + cs: + name: AXA + sector: finance + #fte: + #name: Orange + #sector: telecommunications + #ca: + #name: Carrefour + #sector: sell + #bnp: + #name: BNP Paribas + #sector: finance + #gle: + #name: Societe Generale + #sector: finance + #aca: + #name: Credit agricole + #sector: finance + lg: + name: Lafarge + sector: construction,materials,real estate + en: + name: Bouygues + sector: engineering + #ac: + #name: Accor + #sector: services + sbf120: [] + srd: [] + nasdaq: + code: NASDAQ + timezone: US/Eastern + benchmark: ^GSPC + nasdaq100: [] + others: [] + london: + ftse100: [] + others: [] diff --git a/dev-requirements.txt b/dev-requirements.txt index 3564f91..777c335 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -4,3 +4,4 @@ flake8==2.1.0 coveralls==0.4.1 ipdb==0.8 piprot==0.5.0 +radon==0.5 diff --git a/intuition/analysis.py b/intuition/analysis.py new file mode 100644 index 0000000..ea71432 --- /dev/null +++ b/intuition/analysis.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# vim:fenc=utf-8 + +''' + Intuition analysis module + ------------------------- + + Provides high level building blocks to extract insights from various datasets + + :copyright (c) 2014 Xavier Bruhiere + :license: Apache 2.0, see LICENSE for more details. +''' + +import os +import shutil +import rpy2.robjects as robjects +import dna.logging + +log = dna.logging.logger(__name__) + + +class Stocks(object): + ''' Produce R report of stocks opportunities ''' + knitr_report = '~/.intuition/assets/report.rnw' + + def __init__(self, report_template=None): + log.info('loading R context') + self.r = robjects.r + self.report_template = report_template \ + or os.path.expanduser(self.knitr_report) + self.r('require("knitr")') + + def clean(self, everything=False): + log.debug('cleaning garbage') + for extension in ['aux', 'log', 'out', 'tex']: + os.remove('report.{}'.format(extension)) + if everything: + os.remove('report.{}'.format('pdf')) + + shutil.rmtree('figure') + + def process(self): + log.info('processing report') + self.r('knit2pdf("{}")'.format(self.report_template)) diff --git a/intuition/api/algorithm.py b/intuition/api/algorithm.py index 12f0929..d91d499 100755 --- a/intuition/api/algorithm.py +++ b/intuition/api/algorithm.py @@ -17,8 +17,12 @@ import abc import datetime as dt from zipline.algorithm import TradingAlgorithm -from zipline.sources import DataFrameSource +import zipline.finance.commission as commission from intuition.errors import AlgorithmEventFailed +import insights.plugins.database as database +import insights.plugins.mobile as mobile +import insights.plugins.hipchat as hipchat +import insights.plugins.messaging as msg class TradingFactory(TradingAlgorithm): @@ -40,7 +44,6 @@ class TradingFactory(TradingAlgorithm): auto = False def __init__(self, *args, **kwargs): - self.data_generator = DataFrameSource self.realworld = kwargs['properties'].get('realworld') TradingAlgorithm.__init__(self, *args, **kwargs) @@ -49,6 +52,24 @@ def _is_interactive(self): return not ( self.realworld and (dt.date.today() > self.datetime.date())) + def use_default_middlewares(self, properties): + if properties.get('interactive'): + self.use(msg.RedisProtocol(self.identity).check) + device = properties.get('mobile') + if device: + self.use(mobile.AndroidPush(device).notify) + if properties.get('save'): + self.use(database.RethinkdbBackend( + table=self.identity, db='portfolios', reset=True) + .save_portfolio) + hipchat_room = properties.get('hipchat') + if hipchat_room: + self.use(hipchat.Bot( + hipchat_room, name=self.identity).notify) + + self.set_commission(commission.PerTrade( + cost=properties.get('commission', 2.5))) + def use(self, func, when='whenever'): ''' Append a middleware to the algorithm ''' #NOTE A middleware Object ? @@ -59,17 +80,6 @@ def use(self, func, when='whenever'): 'args': func.func_code.co_varnames, 'when': when}) - def trade(self, source, sim_params=None): - if isinstance(source, dict): - source = self.data_generator(source) - - return self.run(source, sim_params) - - #TODO How can I use several sources ? - def set_data_generator(self, generator_class): - ''' Register a data source to the algorithm ''' - self.data_generator = generator_class - #NOTE I'm not superfan of initialize + warm def warm(self, data): ''' Called at the first handle_data frame ''' @@ -77,7 +87,7 @@ def warm(self, data): @abc.abstractmethod def event(self, data): - ''' Users should overwrite this method ''' + ''' User should overwrite this method ''' pass def handle_data(self, data): @@ -121,7 +131,7 @@ def process_orders(self, orderbook): ''' Default and costant orders processor. Overwrite it for more sophisticated strategies ''' for stock, alloc in orderbook.iteritems(): - self.logger.debug('{}: Ordered {} {} stocks'.format( + self.logger.info('{}: Ordered {} {} stocks'.format( self.datetime, stock, alloc)) if isinstance(alloc, int): self.order(stock, alloc) diff --git a/intuition/api/context.py b/intuition/api/context.py index 11342a8..289c060 100644 --- a/intuition/api/context.py +++ b/intuition/api/context.py @@ -18,8 +18,6 @@ import pandas as pd import dna.logging import intuition.utils -from intuition.errors import InvalidConfiguration -import intuition.data.utils as datautils EMPTY_DATES = pd.date_range('2000/01/01', periods=0, tz=pytz.utc) @@ -76,23 +74,18 @@ def _normalize_context(self, context): if isinstance(context['end'], dt.date): context['end'] = dt.date.strftime( context['end'], format='%Y-%m-%d') - #context['frequency'] = intuition.constants.PANDAS_FREQ[ - #context.get('frequency', 'daily')] context['frequency'] = context.get('frequency', 'D') - exchange = datautils.detect_exchange(context['universe']) - # TODO Check if 'frequency' available trading_dates = self._build_trading_timeline( context.pop('start', None), context.pop('end', None), - context['frequency'], exchange) + context['frequency']) - context['exchange'] = exchange context['index'] = trading_dates context['live'] = (dt.datetime.now(tz=pytz.utc) < trading_dates[-1]) # TODO Frequency for live trading (and backtesting ?) - def _build_trading_timeline(self, start, end, freq, exchange): + def _build_trading_timeline(self, start, end, freq): now = dt.datetime.now(tz=pytz.utc) if not start: @@ -159,11 +152,7 @@ def _build_trading_timeline(self, start, end, freq, exchange): start=start, end=end, freq=freq) - #TODO Use zipline style to filter instead - bt_dates = datautils.filter_market_hours(bt_dates, exchange) - live_dates = datautils.filter_market_hours(live_dates, exchange) - trading_timeline = bt_dates + live_dates - return trading_timeline + return bt_dates + live_dates def _normalize_strategy(self, strategy): ''' some contexts only retrieves strings, giving back right type ''' @@ -172,20 +161,13 @@ def _normalize_strategy(self, strategy): strategy[k] = True elif v == 'false' or v is None: strategy[k] = False - #else: - #try: - #strategy[k] = float(v) - #except ValueError: - #pass - - def validate(self, config): - self.log.info('validating configuration', config=config) - try: - assert intuition.constants.CONFIG_SCHEMA.validate(config) - except: - raise InvalidConfiguration(config=config, module=__name__) - - def build(self, validate=False): + else: + try: + strategy[k] = float(v) + except ValueError: + pass + + def build(self): context = self.load() algorithm = context.pop('algorithm', {}) @@ -205,7 +187,4 @@ def build(self, validate=False): 'data': data } - if validate: - self.validate(context) - return context, strategy diff --git a/intuition/api/data_source.py b/intuition/api/datafeed.py similarity index 51% rename from intuition/api/data_source.py rename to intuition/api/datafeed.py index 3c09108..70a2bfe 100755 --- a/intuition/api/data_source.py +++ b/intuition/api/datafeed.py @@ -14,13 +14,12 @@ ''' -import abc +#import abc import pandas as pd +import dna.logging from zipline.sources.data_source import DataSource from zipline.gens.utils import hash_args -import dna.logging import intuition.utils as utils -from intuition.data.utils import smart_selector from intuition.errors import LoadDataFailed @@ -45,105 +44,48 @@ class HybridDataFactory(DataSource): ''' Surcharge of zipline.DataSource, switching automatically between live stream and backtest sources - - Configuration options: - - sids : list of values representing simulated internal sids - It can be an explicit list of symbols, or a universe like nyse,20 - (that will pick up 20 random symbols from nyse exchange) - start : start date - delta : timedelta between internal events - filter : filter to remove the sids ''' - __metaclass__ = abc.ABCMeta - backtest = None live = None - switched = False - wait_interval = 15 - - def __init__(self, data_descriptor, **kwargs): - assert isinstance(data_descriptor['index'], - pd.tseries.index.DatetimeIndex) + _is_live = False + def __init__(self, **kwargs): self.log = dna.logging.logger(__name__) + assert 'backtest' in kwargs + assert isinstance(kwargs.get('index'), + pd.tseries.index.DatetimeIndex) + # Unpack config dictionary with default values. - self.sids = smart_selector( - kwargs.get('sids', data_descriptor['universe'])) - self.start = kwargs.get('start', data_descriptor['index'][0]) - self.end = kwargs.get('end', data_descriptor['index'][-1]) - self.index = data_descriptor['index'] + self.sids = kwargs.get('sids') or kwargs.get('universe').sids + self.start = kwargs.get('start', kwargs['index'][0]) + self.end = kwargs.get('end', kwargs['index'][-1]) + self.index = kwargs['index'] # Hash_value for downstream sorting. - self.arg_string = hash_args(data_descriptor, **kwargs) - - # Check provided informations - assert isinstance(self.sids, list) - + self.arg_string = hash_args(**kwargs) self._raw_data = None - self.initialize(data_descriptor, **kwargs) - - if hasattr(self.backtest, 'mapping'): - self.current_mapping = self.backtest.mapping - else: - self.current_mapping = self.mapping + # TODO Fails if the class as no get_data method + if 'backtest' in kwargs: + self.backtest = kwargs['backtest'](self.sids, kwargs) + if 'live' in kwargs: + self.live = kwargs['live'](self.sids, kwargs) @property def mapping(self): - return { - 'dt': (lambda x: x, 'dt'), - 'sid': (lambda x: x, 'sid'), - 'price': (float, 'price'), - 'volume': (int, 'volume'), - } - - @abc.abstractmethod - def initialize(self, data_descriptor, **kwargs): - ''' Abstract method for user custom initialization''' - pass - - @property - def instance_hash(self): - return self.arg_string - - @property - def raw_data(self): - if not self._raw_data: - self._raw_data = self.raw_data_gen() - return self._raw_data - - @abc.abstractmethod - def backtest_data(self): - ''' Users should overwrite this method ''' - pass - - @abc.abstractmethod - def live_data(self): - ''' Users should overwrite this method ''' - pass - - def _switch_context(self): - self.log.info( - 'switching from backtest to live mode') - self.switched = True - if self.live: - if hasattr(self.live, 'mapping'): - self.current_mapping = self.live.mapping - - def apply_mapping(self, raw_row): - row = {target: mapping_func(raw_row[source_key]) - for target, (mapping_func, source_key) - in self.current_mapping.items()} - row.update({'source_id': self.get_hash()}) - row.update({'type': self.event_type}) - return row + if self._is_live: + return self.live.mapping + else: + return self.backtest.mapping def raw_data_gen(self): - if self.backtest: + # The first date is usually a few seconds before now, + # so we compare to the next one + if self.backtest and not utils.is_live(self.index[1]): try: - bt_data = self.backtest_data() + bt_data = self.backtest.get_data( + self.sids, self.start, self.end) except Exception as error: raise LoadDataFailed(sids=self.sids, reason=error) else: @@ -151,26 +93,36 @@ def raw_data_gen(self): for date in self.index: self.log.debug('--> next tick {}'.format(date)) - is_live = utils.next_tick(date) - if not is_live: + self._is_live = utils.next_tick(date) + if not self._is_live: date = date.replace(hour=0) - if isinstance(bt_data, pd.DataFrame): if date not in bt_data.index: continue for sid in self.sids: series = bt_data.ix[date] yield _build_event(date, sid, series) + elif isinstance(bt_data, pd.Panel): - df = self.data.major_xs(date) + if date not in bt_data.major_axis: + continue + df = bt_data.major_xs(date) for sid, series in df.iterkv(): yield _build_event(date, sid, series) else: - if not self.switched: - self._switch_context() try: - snapshot = self.live_data() + snapshot = self.live.get_data(self.sids) except Exception as error: raise LoadDataFailed(sids=self.sids, reason=error) for sid, series in snapshot.iterkv(): yield _build_event(date, sid, series) + + @property + def instance_hash(self): + return self.arg_string + + @property + def raw_data(self): + if not self._raw_data: + self._raw_data = self.raw_data_gen() + return self._raw_data diff --git a/intuition/cli.py b/intuition/cli.py index b996d17..47259a7 100644 --- a/intuition/cli.py +++ b/intuition/cli.py @@ -14,6 +14,8 @@ import os import dna.logging from intuition import __version__ +import intuition.utils as utils +import intuition.api.datafeed as datafeed from intuition.core.engine import Simulation import intuition.core.configuration as setup @@ -28,29 +30,39 @@ def intuition(args): - Environment (global informations like third party access) ''' - # Use the provided conext builder to fill the config dicts - configuration, strategy = setup.context(args['context']) - - # Backtest or live engine. - # Registers configuration and setups data client - engine = Simulation() - - # Setup quotes data and financial context (location, market, ...) - # from user parameters. Wraps _configure_context() you can use directly - # for better understanding - engine.configure_environment( - configuration['index'][-1], - configuration['exchange']) - - # Wire togetether modules and initialize them - engine.build(args['session'], configuration['modules'], strategy) - - data = {'universe': configuration['universe'], - 'index': configuration['index']} - data.update(strategy['data']) - # See intuition/core/analyze.py for details of analyzes - # which is an Analyzes object - return engine.run(data, args['bot']) + # Use the provided context builder to fill the config dicts + #configuration, strategy = setup.context(args['context']) + with setup.Context(args['context']) as context: + + # Backtest or live engine. + # Registers configuration and setups data client + simulation = Simulation() + + # Setup quotes data and financial context (location, market, ...) from + # user parameters. Wraps _configure_context() you can use directly for + # better understanding + simulation.configure_environment( + context['config']['index'][-1], + context['market']) + + # Wire togetether modules and initialize them + simulation.build(args['session'], + context['config']['modules'], + context['strategy']) + + # Build data generator + #TODO How can I use several sources ? + data = {'universe': context['market'], + 'index': context['config']['index']} + data.update(context['strategy']['data']) + if 'backtest' in context['config']['modules']: + data['backtest'] = utils.intuition_module( + context['config']['modules']['backtest']) + if 'live' in context['config']['modules']: + data['live'] = utils.intuition_module( + context['config']['modules']['live']) + + return simulation(datafeed.HybridDataFactory(**data), args['bot']) def main(): diff --git a/intuition/constants.py b/intuition/constants.py index 7c2fa6e..e7c90c6 100644 --- a/intuition/constants.py +++ b/intuition/constants.py @@ -17,16 +17,19 @@ #TODO More strict validation CONFIG_SCHEMA = Schema({ 'universe': basestring, - 'exchange': basestring, 'index': object, Optional('_id'): object, #Optional('id'): Use(basestring, error='invalid identity'), Optional('id'): basestring, Optional('live'): bool, Optional('frequency'): basestring, + Optional('_id'): object, + Optional('__v'): object, 'modules': { 'algorithm': basestring, - 'data': basestring, + # TODO It will be at least one + Optional('backtest'): basestring, + Optional('live'): basestring, Optional('manager'): Or(basestring, None)}}) diff --git a/intuition/core/analyzes.py b/intuition/core/analyzes.py index 6371a09..7043d04 100755 --- a/intuition/core/analyzes.py +++ b/intuition/core/analyzes.py @@ -19,14 +19,13 @@ from zipline.data.benchmarks import get_benchmark_returns import intuition.utils from intuition.core.finance import qstk_get_sharpe_ratio -from intuition.data.data import Exchanges log = dna.logging.logger(__name__) class Analyze(): ''' Handle backtest results and performances measurments ''' - def __init__(self, params, results, metrics, exchange=None): + def __init__(self, params, results, metrics, benchmark='^GSPC'): # NOTE Temporary # Simulation parameters self.sim_params = params @@ -35,14 +34,14 @@ def __init__(self, params, results, metrics, exchange=None): # Simulation rolling performance self.metrics = metrics # Market where we traded - self.exchange = exchange + self.benchmark = benchmark - def build_report(self, show=False): + def build_report(self, timestamp='one_month', show=False): # Get daily, cumulative and not, returns of portfolio and benchmark # NOTE Temporary fix before intuition would be able to get benchmark # data on live trading try: - bm_sym = Exchanges[self.exchange]['symbol'] + bm_sym = self.benchmark returns_df = self.get_returns(benchmark=bm_sym) skip = False except: @@ -66,7 +65,7 @@ def build_report(self, show=False): report['benchmark_perfs'] = \ returns_df['benchmark_c_return'][-1] * 100.0 - perfs = self.overall_metrics('one_month') + perfs = self.overall_metrics(timestamp) for k, v in perfs.iteritems(): report[k] = v @@ -96,7 +95,7 @@ def rolling_performances(self, timestamp='one_month'): perfs = {} length = range(len(self.metrics[timestamp])) index = self._get_index(self.metrics[timestamp]) - perf_keys = self.metrics['one_month'][0].keys() + perf_keys = self.metrics[timestamp][0].keys() perf_keys.pop(perf_keys.index('period_label')) perfs['period'] = np.array( diff --git a/intuition/core/configuration.py b/intuition/core/configuration.py index eecd741..abac16b 100755 --- a/intuition/core/configuration.py +++ b/intuition/core/configuration.py @@ -13,11 +13,14 @@ import os import argparse +from schematics.types import StringType, URLType import dna.logging import dna.utils from intuition import __version__, __licence__ import intuition.constants import intuition.utils as utils +import intuition.data.universe as universe +from intuition.errors import InvalidConfiguration log = dna.logging.logger(__name__) @@ -51,12 +54,41 @@ def parse_commandline(): 'bot': args.bot} -def context(driver): - driver = driver.split('://') - context_builder = utils.intuition_module(driver[0]) +class Context(object): + ''' Load and control configuration ''' - log.info('building context', driver=driver[0], data=driver[1]) - return context_builder(driver[1]).build(validate=True) + def __init__(self, access): + # Hold infos to reach the config formatted like an url path + StringType(regex='.*://\w').validate(access) + self._ctx_module = access.split('://')[0] + self._ctx_infos = access.split('://')[1] + URLType().validate('http://{}'.format(self._ctx_infos)) + + def __enter__(self): + Loader = utils.intuition_module(self._ctx_module) + + log.info('building context', + driver=self._ctx_module, data=self._ctx_infos) + config, strategy = Loader(self._ctx_infos).build() + + # TODO Validate strategy as well + self._validate(config) + + market = universe.Market() + market.parse_universe_description(config.pop('universe')) + config['index'] = market.filter_open_hours(config['index']) + + return {'config': config, 'strategy': strategy, 'market': market} + + def __exit__(self, type, value, traceback): + pass + + def _validate(self, config): + log.info('validating configuration', config=config) + try: + assert intuition.constants.CONFIG_SCHEMA.validate(config) + except: + raise InvalidConfiguration(config=config, module=__name__) def logfile(session_id): diff --git a/intuition/core/engine.py b/intuition/core/engine.py index fee49c2..1cdd1ca 100755 --- a/intuition/core/engine.py +++ b/intuition/core/engine.py @@ -12,12 +12,11 @@ ''' -from zipline.finance.trading import TradingEnvironment -from zipline.utils.factory import create_simulation_parameters import dna.utils import dna.logging +from zipline.finance.trading import TradingEnvironment +from zipline.utils.factory import create_simulation_parameters import intuition.constants as constants -from intuition.data.data import Exchanges from intuition.data.loader import LiveBenchmark from intuition.core.analyzes import Analyze import intuition.utils as utils @@ -39,10 +38,6 @@ def __new__(self, identity, modules, strategy_conf): trading_algo.set_logger(dna.logging.logger('algo.' + identity)) - if modules['data']: - trading_algo.set_data_generator( - utils.intuition_module(modules['data'])) - # Use a portfolio manager if modules.get('manager'): log.info('initializing manager {}'.format(modules['manager'])) @@ -68,19 +63,15 @@ def _get_benchmark_handler(self, last_trade, freq='minutely'): last_trade, frequency=freq).surcharge_market_data \ if utils.is_live(last_trade) else None - def configure_environment(self, last_trade, exchange): + def configure_environment(self, last_trade, market): ''' Prepare benchmark loader and trading context ''' # Setup the trading calendar from market informations - self.exchange = exchange - if exchange in Exchanges: - self.context = TradingEnvironment( - bm_symbol=Exchanges[exchange]['symbol'], - exchange_tz=Exchanges[exchange]['timezone'], - load=self._get_benchmark_handler(last_trade)) - else: - raise NotImplementedError( - 'exchange {} not supported'.format(exchange)) + self.benchmark = market.benchmark + self.context = TradingEnvironment( + bm_symbol=market.benchmark, + exchange_tz=market.timezone, + load=self._get_benchmark_handler(last_trade)) def build(self, identity, modules, strategy=constants.DEFAULT_CONFIG): ''' @@ -90,20 +81,20 @@ def build(self, identity, modules, strategy=constants.DEFAULT_CONFIG): self.engine = TradingEngine(identity, modules, strategy) self.initial_cash = strategy['manager'].get('cash', None) - def run(self, data, auto=False): + def __call__(self, datafeed, auto=False): ''' wrap zipline.run() with finer control ''' self.engine.auto = auto #FIXME crash if trading one day that is not a trading day with self.context: sim_params = create_simulation_parameters( capital_base=self.initial_cash, - start=data['index'][0], - end=data['index'][-1]) + start=datafeed.start, + end=datafeed.end) - daily_stats = self.engine.trade(data, sim_params=sim_params) + daily_stats = self.engine.run(datafeed, sim_params) return Analyze( params=sim_params, - exchange=self.exchange, results=daily_stats, - metrics=self.engine.risk_report) + metrics=self.engine.risk_report, + benchmark=self.benchmark) diff --git a/intuition/data/data.py b/intuition/data/data.py index 8e91f5b..bbc60f1 100644 --- a/intuition/data/data.py +++ b/intuition/data/data.py @@ -10,13 +10,7 @@ ''' -FX_PAIRS = ['EUR/USD', 'USD/JPY', 'GBP/USD', - 'EUR/GBP', 'USD/CHF', 'EUR/JPY', - 'EUR/CHF', 'USD/CAD', 'AUD/USD', - 'GBP/JPY', 'AUD/JPY', 'AUD/NZD', - 'CAD/JPY', 'CHF/JPY', 'NZD/USD'] - - +""" # World exchanges caracteristics Exchanges = { # Market code, from yahoo stock code to google market code (needed for @@ -27,10 +21,10 @@ 'timezone': 'Europe/London', 'code': 1001, 'google_market': 'EPA'}, - 'forex': {'symbol': '^FCHI', - 'timezone': 'Europe/London', - 'indexes': [], - 'code': 1002}, + 'forex': {'symbol': '^GSPC', + 'timezone': 'US/Eastern', + 'code': 1002, + 'indexes': []}, 'nasdaq': {'symbol': '^GSPC', 'timezone': 'US/Eastern', 'code': 1003, @@ -41,14 +35,7 @@ 'code': 1004, 'google_market': 'NYSE'} } - - -class Fields: - QUOTES = ['open', 'low', 'high', 'close', 'volume', 'adj_close'] - - -#TODO Same for google json and xml retrieving -googleCode = dict() +""" yahooCode = {'ask': 'a', 'average daily volume': 'a2', 'ask size': 'a5', diff --git a/intuition/data/forex.py b/intuition/data/forex.py index 44bc521..4c91e6b 100755 --- a/intuition/data/forex.py +++ b/intuition/data/forex.py @@ -12,86 +12,75 @@ import os import requests +import string +import random from pandas import DataFrame, Series import dna.logging log = dna.logging.logger(__name__) -def forex_rates(user, password, pairs='', fmt='csv'): - url = 'http://webrates.truefx.com/rates/connect.html' - params = '?&q=ozrates&c={}&f={}&s=n'.format(','.join(pairs), fmt) - auth = requests.get(url + params, - auth=(user, password)) +def _clean_pairs(pairs): + if not isinstance(pairs, list): + pairs = [pairs] + return ','.join(map(str.upper, pairs)) - if auth.ok: - log.debug('[{}] Request successful {}:{}' - .format(auth.headers['date'], auth.reason, auth.status_code)) - #return auth.content.split('\n')[:-2] - return auth.content.split('\n') - -def _fx_mapping(raw_response): +def _fx_mapping(raw_rates): ''' Map raw output to clearer labels ''' - dict_data = dict() - for pair in raw_response: - pair = pair.split(',') - #FIXME Timestamp = year 45173 - dict_data[pair[0]] = {'TimeStamp': pair[1], - 'Bid.Price': float(pair[2] + pair[3]), - 'Ask.Price': float(pair[4] + pair[5]), - 'High': float(pair[6]), - 'Low': float(pair[7])} - return dict_data + return {pair[0].lower(): { + 'timeStamp': pair[1], + 'bid': float(pair[2] + pair[3]), + 'ask': float(pair[4] + pair[5]), + 'high': float(pair[6]), + 'low': float(pair[7]) + } for pair in map(lambda x: x.split(','), raw_rates)} #FIXME 'Not authorized' mode works weird -class ConnectTrueFX(object): - auth_url = ('http://webrates.truefx.com/rates/connect.html?\ - u={}&p={}&q=ozrates&c={}&f={}') - query_url = ('http://webrates.truefx.com/rates/connect.html?\ - id={}&f={}&c={}') - - def __init__(self, credentials='', pairs=[], fmt='csv'): - #NOTE Without authentification you still can access some data - #FIXME Not authorized response prevent from downloading quotes. - # However later you indeed retrieve 10 defaults - self._code = None +class TrueFX(object): + + _api_url = 'http://webrates.truefx.com/rates/connect.html' + _full_snapshot = 'y' + _output_format = 'csv' + + def __init__(self, credentials='', pairs=[]): if not credentials: log.info('No credentials provided, inspecting environment') credentials = os.environ.get('TRUEFX_API', ':') - credentials = credentials.split(':') - auth = requests.get( - self.auth_url.format(credentials[0], - credentials[1], - ','.join(pairs), fmt)) - if auth.ok: - log.debug('[{}] Authentification successful {}:{}' - .format(auth.headers['date'], - auth.reason, - auth.status_code)) - log.debug('Got: {}'.format(auth.content)) + self._user = credentials.split(':')[0] + self._pwd = credentials.split(':')[1] + self.state_pairs = _clean_pairs(pairs) + #self._qualifier = 'ozrates' + self._qualifier = ''.join( + random.choice(string.ascii_lowercase) for _ in range(8)) + + def connect(self): + payload = { + 'u': self._user, + 'p': self._pwd, + 'q': self._qualifier, + 'c': self.state_pairs, + 'f': self._output_format, + 's': self._full_snapshot + } + auth = requests.get(self._api_url, params=payload) - ### Remove '\r\n' - self._code = auth.content[:-2] + if auth.ok: + log.debug('Truefx authentification successful') + # Remove '\r\n' + self._session = auth.content[:-2] + return auth.ok - def query_trueFX(self, pairs='', fmt='csv'): + def query_rates(self, pairs=[]): ''' Perform a request against truefx data ''' - if isinstance(pairs, str): - pairs = [pairs] - response = requests.get( - self.query_url.format(self._code, fmt, ','.join(pairs))) + # If no pairs, TrueFx will use the ones given the last time + payload = {'id': self._session} + if pairs: + payload['c'] = _clean_pairs(pairs) + response = requests.get(self._api_url, params=payload) mapped_data = _fx_mapping(response.content.split('\n')[:-2]) - if len(mapped_data) == 1: - return Series(mapped_data) - return DataFrame(mapped_data) - - def is_active(self): - ''' Indicate wether the connection is still on or not ''' - return isinstance(self._code, str) and (self._code != 'not authorized') - - def __del__(self): - #TODO Deconnection - pass + return Series(mapped_data) if len(mapped_data) == 1 \ + else DataFrame(mapped_data) diff --git a/intuition/data/loader.py b/intuition/data/loader.py index e3b21ea..0fc9d8b 100755 --- a/intuition/data/loader.py +++ b/intuition/data/loader.py @@ -14,7 +14,6 @@ import pandas as pd from collections import OrderedDict import zipline.data.loader as zipline -import intuition.data.data as data class LiveBenchmark(object): @@ -47,12 +46,6 @@ def _load_live_market_data(self, bm_symbol='^GSPC'): #event_dt = datetime.today().replace(tzinfo=pytz.utc) event_dt = self.normalize_date(datetime.now()) - #TODO Handle invalid code - for exchange, infos in data.Exchanges.iteritems(): - if infos['symbol'] == bm_symbol: - code = data.Exchanges[exchange]['code'] - break - bm_returns, tr_curves = zipline.load_market_data(bm_symbol) dates = pd.date_range(event_dt, @@ -64,7 +57,8 @@ def _load_live_market_data(self, bm_symbol='^GSPC'): for i, c in enumerate(tr_curves.values())), key=lambda t: t[0])) - bm_fake = pd.Series([code] * len(dates), index=dates) + # NOTE the code concept is deprecated + bm_fake = pd.Series([1001] * len(dates), index=dates) for i, dt in enumerate(tr_curves.keys()): pd.Timestamp(event_dt + i * self.offset) diff --git a/intuition/data/quandl.py b/intuition/data/quandl.py index 3a91d78..285a4d2 100755 --- a/intuition/data/quandl.py +++ b/intuition/data/quandl.py @@ -19,53 +19,70 @@ log = dna.logging.logger(__name__) -def build_quandl_code(code, market, provider='GOOG'): - # known providers: YAHOO, GOOG - return '{}/{}_{}'.format(provider, market, code).upper() - - -# Ressources : http://www.quandl.com/help/api/resources -# Or the search API : https://github.com/quandl/Python -# Seems to be {PROVIDER}/{MARKET_SUFFIX}_{GOOGLE_SYMBOL} -def use_quandl_symbol(fct): - def decorator(self, symbol, **kwargs): - - dot_pos = symbol.find('.') - if dot_pos > 0: - market = symbol[dot_pos+1:] - provider = 'YAHOO' - symbol = symbol[:dot_pos] +def _clean_sid(sid): + sid = str(sid).lower() + # Remove market extension + dot_pos = sid.find('.') + sid = sid[:dot_pos] if dot_pos > 0 else sid + # Remove forex slash + return sid.replace('/', '') + + +def _build_quandl_code(symbol): + dot_pos = symbol.find('.') + slash_pos = symbol.find('/') + if dot_pos > 0: + market = symbol[dot_pos+1:] + provider = 'YAHOO' + symbol = symbol[:dot_pos] + code = '{}_{}'.format(market, symbol) + else: + if slash_pos > 0: + pair = symbol.split('/') + provider = 'QUANDL' + code = '{}{}'.format(pair[0], pair[1]) else: market = 'NASDAQ' provider = 'GOOG' - quandl_symbol = build_quandl_code(symbol, market, provider) - - return fct(self, quandl_symbol, **kwargs) + code = '{}_{}'.format(market, symbol) + return '{}/{}'.format(provider, code).upper() + + +def use_quandl_symbols(fct): + def decorator(self, symbols, **kwargs): + if not isinstance(symbols, list): + symbols = [symbols] + quandl_symbols = map(_build_quandl_code, symbols) + raw_data = fct(self, quandl_symbols, **kwargs) + + data = {} + for sid in symbols: + data[sid] = raw_data.filter(regex='.*{}.*'.format( + _clean_sid(sid).upper())) + data[sid].columns = map( + lambda x: x.replace(' ', '_').lower().split('_-_')[-1], + data[sid].columns) + return data return decorator -def multi_codes(fct): - ''' - Decorator that allows to use Data.Quandl.fetch with multi codes and get a - panel of it - ''' - def build_panel(self, codes, **kwargs): - if isinstance(codes, str): - # One code, do nothing special - data = fct(self, codes, **kwargs) - elif isinstance(codes, list): - # Multi codes, build a panel - tmp_data = {} - for code in codes: - tmp_data[code] = fct(self, code, **kwargs) - data = pd.Panel(tmp_data) - else: - raise TypeError('quandl codes must be one string or a list') - - #NOTE The algorithm can't detect missing values this way... - #NOTE An interpolate() method could be more powerful - return data.fillna(method='pad') - return build_panel +def fractionate_request(fct): + def inner(self, symbols, **kwargs): + tolerance_window = 10 + cursor = 0 + data = {} + while cursor < len(symbols): + if cursor + tolerance_window > len(symbols): + limit = len(symbols) + else: + limit = cursor + tolerance_window + symbols_fraction = symbols[cursor:limit] + cursor += tolerance_window + data_fraction = fct(self, symbols_fraction, **kwargs) + for sid, df in data_fraction.iteritems(): + data[sid] = df + return pd.Panel(data).fillna(method='pad') + return inner class DataQuandl(object): @@ -76,9 +93,8 @@ def __init__(self, quandl_key=''): self.quandl_key = quandl_key if quandl_key != '' \ else os.environ["QUANDL_API_KEY"] - #TODO Use of search feature for more powerfull and flexible use - @multi_codes - @use_quandl_symbol + @fractionate_request + @use_quandl_symbols def fetch(self, code, **kwargs): ''' Quandl entry point in datafeed object @@ -89,15 +105,18 @@ def fetch(self, code, **kwargs): if 'authtoken' in kwargs: self.quandl_key = kwargs.pop('authtoken') - # Harmonization: Quandl call start_date trim_start - if 'start_date' in kwargs: - kwargs['trim_start'] = kwargs.pop('start_date') - if 'end_date' in kwargs: - kwargs['trim_end'] = kwargs.pop('end_date') + # Harmonization: Quandl call start trim_start + if 'start' in kwargs: + kwargs['trim_start'] = kwargs.pop('start') + if 'end' in kwargs: + kwargs['trim_end'] = kwargs.pop('end') try: data = Quandl.get(code, authtoken=self.quandl_key, **kwargs) + # FIXME With a symbol not found, insert a not_found column data.index = data.index.tz_localize(pytz.utc) + #data.columns = map( + #lambda x: x.replace(' ', '_').lower(), data.columns) except: log.error('** unable to fetch %s from Quandl' % code) data = pd.DataFrame() diff --git a/intuition/data/remote.py b/intuition/data/remote.py index fed7bf8..46a06ba 100755 --- a/intuition/data/remote.py +++ b/intuition/data/remote.py @@ -21,8 +21,7 @@ import dna.logging from zipline.utils.factory import load_from_yahoo, load_bars_from_yahoo from intuition.data.utils import ( - use_google_symbol, invert_dataframe_axis) -import intuition.utils + use_google_symbol, invert_dataframe_axis, apply_mapping) log = dna.logging.logger(__name__) @@ -83,7 +82,7 @@ def fetch_equities_snapshot(self, *args, **kwargs): current infromations about given equitiy names ______________________________________________ Parameters - args: tuple + args: tuple | list company symbols to consider kwargs['level']: int Quantity of information level @@ -143,7 +142,10 @@ def snapshot_yahoo_pandas(symbols): ''' if isinstance(symbols, str): symbols = [symbols] - return get_quote_yahoo(symbols) + # TODO lower() columns + data = get_quote_yahoo(symbols) + data.columns = map(str.lower, data.columns) + return data #NOTE Can use symbol with market: 'goog:nasdaq', any difference ? @@ -163,7 +165,8 @@ def snapshot_google_light(symbols): for i, quote in enumerate(json_infos): #FIXME nasdaq and nyse `symbols` are in capital, not cac40 if quote['t'].lower() in map(str.lower, symbols): - snapshot[symbols[i]] = intuition.utils.apply_mapping( + #snapshot[symbols[i]] = intuition.utils.apply_mapping( + snapshot[symbols[i]] = apply_mapping( quote, google_light_mapping) else: log.warning('Unknown symbol {}, ignoring...'.format( diff --git a/intuition/data/universe.py b/intuition/data/universe.py new file mode 100755 index 0000000..21f6165 --- /dev/null +++ b/intuition/data/universe.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +# vim:fenc=utf-8 + +''' + Universe manager + ---------------- + + It knows everything about market and smartly handle user input + + :copyright (c) 2014 Xavier Bruhiere + :license: Apache 2.0, see LICENSE for more details. +''' + + +import os +import random +import yaml +import dna.logging + +log = dna.logging.logger(__name__) + + +class Market(object): + ''' + Knows everything about market and smartly handle user input + ''' + + # TODO Read default from market.yml or set by the user + benchmark = '^GSPC' + timezone = 'US/Eastern' + raw_description = None + + def __init__(self): + log.info('loading market scheme') + self.scheme = self._read_market_scheme() + + def _read_market_scheme(self): + ''' Load market yaml description ''' + path = os.path.expanduser('~/.intuition/data/market.yml') + return yaml.load(open(path, 'r')) + + def _detect_exchange(self, description): + ''' Guess from the description and the market scheme ''' + pass + + def filter_open_hours(self, index): + ''' Remove market specific closed hours ''' + return index + + def _lookup_sids(self, market, limit=-1): + if market == 'forex': + self.timezone = self.scheme[market]['timezone'] + sids_list = self.scheme[market]['pairs'] + else: + market = market.split(':') + market_scheme = self.scheme + for key in market[:-1]: + market_scheme = market_scheme[key] + sids_list = map( + lambda x: x + '.pa', market_scheme[market[-1]].keys()) + self.timezone = market_scheme['timezone'] + self.benchmark = market_scheme['benchmark'] + + random.shuffle(sids_list) + return sids_list[:limit] if limit > 0 else sids_list + + # TODO multi exchange, sector and other criterias + def parse_universe_description(self, description): + ''' Semantic + + - 'sid1,sid2,sid2,...' + - 'exchange' : every sids of the exchange + - 'exchange,n' : n random sids of the exchange + + where exchange is a combination of 'type:index:submarket' + ''' + self.raw_description = description + description = description.split(',') + self.exchange = description[0] + + n = int(description[1]) if len(description) == 2 else -1 + self.sids = self._lookup_sids(description[0], n) + + def __str__(self): + return '= pd.datetools.Day(): + # Daily or lower frequency, no hours filter required + return dates + if exchange == 'paris': + selector = ((dates.hour > 6) & (dates.hour < 16)) | \ + ((dates.hour == 17) & (dates.minute < 31)) + elif exchange == 'london': + selector = ((dates.hour > 8) & (dates.hour < 16)) | \ + ((dates.hour == 16) & (dates.minute > 31)) + elif exchange == 'tokyo': + selector = ((dates.hour > 0) & (dates.hour < 6)) + elif exchange == 'nasdaq' or exchange == 'nyse': + selector = ((dates.hour > 13) & (dates.hour < 21)) | \ + ((dates.hour == 13) & (dates.minute > 31)) + else: + # Forex or Unknown market, return as is + return dates + + # Pandas dataframe filtering mechanism + index = dates[selector] + if not index.size: + raise ExchangeIsClosed(exchange=exchange, dates=dates) + return index +""" diff --git a/intuition/data/utils.py b/intuition/data/utils.py index cf82b91..e07f457 100755 --- a/intuition/data/utils.py +++ b/intuition/data/utils.py @@ -12,6 +12,7 @@ import os import random +import yaml import pandas as pd import intuition.data.data as data from intuition.errors import ExchangeIsClosed @@ -27,83 +28,6 @@ def apply_mapping(raw_row, mapping): return row -#TODO This is quick and dirty -def detect_exchange(universe): - if not isinstance(universe, list): - universe = universe.split(',') - if universe[0] in data.Exchanges: - exchange = universe[0] - else: - if universe[0].find('/') > 0: - exchange = 'forex' - elif universe[0].find('.pa') > 0: - exchange = 'paris' - else: - exchange = 'nasdaq' - return exchange - - -def market_sids_list(exchange, n=-1): - if exchange == 'forex': - sids_list = data.FX_PAIRS - else: - csv_file = '{}.csv'.format( - os.path.join( - os.environ['HOME'], '.intuition/data', exchange.lower())) - df = pd.read_csv(csv_file) - sids_list = df['Symbol'].tolist() - - random.shuffle(sids_list) - if n > 0: - sids_list = sids_list[:n] - return sids_list - - -def smart_selector(sids): - if not isinstance(sids, list): - sids = sids.split(',') - if sids[0] in data.Exchanges: - if len(sids) == 2: - n = int(sids[1]) - else: - n = -1 - sids = market_sids_list(sids[0], n) - - #return sids - return map(str.lower, map(str, sids)) - - -#TODO Complete with : -# http://en.wikipedia.org/wiki/List_of_stock_exchange_opening_times -# http://en.wikipedia.org/wiki/List_of_S&P_500_companies -def filter_market_hours(dates, exchange): - ''' Only return market open hours from UTC timezone''' - #NOTE UTC times ! Beware of summer and winter hours - if dates.freq >= pd.datetools.Day(): - # Daily or lower frequency, no hours filter required - return dates - if exchange == 'paris': - selector = ((dates.hour > 6) & (dates.hour < 16)) | \ - ((dates.hour == 17) & (dates.minute < 31)) - elif exchange == 'london': - selector = ((dates.hour > 8) & (dates.hour < 16)) | \ - ((dates.hour == 16) & (dates.minute > 31)) - elif exchange == 'tokyo': - selector = ((dates.hour > 0) & (dates.hour < 6)) - elif exchange == 'nasdaq' or exchange == 'nyse': - selector = ((dates.hour > 13) & (dates.hour < 21)) | \ - ((dates.hour == 13) & (dates.minute > 31)) - else: - # Forex or Unknown market, return as is - return dates - - # Pandas dataframe filtering mechanism - index = dates[selector] - if not index.size: - raise ExchangeIsClosed(exchange=exchange, dates=dates) - return index - - def invert_dataframe_axis(fct): ''' Make dataframe index column names, diff --git a/logs.doc.md b/logs.doc.md deleted file mode 100644 index 91a8d84..0000000 --- a/logs.doc.md +++ /dev/null @@ -1,8 +0,0 @@ -Logs management -=============== - -* Stored in ~/.intuition/logs/.log -* Level is set with the `$LOG` environment variable (default: warning) -* Logs are structured in json events, with at least "event", "id" and - "timestamp" fields. Others are optional and used by the application. - Thoses events are indexed with recoreded time, source and debug level diff --git a/requirements.txt b/requirements.txt index d7f10bb..d135470 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ beautifulsoup4==4.3.2 ystockquote blist==1.3.4 +schematics==0.9-4 schema==0.2.0 Cython==0.20.1 numpy==1.8.0 diff --git a/setup.py b/setup.py index b6bbe78..f141373 100644 --- a/setup.py +++ b/setup.py @@ -13,11 +13,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import os from glob import glob - from setuptools import setup, find_packages - from intuition import ( __version__, __author__, __licence__, __project__ ) @@ -38,6 +37,7 @@ def get_requirements(): 'Cython>=0.20.1', 'ystockquote', 'numpy>=1.8.0', + 'schematics>=0.9-4', 'schema==0.2.0', 'python-dateutil>=2.2', 'pytz>=2013.9', @@ -87,7 +87,7 @@ def long_description(): 'Topic :: System :: Distributed Computing', ], data_files=[(os.path.expanduser('~/.intuition/data'), glob('./data/*')), - (os.path.expanduser('~/.intuition/logs'), './logs.doc.md')], + (os.path.expanduser('~/.intuition/logs'), ['logs.doc.md'])], dependency_links=[ 'http://github.com/pydata/pandas/tarball/master#egg=pandas-0.13.0.dev', 'http://github.com/quantopian/zipline/tarball/master#egg=zipline-0.5.11.dev'] diff --git a/tests/test_configuration.py b/tests/test_configuration.py deleted file mode 100644 index 5250484..0000000 --- a/tests/test_configuration.py +++ /dev/null @@ -1,55 +0,0 @@ -''' -Tests for intuition.core.configuration -''' - -import unittest -import os -import intuition.core.configuration as configuration -from dna.errors import ImportContextFailed -from intuition.errors import InvalidConfiguration - - -class ConfigurationTestCase(unittest.TestCase): - - #FIXME File only on my local computer - good_driver = 'insights.contexts.file::intuition/conf.yaml' - bad_driver = 'no.where.file::intuition/conf.yaml' - bad_config = 'insights.contexts.file::/fake.yaml' - - def test_parse_command_line(self): - try: - args = None - args = configuration.parse_commandline() - except SystemExit: - self.assertTrue(args is None) - pass - - def test_load_context(self): - if os.path.exists(self.good_driver): - conf, strat = configuration.context(self.good_driver) - self.assertIsInstance(strat, dict) - self.assertIsInstance(conf, dict) - else: - pass - - def test_load_bad_configuration(self): - self.assertRaises( - InvalidConfiguration, configuration.context, self.bad_config) - - def test_loaded_configuration(self): - if os.path.exists(self.good_driver): - conf, strat = configuration.context(self.good_driver) - for field in ['manager', 'algorithm', 'data']: - self.assertIn(field, strat) - for field in ['index', 'live', 'exchange']: - self.assertIn(field, conf) - else: - pass - - def test_absent_driver_context_load(self): - self.assertRaises( - ImportContextFailed, configuration.context, self.bad_driver) - - def test_logfile(self): - logfile = configuration.logfile('fake_id') - self.assertIn('.intuition/logs/fake_id.log', logfile) diff --git a/tests/test_setup.py b/tests/test_setup.py new file mode 100644 index 0000000..ece2a0c --- /dev/null +++ b/tests/test_setup.py @@ -0,0 +1,79 @@ +''' +Tests for intuition.core.configuration +''' + +import unittest +import pandas as pd +import intuition.core.configuration as configuration +from dna.errors import ImportContextFailed +from intuition.errors import InvalidConfiguration + + +class ConfigurationTestCase(unittest.TestCase): + + # FIXME Needs insights installed for these tests + good_driver = '{}://{}'.format( + 'insights.contexts.file.FileContext', + 'intuition.io/../config/backtest.yaml') + good_driver = '{}://{}'.format( + 'insights.contexts.file.FileContext', + 'intuition.io/../config/backtest.yaml') + good_driver = '{}://{}'.format( + 'insights.contexts.file.FileContext', + 'intuition.io/../config/backtest.yaml') + bad_driver = 'no.file.FileContext://intuition.io/../config/backtest.yaml' + bad_config = 'insights.contexts.file.FileContext://intuition.io/fake.yaml' + bad_formatted_config = 'insights.contexts.file.FileContext::/fake.yaml' + + def test_logfile(self): + logfile = configuration.logfile('fake_id') + self.assertIn('.intuition/logs/fake_id.log', logfile) + + def test_parse_command_line(self): + try: + args = None + args = configuration.parse_commandline() + except SystemExit: + self.assertTrue(args is None) + pass + + def test_load_context(self): + with configuration.Context(self.good_driver) as context: + self.assertIsInstance(context, dict) + self.assertIsInstance(context['strategy'], dict) + self.assertIsInstance(context['config'], dict) + + def test_validate_bad_config(self): + bad_config = {} + ctx = configuration.Context(self.bad_driver) + self.assertRaises( + InvalidConfiguration, ctx._validate, bad_config) + + def test_validate_good_config(self): + good_config = { + 'universe': 'nasdaq,4', + 'exchange': 'nasdaq', + 'index': pd.date_range('2014/2/3', periods=30), + 'modules': { + 'algorithm': 'dualma', + 'data': 'database' + } + } + ctx = configuration.Context(self.bad_driver) + self.assertIsNone(ctx._validate(good_config)) + + def test_load_bad_configuration(self): + # TODO Write an invalid file + pass + + def test_loaded_configuration(self): + with configuration.Context(self.good_driver) as context: + for field in ['manager', 'algorithm', 'data']: + self.assertIn(field, context['strategy']) + for field in ['index', 'live', 'exchange']: + self.assertIn(field, context['config']) + + def test_absent_driver_context_load(self): + ctx = configuration.Context(self.bad_driver) + self.assertRaises( + ImportContextFailed, ctx.__enter__) diff --git a/tests/test_utils.py b/tests/test_utils.py index 6d695d6..5512413 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -45,12 +45,15 @@ def _validate_dates_index(self, index): def test_build_date_index(self): self._validate_dates_index(utils.build_date_index()) - self._validate_dates_index(utils.build_date_index(start='2011/01/01')) - self._validate_dates_index(utils.build_date_index(end='2014/01/01')) - self._validate_dates_index( - utils.build_date_index(end='2014/01/01', freq='M')) - self._validate_dates_index( - utils.build_date_index(start='2011', end='2014/01/01')) + self._validate_dates_index(utils.build_date_index( + start=datetime(2011, 1, 1, tzinfo=pytz.utc))) + self._validate_dates_index(utils.build_date_index( + end=datetime(2014, 1, 1, tzinfo=pytz.utc))) + self._validate_dates_index(utils.build_date_index( + end=datetime(2014, 1, 1, tzinfo=pytz.utc), freq='M')) + self._validate_dates_index(utils.build_date_index( + start=datetime(2011, 1, 1, tzinfo=pytz.utc), + end=datetime(2014, 1, 1, tzinfo=pytz.utc))) def test_utc_date_to_epoch(self): utc_date = datetime(2012, 12, 1, tzinfo=pytz.utc) @@ -69,6 +72,24 @@ def test_epoch_conversions(self): new_date = utils.epoch_to_date(epoch) self.assertEqual(date, new_date) - def test_wait_next_tick(self): + def test_wait_backtest_next_tick(self): old_dt = datetime(2010, 12, 1, tzinfo=pytz.utc) self.assertFalse(utils.next_tick(old_dt)) + + def test_wait_live_next_tick(self): + pass + + def test_import_intuition_module(self): + pass + + def test_import_bad_intuition_module(self): + pass + + def test_truncate_float(self): + original_float = 3.232999 + truncated_float = utils.truncate(original_float, n=2) + self.assertIsInstance(truncated_float, float) + self.assertGreater(original_float, truncated_float) + + def test_truncate_str(self): + self.assertEqual(utils.truncate('nofloat'), 'nofloat') diff --git a/wercker.yml b/wercker.yml index 5a207f9..d0cbe6d 100644 --- a/wercker.yml +++ b/wercker.yml @@ -21,7 +21,8 @@ build: - script: name: run tests code: | - make tests + flake8 tests intuition + nosetests -w tests after-steps: - hipchat-notify: