Merge branch 'rc-0.1.dev7'

.github/ISSUE_TEMPLATE.md

@@ -1,4 +1,4 @@
-Dear Zipline Maintainers,
+Dear Catalyst Maintainers,
 
 Before I tell you about my issue, let me describe my environment:
 
@@ -7,7 +7,7 @@ Before I tell you about my issue, let me describe my environment:
 * Operating System: (Windows Version or `$ uname --all`)
 * Python Version: `$ python --version`
 * Python Bitness: `$ python -c 'import math, sys;print(int(math.log(sys.maxsize + 1, 2) + 1))'`
-* How did you install Zipline: (`pip`, `conda`, or `other (please explain)`)
+* How did you install Catalyst: (`pip`, `conda`, or `other (please explain)`)
 * Python packages: `$ pip freeze` or `$ conda list`
 
 Now that you know a little about me, let me tell you about the issue I am

README.rst

@@ -1 +1 @@
-All the documentation for `Catalyst <https://github.com/enigmampc/catalyst>`_ can be found in the `catalyst-docs wiki <https://github.com/enigmampc/catalyst-docs/wiki>`_.
+All the documentation for `Catalyst <https://github.com/enigmampc/catalyst>`_ can be found in the `catalyst-docs wiki <https://github.com/enigmampc/catalyst-docs/wiki>`_.

catalyst/__main__.py

@@ -123,7 +123,7 @@ def _(*args, **kwargs):
 )
 @click.option(
     '--data-frequency',
-    type=click.Choice({'daily', 'minute'}),
+    type=click.Choice({'daily', '5-minute', 'minute'}),
     default='daily',
     show_default=True,
     help='The data frequency of the simulation.',
@@ -290,6 +290,13 @@ def catalyst_magic(line, cell=None):
     show_default=True,
     help='The data bundle to ingest.',
 )
+@click.option(
+    '-c',
+    '--compile-locally',
+    is_flag=True,
+    default=False,
+    help='Download dataset from source and compile bundle locally.',
+)
 @click.option(
     '--assets-version',
     type=int,
@@ -301,7 +308,7 @@ def catalyst_magic(line, cell=None):
     default=True,
     help='Print progress information to the terminal.'
 )
-def ingest(bundle, assets_version, show_progress):
+def ingest(bundle, compile_locally, assets_version, show_progress):
     """Ingest the data for the given bundle.
     """
     bundles_module.ingest(
@@ -310,6 +317,7 @@ def ingest(bundle, assets_version, show_progress):
         pd.Timestamp.utcnow(),
         assets_version,
         show_progress,
+        compile_locally,
     )
 
 

catalyst/algorithm.py

@@ -133,7 +133,10 @@
 import catalyst.protocol
 from catalyst.sources.requests_csv import PandasRequestsCSV
 
-from catalyst.gens.sim_engine import MinuteSimulationClock
+from catalyst.gens.sim_engine import (
+    MinuteSimulationClock,
+    FiveMinuteSimulationClock,
+)
 from catalyst.sources.benchmark_source import BenchmarkSource
 from catalyst.catalyst_warnings import ZiplineDeprecationWarning
 
@@ -170,7 +173,7 @@ class TradingAlgorithm(object):
     algo_filename : str, optional
         The filename for the algoscript. This will be used in exception
         tracebacks. default: '<string>'.
-    data_frequency : {'daily', 'minute'}, optional
+    data_frequency : {'daily', '5-minute', 'minute'}, optional
         The duration of the bars.
     instant_fill : bool, optional
         Whether to fill orders immediately or on next bar. default: False
@@ -223,7 +226,7 @@ def __init__(self, *args, **kwargs):
             script : str
                 Algoscript that contains initialize and
                 handle_data function definition.
-            data_frequency : {'daily', 'minute'}
+            data_frequency : {'daily', '5-minute', 'minute'}
                The duration of the bars.
             capital_base : float <default: 1.0e5>
                How much capital to start with.
@@ -305,7 +308,10 @@ def __init__(self, *args, **kwargs):
         self.asset_finder = self.trading_environment.asset_finder
 
         # Initialize Pipeline API data.
-        self.init_engine(kwargs.pop('get_pipeline_loader', None))
+        self.init_engine(
+            kwargs.pop('get_pipeline_loader', None),
+            self.sim_params.data_frequency,
+        )
         self._pipelines = {}
         # Create an always-expired cache so that we compute the first time data
         # is requested.
@@ -419,16 +425,28 @@ def noop(*args, **kwargs):
 
         self.restrictions = NoRestrictions()
 
-    def init_engine(self, get_loader):
+    def init_engine(self, get_loader, data_frequency):
         """
         Construct and store a PipelineEngine from loader.
 
         If get_loader is None, constructs an ExplodingPipelineEngine
         """
         if get_loader is not None:
+            if data_frequency == 'daily':
+                all_dates = self.trading_calendar.all_sessions
+            elif data_frequency == '5-minute':
+                all_dates = self.trading_calendar.all_five_minutes
+            elif data_frequency == 'minute':
+                all_dates = self.trading_calendar.all_minutes
+            else:
+                raise ValueError(
+                    'Cannot initialize engine with '
+                    'data frequency: {}'.format(data_frequency)
+                )
+
             self.engine = SimplePipelineEngine(
                 get_loader,
-                self.trading_calendar.all_sessions,
+                all_dates,
                 self.asset_finder,
             )
         else:
@@ -449,7 +467,7 @@ def before_trading_start(self, data):
         self._in_before_trading_start = True
 
         with handle_non_market_minutes(data) if \
-                self.data_frequency == "minute" else ExitStack():
+                self.data_frequency in ('minute', '5-minute') else ExitStack():
             self._before_trading_start(self, data)
 
         self._in_before_trading_start = False
@@ -505,10 +523,11 @@ def _create_clock(self):
         market_closes = trading_o_and_c['market_close']
         minutely_emission = False
 
-        if self.sim_params.data_frequency == 'minute':
+        if self.sim_params.data_frequency in set(('minute', '5-minute')):
             market_opens = trading_o_and_c['market_open']
 
-            minutely_emission = self.sim_params.emission_rate == "minute"
+            minutely_emission = self.sim_params.emission_rate in \
+                set(('minute', '5-minute'))
         else:
             # in daily mode, we want to have one bar per session, timestamped
             # as the last minute of the session.
@@ -528,10 +547,19 @@ def _create_clock(self):
         # FIXME generalize these values
         before_trading_start_minutes = days_at_time(
             self.sim_params.sessions,
-            time(8, 45),
-            "US/Eastern"
+            time(0, 0),
+            'UTC',
         )
 
+        if self.sim_params.data_frequency == '5-minute':
+            return FiveMinuteSimulationClock(
+                self.sim_params.sessions,
+                execution_opens,
+                execution_closes,
+                before_trading_start_minutes,
+                minute_emission=minutely_emission,
+            )
+
         return MinuteSimulationClock(
             self.sim_params.sessions,
             execution_opens,
@@ -660,8 +688,11 @@ def run(self, data=None, overwrite_sim_params=True):
                     # Assume data is daily if timestamp times are
                     # standardized, otherwise assume minute bars.
                     times = data.major_axis.time
-                    if np.all(times == times[0]):
+                    time_count = times.nunique()
+                    if time_count == 1:
                         self.sim_params.data_frequency = 'daily'
+                    elif time_count == 288:
+                        self.sim_params.data_frequency = '5-minute'
                     else:
                         self.sim_params.data_frequency = 'minute'
 
@@ -683,6 +714,8 @@ def run(self, data=None, overwrite_sim_params=True):
 
                 if self.sim_params.data_frequency == 'daily':
                     equity_reader_arg = 'equity_daily_reader'
+                elif self.sim_params.data_frequency == '5-minute':
+                    equity_daily_reader = 'equity_5_minute_reader'
                 elif self.sim_params.data_frequency == 'minute':
                     equity_reader_arg = 'equity_minute_reader'
                 equity_reader = PanelBarReader(
@@ -926,9 +959,9 @@ def get_environment(self, field='platform'):
                   The arena from the simulation parameters. This will normally
                   be ``'backtest'`` but some systems may use this distinguish
                   live trading from backtesting.
-              data_frequency : {'daily', 'minute'}
+              data_frequency : {'daily', '5-minute', 'minute'}
                   data_frequency tells the algorithm if it is running with
-                  daily data or minute data.
+                  daily, minute, or five-minute mode.
               start : datetime
                   The start date for the simulation.
               end : datetime
@@ -1102,12 +1135,19 @@ def schedule_function(self,
                           'date_rule. You should use keyword argument '
                           'time_rule= when calling schedule_function without '
                           'specifying a date_rule', stacklevel=3)
+
+        freq = self.sim_params.data_frequency
 
         date_rule = date_rule or date_rules.every_day()
-        time_rule = ((time_rule or time_rules.every_minute())
-                     if self.sim_params.data_frequency == 'minute' else
-                     # If we are in daily mode the time_rule is ignored.
-                     time_rules.every_minute())
+        if freq is 'daily':
+            # ignore time rule in daily mode
+            time_rule = time_rules.every_minute()
+        else:
+            # use provided time rule or default to every minute or 5 minutes
+            # based on desired data frequency.
+            time_rule = time_rule or (time_rules.every_5_minutes()
+                                      if freq is '5-minute' else
+                                      time_rules.every_minute())
 
         # Check the type of the algorithm's schedule before pulling calendar
         # Note that the ExchangeTradingSchedule is currently the only
@@ -1782,7 +1822,7 @@ def data_frequency(self):
 
     @data_frequency.setter
     def data_frequency(self, value):
-        assert value in ('daily', 'minute')
+        assert value in ('daily', '5-minute', 'minute')
         self.sim_params.data_frequency = value
 
     @api_method

catalyst/data/_minute_bar_internal.pyx

@@ -35,6 +35,17 @@ def minute_value(ndarray[long_t, ndim=1] market_opens,
 
     return market_opens[q] + r
 
+@cython.cdivision(True)
+def five_minute_value(ndarray[long_t, ndim=1] market_opens,
+                      Py_ssize_t pos,
+                      short five_minutes_per_day):
+
+    cdef short q, r
+    q = cython.cdiv(pos, five_minutes_per_day)
+    r = cython.cmod(pos, five_minutes_per_day)
+
+    return market_opens[q] + r
+
 def find_position_of_minute(ndarray[long_t, ndim=1] market_opens,
                             ndarray[long_t, ndim=1] market_closes,
                             long_t minute_val,
@@ -88,6 +99,26 @@ def find_position_of_minute(ndarray[long_t, ndim=1] market_opens,
 
     return (market_open_loc * minutes_per_day) + delta
 
+def find_position_of_five_minute(ndarray[long_t, ndim=1] market_opens,
+                                 ndarray[long_t, ndim=1] market_closes,
+                                 long_t five_minute_val,
+                                 short five_minutes_per_day,
+                                 bool forward_fill):
+
+    cdef Py_ssize_t market_open_loc, market_open, delta
+
+    market_open_loc = \
+        searchsorted(market_opens, five_minute_val, side='right') - 1
+    market_open = market_opens[market_open_loc]
+    market_close = market_closes[market_open_loc]
+
+    if not forward_fill and ((five_minute_val - market_open) >= five_minutes_per_day):
+        raise ValueError("Given five minutes is not between an open and a close")
+
+    delta = int_min(five_minute_val - market_open, market_close - market_open)
+
+    return (market_open_loc * five_minutes_per_day) + delta
+
 def find_last_traded_position_internal(
         ndarray[long_t, ndim=1] market_opens,
         ndarray[long_t, ndim=1] market_closes,
@@ -157,3 +188,51 @@ def find_last_traded_position_internal(
     # we've gone to the beginning of this asset's range, and still haven't
     # found a trade event
     return -1
+
+def find_last_traded_five_minute_position_internal(
+        ndarray[long_t, ndim=1] market_opens,
+        ndarray[long_t, ndim=1] market_closes,
+        long_t end_five_minute,
+        long_t start_five_minute,
+        volumes,
+        short five_minutes_per_day):
+    cdef Py_ssize_t minute_pos, current_minute, q
+
+    five_minute_pos = int_min(
+        find_position_of_five_minute(
+            market_opens,
+            market_closes,
+            end_five_minute,
+            five_minutes_per_day,
+            True,
+        ),
+        len(volumes) - 1,
+    )
+
+    while five_minute_pos >= 0:
+        current_five_minute = five_minute_value(
+            market_opens, five_minute_pos, five_minutes_per_day
+        )
+
+        q = cython.cdiv(five_minute_pos, five_minutes_per_day)
+        if current_five_minute > market_closes[q]:
+            five_minute_pos = find_position_of_five_minute(
+                market_opens,
+                market_closes,
+                market_closes[q],
+                five_minutes_per_day,
+                False,
+            )
+            continue
+
+        if current_five_minute < start_five_minute:
+            return -1
+
+        if volumes[five_minute_pos] != 0:
+            return five_minute_pos
+
+        five_minute_pos -= 1
+
+    # we've gone to the beginning of this asset's range, and still haven't
+    # found a trade event
+    return -1