ENH: Use dividend adjusted history

tests/history_cases.py

@@ -45,28 +45,28 @@ def mixed_frequency_expected_data(count, frequency):
     to_utc('2013-07-03 9:31AM'), 600,
 )
 ONE_MINUTE_PRICE_ONLY_SPECS = [
-    HistorySpec(1, '1m', 'price', True, data_frequency='minute'),
+    HistorySpec(1, '1m', 'price', True, False, data_frequency='minute'),
 ]
 DAILY_OPEN_CLOSE_SPECS = [
-    HistorySpec(3, '1d', 'open_price', False, data_frequency='minute'),
-    HistorySpec(3, '1d', 'close_price', False, data_frequency='minute'),
+    HistorySpec(3, '1d', 'open_price', False, False, data_frequency='minute'),
+    HistorySpec(3, '1d', 'close_price', False, False, data_frequency='minute'),
 ]
 ILLIQUID_PRICES_SPECS = [
-    HistorySpec(3, '1m', 'price', False, data_frequency='minute'),
-    HistorySpec(5, '1m', 'price', True, data_frequency='minute'),
+    HistorySpec(3, '1m', 'price', False, False, data_frequency='minute'),
+    HistorySpec(5, '1m', 'price', True, False, data_frequency='minute'),
 ]
 MIXED_FREQUENCY_SPECS = [
-    HistorySpec(1, '1m', 'price', False, data_frequency='minute'),
-    HistorySpec(2, '1m', 'price', False, data_frequency='minute'),
-    HistorySpec(2, '1d', 'price', False, data_frequency='minute'),
+    HistorySpec(1, '1m', 'price', False, False, data_frequency='minute'),
+    HistorySpec(2, '1m', 'price', False, False, data_frequency='minute'),
+    HistorySpec(2, '1d', 'price', False, False, data_frequency='minute'),
 ]
 MIXED_FIELDS_SPECS = [
-    HistorySpec(3, '1m', 'price', True, data_frequency='minute'),
-    HistorySpec(3, '1m', 'open_price', True, data_frequency='minute'),
-    HistorySpec(3, '1m', 'close_price', True, data_frequency='minute'),
-    HistorySpec(3, '1m', 'high', True, data_frequency='minute'),
-    HistorySpec(3, '1m', 'low', True, data_frequency='minute'),
-    HistorySpec(3, '1m', 'volume', True, data_frequency='minute'),
+    HistorySpec(3, '1m', 'price', True, False, data_frequency='minute'),
+    HistorySpec(3, '1m', 'open_price', True, False, data_frequency='minute'),
+    HistorySpec(3, '1m', 'close_price', True, False, data_frequency='minute'),
+    HistorySpec(3, '1m', 'high', True, False, data_frequency='minute'),
+    HistorySpec(3, '1m', 'low', True, False, data_frequency='minute'),
+    HistorySpec(3, '1m', 'volume', True, False, data_frequency='minute'),
 ]
 
 

tests/test_history.py

@@ -137,6 +137,7 @@ def get_index_at_dt(case_input):
         case_input['frequency'],
         None,
         False,
+        False,
         data_frequency='minute',
     )
     return history.index_at_dt(history_spec, case_input['algo_dt'])
@@ -230,13 +231,15 @@ def test_multiple_specs_on_same_bar(self):
             frequency='1m',
             field='price',
             ffill=True,
+            dividend_adjusted=False,
             data_frequency='minute'
         )
         no_fill_spec = history.HistorySpec(
             bar_count=3,
             frequency='1m',
             field='price',
             ffill=False,
+            dividend_adjusted=False,
             data_frequency='minute'
         )
 
@@ -280,6 +283,7 @@ def test_container_nans_and_daily_roll(self):
             frequency='1d',
             field='price',
             ffill=True,
+            dividend_adjusted=False,
             data_frequency='minute'
         )
         specs = {spec.key_str: spec}
@@ -1102,6 +1106,7 @@ def test_history_grow_length(self,
             frequency=freq,
             field=field,
             ffill=True,
+            dividend_adjusted=False,
             data_frequency=data_frequency,
         )
         specs = {spec.key_str: spec}
@@ -1131,13 +1136,15 @@ def test_history_grow_length(self,
                 frequency=freq,
                 field=field,
                 ffill=True,
+                dividend_adjusted=False,
                 data_frequency=data_frequency,
             ),
             history.HistorySpec(
                 bar_count=bar_count + 2,
                 frequency=freq,
                 field=field,
                 ffill=True,
+                dividend_adjusted=False,
                 data_frequency=data_frequency,
             ),
         )
@@ -1167,6 +1174,7 @@ def test_history_add_field(self, bar_count, freq, pair, data_frequency):
             frequency=freq,
             field=first,
             ffill=True,
+            dividend_adjusted=False,
             data_frequency=data_frequency,
         )
         specs = {spec.key_str: spec}
@@ -1195,6 +1203,7 @@ def test_history_add_field(self, bar_count, freq, pair, data_frequency):
             frequency=freq,
             field=second,
             ffill=True,
+            dividend_adjusted=False,
             data_frequency=data_frequency,
         )
 
@@ -1224,6 +1233,7 @@ def test_history_add_freq(self, bar_count, pair, field, data_frequency):
             frequency=first,
             field=field,
             ffill=True,
+            dividend_adjusted=False,
             data_frequency=data_frequency,
         )
         specs = {spec.key_str: spec}
@@ -1252,6 +1262,7 @@ def test_history_add_freq(self, bar_count, pair, field, data_frequency):
             frequency=second,
             field=field,
             ffill=True,
+            dividend_adjusted=False,
             data_frequency=data_frequency,
         )
 

tests/test_perf_tracking.py

@@ -174,12 +174,23 @@ def calculate_results(host,
     perf_tracker = perf.PerformanceTracker(host.sim_params)
 
     if dividend_events is not None:
+
         dividend_frame = pd.DataFrame(
             [
                 event.to_series(index=zp.DIVIDEND_FIELDS)
                 for event in dividend_events
             ],
         )
+
+        # The 'id' field is set in the trading algorithm by update dividend
+        # For testing purposes, we are building the dividend frame and
+        # injecting it into the perf_tracker directly. Thus, we need to add
+        # the 'id' field here manually.
+        dividend_frame['id'] = np.arange(
+            len(dividend_frame)
+        )
+        dividend_frame.sort(['pay_date', 'ex_date']).set_index('id',
+                                                               drop=False)
         perf_tracker.update_dividends(dividend_frame)
 
     # Raw trades

zipline/algorithm.py

@@ -1021,6 +1021,8 @@ def get_history_spec(self, bar_count, frequency, field, ffill,
                     self.sim_params.data_frequency,
                     bar_data=self._most_recent_data,
                 )
+
+            self.history_container.update_dividends(self.dividend_frame)
             self.history_container.ensure_spec(
                 spec, self.datetime, self._most_recent_data,
             )

zipline/finance/performance/tracker.py

@@ -62,7 +62,6 @@
 import pickle
 from six import iteritems
 
-import numpy as np
 import pandas as pd
 from pandas.tseries.tools import normalize_date
 
@@ -211,7 +210,7 @@ def set_date(self, date):
 
     def update_dividends(self, dividend_frame):
         """
-        Set the dividend_frame to the updated dividend frame. 
+        Set the dividend_frame to the updated dividend frame.
         DataFrame columns should contain at least the entries in
         zp.DIVIDEND_FIELDS.
         """

zipline/gens/tradesimulation.py

@@ -197,7 +197,6 @@ def _process_snapshot(self, dt, snapshot, instant_fill):
         occurring in the next snapshot.  This is the more conservative model,
         and as such it is the default behavior in TradingAlgorithm.
         """
-
         # Flags indicating whether we saw any events of type TRADE and type
         # BENCHMARK.  Respectively, these control whether or not handle_data is
         # called for this snapshot and whether we emit a perf message for this

zipline/history/history_container.py

@@ -20,7 +20,6 @@
 import numpy as np
 import pandas as pd
 from six import itervalues, iteritems, iterkeys
-from datetime import date, timedelta
 
 from . history import HistorySpec
 
@@ -487,7 +486,6 @@ def _create_panel(self, dt, spec, env=None):
             window=window,
             items=self.fields,
             sids=self.sids,
-            dividend_adjusted=spec.dividend_adjusted,
             initial_dates=date_buf,
         )
 
@@ -717,7 +715,7 @@ def frame_from_bardata(self, data, algo_dt):
         """
         data = data._data
         frame_data = np.empty((len(self.fields), len(self.sids))) * np.nan
-        
+
         for j, sid in enumerate(self.sids):
             sid_data = data.get(sid)
 
@@ -747,7 +745,7 @@ def update(self, data, algo_dt):
 
     def update_dividends(self, dividend_frame):
         """
-        Set the dividend_frame to the updated dividend frame. 
+        Set the dividend_frame to the updated dividend frame.
         DataFrame columns should contain at least the entries in
         zp.DIVIDEND_FIELDS.
         """
@@ -944,6 +942,10 @@ def get_history(self, history_spec, algo_dt):
                                                    columns=self.sids)
 
         if do_dividend_adjust:
+            # return the non adjusted price if there are no dividends
+            if self.dividend_frame.empty:
+                return history_output
+
             for security_object in history_output.columns.values:
                 dividends = get_dividends_for_security(self.dividend_frame,
                                                        security_object,
@@ -970,9 +972,9 @@ def get_history(self, history_spec, algo_dt):
 
                 history_output[security_object] = adjusted_pricing
 
-
         return history_output
 
+
 def get_dividends_for_security(dividend_frame, security_object, algo_dt):
     # get the dividends for this security
     dividends = dividend_frame[dividend_frame['sid'] == security_object]
@@ -984,11 +986,12 @@ def get_dividends_for_security(dividend_frame, security_object, algo_dt):
 
     return dividends
 
+
 def get_dividend_multiplier(close_price_day_before_ex_date,
                             dividend_amt):
-
     return 1 - (dividend_amt/close_price_day_before_ex_date)
 
+
 def fast_build_history_output(buffer_frame,
                               last_period,
                               algo_dt,

zipline/protocol.py

@@ -334,7 +334,8 @@ def __contains__(self, name):
     def __repr__(self):
         return "SIDData({0})".format(self.__dict__)
 
-    def _get_buffer(self, bars, field='price', raw=False):
+    def _get_buffer(self, bars, field='price', raw=False,
+                    dividend_adjusted=False):
         """
         Gets the result of history for the given number of bars and field.
 
@@ -357,6 +358,7 @@ def _get_buffer(self, bars, field='price', raw=False):
             # then we need to get more history.
             hst = algo.history(
                 bars, self._freqstr, field, ffill=True,
+                dividend_adjusted=dividend_adjusted,
             )
             # Assert that the column holds ints, not security objects.
             if not isinstance(self._sid, str):
@@ -441,19 +443,22 @@ def minute_get_bars(days, env=None):
     def mavg(self, days):
         bars = self._get_bars(days)
         max_bars = self._get_max_bars(days)
-        prices = self._get_buffer(max_bars, raw=True)[-bars:]
+        prices = self._get_buffer(max_bars, raw=True,
+                                  dividend_adjusted=True)[-bars:]
         return nanmean(prices)
 
     def stddev(self, days):
         bars = self._get_bars(days)
         max_bars = self._get_max_bars(days)
-        prices = self._get_buffer(max_bars, raw=True)[-bars:]
+        prices = self._get_buffer(max_bars, raw=True,
+                                  dividend_adjusted=True)[-bars:]
         return nanstd(prices, ddof=1)
 
     def vwap(self, days):
         bars = self._get_bars(days)
         max_bars = self._get_max_bars(days)
-        prices = self._get_buffer(max_bars, raw=True)[-bars:]
+        prices = self._get_buffer(max_bars, raw=True,
+                                  dividend_adjusted=True)[-bars:]
         vols = self._get_buffer(max_bars, field='volume', raw=True)[-bars:]
 
         vol_sum = nansum(vols)
@@ -470,7 +475,8 @@ def returns(self):
         now = algo.datetime
         if now != self._returns_cache_dt:
             self._returns_cache_dt = now
-            self._returns_cache = algo.history(2, '1d', 'price', ffill=True)
+            self._returns_cache = algo.history(2, '1d', 'price', ffill=True,
+                                               dividend_adjusted=True)
 
         hst = self._returns_cache[self._sid]
         return (hst.iloc[-1] - hst.iloc[0]) / hst.iloc[0]

zipline/utils/data.py

@@ -38,7 +38,6 @@ def __init__(self,
                  window,
                  items,
                  sids,
-                 dividend_adjusted,
                  cap_multiple=2,
                  dtype=np.float64,
                  initial_dates=None):
@@ -50,7 +49,6 @@ def __init__(self,
         self.minor_axis = _ensure_index(sids)
 
         self.cap_multiple = cap_multiple
-        self.dividend_adjusted = dividend_adjusted
 
         self.dtype = dtype
         if initial_dates is None: