Merge 6542b0d into 129d16f

.travis.yml

@@ -9,8 +9,7 @@ env:
     # ANACONDA_TOKEN with api:write
     - secure: "RJJjjQloUjCSkhII93QM+YAsr6YYq7hPFvlbnT07ogn1NeUPsVCyJ97oiZfKtPgdbd24hdQP/CHfB0HgTTES8n996tN3QWc4hZj0e10kFyIlas9qnkrRYRR1jxGShBDXrLdx/tGh8z1qnnCm1fy+fDhAF7Zerouwy4EA2YEzxdE="
   matrix:
-    - PANDAS_VERSION=0.16.1 NUMPY_VERSION=1.9.2 SCIPY_VERSION=0.15.1
-    - PANDAS_VERSION=0.17.1 NUMPY_VERSION=1.10.4 SCIPY_VERSION=0.16.1
+    - NUMPY_VERSION=1.11.1 SCIPY_VERSION=0.17.1
 cache:
   directories:
     - $HOME/.cache/.pip/
@@ -20,14 +19,11 @@ before_install:
   - chmod +x miniconda.sh
   - ./miniconda.sh -b -p $HOME/miniconda
   - export PATH="$HOME/miniconda/bin:$PATH"
-  - sed -i "s/numpy==.*/numpy==$NUMPY_VERSION/" etc/requirements.txt
-  - sed -i "s/pandas==.*/pandas==$PANDAS_VERSION/" etc/requirements.txt
-  - sed -i "s/scipy==.*/scipy==$SCIPY_VERSION/" etc/requirements.txt
 install:
   - conda install conda=4.0.9 conda-build=1.19.2 anaconda-client=1.3.1 --yes
 
   - TALIB_VERSION=$(cat ./etc/requirements_talib.txt | sed "s/TA-Lib==\(.*\)/\1/")
-  - conda create -n testenv --yes -c quantopian pip python=$TRAVIS_PYTHON_VERSION numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION libgfortran=1.0 ta-lib=$TALIB_VERSION
+  - conda create -n testenv --yes -c quantopian pip python=$TRAVIS_PYTHON_VERSION numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION libgfortran=3.0 ta-lib=$TALIB_VERSION
   - source activate testenv
   - IFS='.' read -r -a NPY_VERSION_ARR <<< "$NUMPY_VERSION"
   - CONDA_NPY=${NPY_VERSION_ARR[0]}${NPY_VERSION_ARR[1]}

appveyor.yml

@@ -15,27 +15,15 @@ environment:
   matrix:
     - PYTHON_VERSION: "2.7"
       PYTHON_ARCH: "64"
-      PANDAS_VERSION: "0.16.1"
-      NUMPY_VERSION: "1.9.2"
-      SCIPY_VERSION: "0.15.1"
+      PANDAS_VERSION: "0.18.1"
+      NUMPY_VERSION: "1.11"
+      SCIPY_VERSION: "0.17.0"
 
     - PYTHON_VERSION: "3.4"
       PYTHON_ARCH: "64"
-      PANDAS_VERSION: "0.16.1"
-      NUMPY_VERSION: "1.9.2"
-      SCIPY_VERSION: "0.15.1"
-
-    - PYTHON_VERSION: "2.7"
-      PYTHON_ARCH: "64"
-      PANDAS_VERSION: "0.17.1"
-      NUMPY_VERSION: "1.10.4"
-      SCIPY_VERSION: "0.16.1"
-
-    - PYTHON_VERSION: "3.4"
-      PYTHON_ARCH: "64"
-      PANDAS_VERSION: "0.17.1"
-      NUMPY_VERSION: "1.10.4"
-      SCIPY_VERSION: "0.16.1"
+      PANDAS_VERSION: "0.18.1"
+      NUMPY_VERSION: "1.11"
+      SCIPY_VERSION: "0.17.0"
 
 # We always use a 64-bit machine, but can build x86 distributions
 # with the PYTHON_ARCH variable (which is used by CMD_IN_ENV).

etc/requirements.txt

@@ -8,13 +8,13 @@ Logbook==0.12.5
 
 # Scientific Libraries
 
-pytz==2015.4
-numpy==1.9.2
+pytz==2016.4
+numpy==1.11.1
 
 # scipy and pandas are required for statsmodels,
 # statsmodels in turn is required for some pandas packages
-scipy==0.15.1
-pandas==0.16.1
+scipy==0.17.
+pandas==0.18.1
 pandas-datareader==0.2.1
 # Needed for parts of pandas.stats
 patsy==0.4.0
@@ -38,6 +38,7 @@ contextlib2==0.4.0
 
 # networkx requires decorator
 decorator==4.0.0
+
 # Graph algorithms used by zipline.pipeline
 networkx==1.9.1
 

setup.py

@@ -148,7 +148,7 @@ def _filter_requirements(lines_iter, filter_names=None,
 
 REQ_UPPER_BOUNDS = {
     'bcolz': '<1',
-    'pandas': '<0.18',
+    'pandas': '<0.19',
 }
 
 

tests/data/test_minute_bars.py

@@ -579,7 +579,7 @@ def test_nans(self):
                 'high': full(9, nan),
                 'low': full(9, nan),
                 'close': full(9, nan),
-                'volume': full(9, 0),
+                'volume': full(9, 0.0),
             },
             index=[minutes])
         self.writer.write_sid(sid, data)
@@ -622,7 +622,7 @@ def test_differing_nans(self):
                 view(float64),
                 'close': ((0b11111111111 << 52) + arange(31, 40, dtype=int64)).
                 view(float64),
-                'volume': full(9, 0),
+                'volume': full(9, 0.0),
             },
             index=[minutes])
         self.writer.write_sid(sid, data)

tests/pipeline/test_blaze.py

@@ -784,6 +784,7 @@ def _test_id_macro(self, df, dshape, expected, finder, add):
             expr,
             loader=loader,
             no_deltas_rule='ignore',
+            no_checkpoints_rule='ignore',
             missing_values=self.missing_values,
         )
 

tests/pipeline/test_column.py

@@ -11,6 +11,7 @@
 from zipline.pipeline import Pipeline
 from zipline.pipeline.data.testing import TestingDataSet as TDS
 from zipline.testing import chrange, temp_pipeline_engine
+from zipline.utils.pandas_utils import ignore_pandas_nan_categorical_warning
 
 
 class LatestTestCase(TestCase):
@@ -71,6 +72,8 @@ def test_latest(self):
             dates_to_test[-1],
         )
         for column in columns:
-            col_result = result[column.name].unstack()
+            with ignore_pandas_nan_categorical_warning():
+                col_result = result[column.name].unstack()
+
             expected_col_result = self.expected_latest(column, cal_slice)
             assert_frame_equal(col_result, expected_col_result)

tests/pipeline/test_engine.py

@@ -25,12 +25,8 @@
     Categorical,
     DataFrame,
     date_range,
-    ewma,
-    ewmstd,
     Int64Index,
     MultiIndex,
-    rolling_apply,
-    rolling_mean,
     Series,
     Timestamp,
 )
@@ -959,15 +955,17 @@ def test_SMA(self):
         # Shift back the raw inputs by a trading day because we expect our
         # computed results to be computed using values anchored on the
         # **previous** day's data.
-        expected_raw = rolling_mean(
+        expected_raw = DataFrame(
             expected_bar_values_2d(
                 dates - self.trading_calendar.day,
                 self.equity_info,
                 'close',
             ),
+        ).rolling(
             window_length,
             min_periods=1,
-        )
+        ).mean(
+        ).values
 
         expected = DataFrame(
             # Truncate off the extra rows needed to compute the SMAs.
@@ -1073,19 +1071,31 @@ def init_class_fixtures(cls):
     def expected_ewma(self, window_length, decay_rate):
         alpha = 1 - decay_rate
         span = (2 / alpha) - 1
-        return rolling_apply(
-            self.raw_data,
-            window_length,
-            lambda window: ewma(window, span=span)[-1],
+
+        # XXX: This is a comically inefficient way to compute a windowed EWMA.
+        # Don't use it outside of testing.  We're using rolling-apply of an
+        # ewma (which is itself a rolling-window function) because we only want
+        # to look at ``window_length`` rows at a time.
+        return self.raw_data.rolling(window_length).apply(
+            lambda subarray: (DataFrame(subarray)
+                              .ewm(span=span)
+                              .mean()
+                              .values[-1])
         )[window_length:]
 
     def expected_ewmstd(self, window_length, decay_rate):
         alpha = 1 - decay_rate
         span = (2 / alpha) - 1
-        return rolling_apply(
-            self.raw_data,
-            window_length,
-            lambda window: ewmstd(window, span=span)[-1],
+
+        # XXX: This is a comically inefficient way to compute a windowed
+        # EWMSTD.  Don't use it outside of testing.  We're using rolling-apply
+        # of an ewma (which is itself a rolling-window function) because we
+        # only want to look at ``window_length`` rows at a time.
+        return self.raw_data.rolling(window_length).apply(
+            lambda subarray: (DataFrame(subarray)
+                              .ewm(span=span)
+                              .std()
+                              .values[-1])
         )[window_length:]
 
     @parameterized.expand([
@@ -1210,7 +1220,7 @@ def test_dollar_volume(self):
         expected_1 = (self.raw_data[5:] ** 2) * 2
         assert_frame_equal(results['dv1'].unstack(), expected_1)
 
-        expected_5 = rolling_mean((self.raw_data ** 2) * 2, window=5)[5:]
+        expected_5 = ((self.raw_data ** 2) * 2).rolling(5).mean()[5:]
         assert_frame_equal(results['dv5'].unstack(), expected_5)
 
         # The following two use USEquityPricing.open and .volume as inputs.
@@ -1220,9 +1230,11 @@ def test_dollar_volume(self):
                           * self.raw_data[5:] * 2).fillna(0)
         assert_frame_equal(results['dv1_nan'].unstack(), expected_1_nan)
 
-        expected_5_nan = rolling_mean((self.raw_data_with_nans
-                                       * self.raw_data * 2).fillna(0),
-                                      window=5)[5:]
+        expected_5_nan = ((self.raw_data_with_nans * self.raw_data * 2)
+                          .fillna(0)
+                          .rolling(5).mean()
+                          [5:])
+
         assert_frame_equal(results['dv5_nan'].unstack(), expected_5_nan)
 
 

tests/pipeline/test_events.py

@@ -153,7 +153,7 @@ class EventIndexerTestCase(ZiplineTestCase):
     @classmethod
     def init_class_fixtures(cls):
         super(EventIndexerTestCase, cls).init_class_fixtures()
-        cls.events = make_events(add_nulls=False).sort('event_date')
+        cls.events = make_events(add_nulls=False).sort_values('event_date')
         cls.events.reset_index(inplace=True)
 
     def test_previous_event_indexer(self):
@@ -354,7 +354,7 @@ def check_previous_value_results(self, column, results):
         # from pandas won't be tz_localized.
         dates = self.trading_days.tz_localize(None)
 
-        for asset, asset_result in results.iterkv():
+        for asset, asset_result in results.iteritems():
             relevant_events = events[events.sid == asset.sid]
             self.assertEqual(len(relevant_events), 2)
 
@@ -397,7 +397,7 @@ def check_next_value_results(self, column, results):
         # Remove timezone info from trading days, since the outputs
         # from pandas won't be tz_localized.
         dates = self.trading_days.tz_localize(None)
-        for asset, asset_result in results.iterkv():
+        for asset, asset_result in results.iteritems():
             relevant_events = events[events.sid == asset.sid]
             self.assertEqual(len(relevant_events), 2)
 

tests/pipeline/test_pipeline_algo.py

@@ -346,7 +346,7 @@ class PipelineAlgorithmTestCase(WithBcolzEquityDailyBarReaderFromCSVs,
     AAPL = 1
     MSFT = 2
     BRK_A = 3
-    assets = ASSET_FINDER_EQUITY_SIDS = AAPL, MSFT, BRK_A
+    ASSET_FINDER_EQUITY_SIDS = AAPL, MSFT, BRK_A
     ASSET_FINDER_EQUITY_SYMBOLS = 'AAPL', 'MSFT', 'BRK_A'
     START_DATE = Timestamp('2014')
     END_DATE = Timestamp('2015')
@@ -403,6 +403,9 @@ def init_class_fixtures(cls):
         )
         cls.dates = cls.raw_data[cls.AAPL].index.tz_localize('UTC')
         cls.AAPL_split_date = Timestamp("2014-06-09", tz='UTC')
+        cls.assets = cls.asset_finder.retrieve_all(
+            cls.ASSET_FINDER_EQUITY_SIDS
+        )
 
     def compute_expected_vwaps(self, window_lengths):
         AAPL, MSFT, BRK_A = self.AAPL, self.MSFT, self.BRK_A
@@ -488,7 +491,7 @@ def compute_expected_vwaps(self, window_lengths):
         (False,),
     ])
     def test_handle_adjustment(self, set_screen):
-        AAPL, MSFT, BRK_A = assets = self.AAPL, self.MSFT, self.BRK_A
+        AAPL, MSFT, BRK_A = assets = self.assets
 
         window_lengths = [1, 2, 5, 10]
         vwaps = self.compute_expected_vwaps(window_lengths)

tests/pipeline/test_technical.py

@@ -197,17 +197,17 @@ def test_fso_expected_basic(self):
         fso = FastStochasticOscillator()
 
         today = pd.Timestamp('2015')
-        assets = np.arange(3, dtype=np.float)
-        out = np.empty(shape=(3,), dtype=np.float)
+        assets = np.arange(3, dtype=np.float64)
+        out = np.empty(shape=(3,), dtype=np.float64)
 
-        highs = np.full((50, 3), 3)
-        lows = np.full((50, 3), 2)
-        closes = np.full((50, 3), 4)
+        highs = np.full((50, 3), 3, dtype=np.float64)
+        lows = np.full((50, 3), 2, dtype=np.float64)
+        closes = np.full((50, 3), 4, dtype=np.float64)
 
         fso.compute(today, assets, out, closes, lows, highs)
 
         # Expected %K
-        assert_equal(out, np.full((3,), 200))
+        assert_equal(out, np.full((3,), 200, dtype=np.float64))
 
     def test_fso_expected_with_talib(self):
         """

tests/resources/rebuild_example_data

@@ -167,8 +167,8 @@ def main(ctx, rebuild_input):
             shutil.rmtree(expected_perf_path)
             shutil.copytree(new_perf_path, expected_perf_path)
 
-            with tarfile.open(example_path, 'w|gz') as tar:
-                tar.add(d.getpath('example_data'), 'example_data')
+        with tarfile.open(example_path, 'w|gz') as tar:
+            tar.add(d.getpath('example_data'), 'example_data')
 
 
 if __name__ == '__main__':

tests/test_algorithm.py

@@ -3293,7 +3293,7 @@ def make_equity_minute_bar_data(cls):
                 'high': minutes_arr + 2,
                 'low': minutes_arr - 1,
                 'close': minutes_arr,
-                'volume': np.full(minutes_count, 1),
+                'volume': np.full(minutes_count, 1.0),
             },
             index=asset_minutes,
         )
@@ -3302,11 +3302,11 @@ def make_equity_minute_bar_data(cls):
     def make_equity_daily_bar_data(cls):
         yield 1, pd.DataFrame(
             {
-                'open': np.full(3, 1),
-                'high': np.full(3, 1),
-                'low': np.full(3, 1),
-                'close': np.full(3, 1),
-                'volume': np.full(3, 1),
+                'open': np.full(3, 1, dtype=np.float64),
+                'high': np.full(3, 1, dtype=np.float64),
+                'low': np.full(3, 1, dtype=np.float64),
+                'close': np.full(3, 1, dtype=np.float64),
+                'volume': np.full(3, 1, dtype=np.float64),
             },
             index=cls.sim_params.sessions,
         )

tests/test_fetcher.py

@@ -297,7 +297,6 @@ def handle_data(context, data):
 
     @parameterized.expand([("unspecified", ""),
                            ("none", "usecols=None"),
-                           ("empty", "usecols=[]"),
                            ("without date", "usecols=['Value']"),
                            ("with date", "usecols=('Value', 'Date')")])
     def test_usecols(self, testname, usecols):
@@ -325,30 +324,9 @@ def initialize(context):
     context.stocks = [sid(3766), sid(25317)]
 
 def handle_data(context, data):
-    if {should_have_data}:
-        try:
-            data.current("urban", "cpi")
-        except (KeyError, ValueError):
-            assert False
-    else:
-        try:
-            data.current("urban", "cpi")
-        except (KeyError, ValueError):
-            assert True
+    data.current("urban", "cpi")
         """
-
-        results = self.run_algo(
-            code.format(
-                usecols=usecols,
-                should_have_data=testname in [
-                    'none',
-                    'unspecified',
-                    'without date',
-                    'with date',
-                ],
-            )
-        )
-
+        results = self.run_algo(code.format(usecols=usecols))
         # 251 trading days in 2006
         self.assertEqual(len(results), 251)