Merge pull request #2680 from quantopian/read-hdf-fix-py3

H5 Minute Reader Bug in Python 3

zipline/data/minute_bars.py

@@ -21,6 +21,7 @@
 from lru import LRU
 import bcolz
 from bcolz import ctable
+import h5py
 from intervaltree import IntervalTree
 import logbook
 import numpy as np
@@ -1373,7 +1374,36 @@ class H5MinuteBarUpdateReader(MinuteBarUpdateReader):
         The path of the HDF5 file from which to source data.
     """
     def __init__(self, path):
-        self._panel = pd.read_hdf(path)
+        # There is a bug in `pandas.read_hdf` whereby in Python 3 it fails to
+        # read the timezone attr of an h5 file if that file was written in
+        # Python 2. Until zipline has dropped Python 2 entirely we are at risk
+        # of hitting this issue. For now, use h5py to read the file instead.
+        # The downside of using h5py directly is that we need to interpret the
+        # attrs manually when creating our panel (specifically the tz attr),
+        # but since we know exactly how the file was written this should be
+        # pretty straightforward.
+        with h5py.File(path, 'r') as f:
+            updates = f['updates']
+            values = updates['block0_values']
+            items = updates['axis0']
+            major = updates['axis1']
+            minor = updates['axis2']
+
+            # Our current version of h5py is unable to read the tz attr in the
+            # tests as it was written by HDFStore. This is fixed in version
+            # 2.10.0 of h5py, but that requires >=Python3.7 on conda, so until
+            # then we should be safe to assume UTC.
+            try:
+                tz = major.attrs['tz'].decode()
+            except OSError:
+                tz = 'UTC'
+
+            self._panel = pd.Panel(
+                data=np.array(values).T,
+                items=np.array(items),
+                major_axis=pd.DatetimeIndex(major, tz=tz, freq='T'),
+                minor_axis=np.array(minor).astype('U'),
+            )
 
     def read(self, dts, sids):
         panel = self._panel[sids, dts, :]