Permalink
Browse files

MAINT: Upgrade numpy and fix warnings.

Mostly fixes ambiguous calls to numpy.full, and uses explicitly-united
NaT values.
  • Loading branch information...
ssanderson authored and richafrank committed Jan 22, 2016
1 parent b651993 commit 5f49fa22cb52bb139a5d5c4ff05f1b12884fe23f
@@ -24,7 +24,7 @@ six==1.9.0
# For fetching remote data
requests==2.9.1
Cython==0.22.1
Cython==0.23.4
# faster OrderedDict
cyordereddict==0.2.2
@@ -41,7 +41,7 @@ networkx==1.9.1
numexpr==2.4.3
# On disk storage format for pipeline data.
bcolz==0.10.0
bcolz==0.12.1
# Command line interface helper
click==4.0.0
136 setup.py
@@ -24,6 +24,7 @@
join,
)
from distutils.version import StrictVersion
from pkg_resources import resource_filename
from setuptools import (
Extension,
find_packages,
@@ -33,43 +34,65 @@
import versioneer
class LazyCythonizingList(list):
cythonized = False
def lazy_cythonize(self):
if self.cythonized:
return
self.cythonized = True
from Cython.Build import cythonize
from numpy import get_include
self[:] = cythonize(
[
Extension(*ext_args, include_dirs=[get_include()])
for ext_args in self
]
class LazyBuildExtCommandClass(dict):
"""
Lazy command class that defers operations requiring Cython and numpy until
they've actually been downloaded and installed by setup_requires.
"""
def __contains__(self, key):
return (
key == 'build_ext'
or super(LazyBuildExtCommandClass, self).__contains__(key)
)
def __iter__(self):
self.lazy_cythonize()
return super(LazyCythonizingList, self).__iter__()
def __getitem__(self, num):
self.lazy_cythonize()
return super(LazyCythonizingList, self).__getitem__(num)
ext_modules = LazyCythonizingList([
('zipline.assets._assets', ['zipline/assets/_assets.pyx']),
('zipline.lib.adjustment', ['zipline/lib/adjustment.pyx']),
('zipline.lib._float64window', ['zipline/lib/_float64window.pyx']),
('zipline.lib._int64window', ['zipline/lib/_int64window.pyx']),
('zipline.lib._uint8window', ['zipline/lib/_uint8window.pyx']),
('zipline.lib.rank', ['zipline/lib/rank.pyx']),
('zipline.data._equities', ['zipline/data/_equities.pyx']),
('zipline.data._adjustments', ['zipline/data/_adjustments.pyx']),
])
def __setitem__(self, key, value):
if key == 'build_ext':
raise AssertionError("build_ext overridden!")
super(LazyBuildExtCommandClass, self).__setitem__(key, value)
def __getitem__(self, key):
if key != 'build_ext':
return super(LazyBuildExtCommandClass, self).__getitem__(key)
from Cython.Distutils import build_ext as cython_build_ext
class build_ext(cython_build_ext):
"""
Custom build_ext command that lazily adds numpy's include_dir to
extensions.
"""
def build_extensions(self):
"""
Lazily append numpy's include directory to Extension includes.
This is done here rather than at module scope because setup.py
may be run before numpy has been installed, in which case
importing numpy and calling `numpy.get_include()` will fail.
"""
numpy_incl = resource_filename('numpy', 'core/include')
for ext in self.extensions:
ext.include_dirs.append(numpy_incl)
# This explicitly calls the superclass method rather than the
# usual super() invocation because distutils' build_class, of
# which Cython's build_ext is a subclass, is an old-style class
# in Python 2, which doesn't support `super`.
cython_build_ext.build_extensions(self)
return build_ext
ext_modules = [
Extension('zipline.assets._assets', ['zipline/assets/_assets.pyx']),
Extension('zipline.lib.adjustment', ['zipline/lib/adjustment.pyx']),
Extension(
'zipline.lib._float64window', ['zipline/lib/_float64window.pyx']
),
Extension('zipline.lib._int64window', ['zipline/lib/_int64window.pyx']),
Extension('zipline.lib._uint8window', ['zipline/lib/_uint8window.pyx']),
Extension('zipline.lib.rank', ['zipline/lib/rank.pyx']),
Extension('zipline.data._equities', ['zipline/data/_equities.pyx']),
Extension('zipline.data._adjustments', ['zipline/data/_adjustments.pyx']),
]
STR_TO_CMP = {
@@ -116,9 +139,8 @@ def _filter_requirements(lines_iter):
yield requirement
REQ_UPPER_BOUNDS = {
'numpy': '<1.10',
}
# We don't currently have any known upper bounds.
REQ_UPPER_BOUNDS = {}
def _with_bounds(req):
@@ -183,11 +205,12 @@ def extras_requires(conda_format=False):
}
def module_requirements(requirements_path, module_names):
def module_requirements(requirements_path, module_names, strict_bounds):
module_names = set(module_names)
found = set()
module_lines = []
for line in read_requirements(requirements_path, strict_bounds=True):
for line in read_requirements(requirements_path,
strict_bounds=strict_bounds):
match = REQ_PATTERN.match(line)
if match is None:
raise AssertionError("Could not parse requirement: '%s'" % line)
@@ -203,38 +226,12 @@ def module_requirements(requirements_path, module_names):
)
return module_lines
def pre_setup():
if not set(sys.argv) & {'install', 'develop', 'egg_info', 'bdist_wheel'}:
return
try:
import pip
if StrictVersion(pip.__version__) < StrictVersion('7.1.0'):
raise AssertionError(
"Zipline installation requires pip>=7.1.0, but your pip "
"version is {version}. \n"
"You can upgrade your pip with "
"'pip install --upgrade pip'.".format(
version=pip.__version__,
)
)
except ImportError:
raise AssertionError("Zipline installation requires pip")
required = ('Cython', 'numpy')
for line in module_requirements('etc/requirements.txt', required):
pip.main(['install', line])
pre_setup()
conda_build = os.path.basename(sys.argv[0]) == 'conda-build'
setup(
name='zipline',
version=versioneer.get_version(),
cmdclass=versioneer.get_cmdclass(),
cmdclass=LazyBuildExtCommandClass(versioneer.get_cmdclass()),
description='A backtester for financial algorithms.',
author='Quantopian Inc.',
author_email='opensource@quantopian.com',
@@ -258,5 +255,10 @@ def pre_setup():
],
install_requires=install_requires(conda_format=conda_build),
extras_require=extras_requires(conda_format=conda_build),
setup_requires=module_requirements(
'etc/requirements.txt',
('Cython', 'numpy'),
strict_bounds=False,
),
url="http://zipline.io",
)
@@ -26,7 +26,7 @@
SID_FIELD_NAME,
TS_FIELD_NAME,
)
from zipline.utils.numpy_utils import make_datetime64D, np_NaT
from zipline.utils.numpy_utils import make_datetime64D, NaTD
from zipline.utils.test_utils import (
make_simple_equity_info,
tmp_asset_finder,
@@ -234,7 +234,7 @@ def _compute_busday_offsets(announcement_dates):
# Set NaTs to 0 temporarily because busday_count doesn't support NaT.
# We fill these entries with NaNs later.
whereNaT = raw_announce_dates == np_NaT
whereNaT = raw_announce_dates == NaTD
raw_announce_dates[whereNaT] = make_datetime64D(0)
# The abs call here makes it so that we can use this function to
@@ -257,7 +257,7 @@ def test_single_factor(self):
check_arrays(
result['f'].unstack().values,
full(result_shape, expected_result),
full(result_shape, expected_result, dtype=float),
)
def test_multiple_rolling_factors(self):
@@ -295,16 +295,16 @@ def test_multiple_rolling_factors(self):
# row-wise sum over an array whose values are all (1 - 2)
check_arrays(
results['short'].unstack().values,
full(shape, -short_factor.window_length),
full(shape, -short_factor.window_length, dtype=float),
)
check_arrays(
results['long'].unstack().values,
full(shape, -long_factor.window_length),
full(shape, -long_factor.window_length, dtype=float),
)
# row-wise sum over an array whose values are all (1 - 3)
check_arrays(
results['high'].unstack().values,
full(shape, -2 * high_factor.window_length),
full(shape, -2 * high_factor.window_length, dtype=float),
)
def test_numeric_factor(self):
@@ -398,13 +398,19 @@ def test_rolling_and_nonrolling(self):
result_shape = (len(result_index),)
check_arrays(
result['sumdiff'],
Series(index=result_index, data=full(result_shape, -3)),
Series(
index=result_index,
data=full(result_shape, -3, dtype=float),
),
)
for name, const in [('open', 1), ('close', 2), ('volume', 3)]:
check_arrays(
result[name],
Series(index=result_index, data=full(result_shape, const)),
Series(
index=result_index,
data=full(result_shape, const, dtype=float),
),
)
def test_loader_given_multiple_columns(self):
@@ -471,19 +477,26 @@ class Loader2DataSet(DataSet):
for name, pipe_col in iteritems(columns)}
index = MultiIndex.from_product([self.dates[2:], self.assets])
def expected_for_col(col):
val = vals[col]
offset = columns[col].window_length - min_window
return concatenate(
[
full(offset * index.levshape[1], nan),
full(
(index.levshape[0] - offset) * index.levshape[1],
val,
float,
)
],
)
expected = DataFrame(
data={col:
concatenate((
full((columns[col].window_length - min_window)
* index.levshape[1],
nan),
full((index.levshape[0]
- (columns[col].window_length - min_window))
* index.levshape[1],
val)))
for col, val in iteritems(vals)},
data={col: expected_for_col(col) for col in vals},
index=index,
columns=columns)
columns=columns,
)
assert_frame_equal(result, expected)
@@ -23,7 +23,12 @@
RSI,
)
from zipline.utils.test_utils import check_allclose, check_arrays
from zipline.utils.numpy_utils import datetime64ns_dtype, float64_dtype, np_NaT
from zipline.utils.numpy_utils import (
datetime64ns_dtype,
float64_dtype,
NaTD,
NaTns,
)
from .base import BasePipelineTestCase
@@ -309,7 +314,7 @@ def test_masked_rankdata_2d(self,
mask = eyemask if use_mask else nomask
if set_missing:
asfloat[:, 2] = nan
asdatetime[:, 2] = np_NaT
asdatetime[:, 2] = NaTns
float_result = masked_rankdata_2d(
data=asfloat,
@@ -321,7 +326,7 @@ def test_masked_rankdata_2d(self,
datetime_result = masked_rankdata_2d(
data=asdatetime,
mask=mask,
missing_value=np_NaT,
missing_value=NaTns,
method=method,
ascending=True,
)
@@ -76,9 +76,9 @@ def setUp(self):
self.h = H()
self.d = DateFactor()
self.fake_raw_data = {
self.f: full((5, 5), 3),
self.g: full((5, 5), 2),
self.h: full((5, 5), 1),
self.f: full((5, 5), 3, float),
self.g: full((5, 5), 2, float),
self.h: full((5, 5), 1, float),
self.d: full((5, 5), 0, dtype='datetime64[ns]'),
}
self.mask = DataFrame(True, index=self.dates, columns=self.assets)
@@ -94,7 +94,7 @@ def check_output(self, expr, expected):
def check_constant_output(self, expr, expected):
self.assertFalse(isnan(expected))
return self.check_output(expr, full((5, 5), expected))
return self.check_output(expr, full((5, 5), expected, float))
def test_validate_good(self):
f = self.f
@@ -435,9 +435,9 @@ def test_math_functions(self):
def test_comparisons(self):
f, g, h = self.f, self.g, self.h
self.fake_raw_data = {
f: arange(25).reshape(5, 5),
g: arange(25).reshape(5, 5) - eye(5),
h: full((5, 5), 5),
f: arange(25, dtype=float).reshape(5, 5),
g: arange(25, dtype=float).reshape(5, 5) - eye(5),
h: full((5, 5), 5, dtype=float),
}
f_data = self.fake_raw_data[f]
g_data = self.fake_raw_data[g]
@@ -479,9 +479,9 @@ def test_boolean_binops(self):
)
self.fake_raw_data = {
f: arange(25).reshape(5, 5),
g: arange(25).reshape(5, 5) - eye(5),
h: full((5, 5), 5),
f: arange(25, dtype=float).reshape(5, 5),
g: arange(25, dtype=float).reshape(5, 5) - eye(5),
h: full((5, 5), 5, dtype=float),
custom_filter: custom_filter_mask,
}
@@ -40,6 +40,7 @@ class SomeFactor(Factor):
dtype = float64_dtype
window_length = 5
inputs = [SomeDataSet.foo, SomeDataSet.bar]
SomeFactorAlias = SomeFactor
Oops, something went wrong.

0 comments on commit 5f49fa2

Please sign in to comment.