From bad7c0f4379d07d6a6f335b7052b2145e62bd0b4 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 2 Mar 2016 08:35:24 -0500 Subject: [PATCH] COMPAT: blacklist numexpr=2.4.4 closes #12489 closes #12511 --- ci/requirements-3.4_SLOW.run | 2 +- doc/source/install.rst | 2 +- doc/source/whatsnew/v0.18.0.txt | 4 ++ pandas/computation/__init__.py | 30 +++++++++++ pandas/computation/eval.py | 14 ++--- pandas/computation/expressions.py | 15 +----- pandas/computation/tests/test_compat.py | 69 +++++++++++++++++++++++++ pandas/computation/tests/test_eval.py | 27 ---------- 8 files changed, 110 insertions(+), 53 deletions(-) create mode 100644 pandas/computation/tests/test_compat.py diff --git a/ci/requirements-3.4_SLOW.run b/ci/requirements-3.4_SLOW.run index f0101d34204a3..f9f226e3f1465 100644 --- a/ci/requirements-3.4_SLOW.run +++ b/ci/requirements-3.4_SLOW.run @@ -9,7 +9,7 @@ html5lib patsy beautiful-soup scipy -numexpr +numexpr=2.4.4 pytables matplotlib lxml diff --git a/doc/source/install.rst b/doc/source/install.rst index 3836180af520f..11b9115aa9c81 100644 --- a/doc/source/install.rst +++ b/doc/source/install.rst @@ -225,7 +225,7 @@ Recommended Dependencies * `numexpr `__: for accelerating certain numerical operations. ``numexpr`` uses multiple cores as well as smart chunking and caching to achieve large speedups. - If installed, must be Version 2.1 or higher. Version 2.4.6 or higher on Windows is highly recommended. + If installed, must be Version 2.1 or higher (excluding a buggy 2.4.4). Version 2.4.6 or higher is highly recommended. * `bottleneck `__: for accelerating certain types of ``nan`` evaluations. ``bottleneck`` uses specialized cython routines to achieve large speedups. diff --git a/doc/source/whatsnew/v0.18.0.txt b/doc/source/whatsnew/v0.18.0.txt index 34e024def332b..049152dba9a30 100644 --- a/doc/source/whatsnew/v0.18.0.txt +++ b/doc/source/whatsnew/v0.18.0.txt @@ -12,6 +12,10 @@ users upgrade to this version. pandas >= 0.18.0 no longer supports compatibility with Python version 2.6 and 3.3 (:issue:`7718`, :issue:`11273`) +.. warning:: + + ``numexpr`` version 2.4.4 will now show a warning and not be used as a computation back-end for pandas because of some buggy behavior. This does not affect other versions (>= 2.1 and >= 2.4.6). (:issue:`12489`) + Highlights include: - Moving and expanding window functions are now methods on Series and DataFrame, diff --git a/pandas/computation/__init__.py b/pandas/computation/__init__.py index e69de29bb2d1d..9e94215eecf62 100644 --- a/pandas/computation/__init__.py +++ b/pandas/computation/__init__.py @@ -0,0 +1,30 @@ + +import warnings +from distutils.version import LooseVersion + +_NUMEXPR_INSTALLED = False + +try: + import numexpr as ne + ver = ne.__version__ + _NUMEXPR_INSTALLED = ver >= LooseVersion('2.1') + + # we specifically disallow 2.4.4 as + # has some hard-to-diagnose bugs + if ver == LooseVersion('2.4.4'): + _NUMEXPR_INSTALLED = False + warnings.warn( + "The installed version of numexpr {ver} is not supported " + "in pandas and will be not be used\n".format(ver=ver), + UserWarning) + + elif not _NUMEXPR_INSTALLED: + warnings.warn( + "The installed version of numexpr {ver} is not supported " + "in pandas and will be not be used\nThe minimum supported " + "version is 2.1\n".format(ver=ver), UserWarning) + +except ImportError: # pragma: no cover + pass + +__all__ = ['_NUMEXPR_INSTALLED'] diff --git a/pandas/computation/eval.py b/pandas/computation/eval.py index d2d16acc27fb6..c3300ffca468e 100644 --- a/pandas/computation/eval.py +++ b/pandas/computation/eval.py @@ -6,11 +6,11 @@ import warnings import tokenize from pandas.core import common as com +from pandas.computation import _NUMEXPR_INSTALLED from pandas.computation.expr import Expr, _parsers, tokenize_string from pandas.computation.scope import _ensure_scope from pandas.compat import string_types from pandas.computation.engines import _engines -from distutils.version import LooseVersion def _check_engine(engine): @@ -35,17 +35,11 @@ def _check_engine(engine): # that won't necessarily be import-able) # Could potentially be done on engine instantiation if engine == 'numexpr': - try: - import numexpr - except ImportError: - raise ImportError("'numexpr' not found. Cannot use " + if not _NUMEXPR_INSTALLED: + raise ImportError("'numexpr' is not installed or an " + "unsupported version. Cannot use " "engine='numexpr' for query/eval " "if 'numexpr' is not installed") - else: - ne_version = numexpr.__version__ - if ne_version < LooseVersion('2.1'): - raise ImportError("'numexpr' version is %s, " - "must be >= 2.1" % ne_version) def _check_parser(parser): diff --git a/pandas/computation/expressions.py b/pandas/computation/expressions.py index 6e33250010c2b..82fbefd5ab608 100644 --- a/pandas/computation/expressions.py +++ b/pandas/computation/expressions.py @@ -9,20 +9,7 @@ import warnings import numpy as np from pandas.core.common import _values_from_object -from distutils.version import LooseVersion - -try: - import numexpr as ne - ver = ne.__version__ - _NUMEXPR_INSTALLED = ver >= LooseVersion('2.1') - if not _NUMEXPR_INSTALLED: - warnings.warn( - "The installed version of numexpr {ver} is not supported " - "in pandas and will be not be used\nThe minimum supported " - "version is 2.1\n".format(ver=ver), UserWarning) - -except ImportError: # pragma: no cover - _NUMEXPR_INSTALLED = False +from pandas.computation import _NUMEXPR_INSTALLED _TEST_MODE = None _TEST_RESULT = None diff --git a/pandas/computation/tests/test_compat.py b/pandas/computation/tests/test_compat.py new file mode 100644 index 0000000000000..80b415739c647 --- /dev/null +++ b/pandas/computation/tests/test_compat.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python + +# flake8: noqa + +import nose +from itertools import product +from distutils.version import LooseVersion + +import pandas as pd +from pandas.util import testing as tm + +from pandas.computation.engines import _engines +import pandas.computation.expr as expr + +ENGINES_PARSERS = list(product(_engines, expr._parsers)) + + +def test_compat(): + # test we have compat with our version of nu + + from pandas.computation import _NUMEXPR_INSTALLED + try: + import numexpr as ne + ver = ne.__version__ + if ver == LooseVersion('2.4.4'): + assert not _NUMEXPR_INSTALLED + elif ver < LooseVersion('2.1'): + with tm.assert_produces_warning(UserWarning, + check_stacklevel=False): + assert not _NUMEXPR_INSTALLED + else: + assert _NUMEXPR_INSTALLED + + except ImportError: + raise nose.SkipTest("not testing numexpr version compat") + + +def test_invalid_numexpr_version(): + for engine, parser in ENGINES_PARSERS: + yield check_invalid_numexpr_version, engine, parser + + +def check_invalid_numexpr_version(engine, parser): + def testit(): + a, b = 1, 2 + res = pd.eval('a + b', engine=engine, parser=parser) + tm.assert_equal(res, 3) + + if engine == 'numexpr': + try: + import numexpr as ne + except ImportError: + raise nose.SkipTest("no numexpr") + else: + if ne.__version__ < LooseVersion('2.1'): + with tm.assertRaisesRegexp(ImportError, "'numexpr' version is " + ".+, must be >= 2.1"): + testit() + elif ne.__version__ == LooseVersion('2.4.4'): + raise nose.SkipTest("numexpr version==2.4.4") + else: + testit() + else: + testit() + + +if __name__ == '__main__': + nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'], + exit=False) diff --git a/pandas/computation/tests/test_eval.py b/pandas/computation/tests/test_eval.py index b70252ed9f35b..518e5dd999391 100644 --- a/pandas/computation/tests/test_eval.py +++ b/pandas/computation/tests/test_eval.py @@ -1782,33 +1782,6 @@ def test_name_error_exprs(): yield check_name_error_exprs, engine, parser -def check_invalid_numexpr_version(engine, parser): - def testit(): - a, b = 1, 2 - res = pd.eval('a + b', engine=engine, parser=parser) - tm.assert_equal(res, 3) - - if engine == 'numexpr': - try: - import numexpr as ne - except ImportError: - raise nose.SkipTest("no numexpr") - else: - if ne.__version__ < LooseVersion('2.1'): - with tm.assertRaisesRegexp(ImportError, "'numexpr' version is " - ".+, must be >= 2.1"): - testit() - else: - testit() - else: - testit() - - -def test_invalid_numexpr_version(): - for engine, parser in ENGINES_PARSERS: - yield check_invalid_numexpr_version, engine, parser - - def check_invalid_local_variable_reference(engine, parser): tm.skip_if_no_ne(engine)