Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

pandas-0.25.0rc0 problem on Python-3.8: 'PandasExprVisitor' object has no attribute 'visit_Constant' #27261

Closed
stonebig opened this issue Jul 6, 2019 · 5 comments · Fixed by #28101
Labels
Milestone

Comments

@stonebig
Copy link
Contributor

@stonebig stonebig commented Jul 6, 2019

Code Sample, a copy-pastable example if possible

import pandas as pd
import numpy as np

idx = pd.date_range('2000', '2005', freq='d', closed='left')
datas = pd.DataFrame({'Color':  [ 'green' if x> 1 else 'red' for x in np.random.randn(len(idx))], 
         'Measure': np.random.randn(len(idx)), 'Year': idx.year},
          index=idx.date)
datas.query('Measure > 0').groupby(['Color','Year']).size().unstack()

blows-up with

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-18-6c03ccdf9365> in <module>
      6          'Measure': np.random.randn(len(idx)), 'Year': idx.year},
      7           index=idx.date)
----> 8 datas.query('Measure > 0').groupby(['Color','Year']).size().unstack()

C:\WinP\bd38\bu\WPy32-3800b2\python-3.8.0b2\lib\site-packages\pandas\core\frame.py in query(self, expr, inplace, **kwargs)
   3190         kwargs["level"] = kwargs.pop("level", 0) + 1
   3191         kwargs["target"] = None
-> 3192         res = self.eval(expr, **kwargs)
   3193 
   3194         try:

C:\WinP\bd38\bu\WPy32-3800b2\python-3.8.0b2\lib\site-packages\pandas\core\frame.py in eval(self, expr, inplace, **kwargs)
   3306             kwargs["target"] = self
   3307         kwargs["resolvers"] = kwargs.get("resolvers", ()) + tuple(resolvers)
-> 3308         return _eval(expr, inplace=inplace, **kwargs)
   3309 
   3310     def select_dtypes(self, include=None, exclude=None):

C:\WinP\bd38\bu\WPy32-3800b2\python-3.8.0b2\lib\site-packages\pandas\core\computation\eval.py in eval(expr, parser, engine, truediv, local_dict, global_dict, resolvers, level, target, inplace)
    320         )
    321 
--> 322         parsed_expr = Expr(expr, engine=engine, parser=parser, env=env, truediv=truediv)
    323 
    324         # construct the engine and evaluate the parsed expression

C:\WinP\bd38\bu\WPy32-3800b2\python-3.8.0b2\lib\site-packages\pandas\core\computation\expr.py in __init__(self, expr, engine, parser, env, truediv, level)
    825         self.env.scope["truediv"] = truediv
    826         self._visitor = _parsers[parser](self.env, self.engine, self.parser)
--> 827         self.terms = self.parse()
    828 
    829     @property

C:\WinP\bd38\bu\WPy32-3800b2\python-3.8.0b2\lib\site-packages\pandas\core\computation\expr.py in parse(self)
    842     def parse(self):
    843         """Parse an expression"""
--> 844         return self._visitor.visit(self.expr)
    845 
    846     @property

C:\WinP\bd38\bu\WPy32-3800b2\python-3.8.0b2\lib\site-packages\pandas\core\computation\expr.py in visit(self, node, **kwargs)
    439         method = "visit_" + node.__class__.__name__
    440         visitor = getattr(self, method)
--> 441         return visitor(node, **kwargs)
    442 
    443     def visit_Module(self, node, **kwargs):

C:\WinP\bd38\bu\WPy32-3800b2\python-3.8.0b2\lib\site-packages\pandas\core\computation\expr.py in visit_Module(self, node, **kwargs)
    445             raise SyntaxError("only a single expression is allowed")
    446         expr = node.body[0]
--> 447         return self.visit(expr, **kwargs)
    448 
    449     def visit_Expr(self, node, **kwargs):

C:\WinP\bd38\bu\WPy32-3800b2\python-3.8.0b2\lib\site-packages\pandas\core\computation\expr.py in visit(self, node, **kwargs)
    439         method = "visit_" + node.__class__.__name__
    440         visitor = getattr(self, method)
--> 441         return visitor(node, **kwargs)
    442 
    443     def visit_Module(self, node, **kwargs):

C:\WinP\bd38\bu\WPy32-3800b2\python-3.8.0b2\lib\site-packages\pandas\core\computation\expr.py in visit_Expr(self, node, **kwargs)
    448 
    449     def visit_Expr(self, node, **kwargs):
--> 450         return self.visit(node.value, **kwargs)
    451 
    452     def _rewrite_membership_op(self, node, left, right):

C:\WinP\bd38\bu\WPy32-3800b2\python-3.8.0b2\lib\site-packages\pandas\core\computation\expr.py in visit(self, node, **kwargs)
    439         method = "visit_" + node.__class__.__name__
    440         visitor = getattr(self, method)
--> 441         return visitor(node, **kwargs)
    442 
    443     def visit_Module(self, node, **kwargs):

C:\WinP\bd38\bu\WPy32-3800b2\python-3.8.0b2\lib\site-packages\pandas\core\computation\expr.py in visit_Compare(self, node, **kwargs)
    742             op = self.translate_In(ops[0])
    743             binop = ast.BinOp(op=op, left=node.left, right=comps[0])
--> 744             return self.visit(binop)
    745 
    746         # recursive case: we have a chained comparison, a CMP b CMP c, etc.

C:\WinP\bd38\bu\WPy32-3800b2\python-3.8.0b2\lib\site-packages\pandas\core\computation\expr.py in visit(self, node, **kwargs)
    439         method = "visit_" + node.__class__.__name__
    440         visitor = getattr(self, method)
--> 441         return visitor(node, **kwargs)
    442 
    443     def visit_Module(self, node, **kwargs):

C:\WinP\bd38\bu\WPy32-3800b2\python-3.8.0b2\lib\site-packages\pandas\core\computation\expr.py in visit_BinOp(self, node, **kwargs)
    561 
    562     def visit_BinOp(self, node, **kwargs):
--> 563         op, op_class, left, right = self._maybe_transform_eq_ne(node)
    564         left, right = self._maybe_downcast_constants(left, right)
    565         return self._maybe_evaluate_binop(op, op_class, left, right)

C:\WinP\bd38\bu\WPy32-3800b2\python-3.8.0b2\lib\site-packages\pandas\core\computation\expr.py in _maybe_transform_eq_ne(self, node, left, right)
    482             left = self.visit(node.left, side="left")
    483         if right is None:
--> 484             right = self.visit(node.right, side="right")
    485         op, op_class, left, right = self._rewrite_membership_op(node, left, right)
    486         return op, op_class, left, right

C:\WinP\bd38\bu\WPy32-3800b2\python-3.8.0b2\lib\site-packages\pandas\core\computation\expr.py in visit(self, node, **kwargs)
    438 
    439         method = "visit_" + node.__class__.__name__
--> 440         visitor = getattr(self, method)
    441         return visitor(node, **kwargs)
    442 

AttributeError: 'PandasExprVisitor' object has no attribute 'visit_Constant'

@stonebig stonebig changed the title pandas-0.25.0rc0 proble on Python-3.8 pandas-0.25.0rc0 problem on Python-3.8 Jul 6, 2019
@stonebig stonebig changed the title pandas-0.25.0rc0 problem on Python-3.8 pandas-0.25.0rc0 problem on Python-3.8: 'PandasExprVisitor' object has no attribute 'visit_Constant' Jul 6, 2019
@stonebig

This comment has been minimized.

Copy link
Contributor Author

@stonebig stonebig commented Jul 6, 2019

well, it's a duplicate of #26318, just now with Pandas-0.25.0rc0

@jorisvandenbossche jorisvandenbossche added this to the 0.25.0 milestone Jul 6, 2019
@jorisvandenbossche

This comment has been minimized.

Copy link
Member

@jorisvandenbossche jorisvandenbossche commented Jul 6, 2019

This is related to the ast.Constant change in Python 3.8 (where Constant is replacing Str, Num, e..): python/cpython#9445

Some related bpo issues with discussion around breakages and suggested solutions: https://bugs.python.org/issue32892, https://bugs.python.org/issue36917

It seems it shouldn't be to hard to fix this for 0.25.

@stonebig can you try if the following patch works?

--- a/pandas/core/computation/expr.py
+++ b/pandas/core/computation/expr.py
@@ -590,6 +590,16 @@ class BaseExprVisitor(ast.NodeVisitor):
         name = self.env.add_tmp([self.visit(e)(self.env) for e in node.elts])
         return self.term_type(name, self.env)
 
+    def visit_Constant(self, node, **kwargs):
+        if isinstance(node.value, str):
+            self.visit_Str(node)
+        elif node.value in {True, False}:
+            self.visit_NameConstant(node)
+        elif isinstance(node.value, (int, float)):
+            self.visit_Num(node)
+        else:
+            raise Exception("...")
+
     visit_Tuple = visit_List
@stonebig

This comment has been minimized.

Copy link
Contributor Author

@stonebig stonebig commented Jul 6, 2019

If I did apply it right, I get now a :

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-2-99840eba54d9> in <module>
----> 1 datas.query('Measure > 0').groupby(['Color','Year']).size().unstack()

C:\WinP\bd38\bu\WPy64-3800b2\python-3.8.0b2.amd64\lib\site-packages\pandas\core\frame.py in query(self, expr, inplace, **kwargs)
   3190         kwargs["level"] = kwargs.pop("level", 0) + 1
   3191         kwargs["target"] = None
-> 3192         res = self.eval(expr, **kwargs)
   3193 
   3194         try:

C:\WinP\bd38\bu\WPy64-3800b2\python-3.8.0b2.amd64\lib\site-packages\pandas\core\frame.py in eval(self, expr, inplace, **kwargs)
   3306             kwargs["target"] = self
   3307         kwargs["resolvers"] = kwargs.get("resolvers", ()) + tuple(resolvers)
-> 3308         return _eval(expr, inplace=inplace, **kwargs)
   3309 
   3310     def select_dtypes(self, include=None, exclude=None):

C:\WinP\bd38\bu\WPy64-3800b2\python-3.8.0b2.amd64\lib\site-packages\pandas\core\computation\eval.py in eval(expr, parser, engine, truediv, local_dict, global_dict, resolvers, level, target, inplace)
    320         )
    321 
--> 322         parsed_expr = Expr(expr, engine=engine, parser=parser, env=env, truediv=truediv)
    323 
    324         # construct the engine and evaluate the parsed expression

C:\WinP\bd38\bu\WPy64-3800b2\python-3.8.0b2.amd64\lib\site-packages\pandas\core\computation\expr.py in __init__(self, expr, engine, parser, env, truediv, level)
    835         self.env.scope["truediv"] = truediv
    836         self._visitor = _parsers[parser](self.env, self.engine, self.parser)
--> 837         self.terms = self.parse()
    838 
    839     @property

C:\WinP\bd38\bu\WPy64-3800b2\python-3.8.0b2.amd64\lib\site-packages\pandas\core\computation\expr.py in parse(self)
    852     def parse(self):
    853         """Parse an expression"""
--> 854         return self._visitor.visit(self.expr)
    855 
    856     @property

C:\WinP\bd38\bu\WPy64-3800b2\python-3.8.0b2.amd64\lib\site-packages\pandas\core\computation\expr.py in visit(self, node, **kwargs)
    439         method = "visit_" + node.__class__.__name__
    440         visitor = getattr(self, method)
--> 441         return visitor(node, **kwargs)
    442 
    443     def visit_Module(self, node, **kwargs):

C:\WinP\bd38\bu\WPy64-3800b2\python-3.8.0b2.amd64\lib\site-packages\pandas\core\computation\expr.py in visit_Module(self, node, **kwargs)
    445             raise SyntaxError("only a single expression is allowed")
    446         expr = node.body[0]
--> 447         return self.visit(expr, **kwargs)
    448 
    449     def visit_Expr(self, node, **kwargs):

C:\WinP\bd38\bu\WPy64-3800b2\python-3.8.0b2.amd64\lib\site-packages\pandas\core\computation\expr.py in visit(self, node, **kwargs)
    439         method = "visit_" + node.__class__.__name__
    440         visitor = getattr(self, method)
--> 441         return visitor(node, **kwargs)
    442 
    443     def visit_Module(self, node, **kwargs):

C:\WinP\bd38\bu\WPy64-3800b2\python-3.8.0b2.amd64\lib\site-packages\pandas\core\computation\expr.py in visit_Expr(self, node, **kwargs)
    448 
    449     def visit_Expr(self, node, **kwargs):
--> 450         return self.visit(node.value, **kwargs)
    451 
    452     def _rewrite_membership_op(self, node, left, right):

C:\WinP\bd38\bu\WPy64-3800b2\python-3.8.0b2.amd64\lib\site-packages\pandas\core\computation\expr.py in visit(self, node, **kwargs)
    439         method = "visit_" + node.__class__.__name__
    440         visitor = getattr(self, method)
--> 441         return visitor(node, **kwargs)
    442 
    443     def visit_Module(self, node, **kwargs):

C:\WinP\bd38\bu\WPy64-3800b2\python-3.8.0b2.amd64\lib\site-packages\pandas\core\computation\expr.py in visit_Compare(self, node, **kwargs)
    752             op = self.translate_In(ops[0])
    753             binop = ast.BinOp(op=op, left=node.left, right=comps[0])
--> 754             return self.visit(binop)
    755 
    756         # recursive case: we have a chained comparison, a CMP b CMP c, etc.

C:\WinP\bd38\bu\WPy64-3800b2\python-3.8.0b2.amd64\lib\site-packages\pandas\core\computation\expr.py in visit(self, node, **kwargs)
    439         method = "visit_" + node.__class__.__name__
    440         visitor = getattr(self, method)
--> 441         return visitor(node, **kwargs)
    442 
    443     def visit_Module(self, node, **kwargs):

C:\WinP\bd38\bu\WPy64-3800b2\python-3.8.0b2.amd64\lib\site-packages\pandas\core\computation\expr.py in visit_BinOp(self, node, **kwargs)
    562     def visit_BinOp(self, node, **kwargs):
    563         op, op_class, left, right = self._maybe_transform_eq_ne(node)
--> 564         left, right = self._maybe_downcast_constants(left, right)
    565         return self._maybe_evaluate_binop(op, op_class, left, right)
    566 

C:\WinP\bd38\bu\WPy64-3800b2\python-3.8.0b2.amd64\lib\site-packages\pandas\core\computation\expr.py in _maybe_downcast_constants(self, left, right)
    498             left = self.term_type(name, self.env)
    499         if (
--> 500             right.is_scalar
    501             and hasattr(right, "value")
    502             and not left.is_scalar

AttributeError: 'NoneType' object has no attribute 'is_scalar'
@jreback jreback modified the milestones: 0.25.0, 1.0 Jul 17, 2019
@stonebig stonebig mentioned this issue Jul 21, 2019
6 of 7 tasks complete
@stonebig

This comment has been minimized.

Copy link
Contributor Author

@stonebig stonebig commented Jul 21, 2019

pandas-1.0 is really near ?

@stonebig

This comment has been minimized.

Copy link
Contributor Author

@stonebig stonebig commented Aug 25, 2019

I confirm the proposed 2 lines patch fixes the issue.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Projects
None yet
4 participants
You can’t perform that action at this time.