Fix tests for newest astroid

asttokens/mark_tokens.py

@@ -23,13 +23,7 @@
 
 from . import util
 from .asttokens import ASTTokens
-from .util import AstConstant
-
-try:
-  import astroid.node_classes as nc
-except Exception:
-  # This is only used for type checking, we don't need it if astroid isn't installed.
-  nc = None
+from .util import AstConstant, astroid_node_classes as nc
 
 if TYPE_CHECKING:
   from .util import AstNode
@@ -88,7 +82,8 @@ def _visit_after_children(self, node, parent_token, token):
     first = token
     last = None
     for child in cast(Callable, self._iter_children)(node):
-      if not first or child.first_token.index < first.index:
+      # astroid slices have especially wrong positions, we don't want them to corrupt their parents.
+      if not first or child.first_token.index < first.index and not util.is_astroid_slice(child):
         first = child.first_token
       if not last or child.last_token.index > last.index:
         last = child.last_token

asttokens/util.py

@@ -24,8 +24,18 @@
 
 from six import iteritems
 
+try:
+  from astroid import nodes as astroid_node_classes
+  getattr(astroid_node_classes, "NodeNG")
+except Exception:
+  try:
+    from astroid import node_classes as astroid_node_classes
+  except Exception:
+    astroid_node_classes = None
+
+
 if TYPE_CHECKING:  # pragma: no cover
-  from astroid.node_classes import NodeNG
+  NodeNG = astroid_node_classes.NodeNG
 
   # Type class used to expand out the definition of AST to include fields added by this library
   # It's not actually used for anything other than type checking though!
@@ -218,6 +228,11 @@ def is_slice(node):
   )
 
 
+def is_astroid_slice(node):
+  # type: (AstNode) -> bool
+  return is_slice(node) and not isinstance(node, ast.AST)
+
+
 # Sentinel value used by visit_tree().
 _PREVISIT = object()
 

setup.cfg

@@ -39,7 +39,7 @@ install_requires =
 setup_requires = setuptools>=44; setuptools_scm[toml]>=3.4.3
 
 [options.extras_require]
-test = astroid<=2.5.3; pytest
+test = astroid; pytest
 
 [options.package_data]
 asttokens = py.typed

tests/test_astroid.py

@@ -2,9 +2,9 @@
 from __future__ import unicode_literals, print_function
 
 import astroid
-from astroid.node_classes import NodeNG
 
 from asttokens import ASTTokens
+from asttokens.util import astroid_node_classes
 from . import test_mark_tokens
 
 
@@ -13,7 +13,7 @@ class TestAstroid(test_mark_tokens.TestMarkTokens):
   is_astroid_test = True
   module = astroid
 
-  nodes_classes = NodeNG
+  nodes_classes = astroid_node_classes.NodeNG
   context_classes = [
     (astroid.Name, astroid.DelName, astroid.AssignName),
     (astroid.Attribute, astroid.DelAttr, astroid.AssignAttr),

tests/test_mark_tokens.py

@@ -19,6 +19,11 @@
 
 from . import tools
 
+try:
+  from astroid.nodes.utils import Position as AstroidPosition
+except Exception:
+  AstroidPosition = ()
+
 
 class TestMarkTokens(unittest.TestCase):
   maxDiff = None
@@ -243,7 +248,7 @@ def test_slices(self):
     # important, so we skip them here.
     self.assertEqual({n for n in m.view_nodes_at(1, 56) if 'Slice:' not in n},
                      { "Subscript:foo[:]", "Name:foo" })
-    self.assertEqual({n for n in m.view_nodes_at(1, 64) if 'Slice:' not in n},
+    self.assertEqual({n for n in m.view_nodes_at(1, 64) if 'Slice:' not in n and 'Tuple:' not in n},
                      { "Subscript:bar[::2, :]", "Name:bar" })
 
   def test_adjacent_strings(self):
@@ -814,6 +819,10 @@ def assert_nodes_equal(self, t1, t2):
     else:
       self.assertEqual(type(t1), type(t2))
 
+    if isinstance(t1, AstroidPosition):
+      # Ignore the lineno/col_offset etc. from astroid
+      return
+
     if isinstance(t1, (list, tuple)):
       self.assertEqual(len(t1), len(t2))
       for vc1, vc2 in zip(t1, t2):