Fix asttokens comma misannotation bug; bump v.0.2.4

lmmx · Jan 2, 2020 · f77f760 · f77f760
1 parent 6e5db6e
commit f77f760
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 1 deletion.
diff --git a/README.md b/README.md
@@ -122,6 +122,9 @@ the process.
 
 # Changelog
 
+- version 0.2.4:
+  - fix bug relating to asttokens misannotating comma tokens' type as 54 (`ERRORTOKEN`) rather than 53
+  (`OP`), by just checking for comma tokens of type 54 matching the string `','`, will inform asttokens devs
 - version 0.2.2:
   - fix bug where names assigned outside of definitions were causing errors
 - version 0.2.1:

diff --git a/mvdef/src/import_util.py b/mvdef/src/import_util.py
@@ -94,6 +94,10 @@ def multilinify_import_stmt_str(import_stmt_str, indent_spaces=4, trailing_comma
                 al_endpos = al_n_tok.endpos
             else:
                 comma_tok = tko.find_token(al_n_tok, tok_type=53, tok_str=",")
+                if comma_tok.type == 0:
+                    # Due to an error in asttokens, sometimes tok_type is given as 54
+                    # although this should be an error (the failure tok_type is 0)
+                    comma_tok = tko.find_token(al_n_tok, tok_type=54, tok_str=",")
                 assert comma_tok.type > 0, f"Unable to find comma token"
                 al_endpos = comma_tok.endpos
         else:
@@ -104,6 +108,10 @@ def multilinify_import_stmt_str(import_stmt_str, indent_spaces=4, trailing_comma
                 al_endpos = al_as_tok.endpos
             else:
                 comma_tok = tko.find_token(al_as_tok, tok_type=53, tok_str=",")
+                if comma_tok.type == 0:
+                    # Due to an error in asttokens, sometimes tok_type is given as 54
+                    # although this should be an error (the failure tok_type is 0)
+                    comma_tok = tko.find_token(al_n_tok, tok_type=54, tok_str=",")
                 assert comma_tok.type > 0, f"Unable to find comma token"
                 al_endpos = comma_tok.endpos
         alias_chunk = import_stmt_str[al_startpos:al_endpos]

diff --git a/setup.py b/setup.py
@@ -5,7 +5,7 @@
 
 setup(
     name="mvdef",
-    version="0.2.3",
+    version="0.2.4",
     author="Louis Maddox",
     author_email="louismmx@gmail.com",
     description="Package to move functions and their import statements between files",