Fixes to column number reporting

mypy/checker.py

@@ -3130,8 +3130,10 @@ def visit_del_stmt(self, s: DelStmt) -> None:
             e = s.expr
             m = MemberExpr(e.base, '__delitem__')
             m.line = s.line
+            m.column = s.column
             c = CallExpr(m, [e.index], [nodes.ARG_POS], [None])
             c.line = s.line
+            c.column = s.column
             self.expr_checker.accept(c, allow_none_return=True)
         else:
             s.expr.accept(self.expr_checker)

mypy/fastparse.py

@@ -184,6 +184,7 @@ def parse(source: Union[str, bytes],
 
 def parse_type_comment(type_comment: str,
                        line: int,
+                       column: int,
                        errors: Optional[Errors],
                        assume_str_is_unicode: bool = True,
                        ) -> Tuple[bool, Optional[Type]]:
@@ -200,8 +201,11 @@ def parse_type_comment(type_comment: str,
     else:
         extra_ignore = TYPE_IGNORE_PATTERN.match(type_comment) is not None
         assert isinstance(typ, ast3_Expression)
-        return extra_ignore, TypeConverter(errors, line=line,
-                             assume_str_is_unicode=assume_str_is_unicode).visit(typ.body)
+        converted = TypeConverter(errors,
+                                  line=line,
+                                  override_column=column,
+                                  assume_str_is_unicode=assume_str_is_unicode).visit(typ.body)
+        return extra_ignore, converted
 
 
 def parse_type_string(expr_string: str, expr_fallback_name: str,
@@ -221,7 +225,7 @@ def parse_type_string(expr_string: str, expr_fallback_name: str,
     code with unicode_literals...) and setting `assume_str_is_unicode` accordingly.
     """
     try:
-        _, node = parse_type_comment(expr_string.strip(), line=line, errors=None,
+        _, node = parse_type_comment(expr_string.strip(), line=line, column=column, errors=None,
                                      assume_str_is_unicode=assume_str_is_unicode)
         if isinstance(node, UnboundType) and node.original_str_expr is None:
             node.original_str_expr = expr_string
@@ -480,7 +484,9 @@ def do_func_def(self, n: Union[ast3.FunctionDef, ast3.AsyncFunctionDef],
                     # PEP 484 disallows both type annotations and type comments
                     if n.returns or any(a.type_annotation is not None for a in args):
                         self.fail(message_registry.DUPLICATE_TYPE_SIGNATURES, lineno, n.col_offset)
-                    translated_args = (TypeConverter(self.errors, line=lineno)
+                    translated_args = (TypeConverter(self.errors,
+                                                     line=lineno,
+                                                     override_column=n.col_offset)
                                        .translate_expr_list(func_type_ast.argtypes))
                     arg_types = [a if a is not None else AnyType(TypeOfAny.unannotated)
                                 for a in translated_args]
@@ -511,11 +517,13 @@ def do_func_def(self, n: Union[ast3.FunctionDef, ast3.AsyncFunctionDef],
         if any(arg_types) or return_type:
             if len(arg_types) != 1 and any(isinstance(t, EllipsisType) for t in arg_types):
                 self.fail("Ellipses cannot accompany other argument types "
-                          "in function type signature.", lineno, 0)
+                          "in function type signature", lineno, n.col_offset)
             elif len(arg_types) > len(arg_kinds):
-                self.fail('Type signature has too many arguments', lineno, 0, blocker=False)
+                self.fail('Type signature has too many arguments', lineno, n.col_offset,
+                          blocker=False)
             elif len(arg_types) < len(arg_kinds):
-                self.fail('Type signature has too few arguments', lineno, 0, blocker=False)
+                self.fail('Type signature has too few arguments', lineno, n.col_offset,
+                          blocker=False)
             else:
                 func_type = CallableType([a if a is not None else
                                           AnyType(TypeOfAny.unannotated) for a in arg_types],
@@ -558,7 +566,8 @@ def do_func_def(self, n: Union[ast3.FunctionDef, ast3.AsyncFunctionDef],
             func_def.body.set_line(lineno)  # TODO: Why?
 
             deco = Decorator(func_def, self.translate_expr_list(n.decorator_list), var)
-            deco.set_line(n.decorator_list[0].lineno)
+            first = n.decorator_list[0]
+            deco.set_line(first.lineno, first.col_offset)
             retval = deco  # type: Union[FuncDef, Decorator]
         else:
             # FuncDef overrides set_line -- can't use self.set_line
@@ -631,7 +640,8 @@ def make_argument(self, arg: ast3.arg, default: Optional[ast3.expr], kind: int,
             if annotation is not None:
                 arg_type = TypeConverter(self.errors, line=arg.lineno).visit(annotation)
             elif type_comment is not None:
-                extra_ignore, arg_type = parse_type_comment(type_comment, arg.lineno, self.errors)
+                extra_ignore, arg_type = parse_type_comment(type_comment, arg.lineno,
+                                                            arg.col_offset, self.errors)
                 if extra_ignore:
                     self.type_ignores.add(arg.lineno)
 
@@ -689,7 +699,8 @@ def visit_Assign(self, n: ast3.Assign) -> AssignmentStmt:
         lvalues = self.translate_expr_list(n.targets)
         rvalue = self.visit(n.value)
         if n.type_comment is not None:
-            extra_ignore, typ = parse_type_comment(n.type_comment, n.lineno, self.errors)
+            extra_ignore, typ = parse_type_comment(n.type_comment, n.lineno, n.col_offset,
+                                                   self.errors)
             if extra_ignore:
                 self.type_ignores.add(n.lineno)
         else:
@@ -723,7 +734,8 @@ def visit_NamedExpr(self, n: NamedExpr) -> None:
     # For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)
     def visit_For(self, n: ast3.For) -> ForStmt:
         if n.type_comment is not None:
-            extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno, self.errors)
+            extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno, n.col_offset,
+                                                           self.errors)
             if extra_ignore:
                 self.type_ignores.add(n.lineno)
         else:
@@ -738,7 +750,8 @@ def visit_For(self, n: ast3.For) -> ForStmt:
     # AsyncFor(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)
     def visit_AsyncFor(self, n: ast3.AsyncFor) -> ForStmt:
         if n.type_comment is not None:
-            extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno, self.errors)
+            extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno, n.col_offset,
+                                                           self.errors)
             if extra_ignore:
                 self.type_ignores.add(n.lineno)
         else:
@@ -769,7 +782,8 @@ def visit_If(self, n: ast3.If) -> IfStmt:
     # With(withitem* items, stmt* body, string? type_comment)
     def visit_With(self, n: ast3.With) -> WithStmt:
         if n.type_comment is not None:
-            extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno, self.errors)
+            extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno,
+                                                           n.col_offset, self.errors)
             if extra_ignore:
                 self.type_ignores.add(n.lineno)
         else:
@@ -783,7 +797,8 @@ def visit_With(self, n: ast3.With) -> WithStmt:
     # AsyncWith(withitem* items, stmt* body, string? type_comment)
     def visit_AsyncWith(self, n: ast3.AsyncWith) -> WithStmt:
         if n.type_comment is not None:
-            extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno, self.errors)
+            extra_ignore, target_type = parse_type_comment(n.type_comment, n.lineno,
+                                                           n.col_offset, self.errors)
             if extra_ignore:
                 self.type_ignores.add(n.lineno)
         else:
@@ -1207,13 +1222,26 @@ class TypeConverter:
     def __init__(self,
                  errors: Optional[Errors],
                  line: int = -1,
+                 override_column: int = -1,
                  assume_str_is_unicode: bool = True,
                  ) -> None:
         self.errors = errors
         self.line = line
+        self.override_column = override_column
         self.node_stack = []  # type: List[AST]
         self.assume_str_is_unicode = assume_str_is_unicode
 
+    def convert_column(self, column: int) -> int:
+        """Apply column override if defined; otherwise return column.
+
+        Column numbers are sometimes incorrect in the AST and the column
+        override can be used to work around that.
+        """
+        if self.override_column < 0:
+            return column
+        else:
+            return self.override_column
+
     def invalid_type(self, node: AST, note: Optional[str] = None) -> RawExpressionType:
         """Constructs a type representing some expression that normally forms an invalid type.
         For example, if we see a type hint that says "3 + 4", we would transform that
@@ -1276,6 +1304,7 @@ def visit_raw_str(self, s: str) -> Type:
         # without needing to create an intermediary `Str` object.
         _, typ = parse_type_comment(s.strip(),
                                     self.line,
+                                    -1,
                                     self.errors,
                                     self.assume_str_is_unicode)
         return typ or AnyType(TypeOfAny.from_error)
@@ -1339,13 +1368,13 @@ def _extract_argument_name(self, n: ast3.expr) -> Optional[str]:
         return None
 
     def visit_Name(self, n: Name) -> Type:
-        return UnboundType(n.id, line=self.line)
+        return UnboundType(n.id, line=self.line, column=self.convert_column(n.col_offset))
 
     def visit_NameConstant(self, n: NameConstant) -> Type:
         if isinstance(n.value, bool):
             return RawExpressionType(n.value, 'builtins.bool', line=self.line)
         else:
-            return UnboundType(str(n.value), line=self.line)
+            return UnboundType(str(n.value), line=self.line, column=n.col_offset)
 
     # Only for 3.8 and newer
     def visit_Constant(self, n: Constant) -> Type:
@@ -1455,14 +1484,14 @@ def visit_Subscript(self, n: ast3.Subscript) -> Type:
 
         value = self.visit(n.value)
         if isinstance(value, UnboundType) and not value.args:
-            return UnboundType(value.name, params, line=self.line,
+            return UnboundType(value.name, params, line=self.line, column=value.column,
                                empty_tuple_index=empty_tuple_index)
         else:
             return self.invalid_type(n)
 
     def visit_Tuple(self, n: ast3.Tuple) -> Type:
         return TupleType(self.translate_expr_list(n.elts), _dummy_fallback,
-                         implicit=True, line=self.line)
+                         implicit=True, line=self.line, column=self.convert_column(n.col_offset))
 
     # Attribute(expr value, identifier attr, expr_context ctx)
     def visit_Attribute(self, n: Attribute) -> Type:

mypy/fastparse2.py

@@ -335,7 +335,7 @@ def visit_Module(self, mod: ast27.Module) -> MypyFile:
     def visit_FunctionDef(self, n: ast27.FunctionDef) -> Statement:
         self.class_and_function_stack.append('F')
         lineno = n.lineno
-        converter = TypeConverter(self.errors, line=lineno,
+        converter = TypeConverter(self.errors, line=lineno, override_column=n.col_offset,
                                   assume_str_is_unicode=self.unicode_literals)
         args, decompose_stmts = self.transform_args(n.args, lineno)
 
@@ -389,11 +389,13 @@ def visit_FunctionDef(self, n: ast27.FunctionDef) -> Statement:
         if any(arg_types) or return_type:
             if len(arg_types) != 1 and any(isinstance(t, EllipsisType) for t in arg_types):
                 self.fail("Ellipses cannot accompany other argument types "
-                          "in function type signature.", lineno, 0)
+                          "in function type signature", lineno, n.col_offset)
             elif len(arg_types) > len(arg_kinds):
-                self.fail('Type signature has too many arguments', lineno, 0, blocker=False)
+                self.fail('Type signature has too many arguments', lineno, n.col_offset,
+                          blocker=False)
             elif len(arg_types) < len(arg_kinds):
-                self.fail('Type signature has too few arguments', lineno, 0, blocker=False)
+                self.fail('Type signature has too few arguments', lineno, n.col_offset,
+                          blocker=False)
             else:
                 any_type = AnyType(TypeOfAny.unannotated)
                 func_type = CallableType([a if a is not None else any_type for a in arg_types],
@@ -569,7 +571,10 @@ def visit_Delete(self, n: ast27.Delete) -> DelStmt:
     def visit_Assign(self, n: ast27.Assign) -> AssignmentStmt:
         typ = None
         if n.type_comment:
-            extra_ignore, typ = parse_type_comment(n.type_comment, n.lineno, self.errors,
+            extra_ignore, typ = parse_type_comment(n.type_comment,
+                                                   n.lineno,
+                                                   n.col_offset,
+                                                   self.errors,
                                                    assume_str_is_unicode=self.unicode_literals)
             if extra_ignore:
                 self.type_ignores.add(n.lineno)
@@ -589,7 +594,10 @@ def visit_AugAssign(self, n: ast27.AugAssign) -> OperatorAssignmentStmt:
     # For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)
     def visit_For(self, n: ast27.For) -> ForStmt:
         if n.type_comment is not None:
-            extra_ignore, typ = parse_type_comment(n.type_comment, n.lineno, self.errors,
+            extra_ignore, typ = parse_type_comment(n.type_comment,
+                                                   n.lineno,
+                                                   n.col_offset,
+                                                   self.errors,
                                                    assume_str_is_unicode=self.unicode_literals)
             if extra_ignore:
                 self.type_ignores.add(n.lineno)
@@ -619,7 +627,10 @@ def visit_If(self, n: ast27.If) -> IfStmt:
     # With(withitem* items, stmt* body, string? type_comment)
     def visit_With(self, n: ast27.With) -> WithStmt:
         if n.type_comment is not None:
-            extra_ignore, typ = parse_type_comment(n.type_comment, n.lineno, self.errors,
+            extra_ignore, typ = parse_type_comment(n.type_comment,
+                                                   n.lineno,
+                                                   n.col_offset,
+                                                   self.errors,
                                                    assume_str_is_unicode=self.unicode_literals)
             if extra_ignore:
                 self.type_ignores.add(n.lineno)

mypy/newsemanal/semanal.py

@@ -3309,6 +3309,7 @@ def visit_call_expr(self, expr: CallExpr) -> None:
             # precedence over the CallExpr semantics.
             expr.analyzed = CastExpr(expr.args[1], target)
             expr.analyzed.line = expr.line
+            expr.analyzed.column = expr.column
             expr.analyzed.accept(self)
         elif refers_to_fullname(expr.callee, 'builtins.reveal_type'):
             if not self.check_fixed_args(expr, 1, 'reveal_type'):
@@ -3516,6 +3517,7 @@ def analyze_type_application(self, expr: IndexExpr) -> None:
         base = expr.base
         expr.analyzed = TypeApplication(base, types)
         expr.analyzed.line = expr.line
+        expr.analyzed.column = expr.column
         # Types list, dict, set are not subscriptable, prohibit this if
         # subscripted either via type alias...
         if isinstance(base, RefExpr) and isinstance(base.node, TypeAlias):

mypy/newsemanal/semanal_newtype.py

@@ -55,7 +55,7 @@ def process_newtype_declaration(self, s: AssignmentStmt) -> bool:
 
         old_type, should_defer = self.check_newtype_args(name, call, s)
         if not call.analyzed:
-            call.analyzed = NewTypeExpr(name, old_type, line=call.line)
+            call.analyzed = NewTypeExpr(name, old_type, line=call.line, column=call.column)
         if old_type is None:
             if should_defer:
                 # Base type is not ready.

mypy/newsemanal/typeanal.py

@@ -92,7 +92,7 @@ def no_subscript_builtin_alias(name: str, propose_alt: bool = True) -> str:
 
 
 class TypeAnalyser(SyntheticTypeVisitor[Type], TypeAnalyzerPluginInterface):
-    """Semantic analyzer for types (semantic analysis pass 2).
+    """Semantic analyzer for types.
 
     Converts unbound types into bound types.
     """

mypy/nodes.py

@@ -491,7 +491,7 @@ def __init__(self, items: List['OverloadPart']) -> None:
         self.unanalyzed_items = items.copy()
         self.impl = None
         if len(items) > 0:
-            self.set_line(items[0].line)
+            self.set_line(items[0].line, items[0].column)
         self.is_final = False
 
     def name(self) -> str:
@@ -2168,11 +2168,13 @@ class NewTypeExpr(Expression):
     # The synthesized class representing the new type (inherits old_type)
     info = None  # type: Optional[TypeInfo]
 
-    def __init__(self, name: str, old_type: 'Optional[mypy.types.Type]', line: int) -> None:
+    def __init__(self, name: str, old_type: 'Optional[mypy.types.Type]', line: int,
+                 column: int) -> None:
         super().__init__()
         self.name = name
         self.old_type = old_type
         self.line = line
+        self.column = column
 
     def accept(self, visitor: ExpressionVisitor[T]) -> T:
         return visitor.visit_newtype_expr(self)

mypy/semanal.py

@@ -2448,6 +2448,7 @@ def process_typevar_declaration(self, s: AssignmentStmt) -> None:
         node.kind = self.current_symbol_kind()
         type_var = TypeVarExpr(name, node.fullname, values, upper_bound, variance)
         type_var.line = call.line
+        type_var.column = call.column
         call.analyzed = type_var
         node.node = type_var
 

mypy/semanal_newtype.py

@@ -34,7 +34,7 @@ def process_newtype_declaration(self, s: AssignmentStmt) -> None:
             return
 
         old_type = self.check_newtype_args(name, call, s)
-        call.analyzed = NewTypeExpr(name, old_type, line=call.line)
+        call.analyzed = NewTypeExpr(name, old_type, line=call.line, column=call.column)
         if old_type is None:
             return
 

mypy/treetransform.py

@@ -493,7 +493,7 @@ def visit_type_alias_expr(self, node: TypeAliasExpr) -> TypeAliasExpr:
         return TypeAliasExpr(node.type, node.tvars, node.no_args)
 
     def visit_newtype_expr(self, node: NewTypeExpr) -> NewTypeExpr:
-        res = NewTypeExpr(node.name, node.old_type, line=node.line)
+        res = NewTypeExpr(node.name, node.old_type, line=node.line, column=node.column)
         res.info = node.info
         return res
 

test-data/unit/check-classes.test

@@ -5171,17 +5171,18 @@ B = G
 [out]
 
 [case testForwardInstanceWithBound]
+# flags: --show-column-numbers
 from typing import TypeVar, Generic
 
 T = TypeVar('T', bound=str)
 class G(Generic[T]): ...
 
 A = G
-x: A[B[int]]
+x: A[B[int]] # E
 B = G
 [out]
-main:7: error: Type argument "builtins.int" of "G" must be a subtype of "builtins.str"
-main:7: error: Type argument "__main__.G[builtins.int]" of "G" must be a subtype of "builtins.str"
+main:8:4: error: Type argument "__main__.G[builtins.int]" of "G" must be a subtype of "builtins.str"
+main:8:6: error: Type argument "builtins.int" of "G" must be a subtype of "builtins.str"
 
 [case testExtremeForwardReferencing]
 from typing import TypeVar, Generic