New semantic analyzer: support named tuples

mypy/newsemanal/semanal.py

@@ -870,6 +870,9 @@ def analyze_class(self, defn: ClassDef) -> None:
                 self.prepare_class_def(defn, info)
             return
 
+        if self.analyze_namedtuple_classdef(defn):
+            return
+
         # Create TypeInfo for class now that base classes and the MRO can be calculated.
         self.prepare_class_def(defn)
 
@@ -880,10 +883,6 @@ def analyze_class(self, defn: ClassDef) -> None:
 
         with self.scope.class_scope(defn.info):
             self.configure_base_classes(defn, base_types)
-
-            if self.analyze_namedtuple_classdef(defn):
-                return
-
             defn.info.is_protocol = is_protocol
             self.analyze_metaclass(defn)
             defn.info.runtime_protocol = False
@@ -904,40 +903,23 @@ def analyze_class_body_common(self, defn: ClassDef) -> None:
         self.leave_class()
 
     def analyze_namedtuple_classdef(self, defn: ClassDef) -> bool:
-        """Analyze class-based named tuple if the NamedTuple base class is present.
-
-        TODO: Move this to NamedTupleAnalyzer?
-
-        Return True only if the class is a NamedTuple class.
-        """
-        named_tuple_info = self.named_tuple_analyzer.analyze_namedtuple_classdef(defn)
-        if named_tuple_info is None:
-            return False
-        # Temporarily clear the names dict so we don't get errors about duplicate names
-        # that were already set in build_namedtuple_typeinfo.
-        nt_names = named_tuple_info.names
-        named_tuple_info.names = SymbolTable()
-
-        self.analyze_class_body_common(defn)
-
-        # Make sure we didn't use illegal names, then reset the names in the typeinfo.
-        for prohibited in NAMEDTUPLE_PROHIBITED_NAMES:
-            if prohibited in named_tuple_info.names:
-                if nt_names.get(prohibited) is named_tuple_info.names[prohibited]:
-                    continue
-                ctx = named_tuple_info.names[prohibited].node
-                assert ctx is not None
-                self.fail('Cannot overwrite NamedTuple attribute "{}"'.format(prohibited),
-                          ctx)
-
-        # Restore the names in the original symbol table. This ensures that the symbol
-        # table contains the field objects created by build_namedtuple_typeinfo. Exclude
-        # __doc__, which can legally be overwritten by the class.
-        named_tuple_info.names.update({
-            key: value for key, value in nt_names.items()
-            if key not in named_tuple_info.names or key != '__doc__'
-        })
-        return True
+        """Check if this class can define a named tuple."""
+        if defn.info and defn.info.tuple_type:
+            # Don't reprocess everything. We just need to process methods defined
+            # in the named tuple class body.
+            is_named_tuple, info = True, defn.info  # type: bool, Optional[TypeInfo]
+        else:
+            is_named_tuple, info = self.named_tuple_analyzer.analyze_namedtuple_classdef(defn)
+        if is_named_tuple:
+            if info is None:
+                self.mark_incomplete(defn.name, defn)
+            else:
+                self.prepare_class_def(defn, info)
+                with self.scope.class_scope(defn.info):
+                    with self.named_tuple_analyzer.save_namedtuple_body(info):
+                        self.analyze_class_body_common(defn)
+            return True
+        return False
 
     def apply_class_plugin_hooks(self, defn: ClassDef) -> None:
         """Apply a plugin hook that may infer a more precise definition for a class."""
@@ -1229,10 +1211,15 @@ def prepare_class_def(self, defn: ClassDef, info: Optional[TypeInfo] = None) ->
             # in globals under mangled unique names
             #
             # TODO: Putting local classes into globals breaks assumptions in fine-grained
-            #       incremental mode and we should avoid it.
+            #       incremental mode and we should avoid it. In general, this logic is too
+            #       ad-hoc and needs to be removed/refactored.
             if '@' not in defn.info._fullname:
                 local_name = defn.info._fullname + '@' + str(defn.line)
-                defn.info._fullname = self.cur_mod_id + '.' + local_name
+                if defn.info.is_named_tuple:
+                    # Module is already correctly set in _fullname for named tuples.
+                    defn.info._fullname += '@' + str(defn.line)
+                else:
+                    defn.info._fullname = self.cur_mod_id + '.' + local_name
             else:
                 # Preserve name from previous fine-grained incremental run.
                 local_name = defn.info._fullname
@@ -1298,7 +1285,9 @@ def configure_base_classes(self,
 
         for base, base_expr in bases:
             if isinstance(base, TupleType):
-                if info.tuple_type:
+                # There may be an existing valid tuple type from previous semanal iterations.
+                # Use equality to check if it is the case.
+                if info.tuple_type and info.tuple_type != base:
                     self.fail("Class has two incompatible bases derived from tuple", defn)
                     defn.has_incompatible_baseclass = True
                 info.tuple_type = base
@@ -1426,11 +1415,15 @@ def expr_to_analyzed_type(self,
                               allow_placeholder: bool = False) -> Optional[Type]:
         if isinstance(expr, CallExpr):
             expr.accept(self)
-            info = self.named_tuple_analyzer.check_namedtuple(expr, None, self.is_func_scope())
-            if info is None:
+            is_named_tuple, info = self.named_tuple_analyzer.check_namedtuple(expr, None,
+                                                                              self.is_func_scope())
+            if not is_named_tuple:
                 # Some form of namedtuple is the only valid type that looks like a call
                 # expression. This isn't a valid type.
                 raise TypeTranslationError()
+            elif not info:
+                self.defer()
+                return None
             assert info.tuple_type, "NamedTuple without tuple type"
             fallback = Instance(info, [])
             return TupleType(info.tuple_type.items, fallback=fallback)
@@ -1888,6 +1881,8 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None:
             for expr in names_modified_by_assignment(s):
                 self.mark_incomplete(expr.name, expr)
             return
+        if self.analyze_namedtuple_assign(s):
+            return
         self.analyze_lvalues(s)
         self.check_final_implicit_def(s)
         self.check_classvar(s)
@@ -1896,14 +1891,30 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None:
         self.check_and_set_up_type_alias(s)
         self.newtype_analyzer.process_newtype_declaration(s)
         self.process_typevar_declaration(s)
-        self.named_tuple_analyzer.process_namedtuple_definition(s, self.is_func_scope())
         self.typed_dict_analyzer.process_typeddict_definition(s, self.is_func_scope())
         self.enum_call_analyzer.process_enum_call(s, self.is_func_scope())
         self.store_final_status(s)
         if not s.type:
             self.process_module_assignment(s.lvalues, s.rvalue, s)
         self.process__all__(s)
 
+    def analyze_namedtuple_assign(self, s: AssignmentStmt) -> bool:
+        """Check if s defines a namedtuple."""
+        if isinstance(s.rvalue, CallExpr) and isinstance(s.rvalue.analyzed, NamedTupleExpr):
+            return True  # This is a valid and analyzed named tuple definition, nothing to do here.
+        if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr):
+            return False
+        lvalue = s.lvalues[0]
+        name = lvalue.name
+        is_named_tuple, info = self.named_tuple_analyzer.check_namedtuple(s.rvalue, name,
+                                                                          self.is_func_scope())
+        if not is_named_tuple:
+            return False
+        # Yes, it's a valid namedtuple, but defer if it is not ready.
+        if not info:
+            self.mark_incomplete(name, lvalue, becomes_typeinfo=True)
+        return True
+
     def analyze_lvalues(self, s: AssignmentStmt) -> None:
         for lval in s.lvalues:
             self.analyze_lvalue(lval,
@@ -3674,21 +3685,24 @@ def record_incomplete_ref(self) -> None:
         self.defer()
         self.num_incomplete_refs += 1
 
-    def mark_incomplete(self, name: str, node: Node) -> None:
+    def mark_incomplete(self, name: str, node: Node,
+                        becomes_typeinfo: bool = False) -> None:
         """Mark a definition as incomplete (and defer current analysis target).
 
         Also potentially mark the current namespace as incomplete.
 
         Args:
             name: The name that we weren't able to define (or '*' if the name is unknown)
             node: The node that refers to the name (definition or lvalue)
+            becomes_typeinfo: Pass this to PlaceholderNode (used by special forms like
+                named tuples that will create TypeInfos).
         """
         self.defer()
         if name == '*':
             self.incomplete = True
         elif name not in self.current_symbol_table() and not self.is_global_or_nonlocal(name):
             fullname = self.qualified_name(name)
-            self.add_symbol(name, PlaceholderNode(fullname, node, False), context=None)
+            self.add_symbol(name, PlaceholderNode(fullname, node, becomes_typeinfo), context=None)
         self.missing_names.add(name)
 
     def is_incomplete_namespace(self, fullname: str) -> bool:

mypy/newsemanal/semanal_main.py

@@ -123,7 +123,7 @@ def process_top_level_function(analyzer: 'NewSemanticAnalyzer',
     analyzer.incomplete_namespaces.add(module)
     while deferred and more_iterations:
         iteration += 1
-        if not incomplete or iteration == MAX_ITERATIONS:
+        if not (deferred or incomplete) or iteration == MAX_ITERATIONS:
             # OK, this is one last pass, now missing names will be reported.
             more_iterations = False
             analyzer.incomplete_namespaces.discard(module)