Merge pull request #1257 from google/google_sync

Google sync

CHANGELOG

@@ -1,3 +1,14 @@
+Version 2022.07.18:
+
+Bug fixes:
+* Look up methods properly on classes with _HAS_DYNAMIC_ATTRIBUTES.
+* Handle .pyi-1 files in load_pytd.Module.is_package().
+* Adjust opcode line numbers for return statements in python 3.10+.
+* Remove optimize.Factorize, which unnecessarily flattens overloaded functions.
+* Fix coroutine signatures in overriding_checks.
+* Handle generic types correctly in signature compatibility checks.
+* Respect NoReturn annotations even when maximum depth is reached.
+
 Version 2022.06.30:
 
 Updates:

docs/faq.md

@@ -22,7 +22,7 @@
       * [How do I annotate *args and <code>**kwargs</code>?](#how-do-i-annotate-args-and-kwargs)
       * [Why are signature mismatches in subclasses bad? {#signature-mismatch}](#why-are-signature-mismatches-in-subclasses-bad-signature-mismatch)
 
-<!-- Added by: rechen, at: 2022-06-22T23:51-07:00 -->
+<!-- Added by: mdemello, at: 2022-07-13T17:03-07:00 -->
 
 <!--te-->
 
@@ -371,7 +371,6 @@ TypeError: func() got multiple values for argument 'y'
 [pep-561-issue]: https://github.com/google/pytype/issues/151
 [typeshed]: https://github.com/python/typeshed
 [typing-faq]: typing_faq.md
-[why-is-pytype-taking-so-long]: #why-is-pytype-taking-so-long
 
 <!-- References with different internal and external versions -->
 

pytype/__version__.py

@@ -1,2 +1,2 @@
 # pylint: skip-file
-__version__ = '2022.06.30'
+__version__ = '2022.07.18'

pytype/abstract/_interpreter_function.py

@@ -642,7 +642,12 @@ def call(self, node, func, args, alias_map=None, new_locals=False,
         not abstract_utils.func_name_is_class_init(self.name)):
       log.info("Maximum depth reached. Not analyzing %r", self.name)
       self._set_callself_maybe_missing_members()
-      return node, self.ctx.new_unsolvable(node)
+      if self.signature.annotations.get("return") == self.ctx.convert.no_return:
+        # TODO(b/147230757): Use all return annotations, not just NoReturn.
+        ret = self.signature.annotations["return"]
+      else:
+        ret = self.ctx.convert.unsolvable
+      return node, ret.to_variable(node)
     args = self._fix_args_for_unannotated_contextmanager_exit(node, func, args)
     args = args.simplify(node, self.ctx, self.signature)
     sig, substs, callargs = self._find_matching_sig(node, args, alias_map)

pytype/annotation_utils.py

@@ -54,6 +54,14 @@ def _get_type_parameter_subst(self, node, annot, substs, instantiate_unbound):
     return self.ctx.convert.merge_classes(vals)
 
   def sub_one_annotation(self, node, annot, substs, instantiate_unbound=True):
+
+    def get_type_parameter_subst(annotation):
+      return self._get_type_parameter_subst(node, annotation, substs,
+                                            instantiate_unbound)
+
+    return self._do_sub_one_annotation(node, annot, get_type_parameter_subst)
+
+  def _do_sub_one_annotation(self, node, annot, get_type_parameter_subst_fn):
     """Apply type parameter substitutions to an annotation."""
     # We push annotations onto 'stack' and move them to the 'done' stack as they
     # are processed. For each annotation, we also track an 'inner_type_keys'
@@ -77,8 +85,8 @@ def sub_one_annotation(self, node, annot, substs, instantiate_unbound=True):
           if cur not in late_annotations:
             param_strings = []
             for t in utils.unique_list(self.get_type_parameters(cur)):
-              s = pytd_utils.Print(self._get_type_parameter_subst(
-                  node, t, substs, instantiate_unbound).get_instance_type(node))
+              s = pytd_utils.Print(
+                  get_type_parameter_subst_fn(t).get_instance_type(node))
               param_strings.append(s)
             expr = f"{cur.expr}[{', '.join(param_strings)}]"
             late_annot = abstract.LateAnnotation(expr, cur.stack, cur.ctx)
@@ -107,11 +115,53 @@ def sub_one_annotation(self, node, annot, substs, instantiate_unbound=True):
                     late_annot.expr.split("[", 1)[0]].append(late_annot)
           done.append(done_annot)
       else:
-        done.append(self._get_type_parameter_subst(
-            node, cur, substs, instantiate_unbound))
+        done.append(get_type_parameter_subst_fn(cur))
     assert len(done) == 1
     return done[0]
 
+  def sub_annotations_for_parameterized_class(self, cls, annotations):
+    """Apply type parameter substitutions to a dictionary of annotations.
+
+    Args:
+      cls: ParameterizedClass that defines type parameter substitutions.
+      annotations: A dictionary of annotations to which type parameter
+        substition should be applied.
+
+    Returns:
+      Annotations with type parameters substituted.
+    """
+    assert isinstance(cls, abstract.ParameterizedClass)
+    formal_type_parameters = cls.get_formal_type_parameters()
+
+    def get_type_parameter_subst(annotation):
+      assert isinstance(annotation, abstract.TypeParameter)
+      # Normally the type parameter module is set correctly at this point.
+      # Except for the case when a method that references this type parameter
+      # is inherited in a subclass that does not specialize this parameter:
+      #   class A(Generic[T]):
+      #     def f(self, t: T): ...
+      #
+      #   class B(Generic[T], A[T]):
+      #     pass
+      #
+      #   class C(B[int]): ...
+      # In this case t in A[T].f will be annotated with T with no module set,
+      # since we don't know the correct module until T is specialized in
+      # B[int].
+      annotation = annotation.with_module(cls.full_name)
+      # Method parameter can be annotated with a typevar that doesn't
+      # belong to the class template:
+      #   class A(Generic[T]):
+      #     def f(self, t: U): ...
+      # In this case we return it as is.
+      return formal_type_parameters.get(annotation.full_name, annotation)
+
+    return {
+        name: self._do_sub_one_annotation(self.ctx.root_node, annot,
+                                          get_type_parameter_subst)
+        for name, annot in annotations.items()
+    }
+
   def get_late_annotations(self, annot):
     if annot.is_late_annotation() and not annot.resolved:
       yield annot

pytype/convert.py

@@ -615,6 +615,8 @@ def _special_constant_to_value(self, name):
       return self.function_type
     elif name == "types.NoneType":
       return self.none_type
+    elif name == "types.CodeType":
+      return self.primitive_classes[types.CodeType]
     else:
       return None
 

pytype/directors/directors.py

@@ -280,6 +280,7 @@ def __init__(self, src_tree, errorlog, filename, disable, code):
     # Store function ranges and return lines to distinguish explicit and
     # implicit returns (the bytecode has a `RETURN None` for implcit returns).
     self._return_lines = set()
+    self.block_returns = None
     self._function_ranges = _BlockRanges({})
     # Parse the source code for directives.
     self._parse_src_tree(src_tree, code)
@@ -313,7 +314,8 @@ def _parse_src_tree(self, src_tree, code):
     else:
       opcode_lines = None
 
-    self._return_lines = visitor.returns
+    self.block_returns = visitor.block_returns
+    self._return_lines = visitor.block_returns.all_returns()
     self._function_ranges = _BlockRanges(visitor.function_ranges)
 
     for line_range, group in visitor.structured_comment_groups.items():

pytype/directors/parser.py

@@ -32,6 +32,9 @@ class LineRange:
   def from_node(cls, node):
     return cls(node.lineno, node.end_lineno)
 
+  def __contains__(self, line):
+    return self.start_line <= line <= self.end_line
+
 
 @dataclasses.dataclass(frozen=True)
 class Call(LineRange):
@@ -66,6 +69,44 @@ class _SourceTree:
   structured_comments: Mapping[int, Sequence[_StructuredComment]]
 
 
+class BlockReturns:
+  """Tracks return statements in with/try blocks."""
+
+  def __init__(self):
+    self._block_ranges = []
+    self._returns = []
+    self._block_returns = {}
+    self._final = False
+
+  def add_block(self, node):
+    line_range = LineRange.from_node(node)
+    self._block_ranges.append(line_range)
+
+  def add_return(self, node):
+    self._returns.append(node.lineno)
+
+  def finalize(self):
+    for br in self._block_ranges:
+      self._block_returns[br.start_line] = sorted(
+          r for r in self._returns if r in br
+      )
+    self._final = True
+
+  def all_returns(self):
+    return set(self._returns)
+
+  def __iter__(self):
+    assert self._final
+    return iter(self._block_returns.items())
+
+  def __repr__(self):
+    return f"""
+      Blocks: {self._block_ranges}
+      Returns: {self._returns}
+      {self._block_returns}
+    """
+
+
 class _ParseVisitor(visitor.BaseVisitor):
   """Visitor for parsing a source tree.
 
@@ -97,8 +138,9 @@ def __init__(self, raw_structured_comments):
     self.variable_annotations = []
     self.decorators = []
     self.defs_start = None
-    self.returns = set()
     self.function_ranges = {}
+    self.block_returns = BlockReturns()
+    self.block_depth = 0
 
   def _add_structured_comment_group(self, start_line, end_line, cls=LineRange):
     """Adds an empty _StructuredComment group with the given line range."""
@@ -171,6 +213,9 @@ def should_add(comment, group):
       if cls is not LineRange:
         group.extend(c for c in structured_comments if should_add(c, group))
 
+  def leave_Module(self, node):
+    self.block_returns.finalize()
+
   def visit_Call(self, node):
     self._process_structured_comments(LineRange.from_node(node), cls=Call)
 
@@ -200,8 +245,22 @@ def visit_Try(self, node):
   def _visit_with(self, node):
     item = node.items[-1]
     end_lineno = (item.optional_vars or item.context_expr).end_lineno
+    if self.block_depth == 1:
+      self.block_returns.add_block(node)
     self._process_structured_comments(LineRange(node.lineno, end_lineno))
 
+  def enter_With(self, node):
+    self.block_depth += 1
+
+  def leave_With(self, node):
+    self.block_depth -= 1
+
+  def enter_AsyncWith(self, node):
+    self.block_depth += 1
+
+  def leave_AsyncWith(self, node):
+    self.block_depth -= 1
+
   def visit_With(self, node):
     self._visit_with(node)
 
@@ -226,8 +285,8 @@ def generic_visit(self, node):
       self._process_structured_comments(LineRange.from_node(node))
 
   def visit_Return(self, node):
+    self.block_returns.add_return(node)
     self._process_structured_comments(LineRange.from_node(node))
-    self.returns.add(node.lineno)
 
   def _visit_decorators(self, node):
     if not node.decorator_list:

pytype/directors/parser_libcst.py

@@ -25,6 +25,9 @@ class LineRange:
   start_line: int
   end_line: int
 
+  def __contains__(self, line):
+    return self.start_line <= line <= self.end_line
+
 
 @dataclasses.dataclass(frozen=True)
 class Call(LineRange):
@@ -53,6 +56,31 @@ class _VariableAnnotation(LineRange):
   annotation: str
 
 
+class BlockReturns:
+  """Tracks return statements in with/try blocks."""
+
+  def __init__(self):
+    self._block_ranges = []
+    self._returns = []
+    self._block_returns = {}
+
+  def add_return(self, pos):
+    self._returns.append(pos.start.line)
+
+  def all_returns(self):
+    return set(self._returns)
+
+  def __iter__(self):
+    return iter(self._block_returns.items())
+
+  def __repr__(self):
+    return f"""
+      Blocks: {self._block_ranges}
+      Returns: {self._returns}
+      {self._block_returns}
+    """
+
+
 class _ParseVisitor(libcst.CSTVisitor):
   """Visitor for parsing a source tree.
 
@@ -80,8 +108,8 @@ def __init__(self):
     self.variable_annotations = []
     self.decorators = []
     self.defs_start = None
-    self.returns = set()
     self.function_ranges = {}
+    self.block_returns = BlockReturns()
 
   def _get_containing_groups(self, start_line, end_line=None):
     """Get _StructuredComment groups that fully contain the given line range."""
@@ -240,7 +268,7 @@ def visit_AnnAssign(self, node):
         _VariableAnnotation(pos.start.line, pos.end.line, annotation))
 
   def visit_Return(self, node):
-    self.returns.add(self._get_position(node).start.line)
+    self.block_returns.add_return(self._get_position(node))
 
   def _visit_decorators(self, node):
     if not node.decorators:

pytype/load_pytd.py

@@ -70,8 +70,6 @@ class Module:
     metadata: The metadata extracted from the picked file.
   """
 
-  _INIT_NAMES = ("__init__.pyi", f"__init__.{pytd_utils.PICKLE_EXT}")
-
   # pylint: disable=redefined-outer-name
   def __init__(self, module_name, filename, ast, metadata=None, pickle=None,
                has_unresolved_pointers=True):
@@ -91,7 +89,10 @@ def is_package(self):
       # imports_map_loader adds os.devnull entries for __init__.py files in
       # intermediate directories.
       return True
-    return self.filename and os.path.basename(self.filename) in self._INIT_NAMES
+    if self.filename:
+      base, _ = os.path.splitext(os.path.basename(self.filename))
+      return base == "__init__"
+    return False
 
 
 class BadDependencyError(Exception):
@@ -209,7 +210,7 @@ def _unpickle_module(self, module):
       newly_loaded_asts.append(loaded_ast)
       m.ast = loaded_ast.ast
       if loaded_ast.is_package:
-        init_file = f"__init__.{pytd_utils.PICKLE_EXT}"
+        init_file = f"__init__{pytd_utils.PICKLE_EXT}"
         if m.filename and os.path.basename(m.filename) != init_file:
           base, _ = os.path.splitext(m.filename)
           m.filename = os.path.join(base, init_file)

pytype/load_pytd_test.py

@@ -19,6 +19,20 @@
 import unittest
 
 
+class ModuleTest(test_base.UnitTest):
+  """Tests for load_pytd.Module."""
+
+  def test_is_package(self):
+    for filename, is_package in [("foo/bar.pyi", False),
+                                 ("foo/__init__.pyi", True),
+                                 ("foo/__init__.pyi-1", True),
+                                 ("foo/__init__.pickled", True),
+                                 (os.devnull, True)]:
+      with self.subTest(filename=filename):
+        mod = load_pytd.Module(module_name=None, filename=filename, ast=None)
+        self.assertEqual(mod.is_package(), is_package)
+
+
 class _LoaderTest(test_base.UnitTest):
 
   @contextlib.contextmanager