Google sync

pytype/CMakeLists.txt

@@ -174,6 +174,7 @@ py_library(
     overlays/classgen.py
     overlays/collections_overlay.py
     overlays/dataclass_overlay.py
+    overlays/flax_overlay.py
     overlays/future_overlay.py
     overlays/six_overlay.py
     overlays/subprocess_overlay.py

pytype/abstract.py

@@ -74,6 +74,12 @@ def __init__(self, name, vm):
     self._all_template_names = None
     self._instance = None
 
+    # The variable or function arg name with the type annotation that this
+    # instance was created from. For example,
+    #   x: str = "hello"
+    # would create an instance of str with from_annotation = 'x'
+    self.from_annotation = None
+
   @property
   def all_template_names(self):
     if self._all_template_names is None:
@@ -1127,9 +1133,9 @@ def update(self, node, other_dict, omit=()):
 class AnnotationsDict(Dict):
   """__annotations__ dict."""
 
-  def __init__(self, vm):
+  def __init__(self, annotated_locals, vm):
     super().__init__(vm)
-    self.annotated_locals = vm.current_annotated_locals
+    self.annotated_locals = annotated_locals
 
   def get_type(self, node, name):
     if name not in self.annotated_locals:
@@ -1486,6 +1492,7 @@ def __init__(self, name, vm):
     super(Function, self).__init__(name, vm)
     self.cls = FunctionPyTDClass(self, vm)
     self.is_attribute_of_class = False
+    self.is_classmethod = False
     self.is_abstract = False
     self.members["func_name"] = self.vm.convert.build_string(
         self.vm.root_cfg_node, name)
@@ -1771,6 +1778,42 @@ def call(self, node, func, args, alias_map=None):
       retvar.PasteVariable(result, node)
       all_mutations.update(mutations)
 
+    if all_mutations and self.vm.options.check_container_types:
+      # Raise an error if:
+      # - An annotation has a type param that is not ambigious or empty
+      # - The mutation adds a type that is not ambiguous or empty
+      def filter_contents(var):
+        # reduces the work compatible_with has to do.
+        return set(x for x in var.data
+                   if not x.isinstance_AMBIGUOUS_OR_EMPTY())
+
+      def compatible_with(existing, new):
+        """Check whether a new type can be added to a container."""
+        for data in existing:
+          if self.vm.matcher.match_from_mro(new.cls, data.cls):
+            return True
+        return False
+
+      errors = collections.defaultdict(dict)
+
+      for obj, name, values in all_mutations:
+        if obj.from_annotation:
+          params = obj.get_instance_type_parameter(name)
+          ps = filter_contents(params)
+          if ps:
+            # check if the container type is being broadened.
+            vs = filter_contents(values)
+            new = [x for x in (vs - ps) if not compatible_with(ps, x)]
+            if new:
+              formal = name.split(".")[-1]
+              errors[obj][formal] = (params, values, obj.from_annotation)
+
+      for obj, errs in errors.items():
+        names = {name for _, _, name in errs.values()}
+        name = list(names)[0] if len(names) == 1 else None
+        self.vm.errorlog.container_type_mismatch(
+            self.vm.frames, obj, errs, name)
+
     node = abstract_utils.apply_mutations(node, all_mutations.__iter__)
     return node, retvar
 
@@ -3139,6 +3182,8 @@ def call(self, node, func, args, new_locals=False, alias_map=None):
           extra_key = (self.get_first_opcode(), name)
           node, callargs[name] = self.vm.init_class(
               node, annotations[name], extra_key=extra_key)
+          for d in callargs[name].data:
+            d.from_annotation = name
     try:
       frame = self.vm.make_frame(
           node, self.code, self.f_globals, self.f_locals, callargs,
@@ -3444,6 +3489,10 @@ def is_abstract(self):
   def is_abstract(self, value):
     self.underlying.is_abstract = value
 
+  @property
+  def is_classmethod(self):
+    return self.underlying.is_classmethod
+
   def repr_names(self, callself_repr=None):
     """Names to use in the bound function's string representation.
 

pytype/analyze.py

@@ -12,6 +12,7 @@
 from pytype import function
 from pytype import metrics
 from pytype import output
+from pytype import special_builtins
 from pytype import state as frame_state
 from pytype import vm
 from pytype.overlays import typing_overlay
@@ -138,7 +139,17 @@ def call_function_in_frame(self, node, var, args, kwargs,
     self.pop_frame(frame)
     return state.node, ret
 
-  def maybe_analyze_method(self, node, val):
+  def _maybe_fix_classmethod_cls_arg(self, node, cls, func, args):
+    sig = func.signature
+    if (args.posargs and sig.param_names and
+        (sig.param_names[0] not in sig.annotations)):
+      # fix "cls" parameter
+      return args._replace(
+          posargs=(cls.AssignToNewVariable(node),) + args.posargs[1:])
+    else:
+      return args
+
+  def maybe_analyze_method(self, node, val, cls=None):
     method = val.data
     fname = val.data.name
     if isinstance(method, abstract.INTERPRETER_FUNCTION_TYPES):
@@ -150,6 +161,8 @@ def maybe_analyze_method(self, node, val):
       else:
         for f in method.iter_signature_functions():
           node, args = self.create_method_arguments(node, f)
+          if f.is_classmethod and cls:
+            args = self._maybe_fix_classmethod_cls_arg(node, cls, f, args)
           node, _ = self.call_function_with_args(node, val, args)
     return node
 
@@ -191,18 +204,23 @@ def _call_with_fake_args(self, node0, funcv):
     log.info("Unable to generate fake arguments for %s", funcv)
     return node, self.new_unsolvable(node)
 
-  def analyze_method_var(self, node0, name, var):
+  def analyze_method_var(self, node0, name, var, cls=None):
     log.info("Analyzing %s", name)
     node1 = node0.ConnectNew(name)
     for val in var.bindings:
-      node2 = self.maybe_analyze_method(node1, val)
+      node2 = self.maybe_analyze_method(node1, val, cls)
       node2.ConnectTo(node0)
     return node0
 
   def bind_method(self, node, name, methodvar, instance_var):
     bound = self.program.NewVariable()
     for m in methodvar.Data(node):
-      bound.AddBinding(m.property_get(instance_var), [], node)
+      if isinstance(m, special_builtins.ClassMethodInstance):
+        m = m.func.data[0]
+        is_cls = True
+      else:
+        is_cls = (m.isinstance_InterpreterFunction() and m.is_classmethod)
+      bound.AddBinding(m.property_get(instance_var, is_cls), [], node)
     return bound
 
   def _instantiate_binding(self, node0, cls):
@@ -218,11 +236,7 @@ def _instantiate_binding(self, node0, cls):
     for b in new.bindings:
       self._analyzed_functions.add(b.data.get_first_opcode())
       node2, args = self.create_method_arguments(node1, b.data)
-      if args.posargs and (
-          b.data.signature.param_names[0] not in b.data.signature.annotations):
-        # fix "cls" parameter
-        args = args._replace(
-            posargs=(cls.AssignToNewVariable(node0),) + args.posargs[1:])
+      args = self._maybe_fix_classmethod_cls_arg(node0, cls, b.data, args)
       node3 = node2.ConnectNew()
       node4, ret = self.call_function_with_args(node3, b, args)
       instance.PasteVariable(ret)
@@ -345,7 +359,7 @@ def analyze_class(self, node, val):
       if name in self._CONSTRUCTORS:
         continue  # We already called this method during initialization.
       b = self.bind_method(node, name, methodvar, instance)
-      node = self.analyze_method_var(node, name, b)
+      node = self.analyze_method_var(node, name, b, val)
     return node
 
   def analyze_function(self, node0, val):

pytype/annotations_util.py

@@ -203,6 +203,8 @@ def apply_annotation(self, state, op, name, value):
     typ = self.extract_annotation(
         state.node, var, name, self.vm.simple_stack(), is_var=True)
     _, value = self.vm.init_class(state.node, typ)
+    for d in value.data:
+      d.from_annotation = name
     return typ, value
 
   def extract_annotation(self, node, var, name, stack, is_var=False):

pytype/compare.py

@@ -119,14 +119,26 @@ def compatible_with(value, logical_value):
   elif isinstance(value, mixin.PythonConstant):
     return bool(value.pyval) == logical_value
   elif isinstance(value, abstract.Instance):
-    # Containers with unset parameters and NoneType instances cannot match True.
     name = value.full_name
     if logical_value and name in _CONTAINER_NAMES:
-      return (
+      # Containers with unset parameters cannot match True.
+      ret = (
           value.has_instance_type_parameter(abstract_utils.T) and
           bool(value.get_instance_type_parameter(abstract_utils.T).bindings))
+      return ret
     elif name == "__builtin__.NoneType":
+      # NoneType instances cannot match True.
       return not logical_value
+    elif name in NUMERIC:
+      # Numeric types can match both True and False
+      return True
+    elif isinstance(value.cls, mixin.Class) and not value.cls.overrides_bool:
+      if getattr(value.cls, "template", None):
+        # A parameterized class can match both True and False, since it might be
+        # an empty container.
+        return True
+      # Objects evaluate to True unless explicitly overridden.
+      return logical_value
     return True
   elif isinstance(value, (abstract.Function, mixin.Class)):
     # Functions and classes always evaluate to True.