Merge pull request #1040 from google/google_sync

Google sync

CHANGELOG

@@ -1,3 +1,19 @@
+Version 2021.11.02:
+
+New features and updates:
+* Remove the --bind-properties flag. Its behavior has been made the default.
+* Take advantage of module aliases to print prettier stub files.
+* Add support for cross-module attr.s wrappers.
+* Add a feature flag, --gen-stub-imports, to improve pyi import handling.
+* Add a bit more support for PEP 612 in stubs.
+
+Bug fixes:
+* Add remove{prefix,suffix} methods for bytes, bytearray.
+* Fix a bug where Errorlog.copy_from() duplicated error details.
+* Fix some issues with handling module aliases in stub files.
+* Support a [not-supported-yet] case in a generic class TypeVar renaming check.
+* Add `__init__` attributes to canonical enum members.
+
 Version 2021.10.25:
 
 New features and updates:

pytype/__version__.py

@@ -1,2 +1,2 @@
 # pylint: skip-file
-__version__ = '2021.10.25'
+__version__ = '2021.11.02'

pytype/abstract/abstract.py

@@ -1557,23 +1557,36 @@ def _build_value(self, node, inner, ellipses):
     # For user-defined generic types, check if its type parameter matches
     # its corresponding concrete type
     if isinstance(base_cls, InterpreterClass) and base_cls.template:
-      for formal in base_cls.template:
-        if (isinstance(formal, TypeParameter) and not formal.is_generic() and
-            isinstance(params[formal.name], TypeParameter)):
-          if formal.name != params[formal.name].name:
-            self.ctx.errorlog.not_supported_yet(
-                self.ctx.vm.frames,
-                "Renaming TypeVar `%s` with constraints or bound" % formal.name)
+      for formal_param in base_cls.template:
+        root_node = self.ctx.root_node
+        param_value = params[formal_param.name]
+        if (isinstance(formal_param, TypeParameter) and
+            not formal_param.is_generic() and
+            isinstance(param_value, TypeParameter)):
+          if formal_param.name == param_value.name:
+            # We don't need to check if a TypeParameter matches itself.
+            continue
+          else:
+            actual = param_value.instantiate(
+                root_node, container=abstract_utils.DUMMY_CONTAINER)
         else:
-          root_node = self.ctx.root_node
-          actual = params[formal.name].instantiate(root_node)
-          bad = self.ctx.matcher(root_node).bad_matches(actual, formal)
-          if bad:
-            formal = self.ctx.annotation_utils.sub_one_annotation(
-                root_node, formal, [{}])
-            self.ctx.errorlog.bad_concrete_type(self.ctx.vm.frames, root_node,
-                                                formal, actual, bad)
-            return self.ctx.convert.unsolvable
+          actual = param_value.instantiate(root_node)
+        bad = self.ctx.matcher(root_node).bad_matches(actual, formal_param)
+        if bad:
+          if not isinstance(param_value, TypeParameter):
+            # If param_value is not a TypeVar, we substitute in TypeVar bounds
+            # and constraints in formal_param for a more helpful error message.
+            formal_param = self.ctx.annotation_utils.sub_one_annotation(
+                root_node, formal_param, [{}])
+            details = None
+          elif isinstance(formal_param, TypeParameter):
+            details = (f"TypeVars {formal_param.name} and {param_value.name} "
+                       "have incompatible bounds or constraints.")
+          else:
+            details = None
+          self.ctx.errorlog.bad_concrete_type(
+              self.ctx.vm.frames, root_node, formal_param, actual, bad, details)
+          return self.ctx.convert.unsolvable
 
     try:
       return abstract_class(base_cls, params, self.ctx, template_params)

pytype/analyze.py

@@ -106,7 +106,8 @@ def infer_types(src,
   if ctx.vm.has_unknown_wildcard_imports or any(
       a in defs for a in abstract_utils.DYNAMIC_ATTRIBUTE_MARKERS):
     if "__getattr__" not in ast:
-      ast = pytd_utils.Concat(ast, builtins.GetDefaultAst())
+      ast = pytd_utils.Concat(
+          ast, builtins.GetDefaultAst(options.gen_stub_imports))
   # If merged with other if statement, triggers a ValueError: Unresolved class
   # when attempts to load from the protocols file
   if options.protocols:

pytype/config.py

@@ -172,6 +172,11 @@ def add_basic_options(o):
       help=(
           "Enable stricter namedtuple checks, such as unpacking and "
           "'typing.Tuple' compatibility. ") + temporary)
+  o.add_argument(
+      "--gen-stub-imports", action="store_true",
+      dest="gen_stub_imports", default=False,
+      help=("Generate import statements (`import x`) rather than constants "
+            "(`x: module`) for module names in stub files. ") + temporary)
 
 
 def add_subtools(o):

pytype/convert.py

@@ -514,20 +514,25 @@ def _load_late_type_module(self, late_type):
   def _load_late_type(self, late_type):
     """Resolve a late type, possibly by loading a module."""
     if late_type.name not in self._resolved_late_types:
-      ast, attr_name = self._load_late_type_module(late_type)
-      if ast is None:
-        log.error("During dependency resolution, couldn't resolve late type %r",
-                  late_type.name)
-        t = pytd.AnythingType()
+      ast = self.ctx.loader.import_name(late_type.name)
+      if ast:
+        t = pytd.Module(name=late_type.name, module_name=late_type.name)
       else:
-        try:
-          cls = pytd.LookupItemRecursive(ast, attr_name)
-        except KeyError:
-          if "__getattr__" not in ast:
-            log.warning("Couldn't resolve %s", late_type.name)
+        ast, attr_name = self._load_late_type_module(late_type)
+        if ast is None:
+          log.error(
+              "During dependency resolution, couldn't resolve late type %r",
+              late_type.name)
           t = pytd.AnythingType()
         else:
-          t = pytd.ToType(cls, allow_functions=True)
+          try:
+            cls = pytd.LookupItemRecursive(ast, attr_name)
+          except KeyError:
+            if "__getattr__" not in ast:
+              log.warning("Couldn't resolve %s", late_type.name)
+            t = pytd.AnythingType()
+          else:
+            t = pytd.ToType(cls, allow_functions=True)
       self._resolved_late_types[late_type.name] = t
     return self._resolved_late_types[late_type.name]
 
@@ -536,7 +541,10 @@ def _create_module(self, ast):
       raise abstract_utils.ModuleLoadError()
     data = (ast.constants + ast.type_params + ast.classes +
             ast.functions + ast.aliases)
-    members = {val.name.rsplit(".")[-1]: val for val in data}
+    members = {}
+    for val in data:
+      name = utils.strip_prefix(val.name, f"{ast.name}.")
+      members[name] = val
     return abstract.Module(self.ctx, ast.name, members, ast)
 
   def _get_literal_value(self, pyval):

pytype/errors.py

@@ -347,7 +347,7 @@ def __getitem__(self, index):
   def copy_from(self, errors, stack):
     for e in errors:
       with _CURRENT_ERROR_NAME.bind(e.name):
-        self.error(stack, e.message, e.details, e.keyword, e.bad_call,
+        self.error(stack, e._message, e.details, e.keyword, e.bad_call,  # pylint: disable=protected-access
                    e.keyword_context)
 
   def is_valid_error_name(self, name):
@@ -923,14 +923,16 @@ def bad_yield_annotation(self, stack, name, annot, is_async):
     self.error(stack, message, details)
 
   @_error_name("bad-concrete-type")
-  def bad_concrete_type(self, stack, node, formal, actual, bad):
+  def bad_concrete_type(self, stack, node, formal, actual, bad, details=None):
     expected, actual, _, protocol_details, nis_details = (
         self._print_as_return_types(node, formal, actual, bad))
-    details = ["       Expected: ", expected, "\n",
-               "Actually passed: ", actual]
-    details.extend(protocol_details + nis_details)
+    full_details = ["       Expected: ", expected, "\n",
+                    "Actually passed: ", actual]
+    if details:
+      full_details.append("\n" + details)
+    full_details.extend(protocol_details + nis_details)
     self.error(
-        stack, "Invalid instantiation of generic class", "".join(details))
+        stack, "Invalid instantiation of generic class", "".join(full_details))
 
   def _show_variable(self, var):
     """Show variable as 'name: typ' or 'pyval: typ' if available."""

pytype/io.py

@@ -136,7 +136,7 @@ def check_or_generate_pyi(options, loader=None):
 
   errorlog = errors.ErrorLog()
   result = pytd_builtins.DEFAULT_SRC
-  ast = pytd_builtins.GetDefaultAst()
+  ast = pytd_builtins.GetDefaultAst(options.gen_stub_imports)
   try:
     src = read_source_file(options.input, options.open_function)
     if options.check:
@@ -235,7 +235,7 @@ def write_pickle(ast, options, loader=None):
     if options.nofail:
       ast = serialize_ast.PrepareForExport(
           options.module_name,
-          pytd_builtins.GetDefaultAst(), loader)
+          pytd_builtins.GetDefaultAst(options.gen_stub_imports), loader)
       log.warning("***Caught exception: %s", str(e), exc_info=True)
     else:
       raise

pytype/io_test.py

@@ -80,9 +80,11 @@ def test_generate_pyi_with_options(self):
       with self._tmpfile(
           "{mod} {path}".format(mod=pyi_name, path=pyi.name)) as imports_map:
         src = "import {mod}; y = {mod}.x".format(mod=pyi_name)
-        options = config.Options.create(imports_map=imports_map.name)
+        options = config.Options.create(imports_map=imports_map.name,
+                                        gen_stub_imports=True)
         _, pyi_string, _ = io.generate_pyi(src, options)
-    self.assertEqual(pyi_string, "{mod}: module\ny: int\n".format(mod=pyi_name))
+    self.assertEqual(pyi_string,
+                     "import {mod}\n\ny: int\n".format(mod=pyi_name))
 
   def test_check_or_generate_pyi__check(self):
     with self._tmpfile("") as f:

pytype/load_pytd.py

@@ -27,6 +27,7 @@
     "python_version": "python_version",
     "pythonpath": "pythonpath",
     "use_typeshed": "typeshed",
+    "gen_stub_imports": "gen_stub_imports",
 }
 
 
@@ -129,8 +130,9 @@ class _ModuleMap:
 
   PREFIX = "pytd:"  # for pytd files that ship with pytype
 
-  def __init__(self, python_version, modules=None):
+  def __init__(self, python_version, modules, gen_stub_imports):
     self.python_version = python_version
+    self.gen_stub_imports = gen_stub_imports
     self._modules: Dict[str, Module] = modules or self._base_modules()
     if self._modules["builtins"].needs_unpickling():
       self._unpickle_module(self._modules["builtins"])
@@ -186,7 +188,7 @@ def get_resolved_modules(self) -> Dict[str, ResolvedModule]:
     return resolved_modules
 
   def _base_modules(self):
-    bltins, typing = builtins.GetBuiltinsAndTyping()
+    bltins, typing = builtins.GetBuiltinsAndTyping(self.gen_stub_imports)
     return {
         "builtins":
         Module("builtins", self.PREFIX + "builtins", bltins,
@@ -371,8 +373,9 @@ def collect_dependencies(cls, mod_ast):
 class _BuiltinLoader:
   """Load builtins from the pytype source tree."""
 
-  def __init__(self, python_version):
+  def __init__(self, python_version, gen_stub_imports):
     self.python_version = python_version
+    self.gen_stub_imports = gen_stub_imports
 
   def _parse_predefined(self, pytd_subdir, module, as_package=False):
     """Parse a pyi/pytd file in the pytype source tree."""
@@ -382,7 +385,8 @@ def _parse_predefined(self, pytd_subdir, module, as_package=False):
     except IOError:
       return None
     ast = parser.parse_string(src, filename=filename, name=module,
-                              python_version=self.python_version)
+                              python_version=self.python_version,
+                              gen_stub_imports=self.gen_stub_imports)
     assert ast.name == module
     return ast
 
@@ -415,6 +419,7 @@ class Loader:
     imports_map: A short_path -> full_name mapping for imports.
     use_typeshed: Whether to use https://github.com/python/typeshed.
     open_function: A custom file opening function.
+    gen_stub_imports: Temporary flag for releasing --gen-stub-imports.
   """
 
   PREFIX = "pytd:"  # for pytd files that ship with pytype
@@ -426,20 +431,22 @@ def __init__(self,
                imports_map=None,
                use_typeshed=True,
                modules=None,
-               open_function=open):
+               open_function=open,
+               gen_stub_imports=True):
     self.python_version = utils.normalize_version(python_version)
-    self._modules = _ModuleMap(self.python_version, modules)
+    self._modules = _ModuleMap(self.python_version, modules, gen_stub_imports)
     self.builtins = self._modules["builtins"].ast
     self.typing = self._modules["typing"].ast
     self.base_module = base_module
     self._path_finder = _PathFinder(imports_map, pythonpath)
-    self._builtin_loader = _BuiltinLoader(self.python_version)
+    self._builtin_loader = _BuiltinLoader(self.python_version, gen_stub_imports)
     self._resolver = _Resolver(self.builtins)
     self.use_typeshed = use_typeshed
     self.open_function = open_function
     self._import_name_cache = {}  # performance cache
     self._aliases = {}
     self._prefixes = set()
+    self.gen_stub_imports = gen_stub_imports
     # Paranoid verification that pytype.main properly checked the flags:
     if imports_map is not None:
       assert pythonpath == [""], pythonpath
@@ -505,7 +512,8 @@ def load_file(self, module_name, filename, mod_ast=None):
       with self.open_function(filename, "r") as f:
         mod_ast = parser.parse_string(
             f.read(), filename=filename, name=module_name,
-            python_version=self.python_version)
+            python_version=self.python_version,
+            gen_stub_imports=self.gen_stub_imports)
     return self._process_module(module_name, filename, mod_ast)
 
   def _process_module(self, module_name, filename, mod_ast):
@@ -721,7 +729,7 @@ def _load_builtin(self, subdir, module_name, third_party_only=False):
   def _load_typeshed_builtin(self, subdir, module_name):
     """Load a pyi from typeshed."""
     loaded = typeshed.parse_type_definition(
-        subdir, module_name, self.python_version)
+        subdir, module_name, self.python_version, self.gen_stub_imports)
     if loaded:
       filename, mod_ast = loaded
       return self.load_file(filename=self.PREFIX + filename,

pytype/matcher.py

@@ -292,7 +292,8 @@ def _match_value_against_type(self, value, other_type, subst, view):
 
     if isinstance(left, abstract.TypeParameterInstance) and (
         isinstance(left.instance, (abstract.CallableClass,
-                                   function.Signature))):
+                                   function.Signature)) or
+        left.instance is abstract_utils.DUMMY_CONTAINER):
       if isinstance(other_type, abstract.TypeParameter):
         new_subst = self._match_type_param_against_type_param(
             left.param, other_type, subst, view)

pytype/output.py

@@ -218,7 +218,8 @@ def value_to_pytd_type(self, node, v, seen, view):
     if isinstance(v, (abstract.Empty, typing_overlay.NoReturn)):
       return pytd.NothingType()
     elif isinstance(v, abstract.TypeParameterInstance):
-      if v.module in self._scopes:
+      if (v.module in self._scopes or
+          v.instance is abstract_utils.DUMMY_CONTAINER):
         return self._typeparam_to_def(node, v.param, v.param.name)
       elif v.instance.get_instance_type_parameter(v.full_name).bindings:
         # The type parameter was initialized. Set the view to None, since we
@@ -271,7 +272,10 @@ def value_to_pytd_type(self, node, v, seen, view):
       return pytd.GenericType(base_type=pytd.NamedType("builtins.type"),
                               parameters=(param,))
     elif isinstance(v, abstract.Module):
-      return pytd.NamedType("builtins.module")
+      if self.ctx.options.gen_stub_imports:
+        return pytd.Alias(v.name, pytd.Module(v.name, module_name=v.full_name))
+      else:
+        return pytd.NamedType("builtins.module")
     elif (self._output_mode >= Converter.OutputMode.LITERAL and
           isinstance(v, abstract.ConcreteValue) and
           isinstance(v.pyval, (int, str, bytes))):
@@ -354,7 +358,9 @@ def value_to_pytd_def(self, node, v, name):
     Returns:
       A PyTD definition.
     """
-    if isinstance(v, abstract.BoundFunction):
+    if self.ctx.options.gen_stub_imports and isinstance(v, abstract.Module):
+      return pytd.Alias(name, pytd.Module(name, module_name=v.full_name))
+    elif isinstance(v, abstract.BoundFunction):
       d = self.value_to_pytd_def(node, v.underlying, name)
       assert isinstance(d, pytd.Function)
       sigs = tuple(sig.Replace(params=sig.params[1:]) for sig in d.signatures)

pytype/overlays/collections_overlay.py

@@ -22,7 +22,8 @@ def namedtuple_ast(name,
                    fields,
                    defaults,
                    python_version=None,
-                   strict_namedtuple_checks=True):
+                   strict_namedtuple_checks=True,
+                   gen_stub_imports=True):
   """Make an AST with a namedtuple definition for the given name and fields.
 
   Args:
@@ -33,7 +34,8 @@ def namedtuple_ast(name,
     strict_namedtuple_checks: Whether to enable a stricter type annotation
       hierarchy for generated NamedType. e.g. Tuple[n*[Any]] instead of tuple.
       This should usually be set to the value of
-        ctx.options.strict_namedtuple_checks
+        ctx.options.strict_namedtuple_checks.
+    gen_stub_imports: Set this to the value of ctx.options.gen_stub_imports.
 
   Returns:
     A pytd.TypeDeclUnit with the namedtuple definition in its classes.
@@ -79,7 +81,8 @@ def _replace(self: {typevar}, **kwds) -> {typevar}: ...
       repeat_any=_repeat_type("typing.Any", num_fields),
       fields_as_parameters=fields_as_parameters,
       field_names_as_strings=field_names_as_strings)
-  return parser.parse_string(nt, python_version=python_version)
+  return parser.parse_string(nt, python_version=python_version,
+                             gen_stub_imports=gen_stub_imports)
 
 
 class CollectionsOverlay(overlay.Overlay):
@@ -303,7 +306,8 @@ class have to be changed to match the number and names of the fields, we
         field_names,
         defaults,
         python_version=self.ctx.python_version,
-        strict_namedtuple_checks=self.ctx.options.strict_namedtuple_checks)
+        strict_namedtuple_checks=self.ctx.options.strict_namedtuple_checks,
+        gen_stub_imports=self.ctx.options.gen_stub_imports)
     mapping = self._get_known_types_mapping()
 
     # A truly well-formed pyi for the namedtuple will have references to the new