Revert tyre changes to the kythe indexer.

tyre is now using the underlying traces library, so these changes to make
xref.indexer track pytd types are no longer needed.

I also added a traces test for converting unknowns to Any, which was covered by
the xref.indexer tests that I'm removing.

PiperOrigin-RevId: 261801244

pytype/tools/traces/traces_test.py

@@ -68,6 +68,14 @@ class Foo(object):
     pyval, = trace.types
     self.assertEqual(pyval.name, "typing.Callable")
 
+  def test_unknown(self):
+    # pytype represents unannotated function parameters as unknowns. Make sure
+    # unknowns don't appear in the traced types.
+    src = traces.trace("def f(x): return x")
+    trace, = (x for x in src.traces[1] if x.op == "LOAD_FAST")
+    pyval, = trace.types
+    self.assertIsInstance(pyval, pytd.AnythingType)
+
 
 class MatchAstVisitorTest(unittest.TestCase):
   """Tests for traces.MatchAstVisitor."""

pytype/tools/xref/debug.py

@@ -17,10 +17,9 @@ def format_def_with_location(defn, loc):
       format_loc(loc), defn.typ.ljust(15), defn.format()))
 
 
-def format_ref(ref, keep_pytype_data=False):
-  suffix = " : %s" % (ref.data,) if keep_pytype_data else ""
-  return ("%s  | %s  %s.%s%s" % (format_loc(
-      ref.location), ref.typ.ljust(15), ref.scope, ref.name, suffix))
+def format_ref(ref):
+  return ("%s  | %s  %s.%s" % (
+      format_loc(ref.location), ref.typ.ljust(15), ref.scope, ref.name))
 
 
 def format_call(call):
@@ -42,11 +41,11 @@ def show_defs(index):
         print(" "*28 + str(defn.doc))
 
 
-def show_refs(index, keep_pytype_data=False):
+def show_refs(index):
   """Show references and associated definitions."""
   indent = "          :  "
   for ref, defn in index.links:
-    print(format_ref(ref, keep_pytype_data))
+    print(format_ref(ref))
     if defn:
       print(indent, defn.format())
       for loc in index.locs[defn.id]:
@@ -61,15 +60,15 @@ def show_calls(index):
     print(format_call(call))
 
 
-def show_index(index, keep_pytype_data=False):
+def show_index(index):
   """Display output in human-readable format."""
 
   def separator():
     print("\n--------------------\n")
 
   show_defs(index)
   separator()
-  show_refs(index, keep_pytype_data)
+  show_refs(index)
   separator()
   show_calls(index)
   separator()

pytype/tools/xref/indexer.py

@@ -12,9 +12,6 @@
 from pytype import io
 from pytype import load_pytd
 from pytype import module_utils
-from pytype.pytd import pytd
-from pytype.pytd import pytd_utils
-from pytype.pytd import visitors
 from pytype.tools.traces import source
 from pytype.tools.traces import traces
 from pytype.tools.traces import visitor as ast_visitor
@@ -119,11 +116,6 @@ def match_opcodes(opcode_traces, lineno, op_match_list):
   return out
 
 
-def _join_types(vals):
-  return pytd_utils.JoinTypes(v.to_type() for v in vals).Visit(
-      visitors.RemoveUnknownClasses())
-
-
 def _unwrap(data):
   assert len(data) == 1
   return data[0]
@@ -868,24 +860,14 @@ def get_doc_offsets(self, doc):
     end = start + doc.length
     return (start, end)
 
-  def finalize(self, keep_pytype_data, pytype_ast):
+  def finalize(self):
     """Postprocess the information gathered by the tree visitor.
 
     Note that these functions need to be run in order; some of them depend on
     information generated by previous ones.
-
-    Args:
-      keep_pytype_data: Whether to preserve the Reference.data field.
-      pytype_ast: A pytd.TypeDeclUnit representing the inferred types.
     """
 
     links = self._lookup_refs()
-    # Finalize refs as early as possible to avoid accidentally copying pointers
-    # to the old `data` field.
-    if keep_pytype_data:
-      # TODO(rechen): Once this code has been sufficiently vetted, remove the
-      # `keep_pytype_data` option and always finalize refs.
-      self.refs, links = self._finalize_refs(self.refs, links, pytype_ast)
     self.links = links
     self._process_deflocs()
     self._process_links(links)
@@ -951,15 +933,6 @@ def get_ref_bounds(self, ref):
     else:
       return self.get_anchor_bounds(ref.location, len(ref.name))
 
-  def get_ref_location_and_python_type(self, ref):
-    if ref.typ == "Attribute":
-      # For an attribute, return information about the attribute itself,
-      # ignoring the object it was accessed on.
-      loc, _ = self.source.get_attr_location(ref.name, ref.location)
-    else:
-      loc = ref.location
-    return loc, ref.data[-1]
-
   def _make_defn_vname(self, defn):
     """Convert a definition into a kythe vname."""
     if isinstance(defn, Remote):
@@ -1050,29 +1023,6 @@ def _process_calls(self, links):
             else:
               assert False, ref
 
-  def _to_pytd(self, vals, pytype_ast):
-    if not vals:
-      return pytd.AnythingType()
-    with io.wrap_pytype_exceptions(PytypeError, filename=self.source.filename):
-      return self.loader.resolve_type(_join_types(vals), pytype_ast)
-
-  def _finalize_refs(self, refs, links, pytype_ast):
-    """Preserve the pytype data in references."""
-    final_refs = []
-    final_links = []
-    final_ref_cache = {}
-    for ref in refs:
-      t = tuple(self._to_pytd(d, pytype_ast) for d in ref.data)
-      final_ref = ref._replace(data=t)
-      final_refs.append(final_ref)
-      final_ref_cache[ref.id] = final_ref
-    for ref, defn in links:
-      # Update ref.data from the final ref cache
-      cached = final_ref_cache[ref.id]
-      new_ref = ref._replace(data=cached.data)
-      final_links.append((new_ref, defn))
-    return final_refs, final_links
-
   def _lookup_remote_symbol(self, ref, defn):
     """Try to look up a definition in an imported module."""
 
@@ -1244,20 +1194,14 @@ def __repr__(self):
     return "%s %s" % (super(VmTrace, self).__repr__(), types_repr)
 
 
-def process_file(options,
-                 source_text=None,
-                 kythe_args=None,
-                 keep_pytype_data=False):
+def process_file(options, source_text=None, kythe_args=None):
   """Process a single file and return cross references.
 
   Args:
     options: A dictionary of pytype options.
     source_text: Optional text of the file; will be read from the file pointed
       to by options.input if not supplied.
     kythe_args: Extra args for generating the kythe index
-    keep_pytype_data: Whether to preserve the Reference.data field. If true, the
-      field will hold the type of the reference as a str or Tuple[str, str] (for
-      attributes). Otherwise, it will be inaccessible.
 
   Returns:
     The Indexer object used for indexing.
@@ -1278,7 +1222,7 @@ def process_file(options,
       store_all_calls=False,
       loader=loader)
   with io.wrap_pytype_exceptions(PytypeError, filename=options.input):
-    pytype_ast, _ = analyze.infer_types(
+    analyze.infer_types(
         src=src,
         filename=options.input,
         errorlog=errorlog,
@@ -1300,5 +1244,5 @@ def process_file(options,
   src_code = source.Code(src, vm.opcode_traces, VmTrace, filename=options.input)
   ix = Indexer(src_code, vm.loader, module_name, kythe_args)
   ix.index(ast_root_node)
-  ix.finalize(keep_pytype_data, pytype_ast)
+  ix.finalize()
   return ix

pytype/tools/xref/indexer_test.py

@@ -19,15 +19,13 @@ class IndexerTest(test_base.TargetIndependentTest):
 
   def index_code(self, code, **kwargs):
     """Generate references from a code string."""
-    keep_pytype_data = kwargs.pop("keep_pytype_data", False)
-
     args = {"version": self.python_version}
     args.update(kwargs)
     with file_utils.Tempdir() as d:
       d.create_file("t.py", code)
       options = config.Options.create(d["t.py"])
       options.tweak(**args)
-      return indexer.process_file(options, keep_pytype_data=keep_pytype_data)
+      return indexer.process_file(options)
 
   def generate_kythe(self, code, **kwargs):
     """Generate a kythe index from a code string."""
@@ -206,107 +204,6 @@ def test_literal_attr(self):
       y = [1, 2].reverse()
     """))
 
-  def test_finalize_refs(self):
-    code = textwrap.dedent("""
-      x = ""
-      def f():
-        return x.upper()
-    """)
-    ix = self.index_code(code, keep_pytype_data=True)
-    expected_refs = (("x", ("str",)), ("x.upper", ("str", "Callable[[], str]")))
-
-    def get_data(ref):
-      if ref.data.__class__ is tuple:
-        return (ref.name, tuple(pytd.Print(t) for t in ref.data))
-      else:
-        return (ref.name, pytd.Print(ref.data))
-
-    self.assertCountEqual((get_data(ref) for ref in ix.refs), expected_refs)
-    self.assertCountEqual((get_data(ref) for ref, _ in ix.links), expected_refs)
-
-  def test_type_map(self):
-    code = textwrap.dedent("""\
-      def f():
-        x = ""
-        return x
-    """)
-    ix = self.index_code(code, keep_pytype_data=True)
-    type_map = output.type_map(ix)
-    self.assertTypeMapEqual(type_map, {(3, 9): "str"})
-
-  def test_type_map_attr(self):
-    code = textwrap.dedent("""\
-      class X:
-        n = 42
-      def f():
-        return X.n
-    """)
-    ix = self.index_code(code, keep_pytype_data=True)
-    type_map = output.type_map(ix)
-    self.assertTypeMapEqual(type_map, {(4, 9): "Type[X]", (4, 11): "int"})
-
-  def test_type_map_multiline_attr(self):
-    code = textwrap.dedent("""\
-      class X:
-        n = 42
-      def f():
-        return (X.
-          n)
-    """)
-    ix = self.index_code(code, keep_pytype_data=True)
-    type_map = output.type_map(ix)
-    self.assertTypeMapEqual(type_map, {(4, 10): "Type[X]", (5, 4): "int"})
-
-  def test_type_map_multiline_dotattr(self):
-    code = textwrap.dedent("""\
-      class X:
-        n = 42
-      def f():
-        return (X
-          .n)
-    """)
-    ix = self.index_code(code, keep_pytype_data=True)
-    type_map = output.type_map(ix)
-    self.assertTypeMapEqual(type_map, {(4, 10): "Type[X]", (5, 5): "int"})
-
-  def test_type_map_missing(self):
-    # For obj.attr, sometimes we fail to find the location of attr. Test that
-    # the type of obj is still accurate.
-    code = textwrap.dedent("""\
-      class X:
-        n = 42
-      def f():
-        return (X
-        {}
-          .n)
-    """).format("\n" * 10)
-    ix = self.index_code(code, keep_pytype_data=True)
-    type_map = output.type_map(ix)
-    self.assertTypeMapEqual(type_map, {(4, 10): "Type[X]"})
-
-  def test_unknown(self):
-    # pytype represents unannotated function parameters as unknowns. Make sure
-    # unknowns don't appear in the type map.
-    code = textwrap.dedent("""\
-      def f(x): return x
-    """)
-    ix = self.index_code(code, keep_pytype_data=True)
-    type_map = output.type_map(ix)
-    self.assertTypeMapEqual(type_map, {(1, 17): "Any"})
-
-  def test_type_resolution(self):
-    code = textwrap.dedent("""\
-      class X:
-        pass
-      Y = X
-    """)
-    ix = self.index_code(code, keep_pytype_data=True)
-    pyval = output.type_map(ix)[(3, 4)]
-    # Make sure we have pytd.ClassType objects with the right .cls pointers.
-    self.assertEqual(pyval.base_type.cls,
-                     self.loader.builtins.Lookup("__builtin__.type"))
-    self.assertEqual(pyval.parameters[0].cls.name, "X")
-
 
 class VmTraceTest(test_base.TargetIndependentTest):
 

pytype/tools/xref/main.py

@@ -30,8 +30,7 @@ def main():
     signal.alarm(options.timeout)
 
   try:
-    ix = indexer.process_file(
-        options, kythe_args=kythe_args, keep_pytype_data=args.debug)
+    ix = indexer.process_file(options, kythe_args=kythe_args)
   except indexer.PytypeError as e:
     print(e.args[0], file=sys.stderr)
     if args.debug:
@@ -41,7 +40,7 @@ def main():
     sys.exit(1)
 
   if args.debug:
-    debug.show_index(ix, keep_pytype_data=True)
+    debug.show_index(ix)
   else:
     output.output_kythe_graph(ix)
 

pytype/tools/xref/output.py

@@ -30,14 +30,3 @@ def json_kythe_graph(index):
 def output_kythe_graph(index):
   for x in json_kythe_graph(index):
     print(x)
-
-
-def type_map(index):
-  """Return a map of (line, col) -> python type for all references."""
-
-  m = {}
-  for ref in index.refs:
-    loc, t = index.get_ref_location_and_python_type(ref)
-    if loc not in m:
-      m[loc] = t
-  return m